一步一步用numpy实现神经网络各种层

1. 首先准备一下数据

if __name__ == "__main__":data = np.array([[2, 1, 0],[2, 2, 0],[5, 4, 1],[4, 5, 1],[2, 3, 0],[3, 2, 0],[6, 5, 1],[4, 1, 0],[6, 3, 1],[7, 4, 1]])x = data[:, :-1]y = data[:, -1]for epoch in range(1000):...

2. 实现Softmax+CrossEntropy层

单独求softmax层有点麻烦, 将softmax+entropy一起求导更方便。

假设对于输入向量$(x_1,x_2,x_3)$, 则对应的Loss为:

$$\begin{array}{rl}L& =-\sum _{i=1}^C{y}_i\ln p^i\\ & =-(y_1\ln p_1+y_2\ln p_2+y_3\ln p_3)\end{array}$$

其中$y_i$为ground truth, 为one-hot vector. $p_i$为输出概率。

$$\begin{gathered} p_1=\frac{e^{x_1}}{e^{x_1}+e^{x_2}+e^{x_3}} \\ {p}_2=\frac{e^{x_2}}{e^{x_1}+e^{x_2}+e^{x_3}} \\ {p}_3=\frac{e^{x_3}}{e^{x_1}+e^{x_2}+e^{x_3}} \end{gathered}$$
则偏导为
$$\begin{array}{rl}\frac{\partial L}{\partial x_1}& =-y_1\frac{1}{p_1}\ast \frac{\partial p_1}{\partial x_1}-y_2\frac{1}{p_2}\ast \frac{\partial p_2}{\partial x_1}-y_3\frac{1}{p_3}\ast \frac{\partial p_3}{\partial x_1}\\ & =-y_1\frac{1}{p_1}\ast \frac{e^{x_1}\ast (e^{x_1}+e^{x_2}+e^{x_3})-e^{x_1}\ast e^{x_1}}{(e^{x_1}+e^{x_2}+e^{x_3}{)}^2}\\ & -y_2\frac{1}{p_2}\ast \frac{-e^{x_2}\ast e^{x_1}}{(e^{x_1}+e^{x_2}+e^{x_3}{)}^2}\\ & -y_3\frac{1}{p_3}\ast \frac{-e^{x_3}\ast e^{x_1}}{(e^{x_1}+e^{x_2}+e^{x_3}{)}^2}\\ & =-y_1\frac{1}{p_1}(p_1\ast p_2+p_1\ast p_3)\\ & -y_2\frac{1}{p_2}(-p_1\ast p_2)\\ & -y_3\frac{1}{p_3}(-p_1\ast p_3)\\ & =-y1(p_2+p_3)+y_2\ast p_2+y_3\ast p_3\\ & =-y_1(1-p_1)+y_2\ast p_1+y_3\ast p_1\\ & =y_1(p_1-1)+y_2\ast p_1+y_3\ast p_1\end{array}$$

同理:
$$\begin{gathered} \frac{\partial L}{\partial x_2}=y_1\ast p_2+y_2(p_2-1)+y_3\ast p_2 \\ \frac{\partial L}{\partial x_3}=y_1\ast p_3+y_2{p}_3+y_3\ast (p_3-1) \end{gathered}$$

当$y_1=1$时, 对应的导数为$(p1-1,p_2,p_3)$. 当$y_2=1$时,对应的导数为:$(p_1,p2-1,p3)$.

例如求得概率为$(0.2,0.3,0.5)$, label为$(0,0,1)$, 则导数为$(0.2,0.3,-0.5)$

python代码为:

注意求softmax时需要np.exp(x-np.max(x, axis=1, keepdims=True))防止指数运算溢出。

class Softmax:def __init__(self, n_classes):self.n_classes = n_classesdef forward(self, x, y):prob = np.exp(x-np.max(x, axis=1, keepdims=True))prob /= np.sum(prob, axis=1, keepdims=True)# 选出y==1位置的概率loss = -np.sum(np.log(prob[np.arange(len(y), y])) / len(y)self.grad = prob.copy()self.grad[np.arange(len(y), y] -= 1"""因为后面求导数都是直接np.sum而不是np.mean, 因此这里mean一次就可以了"""self.grad /= len(y)  return prob, lossdef backward(self):return self.grad

3. 单独的CrossEntropy

python代码为:

class Entropy:def __init__(self, n_classes):self.n_classes = n_classesself.grad = Nonedef forward(self, x, y):# x: (b, c), y: (b)b = y.shape[0]one_hot_y = np.zeros((b, self.n_classes))one_hot_y[range(len(y)), y] = 1self.grad = one_hot_y * -1 / xreturn np.mean(-one_hot_y * np.log(x), axis=0)def backward(self):return self.grad

2. 单独的Softmax层

from einops import repeat, rearrange, einsum
class Softmax:def __init__(self):def forward(self, x):# x: (b, c)x_exp = np.exp(x)self.output = x_xep / np.sum(x_exp, axis=1, keep_dims=True)return self.outputdef backward(self, prev_grad):b, c = self.output.shapeo = repeat(self.output, 'b c -> b c r', r=c)I = repeat(np.eye(x.shape[1]), 'c1 c2 -> b c1 c2', b=b)self.grad = o * (I - rearrange(o, 'b c1 c2 -> b c2 c1'))return einsum(self.grad, grad[..., None], 'b c c, b c m -> b c m')[..., 0]		

3. Linear层

注意更新$w$时用的$d_w$, 但是往上一层传递的是$d_x$。因为上一层需要$dL/d_{out}$, 而本层的输入$x$即是上一次层的输出$dL/d_{out}=dL/dx$

class Linear:def __init__(self, in_channels, out_channels, lr):self.lr = lrself.w = np.random.rand(in_channels, out_channels)self.b = np.random.rand(out_channels)def forward(self, x):self.x = xreturn x@self.w + self.bdef backward(self, grad):dx = einsum(prev_grad, rearrange(self.w, 'w1 w2 -> w2 w1'), 'c1 b, b c2 -> c1 c2')dw = einsum(rearrange(self.x, 'b c -> c b'), prev_grad, 'c1 b, b c2 -> c1 c2')db = np.sum(prev_grad, axis=0)self.w -= self.lr * dwself.b -= self.lr * db"""注意这里往上一层传递的是dx, 因为上一层需要dL/d_out, 而本层的输入x即是上一次层的输出dL/d_out = dL/dx"""return dx

5. 完整训练代码

from einops import *
import numpy as npclass Softmax:def __init__(self, train=True):self.grad = Noneself.train = traindef forward(self, x, y):prob = np.exp(x-np.max(x, axis=1, keepdims=True))prob /= np.sum(prob, axis=1, keepdims=True)if self.train:loss = -np.sum(np.log(prob[range(len(y)), y]))/len(y)self.grad = prob.copy()self.grad[range(len(y)), y] -= 1self.grad /= len(y)return prob, losselse:return probdef backward(self):return self.gradclass Linear:def __init__(self, in_channels, out_channels, lr):self.w = np.random.rand(in_channels, out_channels)self.b = np.random.rand(out_channels)self.lr = lrdef forward(self, x):self.x = xoutput = einsum(x, self.w, 'b c1, c1 c2 -> b c2') + self.breturn outputdef backward(self, prev_grad):cur_grad = einsum(rearrange(self.x, 'b c -> c b'), prev_grad, 'c1 b, b c2 -> c1 c2')self.w -= self.lr * cur_gradself.b -= self.lr * np.sum(prev_grad, axis=0)return cur_gradclass Network:def __init__(self, in_channels, out_channels, n_classes, lr):self.lr = lrself.linear = Linear(in_channels, out_channels, lr)self.softmax = Softmax()def forward(self, x, y=None):out = self.linear.forward(x)out = self.softmax.forward(out, y)return outdef backward(self):grad = self.softmax.backward()grad = self.linear.backward(grad)return gradif __name__ == "__main__":data = np.array([[2, 1, 0],[2, 2, 0],[5, 4, 1],[4, 5, 1],[2, 3, 0],[3, 2, 0],[6, 5, 1],[4, 1, 0],[6, 3, 1],[7, 4, 1]])# x = np.concatenate([np.array([[1]] * data.shape[0]), data[:, :2]], axis=1)x = data[:, :-1]y = data[:, -1:].flatten()net = Network(2, 2, 2, 0.1)# loss_fn = CrossEntropy(n_classes=2)for epoch in range(500):prob, loss = net.forward(x, y)# loss = loss_fn.forward(out, y)# grad_ = loss_fn.backward()grad = net.backward()print(loss)net.softmax.train = Falseprint(net.forward(np.array([[0, 0], [0, 4], [8, 6], [10, 10]])), y)