DL Homework 10

習題6-1P?推導RNN反向傳播算法BPTT.

習題6-2?推導公式(6.40)和公式(6.41)中的梯度

習題6-3?當使用公式(6.50)作為循環神經網絡的狀態更新公式時，分析其可能存在梯度爆炸的原因并給出解決方法．

? ? ? ? ?當然，因為我數學比較菜，我看了好半天還是沒看懂怎么庫庫推過來的，等我慢慢研究，這個博客會定時更新的，哭死

????????但是可能存在梯度爆炸的原因還是比較明確的：RNN發生梯度消失和梯度爆炸的原因如圖所示，將公式改為上式后當γ<1時，t-k趨近于無窮時，γ不會趨近于零，解決了梯度消失問題，但是梯度爆炸仍然存在。當γ>1時，隨著傳播路徑的增加，γ趨近于無窮，產生梯度爆炸。

????????如果時刻t的輸出yt依賴于時刻k的輸入xk，當間隔t-k比較大時，簡單神經網絡很難建模這種長距離的依賴關系，稱為長程依賴問題（Long-Term ependencies Problem）

????????由于梯度爆炸或消失問題，實際上只能學習到短周期的依賴關系。

? ? ? ? 至于改進方案老師給出了兩種，一種是較為直接的修改選取合適參數，同時使用非飽和激活函數，盡量使得 $diag({f}'(z_{\tau }))U^T\approx 1$ 需要足夠的人工調參經驗，限制了模型的廣泛應用.

? ? ? ? 另一種則是比較有效的改進模型

權重衰減通過給參數增加L1或L2范數的正則化項來限制參數的取值范圍，從而使得 γ ≤ 1．
梯度截斷當梯度的模大于一定閾值時，就將它截斷成為一個較小的數

習題6-2P?設計簡單RNN模型，分別用Numpy、Pytorch實現反向傳播算子，并代入數值測試.

1.反向求導的函數

import numpy as np
import torch.nn# GRADED FUNCTION: rnn_cell_forward
def softmax(a):exp_a = np.exp(a)sum_exp_a = np.sum(exp_a)y = exp_a / sum_exp_areturn ydef rnn_cell_forward(xt, a_prev, parameters):Wax = parameters["Wax"]Waa = parameters["Waa"]Wya = parameters["Wya"]ba = parameters["ba"]by = parameters["by"]a_next = np.tanh(np.dot(Wax, xt) + np.dot(Waa, a_prev) + ba)yt_pred = softmax(np.dot(Wya, a_next) + by)cache = (a_next, a_prev, xt, parameters)return a_next, yt_pred, cachedef rnn_cell_backward(da_next, cache):(a_next, a_prev, xt, parameters) = cacheWax = parameters["Wax"]Waa = parameters["Waa"]Wya = parameters["Wya"]ba = parameters["ba"]by = parameters["by"]dtanh = (1 - a_next * a_next) * da_next dxt = np.dot(Wax.T, dtanh)dWax = np.dot(dtanh, xt.T)da_prev = np.dot(Waa.T, dtanh)dWaa = np.dot(dtanh, a_prev.T)dba = np.sum(dtanh, keepdims=True, axis=-1)  gradients = {"dxt": dxt, "da_prev": da_prev, "dWax": dWax, "dWaa": dWaa, "dba": dba}return gradients# GRADED FUNCTION: rnn_forward
np.random.seed(1)
xt = np.random.randn(3, 10)
a_prev = np.random.randn(5, 10)
Wax = np.random.randn(5, 3)
Waa = np.random.randn(5, 5)
Wya = np.random.randn(2, 5)
ba = np.random.randn(5, 1)
by = np.random.randn(2, 1)
parameters = {"Wax": Wax, "Waa": Waa, "Wya": Wya, "ba": ba, "by": by}a_next, yt, cache = rnn_cell_forward(xt, a_prev, parameters)da_next = np.random.randn(5, 10)
gradients = rnn_cell_backward(da_next, cache)
print("gradients[\"dxt\"][1][2] =", gradients["dxt"][1][2])
print("gradients[\"dxt\"].shape =", gradients["dxt"].shape)
print("gradients[\"da_prev\"][2][3] =", gradients["da_prev"][2][3])
print("gradients[\"da_prev\"].shape =", gradients["da_prev"].shape)
print("gradients[\"dWax\"][3][1] =", gradients["dWax"][3][1])
print("gradients[\"dWax\"].shape =", gradients["dWax"].shape)
print("gradients[\"dWaa\"][1][2] =", gradients["dWaa"][1][2])
print("gradients[\"dWaa\"].shape =", gradients["dWaa"].shape)
print("gradients[\"dba\"][4] =", gradients["dba"][4])
print("gradients[\"dba\"].shape =", gradients["dba"].shape)
gradients["dxt"][1][2] = -0.4605641030588796
gradients["dxt"].shape = (3, 10)
gradients["da_prev"][2][3] = 0.08429686538067724
gradients["da_prev"].shape = (5, 10)
gradients["dWax"][3][1] = 0.39308187392193034
gradients["dWax"].shape = (5, 3)
gradients["dWaa"][1][2] = -0.28483955786960663
gradients["dWaa"].shape = (5, 5)
gradients["dba"][4] = [0.80517166]
gradients["dba"].shape = (5, 1)

# GRADED FUNCTION: rnn_forward
def rnn_forward(x, a0, parameters):caches = []n_x, m, T_x = x.shapen_y, n_a = parameters["Wya"].shapea = np.zeros((n_a, m, T_x))y_pred = np.zeros((n_y, m, T_x))a_next = a0for t in range(T_x):a_next, yt_pred, cache = rnn_cell_forward(x[:, :, t], a_next, parameters)a[:, :, t] = a_nexty_pred[:, :, t] = yt_predcaches.append(cache)caches = (caches, x)return a, y_pred, cachesnp.random.seed(1)
x = np.random.randn(3, 10, 4)
a0 = np.random.randn(5, 10)
Waa = np.random.randn(5, 5)
Wax = np.random.randn(5, 3)
Wya = np.random.randn(2, 5)
ba = np.random.randn(5, 1)
by = np.random.randn(2, 1)
parameters = {"Waa": Waa, "Wax": Wax, "Wya": Wya, "ba": ba, "by": by}a, y_pred, caches = rnn_forward(x, a0, parameters)
print("a[4][1] = ", a[4][1])
print("a.shape = ", a.shape)
print("y_pred[1][3] =", y_pred[1][3])
print("y_pred.shape = ", y_pred.shape)
print("caches[1][1][3] =", caches[1][1][3])
print("len(caches) = ", len(caches))

?用numpy和pytorh去實現反向傳播算子，并且二者對比

class RNNCell:def __init__(self, weight_ih, weight_hh,bias_ih, bias_hh):self.weight_ih = weight_ihself.weight_hh = weight_hhself.bias_ih = bias_ihself.bias_hh = bias_hhself.x_stack = []self.dx_list = []self.dw_ih_stack = []self.dw_hh_stack = []self.db_ih_stack = []self.db_hh_stack = []self.prev_hidden_stack = []self.next_hidden_stack = []# temporary cacheself.prev_dh = Nonedef __call__(self, x, prev_hidden):self.x_stack.append(x)next_h = np.tanh(np.dot(x, self.weight_ih.T)+ np.dot(prev_hidden, self.weight_hh.T)+ self.bias_ih + self.bias_hh)self.prev_hidden_stack.append(prev_hidden)self.next_hidden_stack.append(next_h)# clean cacheself.prev_dh = np.zeros(next_h.shape)return next_hdef backward(self, dh):x = self.x_stack.pop()prev_hidden = self.prev_hidden_stack.pop()next_hidden = self.next_hidden_stack.pop()d_tanh = (dh + self.prev_dh) * (1 - next_hidden ** 2)self.prev_dh = np.dot(d_tanh, self.weight_hh)dx = np.dot(d_tanh, self.weight_ih)self.dx_list.insert(0, dx)dw_ih = np.dot(d_tanh.T, x)self.dw_ih_stack.append(dw_ih)dw_hh = np.dot(d_tanh.T, prev_hidden)self.dw_hh_stack.append(dw_hh)self.db_ih_stack.append(d_tanh)self.db_hh_stack.append(d_tanh)return self.dx_listif __name__ == '__main__':np.random.seed(123)torch.random.manual_seed(123)np.set_printoptions(precision=6, suppress=True)rnn_PyTorch = torch.nn.RNN(4, 5).double()rnn_numpy = RNNCell(rnn_PyTorch.all_weights[0][0].data.numpy(),rnn_PyTorch.all_weights[0][1].data.numpy(),rnn_PyTorch.all_weights[0][2].data.numpy(),rnn_PyTorch.all_weights[0][3].data.numpy())nums = 3x3_numpy = np.random.random((nums, 3, 4))x3_tensor = torch.tensor(x3_numpy, requires_grad=True)h3_numpy = np.random.random((1, 3, 5))h3_tensor = torch.tensor(h3_numpy, requires_grad=True)dh_numpy = np.random.random((nums, 3, 5))dh_tensor = torch.tensor(dh_numpy, requires_grad=True)h3_tensor = rnn_PyTorch(x3_tensor, h3_tensor)h_numpy_list = []h_numpy = h3_numpy[0]for i in range(nums):h_numpy = rnn_numpy(x3_numpy[i], h_numpy)h_numpy_list.append(h_numpy)h3_tensor[0].backward(dh_tensor)for i in reversed(range(nums)):rnn_numpy.backward(dh_numpy[i])print("numpy_hidden :\n", np.array(h_numpy_list))print("tensor_hidden :\n", h3_tensor[0].data.numpy())print("------")print("dx_numpy :\n", np.array(rnn_numpy.dx_list))print("dx_tensor :\n", x3_tensor.grad.data.numpy())print("------")print("dw_ih_numpy :\n",np.sum(rnn_numpy.dw_ih_stack, axis=0))print("dw_ih_tensor :\n",rnn_PyTorch.all_weights[0][0].grad.data.numpy())print("------")print("dw_hh_numpy :\n",np.sum(rnn_numpy.dw_hh_stack, axis=0))print("dw_hh_tensor :\n",rnn_PyTorch.all_weights[0][1].grad.data.numpy())print("------")print("db_ih_numpy :\n",np.sum(rnn_numpy.db_ih_stack, axis=(0, 1)))print("db_ih_tensor :\n",rnn_PyTorch.all_weights[0][2].grad.data.numpy())print("------")print("db_hh_numpy :\n",np.sum(rnn_numpy.db_hh_stack, axis=(0, 1)))print("db_hh_tensor :\n",rnn_PyTorch.all_weights[0][3].grad.data.numpy())