1.從零開始實現LSTM
#從零開始實現長短期記憶網絡
import torch
from torch import nn
from d2l import torch as d2l#加載時光機器數據集
batch_size,num_steps = 32,35
train_iter,vocab = d2l.load_data_time_machine(batch_size,num_steps)#1.定義和初始化模型參數：
#超參數num_hiddens定義隱藏單元的數量。按照標準差0.01的高斯分布初始化權重，并將偏置項設為0。
def get_lstm_params(vocab_size,num_hiddens,device):num_inputs = num_outputs = vocab_sizedef normal(shape):return torch.randn(size=shape,device=device)*0.01def three():return (normal((num_inputs,num_hiddens)),normal((num_hiddens,num_hiddens)),torch.zeros(num_hiddens,device=device))W_xi,W_hi,b_i = three() #輸入門參數W_xf,W_hf,b_f = three() #遺忘門參數W_xo,W_ho,b_o = three() #輸出門參數W_xc,W_hc,b_c = three() #候選記憶元參數#輸出層參數W_hq = normal((num_hiddens,num_outputs))b_q = torch.zeros(num_outputs,device=device)#附加梯度params = [W_xi,W_hi,b_i,W_xf,W_hf,b_f,W_xo,W_ho,b_o,W_xc,W_hc,b_c,W_hq,b_q]for param in params:param.requires_grad_(True)return params
#2.定義模型
#在初始化函數中，長短期記憶網絡的隱狀態需要返回一個額外的記憶元，單元的值為0，形狀為（批量大小，隱藏單元數）
def init_lstm_state(batch_size,num_hiddens,device):return (torch.zeros((batch_size,num_hiddens),device=device),torch.zeros((batch_size,num_hiddens),device=device))
#實際模型的定義與前面討論的一樣：提供三個門和一個額外的記憶元。
#只有隱狀態才會傳遞到輸出層，而記憶元mathbf{C}_t不直接參與輸出計算。
def lstm(inputs,state,params):[W_xi,W_hi,b_i,W_xf,W_hf,b_f,W_xo,W_ho,b_o,W_xc,W_hc,b_c,W_hq,b_q] = params(H,C) = stateoutputs = []for X in inputs:I = torch.sigmoid((X @ W_xi) + (H @ W_hi) + b_i)F = torch.sigmoid((X @ W_xf) + (H @ W_hf) + b_f)O = torch.sigmoid((X @ W_xo) + (H @ W_ho) + b_o)C_tilda = torch.tanh((X @ W_xc) + (H @ W_hc) + b_c)C = F * C + I * C_tildaH = O * torch.tanh(C)Y = (H @ W_hq) + b_qoutputs.append(Y)return torch.cat(outputs,dim=0),(H,C)
#3.訓練和預測
vocab_size,num_hiddens,device = len(vocab),256,d2l.try_gpu()
num_epochs,lr = 500,1
model = d2l.RNNModelScratch(len(vocab),num_hiddens,device,get_lstm_params,init_lstm_state,lstm)
print(d2l.train_ch8(model,train_iter,vocab,lr,num_epochs,device))
d2l.plt.show()
2.簡潔實現LSTM
#簡潔實現長短期記憶網絡
import torch
from torch import nn
from d2l import torch as d2l#加載時光機器數據集
batch_size,num_steps = 32,35
train_iter,vocab = d2l.load_data_time_machine(batch_size,num_steps)vocab_size,num_hiddens,device = len(vocab),256,d2l.try_gpu()
num_epochs,lr = 500,1num_inputs = vocab_size
lstm_layer = nn.LSTM(num_inputs,num_hiddens)
model = d2l.RNNModel(lstm_layer,len(vocab))
model = model.to(device)
print(d2l.train_ch8(model,train_iter,vocab,lr,num_epochs,device))
d2l.plt.show()
)
