Python訓練第四十四天

DAY 44 預訓練模型

知識點回顧：

預訓練的概念
常見的分類預訓練模型
圖像預訓練模型的發展史
預訓練的策略
預訓練代碼實戰：resnet18

作業：

嘗試在cifar10對比如下其他的預訓練模型，觀察差異，盡可能和他人選擇的不同
嘗試通過ctrl進入resnet的內部，觀察殘差究竟是什么

一、預訓練的概念

我們之前在訓練中發現，準確率最開始隨著epoch的增加而增加。隨著循環的更新，參數在不斷發生更新。

所以參數的初始值對訓練結果有很大的影響：

1. 如果最開始的初始值比較好，后續訓練輪數就會少很多

2. 很有可能陷入局部最優值，不同的初始值可能導致陷入不同的局部最優值

我們之前在訓練中發現，準確率最開始隨著epoch的增加而增加。隨著循環的更新，參數在不斷發生更新。

所以參數的初始值對訓練結果有很大的影響：

1. 如果最開始的初始值比較好，后續訓練輪數就會少很多

2. 很有可能陷入局部最優值，不同的初始值可能導致陷入不同的局部最優值

現在再來看下之前一直用的cifar10數據集，他是不是就很明顯不適合作為預訓練數據集？

1. 規模過小：僅 10 萬張圖像，且尺寸小（32x32），無法支撐復雜模型學習通用視覺特征；

2. 類別單一：僅 10 類（飛機、汽車等），泛化能力有限；

這里給大家介紹一個常常用來做預訓練的數據集，ImageNet，ImageNet 1000 個類別，有 1.2 億張圖像，尺寸 224x224，數據集大小 1.4G。

三、常見的分類預訓練模型介紹

3.1 預訓練模型的訓練策略

那么什么模型會被選為預訓練模型呢？比如一些調參后表現很好的cnn神經網絡（固定的神經元個數+固定的層數等）。

所以調用預訓練模型做微調，本質就是用這些固定的結構+之前訓練好的參數接著訓練

所以需要找到預訓練的模型結構并且加載模型參數

相較于之前用自己定義的模型有以下幾個注意點

1. 需要調用預訓練模型和加載權重

2. 需要resize 圖片讓其可以適配模型

3. 需要修改最后的全連接層以適應數據集

其中，訓練過程中，為了不破壞最開始的特征提取器的參數，最開始往往先凍結住特征提取器的參數，然后訓練全連接層，大約在5-10個epoch后解凍訓練。

主要做特征提取的部分叫做backbone骨干網絡；負責融合提取的特征的部分叫做Featue Pyramid Network（FPN）；負責輸出的預測部分的叫做Head。

import torch
import torch.nn as nn
import torch.optim as optim
from torchvision import datasets, transforms
from torch.utils.data import DataLoader
import matplotlib.pyplot as plt# 設置中文字體支持
plt.rcParams["font.family"] = ["SimHei"]
plt.rcParams['axes.unicode_minus'] = False  # 解決負號顯示問題# 檢查GPU是否可用
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"使用設備: {device}")# 1. 數據預處理（訓練集增強，測試集標準化）
train_transform = transforms.Compose([transforms.RandomCrop(32, padding=4),transforms.RandomHorizontalFlip(),transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),transforms.RandomRotation(15),transforms.ToTensor(),transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))
])test_transform = transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))
])# 2. 加載CIFAR-10數據集
train_dataset = datasets.CIFAR10(root='./data',train=True,download=True,transform=train_transform
)test_dataset = datasets.CIFAR10(root='./data',train=False,transform=test_transform
)# 3. 創建數據加載器（可調整batch_size）
batch_size = 64
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)# 4. 訓練函數（支持學習率調度器）
def train(model, train_loader, test_loader, criterion, optimizer, scheduler, device, epochs):model.train()  # 設置為訓練模式train_loss_history = []test_loss_history = []train_acc_history = []test_acc_history = []all_iter_losses = []iter_indices = []for epoch in range(epochs):running_loss = 0.0correct_train = 0total_train = 0for batch_idx, (data, target) in enumerate(train_loader):data, target = data.to(device), target.to(device)optimizer.zero_grad()output = model(data)loss = criterion(output, target)loss.backward()optimizer.step()# 記錄Iteration損失iter_loss = loss.item()all_iter_losses.append(iter_loss)iter_indices.append(epoch * len(train_loader) + batch_idx + 1)# 統計訓練指標running_loss += iter_loss_, predicted = output.max(1)total_train += target.size(0)correct_train += predicted.eq(target).sum().item()# 每100批次打印進度if (batch_idx + 1) % 100 == 0:print(f"Epoch {epoch+1}/{epochs} | Batch {batch_idx+1}/{len(train_loader)} "f"| 單Batch損失: {iter_loss:.4f}")# 計算 epoch 級指標epoch_train_loss = running_loss / len(train_loader)epoch_train_acc = 100. * correct_train / total_train# 測試階段model.eval()correct_test = 0total_test = 0test_loss = 0.0with torch.no_grad():for data, target in test_loader:data, target = data.to(device), target.to(device)output = model(data)test_loss += criterion(output, target).item()_, predicted = output.max(1)total_test += target.size(0)correct_test += predicted.eq(target).sum().item()epoch_test_loss = test_loss / len(test_loader)epoch_test_acc = 100. * correct_test / total_test# 記錄歷史數據train_loss_history.append(epoch_train_loss)test_loss_history.append(epoch_test_loss)train_acc_history.append(epoch_train_acc)test_acc_history.append(epoch_test_acc)# 更新學習率調度器if scheduler is not None:scheduler.step(epoch_test_loss)# 打印 epoch 結果print(f"Epoch {epoch+1} 完成 | 訓練損失: {epoch_train_loss:.4f} "f"| 訓練準確率: {epoch_train_acc:.2f}% | 測試準確率: {epoch_test_acc:.2f}%")# 繪制損失和準確率曲線plot_iter_losses(all_iter_losses, iter_indices)plot_epoch_metrics(train_acc_history, test_acc_history, train_loss_history, test_loss_history)return epoch_test_acc  # 返回最終測試準確率# 5. 繪制Iteration損失曲線
def plot_iter_losses(losses, indices):plt.figure(figsize=(10, 4))plt.plot(indices, losses, 'b-', alpha=0.7)plt.xlabel('Iteration（Batch序號）')plt.ylabel('損失值')plt.title('訓練過程中的Iteration損失變化')plt.grid(True)plt.show()# 6. 繪制Epoch級指標曲線
def plot_epoch_metrics(train_acc, test_acc, train_loss, test_loss):epochs = range(1, len(train_acc) + 1)plt.figure(figsize=(12, 5))# 準確率曲線plt.subplot(1, 2, 1)plt.plot(epochs, train_acc, 'b-', label='訓練準確率')plt.plot(epochs, test_acc, 'r-', label='測試準確率')plt.xlabel('Epoch')plt.ylabel('準確率 (%)')plt.title('準確率隨Epoch變化')plt.legend()plt.grid(True)# 損失曲線plt.subplot(1, 2, 2)plt.plot(epochs, train_loss, 'b-', label='訓練損失')plt.plot(epochs, test_loss, 'r-', label='測試損失')plt.xlabel('Epoch')plt.ylabel('損失值')plt.title('損失值隨Epoch變化')plt.legend()plt.grid(True)plt.tight_layout()plt.show()
# 導入ResNet模型
from torchvision.models import resnet18# 定義ResNet18模型（支持預訓練權重加載）
def create_resnet18(pretrained=True, num_classes=10):# 加載預訓練模型（ImageNet權重）model = resnet18(pretrained=pretrained)# 修改最后一層全連接層，適配CIFAR-10的10分類任務in_features = model.fc.in_featuresmodel.fc = nn.Linear(in_features, num_classes)# 將模型轉移到指定設備（CPU/GPU）model = model.to(device)return model
# 創建ResNet18模型（加載ImageNet預訓練權重，不進行微調）
model = create_resnet18(pretrained=True, num_classes=10)
model.eval()  # 設置為推理模式# 測試單張圖片（示例）
from torchvision import utils# 從測試數據集中獲取一張圖片
dataiter = iter(test_loader)
images, labels = next(dataiter)
images = images[:1].to(device)  # 取第1張圖片# 前向傳播
with torch.no_grad():outputs = model(images)_, predicted = torch.max(outputs.data, 1)# 顯示圖片和預測結果
plt.imshow(utils.make_grid(images.cpu(), normalize=True).permute(1, 2, 0))
plt.title(f"預測類別: {predicted.item()}")
plt.axis('off')
plt.show()

import torch
import torch.nn as nn
import torch.optim as optim
from torchvision import datasets, transforms, models
from torch.utils.data import DataLoader
import matplotlib.pyplot as plt
import os# 設置中文字體支持
plt.rcParams["font.family"] = ["SimHei"]
plt.rcParams['axes.unicode_minus'] = False  # 解決負號顯示問題# 檢查GPU是否可用
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"使用設備: {device}")# 1. 數據預處理（訓練集增強，測試集標準化）
train_transform = transforms.Compose([transforms.RandomCrop(32, padding=4),transforms.RandomHorizontalFlip(),transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),transforms.RandomRotation(15),transforms.ToTensor(),transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))
])test_transform = transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))
])# 2. 加載CIFAR-10數據集
train_dataset = datasets.CIFAR10(root='./data',train=True,download=True,transform=train_transform
)test_dataset = datasets.CIFAR10(root='./data',train=False,transform=test_transform
)# 3. 創建數據加載器
batch_size = 64
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)# 4. 定義ResNet18模型
def create_resnet18(pretrained=True, num_classes=10):model = models.resnet18(pretrained=pretrained)# 修改最后一層全連接層in_features = model.fc.in_featuresmodel.fc = nn.Linear(in_features, num_classes)return model.to(device)# 5. 凍結/解凍模型層的函數
def freeze_model(model, freeze=True):"""凍結或解凍模型的卷積層參數"""# 凍結/解凍除fc層外的所有參數for name, param in model.named_parameters():if 'fc' not in name:param.requires_grad = not freeze# 打印凍結狀態frozen_params = sum(p.numel() for p in model.parameters() if not p.requires_grad)total_params = sum(p.numel() for p in model.parameters())if freeze:print(f"已凍結模型卷積層參數 ({frozen_params}/{total_params} 參數)")else:print(f"已解凍模型所有參數 ({total_params}/{total_params} 參數可訓練)")return model# 6. 訓練函數（支持階段式訓練）
def train_with_freeze_schedule(model, train_loader, test_loader, criterion, optimizer, scheduler, device, epochs, freeze_epochs=5):"""前freeze_epochs輪凍結卷積層，之后解凍所有層進行訓練"""train_loss_history = []test_loss_history = []train_acc_history = []test_acc_history = []all_iter_losses = []iter_indices = []# 初始凍結卷積層if freeze_epochs > 0:model = freeze_model(model, freeze=True)for epoch in range(epochs):# 解凍控制：在指定輪次后解凍所有層if epoch == freeze_epochs:model = freeze_model(model, freeze=False)# 解凍后調整優化器（可選）optimizer.param_groups[0]['lr'] = 1e-4  # 降低學習率防止過擬合model.train()  # 設置為訓練模式running_loss = 0.0correct_train = 0total_train = 0for batch_idx, (data, target) in enumerate(train_loader):data, target = data.to(device), target.to(device)optimizer.zero_grad()output = model(data)loss = criterion(output, target)loss.backward()optimizer.step()# 記錄Iteration損失iter_loss = loss.item()all_iter_losses.append(iter_loss)iter_indices.append(epoch * len(train_loader) + batch_idx + 1)# 統計訓練指標running_loss += iter_loss_, predicted = output.max(1)total_train += target.size(0)correct_train += predicted.eq(target).sum().item()# 每100批次打印進度if (batch_idx + 1) % 100 == 0:print(f"Epoch {epoch+1}/{epochs} | Batch {batch_idx+1}/{len(train_loader)} "f"| 單Batch損失: {iter_loss:.4f}")# 計算 epoch 級指標epoch_train_loss = running_loss / len(train_loader)epoch_train_acc = 100. * correct_train / total_train# 測試階段model.eval()correct_test = 0total_test = 0test_loss = 0.0with torch.no_grad():for data, target in test_loader:data, target = data.to(device), target.to(device)output = model(data)test_loss += criterion(output, target).item()_, predicted = output.max(1)total_test += target.size(0)correct_test += predicted.eq(target).sum().item()epoch_test_loss = test_loss / len(test_loader)epoch_test_acc = 100. * correct_test / total_test# 記錄歷史數據train_loss_history.append(epoch_train_loss)test_loss_history.append(epoch_test_loss)train_acc_history.append(epoch_train_acc)test_acc_history.append(epoch_test_acc)# 更新學習率調度器if scheduler is not None:scheduler.step(epoch_test_loss)# 打印 epoch 結果print(f"Epoch {epoch+1} 完成 | 訓練損失: {epoch_train_loss:.4f} "f"| 訓練準確率: {epoch_train_acc:.2f}% | 測試準確率: {epoch_test_acc:.2f}%")# 繪制損失和準確率曲線plot_iter_losses(all_iter_losses, iter_indices)plot_epoch_metrics(train_acc_history, test_acc_history, train_loss_history, test_loss_history)return epoch_test_acc  # 返回最終測試準確率# 7. 繪制Iteration損失曲線
def plot_iter_losses(losses, indices):plt.figure(figsize=(10, 4))plt.plot(indices, losses, 'b-', alpha=0.7)plt.xlabel('Iteration（Batch序號）')plt.ylabel('損失值')plt.title('訓練過程中的Iteration損失變化')plt.grid(True)plt.show()# 8. 繪制Epoch級指標曲線
def plot_epoch_metrics(train_acc, test_acc, train_loss, test_loss):epochs = range(1, len(train_acc) + 1)plt.figure(figsize=(12, 5))# 準確率曲線plt.subplot(1, 2, 1)plt.plot(epochs, train_acc, 'b-', label='訓練準確率')plt.plot(epochs, test_acc, 'r-', label='測試準確率')plt.xlabel('Epoch')plt.ylabel('準確率 (%)')plt.title('準確率隨Epoch變化')plt.legend()plt.grid(True)# 損失曲線plt.subplot(1, 2, 2)plt.plot(epochs, train_loss, 'b-', label='訓練損失')plt.plot(epochs, test_loss, 'r-', label='測試損失')plt.xlabel('Epoch')plt.ylabel('損失值')plt.title('損失值隨Epoch變化')plt.legend()plt.grid(True)plt.tight_layout()plt.show()# 主函數：訓練模型
def main():# 參數設置epochs = 40  # 總訓練輪次freeze_epochs = 5  # 凍結卷積層的輪次learning_rate = 1e-3  # 初始學習率weight_decay = 1e-4  # 權重衰減# 創建ResNet18模型（加載預訓練權重）model = create_resnet18(pretrained=True, num_classes=10)# 定義優化器和損失函數optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)criterion = nn.CrossEntropyLoss()# 定義學習率調度器scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=2, verbose=True)# 開始訓練（前5輪凍結卷積層，之后解凍）final_accuracy = train_with_freeze_schedule(model=model,train_loader=train_loader,test_loader=test_loader,criterion=criterion,optimizer=optimizer,scheduler=scheduler,device=device,epochs=epochs,freeze_epochs=freeze_epochs)print(f"訓練完成！最終測試準確率: {final_accuracy:.2f}%")# # 保存模型# torch.save(model.state_dict(), 'resnet18_cifar10_finetuned.pth')# print("模型已保存至: resnet18_cifar10_finetuned.pth")if __name__ == "__main__":main()

import torch
import torch.nn as nn
import torch.optim as optim
from torchvision import datasets, transforms
from torchvision.models import resnet18, densenet121
from torchsummary import summary  # 查看模型結構
import matplotlib.pyplot as plt# 設備配置
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")# CIFAR10 數據預處理
transform = transforms.Compose([transforms.RandomCrop(32, padding=4),transforms.RandomHorizontalFlip(),transforms.ToTensor(),transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
train_set = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)
test_set = datasets.CIFAR10(root='./data', train=False, download=True, transform=transforms.ToTensor())
train_loader = torch.utils.data.DataLoader(train_set, batch_size=128, shuffle=True)
test_loader = torch.utils.data.DataLoader(test_set, batch_size=128, shuffle=False)class DenseNetC10(nn.Module):def __init__(self, num_classes=10):super(DenseNetC10, self).__init__()# 壓縮原版 DenseNet121，減少層數和通道數self.features = nn.Sequential(nn.Conv2d(3, 32, kernel_size=3, padding=1, bias=False),nn.BatchNorm2d(32),nn.ReLU(inplace=True),# 3個密集塊，每個塊含3層self._make_dense_block(32, 32, num_layers=3),self._make_dense_block(64, 32, num_layers=3),self._make_dense_block(96, 32, num_layers=3),nn.BatchNorm2d(128),nn.ReLU(inplace=True),nn.AdaptiveAvgPool2d((1, 1)))self.classifier = nn.Linear(128, num_classes)def _make_dense_block(self, in_channels, growth_rate, num_layers):layers = []for _ in range(num_layers):layers.append(nn.Conv2d(in_channels, growth_rate, kernel_size=3, padding=1, bias=False))layers.append(nn.BatchNorm2d(growth_rate))layers.append(nn.ReLU(inplace=True))in_channels += growth_ratereturn nn.Sequential(*layers)def forward(self, x):features = self.features(x)out = features.view(features.size(0), -1)out = self.classifier(out)return out# 初始化模型
models = {'DenseNet-C10': DenseNetC10().to(device),'MobileViT': MobileViT().to(device),'RepVGG': RepVGG().to(device),'ResNet18': resnet18(pretrained=False, num_classes=10).to(device)  # 對比基準
}# 訓練超參數
criterion = nn.CrossEntropyLoss()
accuracies = {}for model_name, model in models.items():print(f'\nTraining {model_name}...')optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4)scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=200)best_acc = 0.0for epoch in range(1, 201):train_model(model, criterion, optimizer, epoch)acc = test_model(model, criterion)if acc > best_acc:best_acc = accaccuracies[model_name] = best_acc# 打印對比結果
print('\nFinal Accuracy Comparison:')
for name, acc in accuracies.items():print(f'{name}: {acc:.2f}%')def visualize_residual(model, data):# 注冊鉤子函數捕捉殘差塊輸出residuals = []def hook(module, input, output):residual = output - input[0]  # 殘差 = 輸出 - 輸入residuals.append(residual.detach().cpu())# 選擇ResNet18的第一個殘差塊（layer1[0]）model.layer1[0].register_forward_hook(hook)with torch.no_grad():model(data.to(device))# 可視化殘差圖（取第一個樣本的第一個通道）residual = residuals[0][0, 0, :, :]  # 形狀(32,32)plt.figure(figsize=(6, 4))plt.subplot(1, 2, 1)plt.imshow(data[0].permute(1, 2, 0))  # 原始圖像plt.title('Input Image')plt.subplot(1, 2, 2)plt.imshow(residual, cmap='coolwarm')  # 殘差熱力圖plt.title('Residual Map')plt.colorbar()plt.show()# 測試殘差可視化（用ResNet18和測試集中的一張圖像）
resnet_model = resnet18(num_classes=10).to(device)
data, _ = next(iter(test_loader))
visualize_residual(resnet_model, data[:1])  # 取第一個樣本

@浙大疏錦行