import osimport zipfilefrom urllib.request import urlretrievefrom tqdm import tqdmclass TqdmUpTo(tqdm):def update_to(self, b=1, bsize=1, tsize=None):if tsize is not None:self.total = tsizeself.update(b * bsize - self.n)url = "https://object.pouta.csc.fi/OPUS-CCAligned/v1/moses/en-zh_CN.txt.zip"save_path = "en-zh.txt.zip"extract_dir = "/kaggle/working/CC"os.makedirs(extract_dir, exist_ok=True)if not os.path.exists(save_path):print(f"開始從 {url} 下載文件...")with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t:urlretrieve(url, save_path, reporthook=t.update_to)print(f"下載完成并保存到 {save_path}")else:print(f"文件已存在: {save_path}")print(f"開始解壓文件到 {extract_dir}...")with zipfile.ZipFile(save_path, 'r') as zip_ref:zip_ref.extractall(extract_dir)print(f"文件已解壓到：{extract_dir}")

import osimport reimport pandas as pddef read_lines(file_path):with open(file_path, "r", encoding="utf-8") as f:return [line.strip() for line in f if line.strip()]en_path = "/kaggle/working/CC/CCAligned.en-zh_CN.en"zh_path = "/kaggle/working/CC/CCAligned.en-zh_CN.zh_CN"en_lines = read_lines(en_path)zh_lines = read_lines(zh_path)min_len = min(len(en_lines), len(zh_lines))en_lines = en_lines[:min_len]zh_lines = zh_lines[:min_len]df = pd.DataFrame({"en": en_lines, "zh": zh_lines})

def find_extreme_sentences(column, length_column):min_idx = df[length_column].idxmin()max_idx = df[length_column].idxmax()min_sentence = df.loc[min_idx, column]max_sentence = df.loc[max_idx, column]min_length = df.loc[min_idx, length_column]max_length = df.loc[max_idx, length_column]return min_sentence, min_length, max_sentence, max_lengthdf['en_length'] = df['en'].apply(lambda x: len(x.split()))?df['zh_length'] = df['zh'].apply(lambda x: len(x))????????print("數據基本統計：")print(f"英文句子平均長度: {df['en_length'].mean():.2f}")print(f"中文句子平均長度: {df['zh_length'].mean():.2f}")min_en, min_en_len, max_en, max_en_len = find_extreme_sentences('en', 'en_length')min_zh, min_zh_len, max_zh, max_zh_len = find_extreme_sentences('zh', 'zh_length')print("\n最短和最長句子：")print(f"最短英文句子({min_en_len}詞): {min_en}")print(f"最長英文句子({max_en_len}詞): {max_en}")print(f"最短中文句子({min_zh_len}字符): {min_zh}")print(f"最長中文句子({max_zh_len}字符): {max_zh}")print("\n結果驗證：")print(f"英文最短句子實際分詞驗證: {len(min_en.split())}")print(f"英文最長句子實際分詞驗證: {len(max_en.split())}")print(f"中文最短句子實際長度驗證: {len(min_zh)}")
print(f"中文最長句子實際長度驗證: {len(max_zh)}")

import matplotlib.pyplot as pltimport seaborn as snsdef plot_length_distribution(df,xlim):xlim=xlimplt.figure(figsize=(10, 6))sns.histplot(df['en_length'], kde=True, bins=30,color='#99ffff', alpha=0.6, label='English (Words)')sns.histplot(df['zh_length'], kde=True, bins=30,color='#99b3ff', alpha=0.6, label='Chinese (Characters)')max_len = max(df['en_length'].max(), df['zh_length'].max())plt.xlim(0, max_len * 1.1)plt.xlim(0,xlim)plt.title('Sentence Length Distribution', fontsize=14, fontweight='bold')plt.xlabel('Length', fontsize=12)plt.ylabel('Frequency', fontsize=12)plt.legend(fontsize=12)plt.tight_layout()plt.show()plot_length_distribution(df,600)

from scipy.stats import skewdef analyze_skewness(column):skewness = skew(column, nan_policy='omit')if skewness > 1:return f"高度右偏態（偏度值: {skewness:.2f}）"elif 0.5 <= skewness <= 1:return f"中度右偏態（偏度值: {skewness:.2f}）"elif skewness < -1:return f"高度左偏態（偏度值: {skewness:.2f}）"elif -1 <= skewness <= -0.5:return f"中度左偏態（偏度值: {skewness:.2f}）"else:return f"近似正態分布（偏度值: {skewness:.2f}）"

def filter_by_length(df,en_min_words=3,?en_max_words=50,?zh_min_chars=3,?zh_max_chars=150，min_ratio=0.5,??max_ratio=2.0,??enforce_ratio=True):if not all(col in df.columns for col in ['en_length', 'zh_length']):raise ValueError("DataFrame必須包含'en_length'和'zh_length'列")if en_min_words >= en_max_words:raise ValueError("en_min_words必須小于en_max_words")if zh_min_chars >= zh_max_chars:raise ValueError("zh_min_chars必須小于zh_max_chars")ratio = df['zh_length'] / df['en_length'].replace(0, 1e-6)mask = ((df['en_length'] >= en_min_words) &(df['en_length'] <= en_max_words) &(df['zh_length'] >= zh_min_chars) &(df['zh_length'] <= zh_max_chars))if enforce_ratio:mask &= (ratio >= min_ratio) & (ratio <= max_ratio)return df.loc[mask].copy()

def clean_duplicates(df):df = df.dropna(subset=['en', 'zh'])?df = df[(df['en'] != "") & (df['zh'] != "")]?df = df.drop_duplicates(subset=['en', 'zh'])?return df

import redef clean_text(text, lang):text = re.sub(r'<[^>]+>', '', text)?text = re.sub(r'&[a-z]+;', '', text)?if lang == 'zh':text = re.sub(r'[^\u4e00-\u9fa5\w\s,.!?，。！？]', '', text)text = re.sub(r'\s+', ' ', text)elif lang == 'en':text = re.sub(r'[^\w\s,.!?]', '', text)text = re.sub(r'\s+', ' ', text)return text.strip()

def clean_mixed_language(df, en_col='en', zh_col='zh',chinese_threshold=0.2, english_threshold=0.1):df_clean = df.copy()to_drop = set()for idx, row in df_clean.iterrows():en_text = str(row[en_col])zh_in_en = len(re.findall(r'[\u4e00-\u9fa5]', en_text))en_length = len(en_text)zh_ratio_in_en = zh_in_en / en_length if en_length > 0 else 0zh_text = str(row[zh_col])en_in_zh = len(re.findall(r'[a-zA-Z]', zh_text))zh_length = len(zh_text)en_ratio_in_zh = en_in_zh / zh_length if zh_length > 0 else 0if (zh_ratio_in_en > chinese_threshold) or (en_ratio_in_zh > english_threshold):to_drop.add(idx)# 成對刪除混雜的句子df_clean = df_clean.drop(index=to_drop).reset_index(drop=True)print(f"原始數據量: {len(df)}")print(f"刪除混雜句子數量: {len(to_drop)}")print(f"清理后數據量: {len(df_clean)}")print(f"中英文句子數量保持: {len(df_clean[en_col])} == {len(df_clean[zh_col])}")return df_clean

print('截斷過長、過短句子')df=filter_by_length(df,en_min_words=1,en_max_words=100,?zh_min_chars=1,zh_max_chars=150,min_ratio=0.67,?max_ratio=1.5? )print(df.shape)print("過濾空行和重復句對...")df = clean_duplicates(df)print(df.shape)print("開始清洗中文文本...")df['zh'] = df['zh'].apply(lambda x: clean_text(x, 'zh'))print("開始清洗英文文本...")df['en'] = df['en'].apply(lambda x: clean_text(x, 'en'))print(df.shape)print("檢查中英句子混雜")df=clean_mixed_language(df, en_col='en', zh_col='zh',chinese_threshold=0.2, english_threshold=0.1)print(df.shape)print(f"清洗后剩余數據形狀：{df.shape} 條")print("清洗后前5條示例：")print(df.head())

plt.figure(figsize=(10, 6))plt.scatter(df['en_length'], df['zh_length'], c='#99b3ff', alpha=0.6, s=100, edgecolors='w', linewidth=1)plt.title('The correlation of sequence lengths', fontsize=14, pad=20,fontweight='bold')plt.xlabel('Length of English', fontsize=12)plt.ylabel('Length of Chinese', fontsize=12)plt.grid(True, linestyle='--', alpha=0.6)plt.show()

import pandas as pdfrom wordcloud import WordCloudimport matplotlib.pyplot as pltfrom nltk.corpus import stopwordsfrom collections import Counterimport nltknltk.download('stopwords')def generate_english_wordcloud(df, text_col='en', max_words=200, custom_stopwords=None):stop_words = set(stopwords.words('english'))stop_words.update(["'s", "n't", "'m", "'re", "'ve"])? # 追加英文縮寫if custom_stopwords:stop_words.update(custom_stopwords)text = ' '.join(df[text_col].dropna().astype(str))words = [word.lower() for word in text.split()if (word.lower() not in stop_words andlen(word) > 2 andword.isalpha())]wc = WordCloud(width=800,height=600,background_color='white',max_words=max_words,colormap='viridis',collocations=False).generate(' '.join(words))plt.figure(figsize=(10, 6))plt.imshow(wc, interpolation='bilinear')plt.axis('off')plt.title("English Word Cloud ", fontsize=18, pad=20,fontweight='bold')plt.tight_layout()plt.show()top_words = Counter(words).most_common(10)print("Top 10高頻詞示例：")print(top_words)generate_english_wordcloud(df)

from torch.utils.data import Datasetimport torchclass TranslationDataset(Dataset):def __init__(self, df, tokenizer, max_length=128):self.df = dfself.tokenizer = tokenizerself.max_length = max_lengthdef __len__(self):return len(self.df)def __getitem__(self, idx):# 獲取源語言（中文）和目標語言（英文）句子source_text = self.df.iloc[idx]['zh']target_text = self.df.iloc[idx]['en']# 對源語言和目標語言進行編碼encoding = self.tokenizer(source_text,padding='max_length',truncation=True,max_length=self.max_length,return_tensors='pt')# 對目標語言進行編碼（注意：這里使用target_text作為labels）target_encoding = self.tokenizer(target_text,padding='max_length',truncation=True,max_length=self.max_length,return_tensors='pt')# 準備模型輸入inputs = {'input_ids': encoding['input_ids'].flatten(),'attention_mask': encoding['attention_mask'].flatten(),'labels': target_encoding['input_ids'].flatten()}return inputs

from sklearn.model_selection import train_test_splittrain_df,temp_df= train_test_split(df, test_size=0.2, random_state=42)val_df,test_df=train_test_split(temp_df,test_size=0.5,random_state=42)

!pip uninstall -y transformers tokenizers sentencepiece!pip install transformers==4.37.0 tokenizers sentencepiece

from torch.utils.data import Dataset, DataLoaderfrom transformers import AutoTokenizer,AutoModelForSeq2SeqLMtokenizer = AutoTokenizer.from_pretrained("Helsinki-NLP/opus-mt-zh-en")model = AutoModelForSeq2SeqLM.from_pretrained("Helsinki-NLP/opus-mt-zh-en")

train_dataset = TranslationDataset(train_df, tokenizer)val_dataset = TranslationDataset(val_df, tokenizer)test_dataset = TranslationDataset(test_df, tokenizer)

batch_size = 32train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)val_dataloader = DataLoader(val_dataset, batch_size=batch_size)test_dataloader = DataLoader(test_dataset, batch_size=batch_size)sample_batch = next(iter(train_dataloader))print('數據加載器測試:')print(f"輸入ID形狀: {sample_batch['input_ids'].shape}")print(f"注意力掩碼形狀: {sample_batch['attention_mask'].shape}")print(f"標簽形狀: {sample_batch['labels'].shape}")

from torch import nnfrom torch.optim import Optimizerfrom torch.optim.lr_scheduler import _LRSchedulerfrom torch.cuda.amp import GradScalerfrom tqdm import tqdmdef train_epoch(model, dataloader, optimizer: Optimizer, scheduler: _LRScheduler, device, epoch, scaler: GradScaler, gradient_accumulation_steps=1):model.train()total_loss = 0.0progress_bar = tqdm(enumerate(dataloader), total=len(dataloader), desc=f'Epoch {epoch+1}')for step, batch in progress_bar:# 數據移動至設備（顯式轉換，避免隱式拷貝）input_ids = batch['input_ids'].to(device, non_blocking=True)attention_mask = batch['attention_mask'].to(device, non_blocking=True)labels = batch['labels'].to(device, non_blocking=True)# 混合精度前向傳播（顯式上下文管理器）with torch.amp.autocast(device_type='cuda'):? # 更新為新的 autocast 調用方式outputs = model(input_ids=input_ids,attention_mask=attention_mask,labels=labels)# 梯度累積：計算當前批次平均損失loss = outputs.loss / gradient_accumulation_steps# 梯度縮放與反向傳播scaler.scale(loss).backward()total_loss += loss.item()? # 記錄原始損失（非平均后的值，便于后續分析）# 梯度更新條件（使用數學取模，邏輯更清晰）if (step + 1) % gradient_accumulation_steps == 0:scaler.step(optimizer)?? # 混合精度更新參數scaler.update()????????? # 更新縮放因子optimizer.zero_grad(set_to_none=True)? # 更高效的梯度清零方式scheduler.step()???????? # 基于步數的調度器更新# 清空進度條緩存（避免顯存泄漏）progress_bar.reset()# 返回 epoch 平均損失（總損失/總批次，而非總樣本數）avg_loss = total_loss / len(dataloader)return avg_loss

from tqdm import tqdmfrom transformers import AdamW, get_linear_schedule_with_warmupfrom torch.cuda.amp import GradScaler, autocastimport numpy as npfrom torch.utils.tensorboard import SummaryWriter# 設置設備device = torch.device("cuda" if torch.cuda.is_available() else "cpu")model.to(device)print(f"使用設備: {device}")# 優化器optimizer = AdamW(model.parameters(), lr=5e-5)# 訓練總步數和預熱步數num_epochs = 10 # 增加訓練輪數total_steps = len(train_dataloader) * num_epochswarmup_steps = int(total_steps * 0.1)? # 10% 的預熱步數# 學習率調度器scheduler = get_linear_schedule_with_warmup(optimizer,num_warmup_steps=warmup_steps,num_training_steps=total_steps)# 混合精度訓練的梯度縮放器scaler = GradScaler()# 創建TensorBoard寫入器writer = SummaryWriter(log_dir='./runs/translation_experiment')

import torchimport numpy as npfrom tqdm import tqdmfrom typing import Dict, Listdef evaluate(model, dataloader, device, max_samples=None):model.eval()predictions = []references = []total_loss = 0? # 這里用于累積驗證集損失num_samples = 0dataloader_iter = iter(dataloader)with torch.no_grad():for _ in tqdm(range(len(dataloader)), desc='Evaluating'):try:batch = next(dataloader_iter)except StopIteration:break# 準備輸入input_ids = batch['input_ids'].to(device)attention_mask = batch['attention_mask'].to(device)labels = batch['labels'].to(device)# 計算損失（和訓練集使用相同的方式）outputs = model(input_ids=input_ids,attention_mask=attention_mask,labels=labels)loss = outputs.losstotal_loss += loss.item() * labels.size(0)? # 累積批次損失# 生成翻譯結果generated_ids = model.generate(input_ids=input_ids,attention_mask=attention_mask,max_length=128,? # 減少最大長度num_beams=4,??? # 降低beam sizeearly_stopping=True)# 解碼預測和參考句子batch_predictions = [tokenizer.decode(g, skip_special_tokens=True) for g in generated_ids]batch_references = [tokenizer.decode(l, skip_special_tokens=True) for l in labels]predictions.extend(batch_predictions)references.extend(batch_references)# 限制評估樣本數量num_samples += len(batch_predictions)if max_samples is not None and num_samples >= max_samples:break# 計算平均驗證集損失avg_loss = total_loss / num_samples if num_samples > 0 else 0# 計算困惑度（這里困惑度計算邏輯保持不變，基于之前的方式）# 計算總token數（和之前計算困惑度時的邏輯一致）total_tokens = sum([len(tokenizer.encode(ref)) for ref in references])perplexity = np.exp(total_loss / total_tokens) if total_tokens > 0 else 0# 計算其他評估指標metrics = {}try:# BLEU分數 (sacrebleu標準格式)from sacrebleu import corpus_bleubleu = corpus_bleu(predictions, [references])? # sacrebleu需要參考譯文列表的列表metrics['BLEU'] = bleu.scoreprint(bleu.format())? # 打印標準BLEU格式輸出except ImportError:print("sacrebleu未安裝，無法計算標準BLEU分數。")try:# chrF分數 (sacrebleu標準格式)from sacrebleu import corpus_chrfchrf = corpus_chrf(predictions, [references])metrics['chrF'] = chrf.scoreprint(chrf.format())? # 打印標準chrF格式輸出except ImportError:print("sacrebleu未安裝，無法計算標準chrF分數。")try:# TER分數 (sacrebleu)from sacrebleu import corpus_terter = corpus_ter(predictions, [references])metrics['TER'] = ter.scoreprint(ter.format())? # 打印標準TER格式輸出except ImportError:print("sacrebleu未安裝，無法計算TER分數。")try:# ROUGE分數 (rouge-score)from rouge_score import rouge_scorerscorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)# 計算每個樣本的rouge分數并取平均值rouge_scores = {'rouge1': [], 'rouge2': [], 'rougeL': []}for pred, ref in zip(predictions, references):scores = scorer.score(ref, pred)for key in rouge_scores:rouge_scores[key].append(scores[key].fmeasure)# 計算平均ROUGE分數for key in rouge_scores:metrics[key.upper()] = np.mean(rouge_scores[key])print(f"ROUGE-{key.upper()}: {metrics[key.upper()]:.4f}")except ImportError:print("rouge-score未安裝，無法計算ROUGE分數。")# 打印困惑度print(f"\n困惑度 (Perplexity): {perplexity:.4f}")# 將平均驗證集損失添加到指標字典中metrics['validation_loss'] = avg_lossreturn {'perplexity': perplexity,**metrics}