Python腳本每天爬取微博熱搜-升級版

在這里插入圖片描述

主要優化內容：

定時任務調整：

將定時任務從每小時改為每10分鐘執行一次

調整了請求延遲時間，從1-3秒減少到0.5-1.5秒

縮短了請求超時時間，從10秒減少到8秒

性能優化：

移除了廣告數據的處理，減少不必要的處理

優化了數據結構，減少內存占用

添加了數據清理功能，自動刪除7天前的數據

用戶體驗改進：

HTML頁面添加了自動刷新功能（每5分鐘）

添加了手動刷新按鈕

增加了統計信息顯示（總熱搜數、時間段數等）

優化了移動端顯示效果

代碼健壯性：

添加了異常處理

優化了數據存儲和讀取邏輯

添加了數據清理機制，防止數據無限增長

界面美化：

添加了網站圖標

優化了顏色方案和布局

改進了響應式設計，適配移動設備

這個優化版本每10分鐘獲取一次微博熱搜，并將新數據追加到當天的HTML文件中，同時保持了去重功能。頁面也會每5分鐘自動刷新，確保用戶看到最新數據。

import requests
from bs4 import BeautifulSoup
import time
import os
from datetime import datetime, timedelta
import schedule
import random
import json
import redef fetch_weibo_hot():"""使用API接口獲取微博熱搜數據（避免HTML結構變化問題）"""api_url = "https://weibo.com/ajax/side/hotSearch"headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36','Referer': 'https://weibo.com/','Cookie': 'XSRF-TOKEN=RKnNBEaQBrp=='  # 替換為你的實際Cookie}try:# 添加隨機延遲避免被封time.sleep(random.uniform(0.5, 1.5))response = requests.get(api_url, headers=headers, timeout=8)response.raise_for_status()# 解析JSON數據data = response.json()# 提取熱搜數據hot_items = []for group in data['data']['realtime']:# 普通熱搜項if 'word' in group:item = {'rank': group.get('rank', ''),'title': group['word'],'hot': group.get('num', '0'),'link': f"https://s.weibo.com/weibo?q={group['word']}",'timestamp': datetime.now().strftime("%Y-%m-%d %H:%M:%S"),'source': 'weibo'}hot_items.append(item)# 只保留前50條熱搜return hot_items[:50]except Exception as e:print(f"🚨 獲取熱搜數據失敗: {e}")return []def load_existing_hot_searches(date_str):"""加載當天已存在的熱搜數據"""json_file = os.path.join("weibo_hot", date_str, "hot_searches.json")if os.path.exists(json_file):try:with open(json_file, 'r', encoding='utf-8') as f:return json.load(f)except Exception as e:print(f"讀取已有熱搜數據失敗: {e}")return []return []def save_hot_data_json(hot_data, date_str):"""保存熱搜數據到JSON文件"""daily_dir = os.path.join("weibo_hot", date_str)if not os.path.exists(daily_dir):os.makedirs(daily_dir)json_file = os.path.join(daily_dir, "hot_searches.json")# 加載已有數據existing_data = load_existing_hot_searches(date_str)existing_titles = {item['title'] for item in existing_data}# 過濾掉已存在的熱搜new_data = [item for item in hot_data if item['title'] not in existing_titles]# 合并數據all_data = existing_data + new_data# 保存到JSON文件with open(json_file, 'w', encoding='utf-8') as f:json.dump(all_data, f, ensure_ascii=False, indent=2)return all_data, len(new_data)def generate_html(hot_data, date_str):"""生成HTML文件"""if not hot_data:return "<html><body><h1>未獲取到熱搜數據</h1></body></html>"# 按時間分組熱搜time_groups = {}for item in hot_data:time_key = item['timestamp'][:13]  # 只取到小時if time_key not in time_groups:time_groups[time_key] = []time_groups[time_key].append(item)# 按時間倒序排列sorted_times = sorted(time_groups.keys(), reverse=True)# 統計信息total_count = len(hot_data)time_count = len(time_groups)html_content = f"""<!DOCTYPE html><html lang="zh-CN"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>微博熱搜榜 {date_str}</title><link rel="icon" href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 100 100%22><text y=%22.9em%22 font-size=%2290%22>🔥</text></svg>"><style>* {{box-sizing: border-box;margin: 0;padding: 0;}}body {{ font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "PingFang SC", "Microsoft YaHei", sans-serif; background-color: #f5f8fa;color: #14171a;line-height: 1.5;padding: 20px;max-width: 1200px;margin: 0 auto;}}.container {{background: white;border-radius: 16px;box-shadow: 0 3px 10px rgba(0, 0, 0, 0.08);overflow: hidden;margin-bottom: 30px;}}.header {{ background: linear-gradient(135deg, #ff6b6b, #5e72eb);color: white;padding: 25px 30px;position: relative;}}.title {{font-size: 28px;font-weight: 700;margin-bottom: 5px;}}.subtitle {{font-size: 16px;opacity: 0.9;}}.stats {{margin-top: 15px;display: flex;gap: 15px;flex-wrap: wrap;}}.stat-item {{background: rgba(255, 255, 255, 0.2);padding: 8px 15px;border-radius: 20px;font-size: 14px;}}.update-time {{position: absolute;top: 25px;right: 30px;background: rgba(0, 0, 0, 0.15);padding: 5px 12px;border-radius: 20px;font-size: 14px;}}.time-section {{margin: 20px 0;padding: 15px;background: #f8f9fa;border-radius: 8px;}}.time-header {{font-size: 18px;font-weight: 600;margin-bottom: 15px;color: #495057;display: flex;align-items: center;justify-content: space-between;}}.time-info {{display: flex;align-items: center;}}.time-count {{background: #e9ecef;padding: 2px 8px;border-radius: 10px;font-size: 14px;margin-left: 10px;}}.hot-list {{padding: 0;}}.hot-item {{display: flex;align-items: center;padding: 16px 20px;border-bottom: 1px solid #e6ecf0;transition: background 0.2s;}}.hot-item:hover {{background-color: #f7f9fa;}}.rank {{width: 36px;height: 36px;line-height: 36px;text-align: center;font-weight: bold;font-size: 16px;background: #f0f2f5;border-radius: 8px;margin-right: 15px;flex-shrink: 0;}}.top1 {{ background: linear-gradient(135deg, #ff9a9e, #fad0c4);color: #d63031;}}.top2 {{ background: linear-gradient(135deg, #a1c4fd, #c2e9fb);color: #0984e3;}}.top3 {{ background: linear-gradient(135deg, #ffecd2, #fcb69f);color: #e17055;}}.hot-content {{flex: 1;min-width: 0;}}.hot-title {{font-size: 17px;font-weight: 500;margin-bottom: 6px;white-space: nowrap;overflow: hidden;text-overflow: ellipsis;}}.hot-stats {{display: flex;align-items: center;color: #657786;font-size: 14px;gap: 15px;flex-wrap: wrap;}}.hot-value {{color: #ff6b6b;font-weight: 700;}}.hot-time {{font-size: 12px;color: #868e96;}}.link {{color: #1da1f2;text-decoration: none;transition: color 0.2s;}}.link:hover {{color: #0d8bda;text-decoration: underline;}}.footer {{text-align: center;padding: 20px;color: #657786;font-size: 13px;border-top: 1px solid #e6ecf0;}}.no-data {{text-align: center;padding: 40px;color: #657786;}}.collapse-btn {{background: #6c757d;color: white;border: none;padding: 5px 10px;border-radius: 4px;cursor: pointer;font-size: 12px;}}.collapse-btn:hover {{background: #5a6268;}}.collapsed .hot-list {{display: none;}}.auto-refresh {{text-align: center;margin: 20px 0;}}.refresh-btn {{background: #28a745;color: white;border: none;padding: 10px 20px;border-radius: 5px;cursor: pointer;font-size: 14px;}}.refresh-btn:hover {{background: #218838;}}@media (max-width: 768px) {{body {{padding: 10px;}}.header {{padding: 20px 15px;}}.title {{font-size: 22px;}}.update-time {{position: static;margin-top: 10px;}}.hot-item {{padding: 14px 15px;}}.hot-title {{font-size: 16px;}}.hot-stats {{flex-direction: column;align-items: flex-start;gap: 5px;}}}}</style></head><body><div class="container"><div class="header"><h1 class="title">微博熱搜榜</h1><div class="subtitle">全天熱點匯總 · 每10分鐘更新</div><div class="stats"><div class="stat-item">📊 總熱搜數: {total_count}</div><div class="stat-item">🕒 時間段: {time_count}</div><div class="stat-item">? 更新頻率: 每10分鐘</div></div><div class="update-time">最后更新: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}</div></div>"""# 添加時間分組的熱搜內容for time_key in sorted_times:time_display = f"{time_key}:00"time_data = time_groups[time_key]html_content += f"""<div class="time-section"><div class="time-header"><div class="time-info">🕒 {time_display}<span class="time-count">{len(time_data)} 條熱搜</span></div><button class="collapse-btn" onclick="this.parentElement.parentElement.classList.toggle('collapsed')">折疊/展開</button></div><div class="hot-list">"""for item in time_data:rank_class = ""if item['rank'] == 1:rank_class = "top1"elif item['rank'] == 2:rank_class = "top2"elif item['rank'] == 3:rank_class = "top3"hot_value = f"<span class='hot-value'>🔥 {item['hot']}</span>" if item['hot'] else ""html_content += f"""<div class="hot-item"><div class="rank {rank_class}">{item['rank']}</div><div class="hot-content"><div class="hot-title"><a href="{item['link']}" class="link" target="_blank">{item['title']}</a></div><div class="hot-stats">{hot_value}<span class="hot-time">📅 {item['timestamp']}</span></div></div></div>"""html_content += """</div></div>"""if not hot_data:html_content += """<div class="no-data"><h3>暫無熱搜數據</h3><p>請檢查網絡連接或稍后再試</p></div>"""html_content += f"""<div class="auto-refresh"><button class="refresh-btn" onclick="location.reload()">🔄 刷新頁面</button><p>頁面每10分鐘自動更新，也可手動刷新</p></div><div class="footer">數據來源: 微博熱搜 ? 每10分鐘自動更新 ? 最后更新: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")} ? 僅供學習參考</div></div><script>// 默認折疊所有時間段的搜索結果document.addEventListener('DOMContentLoaded', function() {{var sections = document.querySelectorAll('.time-section');sections.forEach(function(section, index) {{if (index > 0) {{ // 保持最新時間段展開section.classList.add('collapsed');}}}});// 設置自動刷新（每5分鐘）setTimeout(function() {{location.reload();}}, 5 * 60 * 1000);}});</script></body></html>"""return html_contentdef cleanup_old_data():"""清理7天前的數據"""try:now = datetime.now()cutoff_date = (now - timedelta(days=7)).strftime("%Y-%m-%d")weibo_hot_dir = "weibo_hot"if os.path.exists(weibo_hot_dir):for date_dir in os.listdir(weibo_hot_dir):if date_dir < cutoff_date:dir_path = os.path.join(weibo_hot_dir, date_dir)import shutilshutil.rmtree(dir_path)print(f"🗑? 已清理過期數據: {date_dir}")except Exception as e:print(f"清理舊數據時出錯: {e}")def save_hot_data():"""保存熱搜數據"""try:# 創建存儲目錄if not os.path.exists("weibo_hot"):os.makedirs("weibo_hot")# 獲取當前日期now = datetime.now()date_str = now.strftime("%Y-%m-%d")print(f"🕒 開始獲取 {now.strftime('%Y-%m-%d %H:%M:%S')} 的熱搜數據...")hot_data = fetch_weibo_hot()if hot_data:print(f"? 成功獲取 {len(hot_data)} 條熱搜數據")# 保存到JSON文件并獲取所有數據all_data, new_count = save_hot_data_json(hot_data, date_str)print(f"📊 已有 {len(all_data)} 條熱搜，新增 {new_count} 條")# 生成HTML內容html_content = generate_html(all_data, date_str)# 保存HTML文件html_file = os.path.join("weibo_hot", date_str, "index.html")with open(html_file, "w", encoding="utf-8") as f:f.write(html_content)print(f"💾 已保存到: {html_file}")# 每周清理一次舊數據#if now.weekday() == 0 and now.hour == 0 and now.minute < 10:  # 每周一凌晨#    cleanup_old_data()else:print("?? 未獲取到熱搜數據，跳過保存")except Exception as e:print(f"? 保存數據時出錯: {e}")def job():"""定時任務"""print("\n" + "="*60)print(f"? 執行定時任務: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")save_hot_data()print("="*60 + "\n")if __name__ == "__main__":print("🔥 微博熱搜爬蟲已啟動 🔥")print("? 首次執行將立即運行，之后每10分鐘執行一次")print("💡 提示: 請確保已更新有效的Cookie")print("="*60)# 立即執行一次job()# 設置定時任務（每10分鐘執行一次）schedule.every(10).minutes.do(job)# 每天凌晨清理一次舊數據schedule.every().day.at("00:00").do(cleanup_old_data)print("? 程序運行中，按Ctrl+C退出...")try:# 保持程序運行while True:schedule.run_pending()time.sleep(30)  # 每30秒檢查一次except KeyboardInterrupt:print("\n👋 程序已手動停止")