Python+Selenium自動化爬取攜程動態加載游記

1. 引言

在旅游行業數據分析、輿情監測或競品研究中，獲取攜程等平臺的游記數據具有重要價值。然而，攜程的游記頁面通常采用動態加載（Ajax、JavaScript渲染），傳統的**Requests**+**BeautifulSoup**方案難以直接獲取完整數據。

解決方案：使用**Selenium**模擬瀏覽器行為，配合**BeautifulSoup**或**lxml**解析動態加載的游記內容。本文將詳細介紹如何利用**Python+Selenium**爬取攜程動態加載的游記，并存儲至**CSV**文件。

2. 技術選型與工具準備

2.1 技術棧

Python 3.8+（推薦使用最新穩定版）
Selenium（瀏覽器自動化工具）
BeautifulSoup4（HTML解析庫）
Pandas（數據存儲與處理）
ChromeDriver（與Chrome瀏覽器配合使用）

2.2 環境安裝

2.3 下載瀏覽器驅動

ChromeDriver：下載地址（需與本地Chrome版本匹配）
GeckoDriver（Firefox）：下載地址

3. 爬取攜程動態加載游記的步驟

3.1 分析攜程游記頁面結構

目標URL示例（以“北京”為例）：

https://you.ctrip.com/travels/beijing1/t3.html

關鍵觀察點：

動態加載：游記列表通過滾動或點擊“加載更多”動態獲取。
Ajax請求：可通過瀏覽器開發者工具（F12→Network→XHR）查看數據接口。
反爬機制：
- User-Agent檢測
- IP限制（需代理或控制請求頻率）
- 登錄驗證（部分內容需登錄）

3.2 Selenium 模擬瀏覽器操作

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import pandas as pd
from bs4 import BeautifulSoup# 配置ChromeDriver路徑
driver_path = "chromedriver.exe"  # 替換為你的驅動路徑
options = webdriver.ChromeOptions()
options.add_argument("--headless")  # 無頭模式（可選）
options.add_argument("--disable-gpu")
options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")driver = webdriver.Chrome(executable_path=driver_path, options=options)

3.3 訪問目標頁面并滾動加載數據

def scroll_to_bottom(driver, max_scroll=5):"""模擬滾動加載"""for _ in range(max_scroll):driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")time.sleep(2)  # 等待數據加載url = "https://you.ctrip.com/travels/beijing1/t3.html"
driver.get(url)
scroll_to_bottom(driver)  # 滾動加載更多游記

3.4 解析游記數據

def parse_travel_notes(driver):soup = BeautifulSoup(driver.page_source, 'html.parser')notes = soup.find_all('div', class_='journalslist')  # 游記列表容器data = []for note in notes:title = note.find('a', class_='journal-title').get_text(strip=True)author = note.find('a', class_='nickname').get_text(strip=True)date = note.find('span', class_='time').get_text(strip=True)views = note.find('span', class_='num').get_text(strip=True)content = note.find('p', class_='journal-content').get_text(strip=True)data.append({"標題": title,"作者": author,"發布時間": date,"閱讀量": views,"內容摘要": content})return datatravel_data = parse_travel_notes(driver)

3.5 存儲數據至CSV

df = pd.DataFrame(travel_data)
df.to_csv("ctrip_travel_notes.csv", index=False, encoding="utf_8_sig")  # 避免中文亂碼

4. 完整代碼實現

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
import pandas as pd
import time
from selenium.webdriver.chrome.options import Optionsdef scroll_to_bottom(driver, max_scroll=5):"""模擬滾動加載"""for _ in range(max_scroll):driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")time.sleep(2)def parse_travel_notes(driver):"""解析游記數據"""soup = BeautifulSoup(driver.page_source, 'html.parser')notes = soup.find_all('div', class_='journalslist')data = []for note in notes:title = note.find('a', class_='journal-title').get_text(strip=True)author = note.find('a', class_='nickname').get_text(strip=True)date = note.find('span', class_='time').get_text(strip=True)views = note.find('span', class_='num').get_text(strip=True)content = note.find('p', class_='journal-content').get_text(strip=True)data.append({"標題": title,"作者": author,"發布時間": date,"閱讀量": views,"內容摘要": content})return datadef main():# 代理配置proxyHost = "www.16yun.cn"proxyPort = "5445"proxyUser = "16QMSOML"proxyPass = "280651"# 初始化瀏覽器配置options = Options()# 設置User-Agentoptions.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")# 設置代理（帶認證）proxy_options = {'proxy': {'http': f'http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}','https': f'https://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}','no_proxy': 'localhost,127.0.0.1'}}# 添加代理擴展（適用于需要認證的代理）from selenium.webdriver.chrome.service import Servicefrom selenium.webdriver.common.proxy import Proxy, ProxyType# 方法1：使用ChromeOptions添加代理（基礎方法，可能不支持認證）# options.add_argument(f'--proxy-server=http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}')# 方法2：使用插件方式添加認證代理（推薦）# 需要先創建一個代理認證插件manifest_json = """{"version": "1.0.0","manifest_version": 2,"name": "Chrome Proxy","permissions": ["proxy","tabs","unlimitedStorage","storage","<all_urls>","webRequest","webRequestBlocking"],"background": {"scripts": ["background.js"]},"minimum_chrome_version":"22.0.0"}"""background_js = """var config = {mode: "fixed_servers",rules: {singleProxy: {scheme: "http",host: "%s",port: parseInt(%s)},bypassList: ["localhost"]}};chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});function callbackFn(details) {return {authCredentials: {username: "%s",password: "%s"}};}chrome.webRequest.onAuthRequired.addListener(callbackFn,{urls: ["<all_urls>"]},['blocking']);""" % (proxyHost, proxyPort, proxyUser, proxyPass)# 創建臨時插件目錄import osimport tempfileimport zipfileplugin_dir = tempfile.mkdtemp()with open(os.path.join(plugin_dir, "manifest.json"), 'w') as f:f.write(manifest_json)with open(os.path.join(plugin_dir, "background.js"), 'w') as f:f.write(background_js)# 打包插件proxy_plugin_path = os.path.join(plugin_dir, "proxy_auth_plugin.zip")with zipfile.ZipFile(proxy_plugin_path, 'w') as zp:zp.write(os.path.join(plugin_dir, "manifest.json"), "manifest.json")zp.write(os.path.join(plugin_dir, "background.js"), "background.js")# 添加插件到ChromeOptionsoptions.add_extension(proxy_plugin_path)try:# 初始化瀏覽器（帶代理）driver = webdriver.Chrome(executable_path="chromedriver.exe", options=options)# 訪問頁面并滾動加載url = "https://you.ctrip.com/travels/beijing1/t3.html"driver.get(url)scroll_to_bottom(driver)# 解析數據travel_data = parse_travel_notes(driver)# 存儲數據df = pd.DataFrame(travel_data)df.to_csv("ctrip_travel_notes.csv", index=False, encoding="utf_8_sig")print("數據爬取完成，已保存至 ctrip_travel_notes.csv")finally:driver.quit()# 清理臨時插件文件import shutilshutil.rmtree(plugin_dir)if __name__ == "__main__":main()