本文繼續接著我的上一篇博客【Python爬蟲】簡單案例介紹3-CSDN博客
目錄
3.4 完整代碼
3.4 完整代碼
此小節給出上述案例的完整代碼，
# encoding=utf-8
import re, json, requests, xlwt, csv
import pandas as pd
from lxml import etree
from bs4 import BeautifulSoup
from openpyxl import Workbook
import numpy as np"""
爬取科普中國-圖文
"""class MySpider(object):"""科普中國-圖文"""def __init__(self):self.base_url = 'https://cloud.kepuchina.cn/newSearch/imageText?s=&start_time=&end_time=&type=1&keyword=&can_down=0&category_id=0&size=21&sort_rule=0&industry_category=0&subject_category=0&kp_category=0&crowd_category=0&spread_category=0&page='self.url = self.base_url + str(0)self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36 Edg/92.0.902.67'}self.index_list = []self.index_article = {}def get(self, url):"""請求并返回網頁源代碼"""try:response = requests.get(url, self.headers)if response.status_code == 200:return response.textexcept Exception as err:print('get():', err)def parse(self, start_page, pages_num):"""解析科普中國網站地址url:param start_page: 開始頁面:param pages_num: 想要爬取的頁面數量:return: """for page in range(start_page, start_page+pages_num):# 將獲取的頁面源碼加載到該對象中soup = BeautifulSoup(self.get(self.base_url + str(page)), 'html.parser')# 拿到數據列表for i in soup.findAll('div', class_="list-block _blockParma"):# 創建 BeautifulSoup 對象soup_i = BeautifulSoup(str(i), 'html.parser')# 提取文章標題和url、副標題、tag、發布者、發布時間title = soup_i.find('a', class_='_title').texttitle_url = soup_i.find('a', class_='_title')['href']subtitle = soup_i.find('p', class_='info').find('a').texttags = [a.text for a in soup_i.find_all('a', class_='typeColor')]publisher = soup_i.find('a', class_='source-txt').text.strip()publish_time = soup_i.find('span', class_='_time').textself.index_article = {"title": title, "title_url": title_url, "subtitle": self.clean(subtitle), "tag": tags, "publisher": publisher, "publish_time": publish_time}# 獲得文章內容文本content和圖片數量以及地址self.parse_page(title_url) if self.index_article not in self.index_list: # 存入列表self.index_list.append(self.index_article)print("已完成" + str(page+1) + "頁的存儲")# self.get_json(str(self.index_list), "1.json")self.save_excel(self.index_list, "result_" + str(start_page) + "_" + str(pages_num) + ".xlsx")def get_json(self, datas_list, filename):"""將列表存儲為json文件:param datas_list: 文章數據列表:param filename: json文件名稱:return:"""with open(filename, 'w') as f:f.write(datas_list)def save_excel(self, inputData, outPutFile):'''將列表數據寫入excel表格文件inputData: 列表，含有多個字典;例如：[{'key_a':'123'},{'key_b':'456'}]outPutFile：輸出文件名，例如：'data.xlsx''''Lable = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L']wb = Workbook()sheet = wb.activesheet.title = "Sheet1"item_0 = inputData[0]i = 0for key in item_0.keys():sheet[Lable[i] + str(1)].value = keyi = i + 1j = 1for item in inputData:k = 0for key in item:try:sheet[Lable[k] + str(j + 1)].value = item[key]except:item[key] = [str(w) for w in item[key]]sheet[Lable[k] + str(j + 1)].value = ' '.join(item[key])k = k + 1j = j + 1wb.save(outPutFile)print('數據寫入完畢!')def parse_page(self, title_url):"""進一步解析頁面，得到頁面的文本content、圖片數量以及地址:param title_url: 文章標題的網頁地址:return:"""response = requests.get(title_url, headers=self.headers)try:if response.status_code == 200:soup = BeautifulSoup(response.text, 'html.parser')# 獲取文章主體內容，根據新HTML結構調整選擇器content_div = soup.find('div', class_='content-box __imgtext-content')if content_div:content = self.clean(content_div.text)else:content = ""# 圖片數量以及地址，過濾掉不需要的圖片來源（如含特定關鍵詞的圖片）img_url = []all_imgs = soup.find_all('img')for img in all_imgs:src = img.get('src')if src and 'kepuyun' in src and 'logo' not in src and 'wechat' not in src and 'weibo' not in src:img_url.append(src)img_num = len(img_url)self.index_article["content"] = contentself.index_article["img_num"] = img_numself.index_article["img_url"] = img_urlelse:print(f"請求失敗，狀態碼: {response.status_code}")except Exception as err:print('parse_page:', err)def clean(self, text):"""清理文本"""text = re.sub(r'\n|\r', '', text).strip().replace(r"\n", "")text = text.split('\ue62b')[0]return textdef main(self):"""主函數:return: """self.parse(0, 1)if __name__ == "__main__":spider = MySpider()spider.main()OK。
