用XPath來做一個簡單的爬蟲，我們嘗試爬取某個貼吧里的所有帖子，并且將該這個帖子里每個樓層發布的圖片下載到本地。

#coding=utf-8
import requests
from lxml import etree
import jsonclass Tieba:def __init__(self,tieba_name):self.tieba_name = tieba_name #接收貼吧名#設置為手機端的UAself.headers = {"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1"}def get_total_url_list(self):'''獲取所有的urllist'''url = "https://tieba.baidu.com/f?kw="+self.tieba_name+"&ie=utf-8&pn={}&"url_list = []for i in range(100): #通過循環拼接100個urlurl_list.append(url.format(i*50))return url_list #返回100個url的urllistdef parse_url(self,url):'''一個發送請求，獲取響應，同時etree處理html'''print("parsing url:",url)response = requests.get(url,headers=self.headers,timeout=10) #發送請求html = response.content.decode() #獲取html字符串html = etree.HTML(html) #獲取element 類型的htmlreturn htmldef get_title_href(self,url):'''獲取一個頁面的title和href'''html = self.parse_url(url)li_temp_list = html.xpath("//li[@class='tl_shadow']") #分組，按照li標簽分組total_items = []for i in li_temp_list: #遍歷分組href = "https:"+i.xpath("./a/@href")[0] if len(i.xpath("./a/@href"))>0 else Nonetext = i.xpath("./a/div[1]/span[1]/text()")text = text[0] if len(text)>0 else Noneitem = dict(  #放入字典href = href,text = text)total_items.append(item)return total_items #返回一個頁面所有的itemdef get_img(self,url):'''獲取一個帖子里面的所有圖片'''html = self.parse_url(url) #返回elemet累心的html，具有xpath方法img_list = html.xpath('//div[@data-class="BDE_Image"]/@data-url')img_list = [i.split("src=")[-1] for i in img_list] #提取圖片的urlimg_list = [requests.utils.unquote(i) for i in img_list]return img_listdef save_item(self,item):'''保存一個item'''with open("teibatupian.txt","a") as f:f.write(json.dumps(item,ensure_ascii=False,indent=2))f.write("\n")def run(self):#1、找到了url規律，url listurl_list = self.get_total_url_list()for url in url_list:#2、遍歷urllist 發送請求，獲得響應，etree處理html# 3、提取title，hreftotal_item = self.get_title_href(url)for item in total_item:href = item["href"]img_list = self.get_img(href) #獲取到了帖子的圖片列表item["img"] = img_list# 4、保存到本地print(item)self.save_item(item)if __name__ == "__main__":tieba = Tieba("貓")tieba.run()