爬取前塵無憂python職位信息并保存到mongo數據庫

１．re實現

import re,os
import requests
from requests.exceptions import RequestException

MAX_PAGE = 10 #最大頁數
KEYWORD = 'python'
headers = {
    'User-Agent':
        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
}
file_name = 're_job51_python.txt'

# 獲取網頁源碼
def getHtml(page):
    try:
        url = 'https://search.51job.com/list/040000,000000,0000,00,9,99,{0},2,{1}.html?'.format(KEYWORD,page)
        response = requests.get(url,headers=headers)
        response.encoding = response.apparent_encoding
        return response.text
    except RequestException:
        print('請求出錯')
        return None

# 解析網頁源碼，得到目標信息
def getTarget(html):
    reg = re.compile(
        r'class="t1 ">.*? <a target="_blank" '
        'title="(.*?)".*? <span class="t2"><a target="_blank" '
        'title="(.*?)".*?<span '
        'class="t3">(.*?)</span>.*?<span '
        'class="t4">(.*?)</span>.*? <span '
        'class="t5">(.*?)</span>',
        re.S)  # 匹配換行符
    target = re.findall(reg,html)
    return target


# 保存到文本中
def save_to_txt(item):
    with open(file_name,'a',newline='') as f:  # newline參數防止兩行之間有空行
        for i in range(len(item)):
            # 最后一個元素換行，非最后則以＇,'隔開
            if i == len(item)-1:
                f.write(item[i])
                f.write('\n')
            else:
                f.write(item[i]+',')

def main():
    # 每次執行前檢查文件是否存在，存在則刪除
    if os.path.exists(file_name):
        os.remove(file_name)

    # 分頁爬取
    for page in range(MAX_PAGE+1):
        html = getHtml(page)
        content = getTarget(html)
        for item in content:
            save_to_txt(item)

if __name__ == '__main__':
    main()

?２．xpath實現

import os
import requests
from requests.exceptions import RequestException
from lxml import etree
import pymongo
from spiders.前程無憂.mongo_config import *

# mongo數據庫設置
client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_DB]

MAX_PAGE = 5
KEYWORD = 'python'
headers = {
    'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '\
    'Chrome/63.0.3239.132 Safari/537.36'
}
file_name = 'xpath_job51_python.txt'

# 獲取網頁
def get_html(page):
    try:
        url = 'https://search.51job.com/list/040000,000000,0000,00,9,99,{},2,{}.html?'.format(KEYWORD,page)
        response = requests.get(url,headers=headers)
        response.encoding = response.apparent_encoding
        return response.text
    except RequestException:
        return None

# 解析網頁
def parse_html(html):
    # 構造xpath解析對象，可自動修整HTML文本
    html = etree.HTML(html)
    # 獲取文本 /text()
    # 獲取屬性　／@href
    # 獲取第i個標簽　／tar_name[i]  從1開始
    # normalize-space－－＞去空格換行符
    # position_name = html.xpath('normalize-space(//div[@class="el"]/p/span/a/text())')

    # 職位名稱，
    position_names = []
    for name in html.xpath('//div[@class="el"]/p/span/a/text()'):
        position_name = name.strip()
        position_names.append(position_name)

    # 職位地址
    position_urls = html.xpath('//div[@class="el"]/p/span/a/@href')

    # 公司名稱
    company_names = html.xpath('//div[@class="el"]/span[1]/a/text()')

    # 公司地址
    company_urls = html.xpath('//div[@class="el"]/span[1]/a/@href')

    # 位置
    locations = html.xpath('//div[@class="el"]/span[@class="t3"]/text()')

    # 薪資
    salarys = html.xpath('//div[@class="el"]/span[@class="t4"]/text()')

    # 發布時間
    release_dates = html.xpath('//div[@class="el"]/span[4]/text()')

    result = zip(position_names,position_urls,company_names,company_urls,locations,salarys,release_dates)
    return result


def save_to_txt(element):
    with open(file_name,'a',newline='') as f:
        for i in range(len(element)):
            # data = ','.join(element[i])
            if i == len(element)-1:
                f.write(element[i])
                f.write('\n')
            else:
                f.write(element[i]+',')


def save_to_mongo(element):
    keys = ['position_name','position_url','company_name',
            'company_url','location','salary','release_date']
    result = dict(zip(keys,list(element)))
    if db[MONGO_TABLE_XPATH].insert(result):
        print('數據成功存儲到mongo數據庫中')
        return True
    return False

    # 遍歷字典元素
    # for k,v in result.items():
    #     print(k,':',v)
    for key in result:
        print(key,':',result[key])



def main():
    if os.path.exists(file_name):
        os.remove(file_name)
    for page in range(1,MAX_PAGE+1):
        html = get_html(page)
        elements = parse_html(html)
        if elements:
            for element in elements:
                save_to_txt(element)
                save_to_mongo(element)

if __name__ == '__main__':
    main()

?