#coding=utf-8
from selenium import webdriver
import time
import os
import re
import sys
import threading
import Queue
import Tkinter as tk
from selenium.common.exceptions import NoSuchElementException
def myinit():
reload(sys)
sys.setdefaultencoding('utf8')
#獲取屏幕分辨率
def getscreeninfo():
#這個沒考慮多顯示器的問題
info={}
root = tk.Tk()
info['width_px'] = root.winfo_screenwidth()
info['height_px'] = root.winfo_screenheight()
info['width_mm'] = root.winfo_screenmmwidth()
info['height_mm'] = root.winfo_screenmmheight()
# 2.54 cm = in
info['width_in'] = info['width_mm'] / 25.4
info['height_in'] = info['height_mm'] / 25.4
info['width_dpi'] = info['width_px']/info['width_in']
info['height_dpi'] = info['height_px']/info['height_in']
return info
########################
#生成多個瀏覽器實例
def makebr(number):
brs=[]
for i in range(number):
br=webdriver.Firefox()
brs.append(br)
return brs
#設置所有瀏覽器實例大小
def setbrsize(brs,x,y):
for i in brs:
i.set_window_size(200,200)
#排列打開的瀏覽器
def layoutbrs(brs,x=0,y=0,dx=10,dy=10,mode=0,hang=0):
#mode 0,橫向, 1,縱向 2,矩形 3,對角線
#hang 0為盡量按正方形排列
screeninfo=getscreeninfo()
if(mode==0):
dy=0
elif(mode==1):
dx=0
if(mode!=2):
for i in brs:
i.set_window_position(x,y)
x=x+dx
y=y+dy
else:
#這個以后有空再弄
pass
#hang=math.ceil(math.sqrt(len(dbrs)))
#關閉所有瀏覽器
def closebr(brs):
for i in brs:
i.quit()
#######################
#獲得指定股票相關數據
def getinfo(mydriver,gourl):
title='//*[@id="name"]'
code='//*[@id="code"]'
hexinshujuxpath="/html/body/div[14]/div[1]/div[4]/div[1]"
restr=":".decode('utf8')
myre=re.compile(restr,re.I|re.M|re.S)
linetext=""
errorinfo=""
mydriver.get(gourl)
try:
gupiaoming=mydriver.find_element_by_xpath(title).text
gupiaocode=mydriver.find_element_by_xpath(code).text
hexinshuju=mydriver.find_element_by_class_name('pad5')
shujuhang=hexinshuju.find_elements_by_tag_name('tr')
for i in range(len(shujuhang)-2):
shujulie=shujuhang[i].find_elements_by_tag_name('td')
tmpshuju=myre.split(shujulie[0].text)
linetext=linetext+"~"+tmpshuju[1]
shuju=myre.split(shujuhang[8].text)
linetext=linetext+"~"+shuju[1]
tmpshuju=myre.split(shujuhang[9].text)
linetext=linetext+"~"+tmpshuju[1]
linetext="%s~%s%s"%(gupiaoming,gupiaocode,linetext)
#print "數據:",linetext
except NoSuchElementException,e:
#print "不是股票"
pass
except Exception:
errorinfo= "非預期錯誤"+gourl
print errorinfo
finally:
return linetext,errorinfo
#獲得所有股票鏈接
def geturls(br):
#通過link對象獲得鏈接地址的text文本
def getlinkurl(linklist):
my=[]
for x in linklist:
my.append(x.get_attribute('href'))
return my
sz=[]
sh=[]
br.get("http://quote.eastmoney.com/stocklist.html")
shxpath="/html/body/div[9]/div[2]/div/ul[1]"
szxpath="/html/body/div[9]/div[2]/div/ul[2]"
shgupiao=br.find_element_by_xpath(shxpath)
szgupiao=br.find_element_by_xpath(szxpath)
shgupiaolist=shgupiao.find_elements_by_tag_name('a')
szgupiaolist=szgupiao.find_elements_by_tag_name('a')
sh=getlinkurl(shgupiaolist)
sz=getlinkurl(szgupiaolist)
return sh,sz
#多線程執行用的函數
def thread_getinfo(br,jobslink_queue,jieguo_queue,errorinfo_queue):
while True:
try:
#獲得隊列里的地址
url=jobslink_queue.get(False) #False =Don't wait
except Queue.Empty:
print "完成退出"
#br.quit()
return
#print url
if(url!=None):
(linetext,errorinfo)=getinfo(br,url)
if(linetext!=""):
jieguo_queue.put(linetext)
if(errorinfo!=""):
errorinfo_queue.put(errorinfo)
#######################
#多線程控制函數
def saveinfoabc(info_filename,error_filename,urllist):
jobslink=Queue.Queue(0)
jieguo=Queue.Queue(0)
errorsinfo=Queue.Queue(0)
for x in urllist[200:250]:#為測試方便這里只取了50,如果要全下載,取消[]就好
jobslink.put(x)
#啟動線程
for x in range(THREAD_NUM):
t=threading.Thread(target=thread_getinfo,args=(brs[x],jobslink,jieguo,errorsinfo))
t.start()
f=open(info_filename,'w')
e=open(error_filename,'w')
mycount=0
while (threading.activeCount()>1) or (not jobslink.empty()):
while jieguo.qsize()>0 or errorsinfo.qsize()>0:
if(jieguo.qsize()>0):
jieguotxt=jieguo.get()
f.write(jieguotxt+"\n")
if(errorsinfo.qsize()>0):
error=errorsinfo.get()
e.write(error+"\n")
mycount=mycount+1
if(mycount%100)==0:
print "%d: 活動線程:%d,剩余連接數:%d,結果剩余條數:%d,錯誤剩余條數:%d"%(mycount,threading.activeCount(),jobslink.qsize(),jieguo.qsize(),errorsinfo.qsize())
time.sleep(0.01)
e.close()
f.close()
print "數據下載完成"
######################
myinit()
br=webdriver.Firefox()
print "獲得所有鏈接地址"
(sh,sz)=geturls(br)
br.quit()
#info_filename='shinfo.txt'
#error_filename='sherror.txt'
THREAD_NUM=10
brs=makebr(THREAD_NUM)
setbrsize(brs,200,300)
layoutbrs(brs ,0,0,80,80,0)
saveinfoabc('shinfo.txt','sherror.txt',sh)
saveinfoabc('szinfo.txt','szerror.txt',sz)
closebr(brs)