記錄最近一次爬蟲和數據可視化練習

0、數據獲取

import pandas as pdyear = [2018,2019,2020,2021,2022]
header = {"User-Agent":"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Mobile Safari/537.36 Edg/111.0.1661.44"
}
for i in year:url = ""df = pd.read_html(url, header=None)[13]# 刪除空行df = df.drop(index=[1])# 刪除最后一列df = df.drop(columns=["久其軟件(002279) 利潤表單位：萬元.5"])# 如果不是2018年，則刪除第一列if i != 2018:df = df.drop(columns=["久其軟件(002279) 利潤表單位：萬元"])df1 = df.T# 寫入csv文件df1.to_csv('2019.csv', header=None, mode="a", index=None)

1、數據可視化

1.1從文件中讀取數據

# 讀取寫入的文件
all_data = pd.read_csv("2019.csv")
# all_data.duplicated().sum() # 查看重復值
# all_data.drop_duplicates(inplace=True)
# all_data.isnull().sum() #  查看空值
# all_data.fillna(0, inplace=True)
# all_data.reset_index(drop=True, inplace=True) # 重置索引

1.2繪圖

這里使用pyecharts來進行繪制，官方文檔，官方demo

繪制折柱混合圖

import pyecharts
from pyecharts.charts import Line, Bar, Grid, WordCloud
import pyecharts.options as opts# 省略數據處理部分bar1 = (Bar().add_xaxis(year).add_yaxis("營業收入", year_income, bar_width=50).set_global_opts(title_opts=opts.TitleOpts(title="營業收入", subtitle="單位（元）"), yaxis_opts=opts.AxisOpts(min_=200000, max_=400000),legend_opts=opts.LegendOpts(is_show=False))
)
line1 = (Line().add_xaxis(year).add_yaxis("營業收入", year_income, linestyle_opts=opts.LineStyleOpts(color="red", width=4))
)bar2 = (Bar().add_xaxis(year).add_yaxis("營業費用", year_fee, bar_width=50).set_global_opts(title_opts=opts.TitleOpts(title="營業費用", subtitle="單位（元）", pos_left="48%"), yaxis_opts=opts.AxisOpts(min_=250000, max_=400000),legend_opts=opts.LegendOpts(is_show=False))
)
line2 = (Line().add_xaxis(year).add_yaxis("營業費用", year_fee, linestyle_opts=opts.LineStyleOpts(color="red", width=4))
)
overlap1 = bar1.overlap(line1)
overlap2 = bar2.overlap(line2)grid = (Grid(init_opts=opts.InitOpts(width="1200px", height="800px", page_title="營收與利潤")).add(overlap1, grid_opts=opts.GridOpts(pos_right="58%")).add(overlap2, grid_opts=opts.GridOpts(pos_left="58%")).render('1.html')
)

繪制標簽切換圖

# 繪圖
def create_line(profit):season = []accu_money = [] # 累積利潤money = [] # 每季度利潤for key in profit:season.append(key)accu_money.append(profit[key])accu_money = accu_money[::-1]money.append(accu_money[0])money.append(accu_money[1] - accu_money[0])money.append(accu_money[2] - accu_money[1])money.append(accu_money[3] - accu_money[2])line = (Line().add_xaxis(season[::-1]).add_yaxis("", money).set_global_opts(title_opts=opts.TitleOpts(title="季度利潤", subtitle="單位（元）")))return linetab = Tab(page_title="季度利潤") # 設置網頁標簽的標題
tab.add(create_line(profit_2018), "2018")
tab.add(create_line(profit_2019), "2019")
tab.add(create_line(profit_2020), "2020")
tab.add(create_line(profit_2021), "2021")
tab.add(create_line(profit_2022), "2022")
tab.render("2.html")

繪制大屏

首先繪制要展示的各種圖表，通過page對象進行整合

# 創建page對象整合布局
from pyecharts.charts import Page# 實例化page對象，指定布局方式
page = Page(layout=Page.DraggablePageLayout, page_title="基于Pyecharts的數據大屏")
page.add(bar1,line1,bar2,line2,create_line(profit_2018),create_line(profit_2019),create_line(profit_2020),create_line(profit_2021),create_line(profit_2022),
)
page.render("4.html")

接著在瀏覽器中調整各圖表的大小和位置，保存json文件后，重新進行渲染

Page.save_resize_html(source="4.html", cfg_file="./chart_config.json", dest="5.html")