Pandas 學習教程

定義

?基本操作

?一維數組操作

二維數組操作

數據選擇過濾

數據處理

數據清洗

數據轉換

數據分析

排序

分組聚合

數據透視表

高級操作

合并數據

時間序列處理

自定義函數調用

數據可視化集成

?數據導出和導入

大數據分塊處理

定義

全稱： 'panel data' and 'python data analysis'

Analy: Series(一維數據)、DataFrame(二維數據)

主要應用：數據清洗：處理缺失數據、重復數據等

數據轉換：改變數據的形狀、結構或格式

數據分析：進行統計、聚合、分組

數據可視化：整合Matplotlib 進行數據可視化

?基本操作

?一維數組操作

import pandas as pda = [1, 2, 3]
mystr = pd.Series(a)
print('列表')
print(mystr)a = ['a', 'b', 'c']
mystr = pd.Series(a, index=['x', 'y', 'z'])
print('自定義列名')
print(mystr)str_dict = {'1': 'a', '2': 'b'}
mystr = pd.Series(str_dict)
print('字典類型')
print(mystr)#############運行結果########################
# 列表
# 0    1
# 1    2
# 2    3
# dtype: int64
# 自定義列名
# x    a
# y    b
# z    c
# dtype: object
# 字典類型
# 1    a
# 2    b
# dtype: object

二維數組操作

import pandas as pd
import numpy as np# 從字典創建
data = {'Name': ['Alice', 'Bob', 'Charlie', 'David'],'Age': [25, 30, 35, 40],'City': ['New York', 'Paris', 'London', 'Tokyo']}
df = pd.DataFrame(data)
print("從字典創建的DataFrame:")
print(df)# 從列表創建
data = [['Apple', 10], ['Banana', 15], ['Orange', 8]]
df_fruits = pd.DataFrame(data, columns=['Fruit', 'Price'])
print("\n從列表創建的DataFrame:")
print(df_fruits)# 查看前幾行
print("\n前兩行:")
print(df.head(2))# 查看基本信息
print("\nDataFrame信息:")
print(df.info())# 查看統計信息
print("\n統計信息:")
print(df.describe())# 查看列名
print("\n列名:")
print(df.columns)# 查看形狀
print("\n形狀(行,列):")
print(df.shape)#############運行結果################
# 從字典創建的DataFrame:
#       Name  Age      City
# 0    Alice   25  New York
# 1      Bob   30     Paris
# 2  Charlie   35    London
# 3    David   40     Tokyo
# 
# 從列表創建的DataFrame:
#     Fruit  Price
# 0   Apple     10
# 1  Banana     15
# 2  Orange      8
# 
# 前兩行:
#     Name  Age      City
# 0  Alice   25  New York
# 1    Bob   30     Paris
# 
# DataFrame信息:
# <class 'pandas.core.frame.DataFrame'>
# RangeIndex: 4 entries, 0 to 3
# Data columns (total 3 columns):
#  #   Column  Non-Null Count  Dtype 
# ---  ------  --------------  ----- 
#  0   Name    4 non-null      object
#  1   Age     4 non-null      int64 
#  2   City    4 non-null      object
# dtypes: int64(1), object(2)
# memory usage: 224.0+ bytes
# None
# 
# 統計信息:
#              Age
# count   4.000000
# mean   32.500000
# std     6.454972
# min    25.000000
# 25%    28.750000
# 50%    32.500000
# 75%    36.250000
# max    40.000000
# 
# 列名:
# Index(['Name', 'Age', 'City'], dtype='object')
# 
# 形狀(行,列):
# (4, 3)

數據選擇過濾

import pandas as pd
import numpy as np# 從字典創建
data = {'Name': ['Alice', 'Bob', 'Charlie', 'David'],'Age': [25, 30, 35, 40],'City': ['New York', 'Paris', 'London', 'Tokyo']}
df = pd.DataFrame(data)# 選擇列
print("\n選擇Name列:")
print(df['Name'])# 選擇多列
print("\n選擇Name和Age列:")
print(df[['Name', 'Age']])# 選擇行
print("\n選擇第二行:")
print(df.iloc[1])# 選擇多行
print("\n選擇前兩行:")
print(df.iloc[0:2])# 條件選擇
print("\n選擇年齡大于30的人:")
print(df[df['Age'] > 30])# 使用loc選擇
print("\n使用loc選擇:")
print(df.loc[df['Age'] > 30, ['Name', 'City']])############運行結果######################
# 選擇Name列:
# 0      Alice
# 1        Bob
# 2    Charlie
# 3      David
# Name: Name, dtype: object
#
# 選擇Name和Age列:
#       Name  Age
# 0    Alice   25
# 1      Bob   30
# 2  Charlie   35
# 3    David   40
#
# 選擇第二行:
# Name      Bob
# Age        30
# City    Paris
# Name: 1, dtype: object
#
# 選擇前兩行:
#     Name  Age      City
# 0  Alice   25  New York
# 1    Bob   30     Paris
#
# 選擇年齡大于30的人:
#       Name  Age    City
# 2  Charlie   35  London
# 3    David   40   Tokyo
#
# 使用loc選擇:
#       Name    City
# 2  Charlie  London
# 3    David   Tokyo

數據處理

數據清洗

import pandas as pd
import numpy as np# 創建有缺失值的DataFrame
data = {'A': [1, 2, np.nan, 4],'B': [5, np.nan, np.nan, 8],'C': [10, 20, 30, 40]}
df_missing = pd.DataFrame(data)
print("\n有缺失值的DataFrame:")
print(df_missing)# 檢查缺失值
print("\n缺失值統計:")
print(df_missing.isnull().sum())# 填充缺失值
print("\n填充缺失值:")
print(df_missing.fillna(value={'A': 0, 'B': df_missing['B'].mean()}))# 刪除缺失值
print("\n刪除包含缺失值的行:")
print(df_missing.dropna())# 刪除全為缺失值的列
print("\n刪除全為缺失值的列:")
print(df_missing.dropna(axis=1, how='all'))#################運行結果#############################
# 有缺失值的DataFrame:
#      A    B   C
# 0  1.0  5.0  10
# 1  2.0  NaN  20
# 2  NaN  NaN  30
# 3  4.0  8.0  40
# 
# 缺失值統計:
# A    1
# B    2
# C    0
# dtype: int64
# 
# 填充缺失值:
#      A    B   C
# 0  1.0  5.0  10
# 1  2.0  6.5  20
# 2  0.0  6.5  30
# 3  4.0  8.0  40
# 
# 刪除包含缺失值的行:
#      A    B   C
# 0  1.0  5.0  10
# 3  4.0  8.0  40
# 
# 刪除全為缺失值的列:
#     C
# 0  10
# 1  20
# 2  30
# 3  40

數據轉換

import pandas as pd# 從字典創建
data = {'Name': ['Alice', 'Bob', 'Charlie', 'David'],'Age': [25, 30, 35, 40],'City': ['New York', 'Paris', 'London', 'Tokyo']}
df = pd.DataFrame(data)
print(df.dtypes)# 類型轉換
df['Age'] = df['Age'].astype('float')
print("\n轉換Age列為浮點型:")
print(df.dtypes)# 重命名列
df_renamed = df.rename(columns={'Name': 'Full Name', 'City': 'Location'})
print("\n重命名列后:")
print(df_renamed.columns)# 替換值
df_replaced = df.replace({'Paris': 'Rome', 'London': 'Berlin'})
print("\n替換值后:")
print(df_replaced)# 應用函數
print("\n應用函數:")
print(df['Age'].apply(lambda x: x + 1))# 字符串操作
print("\n字符串操作:")
print(df['Name'].str.upper())

數據分析

排序

import pandas as pd# 從字典創建
data = {'Name': ['Alice', 'Bob', 'Charlie', 'David'],'Age': [25, 30, 35, 40],'City': ['New York', 'Paris', 'London', 'Tokyo']}
df = pd.DataFrame(data)
# 按列排序
print("\n按Age降序排序:")
print(df.sort_values('Age', ascending=False))  # ascending=False, 反序# 多列排序
print("\n先按City升序，再按Age降序排序:")
print(df.sort_values(['City', 'Age'], ascending=[True, False]))

分組聚合

import pandas as pd# 創建示例數據
data = {'Department': ['Sales', 'Sales', 'IT', 'IT', 'HR', 'HR'],'Employee': ['Alice', 'Bob', 'Charlie', 'David', 'Eve', 'Frank'],'Salary': [5000, 6000, 7000, 8000, 5500, 6500]}
df_company = pd.DataFrame(data)# 分組聚合
print("\n按部門計算平均工資:")
print(df_company.groupby('Department')['Salary'].mean())# 多聚合函數
print("\n按部門計算工資的多種統計量:")
print(df_company.groupby('Department')['Salary'].agg(['mean', 'max', 'min', 'count']))# 多列分組
print("\n按部門分組并顯示員工列表:")
print(df_company.groupby('Department')['Employee'].apply(list))#############運行結果##################
# 按部門計算平均工資:
# Department
# HR       6000.0
# IT       7500.0
# Sales    5500.0
# Name: Salary, dtype: float64
# 
# 按部門計算工資的多種統計量:
#               mean   max   min  count
# Department                           
# HR          6000.0  6500  5500      2
# IT          7500.0  8000  7000      2
# Sales       5500.0  6000  5000      2
# 
# 按部門分組并顯示員工列表:
# Department
# HR           [Eve, Frank]
# IT       [Charlie, David]
# Sales        [Alice, Bob]
# Name: Employee, dtype: object

數據透視表

import pandas as pd
import numpy as np# 創建示例數據
data = {'Date': ['2023-01-01', '2023-01-01', '2023-01-02', '2023-01-02'],'Product': ['A', 'B', 'A', 'B'],'Sales': [100, 150, 200, 50]}
df_sales = pd.DataFrame(data)# 創建透視表
pivot = pd.pivot_table(df_sales, values='Sales', index='Date', columns='Product', aggfunc=np.sum)
print("\n透視表:")
print(pivot)# 添加匯總
pivot_margins = pd.pivot_table(df_sales, values='Sales', index='Date', columns='Product',aggfunc=np.sum, margins=True, margins_name='Total')
print("\n帶匯總的透視表:")
print(pivot_margins)# 透視表:
# Product       A    B
# Date                
# 2023-01-01  100  150
# 2023-01-02  200   50
# 
# 帶匯總的透視表:
# Product       A    B  Total
# Date                       
# 2023-01-01  100  150    250
# 2023-01-02  200   50    250
# Total       300  200    500

高級操作

合并數據

import pandas as pd# 創建兩個DataFrame
df1 = pd.DataFrame({'key': ['A', 'B', 'C'], 'value1': [1, 2, 3]})
df2 = pd.DataFrame({'key': ['B', 'C', 'D'], 'value2': [4, 5, 6]})# 內連接
print("\n內連接:")
print(pd.merge(df1, df2, on='key', how='inner'))# 左連接
print("\n左連接:")
print(pd.merge(df1, df2, on='key', how='left'))# 左連接
print("\n右連接:")
print(pd.merge(df1, df2, on='key', how='right'))# 外連接
print("\n外連接:")
print(pd.merge(df1, df2, on='key', how='outer'))# 連接多個鍵
df3 = pd.DataFrame({'key1': ['A', 'B', 'C'], 'key2': ['X', 'Y', 'Z'], 'value': [10, 20, 30]})
df4 = pd.DataFrame({'key1': ['B', 'C', 'D'], 'key2': ['Y', 'Z', 'W'], 'value': [40, 50, 60]})
print("\n多鍵連接:")
print(pd.merge(df3, df4, on=['key1', 'key2'], how='inner'))

時間序列處理

import pandas as pd
import numpy as np# 創建時間序列數據
dates = pd.date_range('20230101', periods=6)
df_time = pd.DataFrame(np.random.randn(6, 4), index=dates, columns=list('ABCD'))
print("\n時間序列DataFrame:")
print(df_time)# 時間索引選擇
print("\n選擇特定日期范圍:")
print(df_time['2023-01-02':'2023-01-04'])# 重采樣
print("\n按月重采樣求均值:")
print(df_time.resample('ME').mean())###################運行結果#########################
# 時間序列DataFrame:
#                    A         B         C         D
# 2023-01-01 -0.800839 -0.953824  0.958557  1.044269
# 2023-01-02  0.203414  1.462324  0.099291  0.457230
# 2023-01-03 -0.113450  0.441178 -0.779749  1.158564
# 2023-01-04 -0.741451 -1.353738  1.906289  2.513789
# 2023-01-05 -1.719984 -0.305153 -1.283332  0.728910
# 2023-01-06 -0.363050 -1.591411 -2.289244 -0.076304
# 
# 選擇特定日期范圍:
#                    A         B         C         D
# 2023-01-02  0.203414  1.462324  0.099291  0.457230
# 2023-01-03 -0.113450  0.441178 -0.779749  1.158564
# 2023-01-04 -0.741451 -1.353738  1.906289  2.513789
# 
# 按月重采樣求均值:
#                    A         B         C         D
# 2023-01-31 -0.589227 -0.383437 -0.231365  0.971077

自定義函數調用

import pandas as pd# 從字典創建
data = {'Name': ['Alice', 'Bob', 'Charlie', 'David'],'Age': [25, 30, 35, 40],'City': ['New York', 'Paris', 'London', 'Tokyo']}
df = pd.DataFrame(data)
# 使用apply應用自定義函數
def age_group(age):if age < 30:return 'Young'elif age < 40:return 'Middle'else:return 'Senior'print("\n應用自定義函數:")
df['Age Group'] = df['Age'].apply(age_group)
print(df)# 使用pipe進行方法鏈
def add_prefix(df, prefix):df['Name'] = prefix + df['Name']return dfprint("\n使用pipe:")
print(df.pipe(add_prefix, prefix='Mr. '))#####################運行結果###########################
# 應用自定義函數:
#       Name  Age      City Age Group
# 0    Alice   25  New York     Young
# 1      Bob   30     Paris    Middle
# 2  Charlie   35    London    Middle
# 3    David   40     Tokyo    Senior
# 
# 使用pipe:
#           Name  Age      City Age Group
# 0    Mr. Alice   25  New York     Young
# 1      Mr. Bob   30     Paris    Middle
# 2  Mr. Charlie   35    London    Middle
# 3    Mr. David   40     Tokyo    Senior

數據可視化集成

import pandas as pdimport matplotlib.pyplot as plt# 創建示例數據
df_plot = pd.DataFrame({'Year': [2010, 2011, 2012, 2013, 2014],'Sales': [100, 120, 150, 180, 200],'Profit': [20, 25, 30, 35, 40]
})# 繪制折線圖
df_plot.plot(x='Year', y=['Sales', 'Profit'], kind='line', title='Sales and Profit Over Years')
plt.show()# 繪制柱狀圖
df_plot.plot(x='Year', y='Sales', kind='bar', title='Sales by Year')
plt.show()# 繪制散點圖
df_plot.plot(x='Sales', y='Profit', kind='scatter', title='Profit vs Sales')
plt.show()

運行結果：

?數據導出和導入

import pandas as pd# 從字典創建
data = {'Name': ['Alice', 'Bob', 'Charlie', 'David'],'Age': [25, 30, 35, 40],'City': ['New York', 'Paris', 'London', 'Tokyo']}
df = pd.DataFrame(data)
# 導出到CSV
df.to_csv('output.csv', index=False)# 從CSV導入
df_imported = pd.read_csv('output.csv')
print("\n從CSV導入的數據:")
print(df_imported.head())# 導出到Excel
df.to_excel('output.xlsx', sheet_name='Sheet1', index=False)# 從Excel導入
df_excel = pd.read_excel('output.xlsx', sheet_name='Sheet1')
print("\n從Excel導入的數據:")
print(df_excel.head())#####################運行結果################################
# 從CSV導入的數據:
#       Name  Age      City
# 0    Alice   25  New York
# 1      Bob   30     Paris
# 2  Charlie   35    London
# 3    David   40     Tokyo
# 
# 從Excel導入的數據:
#       Name  Age      City
# 0    Alice   25  New York
# 1      Bob   30     Paris
# 2  Charlie   35    London
# 3    David   40     Tokyo

大數據分塊處理

import pandas as pd# 從字典創建
data = {'Name': ['Alice', 'Bob', 'Charlie', 'David'],'Age': [25, 30, 35, 40],'City': ['Paris', 'Paris', 'London', 'London']}
df = pd.DataFrame(data)
# 導出到CSV
df.to_csv('output.csv', index=False)# 分塊讀取大文件
chunk_size = 3
chunks = pd.read_csv('output.csv', chunksize=chunk_size)# 分塊處理
results = []
for chunk in chunks:processed = chunk.groupby('City')['Age'].sum()results.append(processed)# 合并結果
final_result = pd.concat(results).groupby(level=0).sum()
print(final_result)############運行結果##########################
# City
# London    75
# Paris     55
# Name: Age, dtype: int64

參考來源： deepseek。。。