#作者：程宏斌
文章目錄
- fluent-bit 1.9.4 轉換測試
- 結論
接上篇：《Fluent Bit系列：字符集轉碼測試（上）》https://blog.csdn.net/qq_40477248/article/details/150776142?spm=1001.2014.3001.5501
fluent-bit 1.9.4 轉換測試
1、測試使用配置文件
[SERVICE]Flush 1Parsers_File parsers.confHTTP_Server OnHTTP_Listen 0.0.0.0HTTP_PORT 3194
[INPUT]Name tailTag regex-fluentDB ./db/regex-fluent.dbRead_from_Head truePath /var/log/pods/logtest/*.logPath_Key pod_log_path
[FILTER]Name modifyMatch *Add paas_log_belong userAdd paas_log_type middlewareAdd paas_collection_type userfileAdd paas_account_id 123456789Add paas_region_id lftstAdd paas_product_id cccAdd paas_instance_name test10Add paas_host_ip 127.0.0.1Add paas_manager_ip 127.0.0.1Add pod_namespace defaultAdd pod_name test-0Add pod_container_name test
[FILTER]Name multilineMatch *multiline.key_content logmultiline.parser multiline-regex-goemitter_mem_buf_limit 2048M
[FILTER]Name luaMatch *script gbk2utf8.luacall convert_gbk_to_utf8
[OUTPUT]Name fileMatch *Path /vdata/logtest
核查數據結果
# 檢查采集結果命令
less /vdata/logtest/regex-fluent #提前獲取中文字符
egrep -n '匹配費率為0，不進行計價|to_number' regex-fluent | head -n 5
以下是采集后的結果被輸入到 /vdata/logtest/regex-fluent 文件中。在Linux系統中，通過使用 grep 命令搜索關鍵字，可以看到文件的第135、137行，確認沒有出現任何 GBK 格式的亂碼。這表明 Fluent Bit 1.9.4 成功地將 GBK 格式轉換為 UTF-8。此外，第24、40、41行的內容也確認了日志被正確合并。 但是我發現這個正則沒有處理日志文件中的空行，導致采集的數據有空白。
[root@cdp-10-191-193-8 logtest]# egrep -n '匹配費率為0，不進行計價|to_number' regex-fluent | head -n 5
24:regex-fluent: [1720149989.173038023, {"pod_log_path":"/var/log/pods/logtest/20230911_filerate-3105-9c659b484-kfp5j.0.log","paas_host_ip":"127.0.0.1","paas_log_belong":"user","pod_namespace":"default","paas_log_type":"middleware","pod_name":"test-0","paas_collection_type":"userfile","pod_container_name":"test","paas_account_id":"123456789","paas_region_id":"lftst","paas_manager_ip":"127.0.0.1","paas_product_id":"ccc","log":"329 2023-09-11 17:03:30 [/data01/heht30/guoc//stl_jsdev/src/load_shm.c:fLoadTBillCycle:401] [INFO]sql is [select \na.bill_cycle_seq, \na.bill_period_id, \nto_number(to_char(nvl(a.start_date, add_months(sysdate, 0)), 'yyyymmdd')), \nto_number(to_char(nvl(a.cutoff_date, add_months(sysdate, 240)), 'yyyymmdd')), \nnvl(a.split_table_postfix,-1), \nnvl(b.latn_id,-1), \nto_number(b.status) ","paas_instance_name":"test10"}]
40:regex-fluent: [1720149989.173041374, {"pod_log_path":"/var/log/pods/logtest/20230911_filerate-3105-9c659b484-kfp5j.0.log","paas_host_ip":"127.0.0.1","paas_log_belong":"user","pod_namespace":"default","paas_log_type":"middleware","pod_name":"test-0","paas_collection_type":"userfile","pod_container_name":"test","paas_account_id":"123456789","paas_region_id":"lftst","paas_manager_ip":"127.0.0.1","paas_product_id":"ccc","log":" to_number(nvl(to_char(active_date,'yyyymmdd'),'19700101')), ","paas_instance_name":"test10"}]
41:regex-fluent: [1720149989.173046930, {"pod_log_path":"/var/log/pods/logtest/20230911_filerate-3105-9c659b484-kfp5j.0.log","paas_host_ip":"127.0.0.1","paas_log_belong":"user","pod_namespace":"default","paas_log_type":"middleware","pod_name":"test-0","paas_collection_type":"userfile","pod_container_name":"test","paas_account_id":"123456789","paas_region_id":"lftst","paas_manager_ip":"127.0.0.1","paas_product_id":"ccc","log":" to_number(to_char(nvl(inactive_date,add_months(sysdate,360)),'yyyymmdd'))","paas_instance_name":"test10"}]
135:regex-fluent: [1720149989.173070565, {"pod_log_path":"/var/log/pods/logtest/20230911_filerate-3105-9c659b484-kfp5j.0.log","paas_host_ip":"127.0.0.1","paas_log_belong":"user","pod_namespace":"default","paas_log_type":"middleware","pod_name":"test-0","paas_collection_type":"userfile","pod_container_name":"test","paas_account_id":"123456789","paas_region_id":"lftst","paas_manager_ip":"127.0.0.1","paas_product_id":"ccc","log":"329 2023-09-11 17:04:53 [/data01/heht30/guoc//stl_jsdev/src/stl_pub.c:CdrRating:761] [INFO]匹配費率為0，不進行計價","paas_instance_name":"test10"}]
137:regex-fluent: [1720149989.173070837, {"pod_log_path":"/var/log/pods/logtest/20230911_filerate-3105-9c659b484-kfp5j.0.log","paas_host_ip":"127.0.0.1","paas_log_belong":"user","pod_namespace":"default","paas_log_type":"middleware","pod_name":"test-0","paas_collection_type":"userfile","pod_container_name":"test","paas_account_id":"123456789","paas_region_id":"lftst","paas_manager_ip":"127.0.0.1","paas_product_id":"ccc","log":"329 2023-09-11 17:04:53 [/data01/heht30/guoc//stl_jsdev/src/stl_pub.c:CdrRating:761] [INFO]匹配費率為0，不進行計價","paas_instance_name":"test10"}]
下面是日志文件的前三行
[root@cdp-10-191-193-8 logtest]# head -n 3 /home/dcos/fluentbit/20230911_filerate-3105-9c659b484-kfp5j.0.log
329 2023-09-11 17:03:29 [/data01/heht30/guoc//stl_jsdev/src/stl_main.c:main:328] [INFO][REPEAT_MESSAGE=2|MESSAGE_SEQUENCE=3105:72:239|THIS_WORKFLOW_ID=72|INSERT_TIME=20230911164216|THIS_NODE_ID=3|GROUP_ID=3105|WORKFLOW_ID=72|FILE_NAME=991000PTSVDA022023090116581100000005.filefmt|PROVINCE_ID=99|FILE_PATH=/jzjs_month/spcp//2799/20230901/99|FMT_NORMAL_REC=10853], ���?�??��=[2023-09-11 17:03:29]329 2023-09-11 17:03:29 [/data01/heht30/guoc//stl_jsdev/src/stl_main.c:main:341] [ALERT]REPEAT_MESSAGE=2|MESSAGE_SEQUENCE=3105:72:239|THIS_WORKFLOW_ID=72|INSERT_TIME=20230911164216|THIS_NODE_ID=3|GROUP_ID=3105|WORKFLOW_ID=72|FILE_NAME=991000PTSVDA022023090116581100000005.filefmt|PROVINCE_ID=99|FILE_PATH=/jzjs_month/spcp//2799/20230901/99|FMT_NORMAL_REC=10853
下面是采集文件的前三行內容，第二行也就是時間戳為“1720149989.173032110”的行，沒有正常處理空白行。
[root@cdp-10-191-193-8 logtest]# head -n 3 /vdata/logtest/regex-fluent
regex-fluent: [1720149989.173026110, {"pod_log_path":"/var/log/pods/logtest/20230911_filerate-3105-9c659b484-kfp5j.0.log","paas_host_ip":"127.0.0.1","paas_log_belong":"user","pod_namespace":"default","paas_log_type":"middleware","pod_name":"test-0","paas_collection_type":"userfile","pod_container_name":"test","paas_account_id":"123456789","paas_region_id":"lftst","paas_manager_ip":"127.0.0.1","paas_product_id":"ccc","log":"329 2023-09-11 17:03:29 [/data01/heht30/guoc//stl_jsdev/src/stl_main.c:main:328] [INFO][REPEAT_MESSAGE=2|MESSAGE_SEQUENCE=3105:72:239|THIS_WORKFLOW_ID=72|INSERT_TIME=20230911164216|THIS_NODE_ID=3|GROUP_ID=3105|WORKFLOW_ID=72|FILE_NAME=991000PTSVDA022023090116581100000005.filefmt|PROVINCE_ID=99|FILE_PATH=/jzjs_month/spcp//2799/20230901/99|FMT_NORMAL_REC=10853], 批價開始時間=[2023-09-11 17:03:29]","paas_instance_name":"test10"}]
regex-fluent: [1720149989.173032110, {"pod_log_path":"/var/log/pods/logtest/20230911_filerate-3105-9c659b484-kfp5j.0.log","paas_host_ip":"127.0.0.1","paas_log_belong":"user","pod_namespace":"default","paas_log_type":"middleware","pod_name":"test-0","paas_collection_type":"userfile","pod_container_name":"test","paas_account_id":"123456789","paas_region_id":"lftst","paas_manager_ip":"127.0.0.1","paas_product_id":"ccc","log":" ","paas_instance_name":"test10"}]
regex-fluent: [1720149989.173032547, {"pod_log_path":"/var/log/pods/logtest/20230911_filerate-3105-9c659b484-kfp5j.0.log","paas_host_ip":"127.0.0.1","paas_log_belong":"user","pod_namespace":"default","paas_log_type":"middleware","pod_name":"test-0","paas_collection_type":"userfile","pod_container_name":"test","paas_account_id":"123456789","paas_region_id":"lftst","paas_manager_ip":"127.0.0.1","paas_product_id":"ccc","log":"329 2023-09-11 17:03:29 [/data01/heht30/guoc//stl_jsdev/src/stl_main.c:main:341] [ALERT]REPEAT_MESSAGE=2|MESSAGE_SEQUENCE=3105:72:239|THIS_WORKFLOW_ID=72|INSERT_TIME=20230911164216|THIS_NODE_ID=3|GROUP_ID=3105|WORKFLOW_ID=72|FILE_NAME=991000PTSVDA022023090116581100000005.filefmt|PROVINCE_ID=99|FILE_PATH=/jzjs_month/spcp//2799/20230901/99|FMT_NORMAL_REC=10853 ","paas_instance_name":"test10"}]
結論
Fluent Bit 1.9.4 和 3.0.2 均能夠通過此 Lua 腳本進行字符集轉換驗證，證明字符集轉碼準確性驗證成功。
由于舊版本正則表達式對空白行不進行處理，接下來所有字符集測試都基于新版本正則表達式。因此，不再使用舊版本和新版本正則表達式進行性能對比測試，因為舊版本正則表達式功能不完善，不再具備測試的必要性。
