單點部署

環境準備

基于Rocky9虛擬機，內存大小為4G

yum -y install lrzsz
useradd elkf
passwd elkf#密碼隨意su - elk
rz 導入包，筆者導使用版本為7.17.8

下載地址：https://www.elastic.co/downloads/past-releases/

tar -xf elasticsearch-7.17.8-linux-x86_64.tar.gz 
tar -xf filebeat-7.17.8-linux-x86_64.tar.gz 
tar -xf kibana-7.17.8-linux-x86_64.tar.gz  
tar -xf logstash-7.17.8-linux-x86_64.tar.gz 

配置elasticsearch

環境地址：/home/elkf/elasticsearch-7.17.8

# 配置以下環境變量
vim ~/.bash_profile
export ES_JAVA_HOME=/home/elkf/elasticsearch-7.17.8/jdk
export ES_HOME=/home/elkf/elasticsearch-7.17.8
source ~/.bash_profile # 配置jvm內存資源分配
vim config/jvm.options
-Xms1g
-Xmx4g# 配置elastic環境變量
vim config/elasticsearch.yml 
network.host: 0.0.0.0
discovery.type: single-node
xpack.security.enabled: false# 啟動elasticsearch
bin/elasticsearch -d# 若啟動成功
curl 127.0.0.1:9200
{"name" : "maxscale","cluster_name" : "elasticsearch","cluster_uuid" : "g6ZSGcSuTzSkthyWX5W90w","version" : {"number" : "7.17.8","build_flavor" : "default","build_type" : "tar","build_hash" : "120eabe1c8a0cb2ae87cffc109a5b65d213e9df1","build_date" : "2022-12-02T17:33:09.727072865Z","build_snapshot" : false,"lucene_version" : "8.11.1","minimum_wire_compatibility_version" : "6.8.0","minimum_index_compatibility_version" : "6.0.0-beta1"},"tagline" : "You Know, for Search"
}

配置kibana

環境地址：/home/elkf/kibana-7.17.8-linux-x86_64

vim config/kibana.yml
server.port:5601
server.host:"0.0.0.0"
elasticsearch.hosts:["http://localhost:9200"]
server.name: "kibana"
kibana.index: ".kibana"
i18n.locale: "zh-CN"#配置中文模式# 啟動kibana
nohup bin/kibana &# ip為虛擬機ip，云端使用云端ip
使用瀏覽器訪問：http://ip:5601

配置logstash

實驗環境路徑：/home/elkf/logstash-7.17.8

# 配置需要收集信息的文件
mkdir test
touch file.txt# 配置logstash收集信息的規則
vim config/pipelines.yml 
input {file {path=> "/home/elkf/logstash-7.17.8/test/file.txt"start_position => "beginning"}
}
output {elasticsearch {hosts => ["127.0.0.1:9200"]index => "system-log-%{+YYY.MM.dd}"}stdout {codec => rubydebug}
}# 可使用絕對路徑來啟動
nohup bin/logstash -f config/piplines.yml &# 使用其他終端測試收集信息是否成功
echo 15 > /home/elkf/logstash-7.17.8/test/file.txt
echo alpha > /home/elkf/logstash-7.17.8/test/file.txt
在kibana查看索引是否有sys-log-timedump

由于單點部署中Logstash完全能夠完成數據收集、過濾、輸出的功能，因此不再部署Filebeat。

集群部署

基于elkf爭對nginx進行日志分析的節點分配

192.168.25.101:elasticsearch

192.168.25.102:kibana

192.168.25.103:logstash

192.168.25.104:nginx+filebeat

elasticsearch部署

單節點推薦至少4G運行內存，否則可能運行失敗

# 導入包見單點部署
# 為方便管理，軟件解壓到/usr/local/目錄下統一管理hostnamectl set-hostname ElasticSearch
tar -xf elasticsearch-7.17.8-linux-x86_64.tar.gz  -C /usr/local
tar -xf kibana-7.17.8-linux-x86_64.tar.gz -C /usr/localcd /usr/local/elasticsearch-7.17.8
# 配置JVM內存，配置內核max_map_count適配elasticsearch集群模式
vim config/jvm.options
-Xms4g
-Xmx4g
vim /etc/sysctl.conf
vm.max_map_count=262144# 配置elastic集群適配內容，注意與單點配置區別
mkdir /var/lib/elasticsearch/
mkdir /var/log/elasticsearch/
vim config/elasticsearch.yml
cluster.name: elkf
node.name: es1
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch/
cluster.initial_master_nodes: ["es1"]# elasticsearch出于安全考慮，不能使用root用戶啟動，為elasticsearch配置相應用戶
useradd elastic
chown -R elastic:elastic /usr/local/elasticsearch-7.17.8/
chown -R elastic:elastic /var/lib/elasticsearch /var/log/elasticsearch
su elastic
bin/elasticsearch -dcurl 192.168.25.101:9200
{"name" : "es1","cluster_name" : "elkf","cluster_uuid" : "uKedNB_VR8e90JOzgjrctg","version" : {"number" : "7.17.8","build_flavor" : "default","build_type" : "tar","build_hash" : "120eabe1c8a0cb2ae87cffc109a5b65d213e9df1","build_date" : "2022-12-02T17:33:09.727072865Z","build_snapshot" : false,"lucene_version" : "8.11.1","minimum_wire_compatibility_version" : "6.8.0","minimum_index_compatibility_version" : "6.0.0-beta1"},"tagline" : "You Know, for Search"}

配置kibana

單節點至少1G內存

# 導入包見單節點部署
hostnamectl set-hostname kibana
tar -xf kibana-7.17.8-linux-x86_64.tar.gz  -C /usr/local
cd /usr/local/kibana-7.17.8-linux-x86_64/
server.port: 5601
server.host: "192.168.25.102"
elasticsearch.hosts: ["http://192.168.25.101:9200"]
server.name: "kibana"
kibana.index: ".kibana"
i18n.locale: "zh-CN"
# 使用root用戶啟動，生產環境建議為其創建單獨用戶，并啟用賬戶授權認證
nohup bin/kibana --allow-root &

配置logstash

單節點至少1G內存

hostnamectl set-hostname logstash
tar -xf logstash-7.17.8-linux-x86_64.tar.gz -C /usr/local/
cd /usr/local/logstash-7.17.8# 測試是否能啟動成功
bin/logstash -e 'input{ stdin{} }output { stdout{} }'# 配置pipelines.yml之后
# 測試pipelines.yml
bin/logstash -f config/pipelines.yml  --config.test_and_exit# 測試返回成功之后啟動logstash
nohup bin/logstash -f config/pipelines.yml &

• 配置pipelines.yml

測試logstash與elasticsearch之間的連接，logstash本機系統日志測試

input {file {path => "var/log/messages"start_position => "beginning"}
}
output {elasticsearch {hosts => ["192.168.25.101:9200"]index => ["system-log-%{+YYYY.MM.dd}"]}stdout {codec => rubydebug}
}

配置filebeat+nginx

hostnamectl set-hostname filebeatnginx
yum -y install nginx
systemctl start nginx# 導入包見單點部署# 部署filebeat 
tar -xf filebeat-7.17.8-linux-x86_64.tar.gz  -C /usr/local
cd /usr/local/filebeat-7.17.8-linux-x86_64/

nginx日志收集實戰

配置logstash

mv /usr/local/logstash-7.17.8/config/pipelines.yml /usr/local/logstash-7.17.8/config/pipelines.yml.bak
vim /usr/local/logstash-7.17.8/config/pipelines.yml

input {beats {port => 5004}
}
output {elasticsearch {hosts => ["192.168.25.101:9200"]index => ["Name1-nginx-access-%{+YYYY.MM.dd}"]}stdout {codec => rubydebug}
}

測試并啟動

/usr/local/logstash-7.17.8/bin/logstash -f /usr/local/logstash-7.17.8/config/pipelines.yml --config.test_and_exitnohup /usr/local/logstash-7.17.8/bin/logstash -f /usr/local/logstash-7.17.8/config/pipelines.yml &

配置filebeat

配置文件：/usr/local/filebeat-7.17.8-linux-x86_64/filebeat.yml

mv /usr/local/filebeat-7.17.8-linux-x86_64/filebeat.yml /usr/local/filebeat-7.17.8-linux-x86_64/filebeat.yml.bak
vim /usr/local/filebeat-7.17.8-linux-x86_64/filebeat.yml

# ============================== Filebeat inputs ===============================
filebeat.inputs:
- type: filestreamid: Name1-nginx-monitorenabled: truepaths:- /var/log/nginx/access.log- /var/log/nginx/error.log
# ================================== General ===================================
tags: ["name1", "nginx"]
# ------------------------------ Logstash Output -------------------------------
output.logstash:hosts: ["192.168.25.103:5004"]

/usr/local/filebeat-7.17.8-linux-x86_64/filebeat