es 集群创建密码

参考这篇博文进行设置：https://juejin.cn/post/7079955586330132487
最后的效果
#curl -XGET 'http://127.0.0.1:9200/_cat/nodes?pretty' -u elastic:gfsdfdsfesfes
172.16.100.6 25 51 0 0.04 0.32 0.22 cdfhilmrstw * node-3
172.16.100.5 25 51 2 0.05 0.22 0.15 cdfhilmrstw - node-2
172.16.100.4 21 51 0 0.10 0.34 0.23 cdfhilmrstw - node-1es配置
cluster.name: k
node.name: node-1
path.data: /data/esdata
path.logs: /data/eslogs
network.host: 0.0.0.0
http.port: 9200
discovery.seed_hosts: ["172.16.100.4", "172.16.100.5", "172.16.100.6"]
cluster.initial_master_nodes: ["node-1","node-2","node-3"]
node.master: true
node.data: truexpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: /data/elasticsearch/config/elastic-certificates.p12
xpack.security.transport.ssl.truststore.path: /data/elasticsearch/config/elastic-certificates.p12

kibana 配置文件以及和nginx配置

配置文件
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.hosts: 
server.basePath: "/elk"
xpack.monitoring.ui.container.elasticsearch.enabled: true
i18n.locale: "zh-CN"
["http://172.16.100.4:9200","http://172.16.100.5:9200","http://172.16.100.6:9200"]
elasticsearch.username: "kibana_system"
elasticsearch.password: "gLUAjdwadwadwwda"server {listen       80 ;listen 443 ssl http2;server_name  elkfront.xxx.com;if ($server_port ~ 80){rewrite ^  https://$host/elk$request_uri? permanent;}ssl_certificate  /server/key/xxx.com.crt;ssl_certificate_key /server/key/xxx.com.key;location / {rewrite ^/$ /elk redirect;}location /elk/ {proxy_pass http://127.0.0.1:5601;proxy_redirect off;proxy_set_header Host $host:9091;proxy_set_header X-Real-IP $remote_addr;proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;proxy_set_header X-Forwarded-Proto $scheme;rewrite ^/elk/(.*)$ /$1 break;}
}

pm2 安装

centos7.9 版本 
以下是在 CentOS 7.9 上使用 nvm 安装 Node.js 的步骤：安装 nvm：bashCopy Codecurl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash在终端中重新加载 shell 配置或打开一个新的终端窗口：bashCopy Codesource ~/.bashrc安装适用于您的系统的 Node.js 版本。例如，您可以尝试安装 v14.17.6：bashCopy Codenvm install v14.17.6使用 nvm 切换到安装的 Node.js 版本：bashCopy Codenvm use v14.17.6确认 Node.js 版本已切换成功：bashCopy Codenode -v现在，您应该能够使用较新版本的 Node.js，而不会出现与 glibc 版本不兼容的错误。npm install pm2 -gpm2 结合pm2 进行脚本控制
#cat logstash_main.sh 
#!/bin/bash
/data/logstash_main/bin/logstash -f /data/logstash_main/config/logstash.conf  >> /data/logs/logstash_main.log  2>&1pm2 start /data/scripts/logstash_main.shredis 端口监控
*/5 * * * *  /bin/bash /importredis.sh >/dev/null 2>&1 &
#cat importredis.sh 
#!/bin/bash
gg=`netstat -an |grep ":6379" |awk '$1 == "tcp" && $NF == "LISTEN" {print $0}'|wc -l`
if [[ ${gg}x = "0x" ]];thensystemctl start redis  >/dev/null 2>&1 &
fi

定期清理索引以及告警

30 13 * * 7 sh /root/delete.sh >/dev/null 2>&1
delete.sh
curl -XDELETE 'http://127.0.0.1:9200/xx-jt*' -u elastic:gLcdsdfsdfesdfe告警部分
#cat http_status.py 
from datetime import datetime, timedelta
from elasticsearch import Elasticsearch
import sys
import requestsindex=sys.argv[1] #要查询的索引
http_code=sys.argv[2] #要统计的状态码
limit=int(sys.argv[3])def send_msg(status,count,index):  #上线通知功能token = "2104vfvdffvfdvsdfM" #chat_id = -73223443232 # 替换成你自己的chat_idtry:ret = requests.post(f'https://api.telegram.org/bot{token}/sendMessage', json={"chat_id": chat_id,"text": "ELK通知:\n状态码: {status} \n数量: {count} \n索引: {index}".format(status=status,count=count,index=index)},proxies=None)print(r.json())except:print("error")passes = Elasticsearch(['http://10.0.0.4:9200','http://10.0.0.5:9200','http://10.0.0.6:9200'],http_auth=('elastic', 'gLUAjIJfesfesfsef'),)
query = {"query": {"bool": {"filter": [{"range": {"@timestamp": {"gte": "now-5m", "lte": "now"}}},{"bool": {"should": [{"term": {"status": http_code}},#{"term": {"status": 502}},#j{"term": {"status": 503}},#{"term": {"status": 504}}],"minimum_should_match": 1}}]}}
}result = es.search(index=index, body=query)print(result['hits']['total'])
count=result['hits']['total'].get('value')
print(count)
if count>limit:print("出现异常记录\n")send_msg(http_code,count,index)  #上线通知功能定时计划任务 # xxxxx日志索引
*/1 * * * * python3 /data/shell/http_status.py xxxxx* 500 20
*/1 * * * * python3 /data/shell/http_status.py xxxxx* 502 20
*/1 * * * * python3 /data/shell/http_status.py xxxxx* 503 20
*/1 * * * * python3 /data/shell/http_status.py xxxxx* 504 20

logstash 配置

 jvm.options 配置
-Xms2g
-Xmx2g#cat logstash.conf 
input {# 从文件读取日志信息redis {host => "10.0.0.7"port => 6379password => "Rcfesfefesfesfes"  #如果没有密码不需要写这个参数key => "nginx1"data_type => "list"db => 0}}filter {json {source => "message"remove_field => ["beat","offset","tags","prospector"] #移除字段，不需要采集}date {match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"] #匹配timestamp字段target => "@timestamp"  #将匹配到的数据写到@timestamp字段中}
}output {if [filetype] == "a1_nginxjson" {elasticsearch {hosts => ["10.0.0.4:9200","10.0.0.5:9200","10.0.0.6:9200"]index => "jxxxxx1-%{+YYYY.MM.dd}"user => elasticpassword => 'gLUAjIJwnuHb1Rp21gWZ'}} else if [filetype] == "a2_nginxjson" {elasticsearch {hosts => ["10.0.0.4:9200","10.0.0.5:9200","10.0.0.6:9200"]index => "jxxxxx-%{+YYYY.MM.dd}"user => elasticpassword => 'fesfefesfadfsd'}} 
} 

filebeat 配置文件

#cat filebeat.yml 
filebeat.inputs:
- type: logenabled: truebackoff: "1s"tail_files: falsepaths:- /home/*.logfields:filetype: log_nginxjsonfields_under_root: true- type: logenabled: truebackoff: "1s"tail_files: falsepaths:- /usr/local/nginx/logs/*.logfields:filetype: a1_nginxjsonfields_under_root: trueoutput.redis:enabled: truehosts: ["10.0.0.7:6379"]password: Rcfesfefesfesfeskey: nginx1db: 0

nginx 的日志索引

        log_format mainJson'{"@timestamp":"$time_iso8601",''"host":"$hostname",''"server_ip":"$server_addr",''"http_x_forwarded_for":"$http_x_forwarded_for",''"domain":"$host",''"url":"$uri",''"referer":"$http_referer",''"args":"$args",''"upstreamtime":"$upstream_response_time",''"responsetime":"$request_time",''"request_method":"$request_method",''"status":"$status",''"size":"$body_bytes_sent",'#'"request_body":"$request_body",''"request_length":"$request_length",''"protocol":"$server_protocol",''"upstreamhost":"$upstream_addr",''"file_dir":"$request_filename",''"http_user_agent":"$http_user_agent",''"remote_addr":"$remote_addr",''"client_ip": "$remote_addr",''"request_uri":"$request_uri",''"request_completion":"$request_completion"''}';

参考网址:https://juejin.cn/post/7079955586330132487