目录
环境
【1】redis配置
【2】filebeat配置
【3】对接logstash配置
【4】验证
【5】安全配置：第一种：kibana-nginx访问控制
【6】第二种：在ES-主节点-配置TLS
【7】kibana配置密码
【8】logstash添加用户密码
环境
| es-01,kibana | 10.0.0.21 |
| es-02 | 10.0.0.22 |
| es-03 | 10.0.0.23 |
| filebeat,nginx | 10.0.0.25 |
| logstash | 10.0.0.26 |
| redis | 10.0.0.27 |
【1】redis配置
[root@redis ~]# vim /etc/redis.conf
.....
bind 0.0.0.0
.....
requirepass 111
.....【2】filebeat配置
[root@filebeat ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: logenabled: truepaths:- /var/log/nginx/access.logtags: ["access"]- type: logenabled: truepaths:- /var/log/nginx/error.logtags: ["error"]output.redis:hosts: ["10.0.0.27:6379"]password: "111"db: 0timeout: 5keys:- key: "filebeat-nginx-access"when.contains:tags: "access"- key: "filebeat-nginx-error"when.contains:tags: "error"[root@filebeat ~]# systemctl restart filebeat.service## 验证redis是否有数据写入,查看key的长度
[root@redis ~]# redis-cli
127.0.0.1:6379> auth 111
OK
127.0.0.1:6379> keys *
1) "filebeat-nginx-access"
2) "filebeat-nginx-error"
127.0.0.1:6379> select 0
OK
127.0.0.1:6379> LLEN filebeat-access
(integer) 19
127.0.0.1:6379> LLEN filebeat-error
(integer) 15
127.0.0.1:6379>【3】对接logstash配置
[root@logstash ~]# cat /etc/logstash/conf.d/file-redis-log.conf
input {redis {host => "10.0.0.27"password => "111"db => "0"data_type => "list"key => "filebeat-access"}redis {host => "10.0.0.27"password => "111"db => "0"data_type => "list"key => "filebeat-error"}
}filter {if "access" in [tags][0] {grok {match => { "message" => "%{COMBINEDAPACHELOG}"}}geoip {source => "clientip"}date {match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]target => "@timestamp"timezone => "Asia/Shanghai"}useragent {source => "agent"target => "useragent"}mutate {convert => ["bytes","integer"]convert => ["response_time", "float"]convert => ["upstream_response_time", "float"]remove_field => ["message"]add_field => { "target_index" => "app-logstash-nginx-access-%{+YYYY.MM.dd}" } }# 提取 referrer 具体的域名 /^"http/if [referrer] =~ /^"http/ {grok {match => { "referrer" => '%{URIPROTO}://%{URIHOST:referrer_host}' }}}# 提取用户请求资源类型以及资源 ID 编号if "test.com" in [referrer_host] {grok {match => { "referrer" => '%{URIPROTO}://%{URIHOST}/(%{NOTSPACE:test_type}/%{NOTSPACE:test_res_id})?' }}}}else if "error" in [tags][0] {date {match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]target => "@timestamp"timezone => "Asia/Shanghai"}mutate {add_field => { "target_index" => "app-logstash-nginx-error-%{+YYYY.MM.dd}" }}}
}output {elasticsearch {hosts => ["10.0.0.21:9200","10.0.0.22:9200","10.0.0.23:9200"]index => "%{[target_index]}"template_overwrite => true}
}[root@logstash ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/file-redis-log.conf -r【4】验证
【5】安全配置：第一种：kibana-nginx访问控制
[root@es-01 ~]# yum -y install httpd-tools.x86_64
[root@es-01 ~]# htpasswd -c -b /etc/nginx/basic_passwd kibana 123123
Adding password for user kibana
[root@es-01 ~]# cat /etc/nginx/basic_passwd
kibana:$apr1$SASZLDcF$mHc2stQeQOCYXeapYhMj7/[root@es-01 ~]# vim /etc/nginx/conf.d/kibana.conf
server {listen 80;server_name nginx.kibana.org;location / {proxy_pass http://127.0.0.1:5601$request_uri;auth_basic "请输入用户名、密码";auth_basic_user_file /etc/nginx/basic_passwd; }
}[root@es-01 ~]# systemctl start nginx
【6】第二种：在ES-主节点-配置TLS
## 在主节点配置TLS
[root@es-01 ~]# /usr/share/elasticsearch/bin/elasticsearch-certutil \
> cert -out /etc/elasticsearch/elasticsearch-certificates.p12 -pass ""[root@es-01 ~]# chmod 660 /etc/elasticsearch/elasticsearch-certificates.p12 ## 编辑配置文件，添加以下内容，所有节点都需要添加
[root@es-01 ~]# vim /etc/elasticsearch/elasticsearch.yml
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: elasticsearch-certificates.p12
xpack.security.transport.ssl.truststore.path: elasticsearch-certificates.p12[root@es-02 ~]# vim /etc/elasticsearch/elasticsearch.yml
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: elasticsearch-certificates.p12
xpack.security.transport.ssl.truststore.path: elasticsearch-certificates.p12[root@es-03 ~]# vim /etc/elasticsearch/elasticsearch.yml
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: elasticsearch-certificates.p12
xpack.security.transport.ssl.truststore.path: elasticsearch-certificates.p12## 将TLS证书拷贝到所有节点
[root@es-01 ~]# scp -rp /etc/elasticsearch/elasticsearch-certificates.p12 root@10.0.0.22:/etc/elasticsearch/
[root@es-01 ~]# scp -rp /etc/elasticsearch/elasticsearch-certificates.p12 root@10.0.0.23:/etc/elasticsearch/## 重新启动所有节点
[root@es-01 ~]# systemctl restart elasticsearch## 主节点运行后，集群配置密码，auto随机密码，interactive手动
[root@es-01 ~]# /usr/share/elasticsearch/bin/elasticsearch-setup-passwords auto
Initiating the setup of passwords for reserved users elastic,apm_system,kibana,logstash_system,beats_system,remote_monitoring_user.
The passwords will be randomly generated and printed to the console.
Please confirm that you would like to continue [y/N]yChanged password for user apm_system
PASSWORD apm_system = IFYVDU9pa0BaYqn8bFdiChanged password for user kibana
PASSWORD kibana = 3S32IGJTj9bpgIy8q4bPChanged password for user logstash_system
PASSWORD logstash_system = EbaDFZhIMqNMgtMWaBFYChanged password for user beats_system
PASSWORD beats_system = hFivaRmv8BUGxr11vkRMChanged password for user remote_monitoring_user
PASSWORD remote_monitoring_user = 68xmYgepf2L40Zr5HPLIChanged password for user elastic
PASSWORD elastic = PpCC9KV8FnoDepE7DYZU[root@es-01 ~]# 
【7】kibana配置密码
[root@es-01 ~]# vim /etc/kibana/kibana.yml
......
elasticsearch.username: "kibana"
elasticsearch.password: "vERKRjujB5kfukYCT77w"[root@es-01 ~]# systemctl restart kibana
【8】logstash添加用户密码
[root@logstash ~]# cat /etc/logstash/conf.d/file-redis-log.conf
input {redis {host => "10.0.0.27"password => "111"db => "0"data_type => "list"key => "filebeat-access"}redis {host => "10.0.0.27"password => "111"db => "0"data_type => "list"key => "filebeat-error"}
}filter {if "access" in [tags][0] {grok {match => { "message" => "%{COMBINEDAPACHELOG}"}}geoip {source => "clientip"}date {match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]target => "@timestamp"timezone => "Asia/Shanghai"}useragent {source => "useragent"target => "useragent"}mutate {convert => ["bytes","integer"]convert => ["response_time", "float"]convert => ["upstream_response_time", "float"]remove_field => ["message"]add_field => { "target_index" => "app-logstash-nginx-access-%{+YYYY.MM.dd}" } }# 提取 referrer 具体的域名 /^"http/if [referrer] =~ /^"http/ {grok {match => { "referrer" => '%{URIPROTO}://%{URIHOST:referrer_host}' }}}# 提取用户请求资源类型以及资源 ID 编号if "test.com" in [referrer_host] {grok {match => { "referrer" => '%{URIPROTO}://%{URIHOST}/(%{NOTSPACE:test_type}/%{NOTSPACE:test_res_id})?' }}}}else if "error" in [tags][0] {date {match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]target => "@timestamp"timezone => "Asia/Shanghai"}mutate {add_field => { "target_index" => "app-logstash-nginx-error-%{+YYYY.MM.dd}" }}}
}output {elasticsearch {hosts => ["10.0.0.21:9200","10.0.0.22:9200","10.0.0.23:9200"]user => "logstash_push_es"password => "111111"index => "%{[target_index]}"template_overwrite => true}stdout {codec => rubydebug}
}
