elk安装

官网下载最新的rpm包安装。

http://blog.51cto.com/liqingbiao/1928653

es安装head

先安装node

wget https://nodejs.org/dist/v0.10.48/node-v0.10.48.tar.gz

加压，make，make install

node --version

git clone https://github.com/mobz/elasticsearch-head.git

cd elasticsearch-head/

npm install

所有依赖包安装成功后，修改 elasticsearch-head 目录下的 Gruntfile.js 文件，在 options 属性内增加 hostname，设置为 0.0.0.0。

connect: {server: {options: {hostname: '0.0.0.0',port: 9100, base: '.', keepalive: true } } }

修改 Elasticsearch 配置文件 config/elasticsearch.yml

在配置文件最后增加两个配置项，这样 elasticsearch-head 插件才可以访问 Elasticsearch 。

http.cors.enabled: true http.cors.allow-origin: "*"

npm run start

http://10.21.8.88:9100/

elk添加ip -map

input{
beats{
port => "5044"
}
}

filter{
#grok{match => { "message" => "%{IP:client_ip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] \"%{WORD:method} /%{NOTSPACE:request_page} HTTP/%{NUMBER:http_version}\" %{NUMBER:status}" }}

grok {match => {"message" => "%{IP:client_ip} %{USER:remote_user} %{USER:remote_auth} \[%{HTTPDATE:timestamp}\] %{QUOTEDSTRING:request} %{NUMBER:status_code} %{NUMBER:body_bytes_sent} %{QUOTEDSTRING:http_referer} %{QUOTEDSTRING:http_user_agent} %{QUOTEDSTRING:remote_addr} %{QUOTEDSTRING:upstream_response_time} %{QUOTEDSTRING:request_time}" }

}

geoip{

source => "client_ip"
target => "geoip"
database => "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-filter-geoip-5.0.3-java/vendor/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]

}
#
mutate {
convert => [ "[geoip][coordinates]", "float" ]
#convert => [ "[request_time]", "float" ]
#convert => [ "[upstream_response_time]", "float" ]
#
}
#

}

output{
elasticsearch{
hosts => ["ip9200"]
index => "logstash-www-%{+YYYY.MM.dd}"
}
}

###############################################################################################

解析mongo日志

logstash配置

input {
beats {
port => "5044"
type => "mongodblog"
}
}

filter {
if [type] == "mongodblog" {
grok {
match => ["message","%{TIMESTAMP_ISO8601:timestamp}\s+I %{WORD:MONGO_ACTION}\s+\[%{WORD:SOCK_ACTION}\]\s+%{GREEDYDATA:body}"]
remove_field => [ "message" ]
}

if [body] =~ "ms$" {
grok {
match => ["body","%{WORD:command_action}\s+%{WORD:dbname}\.\$?%{WORD:collname}\s+%{GREEDYDATA:command_content}\s+%{NUMBER:time_spend}ms"]
}
}

date {
match => [ "timestamp", "UNIX", "YYYY-MM-dd HH:mm:ss", "ISO8601"]
remove_field => [ "timestamp" ]
}

mutate {
remove_field => ["message"]
}
}
}

output {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "mongo-%{+YYYY.MM.dd}"
}
}

参考

日志格式

2018-03-06T03:11:51.338+0800 I COMMAND  [conn1978967] command top_fba.$cmd command: createIndexes { createIndexes: "top_amazon_fba_inventory_data_2018-03-06", indexes: [ { key: { sellerId: 1,sku: 1, updateTime: 1 }, name: "sellerId_1_sku_1_updateTime_1" } ] } keyUpdates:0 writeConflicts:0 numYields:0 reslen:113 locks:{ Global: { acquireCount: { r: 3, w: 3 } }, Database: { acquir
eCount: { w: 2, W: 1 } }, Collection: { acquireCount: { w: 1 } }, Metadata: { acquireCount: { w: 2 } }, oplog: { acquireCount: { w: 2 } } } protocol:op_query 5751ms