部署监控
version: '3.7'services:
#dingtalkdingtalk:image: timonwong/prometheus-webhook-dingtalk:latestcontainer_name: dingtalkrestart: alwayscommand:- '--config.file=/etc/prometheus-webhook-dingtalk/config.yml'volumes:- /data/monitor/dingtalk/config.yml:/etc/prometheus-webhook-dingtalk/config.yml- /etc/localtime:/etc/localtime:roports:- "8060:8060"
#alertmanageralertmanager:image: prom/alertmanager:latestcontainer_name: alertmanagerrestart: alwaysvolumes:- /data/monitor/alertmanager/config/alertmanager.yml:/etc/alertmanager/alertmanager.ymlports:- "9093:9093"
#prometheusprometheus:image: prom/prometheuscontainer_name: prometheusrestart: alwaysports:- "9090:9090"volumes:- /data/monitor/promethues/prometheus.yml:/etc/prometheus/prometheus.yml- /data/monitor/promethues/alert.yml:/etc/prometheus/rule.yml- /etc/localtime:/etc/localtime:ro
#grafanagrafana:image: grafana/grafanacontainer_name: grafanarestart: alwaysports:- "3000:3000"volumes:- /data/monitor/grafana:/var/lib/grafana
#node-exporternode-exporter:image: prom/node-exportercontainer_name: node-exporterrestart: alwaysports:- "9100:9100"volumes:- /proc:/host/proc:ro- /sys:/host/sys:ro- /:/rootfs:ro
Dingtalk配置文件
/data/monitor/dingtalk/config.yml
templates:- /etc/prometheus-webhook-dingtalk/templates/templates.tmpltargets: #配置多个接收方webhook2:url: https://oapi.dingtalk.com/robot/send?access_token=钉钉tokensecret: 钉钉加签Alertmanager配置文件
/data/monitor/alertmanager/config/alertmanager.yml
global:resolve_timeout: 5msmtp_smarthost: 'smtp.qiye.163.com:465' #邮箱smtp服务器代理，启用SSL发信, 端口一般是465smtp_from: 'user@163.com' #发送邮箱名称smtp_auth_username: 'user@163.com' #邮箱名称smtp_auth_password: 'password' #邮箱密码或授权码smtp_require_tls: falseroute:receiver: 'default'group_wait: 10sgroup_interval: 1mrepeat_interval: 1hgroup_by: ['alertname']inhibit_rules:
- source_match:severity: 'critical'target_match:severity: 'warning'equal: ['alertname', 'instance']receivers:
- name: 'default'webhook_configs:- url: 'http://dingtalk-IP:8060/dingtalk/webhook2/send' #webhoo2匹配dingtalk targetssend_resolved: true
Prometheus配置prometheus文件
/data/monitor/promethues/prometheus.yml
global:scrape_interval: 60sevaluation_interval: 60s
alerting:alertmanagers:- static_configs:- targets: ['IP:9093']
rule_files:- "/etc/prometheus/rule.yml"- "rules/*.yml"scrape_configs:- job_name: prometheusstatic_configs:- targets: ['localhost:9090']labels:instance: prometheus- job_name: litestatic_configs:- targets: ['IP:9100']labels:env: dev- job_name: redis_exporterstatic_configs:- targets: ['IP:9121']labels:env: devident: redis- job_name: mysql_exporterstatic_configs:- targets: ['IP:9104']labels:env: devident: mysql- job_name: emqx_exportermetrics_path: /api/v5/prometheus/statsscrape_interval: 5shonor_labels: truestatic_configs:- targets: ['IP:18083']- job_name: 'alertmanager'scrape_interval: 15sstatic_configs:- targets: ['IP:9100']
Prometheus配置alert文件
/data/monitor/promethues/alert.yml
groups:
- name: 服务器主机信息监控告警rules:- alert: 公司内部服务器监控expr: up {job="公司内部服务器"} == 0for: 0mlabels:severity: 非常严重annotations:description: "监控的目标已丢失，请检查服务器自身或node_exporter服务"- alert: "内存报警"expr: 100 - ((node_memory_MemAvailable_bytes * 100) / node_memory_MemTotal_bytes) > 10for: 1m # 告警持续时间，超过这个时间才会发送给alertmanagerlabels:severity: 严重annotations:summary: "{{ $labels.instance }} 内存使用率过高，请尽快处理！"description: "{{ $labels.instance }}内存使用率超过95%,当前使用率{{ $value }}%."- alert: "磁盘空间报警"expr: (1 - node_filesystem_avail_bytes{fstype=~"ext4|xfs"} / node_filesystem_size_bytes{fstype=~"ext4|xfs"}) * 100 > 20for: 60slabels:severity: 严重annotations:summary: "{{ $labels.instance }}磁盘空间使用超过95%了"description: "{{ $labels.instance }}磁盘使用率超过95%,当前使用率{{ $value }}%."- alert: "CPU报警"expr: 100-(avg(irate(node_cpu_seconds_total{mode="idle"}[5m])) by(instance)* 100) > 5for: 120slabels:severity: 严重instance: "{{ $labels.instance }}"annotations:summary: "{{$labels.instance}}CPU使用率超过95%了"description: "{{ $labels.instance }}CPU使用率超过95%,当前使用率{{ $value }}%."- alert: "磁盘IO性能报警"expr: ((irate(node_disk_io_time_seconds_total[30m]))* 100) > 95for: 3mlabels:severity: 严重annotations:summary: "{{$labels.instance}} 流入磁盘IO使用率过高，请尽快处理！"description: "{{$labels.instance}} 流入磁盘IO大于95%,当前使用率{{ $value }}%."
)
)
