参考 https://blog.51cto.com/u_11880730/11223693
https://github.com/prometheus/node_exporter
tar zxvf node_exporter-1.8.2.linux-amd64.tar.gz -C /usr/local/
cd /usr/local/
mv node_exporter-1.0.1.linux-amd64 node_exporter
vim /usr/lib/systemd/system/node_exporter.service
[Unit]
Description=node_exporter
After=network.target[Service]
Type=simple
ExecStart=/usr/local/node_exporter/node_exporter[Install]
WantedBy=multi-user.target
http://172.17.13.1:9100/metrics
vim external-nodeexporter-service.yaml
apiVersion: v1
kind: Endpoints
metadata:labels:app.kubernetes.io/name: external-node-exporter #endpoints 资源Labels标签name: external-node-exporter #endpoints 名称namespace: monitoring #所属命名空间
subsets: #用于指定一个或者多个IP,代表后端服务实例的地址
- addresses:- ip: 172.17.13.1- ip: 172.17.13.2- ip: 172.17.13.3- ip: 172.17.13.4- ip: 172.17.13.6- ip: 172.17.13.7- ip: 172.17.13.8- ip: 172.17.13.9- ip: 172.17.13.13ports: #描述IP所使用的端口号信息- name: http-metricsport: 9100protocol: TCP
---
apiVersion: v1
kind: Service
metadata:labels:app.kubernetes.io/name: external-node-exporter #同样是用于指定svc资源的标签Labelsname: external-node-exporter #定义svc资源的名称namespace: monitoring #定义所属命名空间
spec:ports: #定义了Svc所监控的端口以及关联的后端Pod端口信息- name: http-metrics #定义了端口的名称port: 9100 #定义了svc所暴露出来的端口protocol: TCP #定义了端口使用的协议targetPort: 9100 #定义了与svc所关联的Pod端口,该端口一般是Pod容器服务的内部端口sessionAffinity: None #定义了会话亲和力类型,表示不启用亲和性,即新的请求可能会被路由到不同的后端Pod上,而不考虑之前请求的路由情况type: ClusterIP #定义Svc访问类型,该类型只能在集群内部使用,适合作为后端服务的访问入口
vim external-nodeexporter-serviceMonitor.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor # prometheus-operator 定义的CRD
metadata:name: external-node-exporter namespace: monitoringlabels:k8s-apps: external-node-exporter
spec:jobLabel: app.kubernetes.io/name selector:matchLabels:app.kubernetes.io/name: external-node-exporternamespaceSelector:matchNames: # 配置需要自动发现的命名空间,可以配置多个- monitoringendpoints:- port: http-metrics # 拉去metric的端口,这个写的是 service的端口名称,即 service yaml的spec.ports.nameinterval: 15s # 拉取metric的时间间隔
vim external-nodeexporter-prometheusRule.yaml
需注意复用的规则应将监控目标修改成外部的node Job名称,上述job名称为external-node-exporter。那么该规则同样要指定该job名称,否则监控规则无法生效
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:labels:app.kubernetes.io/component: exporterapp.kubernetes.io/name: external-node-exporterapp.kubernetes.io/part-of: kube-prometheusapp.kubernetes.io/version: 1.8.2prometheus: k8srole: alert-rulesname: external-node-exporter-rulesnamespace: monitoring
spec:groups:- name: external-node-exporterrules:- alert: NodeFilesystemSpaceFillingUpannotations:description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingupsummary: Filesystem is predicted to run out of space within the next 24 hours.expr: |(node_filesystem_avail_bytes{job="external-node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="external-node-exporter",fstype!="",mountpoint!=""} * 100 < 15andpredict_linear(node_filesystem_avail_bytes{job="external-node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0andnode_filesystem_readonly{job="external-node-exporter",fstype!="",mountpoint!=""} == 0)for: 1hlabels:severity: warning- alert: NodeFilesystemSpaceFillingUpannotations:description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up fast.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingupsummary: Filesystem is predicted to run out of space within the next 4 hours.expr: |(node_filesystem_avail_bytes{job="external-node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="external-node-exporter",fstype!="",mountpoint!=""} * 100 < 10andpredict_linear(node_filesystem_avail_bytes{job="external-node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0andnode_filesystem_readonly{job="external-node-exporter",fstype!="",mountpoint!=""} == 0)for: 1hlabels:severity: critical- alert: NodeFilesystemAlmostOutOfSpaceannotations:description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspacesummary: Filesystem has less than 5% space left.expr: |(node_filesystem_avail_bytes{job="external-node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="external-node-exporter",fstype!="",mountpoint!=""} * 100 < 5andnode_filesystem_readonly{job="external-node-exporter",fstype!="",mountpoint!=""} == 0)for: 30mlabels:severity: warning- alert: NodeFilesystemAlmostOutOfSpaceannotations:description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspacesummary: Filesystem has less than 3% space left.expr: |(node_filesystem_avail_bytes{job="external-node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="external-node-exporter",fstype!="",mountpoint!=""} * 100 < 3andnode_filesystem_readonly{job="external-node-exporter",fstype!="",mountpoint!=""} == 0)for: 30mlabels:severity: critical- alert: NodeFilesystemFilesFillingUpannotations:description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingupsummary: Filesystem is predicted to run out of inodes within the next 24 hours.expr: |(node_filesystem_files_free{job="external-node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="external-node-exporter",fstype!="",mountpoint!=""} * 100 < 40andpredict_linear(node_filesystem_files_free{job="external-node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0andnode_filesystem_readonly{job="external-node-exporter",fstype!="",mountpoint!=""} == 0)for: 1hlabels:severity: warning- alert: NodeFilesystemFilesFillingUpannotations:description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up fast.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingupsummary: Filesystem is predicted to run out of inodes within the next 4 hours.expr: |(node_filesystem_files_free{job="external-node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="external-node-exporter",fstype!="",mountpoint!=""} * 100 < 20andpredict_linear(node_filesystem_files_free{job="external-node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0andnode_filesystem_readonly{job="external-node-exporter",fstype!="",mountpoint!=""} == 0)for: 1hlabels:severity: critical- alert: NodeFilesystemAlmostOutOfFilesannotations:description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffilessummary: Filesystem has less than 5% inodes left.expr: |(node_filesystem_files_free{job="external-node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="external-node-exporter",fstype!="",mountpoint!=""} * 100 < 5andnode_filesystem_readonly{job="external-node-exporter",fstype!="",mountpoint!=""} == 0)for: 1hlabels:severity: warning- alert: NodeFilesystemAlmostOutOfFilesannotations:description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffilessummary: Filesystem has less than 3% inodes left.expr: |(node_filesystem_files_free{job="external-node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="external-node-exporter",fstype!="",mountpoint!=""} * 100 < 3andnode_filesystem_readonly{job="external-node-exporter",fstype!="",mountpoint!=""} == 0)for: 1hlabels:severity: critical- alert: NodeNetworkReceiveErrsannotations:description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} receive errors in the last two minutes.'runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworkreceiveerrssummary: Network interface is reporting many receive errors.expr: |rate(node_network_receive_errs_total{job="external-node-exporter"}[2m]) / rate(node_network_receive_packets_total{job="external-node-exporter"}[2m]) > 0.01for: 1hlabels:severity: warning- alert: NodeNetworkTransmitErrsannotations:description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} transmit errors in the last two minutes.'runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworktransmiterrssummary: Network interface is reporting many transmit errors.expr: |rate(node_network_transmit_errs_total{job="external-node-exporter"}[2m]) / rate(node_network_transmit_packets_total{job="external-node-exporter"}[2m]) > 0.01for: 1hlabels:severity: warning- alert: NodeHighNumberConntrackEntriesUsedannotations:description: '{{ $value | humanizePercentage }} of conntrack entries are used.'runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodehighnumberconntrackentriesusedsummary: Number of conntrack are getting close to the limit.expr: |(node_nf_conntrack_entries{job="external-node-exporter"} / node_nf_conntrack_entries_limit) > 0.75labels:severity: warning- alert: NodeTextFileCollectorScrapeErrorannotations:description: Node Exporter text file collector on {{ $labels.instance }} failed to scrape.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodetextfilecollectorscrapeerrorsummary: Node Exporter text file collector failed to scrape.expr: |node_textfile_scrape_error{job="external-node-exporter"} == 1labels:severity: warning- alert: NodeClockSkewDetectedannotations:description: Clock at {{ $labels.instance }} is out of sync by more than 0.05s. Ensure NTP is configured correctly on this host.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodeclockskewdetectedsummary: Clock skew detected.expr: |(node_timex_offset_seconds{job="external-node-exporter"} > 0.05andderiv(node_timex_offset_seconds{job="external-node-exporter"}[5m]) >= 0)or(node_timex_offset_seconds{job="external-node-exporter"} < -0.05andderiv(node_timex_offset_seconds{job="external-node-exporter"}[5m]) <= 0)for: 10mlabels:severity: warning- alert: NodeRAIDDegradedannotations:description: RAID array '{{ $labels.device }}' at {{ $labels.instance }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddegradedsummary: RAID Array is degraded.expr: |node_md_disks_required{job="external-node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} - ignoring (state) (node_md_disks{state="active",job="external-node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}) > 0for: 15mlabels:severity: critical- alert: NodeRAIDDiskFailureannotations:description: At least one device in RAID array at {{ $labels.instance }} failed. Array '{{ $labels.device }}' needs attention and possibly a disk swap.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddiskfailuresummary: Failed device in RAID array.expr: |node_md_disks{state="failed",job="external-node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} > 0labels:severity: warning- alert: NodeFileDescriptorLimitannotations:description: File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimitsummary: Kernel is predicted to exhaust file descriptors limit soon.expr: |(node_filefd_allocated{job="external-node-exporter"} * 100 / node_filefd_maximum{job="external-node-exporter"} > 70)for: 15mlabels:severity: warning- alert: NodeFileDescriptorLimitannotations:description: File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimitsummary: Kernel is predicted to exhaust file descriptors limit soon.expr: |(node_filefd_allocated{job="external-node-exporter"} * 100 / node_filefd_maximum{job="external-node-exporter"} > 90)for: 15mlabels:severity: critical- alert: NodeCPUHighUsageannotations:description: |CPU usage at {{ $labels.instance }} has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodecpuhighusagesummary: High CPU usage.expr: |sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="external-node-exporter", mode!="idle"}[2m]))) * 100 > 90for: 15mlabels:severity: info- alert: NodeSystemSaturationannotations:description: |System load per core at {{ $labels.instance }} has been above 2 for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.This might indicate this instance resources saturation and can cause it becoming unresponsive.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemsaturationsummary: System saturated, load per core is very high.expr: |node_load1{job="external-node-exporter"}/ count without (cpu, mode) (node_cpu_seconds_total{job="external-node-exporter", mode="idle"}) > 2for: 15mlabels:severity: warning- alert: NodeMemoryMajorPagesFaultsannotations:description: |Memory major pages are occurring at very high rate at {{ $labels.instance }}, 500 major page faults per second for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.Please check that there is enough memory available at this instance.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodememorymajorpagesfaultssummary: Memory major page faults are occurring at very high rate.expr: |rate(node_vmstat_pgmajfault{job="external-node-exporter"}[5m]) > 500for: 15mlabels:severity: warning- alert: NodeMemoryHighUtilizationannotations:description: |Memory is filling up at {{ $labels.instance }}, has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodememoryhighutilizationsummary: Host is running out of memory.expr: |100 - (node_memory_MemAvailable_bytes{job="external-node-exporter"} / node_memory_MemTotal_bytes{job="external-node-exporter"} * 100) > 90for: 15mlabels:severity: warning- alert: NodeDiskIOSaturationannotations:description: |Disk IO queue (aqu-sq) is high on {{ $labels.device }} at {{ $labels.instance }}, has been above 10 for the last 30 minutes, is currently at {{ printf "%.2f" $value }}.This symptom might indicate disk saturation.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodediskiosaturationsummary: Disk IO queue is high.expr: |rate(node_disk_io_time_weighted_seconds_total{job="external-node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) > 10for: 30mlabels:severity: warning- alert: NodeSystemdServiceFailedannotations:description: Systemd service {{ $labels.name }} has entered failed state at {{ $labels.instance }}runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemdservicefailedsummary: Systemd service has entered failed state.expr: |node_systemd_unit_state{job="external-node-exporter", state="failed"} == 1for: 5mlabels:severity: warning- alert: NodeBondingDegradedannotations:description: Bonding interface {{ $labels.master }} on {{ $labels.instance }} is in degraded state due to one or more slave failures.runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodebondingdegradedsummary: Bonding interface is degradedexpr: |(node_bonding_slaves - node_bonding_active) != 0for: 5mlabels:severity: warning- name: external-external-node-exporter.rulesrules:- expr: |count without (cpu, mode) (node_cpu_seconds_total{job="external-node-exporter",mode="idle"})record: instance:node_num_cpu:sum- expr: |1 - avg without (cpu) (sum without (mode) (rate(node_cpu_seconds_total{job="external-node-exporter", mode=~"idle|iowait|steal"}[5m])))record: instance:node_cpu_utilisation:rate5m- expr: |(node_load1{job="external-node-exporter"}/instance:node_num_cpu:sum{job="external-node-exporter"})record: instance:node_load1_per_cpu:ratio- expr: |1 - ((node_memory_MemAvailable_bytes{job="external-node-exporter"}or(node_memory_Buffers_bytes{job="external-node-exporter"}+node_memory_Cached_bytes{job="external-node-exporter"}+node_memory_MemFree_bytes{job="external-node-exporter"}+node_memory_Slab_bytes{job="external-node-exporter"}))/node_memory_MemTotal_bytes{job="external-node-exporter"})record: instance:node_memory_utilisation:ratio- expr: |rate(node_vmstat_pgmajfault{job="external-node-exporter"}[5m])record: instance:node_vmstat_pgmajfault:rate5m- expr: |rate(node_disk_io_time_seconds_total{job="external-node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m])record: instance_device:node_disk_io_time_seconds:rate5m- expr: |rate(node_disk_io_time_weighted_seconds_total{job="external-node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m])record: instance_device:node_disk_io_time_weighted_seconds:rate5m- expr: |sum without (device) (rate(node_network_receive_bytes_total{job="external-node-exporter", device!="lo"}[5m]))record: instance:node_network_receive_bytes_excluding_lo:rate5m- expr: |sum without (device) (rate(node_network_transmit_bytes_total{job="external-node-exporter", device!="lo"}[5m]))record: instance:node_network_transmit_bytes_excluding_lo:rate5m- expr: |sum without (device) (rate(node_network_receive_drop_total{job="external-node-exporter", device!="lo"}[5m]))record: instance:node_network_receive_drop_excluding_lo:rate5m- expr: |sum without (device) (rate(node_network_transmit_drop_total{job="external-node-exporter", device!="lo"}[5m]))record: instance:node_network_transmit_drop_excluding_lo:rate5m
grafana
Node 模版ID1860