alertmanager add enterprise micro-channel alarm monitoring alarm

Prometheus machine: 172.27.143.155
alertmanager machine: 172.27.143.150

First, the above configuration and Prometheus grafana service machine 155 above
the next service configuration alermanager
. 1, wget https://github.com/prometheus/alertmanager/releases/download/v0.20.0/alertmanager-0.20.0.linux-amd64 .tar.gz

2, tar zxf Alert Manager 0.20.0.linux amd64.tar.gz-
3, etc. Alert Manager 0.20.0.linux-amd64 / usr / local / alert manager
4, VIM alertmanager.yml

5、vim /etc/alertmanager/template/wechat.tmpl

{{ define "wechat.default.message" }}
{{ range $i, $alert :=.Alerts }}
========监控报警==========
告警状态：{{ .Status }}
告警级别：{{ $alert.Labels.severity }}
告警类型：{{ $alert.Labels.alertname }}
告警应用：{{ $alert.Annotations.summary }}
告警主机：{{ $alert.Labels.instance }}
告警详情：{{ $alert.Annotations.description }}
触发阀值：{{ $alert.Annotations.value }}
告警时间：{{ $alert.StartsAt.Format "2006-01-02 15:04:05" }}
========end==========
{{ end }}
{{ end }}

6、完成之后启动服务
nohup ./alertmanager &

接下来配置Prometheus服务
修改配置文件

需要创建一个 rules目录
里面又2个文件，一个时主机监控，一个时容器监控
1）cat host_sys.yml

groups:

• name: Host
  rules:
  • alert: Memory Usage
    expr: (node_memory_MemTotal_bytes - (node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes)) / node_memory_MemTotal_bytes * 100 > 2
    for: 1m
    labels:
    name: Memory
    severity: Warning
    annotations:
    summary: " {{ $labels.appname }} "
    description: "宿主机内存使用率超过80%."
    value: "{{ $value }}"
  • alert: CPU Usage
    expr: sum(avg without (cpu)(irate(node_cpu_seconds_total{mode!='idle'}[5m]))) by (instance,appname) > 0.05
    for: 1m
    labels:
    name: CPU
    severity: Warning
    annotations:
    summary: " {{ $labels.appname }} "
    description: "宿主机CPU使用率超过65%."
    value: "{{ $value }}"
  • alert: HostLoad
    expr: node_load5 > 4
    for: 1m
    labels:
    name: Load
    severity: Warning
    annotations:
    summary: "{{ $labels.appname }} "
    description: " 主机负载5分钟超过4."
    value: "{{ $value }}"
  • alert: Filesystem Usage
    expr: 1-(node_filesystem_free_bytes / node_filesystem_size_bytes) > 0.3
    for: 1m
    labels:
    name: Disk
    severity: Warning
    annotations:
    summary: " {{ $labels.appname }} "
    description: " 宿主机 [ {{ $labels.mountpoint }} ]分区使用超过80%."
    value: "{{ $value }}%"
  • alert: Diskio writes
    expr: irate(node_disk_writes_completed_total{job=~"Host"}[1m]) > 50
    for: 1m
    labels:
    name: Diskio
    severity: Warning
    annotations:
    summary: " {{ $labels.appname }} "
    description: " 宿主机 [{{ $labels.device }}]磁盘1分钟平均写入IO负载较高."
    value: "{{ $value }}iops"
  • alert: Diskio reads
    expr: irate(node_disk_reads_completed_total{job=~"Host"}[1m]) > 5
    for: 1m
    labels:
    name: Diskio
    severity: Warning
    annotations:
    summary: " {{ $labels.appname }} "
    description: " 宿主机 [{{ $labels.device }}]磁盘1分钟平均读取IO负载较高."
    value: "{{ $value }}iops"
  • alert: Network_receive
    expr: irate(node_network_receive_bytes_total{device!~"lo|bond[0-9]|cbr[0-9]|veth.|virbr.|ovs-system"}[5m]) / 1048576 > 5
    for: 1m
    labels:
    name: Network_receive
    severity: Warning
    annotations:
    summary: " {{ $labels.appname }} "
    description: " 宿主机 [{{ $labels.device }}] 网卡5分钟平均接收流量超过5Mbps."
    value: "{{ $value }}Mbps"
  • alert: Network_transmit
    expr: irate(node_network_transmit_bytes_total{device!~"lo|bond[0-9]|cbr[0-9]|veth.|virbr.|ovs-system"}[5m]) / 1048576 > 5
    for: 1m
    labels:
    name: Network_transmit
    severity: Warning
    annotations:
    summary: " {{ $labels.appname }} "
    description: " 宿主机 [{{ $labels.device }}] 网卡5分钟内平均发送流量超过5Mbps."
    value: "{{ $value }}Mbps"

2） cat container_sys.yml

groups:

• name: Container
  rules:
  • alert: CPU Usage
    expr: (sum by(name,instance) (rate(container_cpu_usage_seconds_total{image!=""}[5m]))*100) > 80
    for: 1m
    labels:
    name: CPU
    severity: Warning
    annotations:
    summary: "{{ $labels.name }} "
    description: " 容器CPU使用超过80%"
    value: "{{ $value }}%"
  • alert: Memory Usage
    expr: (container_memory_usage_bytes{name=~".+"} - container_memory_cache{name=~".+"}) / container_spec_memory_limit_bytes{name=~".+"} * 100 > 80
    for: 1m
    labels:
    name: Memory
    severity: Warning
    annotations:
    summary: "{{ $labels.name }} "
    description: " 容器内存使用超过80%."
    value: "{{ $value }}%"
  • alert: Network_receive
    expr: irate(container_network_receive_bytes_total{name=~".+",interface=~"eth.+"}[5m]) / 1048576 > 5
    for: 1m
    labels:
    name: Network_receive
    severity: Warning
    annotations:
    summary: "{{ $labels.name }} "
    description: "容器 [{{ $labels.device }}] 网卡5分钟平均接收流量超过5Mbps."
    value: "{{ $value }}Mbps"
  • Alert: Network_transmit
    expr: irate (container_network_transmit_bytes_total {. "+" name = ~, interface = ~ "Eth +."} [5m]) / 1048576>. 5
    for: 1M
    Labels:
    name: Network_transmit
    Severity: Warning
    Annotations:
    Summary: " labels.name}} {$ { "
    Description:" container [{{$ labels.device}}] NIC 5 minutes the average transmission traffic than 5Mbps. "
    value:" {{value}} $ Mbps "

After configuration is complete restart the service Prometheus

Wait a minute to verify the effect

Recovery is the following

Monitoring container is complete