记一次open-falcon手动push数据

nginx状态码监控

#falcon的agent部署目录
[root@nginxtest scripts]# pwd
/home/falcon/scripts

[root@nginxtest scripts]# cat sentry_count.sh 
#!/bin/bash
#sentry数量统计

ts=`date +%s`
t=`LC_ALL=en_US.UTF-8 date '+%d/%b/%Y:%H:%M' -d "-1 minute"`
valus=`tail -1000000 /usr/local/nginx/logs/access.log |egrep 'sentry.chuntianu.me' |egrep 'POST'|wc -l`
curl -X POST -d "[{\"metric\": \"sentry_nums\", \"endpoint\": \"nginxtest\", \"timestamp\": $ts,\"step\": 60,\"value\": $valus,\"counterType\": \"GAUGE\",\"tags\": \"request=post,nginx_cluster=nginxtest\"}]" http://127.0.0.1:1988/v1/push

[root@nginxtest scripts]# cat nginx.sh 
#!/bin/bash
#访问状态统计
codes="302 400 401 403 404 500 502 503"

for code in $codes;do

  ts=`date +%s`;
  t=`LC_ALL=en_US.UTF-8 date '+%d/%b/%Y:%H:%M' -d "-1 minute"`
  valus=`tail -1000000 /usr/local/nginx/logs/access.log | grep -v 'tvdoctor' | grep $t | awk '{print $9}' | grep $code | wc -l`
  curl -X POST -d "[{\"metric\": \"nginx_code\", \"endpoint\": \"nginxtest\", \"timestamp\": $ts,\"step\": 60,\"value\": $valus,\"counterType\": \"GAUGE\",\"tags\": \"code=$code,nginx_cluster=nginxtest\"}]" http://127.0.0.1:1988/v1/push

调用数据

在open-falcon的expression配置相关的表达式即可！

该表达式的功能是采集sentry消息的数量，如果数量连续3次大于100则告警！

参考文档

http://book.open-falcon.org/zh/usage/data-push.html
http://book.open-falcon.org/zh/philosophy/tags-and-hostgroup.html
https://github.com/GuyCheung/falcon-ngx_metric

---

etcd监控

action1

#!/bin/bash

#端口监控
CONN_STATUS=`netstat -lntp |egrep ':2379'|egrep -v '127.0.0.1' |wc -l`
curl -X POST -d "[{\"metric\":etcd_listen \"CONN_STATUS\", \"endpoint\": etcd_clusters, \"db\", \"timestamp\": $ts,\"step\": 60,\"value\": $CONN_STATUS,\"counterType\": \"GAUGE\",\"tags\": \"service=etcd,port=2379\"}]" http://127.0.0.1:1988/v1/push

curl --cacert /home/etcd/ca/ca.pem --cert /home/etcd/ca/client.pem --key /home/etcd/ca/client-key.pem https://127.0.0.1:2379/metrics 2>/dev/null |grep -v '#' >/tmp/etcd_monitor.txt
etcd_items="/tmp/etcd_monitor.txt"
ts=`date +%s`;

#"领导者是否存在; 领导者变化的数量; 共识提案的总数; 共识提案的总数; 目前正在处理的提案数量; 失败提案总数"
items1="has_leader leader_changes_seen_total proposals_applied_total proposals_committed_total proposals_failed_total proposals_pending"

#"wal调用的fsync总延迟时间; 后端调用总延迟时间"
items2="wal_fsync_duration_seconds_count backend_commit_duration_seconds_count"

#"发送给具有ID的对等体的总字节数; 从具有ID的对等方收到的总字节数; 发送给grpc客户端的总字节数; 接收到grpc客户端的总字节数; 同级之间总的往返时间"
items3="peer_sent_bytes_total peer_received_bytes_total grpc_received_bytes_total grpc_sent_bytes_total peer_round_trip_time_seconds_count"

#"打开文件描述符的数量; 允许打开文件描述符的最大数量"
items4="process_open_fds process_max_fds"

all_items="items1 items2 items3 items4"

for item in $all_items;do

#sever监控
  if [ $item == items1 ];then
    for i in $items1;do
    valus=`cat $etcd_items |grep "$i" |awk '{print $2}'`
  curl -X POST -d "[{\"metric\":etcd_server, \"etcd_status\", \"endpoint\": etcd_cluster, \"timestamp\": $ts,\"step\": 60,\"value\": $valus,\"counterType\": \"GAUGE\",\"tags\": \"item=etcd_server_$i,service=etcd\"}]" http://127.0.0.1:1988/v1/push
    done

#磁盘
  elif [ $item == items2 ];then
    for i in $items2;do
      valus=`cat $etcd_items |grep "$i" |awk '{print $2}'`
  curl -X POST -d "[{\"metric\":etcd_disk, \"etcd_status\", \"endpoint\": etcd_cluster, \"timestamp\": $ts,\"step\": 60,\"value\": $valus,\"counterType\": \"GAUGE\",\"tags\": \"item=etcd_server_$i,service=etcd\"}]" http://127.0.0.1:1988/v1/push
    done

#网络
  elif [ $item == items3 ];then
    for i in $items3;do
      valus=`cat $etcd_items |grep "$i" |awk '{print $2}'`
  curl -X POST -d "[{\"metric\":etcd_net, \"etcd_status\", \"endpoint\": etcd_cluster, \"timestamp\": $ts,\"step\": 60,\"value\": $valus,\"counterType\": \"GAUGE\",\"tags\": \"item=etcd_server_$i,service=etcd\"}]" http://127.0.0.1:1988/v1/push
    done

#其他
  elif [ $item == items4 ];then
    for i in $items4;do
      valus=`cat $etcd_items |grep "$i" |awk '{print $2}'`
  curl -X POST -d "[{\"metric\":etcd_fd, \"etcd_status\", \"endpoint\": etcd_cluster, \"timestamp\": $ts,\"step\": 60,\"value\": $valus,\"counterType\": \"GAUGE\",\"tags\": \"item=etcd_server_$i,service=etcd\"}]" http://127.0.0.1:1988/v1/push
    done
  fi
done

采集某时间段内nginx日志

#!/bin/bash
#sentry数量统计

apis="3 24"
for api in $apis;do
  ts=`date +%s`;
  t1=`LC_ALL=en_US.UTF-8 date '+%d\/%b\/%Y:%H:%M' -d "-3 minute"` 
  t2=`LC_ALL=en_US.UTF-8 date '+%d\/%b\/%Y:%H:%M' -d "-1 minute"` 
  if [ $api -eq 3 ];then
    project=medweb_online
  elif [ $api -eq 24 ];then
    project=op
  fi

  valus=`tail -1000000 /usr/local/nginx/logs/access.log |egrep 'sentry.chunyu.me' |sed -n "/$t1/,/$t2/"p |egrep 'POST'|egrep "/api/$api/" |wc -l`
  curl -X POST -d "[{\"metric\": \"sentry\", \"endpoint\": \"nginxtest\", \"timestamp\": $ts,\"step\": 60,\"value\": $valus,\"type\":\"sentry\",\"counterType\": \"GAUGE\",\"tags\": \"project=$project\"}]" http://127.0.0.1:1988/v1/push

#脚本测试
echo $t
echo $t1
echo $t2
echo $api:$project:$valus
done

etcd监控

#!/bin/bash

monit_keys=`curl --cacert /home/etcd/ca/ca.pem --cert /home/etcd/ca/client.pem --key /home/etcd/ca/client-key.pem https://127.0.0.1:2379/metrics 2>/dev/null |grep -v '#'`
ts=`date +%s`;
#"领导者是否存在; 领导者变化的数量; 共识提案的总数; 共识提案的总数; 目前正在处理的提案数量; 失败提案总数; wal调用的fsync总延迟时间; 后端调用总延迟时间; 发送给具有ID的对等体的总字节数; 发送给grpc客户端的总
字节数; 接收到grpc客户端的总字节数; 同级之间总的往返时间; 打开文件描述符的数量; 允许打开文件描述符的最大数量"
items="etcd_server_has_leader etcd_server_leader_changes_seen_total etcd_server_proposals_applied_total etcd_server_proposals_committed_total etcd_server_proposals_failed_total etcd_server_proposals_pending etcd_disk_wal_fsync_duration_seconds_count etcd_disk_backend_commit_duration_seconds_count etcd_network_peer_sent_bytes_total etcd_network_client_grpc_received_bytes_total etcd_network_client_grpc_sent_bytes_total etcd_network_peer_round_trip_time_seconds_count process_open_fds process_max_fds"

for item in $items;do
  valus=`echo $monit_keys|xargs -n2 |grep "$item" |awk '{print $2}' |uniq`
  if [ "$item"x = "etcd_network_peer_sent_bytes_total"x ];then
    valus=`echo $valus |awk '{print $1}' |awk -F'e+' '{print $1}'`
    valus=$(printf "%.f" `echo "$valus*10000000"|bc`)
  fi

  if [ "$item"x == "etcd_network_peer_received_bytes_total"x ];then
    valus=`echo $valus |awk '{print $1}' |awk -F'e+' '{print $1}'`
    valus=$(printf "%.f" `echo "$valus*1000000"|bc`)
  fi
  curl -X POST -d "[{\"metric\": \"$item\", \"endpoint\": \"configcenter\", \"timestamp\": $ts,\"step\": 60,\"value\": $valus,\"counterType\": \"GAUGE\",\"tags\": \"etcd_cluster=configcenter,srv=etcd\"}]" http://127.0.0.1:1988/v1/push

done

注意：本段代码的重点在“valus=echo $monit_keys|xargs -n2 |grep "$item" |awk '{print $2}' |uniq”中 “uniq” 的使用，实际环境中有部分关键字在“$monit_keys”中会重复出现，如果不加uniq会同时有多个数据被push，这样就会导致curl POST数据是报错“cannot decode body”。当然也如果通过关键字过滤出来的monit_keys完全一致，也可以写成“monit_keys=curl --cacert /home/etcd/ca/ca.pem --cert /home/etcd/ca/client.pem --key /home/etcd/ca/client-key.pem https://127.0.0.1:2379/metrics 2>/dev/null |grep -v '#' |uniq”,但是在本脚本中过滤出来的keys值不完全一致，但是value相同，所以只能加在“valus=...”。