部署环境 $ kubectl get node NAME STATUS ROLES AGE VERSION master01 Ready master 13d v1.14.0 master02 Ready master 13d v1.14.0 master03 Ready master 13d v1.14.0 node01 Ready <none> 13d v1.14.0 node02 Ready <none> 13d v1.14.0 node03 Ready <none> 13d v1.14.0
目录结构 ├── control │ ├── alertmanager 组件1 │ │ ├── config.yaml │ │ ├── deploymen.yaml │ │ └── service.yaml │ ├── grafana 组件2 │ │ ├── deployment.yaml │ │ ├── pvc.yaml │ │ ├── pv.yaml │ │ └── service.yaml │ ├── namespace │ │ └── namespace.yaml │ ├── node-exporter 组件3 │ │ ├── node-exporter-service.yaml │ │ └── node-exporter.yaml │ └── prometheus 组件4 │ ├── configmap │ │ ├── config.yaml │ │ ├── config.yaml.bak │ │ ├── prometheus.yaml │ │ ├── rules-down.yml │ │ └── rules-load.yml.bak │ ├── deployment.yaml │ ├── pvc.yaml │ ├── pv.yaml │ ├── rbac.yaml │ └── service.yaml
一、创建名称空间
mkdir control/{alertmanager,grafana,namespace,node-exporter,prometheus}
cd control/namespace
cat namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
name: ns-monitor
labels:
name: ns-monitor
# 生成配置文件
kubectl create -f namespace.yaml
二、部署prometheus node exporter （在master01 操作） cd /root/control/node-exporter
创建 node exporter pod
$ cat node-exporter.yaml
apiVersion: apps/v1beta2
kind: DaemonSet
metadata:
name: node-exporter
namespace: ns-monitor
labels:
app: node-exporter
spec:
revisionHistoryLimit: 10
selector:
matchLabels:
app: node-exporter
template:
metadata:
labels:
app: node-exporter
spec:
containers:
- name: node-exporter
image: prom/node-exporter:v0.16.0
ports:
- containerPort: 9100
protocol: TCP
name: http
hostNetwork: true
hostPID: true
tolerations: # 在master 节点也会创建pod
- effect: NoSchedule
operator: Exists
# 生成配置文件
kubectl create -f node-exporter.yaml
创建node exporter pod service
$ cat node-exporter-service.yaml
apiVersion: v1
kind: Service
metadata:
name: node-exporter-service
namespace: ns-monitor
labels:
app: node-exporter-service
spec:
ports:
- name: http
port: 9100
nodePort: 31672
protocol: TCP
type: NodePort
selector:
app: node-exporter
# 生成配置文件
kubectl create -f node-exporter-service.yaml
三、部署prometheus (在master01 操作) cd control/prometheus
创建prometheus pv (前提在各个node 节点创建目录/nfs/prometheus/data)
$ cat pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: "prometheus-data-pv"
labels:
name: "prometheus-data-pv"
release: stable
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Recycle
hostPath:
path: /nfs/prometheus/data
#生成配置文件
kubectl create -f pv.yaml
创建 prometheus pvc
$ cat pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: prometheus-data-pvc
namespace: ns-monitor
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Gi
selector:
matchLabels:
name: prometheus-data-pv
release: stable
#生成配置文件
kubectl create -f pvc.yaml
创建 rbac 认证,因为我们需要在 prometheus 中去访问 Kubernetes 的相关信息，所以我们这里管理了一个名为 prometheus 的 serviceAccount 对象
$ cat rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: prometheus
rules:
- apiGroups: [""]
resources:
- nodes
- nodes/proxy
- services
- endpoints
- pods
verbs:
- get
- watch
- list
- apiGroups:
- extensions
resources:
- ingresses
verbs:
- get
- watch
- list
- nonResourceURLs: ["/metrics"]
verbs:
- get
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: prometheus
namespace: ns-monitor
labels:
app: prometheus
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: prometheus
subjects:
- kind: ServiceAccount
name: prometheus
namespace: ns-monitor
roleRef:
kind: ClusterRole
name: prometheus
apiGroup: rbac.authorization.k8s.io
#生成配置文件
kubectl create -f rbac.yaml
创建prometheus 主配置文件 configmap
cd /root/control/prometheus/configmap
$ cat config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus-conf
namespace: ns-monitor
labels:
app: prometheus
data:
prometheus.yml: |-
# my global config
global:
scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
# scrape_timeout is set to the global default (10s).
# Alertmanager configuration
alerting:
alertmanagers:
- static_configs:
- targets:
- 'alertmanager-service.ns-monitor:9093' #如果报错，可以先注释此行，后续会添加此配置
# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
- "/etc/prometheus/rules/nodedown.rule.yml"
scrape_configs:
- job_name: 'prometheus'
static_configs:
- targets: ['localhost:9090']
- job_name: 'grafana'
static_configs:
- targets:
- 'grafana-service.ns-monitor:3000'
- job_name: 'kubernetes-apiservers'
kubernetes_sd_configs:
- role: endpoints
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
action: keep
regex: default;kubernetes;https
- job_name: 'kubernetes-nodes'
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- role: node
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics
- job_name: 'kubernetes-cadvisor'
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- role: node
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
- job_name: 'kubernetes-service-endpoints'
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
action: replace
target_label: __scheme__
regex: (https?)
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
action: replace
target_label: __address__
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_service_name]
action: replace
target_label: kubernetes_name
- job_name: 'kubernetes-services'
metrics_path: /probe
params:
module: [http_2xx]
kubernetes_sd_configs:
- role: service
relabel_configs:
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
action: keep
regex: true
- source_labels: [__address__]
target_label: __param_target
- target_label: __address__
replacement: blackbox-exporter.example.com:9115
- source_labels: [__param_target]
target_label: instance
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_service_name]
target_label: kubernetes_name
- job_name: 'kubernetes-ingresses'
metrics_path: /probe
params:
module: [http_2xx]
kubernetes_sd_configs:
- role: ingress
relabel_configs:
- source_labels: [__meta_kubernetes_ingress_annotation_prometheus_io_probe]
action: keep
regex: true
- source_labels: [__meta_kubernetes_ingress_scheme,__address__,__meta_kubernetes_ingress_path]
regex: (.+);(.+);(.+)
replacement: ${1}://${2}${3}
target_label: __param_target
- target_label: __address__
replacement: blackbox-exporter.example.com:9115
- source_labels: [__param_target]
target_label: instance
- action: labelmap
regex: __meta_kubernetes_ingress_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_ingress_name]
target_label: kubernetes_name
- job_name: 'kubernetes-pods'
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
action: replace
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
target_label: __address__
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_pod_name]
action: replace
target_label: kubernetes_pod_name
# 生成配置文件
kubectl create -f config.yaml
创建 prometheus rules 配置文件
$ cat rules-down.yml
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus-rules
namespace: ns-monitor
labels:
app: prometheus
data:
nodedown.rule.yml: |
groups:
- name: test-rule
rules:
- alert: NodeDown
expr: up == 0
for: 1m
labels:
team: node
annotations:
summary: "{{$labels.instance}}: down detected"
description: "{{$labels.instance}}: is down 1m (current value is: {{ $value }}"
#生成配置文件
kubectl create -f rules-down.yml
创建 prometheus pod
$ cat deployment.yaml
apiVersion: apps/v1beta2
kind: Deployment
metadata:
name: prometheus
namespace: ns-monitor
labels:
app: prometheus
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app: prometheus
template:
metadata:
labels:
app: prometheus
spec:
serviceAccountName: prometheus
securityContext:
runAsUser: 0
containers:
- name: prometheus
image: prom/prometheus:latest
volumeMounts:
- mountPath: /prometheus
name: prometheus-data-volume
- mountPath: /etc/prometheus/prometheus.yml
name: prometheus-conf-volume
subPath: prometheus.yml
- mountPath: /etc/prometheus/rules
name: prometheus-rules-volume
ports:
- containerPort: 9090
protocol: TCP
volumes:
- name: prometheus-data-volume
persistentVolumeClaim:
claimName: prometheus-data-pvc
- name: prometheus-conf-volume
configMap:
name: prometheus-conf
- name: prometheus-rules-volume
configMap:
name: prometheus-rules
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
# 生成配置文件
kubectl create -f deployment.yaml
创建 prometheus service
$ cat service.yaml
apiVersion: v1
kind: Service
metadata:
annotations:
prometheus.io/scrape: 'true'
labels:
app: prometheus
name: prometheus-service
namespace: ns-monitor
spec:
ports:
- port: 9090
targetPort: 9090
selector:
app: prometheus
type: NodePort
# 生成配置文件
kubectl create -f service.yaml
查看prometheus
四、部署 grafana (在master 01操作) cd /root/control/grafana
创建pv (前提是在各node 节点创建数据目录/nfs/grafana/data )
$ cat pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: "grafana-data-pv"
labels:
name: grafana-data-pv
release: stable
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Recycle
hostPath:
path: /nfs/grafana/data
#生成配置文件
kubectl create -f pv.yaml
创建pvc
$ cat pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: grafana-data-pvc
namespace: ns-monitor
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Gi
selector:
matchLabels:
name: grafana-data-pv
release: stable
# 生成配置文件
kubectl create -f pvc.yaml
创建 grafana pod
$ cat deployment.yaml
apiVersion: apps/v1beta2
kind: Deployment
metadata:
name: grafana
namespace: ns-monitor
labels:
app: grafana
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app: grafana
template:
metadata:
labels:
app: grafana
spec:
securityContext:
runAsUser: 0
containers:
- name: grafana
image: grafana/grafana:latest
env:
- name: GF_AUTH_BASIC_ENABLED
value: "true"
- name: GF_AUTH_ANONYMOUS_ENABLED
value: "false"
readinessProbe:
httpGet:
path: /login
port: 3000
volumeMounts:
- mountPath: /var/lib/grafana
name: grafana-data-volume
ports:
- containerPort: 3000
protocol: TCP
volumes:
- name: grafana-data-volume
persistentVolumeClaim:
claimName: grafana-data-pvc
#生成配置文件
kubectl create -f deployment.yaml
创建 grafana pod service
$ cat service.yaml
apiVersion: v1
kind: Service
metadata:
labels:
app: grafana
name: grafana-service
namespace: ns-monitor
spec:
ports:
- port: 3000
targetPort: 3000
selector:
app: grafana
type: NodePort
#生成配置文件
kubectl create -f service.yaml
五、部署 alertmanager cd /root/control/alertmanager
创建 alertmanager 主配置文件 configmap
$ cat config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: alert-config
namespace: ns-monitor
data:
config.yml: |-
global:
smtp_smarthost: 'smtp.exmail.qq.com:465'
smtp_from: '[email protected]'
smtp_auth_username: '[email protected]'
smtp_auth_password: 'yNE8wZDfYsadsadsad13ctr65Gra'
smtp_require_tls: false
route:
group_by: ['alertname', 'cluster']
group_wait: 30s
group_interval: 5m
repeat_interval: 5m
receiver: default
routes:
- receiver: email
group_wait: 10s
match:
team: node
receivers:
- name: 'default'
email_configs:
- to: '[email protected]'
send_resolved: true
- name: 'email'
email_configs:
- to: '[email protected]'
send_resolved: true
# 生成配置文件
kubectl create -f config.yaml
创建 alertmanager pod
$ cat deploymen.yaml
apiVersion: apps/v1beta2
kind: Deployment
metadata:
name: alertmanager
namespace: ns-monitor
labels:
app: alertmanager
spec:
replicas: 1
selector:
matchLabels:
app: alertmanager
template:
metadata:
labels:
app: alertmanager
spec:
containers:
- name: alertmanager
image: prom/alertmanager:v0.15.3
imagePullPolicy: IfNotPresent
args:
- "--config.file=/etc/alertmanager/config.yml"
- "--storage.path=/alertmanager/data"
ports:
- containerPort: 9093
name: http
volumeMounts:
- mountPath: "/etc/alertmanager"
name: alertcfg
resources:
requests:
cpu: 100m
memory: 256Mi
volumes:
- name: alertcfg
configMap:
name: alert-config
# 生成配置文件
kubectl create -f deploymen.yaml
创建 alertmanager service
$ cat service.yaml
apiVersion: v1
kind: Service
metadata:
labels:
app: alertmanager
name: alertmanager-service
namespace: ns-monitor
spec:
ports:
- port: 9093
targetPort: 9093
selector:
app: alertmanager
type: NodePort
# 生成配置文件
kubectl create -f service.yaml
