申明:
- 我想把我收集的日志中不需要的数据,在kibana上不展示。
- 我想索引名字按照日志中某个字段定义名称
- 我想把日志几行合并为一行在kibana上展示
- 我想把几个索引的某个关键字匹配的内容写入一个新的索引
1. 安装基础环境
- 安装es:
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: es-cluster
namespace: kube-logging
spec:
serviceName: elasticsearch
replicas: 1
selector:
matchLabels:
app: elasticsearch
template:
metadata:
labels:
app: elasticsearch
spec:
containers:
- name: elasticsearch
image: elasticsearch:7.12.1
imagePullPolicy: IfNotPresent
resources:
limits:
cpu: 1000m
requests:
cpu: 100m
ports:
- containerPort: 9200
name: rest
protocol: TCP
- containerPort: 9300
name: inter-node
protocol: TCP
volumeMounts:
- name: data
mountPath: /usr/share/elasticsearch/data
env:
- name: cluster.name
value: k8s-logs
- name: node.name
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: discovery.seed_hosts
value: "es-cluster-0.elasticsearch"
- name: cluster.initial_master_nodes
value: "es-cluster-0"
- name: ES_JAVA_OPTS
value: "-Xms2g -Xmx2g"
initContainers:
- name: fix-permissions
image: busybox
imagePullPolicy: IfNotPresent
command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
securityContext:
privileged: true
volumeMounts:
- name: data
mountPath: /usr/share/elasticsearch/data
- name: increase-vm-max-map
image: busybox
imagePullPolicy: IfNotPresent
command: ["sysctl", "-w", "vm.max_map_count=262144"]
securityContext:
privileged: true
- name: increase-fd-ulimit
image: busybox
imagePullPolicy: IfNotPresent
command: ["sh", "-c", "ulimit -n 65536"]
securityContext:
privileged: true
volumes:
- name: data
hostPath:
path: /home/es/
- es-svc
apiVersion: v1
kind: Service
metadata:
name: elasticsearch
namespace: kube-logging
labels:
app: elasticsearch
spec:
selector:
app: elasticsearch
clusterIP: None
ports:
- port: 9200
name: rest
- port: 9300
name: inter-node
- filebeat部署
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: filebeat
namespace: kube-logging
labels:
k8s-app: filebeat
spec:
selector:
matchLabels:
k8s-app: filebeat
template:
metadata:
labels:
k8s-app: filebeat
spec:
serviceAccountName: filebeat
terminationGracePeriodSeconds: 30
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: filebeat
image: elastic/filebeat:7.6.2
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
env:
- name: ELASTICSEARCH_HOST
value: elasticsearch
- name: ELASTICSEARCH_PORT
value: "9200"
- name: ELASTICSEARCH_USERNAME
value: elastic
- name: ELASTICSEARCH_PASSWORD
value: changeme
- name: ELASTIC_CLOUD_ID
value:
- name: ELASTIC_CLOUD_AUTH
value:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
securityContext:
runAsUser: 0
resources:
limits:
cpu: 500m
memory: 500Mi
requests:
cpu: 100m
memory: 100Mi
volumeMounts:
- name: config
mountPath: /etc/filebeat.yml
readOnly: true
subPath: filebeat.yml
- name: data
mountPath: /usr/share/filebeat/data
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: varlog
mountPath: /var/log
readOnly: true
volumes:
- name: config
configMap:
defaultMode: 0600
name: filebeat-config
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: varlog
hostPath:
path: /var/log
- name: data
hostPath:
path: /var/lib/filebeat-data
type: DirectoryOrCreate
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: filebeat
subjects:
- kind: ServiceAccount
name: filebeat
namespace: kube-logging
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: filebeat
labels:
k8s-app: filebeat
rules:
- apiGroups: [""]
resources:
- namespaces
- pods
verbs:
- get
- watch
- list
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: kube-logging
labels:
k8s-app: filebeat
- filebeat-config文件
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: kube-logging
labels:
k8s-app: filebeat
data:
filebeat.yml: |-
filebeat.inputs:
- type: container
paths:
- /var/log/containers/*.log
- /var/log/test/*.log
output.logstash:
hosts: ['logstash.kube-logging.svc.cluster.local:5044']
- logstash部署
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: logstash
namespace: kube-logging
spec:
selector:
matchLabels:
app: logstash
template:
metadata:
labels:
app: logstash
spec:
hostname: logstash
containers:
- name: logstash
ports:
- containerPort: 5044
name: logstash
image: logstash:v1
volumeMounts:
- name: logstash-config
mountPath: /usr/share/logstash/pipeline/
command:
- logstash
volumes:
- name: logstash-config
configMap:
name: logstash-config
items:
- key: logstash.conf
path: logstash.conf
---
kind: Service
apiVersion: v1
metadata:
name: logstash
namespace: kube-logging
spec:
clusterIP: None
selector:
app: logstash
ports:
- protocol: TCP
port: 5044
name: logstash
- logtash 配置
---
apiVersion: v1
kind: ConfigMap
metadata:
name: logstash-config
namespace: kube-logging
data:
logstash.conf: |-
input {
beats {
port => "5044"
}
}
filter {
if [agent][id] == "7e857655-7475-4eb2-866e-d2939fe1ba03" {
if [message] =~ /info|test/ {
drop {
}
}
}
multiline {
pattern => "^\s*(\d{
4}|\d{
2})\-(\d{
2}|[a-zA-Z]{
3})\-(\d{
2}|\d{
4})|^(.*)(\d{
4}|\d{
2})\-(\d{
2}|[a-zA-Z]{
3})\-(\d{
2}|\d{
4})\s(\d{
2}):(\d{
1,2})|^{
(.*)}|^\d{
4}|^(.*)(\d{
4}|\d{
2})\-(\d{
2}|[a-zA-Z]{
3})\-(\d{
2}|\d{
4})\s(\d{
2}):(\d{
1,2})"
negate => true
what => "previous"
}
}
output {
elasticsearch {
hosts => "elasticsearch.kube-logging.svc.cluster.local:9200"
index => "my_index-%{
[agent][id]}-%{
+yyyy.MM.dd}"
}
}
- 启动
kubectl apply -f ./
- 多匹配条件
apiVersion: v1
kind: ConfigMap
metadata:
name: logstash-config
namespace: logging
data:
logstash.conf: |-
input {
beats {
port => "5044"
}
}
filter {
if [kubernetes][labels][app_kubernetes_io/name] == "dtk-go-open-api" {
if [message] =~ /---/ {
drop {
}
}
}
if [kubernetes][labels][app_kubernetes_io/name] == "dtk-go-taobao-api" or [kubernetes][labels][app_kubernetes_io/name] == "dtk-go-taobao-api-h" {
if [message] =~ /CheckNewGoodsSign|CheckNewGoodsSign2/ {
mutate {
add_field => {
"new_index" => "CheckNewGoodsSign-index" }
}
}
}
}
output {
if [new_index] == "CheckNewGoodsSign-index" {
elasticsearch {
hosts => "elasticsearch.logging.svc.cluster.local:9200"
index => "checknewgoodssign-%{
+yyyy.MM.dd}"
}
}
else {
elasticsearch {
hosts => "elasticsearch.logging.svc.cluster.local:9200"
index => "%{
[kubernetes][labels][app_kubernetes_io/name]}-%{
+yyyy.MM.dd}"
}
}
}
- 更新
apiVersion: v1
kind: ConfigMap
metadata:
name: logstash-config
namespace: logging
data:
logstash.conf: |-
input {
beats {
port => "5044"
}
}
filter {
if [kubernetes][labels][app_kubernetes_io/name] == "dtk-go-open-api" {
if [message] =~ /---/ {
drop {
}
}
}
if [kubernetes][labels][app_kubernetes_io/name] == "dtk-go-taobao-api-h" {
if [message] =~ /cloud.go:2279|cloud.go:2226/ {
drop {
}
}
}
if [kubernetes][labels][app_kubernetes_io/name] == "dtk-go-taobao-api" {
if [message] =~ /-----auth_id-|-------11111----/{
drop {
}
}
}
if [kubernetes][labels][app_kubernetes_io/name] == "dtk-php-api-android-cms" {
drop {
}
}
if ![kubernetes][labels][app_kubernetes_io/name] {
drop {
}
}
if [kubernetes][labels][app_kubernetes_io/name] == "dtk-go-taobao-api" or [kubernetes][labels][app_kubernetes_io/name] == "dtk-go-taobao-api-h" {
if [message] =~ /CheckNewGoodsSign|CheckNewGoodsSign2/ {
mutate {
add_field => {
"new_index" => "CheckNewGoodsSign-index" }
}
}
}
}
output {
if [new_index] == "CheckNewGoodsSign-index" {
elasticsearch {
hosts => "elasticsearch.logging.svc.cluster.local:9200"
index => "checknewgoodssign-%{
+yyyy.MM.dd}"
}
}
else if ![kubernetes][labels][app_kubernetes_io/name] {
elasticsearch {
hosts => "elasticsearch.logging.svc.cluster.local:9200"
index => "index-unknown-%{
+yyyy.MM.dd}"
}
}
else {
elasticsearch {
hosts => "elasticsearch.logging.svc.cluster.local:9200"
index => "%{
[kubernetes][labels][app_kubernetes_io/name]}-%{
+yyyy.MM.dd}"
}
}
}