目次
5. Elasticsearchはデータ+ストレージ用のnfsを保存します（pvの自動登録の詳細は前回の記事を参照してください）
1.NFSストレージ
参考リンク：k8s構成hadoopクラスタ、ストレージとしてのnfs_Crazy snailのブログ - CSDNブログ
2. イメージを展開してタグを作成する
docker pull docker.elastic.co/elasticsearch/elasticsearch:7.17.2
docker pull docker.elastic.co/kibana/kibana:7.17.2
docker pull docker.elastic.co/logstash/logstash:7.17.2
docker pull docker.elastic.co/beats/filebeat:7.17.2
docker pull redis
#注：提前下载，其他master将镜像copy过去load比较省时3. Filebeat がデータを収集する
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: logging
data:
filebeat.yml: |
filebeat.inputs:
- type: log
paths:
- /var/log/nginx
document_type: k8s-nginx
setup.template.name: "k8s-nginx"
setup.template.pattern: "k8s-nginx-*"
output.elasticsearch:
hosts: ["elasticsearch:9200"]
index: "k8s-nginx-%{+yyyy.MM.dd}"
# output.logstash:
# hosts: ['logging-logstash:5044']
# enabled: true
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: filebeat
namespace: dev
spec:
selector:
matchLabels:
app: filebeat
template:
metadata:
labels:
k8s-app: filebeat
app: filebeat
spec:
terminationGracePeriodSeconds: 30
containers:
- name: filebeat
image: docker.elastic.co/beats/filebeat:7.17.2
imagePullPolicy: IfNotPresent
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
volumeMounts:
- name: config
mountPath: /etc/filebeat.yml
readOnly: true
subPath: filebeat.yml
- name: log
mountPath: /var/log/
volumes:
- name: config
configMap:
defaultMode: 0755
name: filebeat-config
- name: log
hostPath:
path: /var/log/
type: Directory
4、logstash フィルター データ
---
apiVersion: v1
kind: ConfigMap
metadata:
name: logstash-configmap
namespace: dev
labels:
k8s-app: logstash-configmap
data:
logstash.conf: |
input {
beats {
port => "5044"
codec => "json"
}
}
filter{
json{
source => "message"
remove_field => "message"
}
}
output {
elasticsearch {
hosts => "elasticsearch:9200"
index => "nginx-json-log-%{+YYYY.MM.dd}"
}
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: logstash
namespace: dev
spec:
replicas: 1
selector:
matchLabels:
k8s-app: logstash
template:
metadata:
labels:
k8s-app: logstash
spec:
containers:
- name: logstash
image: docker.elastic.co/logstash/logstash:7.17.2
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5044
volumeMounts:
- name: config-volume
mountPath: /usr/share/logstash/pipeline/
volumes:
- name: config-volume
configMap:
name: logstash-configmap
items:
- key: logstash.conf
path: logstash.conf
---
apiVersion: v1
kind: Service
metadata:
name: logstash
namespace: dev
spec:
ports:
- port: 5044
targetPort: 5044
protocol: TCP
selector:
k8s-app: logstash
type: ClusterIP5. Elasticsearchはデータ+ストレージ用のnfsを保存します（pvの自動登録の詳細は前回の記事を参照してください）
apiVersion: v1
kind: Service
metadata:
name: elasticsearch
namespace: dev
labels:
app: elasticsearch
spec:
selector:
k8s-app: elasticsearch
clusterIP: None
ports:
- port: 9200
name: db
- port: 9300
name: inter
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: elasticsearch
namespace: dev
labels:
k8s-app: elasticsearch
spec:
serviceName: elasticsearch
selector:
matchLabels:
k8s-app: elasticsearch
template:
metadata:
labels:
k8s-app: elasticsearch
spec:
containers:
- image: docker.elastic.co/elasticsearch/elasticsearch:7.17.2
name: elasticsearch
resources:
limits:
cpu: 1
memory: 2Gi
requests:
cpu: 0.5
memory: 500Mi
env:
- name: "discovery.type"
value: "single-node"
- name: ES_JAVA_OPTS
value: "-Xms512m -Xmx2g"
ports:
- containerPort: 9200
name: db
protocol: TCP
- name: inter
containerPort: 9300
volumeMounts:
- name: elasticsearch-data
mountPath: /usr/share/elasticsearch/data
volumeClaimTemplates:
- metadata:
name: elasticsearch-data
spec:
storageClassName: "nfs-storage"
accessModes: [ "ReadWriteMany" ]
resources:
requests:
storage: 1Gi
6、kibana表示データ
apiVersion: apps/v1
kind: Deployment
metadata:
name: kibana
namespace: dev
labels:
k8s-app: kibana
spec:
replicas: 1
selector:
matchLabels:
k8s-app: kibana
template:
metadata:
labels:
k8s-app: kibana
spec:
containers:
- name: kibana
image: docker.elastic.co/kibana/kibana:7.17.2
resources:
limits:
cpu: 1
memory: 1G
requests:
cpu: 0.5
memory: 500Mi
env:
- name: ELASTICSEARCH_HOSTS
value: http://elasticsearch:9200
ports:
- containerPort: 5601
protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
name: kibana
namespace: dev
spec:
ports:
- port: 5601
protocol: TCP
targetPort: 5601
nodePort: 30000
type: NodePort
selector:
k8s-app: kibana
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: kibana
namespace: dev
spec:
rules:
- host: "dev。kibana.cn"
http:
paths:
- path: /kibana
pathType: Prefix
backend:
service:
name: kibana
port:
number: 56017、インストールを確認します
[root@master-01 elk]# kubectl get pv,pvc -n dev |grep ela
persistentvolume/pvc-9f405d08-2669-45de-81bb-e37f45ac53b7 1Gi RWX Delete Bound dev/elasticsearch-data-elasticsearch-0 nfs-storage 146m
persistentvolumeclaim/elasticsearch-data-elasticsearch-0 Bound pvc-9f405d08-2669-45de-81bb-e37f45ac53b7 1Gi RWX nfs-storage 146m
[root@master-01 elk]# kubectl get pod -n dev |grep filebeat
filebeat-5255b 1/1 Running 0 151m
filebeat-5fdgr 1/1 Running 0 151m
filebeat-bv94f 1/1 Running 0 151m
filebeat-fcljd 1/1 Running 0 151m
filebeat-jbt87 1/1 Running 0 151m
filebeat-p7jgx 1/1 Running 0 151m
[root@master-01 elk]# kubectl get pod -n dev |grep elasticsearch
elasticsearch-0 1/1 Running 0 147m
[root@master-01 elk]# kubectl get pod -n dev |grep kibana
kibana-5f5cb8b78d-g6fvz 1/1 Running 0 147m
[root@master-01 elk]# kubectl get pod -n dev |grep logstash
logstash-77bf676fb6-6rr9s 1/1 Running 0 150m