kubernetes event 收集

kubernetes event 收集

背景

刚开始准备使用 kubernetes 的官方 python 库，但是这个 python 官方库一直落后于 kubernetes 的正式版本好几个版本,而且用这个库的时候监听 event 老是报错，所以决定使用 kubernetes 官方的 client-go 这个库。

代码介绍

我是参考 (kube-eventer)[https://github.com/AliyunContainerService/kube-eventer.git] 这个代码写的，没用这个是因为我想把我们几个集群的 event 都收集到一个 elasticsearch 的 index，然后通过集群名区分。

下面是改写好的代码，本文用的 client-go 的 git hash ea0a6e11838c4413b5d51574a50da6312d572658，集群名是通过 KUBECONFIG 这个环境变量传给程序的，比如 /ops/kubernetes/bj 集群名就是 bj。

package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/elastic/go-elasticsearch/v8"
	"github.com/elastic/go-elasticsearch/v8/esapi"
	"io/ioutil"
	kubeapi "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kubewatch "k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/kubernetes"
	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/util/homedir"
	"k8s.io/klog"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

const IndexName = "k8s-event"

type EsEvent struct {
	Cluster   string `json:"cluster"`
	Time      string `json:"time"`
	Namespace string `json:"namespace"`
	Kind      string `json:"kind"`
	Name      string `json:"name"`
	Type      string `json:"type"`
	Reason    string `json:"reason"`
	Message   string `json:"message"`
}

func NewEsEvent(event kubeapi.Event) (e EsEvent) {
	e.Cluster = GetClusterName()
	e.Namespace = event.InvolvedObject.Namespace
	e.Kind = event.InvolvedObject.Kind
	e.Name = event.InvolvedObject.Name
	e.Time = time.Now().Format(time.RFC3339)
	e.Type = event.Type
	e.Message = event.Message
	return
}

func GetClusterName() string {
	KUBECONFIG := os.Getenv("KUBECONFIG")
	if KUBECONFIG == "" {
		return "test"
	}
	return strings.Split(KUBECONFIG, "/")[len(strings.Split(KUBECONFIG, "/"))-1]
}

func (e *EsEvent) Req(es *elasticsearch.Client) {
	data, err := json.Marshal(e)
	if err != nil {
		klog.Fatal(err)
	}
	fmt.Printf("%s\n\n", data)
	req := esapi.IndexRequest{
		Index: IndexName,
		Body:  strings.NewReader(string(data)),
	}

	res, err := req.Do(context.Background(), es)
	CheckErr(err)
	defer res.Body.Close()

	if res.IsError() {
		body, _ := ioutil.ReadAll(res.Body)
		klog.Fatalln(string(body), res.Status())
	}
}

func CheckErr(err error) {
	if err != nil {
		klog.Fatalln(err)
	}
}

func main() {
	eventClient := GetKubeClient()
	es := GetEsClient()
	for {
		events, err := eventClient.List(metav1.ListOptions{})
		if err != nil {
			klog.Errorf("Failed to load events: %v", err)
			time.Sleep(time.Second)
			continue
		}
		// Do not write old events.

		resourceVersion := events.ResourceVersion

		watcher, err := eventClient.Watch(
			metav1.ListOptions{
				Watch:           true,
				ResourceVersion: resourceVersion})
		if err != nil {
			klog.Errorf("Failed to start watch for new events: %v", err)
			time.Sleep(time.Second)
			continue
		}

		watchChannel := watcher.ResultChan()
		// Inner loop, for update processing.
	inner_loop:
		for {
			select {
			case watchUpdate, ok := <-watchChannel:
				if !ok {
					klog.Errorf("Event watch channel closed")
					break inner_loop
				}

				if watchUpdate.Type == kubewatch.Error {
					if status, ok := watchUpdate.Object.(*metav1.Status); ok {
						klog.Errorf("Error during watch: %#v", status)
						break inner_loop
					}
					klog.Errorf("Received unexpected error: %#v", watchUpdate.Object)
					break inner_loop
				}

				if event, ok := watchUpdate.Object.(*kubeapi.Event); ok {
					switch watchUpdate.Type {
					case kubewatch.Added, kubewatch.Modified:
						data, err := json.Marshal(event)
						if err != nil {
							klog.Fatal(err)
						}
						fmt.Printf("%s\n\n", data)
						esevent := NewEsEvent(*event)
						esevent.Req(es)
					case kubewatch.Deleted:
						// Deleted events are silently ignored.
					default:
						klog.Warningf("Unknown watchUpdate.Type: %#v", watchUpdate.Type)
					}
				} else {
					klog.Errorf("Wrong object received: %v", watchUpdate)
				}

				//case <-this.stopChannel:
				//	klog.Infof("Event watching stopped")
				//	return
			}
		}
	}
}

func GetEsClient() (*elasticsearch.Client) {
	EsUrl := "http://192.168.56.11:9200"
	if runtime.GOOS == "darwin" {
		EsUrl = "http://es.wis.com"
	}
	cfg := elasticsearch.Config{
		Addresses: []string{
			EsUrl,
		},
	}
	es, err := elasticsearch.NewClient(cfg)
	if err != nil {
		klog.Fatalln(err)
	}
	return es
}

func GetKubeClient() (v1.EventInterface ) {
	var kubeconfig *string
	if home := homedir.HomeDir(); home != "" {
		kubeconfig = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file")
	} else {
		kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file")
	}
	flag.Parse()
	if KUBECONFIG := os.Getenv("KUBECONFIG"); KUBECONFIG != "" {
		*kubeconfig = KUBECONFIG
	}

	config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
	if err != nil {
		panic(err)
	}
	clientset, err := kubernetes.NewForConfig(config)
	CheckErr(err)
	return clientset.CoreV1().Events(kubeapi.NamespaceAll)
}

总结

最后写完发现，之前用 python 库的时候应该是没做错误处理，所以收集 event 有问题，不过这个过程中自己也学习到了好多 go 的编程知识。