Antecedent Review
We previously completed kafka consumption logic implementations, and put elasticsearch news, then check out our logs kibana visualization tools.
This section target
Previously just finished kafka news consumption, and put elastic, this will improve the project so that it supports hot update is that when config.yaml in monitoring the change log, or etcd data has changed dynamically start monitor new log coroutine close cancel the log monitoring coroutine.
New Co Chengzi Qi movable control variable
Add the following code kafkaconsumer.go
var topicMap map[string]map[int32]*TopicData
var topicSet map[string]bool
var etcd_topicSet map[string]bool
var etcd_topicMap map[string]map[int32]*TopicData
var topicChan chan *TopicPart
var etcd_topicChan chan *TopicPart
var consumer_list []sarama.Consumer
var etcdcli *clientv3.Client
func init() {
topicMap = make(map[string]map[int32]*TopicData)
etcd_topicMap = make(map[string]map[int32]*TopicData)
topicSet = make(map[string]bool)
etcd_topicSet = make(map[string]bool)
topicChan = make(chan *TopicPart, 20)
etcd_topicChan = make(chan *TopicPart, 20)
consumer_list = make([]sarama.Consumer, 0, 20)
}
topicMap topic and for logging parameters stored config.yaml coroutine directly recorded, etcd_topicMap used to record and topic parameters etcd coroutine recorded.
topicSet for logging topic config.yaml directly recorded, etcd_topicSet topic etcd for recording recorded.
When the monitoring topicChan log write abnormal elastic coroutine crash, to return the topic information Chan, we then find the corresponding topic restarted with the coroutine topicMap.
The topic etcd_topicChan coroutine stored etcd, the monitoring etlastic process, if the coroutine crashes, etcd_topicMap find the corresponding topic restarted with the coroutine.
consumer_list saved kafka list of consumers.
etcd etcdcli the client for handling etcd reader.
The generated topic set configured log config
func ConstructTopicSet() map[string]bool {
topicSetTmp := make(map[string]bool)
configtopics, _ := logconfig.ReadConfig(logconfig.InitVipper(), "collectlogs")
if configtopics == nil {
goto CONFTOPIC
}
for _, configtopic := range configtopics.([]interface{}) {
confmap := configtopic.(map[interface{}]interface{})
for key, val := range confmap {
if key.(string) == "logtopic" {
topicSetTmp[val.(string)] = true
}
}
}
CONFTOPIC:
return topicSetTmp
}
etcdconsumer.go GetTopicSet read by the topic from etcd life and set
acquisition val according etcd config configured key, and then generates a set acquired topic
func GetTopicSet(cli *clientv3.Client) (interface{}, error) {
etcdKeys, etcdres := logconfig.ReadConfig(logconfig.InitVipper(), "etcdkeys")
if !etcdres {
fmt.Println("read config etcdkeys failed")
return nil, errors.New("read config etcdkeys failed")
}
fmt.Println(reflect.TypeOf(etcdKeys))
topicSet := make(map[string]bool)
for _, keyval := range etcdKeys.([]interface{}) {
ctxtime, cancel := context.WithTimeout(context.Background(), time.Second)
resp, err := cli.Get(ctxtime, keyval.(string))
cancel()
if err != nil {
fmt.Println("get failed, err:", err)
continue
}
for _, ev := range resp.Kvs {
fmt.Printf("%s : %s ...\n", ev.Key, ev.Value)
etcdLogConf := make([]*etcdlogconf.EtcdLogConf, 0, 20)
unmarsherr := json.Unmarshal(ev.Value, &etcdLogConf)
if unmarsherr != nil {
fmt.Println("unmarshal error !, error is ", unmarsherr)
continue
}
for _, etcdval := range etcdLogConf {
topicSet[etcdval.Topic] = true
}
}
}
return topicSet, nil
}
Set into the map, and then starts the configuration coroutine
func ConvertSet2Map(consumer sarama.Consumer, topicSet map[string]bool,
topicMaps map[string]map[int32]*TopicData, topic_chan chan *TopicPart) {
for key, _ := range topicSet {
partitionList, err := consumer.Partitions(key)
if err != nil {
fmt.Println("get consumer partitions failed")
fmt.Println("error is ", err.Error())
continue
}
for partition := range partitionList {
pc, err := consumer.ConsumePartition(key, int32(partition), sarama.OffsetNewest)
if err != nil {
fmt.Println("consume partition error is ", err.Error())
continue
}
// defer pc.AsyncClose()
topicData := new(TopicData)
topicData.Ctx, topicData.Cancel = context.WithCancel(context.Background())
topicData.KafConsumer = pc
topicData.TPartition = new(TopicPart)
topicData.TPartition.Partition = int32(partition)
topicData.TPartition.Topic = key
_, okm := topicMaps[key]
if !okm {
topicMaps[key] = make(map[int32]*TopicData)
}
topicMaps[key][int32(partition)] = topicData
go PutIntoES(topicData, topic_chan)
}
}
}
Kafka read from the message, and call the above function starts monitoring es coroutine
Kafka read information from, and then generates a map set according to the configuration and start monitoring es coroutine
func ConsumeTopic(consumer sarama.Consumer) {
ConvertSet2Map(consumer, topicSet, topicMap, topicChan)
ConvertSet2Map(consumer, etcd_topicSet, etcd_topicMap, etcd_topicChan)
//监听配置文件
ctx, cancel := context.WithCancel(context.Background())
pathChan := make(chan interface{})
etcdChan := make(chan interface{})
go logconfig.WatchConfig(ctx, logconfig.InitVipper(), pathChan, etcdChan)
defer func(cancel context.CancelFunc) {
consumer_once.Do(func() {
if err := recover(); err != nil {
fmt.Println("consumer main goroutine panic, ", err)
}
cancel()
})
}(cancel)
for {
select {
// coroutine crash detection monitoring path, restart
Case topicpart: = <-topicChan:
fmt.Printf ( "the receive goroutines the Exited, Topic% S IS, IS Partition% D \ n-",
topicpart.Topic, topicpart.Partition)
/ / restart reading consumer data coroutine
Val, OK: = topicMap [topicpart.Topic]
IF OK {!
Continue
}
TP, OK: Val = [topicpart.Partition]
! IF OK {
Continue
}
tp.Ctx, TP. = context.WithCancel the Cancel (context.Background ())
Go PutIntoES (TP, topicChan)
// parse the configuration etcd detection, monitoring path coroutine crash, reboot
Case topicpart: = <-etcd_topicChan:
fmt.Printf ( "the receive goroutines the Exited, Topic% S IS, IS Partition% D \ n-",
topicpart.Topic, topicpart.Partition)
// read the data consumer restart coroutine
val, ok: = etcd_topicMap [topicpart.Topic ]
IF OK {!
Continue
}
TP, OK: Val = [topicpart.Partition]
! IF OK {
Continue
}
tp.Ctx, tp.Cancel = context.WithCancel (context.Background ( ))
Go PutIntoES (TP, etcd_topicChan)
// returns the monitor detector configured update VIPPER
Case pathchange, OK: = <-pathChan:
! {IF OK
fmt.Println ( "VIPPER goroutines the Exited Watch")
GOTO loopend
}
// FMT. println (pathchange)
topicSetTemp: = the make (Map [String] BOOL)
for _, chval:. = Range pathchange ([] interface {}) {
for logkey, logval:. = Range chval (Map [interface {}] interface { }) {
if logkey.(string) == "logtopic" {
topicSetTemp[logval.(string)] = true
}
}
}
UpdateTopicLogRoutine(topicSetTemp)
//fmt.Println(topicSetTemp)
case etcdchange, ok := <-etcdChan:
if !ok {
fmt.Println("vipper watch goroutine extied")
goto LOOPEND
}
fmt.Println(etcdchange)
topicsetTemp, err := etcdconsumer.GetTopicSet(etcdcli)
if err != nil {
continue
}
UpdateEtcdTopicLogRoutine(topicsetTemp.(map[string]bool))
}
}
LOOPEND:
fmt.Printf("for exited ")
}
go logconfig.WatchConfig start coroutine, call vipper monitor configuration, topic when equipped with updated consumer coroutine handling updates.
At the same time when child support coroutine abnormal collapse, consumer coroutine restart the coroutine.
By kibana log information
kibana in ManageMent management, and new index, elastic is the index we set topic, so we have a few new index,
etcd_log, golang_log, logdir2.
Then kibana can be seen in several of the index log information
Source download
https://github.com/secondtonone1/golang-/tree/master/logcatchsys
thank my public concern No.