dsMin/dsDataex/MyTask/Kafka2ES/Kafka2ESService/Kafka2ESService.go

package Kafka2ESService

import (
	"dsDataex/MyTask/Kafka2ES/Kafka2ESDAO"
	"dsDataex/Utils/ConfigUtil"
	"dsDataex/Utils/KafkaUtil"
	"fmt"
	"github.com/go-co-op/gocron"
	"reflect"
	"sync"
	"time"
)

var ChanTopic chan []string
var LstTopic []string

var loc sync.Mutex

func ServiceStart() {

	cronMan := gocron.NewScheduler(time.UTC)

	cronMan.Every(5).Seconds().StartImmediately().Do(DBWatch)

	cronMan.Every(10).Seconds().Do(LogProcess)

	cronMan.StartAsync()

	defer func() {
		if err := recover(); err != nil {
			fmt.Println("Kafka2ESService Panic Recover :", err)

			cronMan.Stop()
		}
	}()

	var procNo = int(ConfigUtil.KafkaProcNo)

	KafkaUtil.ChanTopicProc = make(map[string][]chan bool)
	KafkaUtil.StateTopicProc = make(map[string][]bool)
	KafkaUtil.CountTopicProc = make(map[string][]int)

	ChanTopic = make(chan []string, 100)

	for topics := range ChanTopic {
		for no := 0; no < len(topics); no++ {

			topic := topics[no]

			if Contains(LstTopic, topic) == -1 {

				LstTopic = append(LstTopic, topic)

				//change by zhangjun 2020-08-02
				cronMan.Every(60).Seconds().SetTag([]string{"kafka_" + topic}).StartImmediately().Do(KafkaProcess, topic, procNo)
				//go KafkaProcess(topic, procNo)

			}
		}

		if len(LstTopic) > len(topics) {
			for no := 0; no < len(LstTopic); no++ {
				if Contains(topics, LstTopic[no]) == -1 {

					//删除任务
					cronMan.RemoveJobByTag("kafka_" + LstTopic[no])

					//关闭子线程
					for no2 := 0; no2 < len(KafkaUtil.ChanTopicProc[LstTopic[no]]); no2++ {
						KafkaUtil.ChanTopicProc[LstTopic[no]][no] <- true
					}

					delete(KafkaUtil.ChanTopicProc, LstTopic[no])
					delete(KafkaUtil.StateTopicProc, LstTopic[no])
				}
			}

			LstTopic = []string{}

			LstTopic = append(LstTopic, topics...)
		}
	}
}

func DBWatch() {

	var _, topics = Kafka2ESDAO.GetTopics()

	ChanTopic <- topics
}

func KafkaProcess(topic string, procNo int) {

	_, f := KafkaUtil.ChanTopicProc[topic]

	if f == false {

		var lst []chan bool
		var lst2 []bool
		var lst3 []int

		for no := 0; no < procNo; no++ {

			var chanProc = make(chan bool, 100)

			lst = append(lst, chanProc)
			lst2 = append(lst2, true)
			lst3 = append(lst3, 0)
		}
		//add by zhangjun 2020-07-30
		loc.Lock()

		KafkaUtil.ChanTopicProc[topic] = lst
		KafkaUtil.StateTopicProc[topic] = lst2
		KafkaUtil.CountTopicProc[topic] = lst3

		loc.Unlock()

		for no := 0; no < procNo; no++ {

			fmt.Printf("Dataex Consume Kafka ,Topic:%s,ConsumerGroup:%s,GoroutineNO:%d.\n", topic, "group_"+topic, no)

			//开启子线程
			go KafkaUtil.Consume(topic, "group_"+topic, no)

			time.Sleep(time.Second * 10)
		}
	} else { //TODO：处理异常子线程！！！

		for no := 0; no < len(KafkaUtil.StateTopicProc[topic]); no++ {
			if KafkaUtil.StateTopicProc[topic][no] == false {
				fmt.Printf("Dataex Consume Kafka ,Topic:%s,ConsumerGroup:%s,GoroutineNO:%d.\n", topic, "group_"+topic, no)

				KafkaUtil.StateTopicProc[topic][no] = true
				KafkaUtil.CountTopicProc[topic][no] = 0

				go KafkaUtil.Consume(topic, "group_"+topic, no)

				time.Sleep(time.Second * 10)
			}
		}
	}
}

func LogProcess()  {

	for k,v:= range KafkaUtil.CountTopicProc{

		if len(KafkaUtil.CountTopicProc[k])>0{
			for no:=0;no< len(v);no++{
				fmt.Println("Kafka Consume Topic:"+k,"【",time.Now(),"】 Process Total:",v[no])
			}
		}
	}
}

func Contains(array interface{}, val interface{}) (index int) {
	index = -1
	switch reflect.TypeOf(array).Kind() {
	case reflect.Slice:
		{
			s := reflect.ValueOf(array)
			for i := 0; i < s.Len(); i++ {
				if reflect.DeepEqual(val, s.Index(i).Interface()) {
					index = i
					return index
				}
			}
		}
	}
	return index
}