|
|
package Kafka2ESService
|
|
|
|
|
|
import (
|
|
|
"dsDataex/MyTask/Kafka2ES/Kafka2ESDAO"
|
|
|
"dsDataex/Utils/ConfigUtil"
|
|
|
"dsDataex/Utils/KafkaUtil"
|
|
|
"fmt"
|
|
|
"github.com/go-co-op/gocron"
|
|
|
"reflect"
|
|
|
"time"
|
|
|
)
|
|
|
|
|
|
var ChanTopic chan []string
|
|
|
var LstTopic []string
|
|
|
|
|
|
func ServiceStart(){
|
|
|
|
|
|
cronMan := gocron.NewScheduler(time.UTC)
|
|
|
|
|
|
cronMan.Every(5).Seconds().StartImmediately().Do(DBWatch)
|
|
|
|
|
|
cronMan.StartAsync()
|
|
|
|
|
|
var procNo=int(ConfigUtil.KafkaProcNo)
|
|
|
|
|
|
KafkaUtil.ChanTopicProc=make(map[string][]chan bool)
|
|
|
KafkaUtil.StateTopicProc=make(map[string][]bool)
|
|
|
|
|
|
ChanTopic=make(chan []string,100)
|
|
|
|
|
|
for topics :=range ChanTopic{
|
|
|
for no :=0;no< len(topics);no++ {
|
|
|
|
|
|
topic:=topics[no]
|
|
|
|
|
|
if Contains(LstTopic, topic) == -1 {
|
|
|
|
|
|
LstTopic = append(LstTopic, topic)
|
|
|
|
|
|
cronMan.Every(60).Seconds().SetTag([]string{"kafka_" + topic}).StartImmediately().Do(KafkaProcess, topic, procNo)
|
|
|
|
|
|
//cronMan.StartAsync()
|
|
|
}
|
|
|
}
|
|
|
|
|
|
if len(LstTopic)> len(topics){
|
|
|
for no:=0;no< len(LstTopic);no++{
|
|
|
if Contains(topics,LstTopic[no])==-1{
|
|
|
|
|
|
//删除任务
|
|
|
cronMan.RemoveJobByTag("kafka_"+LstTopic[no])
|
|
|
|
|
|
//关闭子线程
|
|
|
for no2:=0;no2< len(KafkaUtil.ChanTopicProc[LstTopic[no]]);no2++{
|
|
|
KafkaUtil.ChanTopicProc[LstTopic[no]][no] <- true
|
|
|
}
|
|
|
|
|
|
delete(KafkaUtil.ChanTopicProc,LstTopic[no])
|
|
|
delete(KafkaUtil.StateTopicProc,LstTopic[no])
|
|
|
}
|
|
|
}
|
|
|
|
|
|
LstTopic=[]string{}
|
|
|
|
|
|
LstTopic=append(LstTopic,topics...)
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
func DBWatch() {
|
|
|
|
|
|
var _,topics = Kafka2ESDAO.GetTopics()
|
|
|
|
|
|
ChanTopic <- topics
|
|
|
}
|
|
|
|
|
|
func KafkaProcess(topic string,procNo int) {
|
|
|
|
|
|
_,f:=KafkaUtil.ChanTopicProc[topic]
|
|
|
|
|
|
if f==false{
|
|
|
|
|
|
var lst []chan bool
|
|
|
var lst2 []bool
|
|
|
|
|
|
for no:=0;no<procNo;no++{
|
|
|
|
|
|
var chanProc =make(chan bool ,100)
|
|
|
|
|
|
lst=append(lst,chanProc)
|
|
|
lst2=append(lst2,true)
|
|
|
}
|
|
|
|
|
|
KafkaUtil.ChanTopicProc[topic]=lst
|
|
|
KafkaUtil.StateTopicProc[topic]=lst2
|
|
|
|
|
|
for no:=0;no<procNo;no++{
|
|
|
|
|
|
fmt.Printf("Dataex Consume Kafka ,Topic:%s,ConsumerGroup:%s,GoroutineNO:%d.\n",topic,"group_"+topic,no)
|
|
|
|
|
|
//开启子线程
|
|
|
go KafkaUtil.Consume(topic,"group_"+topic,no)
|
|
|
}
|
|
|
}else {//TODO：处理异常子线程！！！
|
|
|
|
|
|
for no:=0;no< len(KafkaUtil.StateTopicProc[topic]);no++{
|
|
|
if KafkaUtil.StateTopicProc[topic][no]==false{
|
|
|
fmt.Println("Dataex Consume Kafka ,Topics :",topic,"group_"+topic,no)
|
|
|
|
|
|
KafkaUtil.StateTopicProc[topic][no]=true
|
|
|
|
|
|
go KafkaUtil.Consume(topic,"group_"+topic,no)
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
func Contains(array interface{}, val interface{}) (index int) {
|
|
|
index = -1
|
|
|
switch reflect.TypeOf(array).Kind() {
|
|
|
case reflect.Slice: {
|
|
|
s := reflect.ValueOf(array)
|
|
|
for i := 0; i < s.Len(); i++ {
|
|
|
if reflect.DeepEqual(val, s.Index(i).Interface()) {
|
|
|
index = i
|
|
|
return index
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
return index
|
|
|
} |
