bigdata
/
dsMin
6
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

kafka 2 es

Browse Source
master
zhangjun 5 years ago
parent 451e6b890c
commit c30298cadf
14 changed files with 87 additions and 52 deletions
Show Stats Download Patch File Download Diff File

4
dsDataex/MyService/DataEX/DataexService/DataexService.go
Unescape View File

@ -11,6 +11,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"strconv" "strconv"
"strings"
"time" "time"
) )
@ -516,6 +517,9 @@ func DataexPage(datasourceCode string,consumeType int,consumeOrgID string,orgID
if flag{ if flag{
var lstEsdata []DataEX.ESData var lstEsdata []DataEX.ESData
//add by zhangjun 2020-08-02
beginTime =strings.Replace(beginTime,"-","/",-1)
if orgID=="-1"{ if orgID=="-1"{
flag,result,lstEsdata= ES7Util.IndexDocPage(datasourceCode,nil,pageNO,beginTime) flag,result,lstEsdata= ES7Util.IndexDocPage(datasourceCode,nil,pageNO,beginTime)

2
dsDataex/MyService/MySwagger/Auth.go
Unescape View File

@ -4,5 +4,5 @@ package MySwagger
type Auth struct { type Auth struct {
SystemID string `json:"system_id" example:"SYS-200201" ` SystemID string `json:"system_id" example:"SYS-200201" `
SystemToken string `json:"system_token" example:"a6ce11eab94df48a6ce11eab94df48e38f73cf7e38f73cf7"` SystemToken string `json:"system_token" example:"a6ce11eab94df48a6ce11eab94df48e38f73cf7e38f73cf7"`
AuthTime string `json:"auth_time" example:"2020-01-02 03:04:05"` AuthTime string `json:"auth_time" example:"2020/01/02 03:04:05"`
} }

4
dsDataex/MyService/MySwagger/DataGet.go
Unescape View File

@ -3,7 +3,7 @@ package MySwagger
type DataGet struct { type DataGet struct {
SystemID string `json:"system_id" example:"SYS-200201"` SystemID string `json:"system_id" example:"SYS-200201"`
AuthToken string `json:"auth_token" example:"DATAEX-TOKEN-a6ce-11ea-b94df48e38f73cf7"` AuthToken string `json:"auth_token" example:"DATAEX-TOKEN-a6ce-11ea-b94df48e38f73cf7"`
DataSource string `json:"data_source" example:"ORG"` DataSource string `json:"data_source" example:"org_school"`
OrgID string `json:"org_id" example:"200201"` OrgID string `json:"org_id" example:"-1"`
DataID string `json:"data_id" example:"202008080008"` DataID string `json:"data_id" example:"202008080008"`
} }

6
dsDataex/MyService/MySwagger/DataPage.go
Unescape View File

@ -3,8 +3,8 @@ package MySwagger
type DataPage struct { type DataPage struct {
SystemID string `json:"system_id" example:"SYS-200201"` SystemID string `json:"system_id" example:"SYS-200201"`
AuthToken string `json:"auth_token" example:"DATAEX-TOKEN-a6ce-11ea-b94df48e38f73cf7"` AuthToken string `json:"auth_token" example:"DATAEX-TOKEN-a6ce-11ea-b94df48e38f73cf7"`
DataSource string `json:"data_source" example:"ORG"` DataSource string `json:"data_source" example:"org_school"`
OrgID string `json:"org_id" example:"200201"` OrgID string `json:"org_id" example:"-1"`
QueryTime string `json:"query_time" example:"2020-01-01 02:03:04"` QueryTime string `json:"query_time" example:"2020/01/01 02:03:04"`
QueryPage int `json:"query_page" example:"0"` QueryPage int `json:"query_page" example:"0"`
} }

6
dsDataex/MyService/MySwagger/DataQuery.go
Unescape View File

@ -3,8 +3,8 @@ package MySwagger
type DataQuery struct { type DataQuery struct {
SystemID string `json:"system_id" example:"SYS-200201"` SystemID string `json:"system_id" example:"SYS-200201"`
AuthToken string `json:"auth_token" example:"DATAEX-TOKEN-a6ce-11ea-b94df48e38f73cf7"` AuthToken string `json:"auth_token" example:"DATAEX-TOKEN-a6ce-11ea-b94df48e38f73cf7"`
DataSource string `json:"data_source" example:"ORG"` DataSource string `json:"data_source" example:"org_school"`
OrgID string `json:"org_id" example:"200201"` OrgID string `json:"org_id" example:"-1"`
QueryConditions []string `json:"query_conditions" example:"subject_code=22"` QueryConditions []string `json:"query_conditions" example:"subject_code=22"`
QueryPage int `json:"query_page" example:"1"` QueryPage int `json:"query_page" example:"0"`
} }

2
dsDataex/MyService/MySwagger/Event.go
Unescape View File

@ -4,7 +4,7 @@ type Event struct {
EventType string `json:"event_type" example:"1001"` EventType string `json:"event_type" example:"1001"`
EventName string `json:"event_name" example:"在线提交作业"` EventName string `json:"event_name" example:"在线提交作业"`
EventTime string `json:"event_time" example:"2020-08-08 18:19:20"` EventTime string `json:"event_time" example:"2020/08/08 18:19:20"`
EventURI string `json:"event_uri" example:"https://edusoa.com/cloud/study/do_homework"` EventURI string `json:"event_uri" example:"https://edusoa.com/cloud/study/do_homework"`
EventSeqNO string `json:"event_seqno" example:"事件顺序号：1/2/3"` EventSeqNO string `json:"event_seqno" example:"事件顺序号：1/2/3"`
EventUserID string `json:"event_userid" example:"用户ID"` EventUserID string `json:"event_userid" example:"用户ID"`

26
dsDataex/MyTask/Kafka2ES/Kafka2ESService/Kafka2ESService.go
Unescape View File

@ -22,6 +22,8 @@ func ServiceStart() {
cronMan.Every(5).Seconds().StartImmediately().Do(DBWatch) cronMan.Every(5).Seconds().StartImmediately().Do(DBWatch)
cronMan.Every(10).Seconds().Do(LogProcess)
cronMan.StartAsync() cronMan.StartAsync()
defer func() { defer func() {
@ -36,6 +38,7 @@ func ServiceStart() {
KafkaUtil.ChanTopicProc = make(map[string][]chan bool) KafkaUtil.ChanTopicProc = make(map[string][]chan bool)
KafkaUtil.StateTopicProc = make(map[string][]bool) KafkaUtil.StateTopicProc = make(map[string][]bool)
KafkaUtil.CountTopicProc = make(map[string][]int)
ChanTopic = make(chan []string, 100) ChanTopic = make(chan []string, 100)
@ -48,9 +51,10 @@ func ServiceStart() {
LstTopic = append(LstTopic, topic) LstTopic = append(LstTopic, topic)
//change by zhangjun 2020-08-02
cronMan.Every(60).Seconds().SetTag([]string{"kafka_" + topic}).StartImmediately().Do(KafkaProcess, topic, procNo) cronMan.Every(60).Seconds().SetTag([]string{"kafka_" + topic}).StartImmediately().Do(KafkaProcess, topic, procNo)
//go KafkaProcess(topic, procNo)
//cronMan.StartAsync()
} }
} }
@ -93,6 +97,7 @@ func KafkaProcess(topic string, procNo int) {
var lst []chan bool var lst []chan bool
var lst2 []bool var lst2 []bool
var lst3 []int
for no := 0; no < procNo; no++ { for no := 0; no < procNo; no++ {
@ -100,12 +105,14 @@ func KafkaProcess(topic string, procNo int) {
lst = append(lst, chanProc) lst = append(lst, chanProc)
lst2 = append(lst2, true) lst2 = append(lst2, true)
lst3 = append(lst3, 0)
} }
//add by zhangjun 2020-07-30 //add by zhangjun 2020-07-30
loc.Lock() loc.Lock()
KafkaUtil.ChanTopicProc[topic] = lst KafkaUtil.ChanTopicProc[topic] = lst
KafkaUtil.StateTopicProc[topic] = lst2 KafkaUtil.StateTopicProc[topic] = lst2
KafkaUtil.CountTopicProc[topic] = lst3
loc.Unlock() loc.Unlock()
@ -116,7 +123,7 @@ func KafkaProcess(topic string, procNo int) {
//开启子线程 //开启子线程
go KafkaUtil.Consume(topic, "group_"+topic, no) go KafkaUtil.Consume(topic, "group_"+topic, no)
time.Sleep(time.Second *5) time.Sleep(time.Second * 10)
} }
} else { //TODO：处理异常子线程！！！ } else { //TODO：处理异常子线程！！！
@ -125,10 +132,23 @@ func KafkaProcess(topic string, procNo int) {
fmt.Printf("Dataex Consume Kafka ,Topic:%s,ConsumerGroup:%s,GoroutineNO:%d.\n", topic, "group_"+topic, no) fmt.Printf("Dataex Consume Kafka ,Topic:%s,ConsumerGroup:%s,GoroutineNO:%d.\n", topic, "group_"+topic, no)
KafkaUtil.StateTopicProc[topic][no] = true KafkaUtil.StateTopicProc[topic][no] = true
KafkaUtil.CountTopicProc[topic][no] = 0
go KafkaUtil.Consume(topic, "group_"+topic, no) go KafkaUtil.Consume(topic, "group_"+topic, no)
time.Sleep(time.Second *5) time.Sleep(time.Second * 10)
}
}
}
}
func LogProcess() {
for k,v:= range KafkaUtil.CountTopicProc{
if len(KafkaUtil.CountTopicProc[k])>0{
for no:=0;no< len(v);no++{
fmt.Println("Kafka Consume Topic:"+k,"【",time.Now(),"】 Process Total:",v[no])
} }
} }
} }

12
dsDataex/MyTask/Kafka2ES/Kafka2ESTask/Kafka2ESTask.go
Unescape View File

@ -160,19 +160,21 @@ func Process(topic string,msg kafka.Message){
myDoc.DataContent["access_cityname"]="" myDoc.DataContent["access_cityname"]=""
} }
//TODO: 创建 ES 索引,此处并发执行，来不及判断、创建索引
//add by zhangjun 2020-07-30
//loc.Lock()
flag,_= ES7Util.IndexExist(myMsg.DatasourceId) flag,_= ES7Util.IndexExist(myMsg.DatasourceId)
//创建 ES 索引
if flag==false { if flag==false {
//add by zhangjun 2020-07-30
loc.Lock()
ES7Util.IndexInit(myMsg.DatasourceId, myDoc.DataContent) ES7Util.IndexInit(myMsg.DatasourceId, myDoc.DataContent)
loc.Unlock()
time.Sleep(time.Millisecond * 100) time.Sleep(time.Millisecond * 100)
} }
//loc.Unlock()
ES7Util.IndexDocAdd(myMsg.DatasourceId, &myDoc) ES7Util.IndexDocAdd(myMsg.DatasourceId, &myDoc)
} }

3
dsDataex/MyTask/main.go
Unescape View File

@ -2,6 +2,7 @@ package main
import ( import (
"dsDataex/MyTask/Kafka2ES/Kafka2ESService" "dsDataex/MyTask/Kafka2ES/Kafka2ESService"
"dsDataex/Utils/CacheUtil"
"fmt" "fmt"
"time" "time"
) )
@ -23,6 +24,8 @@ func main() {
//go proc() //go proc()
//time.Sleep(60 * time.Minute) //time.Sleep(60 * time.Minute)
CacheUtil.OrgtreeCacheInit()
Kafka2ESService.ServiceStart() Kafka2ESService.ServiceStart()
//db, _ := geoip2.Open("GeoLite2/GeoLite2-City.mmdb") //db, _ := geoip2.Open("GeoLite2/GeoLite2-City.mmdb")

8
dsDataex/Utils/KafkaUtil/KafkaUtil.go
Unescape View File

@ -26,6 +26,8 @@ var ChanTopicProc map[string][]chan bool
//记录 consume 子线程状态 ( 10 分钟内是否成功执行 ReadMessage，不执行自动关闭子线程 ) //记录 consume 子线程状态 ( 10 分钟内是否成功执行 ReadMessage，不执行自动关闭子线程 )
var StateTopicProc map[string][]bool var StateTopicProc map[string][]bool
var CountTopicProc map[string][]int
func init() { func init() {
KafkaClient, _ := kafka.DialLeader(context.Background(), "tcp", ConfigUtil.KafkaBrokers[0], "__consumer_offsets", 0) KafkaClient, _ := kafka.DialLeader(context.Background(), "tcp", ConfigUtil.KafkaBrokers[0], "__consumer_offsets", 0)
@ -200,6 +202,8 @@ myLoop:
r.Close() r.Close()
StateTopicProc[topic][index] = false StateTopicProc[topic][index] = false
fmt.Printf("KafkaUtil Consume Close ,Topic:%s,ConsumerGroup:%s,GoroutineNO:%d.\n", topic, group, index)
return return
} }
//case <- ticker.C: //case <- ticker.C:
@ -216,12 +220,14 @@ myLoop:
msg, err := r.ReadMessage( ctx ) msg, err := r.ReadMessage( ctx )
if err != nil { if err != nil {
//fmt.Println("KafkaUtil ReadMessage Error :",err.Error()) fmt.Println("KafkaUtil ReadMessage Error :",err.Error())
break myLoop break myLoop
} }
Kafka2ESTask.Process(topic, msg) Kafka2ESTask.Process(topic, msg)
//count++ //count++
CountTopicProc[topic][index] ++
//fmt.Printf("message at partiton %d offset %d: %s ==> %s\n",msg.Partition, msg.Offset, string(msg.Key), string(msg.Value)) //fmt.Printf("message at partiton %d offset %d: %s ==> %s\n",msg.Partition, msg.Offset, string(msg.Key), string(msg.Value))
} }
} }

20
dsDataex/docs/docs.go
Unescape View File

@ -1414,7 +1414,7 @@ var doc = `{
"properties": { "properties": {
"auth_time": { "auth_time": {
"type": "string", "type": "string",
"example": "2020-01-02 03:04:05" "example": "2020/01/02 03:04:05"
}, },
"system_id": { "system_id": {
"type": "string", "type": "string",
@ -1485,11 +1485,11 @@ var doc = `{
}, },
"data_source": { "data_source": {
"type": "string", "type": "string",
"example": "ORG" "example": "org_school"
}, },
"org_id": { "org_id": {
"type": "string", "type": "string",
"example": "200201" "example": "-1"
}, },
"system_id": { "system_id": {
"type": "string", "type": "string",
@ -1578,11 +1578,11 @@ var doc = `{
}, },
"data_source": { "data_source": {
"type": "string", "type": "string",
"example": "ORG" "example": "org_school"
}, },
"org_id": { "org_id": {
"type": "string", "type": "string",
"example": "200201" "example": "-1"
}, },
"query_page": { "query_page": {
"type": "integer", "type": "integer",
@ -1590,7 +1590,7 @@ var doc = `{
}, },
"query_time": { "query_time": {
"type": "string", "type": "string",
"example": "2020-01-01 02:03:04" "example": "2020/01/01 02:03:04"
}, },
"system_id": { "system_id": {
"type": "string", "type": "string",
@ -1607,11 +1607,11 @@ var doc = `{
}, },
"data_source": { "data_source": {
"type": "string", "type": "string",
"example": "ORG" "example": "org_school"
}, },
"org_id": { "org_id": {
"type": "string", "type": "string",
"example": "200201" "example": "-1"
}, },
"query_conditions": { "query_conditions": {
"type": "array", "type": "array",
@ -1624,7 +1624,7 @@ var doc = `{
}, },
"query_page": { "query_page": {
"type": "integer", "type": "integer",
"example": 1 "example": 0
}, },
"system_id": { "system_id": {
"type": "string", "type": "string",
@ -1749,7 +1749,7 @@ var doc = `{
}, },
"event_time": { "event_time": {
"type": "string", "type": "string",
"example": "2020-08-08 18:19:20" "example": "2020/08/08 18:19:20"
}, },
"event_type": { "event_type": {
"type": "string", "type": "string",

20
dsDataex/docs/swagger.json
Unescape View File

@ -1399,7 +1399,7 @@
"properties": { "properties": {
"auth_time": { "auth_time": {
"type": "string", "type": "string",
"example": "2020-01-02 03:04:05" "example": "2020/01/02 03:04:05"
}, },
"system_id": { "system_id": {
"type": "string", "type": "string",
@ -1470,11 +1470,11 @@
}, },
"data_source": { "data_source": {
"type": "string", "type": "string",
"example": "ORG" "example": "org_school"
}, },
"org_id": { "org_id": {
"type": "string", "type": "string",
"example": "200201" "example": "-1"
}, },
"system_id": { "system_id": {
"type": "string", "type": "string",
@ -1563,11 +1563,11 @@
}, },
"data_source": { "data_source": {
"type": "string", "type": "string",
"example": "ORG" "example": "org_school"
}, },
"org_id": { "org_id": {
"type": "string", "type": "string",
"example": "200201" "example": "-1"
}, },
"query_page": { "query_page": {
"type": "integer", "type": "integer",
@ -1575,7 +1575,7 @@
}, },
"query_time": { "query_time": {
"type": "string", "type": "string",
"example": "2020-01-01 02:03:04" "example": "2020/01/01 02:03:04"
}, },
"system_id": { "system_id": {
"type": "string", "type": "string",
@ -1592,11 +1592,11 @@
}, },
"data_source": { "data_source": {
"type": "string", "type": "string",
"example": "ORG" "example": "org_school"
}, },
"org_id": { "org_id": {
"type": "string", "type": "string",
"example": "200201" "example": "-1"
}, },
"query_conditions": { "query_conditions": {
"type": "array", "type": "array",
@ -1609,7 +1609,7 @@
}, },
"query_page": { "query_page": {
"type": "integer", "type": "integer",
"example": 1 "example": 0
}, },
"system_id": { "system_id": {
"type": "string", "type": "string",
@ -1734,7 +1734,7 @@
}, },
"event_time": { "event_time": {
"type": "string", "type": "string",
"example": "2020-08-08 18:19:20" "example": "2020/08/08 18:19:20"
}, },
"event_type": { "event_type": {
"type": "string", "type": "string",

20
dsDataex/docs/swagger.yaml
Unescape View File

@ -3,7 +3,7 @@ definitions:
MySwagger.Auth: MySwagger.Auth:
properties: properties:
auth_time: auth_time:
example: "2020-01-02 03:04:05" example: 2020/01/02 03:04:05
type: string type: string
system_id: system_id:
example: SYS-200201 example: SYS-200201
@ -53,10 +53,10 @@ definitions:
example: "202008080008" example: "202008080008"
type: string type: string
data_source: data_source:
example: ORG example: org_school
type: string type: string
org_id: org_id:
example: "200201" example: "-1"
type: string type: string
system_id: system_id:
example: SYS-200201 example: SYS-200201
@ -118,16 +118,16 @@ definitions:
example: DATAEX-TOKEN-a6ce-11ea-b94df48e38f73cf7 example: DATAEX-TOKEN-a6ce-11ea-b94df48e38f73cf7
type: string type: string
data_source: data_source:
example: ORG example: org_school
type: string type: string
org_id: org_id:
example: "200201" example: "-1"
type: string type: string
query_page: query_page:
example: 0 example: 0
type: integer type: integer
query_time: query_time:
example: "2020-01-01 02:03:04" example: 2020/01/01 02:03:04
type: string type: string
system_id: system_id:
example: SYS-200201 example: SYS-200201
@ -139,10 +139,10 @@ definitions:
example: DATAEX-TOKEN-a6ce-11ea-b94df48e38f73cf7 example: DATAEX-TOKEN-a6ce-11ea-b94df48e38f73cf7
type: string type: string
data_source: data_source:
example: ORG example: org_school
type: string type: string
org_id: org_id:
example: "200201" example: "-1"
type: string type: string
query_conditions: query_conditions:
example: example:
@ -151,7 +151,7 @@ definitions:
type: string type: string
type: array type: array
query_page: query_page:
example: 1 example: 0
type: integer type: integer
system_id: system_id:
example: SYS-200201 example: SYS-200201
@ -236,7 +236,7 @@ definitions:
example: 事件顺序号：1/2/3 example: 事件顺序号：1/2/3
type: string type: string
event_time: event_time:
example: "2020-08-08 18:19:20" example: 2020/08/08 18:19:20
type: string type: string
event_type: event_type:
example: "1001" example: "1001"

Write Preview
Loading…
Cancel
Save