diff --git a/dsSupport/Utils/CacheUtil/DaoCache.go b/dsSupport/Utils/CacheUtil/DaoCache.go index 52d38759..b5b0dd60 100644 --- a/dsSupport/Utils/CacheUtil/DaoCache.go +++ b/dsSupport/Utils/CacheUtil/DaoCache.go @@ -144,10 +144,12 @@ func batchReadRedis(ids []string, prefix string) ([]map[string]interface{}, []st func count(baseSql string, pkField string, args []interface{}) (int32, error) { //替换掉分页代码，获取数据量总数 countSql := strings.Replace(baseSql, "SELECT "+pkField+" ", "SELECT count(*) as count ", -1) + fmt.Println("countSql==", countSql) var count int32 //去掉最后的limit+offset args = args[0 : len(args)-2] _, err := DbUtil.Engine.SQL(countSql, args...).Get(&count) + fmt.Println("err==", err) if err != nil { LogUtil.Error(ErrorConst.SqlQueryError, err.Error()) return 0, err @@ -177,6 +179,7 @@ func Page(baseSql string, args ...interface{}) ([]map[string]interface{}, int32, conditionSql := fmt.Sprintf("%s", " limit ? offset ? ") //分页的语句 pageSql := fmt.Sprintf("%s %s", baseSql, conditionSql) + fmt.Println("pageSql=", pageSql) //分页数据 list, err := DbUtil.Engine.SQL(pageSql, args...).Query().List() diff --git a/dsSupport/Utils/ES7Util/DataEX/ESData.go b/dsSupport/Utils/ES7Util/DataEX/ESData.go new file mode 100644 index 00000000..bf872a1b --- /dev/null +++ b/dsSupport/Utils/ES7Util/DataEX/ESData.go @@ -0,0 +1,64 @@ +package DataEX + +import ( + "fmt" + "time" +) + +type ESData struct { + + SystemId string `json:"system_id"` + DatasourceId string `json:"datasource_id"` + ProvinceId string `json:"province_code"` + ProvinceName string `json:"province_name"` + CityId string `json:"city_code"` + CityName string `json:"city_name"` + AreaId string `json:"district_code"` + AreaName string `json:"district_name"` + BureauId string `json:"bureau_id"` + RegionId string `json:"region_id"` + MainId string `json:"main_id"` + OrgId string `json:"org_id"` + OrgName string `json:"org_name" ` + OrgType int `json:"org_type"` + SchoolType string `json:"school_type"` + SchoolTypeName string `json:"school_typename"` + DeptId string `json:"dept_id"` + StageId string `json:"stage_id"` + GradeId string `json:"grade_id"` + ClassId string `json:"class_id"` + DataId string `json:"data_id"` + FileUri string `json:"file_uri"` + BeginTime JsonDate `json:"begin_time"` + EndTime JsonDate `json:"end_time"` + DelFlag int `json:"del_flag"` + EnableFlag int `json:"enable_flag"` + + DataContent map[string]interface{} `json:"data_content"` +} + + +//add by zhangjun 2020-07-01 +type JsonDate time.Time + +const ( + timeFormat = "2006/01/02 15:04:05" +) + +// JsonDate反序列化 +func (t *JsonDate) UnmarshalJSON(data []byte) (err error) { + newTime, err := time.ParseInLocation("\""+timeFormat+"\"", string(data), time.Local) + *t = JsonDate(newTime) + return +} + +// JsonDate序列化 +func (t JsonDate) MarshalJSON() ([]byte, error) { + timeStr := fmt.Sprintf("\"%s\"", time.Time(t).Format(timeFormat)) + return []byte(timeStr),nil +} + +// string方法 +func (t JsonDate) String() string { + return time.Time(t).Format(timeFormat) +} diff --git a/dsSupport/Utils/ES7Util/DataEX/KafkaData.go b/dsSupport/Utils/ES7Util/DataEX/KafkaData.go new file mode 100644 index 00000000..5b12ec07 --- /dev/null +++ b/dsSupport/Utils/ES7Util/DataEX/KafkaData.go @@ -0,0 +1,24 @@ +package DataEX + +type KafkaData struct { + + SystemId string `json:"system_id"` + DatasourceId string `json:"datasource_id"` + DataId string `json:"data_id"` + + UserID string `json:"user_id"` + Identity string `json:"identity"` + AccessID string `json:"access_id"` + AccessIP string `json:"access_ip"` + AccessWay string `json:"access_way"` + + EventType string `json:"event_type"` + EventName string `json:"event_name"` + EventTime string `json:"event_time"` + EventURI string `json:"event_uri"` + EventSeqNO string `json:"event_seqno"` + CollectTime string `json:"collect_time"` + + UserProperty map[string]interface{} `json:"user_property"` + EventProperty map[string]interface{} `json:"event_property"` +} diff --git a/dsSupport/Utils/ES7Util/ES7Util.go b/dsSupport/Utils/ES7Util/ES7Util.go new file mode 100644 index 00000000..af547969 --- /dev/null +++ b/dsSupport/Utils/ES7Util/ES7Util.go @@ -0,0 +1,678 @@ +package ES7Util + +import ( + "context" + "dsSupport/Utils/CommonUtil" + "dsSupport/Utils/ConfigUtil" + "dsSupport/Utils/ES7Util/DataEX" + "dsSupport/Utils/ES7Util/MySwagger" + "dsSupport/Utils/RedisUtil" + "encoding/json" + "fmt" + "github.com/olivere/elastic/v7" + "reflect" + "strconv" + "strings" + "time" +) + +var ES7Client *elastic.Client +var ServerVersion string +var CTX context.Context + +/** + * @Author zhangjun + * @Description ES7 客户端工具初始化 + * @Date 2020-06-12 16:34 + * @Param + * @return + **/ +func init() { + //初始化ES Client，使用配置文件 + ES7Client, _ = elastic.NewClient(elastic.SetURL(ConfigUtil.ESAddress...)) + + CTX=context.Background() + + //Ping，获取ES server启动信息 + info, _, _ := ES7Client.Ping(ConfigUtil.ESAddress[0]).Do(CTX) + + ServerVersion =ES7Client.String() +"【"+ info.Version.Number+"】" +} + +/** + * @Author zhangjun + * @Description 创建ES索引 + * @Date 2020-06-12 14:31 + * @Param indexName string ES索引名称【相当于数据库表名】 + * @Param dataMap string ES索引Mapping【相当于数据库表定义】 + * @return bool 成功/失败 + * @return error 异常 + **/ +func IndexCreate(indexName string,dataMap string) (bool,error){ + + const emptyMap = ` +{ + "settings":{ + "number_of_shards": 3, + "number_of_replicas": 1 + }, + "mappings":{ + } +}` + if dataMap==""{ + dataMap=emptyMap + } + + createIndex, err := ES7Client.CreateIndex(indexName).Body(dataMap).Pretty(true).Do(CTX) + + if createIndex.Acknowledged { + + return true,nil + } else{ + + return false,err + } +} + +/** + * @Author zhangjun + * @Description 判断ES索引是否存在,并同步到 Redis 缓存 + * @Date 2020-06-12 14:42 + * @Param indexName string ES索引名称【相当于数据库表名】 + * @return bool 成功/失败 + * @return error 异常 + **/ +func IndexExist(indexName string) (bool,error){ + + count,_:= RedisUtil.RedisClient.Exists("es_index_"+indexName).Result() + + if count>0 { + + return true,nil + }else{ + + exists, _ := ES7Client.IndexExists(indexName).Do(CTX) + + if exists { + RedisUtil.SET("es_index_"+indexName,indexName,10*time.Hour) + + return true,nil + }else { + + return false,nil + } + } +} + +/** + * @Author zhangjun + * @Description 使用一条索引数据创建索引，并删除这条数据 + * @Date 2020-06-12 14:38 + * @Param indexName string ES索引名称【相当于数据库表名】 + * @Param data map[string]interface{} 一条索引数据 + * @return bool 成功/失败 + * @return error 异常 + **/ +func IndexInit(indexName string,dataex map[string]interface{}) (bool,error) { + + IndexCreate(indexName,"") + + begin:= time.Now(); + end:=time.Date(9999,9,9,9,9,9,0,begin.Location()) + + //dataex := make(map[string]interface{}) + //dataex["org_id"]="123" + //dataex["org_name"]="长春市第一中学" + //dataex["org_type"]=2 + var myMap=make(map[string]interface{}) + + for key, val := range dataex { + switch val.(type){ + case int,int32,int64: + new,_:=strconv.ParseFloat(val.(string),64) + myMap[key]=new +0.1 + case float64: + new,_:=val.(float64) + myMap[key]=new +0.1 + default: + myMap[key]=val + } + } + + dataJson := DataEX.ESData{SystemId: "BASE001", + DatasourceId: indexName, + ProvinceId: "220000", + CityId: "220100", + AreaId: "220101", + RegionId: "R22010101", + BureauId: "B22010101", + MainId: "M2201010101", + OrgId: "O22010101010101", + OrgType: 1, + GradeId: "G2020", + ClassId: "C202001", + DeptId: "D20200001", + StageId: "S0001", + DataId: "D22010101010101", + DataContent: myMap, + FileUri: "/path/file", + BeginTime: DataEX.JsonDate(begin) , + EndTime: DataEX.JsonDate(end), + DelFlag: -1, + EnableFlag: 1, + } + + result, err := ES7Client.Index().Index(indexName).Id("2020-2020").BodyJson(dataJson).Do(CTX) + + if result.Result=="created" { + //change by zhangjun 2020-06-30 + //ES7Client.Delete().Index(indexName).Id("2020-2020").Do(CTX) + + res,_:= ES7Client.GetMapping().Index(indexName).Do(CTX) + + bytes,_:=json.Marshal( res[indexName] ) + mapping :=string(bytes) + + new :=strings.Replace(mapping,"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}},","",-1) + new=strings.Replace(new,"{\"mappings\"","{\"settings\":{\"number_of_shards\": 3,\"number_of_replicas\": 1},\"mappings\"",-1) + new=strings.Replace(new,"\"text\"","\"keyword\"",-1) + + //Delete the dafault date format + //new=strings.Replace(new ,"\"format\":\"yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis\",","",-1) + //new=strings.Replace(new,"\"date\"}","\"date\",\"format\":\"yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis\"}",-1) + + ES7Client.DeleteIndex(indexName).Do(CTX) + + ES7Client.CreateIndex(indexName).Body(new).Pretty(true).Do(CTX) + + return true,nil + }else{ + + return false,err + } +} + +/** + * @Author zhangjun + * @Description 获取ES索引 Mapping 结构 + * @Date 2020-06-12 17:07 + * @Param indexName string ES索引名称【相当于数据库表名】 + * @return map[string]interface mapping data_content数据 + **/ +func IndexDatamap(indexName string) map[string]interface{} { + result,_:= ES7Client.GetMapping().Index(indexName).Do(CTX) + + return result["data_content"].(map[string]interface{}) +} + +/** + * @Author zhangjun + * @Description 添加文档 + * @Date 2020-06-15 16:52 + * @Param indexName string + * @Param data ESData 文档数据 + * @return + **/ +func IndexDocAdd(indexName string,indexData *DataEX.ESData) (bool,string,error){ + indexData.BeginTime = DataEX.JsonDate(time.Now()); + indexData.EndTime =DataEX.JsonDate(time.Date(9999,9,9,9,9,9,0,time.Now().Location())) + + newid:=CommonUtil.GetUUID() + + result, err := ES7Client.Index().Index(indexName).Id(newid).BodyJson(indexData).Do(CTX) + + if result.Result=="created" { + return true,"文档操作成功",nil + }else{ + return false,"文档操作失败",err + } +} + +/** + * @Author zhangjun + * @Description Kafka 2 ES + * @Date 2020-08-02 02:44 + * @Param + * @return + **/ +func IndexDocAdd2(indexName string,indexData *DataEX.ESData){ + + //defer func() { + // if err := recover(); err != nil { + // fmt.Println("IndexDocAdd2 Panic Recover :", err) + // } + //}() + + indexData.BeginTime = DataEX.JsonDate(time.Now()); + + indexData.EndTime =DataEX.JsonDate(time.Date(9999,9,9,9,9,9,0,time.Now().Location())) + + res, err := ES7Client.Index().Index(indexName).Id(indexData.DataId).BodyJson(indexData).Do(CTX) + + if err != nil { + fmt.Println("IndexDocAdd2 Error :" +err.Error()) + }else { + if res.Result == "created" { + //fmt.Println("IndexDocAdd2 Result :" ,res) + } + if res.Result == "updated"{ + //fmt.Println("IndexDocAdd2 Result :" ,res) + } + } +} + +/** + * @Author zhangjun + * @Description 批量添加文档 + * @Date 2020-06-17 09:31 + * @Param indexName string + * @Param indexData 【】ESData 文档数据数组 + * @return bool 成功/失败 + * @return error 错误 + * @return 【】string 成功数据ID数组 + * @return 【】map【string】string 失败数据map数组 + **/ +func IndexDocAddBatch(indexName string,indexData []*DataEX.ESData) (bool,error,[]string,[]MySwagger.FailResult){ + + var requests []elastic.BulkableRequest + var tempMap =make(map[string]string) + var IDs []string + var Fails []MySwagger.FailResult + + for no:=0;no< len(indexData);no++{ + //indexData[no].BeginTime = time.Now(); + //indexData[no].EndTime =time.Date(9999,9,9,9,9,9,0,time.Now().Location()) + + newid:=CommonUtil.GetUUID() + + //文档ID 绑定 数据ID + tempMap[newid]=indexData[no].DataId + IDs=append(IDs,indexData[no].DataId) + + req := elastic.NewBulkIndexRequest() + + req.Index(indexName).Id(newid).Doc(indexData[no]) + + requests=append(requests,req) + } + + //批量添加索引文档，只返回错误记录 + result, _ := ES7Client.Bulk().Index(indexName).Add(requests...).FilterPath("items.*.error").Do(CTX) + //result, err := ES7Client.Bulk().Index(indexName).Add(requests...).Do(CTX) + + ES7Client.Refresh(indexName).Do(CTX) + + if len(result.Items) ==0 { + + return true,nil,IDs,nil + }else { + for no:=0;no< len(result.Items);no++{ + + reason :=result.Items[no]["index"].Error.Reason + + docID := reason[strings.Index(reason,"document with id '")+ 18:strings.Index(reason,"document with id '") + 18 + 36] + dataID:= tempMap[docID] + + IDs=remove(IDs,dataID) + + var fail MySwagger.FailResult + fail.FailID=dataID + fail.FailReason=reason + + Fails =append(Fails,fail) + } + + return true,nil,IDs,Fails + } +} + +/** + * @Author zhangjun + * @Description 逻辑删除相同dataID的文档 + * @Date 2020-06-16 15:59 + * @Param indexName string 索引名称 + * @Param dataID string 数据ID + * @return + **/ +func IndexDocDel(indexName string,dataID string) (bool,string,error){ + + //change by zhangjun 2020-07-01 + //term := elastic.NewTermQuery("data_id.keyword", dataID) + term := elastic.NewTermQuery("data_id", dataID) + term2 := elastic.NewTermQuery("enable_flag", 1) + + query:=elastic.NewBoolQuery() + query.Filter(term).Filter(term2) + + script := elastic.NewScript("ctx._source.enable_flag = 0 ; ctx._source.end_time = params.now ").Param("now", time.Now()) + + response,err:= ES7Client.UpdateByQuery().Index(indexName).Query(query).Script(script).Do(CTX) + + if err !=nil{ + return false,"文档修改失败",err + } + + if response.Updated >0{ + return true,"文档修改成功",nil + } + return true,"文档修改成功，没有符合筛选条件的文档",nil +} + +/** + * @Author zhangjun + * @Description 批量逻辑删除 + * @Date 2020-06-17 09:33 + * @Param indexName string 索引名称 + * @Param dataIDs 【】string 数据ID数组 + * @return + **/ +func IndexDocDelBatch(indexName string, end time.Time, dataIDs []string) (bool,string,error){ + + //change by zhangjun 2020-07-01 + //term := elastic.NewTermsQuery("data_id.keyword", string2interface(dataIDs)...) + term := elastic.NewTermsQuery("data_id", string2interface(dataIDs)...) + term2 := elastic.NewTermQuery("enable_flag", 1) + + query:=elastic.NewBoolQuery() + query.Filter(term).Filter(term2) + + script := elastic.NewScript("ctx._source.enable_flag = 0 ; ctx._source.end_time = params.now ").Param("now", DataEX.JsonDate(end)) + + response,err:= ES7Client.UpdateByQuery().Index(indexName).Query(query).Script(script).Do(CTX) + + ES7Client.Refresh(indexName).Do(CTX) + + if err !=nil{ + return false,"文档批量修改失败",err + } + + if response.Updated >0{ + return true,"文档批量修改成功",nil + } + return true,"文档批量修改成功,没有符合筛选条件的文档",nil +} + +func IndexDocDelBatch2(indexName string,end time.Time, dataIDs []string) (bool,string,error){ + + //change by zhangjun 2020-07-01 + //term := elastic.NewTermsQuery("data_id.keyword", string2interface(dataIDs)...) + term := elastic.NewTermsQuery("data_id", string2interface(dataIDs)...) + term2 := elastic.NewTermQuery("enable_flag", 1) + term3 := elastic.NewRangeQuery("begin_time").Lt(end) + + query:=elastic.NewBoolQuery() + query.Filter(term).Filter(term2).Filter(term3) + + script := elastic.NewScript("ctx._source.enable_flag = 0 ; ctx._source.end_time = params.now ").Param("now", DataEX.JsonDate(end)) + + response,err:= ES7Client.UpdateByQuery().Index(indexName).Query(query).Script(script).Do(CTX) + + ES7Client.Refresh(indexName).Do(CTX) + + if err !=nil{ + return false,"文档批量修改失败",err + } + + if response.Updated >0{ + return true,"文档批量修改成功",nil + } + return true,"文档批量修改成功,没有符合筛选条件的文档",nil +} + +func IndexDocDelRestore(indexName string, end time.Time, dataIDs []string) (bool,string,error){ + + //change by zhangjun 2020-07-01 + //term := elastic.NewTermsQuery("data_id.keyword", string2interface(dataIDs)...) + term := elastic.NewTermsQuery("data_id", string2interface(dataIDs)...) + term2 := elastic.NewTermQuery("enable_flag", 0) + term3 := elastic.NewTermQuery("end_time",end) + + query:=elastic.NewBoolQuery() + query.Filter(term).Filter(term2).Filter(term3) + + newEnd :=time.Date(9999,9,9,9,9,9,0,time.Now().Location()) + + script := elastic.NewScript("ctx._source.enable_flag = 1 ; ctx._source.end_time = params.now ").Param("now", DataEX.JsonDate(newEnd)) + + response,err:= ES7Client.UpdateByQuery().Index(indexName).Query(query).Script(script).Do(CTX) + + ES7Client.Refresh(indexName).Do(CTX) + + if err !=nil{ + return false,"文档批量修改失败",err + } + + if response.Updated >0{ + return true,"文档批量修改成功",nil + } + return true,"文档批量修改成功,没有符合筛选条件的文档",nil +} + +func IndexDocGet(indexName string,dataID string) (bool,string,DataEX.ESData) { + var data DataEX.ESData + //change by zhangjun 2020-07-01 + //term := elastic.NewTermQuery("data_id.keyword", dataID) + term := elastic.NewTermQuery("data_id", dataID) + term2 := elastic.NewTermQuery("enable_flag", 1) + query:=elastic.NewBoolQuery() + query.Filter(term).Filter(term2) + + result,err:=ES7Client.Search().Index(indexName).Query(query).Do(CTX) + + if err == nil{ + var ttyp DataEX.ESData + for _, item := range result.Each(reflect.TypeOf(ttyp)) { + + data = item.(DataEX.ESData) + break + } + return true,"获取数据成功", data + }else{ + return false,"获取数据失败", data + } +} + +func IndexDocPage(indexName string,orgIDs []string,page int,begin string) (bool,string,[]DataEX.ESData) { + var list []DataEX.ESData + + term2 := elastic.NewTermQuery("enable_flag", 1) + + query:=elastic.NewBoolQuery() + query.Filter(term2) + + if len(orgIDs)!=0{ + //change by zhangjun 2020-07-01 + //term := elastic.NewTermsQuery("org_id.keyword", string2interface(orgIDs)...) + term := elastic.NewTermsQuery("org_id", string2interface(orgIDs)...) + query.Filter(term) + } + + if begin!=""{ + term3 := elastic.NewRangeQuery("begin_time").Gte(begin) + query.Filter(term3) + } + + result,err:=ES7Client.Search().Index(indexName).Query(query).From(page * 100).Size(100).Sort("begin_time",true).Do(CTX) + + if err == nil{ + var ttyp DataEX.ESData + for _, item := range result.Each(reflect.TypeOf(ttyp)) { + + data := item.(DataEX.ESData) + list=append(list,data) + } + return true,"获取数据成功", list + }else{ + return false,"获取数据失败", list + } +} + +func IndexDocQuery(indexName string,orgIDs []string,page int,conditions []string) (bool,string,[]DataEX.ESData) { + var list []DataEX.ESData + + term2 := elastic.NewTermQuery("enable_flag", 1) + + query:=elastic.NewBoolQuery() + query.Filter(term2) + + if len(orgIDs)!=0{ + //change by zhangjun 2020-07-01 + //term := elastic.NewTermsQuery("org_id.keyword", string2interface(orgIDs)...) + term := elastic.NewTermsQuery("org_id", string2interface(orgIDs)...) + query.Filter(term) + } + + if len(conditions)>0{ + for no:=0;no< len(conditions);no++{ + if strings.Contains(conditions[no],">="){ + arr:=strings.Split(conditions[no],">=") + term3 := elastic.NewRangeQuery(strings.TrimSpace(arr[0])).Gte(strings.TrimSpace(arr[1])) + query.Filter(term3) + continue + } + if strings.Contains(conditions[no],"<="){ + arr:=strings.Split(conditions[no],"<=") + term3 := elastic.NewRangeQuery(strings.TrimSpace(arr[0])).Lte(strings.TrimSpace(arr[1])) + query.Filter(term3) + continue + } + if strings.Contains(conditions[no],"="){ + arr:=strings.Split(conditions[no],"=") + term3 := elastic.NewTermQuery(strings.TrimSpace(arr[0]) ,strings.TrimSpace(arr[1])) + query.Filter(term3) + continue + } + if strings.Contains(conditions[no],">"){ + arr:=strings.Split(conditions[no],">") + term3 := elastic.NewRangeQuery(strings.TrimSpace(arr[0])).Gt(strings.TrimSpace(arr[1])) + query.Filter(term3) + continue + } + if strings.Contains(conditions[no],"<"){ + arr:=strings.Split(conditions[no],"<") + term3 := elastic.NewRangeQuery(strings.TrimSpace(arr[0])).Lt(strings.TrimSpace(arr[1])) + query.Filter(term3) + continue + } + } + } + + result,err:=ES7Client.Search().Index(indexName).Query(query).From(page * 100).Size(100).Sort("begin_time",true).Do(CTX) + + if err == nil{ + var ttyp DataEX.ESData + for _, item := range result.Each(reflect.TypeOf(ttyp)) { + + data := item.(DataEX.ESData) + list=append(list,data) + } + return true,"获取数据成功", list + }else{ + return false,"获取数据失败", list + } +} + +func remove(array []string,del string)[]string { + var result []string + + for no:=0;no< len(array);no++{ + if array[no]!=del{ + result=append(result,array[no]) + } + } + return result +} + +func IndexRefresh(indexName string){ + + ES7Client.Refresh(indexName).Do(CTX) +} + +/** + * @Author zhangjun + * @Description 字符数组转换为接口数组，否则terms query会报错，很坑！！！！ + * @Date 2020-06-17 16:10 + * @Param + * @return + **/ +func string2interface(arr []string) []interface{} { + + var result []interface{} + for no:=0;no< len(arr);no++{ + result=append(result,arr[no]) + } + return result +} + +func SearchDocPage(indexName string,orgIDs []string,page int,begin string, conditions map[string]interface{}, sort map[string]interface{}) (bool,string,[]DataEX.ESData) { + var list []DataEX.ESData + var field string + var ascending bool + + term2 := elastic.NewTermQuery("enable_flag", 1) + + query:=elastic.NewBoolQuery() + query.Filter(term2) + + if len(orgIDs)!=0{ + term := elastic.NewTermsQuery("org_id", string2interface(orgIDs)...) + query.Filter(term) + } + + if begin!=""{ + term3 := elastic.NewRangeQuery("begin_time").Gte(begin) + query.Filter(term3) + } + + if conditions != nil { + for k, v := range conditions { + condition := elastic.NewTermsQuery(k, v) + query.Filter(condition) + } + } + + if sort == nil { + field = "begin_time" + ascending = true + } else { + field = sort["field"].(string) + ascending = sort["ascending"].(bool) + } + + result,err:=ES7Client.Search().Index(indexName).Query(query).From(page * 100).Size(100).Sort(field, ascending).Do(CTX) + + if err == nil{ + var ttyp DataEX.ESData + for _, item := range result.Each(reflect.TypeOf(ttyp)) { + + data := item.(DataEX.ESData) + list=append(list,data) + } + return true,"获取数据成功", list + }else{ + return false,"获取数据失败", list + } +} + +/** + * @Author wangshuai + * @Description 获取ES索引 Mapping 结构 + * @Date 2020-07-31 17:07 + * @Param indexName string ES索引名称【相当于数据库表名】 + * @return map[string]interface mapping data_content数据 + **/ +func IndexDataContentMapping(indexName string) map[string]interface{} { + result,_:= ES7Client.GetMapping().Index(indexName).Do(CTX) + //fmt.Println(result["user_student"].(map[string]interface{})["mappings"].(map[string]interface{})["properties"].(map[string]interface{})["data_content"].(map[string]interface{})["properties"]) + return result[indexName].(map[string]interface{})["mappings"].(map[string]interface{})["properties"].(map[string]interface{})["data_content"].(map[string]interface{})["properties"].(map[string]interface{}) +} + +func GetDocCount(indexName string) (int64, error) { + count, err := ES7Client.Count(indexName).Do(CTX) + + if err != nil { + return count, nil + }else { + return count, err + } +} \ No newline at end of file diff --git a/dsSupport/Utils/ES7Util/MySwagger/Auth.go b/dsSupport/Utils/ES7Util/MySwagger/Auth.go new file mode 100644 index 00000000..8306d536 --- /dev/null +++ b/dsSupport/Utils/ES7Util/MySwagger/Auth.go @@ -0,0 +1,8 @@ +package MySwagger + + +type Auth struct { + SystemID string `json:"system_id" example:"SYS-200201" ` + SystemToken string `json:"system_token" example:"a6ce11eab94df48a6ce11eab94df48e38f73cf7e38f73cf7"` + AuthTime string `json:"auth_time" example:"2020/01/02 03:04:05"` +} \ No newline at end of file diff --git a/dsSupport/Utils/ES7Util/MySwagger/Data.go b/dsSupport/Utils/ES7Util/MySwagger/Data.go new file mode 100644 index 00000000..949456c2 --- /dev/null +++ b/dsSupport/Utils/ES7Util/MySwagger/Data.go @@ -0,0 +1,9 @@ +package MySwagger + +type Data struct { + OrgID string `json:"org_id" example:"200201"` + DataID string `json:"data_id" example:"202008080008"` + Data string `json:"data" example:"{id:12300,name:zhangj,org_id:123001}"` + DelFlag int `json:"del_flag" example:"0"` +} + diff --git a/dsSupport/Utils/ES7Util/MySwagger/DataCollect.go b/dsSupport/Utils/ES7Util/MySwagger/DataCollect.go new file mode 100644 index 00000000..642ae9c2 --- /dev/null +++ b/dsSupport/Utils/ES7Util/MySwagger/DataCollect.go @@ -0,0 +1,10 @@ +package MySwagger + +type DataCollect struct { + + AccessToken string `json:"access_token" example:"system_01##20200102030405##a6ce11eab94df48a6ce11eab" ` + DataSource string `json:"data_source" example:"log_login" ` + + UserDatas []User `json:"user_datas" ` + EventDatas []Event `json:"event_datas" ` +} \ No newline at end of file diff --git a/dsSupport/Utils/ES7Util/MySwagger/DataGet.go b/dsSupport/Utils/ES7Util/MySwagger/DataGet.go new file mode 100644 index 00000000..5c34e1bf --- /dev/null +++ b/dsSupport/Utils/ES7Util/MySwagger/DataGet.go @@ -0,0 +1,9 @@ +package MySwagger + +type DataGet struct { + SystemID string `json:"system_id" example:"SYS-200201"` + AuthToken string `json:"auth_token" example:"DATAEX-TOKEN-a6ce-11ea-b94df48e38f73cf7"` + DataSource string `json:"data_source" example:"org_school"` + OrgID string `json:"org_id" example:"-1"` + DataID string `json:"data_id" example:"202008080008"` +} diff --git a/dsSupport/Utils/ES7Util/MySwagger/DataGetResult.go b/dsSupport/Utils/ES7Util/MySwagger/DataGetResult.go new file mode 100644 index 00000000..7421fb78 --- /dev/null +++ b/dsSupport/Utils/ES7Util/MySwagger/DataGetResult.go @@ -0,0 +1,8 @@ +package MySwagger + +type DataGetResult struct { + Success bool `json:"success" example:"true"` + Datas []Data `json:"datas"` + Message string `json:"message" example:"操作成功"` +} + diff --git a/dsSupport/Utils/ES7Util/MySwagger/DataIn.go b/dsSupport/Utils/ES7Util/MySwagger/DataIn.go new file mode 100644 index 00000000..8d6ee6ab --- /dev/null +++ b/dsSupport/Utils/ES7Util/MySwagger/DataIn.go @@ -0,0 +1,8 @@ +package MySwagger + +type DataIn struct { + SystemID string `json:"system_id" example:"SYS-200201" ` + AuthToken string `json:"auth_token" example:"DATAEX-TOKEN-a6ce-11ea-b94df48e38f73cf7" ` + DataSource string `json:"data_source" example:"ORG" ` + Datas []Data `json:"datas" ` +} diff --git a/dsSupport/Utils/ES7Util/MySwagger/DataInResult.go b/dsSupport/Utils/ES7Util/MySwagger/DataInResult.go new file mode 100644 index 00000000..0680cf01 --- /dev/null +++ b/dsSupport/Utils/ES7Util/MySwagger/DataInResult.go @@ -0,0 +1,8 @@ +package MySwagger + +type DataInResult struct { + Success bool `json:"success" example:"true"` + SuccessIDs []string `json:"success_ids" example:"202008080008,202008080009,202008080010"` + FailResults []FailResult `json:"fail_results"` + Message string `json:"message" example:"操作成功"` + } diff --git a/dsSupport/Utils/ES7Util/MySwagger/DataPage.go b/dsSupport/Utils/ES7Util/MySwagger/DataPage.go new file mode 100644 index 00000000..5bb43aa0 --- /dev/null +++ b/dsSupport/Utils/ES7Util/MySwagger/DataPage.go @@ -0,0 +1,10 @@ +package MySwagger + +type DataPage struct { + SystemID string `json:"system_id" example:"SYS-200201"` + AuthToken string `json:"auth_token" example:"DATAEX-TOKEN-a6ce-11ea-b94df48e38f73cf7"` + DataSource string `json:"data_source" example:"org_school"` + OrgID string `json:"org_id" example:"-1"` + QueryTime string `json:"query_time" example:"2020/01/01 02:03:04"` + QueryPage int `json:"query_page" example:"0"` +} \ No newline at end of file diff --git a/dsSupport/Utils/ES7Util/MySwagger/DataQuery.go b/dsSupport/Utils/ES7Util/MySwagger/DataQuery.go new file mode 100644 index 00000000..92cb10a2 --- /dev/null +++ b/dsSupport/Utils/ES7Util/MySwagger/DataQuery.go @@ -0,0 +1,10 @@ +package MySwagger + +type DataQuery struct { + SystemID string `json:"system_id" example:"SYS-200201"` + AuthToken string `json:"auth_token" example:"DATAEX-TOKEN-a6ce-11ea-b94df48e38f73cf7"` + DataSource string `json:"data_source" example:"org_school"` + OrgID string `json:"org_id" example:"-1"` + QueryConditions []string `json:"query_conditions" example:"subject_code=22"` + QueryPage int `json:"query_page" example:"0"` +} \ No newline at end of file diff --git a/dsSupport/Utils/ES7Util/MySwagger/Event.go b/dsSupport/Utils/ES7Util/MySwagger/Event.go new file mode 100644 index 00000000..733ea06f --- /dev/null +++ b/dsSupport/Utils/ES7Util/MySwagger/Event.go @@ -0,0 +1,13 @@ +package MySwagger + +type Event struct { + + EventType string `json:"event_type" example:"1001"` + EventName string `json:"event_name" example:"在线提交作业"` + EventTime string `json:"event_time" example:"2020/08/08 18:19:20"` + EventURI string `json:"event_uri" example:"https://edusoa.com/cloud/study/do_homework"` + EventSeqNO string `json:"event_seqno" example:"事件顺序号：1/2/3"` + EventUserID string `json:"event_userid" example:"用户ID"` + + Properties string `json:"properties" example:"{res_id:12300,res_name:初一语文一单元作业,teacher_id:123001,time_spend:100s}"` +} diff --git a/dsSupport/Utils/ES7Util/MySwagger/FailResult.go b/dsSupport/Utils/ES7Util/MySwagger/FailResult.go new file mode 100644 index 00000000..4ecc35ec --- /dev/null +++ b/dsSupport/Utils/ES7Util/MySwagger/FailResult.go @@ -0,0 +1,6 @@ +package MySwagger + +type FailResult struct { + FailID string `json:"fail_id" example:"202006060001"` + FailReason string `json:"fail_reason" example:"数据ORG_ID访问权限验证失败，无法修改数据"` +} diff --git a/dsSupport/Utils/ES7Util/MySwagger/Result.go b/dsSupport/Utils/ES7Util/MySwagger/Result.go new file mode 100644 index 00000000..f2f392cf --- /dev/null +++ b/dsSupport/Utils/ES7Util/MySwagger/Result.go @@ -0,0 +1,6 @@ +package MySwagger + +type Result struct { + Success bool `json:"success" example:"false"` + Message string `json:"message" example:"接入系统票据验证失败"` +} diff --git a/dsSupport/Utils/ES7Util/MySwagger/User.go b/dsSupport/Utils/ES7Util/MySwagger/User.go new file mode 100644 index 00000000..95c312d1 --- /dev/null +++ b/dsSupport/Utils/ES7Util/MySwagger/User.go @@ -0,0 +1,12 @@ +package MySwagger + +type User struct { + + UserID string `json:"user_id" example:"用户ID"` + Identity string `json:"identity" example:"用户身份（1：教师，2：学生，3：家长，4：管理员，5：访客）"` + AccessID string `json:"access_id" example:"访客ID（用户未登录）"` + AccessIP string `json:"access_ip" example:"10.10.8.88"` + AccessWay string `json:"access_way" example:"app/web/weixin/dingding"` + + Properties string `json:"properties" example:"{group_id:12300,group_name:课外学习一组,user_level:2,login_time:2020/08/08 12:13:14}"` +} diff --git a/dsSupport/Utils/ErrorConst/ErrorConst.go b/dsSupport/Utils/ErrorConst/ErrorConst.go new file mode 100644 index 00000000..c4ca5d0e --- /dev/null +++ b/dsSupport/Utils/ErrorConst/ErrorConst.go @@ -0,0 +1,11 @@ +package ErrorConst + +var +( + SqlQueryError = "SqlQueryError" + SqlUpdateError = "SqlUpdateError" + CreateRedisError = "CreateRedisError" + QueryRedisError = "QueryRedisError" + AnalyzeCookieError = "AnalyzeCookieError" + ConvertError = "ConvertError" +)