diff --git a/Logs/dsBaseRpc.log b/Logs/dsBaseRpc.log index 064e89f4..cd37513b 100644 --- a/Logs/dsBaseRpc.log +++ b/Logs/dsBaseRpc.log @@ -52229,3 +52229,97 @@ [Error]2020/09/07 11:14:10 SqlQueryError MISCONF Redis is configured to save RDB snapshots, but it is currently not able to persist on disk. Commands that may modify the data set are disabled, because this instance is configured to report errors during writes if RDB snapshotting fails (stop-writes-on-bgsave-error option). Please check the Redis logs for details about the RDB error. [Error]2020/09/07 11:14:11 SqlQueryError MISCONF Redis is configured to save RDB snapshots, but it is currently not able to persist on disk. Commands that may modify the data set are disabled, because this instance is configured to report errors during writes if RDB snapshotting fails (stop-writes-on-bgsave-error option). Please check the Redis logs for details about the RDB error. [Error]2020/09/07 11:14:11 SqlQueryError MISCONF Redis is configured to save RDB snapshots, but it is currently not able to persist on disk. Commands that may modify the data set are disabled, because this instance is configured to report errors during writes if RDB snapshotting fails (stop-writes-on-bgsave-error option). Please check the Redis logs for details about the RDB error. +[Error]2020/09/14 13:45:49 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 13:45:49 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 13:45:49 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 13:45:49 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 13:45:49 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 13:45:49 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 13:45:49 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 13:45:49 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 13:46:23 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 13:46:23 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 13:48:08 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 13:48:08 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 13:51:51 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 13:51:51 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 13:58:52 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 13:58:52 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:11:16 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:11:16 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:11:16 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:11:16 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:11:16 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:11:16 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:11:16 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:11:16 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:11:39 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:11:39 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:12:29 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:12:29 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:12:29 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:12:29 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:12:29 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:12:29 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:12:29 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:12:29 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:12:41 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:12:41 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:14:20 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:14:20 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:14:20 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:14:20 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:14:20 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:14:20 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:14:20 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:14:20 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:14:24 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:14:24 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:24:40 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:24:40 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:24:40 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:24:40 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:24:40 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:24:40 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:24:40 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:24:40 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:25:14 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:25:14 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:25:14 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:25:14 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:25:14 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:25:14 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:25:14 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:25:14 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:25:50 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:25:50 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:25:50 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:25:50 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:25:50 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:25:50 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:25:50 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:25:50 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:27:10 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:27:10 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:27:10 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:27:10 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:27:10 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:27:10 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:27:10 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:27:10 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:27:47 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:27:47 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:27:47 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:27:47 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:27:47 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:27:47 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:27:47 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:27:47 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:28:31 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:28:31 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:28:31 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:28:31 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:28:31 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:28:31 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:28:31 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) +[Error]2020/09/14 14:28:31 SqlQueryError Error 1045: Access denied for user 'root'@'localhost' (using password: YES) diff --git a/dsDataex/Config/Config.ini b/dsDataex/Config/Config.ini index 0160fad2..08612626 100644 --- a/dsDataex/Config/Config.ini +++ b/dsDataex/Config/Config.ini @@ -3,7 +3,7 @@ ip = 127.0.0.1 port = 3306 database = db_dataex user = root -pwd = dsideal +pwd = ;ip = 10.10.14.187 ;port = 22066 diff --git a/dsDataex/MyService/DataEX/DataexDAO/DataexDAO.go b/dsDataex/MyService/DataEX/DataexDAO/DataexDAO.go index f49ae0f8..fd89d439 100644 --- a/dsDataex/MyService/DataEX/DataexDAO/DataexDAO.go +++ b/dsDataex/MyService/DataEX/DataexDAO/DataexDAO.go @@ -384,4 +384,47 @@ func SaveDataError2(fails []MySwagger.FailResult,datas []MySwagger.Data ,SystemI } return true, "数据添加成功", nil +} + +/** + * @title GetMetadataResults + * @Description 获取元数据集合 + * @Author wangshuai + * @Date 2020-09-16 + * @Param conditions string 查询条件 + * @Return bool 执行结果 + * @Return string 提示信息 + * @Return int 条数 + * @Return []map[string]interface{} 元数据集合 + * @Return error 错误信息 + */ +func GetMetadataResults(conditions string) (bool, string, int, []map[string]interface{}, error) { + sql := "SELECT * FROM t_dataex_metadata WHERE 1 = 1" + conditions + " ORDER BY create_time DESC, change_time DESC" + //数据条数 + count, _ := DbUtil.Engine.SQL(sql).Query().Count() + //分页数据 + list, err := DbUtil.Engine.SQL(sql).Query().List() + if list != nil { + return true, "数据获取成功", count, list, err + } else { + return false, "数据获取失败，数据源不存在", count, nil, nil + } +} + +/** + * @title GetJyt2012CountByParentIdAndDicValue + * @Description 根据parent_id和字典项值获取字典数量 + * @Author wangshuai + * @Date 2020-09-16 + * @Param parentId string 字典项父ID + * @Param dicValue string 字典项值 + * @Return int64 数量 + */ +func GetJyt2012CountByParentIdAndDicValue(parentId string, dicValue string) int64 { + business := new(models.TDataexJyt2012) + total, err := db.Where("parent_id =? AND dic_value =?", parentId, dicValue).Count(business) + if err != nil { + LogUtil.Error(ErrorConst.SqlQueryError, "数据库操作错误："+err.Error()) + } + return total } \ No newline at end of file diff --git a/dsDataex/MyService/DataEX/DataexService/DataexService.go b/dsDataex/MyService/DataEX/DataexService/DataexService.go index 590cf6be..e4f3df78 100644 --- a/dsDataex/MyService/DataEX/DataexService/DataexService.go +++ b/dsDataex/MyService/DataEX/DataexService/DataexService.go @@ -384,8 +384,13 @@ func DataexSetBatch(systemID string, datas []MySwagger.Data,datasource *models.T var failIDs []string var dataContentFailIDs []map[string]string - //add by wangshuai - //_, datasourceId := ValidationUtil.GetDatasourceIdByCode(datasource.DatasourceCode) + // 校验ESDataContent + // add by wangshuai 2020-09-16 + //_, _, successDatas, failureIDs := ValidationUtil.ValidESDataContent(esData.DatasourceId, datas) + // 将校验通过数据赋值给datas + //datas = successDatas + // 将校验失败数据赋值给failIDs + //failIDs = failureIDs //四、循环添加索引文档 for no:=0;no< len(datas) && no<1000 ;no++{ diff --git a/dsDataex/Utils/ValidationUtil/ValidationUtil.go b/dsDataex/Utils/ValidationUtil/ValidationUtil.go index a5e8798e..08fcb598 100644 --- a/dsDataex/Utils/ValidationUtil/ValidationUtil.go +++ b/dsDataex/Utils/ValidationUtil/ValidationUtil.go @@ -1,74 +1,113 @@ package ValidationUtil import ( - "dsDataex/MyModel_del/DataSource/DatasourceService" - "dsDataex/MyModel_del/JYT2012/Jyt2012DAO" - "dsDataex/MyModel_del/MetaData/MetadataDAO" - "dsDataex/MyModel_del/MySwagger" - "strconv" + "dsDataex/MyService/DataEX" + "dsDataex/MyService/DataEX/DataexDAO" + "dsDataex/MyService/MySwagger" + "dsDataex/Utils/CommonUtil" + "encoding/json" "strings" ) -func ValidESDataContent(datasourceId string, dataContent map[string]interface{}) (bool, string) { +/** + * @title ValidESDataContent + * @Description 校验ESDataContent + * @Author wangshuai + * @Date 2020-09-16 + * @Param datasourceCode string 数据源CODE + * @Param datas []MySwagger.Data Data + * @Return bool 校验是否完全通过 + * @Return string 提示信息 + * @Return []MySwagger.Data 校验通过数据集合 + * @Return []string 校验失败ID集合 + */ +func ValidESDataContent(datasourceCode string, datas []MySwagger.Data) (bool, string, []MySwagger.Data, []string) { + var dataContent map[string]interface{} var fails string - var query MySwagger.MetadataQuery - r := true + var failIDs []string + var successDatas []MySwagger.Data + var conditions string + var esData DataEX.ESData + var r bool - query.Conditions = " AND datasource_id=" + datasourceId - result, _, _, metadatas, _ := MetadataDAO.GetMetadataResults(query) + conditions = " AND datasource_id='" + datasourceCode + "'" + // 获取该数据源对应的元数据集合 + result, _, _, metadatas, _ := DataexDAO.GetMetadataResults(conditions) if result == true { - for _, v := range metadatas { - for _, vv := range v { - if _, ok := dataContent[vv.(string)]; ok { - res, mes, _ := ValidESDataContentItem(v, vv.(string), dataContent[vv.(string)]) - if res == false { - //fail := make(map[string]interface{}) - //fail[vv.(string)] = mes - fails += vv.(string) + mes + " " + for no := 0; no < len(datas) && no < 1000; no++ { + r = true + var jsonData map[string]interface{} + json.Unmarshal([]byte(datas[no].Data), &jsonData) + esData.DataContent = jsonData + dataContent = esData.DataContent + for _, v := range metadatas { + // 判断集合中是否含有该项数据 + if _, ok := dataContent[v["item_name"].(string)]; ok { + // 校验数据 + res, mes, _ := ValidESDataContentItem(v, v["item_name"].(string), dataContent[v["item_name"].(string)]) + if res == false { + fails += v["item_name"].(string) + mes + " " r = false + // 将校验失败的ID集合写入到failIDs + failIDs = append(failIDs, datas[no].DataID) + continue } } } + if r == true { + // 将校验通过的数据集合写入到successDatas + successDatas = append(successDatas, datas[no]) + } } } + // 首尾去空格 fails = strings.TrimRight(fails, " ") - return r, fails + return r, fails, successDatas, failIDs } +/** + * @title ValidESDataContentItem + * @Description 校验ESDataContent每项元数据 + * @Author wangshuai + * @Date 2020-09-16 + * @Param dataContentItem map[string]interface{} 数据源CODE + * @Param itemName string 校验项名称 + * @Param itemValue interface{} 校验项值 + * @Return bool 校验是否通过 + * @Return string 校验提示信息 + * @Return error 错误信息 + */ func ValidESDataContentItem(dataContentItem map[string]interface{}, itemName string, itemValue interface{}) (bool, string, error) { - //var where string - //where = "datasource_id='" + datasourceId + "' AND item_name='" + itemName + "'" - //result, _, row, _ := MetadataDAO.GetMetadataRow(where) if dataContentItem != nil { dicId := dataContentItem["dic_id"] - itemLength, _ := strconv.Atoi(dataContentItem["item_length"].(string)) - checkName, _ := strconv.Atoi(dataContentItem["check_name"].(string)) - checkDic, _ := strconv.Atoi(dataContentItem["check_dic"].(string)) - checkType, _ := strconv.Atoi(dataContentItem["check_type"].(string)) - checkPattern, _ := strconv.Atoi(dataContentItem["check_pattern"].(string)) - checkExist, _ := strconv.Atoi(dataContentItem["check_exist"].(string)) + itemLength := dataContentItem["item_length"].(int64) + checkName := dataContentItem["check_name"].(int64) + checkDic := dataContentItem["check_dic"].(int64) + checkType := dataContentItem["check_type"].(int64) + checkPattern := dataContentItem["check_pattern"].(int64) + checkExist := dataContentItem["check_exist"].(int64) - if itemLength > 0 { - if len(itemValue.(string)) > itemLength { + if itemLength > 0 { // 校验长度 + if CommonUtil.ConvertIntToInt64(len(itemValue.(string))) > itemLength { return false, "最大长度超出", nil } else { return true, "ok", nil } - } else if checkName > 0 { + } else if checkName > 0 { // 校验名称 return true, "ok", nil - } else if checkDic > 0 { + } else if checkDic > 0 { // 校验字典 if CheckDic(dicId.(string), itemValue) == false { return false, "字典检测不通过", nil } else { return true, "ok", nil } - } else if checkType > 0 { + } else if checkType > 0 { // 校验类型 return true, "ok", nil - } else if checkPattern > 0 { + } else if checkPattern > 0 { // 校验规则 return true, "ok", nil - } else if checkExist > 0 { + } else if checkExist > 0 { // 校验必填项 if itemValue == nil { return false, itemName + "是必填项", nil } else { @@ -82,25 +121,47 @@ func ValidESDataContentItem(dataContentItem map[string]interface{}, itemName str } } -func GetDatasourceIdByCode(datasourceCode string) (bool, string) { - res, _, datasourceId, _ := DatasourceService.GetDatasourceIdByCode(datasourceCode) - if res == true { - return res, datasourceId.(string) - } else { - return res, "" - } -} - -//func CheckType(esType string, sqlType string) bool { -// -//} - -// 字典校验 +/** + * @title CheckDic + * @Description 字典校验 + * @Author wangshuai + * @Date 2020-09-16 + * @Param dicId string 字典ID + * @Param itemValue interface{} 字典项值 + * @Return bool 校验是否通过 + */ func CheckDic(dicId string, itemValue interface{}) bool { - total := Jyt2012DAO.GetJyt2012CountByParentIdAndDicValue(dicId, itemValue.(string)) + total := DataexDAO.GetJyt2012CountByParentIdAndDicValue(dicId, itemValue.(string)) if total > 0 { return true } else { return false } } + +// 暂时作废 2020-09-16 +func ValidESDataContentBak(datasourceCode string, dataContent map[string]interface{}) (bool, string) { + var fails string + var conditions string + r := true + + conditions = " AND datasource_id=" + datasourceCode + result, _, _, metadatas, _ := DataexDAO.GetMetadataResults(conditions) + if result == true { + for _, v := range metadatas { + for _, vv := range v { + if _, ok := dataContent[vv.(string)]; ok { + res, mes, _ := ValidESDataContentItem(v, vv.(string), dataContent[vv.(string)]) + if res == false { + fails += vv.(string) + mes + " " + + r = false + } + } + } + } + } + fails = strings.TrimRight(fails, " ") + + return r, fails +} diff --git a/dsSupport/MyModel/DataSource/DatasourceOpenAPI/DatasourceOpenAPI.go b/dsSupport/MyModel/DataSource/DatasourceOpenAPI/DatasourceOpenAPI.go index e021e466..f7d57b07 100644 --- a/dsSupport/MyModel/DataSource/DatasourceOpenAPI/DatasourceOpenAPI.go +++ b/dsSupport/MyModel/DataSource/DatasourceOpenAPI/DatasourceOpenAPI.go @@ -249,7 +249,7 @@ func ReadESDoc(c *gin.Context) { conditions := raw.Conditions sort := raw.Sort - success, message, esdata := DatasourceService.ReadESDoc(datasourceCode, orgIDs, page, begin, conditions, sort) + success, message, count, esdata := DatasourceService.ReadESDoc(datasourceCode, orgIDs, page, begin, conditions, sort) //fmt.Println("esdata=", esdata) if success { c.JSON(http.StatusOK, MySwagger.Result{ @@ -257,6 +257,7 @@ func ReadESDoc(c *gin.Context) { Fail: false, Message: message, Data: esdata, + Total: count, }) return diff --git a/dsSupport/MyModel/DataSource/DatasourceService/DatasourceService.go b/dsSupport/MyModel/DataSource/DatasourceService/DatasourceService.go index f237249c..3fbba1c8 100644 --- a/dsSupport/MyModel/DataSource/DatasourceService/DatasourceService.go +++ b/dsSupport/MyModel/DataSource/DatasourceService/DatasourceService.go @@ -10,6 +10,7 @@ import ( "dsSupport/Utils/ES7Util" "dsSupport/models" "encoding/json" + "fmt" "html" "strconv" "strings" @@ -178,7 +179,9 @@ func GetESDoc(datasourceCode string, dataId string) (bool, string, map[string]in return result, message, m } -func ReadESDoc(datasourceCode string, orgIDs []string, page int, begin string, conditions map[string]interface{}, sort map[string]interface{}) (bool, string, []map[string]interface{}) { +func ReadESDoc(datasourceCode string, orgIDs []string, page int, begin string, conditions map[string]interface{}, sort map[string]interface{}) (bool, string, int, []map[string]interface{}) { + count, _ := ES7Util.GetDocCount(datasourceCode, "", nil) + fmt.Println(count) result, message, esdata := ES7Util.SearchDocPage(datasourceCode, orgIDs, page, begin, conditions, sort) var query = "AND datasource_code='" + datasourceCode + "'" res, _, data, _ := DatasourceDAO.GetDatasourceRow(query) @@ -198,7 +201,7 @@ func ReadESDoc(datasourceCode string, orgIDs []string, page int, begin string, c esDatas = append(esDatas, esData) } - return result, message, esDatas + return result, message, count, esDatas } func IsDatasourceExistsById(id string) bool { diff --git a/dsSupport/MyModel/DataStatistic/DatastatisticService/DatastatisticService.go b/dsSupport/MyModel/DataStatistic/DatastatisticService/DatastatisticService.go index 61cf37a6..b91c2a9e 100644 --- a/dsSupport/MyModel/DataStatistic/DatastatisticService/DatastatisticService.go +++ b/dsSupport/MyModel/DataStatistic/DatastatisticService/DatastatisticService.go @@ -5,6 +5,7 @@ import ( "dsSupport/MyModel/DataSource/DatasourceService" "dsSupport/MyModel/MySwagger" "dsSupport/Utils/ES7Util" + "fmt" ) type ESDataContent struct { @@ -21,13 +22,14 @@ func GetESDocCount() (bool, string, int, []map[string]interface{}) { linksystems, c, _ := AccessSystemDao.ListApp("", 1, 1000) if c > 0 { for _, v := range linksystems { - var num int64 = 0 + var num int = 0 var sw MySwagger.DatasourceSwag - sw.SystemId = v["app_id"].(string) + sw.SystemId = v["app_code"].(string) rr, _, _, datasources, _ := DatasourceService.GetDatasourceResults(sw) if rr == true { for _, vv := range datasources { - result, _ := ES7Util.GetDocCount(vv["datasource_code"].(string)) + fmt.Println(vv["datasource_code"]) + result, _ := ES7Util.GetDocCount(vv["datasource_code"].(string), "", nil) num += result _, _, esdata := ES7Util.GetLatestDoc(vv["datasource_code"].(string)) @@ -36,7 +38,7 @@ func GetESDocCount() (bool, string, int, []map[string]interface{}) { data["system_id"] = v["app_id"] data["datasource_code"] = vv["datasource_code"] data["datasource_name"] = vv["datasource_name"] - data["datasource_id"] = vv["id"] + //data["datasource_id"] = vv["id"] data["storage_time"] = esdata.BeginTime data["num"] = result datas = append(datas, data) diff --git a/dsSupport/MyModel/MetaData/MetadataService/MetadataService.go b/dsSupport/MyModel/MetaData/MetadataService/MetadataService.go index 3ba7ad70..c2c299f0 100644 --- a/dsSupport/MyModel/MetaData/MetadataService/MetadataService.go +++ b/dsSupport/MyModel/MetaData/MetadataService/MetadataService.go @@ -195,7 +195,7 @@ func CreateMetadataES(indexName string) (bool, string, error) { b := models.TDataexMetadata{} b.Id = CommonUtil.GetUUID() - b.DatasourceId = html.EscapeString(datasourceId.(string)) + b.DatasourceId = html.EscapeString(indexName) b.ItemName = html.EscapeString(k) //b.DicId = html.EscapeString("BA710E2E-895B-4563-9377-0CA5B3DE1CFD") b.ItemType = itemType diff --git a/dsSupport/Utils/CommonUtil/CommonUtil.go b/dsSupport/Utils/CommonUtil/CommonUtil.go index fa7669c7..358e5720 100644 --- a/dsSupport/Utils/CommonUtil/CommonUtil.go +++ b/dsSupport/Utils/CommonUtil/CommonUtil.go @@ -235,6 +235,22 @@ func ConvertIntToInt64(i int) int64 { return s64 } +/** + * 将int64数据类型转为int + * + * @Author wangshuai + * @Date 2020-09-16 + * + * @Param i64 int64 INT64 + * + * @Return int + */ +func ConvertInt64ToInt(i64 int64) int { + strInt64 := strconv.FormatInt(i64, 10) + i, _ := strconv.Atoi(strInt64) + return i +} + /** 功能：将字符串转为整数 作者：黄海 diff --git a/dsSupport/Utils/ES7Util/ES7Util.go b/dsSupport/Utils/ES7Util/ES7Util.go index 91c663a9..a94a3f0e 100644 --- a/dsSupport/Utils/ES7Util/ES7Util.go +++ b/dsSupport/Utils/ES7Util/ES7Util.go @@ -604,6 +604,20 @@ func string2interface(arr []string) []interface{} { return result } +/** + * @title SearchDocPage + * @Description 检索文档 + * @Author wangshuai + * @Date 2020-09-16 + * @Param indexName string 数据源CODE + * @Param orgIDs []string 机构ID集合 + * @Param begin string 开始时间 + * @Param conditions map[string]interface{} 查询条件 + * @Param sort map[string]interface{} 排序 + * @Return bool 执行结果 + * @Return string 提示信息 + * @Return []DataEX.ESData 文档集合 + */ func SearchDocPage(indexName string,orgIDs []string,page int,begin string, conditions map[string]interface{}, sort map[string]interface{}) (bool,string,[]DataEX.ESData) { var list []DataEX.ESData var field string @@ -655,20 +669,50 @@ func SearchDocPage(indexName string,orgIDs []string,page int,begin string, condi } /** - * @Author wangshuai + * @title IndexDataContentMapping * @Description 获取ES索引 Mapping 结构 - * @Date 2020-07-31 17:07 - * @Param indexName string ES索引名称【相当于数据库表名】 - * @return map[string]interface mapping data_content数据 - **/ + * @Author wangshuai + * @Date 2020-09-16 + * @Param indexName string 数据源CODE + * @Return map[string]interface{} data_content mapping 数据 + */ func IndexDataContentMapping(indexName string) map[string]interface{} { result,_:= ES7Client.GetMapping().Index(indexName).Do(CTX) //fmt.Println(result["user_student"].(map[string]interface{})["mappings"].(map[string]interface{})["properties"].(map[string]interface{})["data_content"].(map[string]interface{})["properties"]) return result[indexName].(map[string]interface{})["mappings"].(map[string]interface{})["properties"].(map[string]interface{})["data_content"].(map[string]interface{})["properties"].(map[string]interface{}) } -func GetDocCount(indexName string) (int64, error) { - count, err := ES7Client.Count(indexName).Do(CTX) +/** + * @title GetDocCount + * @Description 获取文档数量 + * @Author wangshuai + * @Date 2020-09-16 + * @Param indexName string 数据源CODE + * @Param begin string 开始时间 + * @Param conditions map[string]interface{} 查询条件 + * @Return int 数量 + * @Return error 错误信息 + */ +func GetDocCount(indexName string, begin string, conditions map[string]interface{}) (int, error) { + term2 := elastic.NewTermQuery("enable_flag", 1) + + query:=elastic.NewBoolQuery() + query.Filter(term2) + + if begin!=""{ + term3 := elastic.NewRangeQuery("begin_time").Gte(begin) + query.Filter(term3) + } + + if conditions != nil { + for k, v := range conditions { + condition := elastic.NewTermsQuery(k, v) + query.Filter(condition) + } + } + + count64, err := ES7Client.Count(indexName).Query(query).Do(CTX) + count := CommonUtil.ConvertInt64ToInt(count64) if err != nil { return count, nil @@ -677,6 +721,16 @@ func GetDocCount(indexName string) (int64, error) { } } +/** + * @title GetLatestDoc + * @Description 获取最新一条文档 + * @Author wangshuai + * @Date 2020-09-16 + * @Param indexName string 数据源CODE + * @Return bool 执行结果 + * @Return string 提示信息 + * @Return DataEX.ESData 文档数据 + */ func GetLatestDoc(indexName string) (bool,string,DataEX.ESData) { var data DataEX.ESData var list []DataEX.ESData @@ -699,12 +753,11 @@ func GetLatestDoc(indexName string) (bool,string,DataEX.ESData) { } } +// 暂时作废 2020-09-16 func GetDoc(indexName string, dataId string) (bool,string,DataEX.ESData) { var data DataEX.ESData var list []DataEX.ESData term2 := elastic.NewTermQuery("enable_flag", 1) - - //query.Filter(term2) term3 := elastic.NewTermQuery("data_id", dataId)