diff --git a/dsDataex/MyService/DataEX/DataexService/DataexService.go b/dsDataex/MyService/DataEX/DataexService/DataexService.go index 44b1249a..1826bcc9 100644 --- a/dsDataex/MyService/DataEX/DataexService/DataexService.go +++ b/dsDataex/MyService/DataEX/DataexService/DataexService.go @@ -9,6 +9,7 @@ import ( "dsDataex/Utils/CommonUtil" "dsDataex/Utils/ES7Util" "dsDataex/Utils/KafkaUtil" + "dsDataex/Utils/ValidationUtil" "encoding/json" "fmt" "strconv" @@ -386,11 +387,11 @@ func DataexSetBatch(systemID string, datas []MySwagger.Data,datasource *models.T //校验ESDataContent //add by wangshuai 2020-09-16 - //_, _, successDatas, failureIDs := ValidationUtil.ValidESDataContent(esData.DatasourceId, datas) + _, _, successDatas, failureIDs := ValidationUtil.ValidESDataContent(esData.DatasourceId, datas) //将校验通过数据赋值给datas - //datas = successDatas + datas = successDatas //将校验失败数据赋值给failIDs - //dataContentFailIDs = failureIDs + dataContentFailIDs = failureIDs //四、循环添加索引文档 for no:=0;no< len(datas) && no<1000 ;no++{ diff --git a/dsDataex/Utils/ValidationUtil/ValidationUtil.go b/dsDataex/Utils/ValidationUtil/ValidationUtil.go index 8cdc6dba..520c7d75 100644 --- a/dsDataex/Utils/ValidationUtil/ValidationUtil.go +++ b/dsDataex/Utils/ValidationUtil/ValidationUtil.go @@ -6,13 +6,15 @@ import ( "dsDataex/MyService/MySwagger" "dsDataex/Utils/CommonUtil" "encoding/json" + "fmt" + "reflect" "regexp" "strings" "time" ) var dics []map[string]interface{} -const TIME_LAYOUT = "2020/09/14 15:29:10" +const TIME_LAYOUT = "2020-09-14 15:29:10" type RegexCheck struct {} @@ -39,6 +41,8 @@ func ValidESDataContent(datasourceCode string, datas []MySwagger.Data) (bool, st var dicIds []string var dicIdsStr string + fmt.Println("datas: ", datas) + conditions = "datasource_id='" + datasourceCode + "'" // 获取该数据源对应的元数据集合 result, _, _, metadatas, _ := DataexDAO.GetMetadataResults(conditions) @@ -65,6 +69,7 @@ func ValidESDataContent(datasourceCode string, datas []MySwagger.Data) (bool, st if _, ok := dataContent[v["item_name"].(string)]; ok { // 校验数据 res, mes, _ := ValidESDataContentItem(v, v["item_name"].(string), dataContent[v["item_name"].(string)]) + //fmt.Println(res, mes) if res == false { failMessages = v["item_name"].(string) + mes + " " r = false @@ -75,7 +80,7 @@ func ValidESDataContent(datasourceCode string, datas []MySwagger.Data) (bool, st fails["fail_id"] = datas[no].DataID fails["fail_msg"] = failMessages dataContentFailIDs = append(dataContentFailIDs,fails) - continue + break } } } @@ -115,44 +120,42 @@ func ValidESDataContentItem(dataContentItem map[string]interface{}, itemName str checkPattern := dataContentItem["check_pattern"].(int64) checkExist := dataContentItem["check_exist"].(int64) + fmt.Println("dataContentItem: ", dataContentItem) + if itemLength > 0 { // 校验长度 if CommonUtil.ConvertIntToInt64(len(itemValue.(string))) > itemLength { + fmt.Println(itemValue.(string) + "最大长度超出") return false, "最大长度超出", nil - } else { - return true, "ok", nil } } else if checkName > 0 { // 校验名称 - return true, "ok", nil + } else if checkDic > 0 { // 校验字典 if CheckDic(itemValue, dicId, dics) == false { + fmt.Println("字典校验不通过") return false, "字典校验不通过", nil - } else { - return true, "ok", nil } } else if checkType > 0 { // 校验类型 if CheckType(itemValue, itemType) == false { + fmt.Println("类型校验不通过") return false, "类型校验不通过", nil - } else { - return true, "ok", nil } } else if checkPattern > 0 { // 校验规则 if CheckPattern(itemValue, itemPattern) == false { + fmt.Println("数据项模式校验不通过") return false, "数据项模式校验不通过", nil - } else { - return true, "ok", nil } } else if checkExist > 0 { // 校验必填项 if itemValue == nil { + fmt.Println("是必填项") return false, itemName + "是必填项", nil - } else { - return true, "ok", nil } - } else { - return true, "ok", nil } } else { + fmt.Println("缺少dataContentItem") return false, "校验失败, 缺少dataContentItem", nil } + + return true, "ok", nil } /** @@ -187,12 +190,17 @@ func CheckDic(itemValue interface{}, dicId string, dics []map[string]interface{} */ func CheckType(itemValue interface{}, itemType string) bool { flag := false + fmt.Println("itemType: ", itemType) if itemType == "string" { // 校验是否是字符串 _, ok := itemValue.(string) if ok { flag = true } + tp := reflect.TypeOf(itemValue) + fmt.Println("tp: ", tp) } else if itemType == "float" { // 校验是否是浮点数 + //tp := reflect.TypeOf(itemValue) + //fmt.Println("tp: ", tp) _, ok := itemValue.(float32) if ok { flag = true @@ -201,6 +209,7 @@ func CheckType(itemValue interface{}, itemType string) bool { if ok1 { flag = true } + //fmt.Println("flag: ", flag) } else if itemType == "datetime" { // 校验时间格式是否正确 _, e := time.Parse(TIME_LAYOUT, itemValue.(string)) if e == nil { diff --git a/dsSupport/MyModel/DataSource/DatasourceDAO/DatasourceDAO.go b/dsSupport/MyModel/DataSource/DatasourceDAO/DatasourceDAO.go index 8e595489..826c7558 100644 --- a/dsSupport/MyModel/DataSource/DatasourceDAO/DatasourceDAO.go +++ b/dsSupport/MyModel/DataSource/DatasourceDAO/DatasourceDAO.go @@ -15,6 +15,7 @@ var db = DbUtil.Engine func GetDatasourceRow(query string) (bool, string, map[string]interface{}, error) { sql := "SELECT * FROM t_dataex_datasource WHERE 1 = 1 " + query + fmt.Println("sql::", sql) var datasource models.TDataexDatasource has, _ := DbUtil.Engine.SQL(sql).Get(&datasource) diff --git a/dsSupport/MyModel/DataSource/DatasourceOpenAPI/DatasourceOpenAPI.go b/dsSupport/MyModel/DataSource/DatasourceOpenAPI/DatasourceOpenAPI.go index 76f0186c..b51131a7 100644 --- a/dsSupport/MyModel/DataSource/DatasourceOpenAPI/DatasourceOpenAPI.go +++ b/dsSupport/MyModel/DataSource/DatasourceOpenAPI/DatasourceOpenAPI.go @@ -255,7 +255,6 @@ func ReadESDoc(c *gin.Context) { sort := raw.Sort success, message, count, esdata := DatasourceService.ReadESDoc(datasourceCode, orgIDs, page, begin, conditions, sort) - //fmt.Println("esdata=", esdata) if success { c.JSON(http.StatusOK, MySwagger.Result{ Success: true, diff --git a/dsSupport/MyModel/DataSource/DatasourceService/DatasourceService.go b/dsSupport/MyModel/DataSource/DatasourceService/DatasourceService.go index bae2db41..4ad24f1a 100644 --- a/dsSupport/MyModel/DataSource/DatasourceService/DatasourceService.go +++ b/dsSupport/MyModel/DataSource/DatasourceService/DatasourceService.go @@ -218,7 +218,6 @@ func GetESDoc(datasourceCode string, dataId string) (bool, string, map[string]in func ReadESDoc(datasourceCode string, orgIDs []string, page int, begin string, conditions map[string]interface{}, sort map[string]interface{}) (bool, string, int, []map[string]interface{}) { count, _ := ES7Util.GetDocCount(datasourceCode, begin, conditions) - fmt.Println(count) result, message, esdata := ES7Util.SearchDocPage(datasourceCode, orgIDs, page, begin, conditions, sort) var query = "AND datasource_code='" + datasourceCode + "'" res, _, data, _ := DatasourceDAO.GetDatasourceRow(query) @@ -228,17 +227,25 @@ func ReadESDoc(datasourceCode string, orgIDs []string, page int, begin string, c } var esDatas []map[string]interface{} - var esData map[string]interface{} + // 写法2 + //esDatas := make([]interface{}, count) + fmt.Println(esDatas) for _, value := range esdata { + esData := make(map[string]interface{}) j, _ := json.Marshal(value) json.Unmarshal(j, &esData) - //esData = make(map[string]interface{}) + esData["datasource_name"] = datasourceName + fmt.Println("esData:", esData) + esDatas = append(esDatas, esData) + // 写法2 + //esDatas[k] = make(map[string]interface{}, 1) + //esDatas[k] = esData } - return result, message, count, esDatas + return result, message, count, nil } //func IsDatasourceExistsByCode(code string) bool { diff --git a/dsSupport/Utils/ES7Util/ES7Util.go b/dsSupport/Utils/ES7Util/ES7Util.go index 938e9bb9..1ccaea59 100644 --- a/dsSupport/Utils/ES7Util/ES7Util.go +++ b/dsSupport/Utils/ES7Util/ES7Util.go @@ -658,6 +658,7 @@ func SearchDocPage(indexName string,orgIDs []string,page int,begin string, condi } result,err:=ES7Client.Search().Index(indexName).Query(query).From(page * 100).Size(100).Sort(field, ascending).Do(CTX) + fmt.Println(result) if err == nil{ var ttyp DataEX.ESData for _, item := range result.Each(reflect.TypeOf(ttyp)) {