diff --git a/dsBaseRpc/models/t_app_base.go b/dsBaseRpc/models/t_app_base.go index 4a63ff48..4e608af8 100644 --- a/dsBaseRpc/models/t_app_base.go +++ b/dsBaseRpc/models/t_app_base.go @@ -6,6 +6,7 @@ type TAppBase struct { AppName string `xorm:"not null comment('系统名称') VARCHAR(100)"` AccessKey string `xorm:"not null comment('AK') CHAR(20)"` SecretKey string `xorm:"not null comment('SK') CHAR(26)"` + AppToken string `xorm:"default 'NULL' comment('系统票据') CHAR(100)"` AppUrl string `xorm:"default 'NULL' comment('系统集成页面调用地址') VARCHAR(1024)"` AppIcon string `xorm:"default 'NULL' comment('系统图标的路径') VARCHAR(1024)"` RedirectUri string `xorm:"default 'NULL' comment('统一认证回调地址') VARCHAR(1024)"` diff --git a/dsBaseRpc/models/t_base_organization_log.go b/dsBaseRpc/models/t_base_organization_log.go index a48e3df0..148ce0a8 100644 --- a/dsBaseRpc/models/t_base_organization_log.go +++ b/dsBaseRpc/models/t_base_organization_log.go @@ -12,7 +12,7 @@ type TBaseOrganizationLog struct { MainSchoolType int32 `xorm:"not null default 1 comment('主校类型，1：普通校，2：主校，3：分校') INT(11)"` MainSchoolId string `xorm:"not null default '''' comment('如果是分校，所属主校的ID,如果是主校，是自己的ID，如果是分校，是主校ID，如果是普通校，也是自己的ID') CHAR(36)"` Xxbxlxm string `xorm:"not null default '''' comment('学校办学类型（学校专有属性）有字典') CHAR(3)"` - Szdcxlxm string `xorm:"not null default '''' comment('学校城乡类型（学校专有属性）有字典') CHAR(2)"` + Szdcxlxm string `xorm:"not null default '''' comment('学校城乡类型（学校专有属性）有字典') CHAR(3)"` Xxjbzm string `xorm:"not null default '''' comment('学校举办者（学校专有属性）有字典') CHAR(3)"` Fzr string `xorm:"not null default '''' comment('机构负责人ID（*一个单位只能有一个负责人）') CHAR(36)"` Fddbr string `xorm:"not null default '''' comment('机构法定代表人') VARCHAR(50)"` diff --git a/dsBaseRpc/models/t_base_teacher_position.go b/dsBaseRpc/models/t_base_teacher_position.go index 08d2db18..63f50e08 100644 --- a/dsBaseRpc/models/t_base_teacher_position.go +++ b/dsBaseRpc/models/t_base_teacher_position.go @@ -2,7 +2,7 @@ package models type TBaseTeacherPosition struct { Id string `xorm:"not null pk comment('主键') CHAR(36)"` - PersonId string `xorm:"not null default '''' comment('人员ID') index(person_id) CHAR(18)"` + PersonId string `xorm:"not null default '''' comment('人员ID') index(person_id) CHAR(36)"` BureauId string `xorm:"not null comment('所在单位ID') index(person_id) CHAR(36)"` PositionId string `xorm:"not null default ''0'' comment('职务ID') CHAR(36)"` BUse int32 `xorm:"not null default 1 comment('是否可用 1：可用 -2：不可用') index(person_id) TINYINT(1)"` diff --git a/dsBaseRpc/models/t_dataex_dataaccess.go b/dsBaseRpc/models/t_dataex_dataaccess.go index 1346808b..ae2bb76b 100644 --- a/dsBaseRpc/models/t_dataex_dataaccess.go +++ b/dsBaseRpc/models/t_dataex_dataaccess.go @@ -8,7 +8,7 @@ type TDataexDataaccess struct { Id string `xorm:"not null pk comment('ID') VARCHAR(36)"` DatasourceId string `xorm:"not null comment('数据源ID') index VARCHAR(36)"` DatasourceCode string `xorm:"default 'NULL' VARCHAR(255)"` - ConsumeSystemid string `xorm:"not null comment('数据使用系统ID') index VARCHAR(36)"` + ConsumeSystemid string `xorm:"default 'NULL' comment('数据使用系统ID') index CHAR(36)"` QueryFlag int32 `xorm:"not null default 1 comment('可查【1：是，-1：否】') INT(11)"` SetFlag int32 `xorm:"not null default -1 comment('可修改【1：是，-1：否】') INT(11)"` ConsumeType int32 `xorm:"not null comment('使用数据范围【1：本机构，2：本机构以及下属机构，-1：不限制】') INT(11)"` diff --git a/dsBaseRpc/models/t_dataex_error.go b/dsBaseRpc/models/t_dataex_error.go index e1daba47..2ecdf23a 100644 --- a/dsBaseRpc/models/t_dataex_error.go +++ b/dsBaseRpc/models/t_dataex_error.go @@ -8,10 +8,11 @@ type TDataexError struct { Id string `xorm:"not null pk comment('ID') VARCHAR(36)"` SystemId string `xorm:"not null comment('数据提供系统ID【输入参数】') VARCHAR(36)"` DatasourceId string `xorm:"not null comment('数据源ID【输入参数】') VARCHAR(36)"` - OrgId string `xorm:"not null comment('数据机构ID【输入参数】') VARCHAR(36)"` - DataId string `xorm:"not null comment('数据ID【输入参数】') VARCHAR(36)"` + OrgId string `xorm:"default 'NULL' comment('数据机构ID【输入参数】') VARCHAR(36)"` + DataId string `xorm:"default 'NULL' comment('数据ID【输入参数】') VARCHAR(36)"` DataContent string `xorm:"default 'NULL' comment('数据内容【Json格式输入参数】') LONGTEXT"` FileUri string `xorm:"default 'NULL' comment('文件地址【文件交换】') VARCHAR(500)"` + ErrorInfo string `xorm:"default 'NULL' comment('错误信息说明') VARCHAR(5000)"` ChangeTime time.Time `xorm:"default 'NULL' comment('最近修改时间') DATETIME"` DeleteTime time.Time `xorm:"default 'NULL' comment('删除时间') DATETIME"` EnableFlag int32 `xorm:"not null default 1 comment('启用标志【默认1，1：启用，-1：禁用】') INT(11)"` diff --git a/dsBaseRpc/models/t_dataex_jyt2012.go b/dsBaseRpc/models/t_dataex_jyt2012.go index 47c395d0..515e6e94 100644 --- a/dsBaseRpc/models/t_dataex_jyt2012.go +++ b/dsBaseRpc/models/t_dataex_jyt2012.go @@ -7,6 +7,7 @@ import ( type TDataexJyt2012 struct { Id string `xorm:"not null pk comment('ID') VARCHAR(36)"` DicName string `xorm:"not null comment('字典/字典项名称') VARCHAR(100)"` + DicValue string `xorm:"not null VARCHAR(100)"` DicType int32 `xorm:"not null default 1 comment('字典类型【1：国标数据、关联数据源，2：国标字典、关联元数据】') INT(11)"` DicInfo string `xorm:"default ''NULL'' comment('字典相信说明') VARCHAR(500)"` RootFlag int32 `xorm:"not null default -1 comment('是否是字典【1：是，-1：否】') INT(11)"` diff --git a/dsBaseRpc/models/t_dw_table.go b/dsBaseRpc/models/t_dw_table.go new file mode 100644 index 00000000..bc01a486 --- /dev/null +++ b/dsBaseRpc/models/t_dw_table.go @@ -0,0 +1,6 @@ +package models + +type TDwTable struct { + TableId int32 `xorm:"not null pk autoincr INT(255)"` + TableName string `xorm:"not null VARCHAR(255)"` +} diff --git a/dsBaseRpc/models/t_dw_table_field.go b/dsBaseRpc/models/t_dw_table_field.go new file mode 100644 index 00000000..7d73919c --- /dev/null +++ b/dsBaseRpc/models/t_dw_table_field.go @@ -0,0 +1,13 @@ +package models + +type TDwTableField struct { + Id int32 `xorm:"not null pk autoincr INT(11)"` + TableId int32 `xorm:"not null comment('哪张表') index INT(11)"` + FieldName string `xorm:"not null comment('字段名称') VARCHAR(255)"` + DataType string `xorm:"not null comment('数据类型') VARCHAR(255)"` + FieldLength int32 `xorm:"not null comment('字段长度') INT(11)"` + DecimalPointLength int32 `xorm:"not null default 0 comment('小数点后长度') INT(11)"` + Comment string `xorm:"default 'NULL' comment('描述') VARCHAR(255)"` + IsPk int32 `xorm:"not null default 0 comment('是否为主键') INT(255)"` + IsNull int32 `xorm:"not null comment('是否可为空') INT(255)"` +} diff --git a/dsSupport/Test/CreatePgTable/CreatePgTable.go b/dsSupport/Test/CreatePgTable/CreatePgTable.go index 5584c509..f6618538 100644 --- a/dsSupport/Test/CreatePgTable/CreatePgTable.go +++ b/dsSupport/Test/CreatePgTable/CreatePgTable.go @@ -3,10 +3,12 @@ package main import ( "dsSupport/Utils/CommonUtil" "dsSupport/Utils/DbUtil" + "dsSupport/Utils/PgUtil" "fmt" ) var db = DbUtil.Engine +var pgDb = PgUtil.Engine func main() { //1、读取每一张需要创建的表 @@ -17,14 +19,28 @@ func main() { tableId := list[i]["table_id"].(int64) tableName := list[i]["table_name"].(string) createTableSql := `DROP TABLE IF EXISTS "public"."` + tableName + `";` + "\r\n" - createTableSql += `CREATE TABLE "public"."` + tableName + `" (` + "\r\n" + pgDb.SQL(createTableSql).Execute() + + createTableSql = `CREATE TABLE "public"."` + tableName + `" (` + "\r\n" //2、根据表名，获取相应的创建表的信息 sql = `select * from t_dw_table_field where table_id=?` list2, _ := db.SQL(sql, tableId).Query().List() + //手动增加两个字段：uuid+enable_flag + _map := make(map[string]interface{}, 0) + _map["data_type"] = "char" + _map["field_length"] = CommonUtil.ConvertIntToInt64(36) + _map["field_name"] = "uuid" + list2 = append(list2, _map) + + _map2 := make(map[string]interface{}, 0) + _map2["data_type"] = "int" + _map2["field_length"] = CommonUtil.ConvertIntToInt64(4) + _map2["field_name"] = "enable_flag" + list2 = append(list2, _map2) + for i2 := range list2 { dataType := list2[i2]["data_type"].(string) fieldLength := list2[i2]["field_length"].(int64) - isNull := list2[i2]["is_null"].(int64) fieldName := list2[i2]["field_name"].(string) var c = "" if dataType == "int" { @@ -32,39 +48,25 @@ func main() { } else { c = dataType + "(" + CommonUtil.ConvertInt64ToString(fieldLength) + ") " + ` COLLATE "pg_catalog"."default"` } - if isNull == 0 { - c += " NOT NULL," - } else { - if i2 < len(list2)-1 { - c += "," - } - } + c += "," createTableSql += `"` + fieldName + `" ` + c + "\r\n" } + createTableSql = createTableSql[0 : len(createTableSql)-3] createTableSql += `);` + "\r\n" - //添加生成字段描述信息 - for i2 := range list2 { - fieldName := list2[i2]["field_name"].(string) - comment := list2[i2]["comment"] - if comment != nil && len(comment.(string)) > 0 { - createTableSql += `COMMENT ON COLUMN "public"."` + tableName + `"."` + fieldName + `" IS '` + comment.(string) + `';` + "\r\n" - } - } - //主键有哪些 - var pks = "" - for i2 := range list2 { - fieldName := list2[i2]["field_name"].(string) - isPk := list2[i2]["is_pk"].(int64) - if isPk == 1 { - pks += fieldName + "," - } + _, err := pgDb.SQL(createTableSql).Execute() + fmt.Println(createTableSql) + if err != nil { + panic(err.Error()) } - //去掉最后一个逗号 - pks = pks[0 : len(pks)-1] + //主键 + var pks = "uuid" //添加主键描述 - createTableSql += `-- 设置主键` + "\r\n" - createTableSql += `ALTER TABLE "public"."` + tableName + `" ADD CONSTRAINT "` + tableName + `_dw_pkey" PRIMARY KEY ("` + pks + `");` + "\r\n" - - fmt.Println(createTableSql) + createTableSql = `ALTER TABLE "public"."` + tableName + `" ADD CONSTRAINT "` + tableName + `_dw_pkey" PRIMARY KEY ("` + pks + `");` + "\r\n" + _, err = pgDb.SQL(createTableSql).Execute() + if err != nil { + panic(err.Error()) + } + //bzlbm + fmt.Println("恭喜，数据仓库中相应表格创建完毕！") } } diff --git a/dsSupport/Test/Elasticsearch/ReadElasticsearch.go b/dsSupport/Test/Elasticsearch/ReadElasticsearch.go deleted file mode 100644 index 1c9f527f..00000000 --- a/dsSupport/Test/Elasticsearch/ReadElasticsearch.go +++ /dev/null @@ -1,71 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "github.com/olivere/elastic/v7" - "log" - "os" - "time" -) - -// 深度分页 -// https://www.cnblogs.com/hello-shf/p/11543453.html - -var client elastic.Client - -func main() { - var host = "http://10.10.14.188:9200/" - esClient, err := elastic.NewClient( - elastic.SetURL(host), - elastic.SetSniff(false), - elastic.SetHealthcheckInterval(10*time.Second), - elastic.SetGzip(true), - elastic.SetErrorLog(log.New(os.Stderr, "ELASTIC ", log.LstdFlags)), - elastic.SetInfoLog(log.New(os.Stdout, "", log.LstdFlags))) - - if err != nil { - panic(err) - } - - //取所有 - CTX := context.Background() - result, err := esClient.Scroll().Index("user_teacher").Size(10).Do(CTX) - if err != nil { - fmt.Println(err.Error()) - } - //第一次的结果集 - for i := range result.Hits.Hits { - resByte, _ := json.Marshal(result.Hits.Hits[i]) - resStr := string(resByte) - fmt.Println(resStr) - } - //继续用的 scoll_id - scrollId := result.ScrollId - - //第一次命中的个数 - var nowCount = int64(len(result.Hits.Hits)) - //总的命中个数 - allCount := result.TotalHits() - - //开始循环 - for { - fmt.Println(nowCount) - //如果还有数据没有获取 - if allCount > nowCount { - result, err = esClient.Scroll().ScrollId(scrollId).Do(CTX) - scrollId = result.ScrollId - nowCount += int64(len(result.Hits.Hits)) - - for i := range result.Hits.Hits { - resByte, _ := json.Marshal(result.Hits.Hits[i]) - resStr := string(resByte) - fmt.Println(resStr) - } - } else { - //没有数据了 - break - } - } -} diff --git a/dsSupport/Test/ElasticsearchToGreenPlum/ElasticsearchToGreenPlum.go b/dsSupport/Test/ElasticsearchToGreenPlum/ElasticsearchToGreenPlum.go new file mode 100644 index 00000000..a349a34f --- /dev/null +++ b/dsSupport/Test/ElasticsearchToGreenPlum/ElasticsearchToGreenPlum.go @@ -0,0 +1,208 @@ +package main + +import ( + "context" + "dsSupport/Utils/CommonUtil" + "dsSupport/Utils/DbUtil" + "dsSupport/Utils/PgUtil" + "encoding/json" + "fmt" + "github.com/olivere/elastic/v7" + "github.com/tidwall/gjson" + "log" + "os" + "time" +) + +// 深度分页 +// https://www.cnblogs.com/hello-shf/p/11543453.html + +//MYSQL数据连接器，用于读取元数据信息 +var mysqlDb = DbUtil.Engine + +//GreenPlum数据连接器，用于写入、修改GPDB的数据 +var pgDb = PgUtil.Engine + +//用于批量一次性插入的数据集合 +var insertStrArray = make([]string, 0) + +//用于批量更新的数据集合，一般是业务主键ID集合，修改enable_falg=0 +var pkStrArray = make([]string, 0) + +//缓存表的结构 +var cacheFieldKeys []string + +func main() { + //TODO + //需要将ES的连接信息修改到配置文件中,目前想来应该是数据交换平台的概念，需要在dsDataex项目中实现，待迁移 + var host = "http://10.10.14.188:9200/" + esClient, err := elastic.NewClient( + elastic.SetURL(host), + elastic.SetSniff(false), + elastic.SetHealthcheckInterval(10*time.Second), + elastic.SetGzip(true), + elastic.SetErrorLog(log.New(os.Stderr, "ELASTIC ", log.LstdFlags)), + elastic.SetInfoLog(log.New(os.Stdout, "", log.LstdFlags)), + ) + if err != nil { + //如果是集成到dsDataex项目中，以协程方式启动，这里就不有使用painc(err)会异常会导致程序退出 + panic(err) + } + //从配置表中获取此索引对应的表中主键是哪个，目前只支持一个主键，无主键和多主键不支持 + sql := "select * from t_dw_table where b_use=1" + tableList, _ := mysqlDb.SQL(sql).Query().List() + for i := range tableList { + tableId := tableList[i]["table_id"].(int64) + indexName := tableList[i]["table_name"].(string) + //预热数据表的列名 + getFields(int(tableId)) + //清空重新导入 + sql = "truncate table " + indexName + pgDb.SQL(sql).Execute() + + // 当前表的主键是什么，目前只支持单业务主键，复合主键的不支持 + sql = `select field_name from t_dw_table_field where table_id=? and is_pk=1` + list, _ := mysqlDb.SQL(sql, tableId).Query().List() + pk := list[0]["field_name"].(string) + //取所有 + CTX := context.Background() + result, err := esClient.Scroll().Index(indexName).Size(200).Do(CTX) + if err != nil { + panic(err) + } + //第一次的结果集 + for i := range result.Hits.Hits { + resByte, _ := json.Marshal(result.Hits.Hits[i].Source) + resStr := string(resByte) + value := gjson.Get(resStr, "data_content") + addRecord(pk, value.String()) + } + //批量执行 + batchSave(indexName, pk) + //继续用的 scoll_id + scrollId := result.ScrollId + //第一次命中的个数 + var nowCount = int64(len(result.Hits.Hits)) + //总的命中个数 + allCount := result.TotalHits() + + //开始循环 + for { + //清空 + insertStrArray = insertStrArray[0:0] + pkStrArray = pkStrArray[0:0] + //如果还有数据没有获取 + if allCount > nowCount { + result, err = esClient.Scroll().ScrollId(scrollId).Do(CTX) + scrollId = result.ScrollId + nowCount += int64(len(result.Hits.Hits)) + + for i := range result.Hits.Hits { + resByte, _ := json.Marshal(result.Hits.Hits[i].Source) + resStr := string(resByte) + value := gjson.Get(resStr, "data_content") + addRecord(pk, value.String()) + } + } else { + //没有数据了 + break + } + //批量执行 + batchSave(indexName, pk) + } + fmt.Println("索引" + indexName + "同步完成！") + } +} + +/** +功能：获取指定表的字段有哪些，应该是一个稳定的排序，因为后面要用到 +作者：黄海 +时间：2020-09-07 +*/ +func getFields(tableId int) { + //清空 + cacheFieldKeys = cacheFieldKeys[0:0] + //填充 + sql := `select field_name from t_dw_table_field where table_id=? order by sort_id,field_name` + list, _ := mysqlDb.SQL(sql, tableId).Query().List() + + for i := range list { + fieldName := list[i]["field_name"].(string) + cacheFieldKeys = append(cacheFieldKeys, fieldName) + } +} + +//后续优化为到达一定的阀值再提交一次，初步定为200一次 +func addRecord(pk string, jsonStr string) { + var mymap map[string]interface{} + if err := json.Unmarshal([]byte(jsonStr), &mymap); err == nil { + //组装pkArray,用来update enable_flag + switch (mymap[pk]).(type) { + case string: + pkStrArray = append(pkStrArray, mymap[pk].(string)) + break + case float64: + pkStrArray = append(pkStrArray, CommonUtil.ConvertInt64ToString(int64(mymap[pk].(float64)))) + break + default: + break + } + //另一个任务是组装insertStrArray,准备批量提交 + var lineSql = "" + //根据key从m中拿元素，就是按顺序拿了 + for _, k := range cacheFieldKeys { + switch mymap[k].(type) { + case string: + lineSql += "'" + mymap[k].(string) + "'," + break + case int64: + lineSql += CommonUtil.ConvertInt64ToString(mymap[k].(int64)) + "," + break + case float64: + lineSql += fmt.Sprintf("%d", int64(mymap[k].(float64))) + "," + break + default: + lineSql += "'-1'," //进入这里一般是有这个字段，但数据源中没有这列数据，按-1填充吧~ + break + } + } + lineSql += "'" + CommonUtil.GetUUID() + "'" + insertStrArray = append(insertStrArray, lineSql) + } else { + fmt.Println(err.Error()) + } +} + +//提交 +func batchSave(tableName string, pkName string) { + var pkStr = "" + //数组去重 + pkStrArray = CommonUtil.RemoveRepeatedElement(pkStrArray) + for i := range pkStrArray { + pkStr += "'" + pkStrArray[i] + "'," + } + pkStr = pkStr[0 : len(pkStr)-1] + sql := `update ` + tableName + " set enable_flag=0 where " + pkName + " in (" + pkStr + ") and enable_flag=1" + pgDb.SQL(sql).Execute() + + //插入 + sql = `insert into ` + tableName + "(" + //根据key从m中拿元素，就是按顺序拿了 + for _, k := range cacheFieldKeys { + sql += k + "," + } + sql += "uuid" + + var lineSql = "" + for i := range insertStrArray { + lineSql += "(" + insertStrArray[i] + ")," + } + lineSql = lineSql[0 : len(lineSql)-1] + sql += ") values " + lineSql + ";" + _, err := pgDb.SQL(sql).Execute() + if err != nil { + fmt.Println(sql) + panic(err) + } + fmt.Println(tableName+"批量执行" + CommonUtil.ConvertIntToString(len(insertStrArray)) + "条.") +} diff --git a/dsSupport/Test/ReadEsExistMapping/ReadEsExistMapping.go b/dsSupport/Test/ReadEsExistMapping/ReadEsExistMapping.go new file mode 100644 index 00000000..32884300 --- /dev/null +++ b/dsSupport/Test/ReadEsExistMapping/ReadEsExistMapping.go @@ -0,0 +1,92 @@ +package main + +import ( + "context" + "dsSupport/Utils/DbUtil" + "dsSupport/models" + "encoding/json" + "fmt" + "github.com/olivere/elastic/v7" + "github.com/tidwall/gjson" + "log" + "os" + "time" +) + +// 深度分页 +// https://www.cnblogs.com/hello-shf/p/11543453.html + +var client elastic.Client +var db = DbUtil.Engine + +func main() { + var host = "http://10.10.14.188:9200/" + esClient, err := elastic.NewClient( + elastic.SetURL(host), + elastic.SetSniff(false), + elastic.SetHealthcheckInterval(10*time.Second), + elastic.SetGzip(true), + elastic.SetErrorLog(log.New(os.Stderr, "ELASTIC ", log.LstdFlags)), + elastic.SetInfoLog(log.New(os.Stdout, "", log.LstdFlags)), + ) + + if err != nil { + panic(err) + } + + //要同步的索引名称，也就是表名称 + indexName := "user_student" + //取所有 + CTX := context.Background() + result, err := esClient.Scroll().Index(indexName).Size(100).Do(CTX) + if err != nil { + panic(err) + } + //第一次的结果集 + for i := range result.Hits.Hits { + resByte, _ := json.Marshal(result.Hits.Hits[i].Source) + resStr := string(resByte) + value := gjson.Get(resStr, "data_content") + //如果是第一次的话而且表不存在的情况下，重新创建表结构 + m := make(map[string]interface{}) + json.Unmarshal([]byte(value.String()), &m) + if len(m) > 0 && i == len(result.Hits.Hits)-1 { + addFieldData(indexName, m) + break + } + } +} + +/** +功能：创建一张INDEX对应的表 +*/ +func addFieldData(tableName string, m map[string]interface{}) { + //换算table_id + sql := `select table_id from t_dw_table where table_name=?` + list, _ := db.SQL(sql, tableName).Query().List() + tableId := list[0]["table_id"].(int64) + //先删除后插入 + sql = `delete from t_dw_table_field where table_id=?` + db.SQL(sql, tableId).Execute() + //准备插入 + for k, v := range m { + model := new(models.TDwTableField) + model.TableId = int32(tableId) + model.FieldName = k + switch v.(type) { + case float64: + model.DataType = "int" + model.FieldLength = 4 + model.DecimalPointLength = 0 + case string: + model.DataType = "varchar" + model.FieldLength = 128 + model.DecimalPointLength = 0 + } + _, err := db.Insert(model) + if err != nil { + panic(err) + } + } + fmt.Println("完成现有Es Mapping的结构反向初始化工作，将手工修改表t_dw_table_field中数据！尤其是主键，一定要设置啊！") +} diff --git a/dsSupport/Utils/CommonUtil/CommonUtil.go b/dsSupport/Utils/CommonUtil/CommonUtil.go index d07f1eac..fa7669c7 100644 --- a/dsSupport/Utils/CommonUtil/CommonUtil.go +++ b/dsSupport/Utils/CommonUtil/CommonUtil.go @@ -363,6 +363,25 @@ func ConvertIntegerArrayToStringArray(nums []int) []string { func ConvertInt64ToString(int64 int64) string { return strconv.FormatInt(int64, 10) } +/** +功能：数组去重 + */ +func RemoveRepeatedElement(arr []string) (newArr []string) { + newArr = make([]string, 0) + for i := 0; i < len(arr); i++ { + repeat := false + for j := i + 1; j < len(arr); j++ { + if arr[i] == arr[j] { + repeat = true + break + } + } + if !repeat { + newArr = append(newArr, arr[i]) + } + } + return +} /** 功能：将int转化为string diff --git a/dsSupport/Utils/PgUtil/PgUtil.go b/dsSupport/Utils/PgUtil/PgUtil.go new file mode 100644 index 00000000..eaac874c --- /dev/null +++ b/dsSupport/Utils/PgUtil/PgUtil.go @@ -0,0 +1,44 @@ +package PgUtil + +import ( + "dsSupport/Const/ErrorConst" + "dsSupport/Utils/LogUtil" + "fmt" + _ "github.com/lib/pq" + "github.com/xormplus/core" + "github.com/xormplus/xorm" + "github.com/xormplus/xorm/log" + "os" + "time" +) + +var Engine *xorm.Engine +var err error + +func init() { + //postgresql + psqlInfo := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", "10.10.14.107", 5432, "dsideal", "DsideaL147258369", "base_db_dev") + //格式 + Engine, err = xorm.NewEngine("postgres", psqlInfo) + if err != nil { + LogUtil.Error(ErrorConst.SqlQueryError, err.Error()) + } + //设置数据库连接池 + Engine.SetMaxIdleConns(200) //设置连接池中的保持连接的最大连接数。 + Engine.SetMaxOpenConns(200) //设置打开数据库的最大连接数，包含正在使用的连接和连接池的连接。 + Engine.SetConnMaxLifetime(time.Minute * 5) + //设置数据时区 + var location *time.Location + location, _ = time.LoadLocation("Asia/Shanghai") + Engine.TZLocation = location + //与 struct的映射方式，这里采用蛇型方法，默认是蛇形 + Engine.SetTableMapper(core.SnakeMapper{}) + + //显示+记录SQL日志 + f, _ := os.Create("./Logs/pg_sql.log") + + logger := log.NewSimpleLogger(f) + logger.SetLevel(log.LOG_DEBUG) + Engine.SetLogger(log.NewLoggerAdapter(logger)) + Engine.ShowSQL(true) // 则会在控制台打印出生成的SQL语句 +} diff --git a/dsSupport/go.mod b/dsSupport/go.mod index 77ac3328..499c2e9e 100644 --- a/dsSupport/go.mod +++ b/dsSupport/go.mod @@ -18,6 +18,7 @@ require ( github.com/go-sql-driver/mysql v1.5.0 github.com/golang/protobuf v1.4.1 github.com/golang/snappy v0.0.1 // indirect + github.com/lib/pq v1.8.0 github.com/oklog/ulid v1.3.1 github.com/olivere/elastic/v7 v7.0.20 github.com/pkg/sftp v1.12.0 @@ -26,6 +27,7 @@ require ( github.com/swaggo/gin-swagger v1.2.0 github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tealeg/xlsx v1.0.5 // indirect + github.com/tidwall/gjson v1.6.1 github.com/xormplus/builder v0.0.0-20200331055651-240ff40009be github.com/xormplus/core v0.0.0-20200308074340-f3bce19d5f31 github.com/xormplus/xorm v0.0.0-20200731130200-6811f3bde592 diff --git a/dsSupport/go.sum b/dsSupport/go.sum index 24781677..465f75cc 100644 --- a/dsSupport/go.sum +++ b/dsSupport/go.sum @@ -105,6 +105,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= @@ -155,6 +157,12 @@ github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFd github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tealeg/xlsx v1.0.5 h1:+f8oFmvY8Gw1iUXzPk+kz+4GpbDZPK1FhPiQRd+ypgE= github.com/tealeg/xlsx v1.0.5/go.mod h1:btRS8dz54TDnvKNosuAqxrM1QgN1udgk9O34bDCnORM= +github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws= +github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= +github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= +github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= +github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= diff --git a/dsSupport/models/t_dw_table.go b/dsSupport/models/t_dw_table.go new file mode 100644 index 00000000..bc01a486 --- /dev/null +++ b/dsSupport/models/t_dw_table.go @@ -0,0 +1,6 @@ +package models + +type TDwTable struct { + TableId int32 `xorm:"not null pk autoincr INT(255)"` + TableName string `xorm:"not null VARCHAR(255)"` +} diff --git a/dsSupport/models/t_dw_table_field.go b/dsSupport/models/t_dw_table_field.go new file mode 100644 index 00000000..a6b485a3 --- /dev/null +++ b/dsSupport/models/t_dw_table_field.go @@ -0,0 +1,12 @@ +package models + +type TDwTableField struct { + Id int32 `xorm:"not null pk autoincr INT(11)"` + TableId int32 `xorm:"not null comment('哪张表') index INT(11)"` + FieldName string `xorm:"not null comment('字段名称') VARCHAR(255)"` + DataType string `xorm:"not null comment('数据类型') VARCHAR(255)"` + FieldLength int32 `xorm:"not null comment('字段长度') INT(11)"` + DecimalPointLength int32 `xorm:"not null default 0 comment('小数点后长度') INT(11)"` + Comment string `xorm:"default 'NULL' comment('描述') VARCHAR(255)"` + IsPk int32 `xorm:"not null default 0 comment('是否为主键') INT(255)"` +}