diff --git a/dsBaseRpc/models/t_app_base.go b/dsBaseRpc/models/t_app_base.go index 4a63ff48..4e608af8 100644 --- a/dsBaseRpc/models/t_app_base.go +++ b/dsBaseRpc/models/t_app_base.go @@ -6,6 +6,7 @@ type TAppBase struct { AppName string `xorm:"not null comment('系统名称') VARCHAR(100)"` AccessKey string `xorm:"not null comment('AK') CHAR(20)"` SecretKey string `xorm:"not null comment('SK') CHAR(26)"` + AppToken string `xorm:"default 'NULL' comment('系统票据') CHAR(100)"` AppUrl string `xorm:"default 'NULL' comment('系统集成页面调用地址') VARCHAR(1024)"` AppIcon string `xorm:"default 'NULL' comment('系统图标的路径') VARCHAR(1024)"` RedirectUri string `xorm:"default 'NULL' comment('统一认证回调地址') VARCHAR(1024)"` diff --git a/dsBaseRpc/models/t_base_organization_log.go b/dsBaseRpc/models/t_base_organization_log.go index a48e3df0..148ce0a8 100644 --- a/dsBaseRpc/models/t_base_organization_log.go +++ b/dsBaseRpc/models/t_base_organization_log.go @@ -12,7 +12,7 @@ type TBaseOrganizationLog struct { MainSchoolType int32 `xorm:"not null default 1 comment('主校类型，1：普通校，2：主校，3：分校') INT(11)"` MainSchoolId string `xorm:"not null default '''' comment('如果是分校，所属主校的ID,如果是主校，是自己的ID，如果是分校，是主校ID，如果是普通校，也是自己的ID') CHAR(36)"` Xxbxlxm string `xorm:"not null default '''' comment('学校办学类型（学校专有属性）有字典') CHAR(3)"` - Szdcxlxm string `xorm:"not null default '''' comment('学校城乡类型（学校专有属性）有字典') CHAR(2)"` + Szdcxlxm string `xorm:"not null default '''' comment('学校城乡类型（学校专有属性）有字典') CHAR(3)"` Xxjbzm string `xorm:"not null default '''' comment('学校举办者（学校专有属性）有字典') CHAR(3)"` Fzr string `xorm:"not null default '''' comment('机构负责人ID（*一个单位只能有一个负责人）') CHAR(36)"` Fddbr string `xorm:"not null default '''' comment('机构法定代表人') VARCHAR(50)"` diff --git a/dsBaseRpc/models/t_base_teacher_position.go b/dsBaseRpc/models/t_base_teacher_position.go index 08d2db18..63f50e08 100644 --- a/dsBaseRpc/models/t_base_teacher_position.go +++ b/dsBaseRpc/models/t_base_teacher_position.go @@ -2,7 +2,7 @@ package models type TBaseTeacherPosition struct { Id string `xorm:"not null pk comment('主键') CHAR(36)"` - PersonId string `xorm:"not null default '''' comment('人员ID') index(person_id) CHAR(18)"` + PersonId string `xorm:"not null default '''' comment('人员ID') index(person_id) CHAR(36)"` BureauId string `xorm:"not null comment('所在单位ID') index(person_id) CHAR(36)"` PositionId string `xorm:"not null default ''0'' comment('职务ID') CHAR(36)"` BUse int32 `xorm:"not null default 1 comment('是否可用 1：可用 -2：不可用') index(person_id) TINYINT(1)"` diff --git a/dsBaseRpc/models/t_dataex_dataaccess.go b/dsBaseRpc/models/t_dataex_dataaccess.go index 1346808b..ae2bb76b 100644 --- a/dsBaseRpc/models/t_dataex_dataaccess.go +++ b/dsBaseRpc/models/t_dataex_dataaccess.go @@ -8,7 +8,7 @@ type TDataexDataaccess struct { Id string `xorm:"not null pk comment('ID') VARCHAR(36)"` DatasourceId string `xorm:"not null comment('数据源ID') index VARCHAR(36)"` DatasourceCode string `xorm:"default 'NULL' VARCHAR(255)"` - ConsumeSystemid string `xorm:"not null comment('数据使用系统ID') index VARCHAR(36)"` + ConsumeSystemid string `xorm:"default 'NULL' comment('数据使用系统ID') index CHAR(36)"` QueryFlag int32 `xorm:"not null default 1 comment('可查【1：是，-1：否】') INT(11)"` SetFlag int32 `xorm:"not null default -1 comment('可修改【1：是，-1：否】') INT(11)"` ConsumeType int32 `xorm:"not null comment('使用数据范围【1：本机构，2：本机构以及下属机构，-1：不限制】') INT(11)"` diff --git a/dsBaseRpc/models/t_dataex_error.go b/dsBaseRpc/models/t_dataex_error.go index e1daba47..2ecdf23a 100644 --- a/dsBaseRpc/models/t_dataex_error.go +++ b/dsBaseRpc/models/t_dataex_error.go @@ -8,10 +8,11 @@ type TDataexError struct { Id string `xorm:"not null pk comment('ID') VARCHAR(36)"` SystemId string `xorm:"not null comment('数据提供系统ID【输入参数】') VARCHAR(36)"` DatasourceId string `xorm:"not null comment('数据源ID【输入参数】') VARCHAR(36)"` - OrgId string `xorm:"not null comment('数据机构ID【输入参数】') VARCHAR(36)"` - DataId string `xorm:"not null comment('数据ID【输入参数】') VARCHAR(36)"` + OrgId string `xorm:"default 'NULL' comment('数据机构ID【输入参数】') VARCHAR(36)"` + DataId string `xorm:"default 'NULL' comment('数据ID【输入参数】') VARCHAR(36)"` DataContent string `xorm:"default 'NULL' comment('数据内容【Json格式输入参数】') LONGTEXT"` FileUri string `xorm:"default 'NULL' comment('文件地址【文件交换】') VARCHAR(500)"` + ErrorInfo string `xorm:"default 'NULL' comment('错误信息说明') VARCHAR(5000)"` ChangeTime time.Time `xorm:"default 'NULL' comment('最近修改时间') DATETIME"` DeleteTime time.Time `xorm:"default 'NULL' comment('删除时间') DATETIME"` EnableFlag int32 `xorm:"not null default 1 comment('启用标志【默认1，1：启用，-1：禁用】') INT(11)"` diff --git a/dsBaseRpc/models/t_dataex_jyt2012.go b/dsBaseRpc/models/t_dataex_jyt2012.go index 47c395d0..515e6e94 100644 --- a/dsBaseRpc/models/t_dataex_jyt2012.go +++ b/dsBaseRpc/models/t_dataex_jyt2012.go @@ -7,6 +7,7 @@ import ( type TDataexJyt2012 struct { Id string `xorm:"not null pk comment('ID') VARCHAR(36)"` DicName string `xorm:"not null comment('字典/字典项名称') VARCHAR(100)"` + DicValue string `xorm:"not null VARCHAR(100)"` DicType int32 `xorm:"not null default 1 comment('字典类型【1：国标数据、关联数据源，2：国标字典、关联元数据】') INT(11)"` DicInfo string `xorm:"default ''NULL'' comment('字典相信说明') VARCHAR(500)"` RootFlag int32 `xorm:"not null default -1 comment('是否是字典【1：是，-1：否】') INT(11)"` diff --git a/dsBaseRpc/models/t_dw_table.go b/dsBaseRpc/models/t_dw_table.go new file mode 100644 index 00000000..bc01a486 --- /dev/null +++ b/dsBaseRpc/models/t_dw_table.go @@ -0,0 +1,6 @@ +package models + +type TDwTable struct { + TableId int32 `xorm:"not null pk autoincr INT(255)"` + TableName string `xorm:"not null VARCHAR(255)"` +} diff --git a/dsBaseRpc/models/t_dw_table_field.go b/dsBaseRpc/models/t_dw_table_field.go new file mode 100644 index 00000000..7d73919c --- /dev/null +++ b/dsBaseRpc/models/t_dw_table_field.go @@ -0,0 +1,13 @@ +package models + +type TDwTableField struct { + Id int32 `xorm:"not null pk autoincr INT(11)"` + TableId int32 `xorm:"not null comment('哪张表') index INT(11)"` + FieldName string `xorm:"not null comment('字段名称') VARCHAR(255)"` + DataType string `xorm:"not null comment('数据类型') VARCHAR(255)"` + FieldLength int32 `xorm:"not null comment('字段长度') INT(11)"` + DecimalPointLength int32 `xorm:"not null default 0 comment('小数点后长度') INT(11)"` + Comment string `xorm:"default 'NULL' comment('描述') VARCHAR(255)"` + IsPk int32 `xorm:"not null default 0 comment('是否为主键') INT(255)"` + IsNull int32 `xorm:"not null comment('是否可为空') INT(255)"` +} diff --git a/dsSupport/Test/CreatePgTable/CreatePgTable.go b/dsSupport/Test/CreatePgTable/CreatePgTable.go index 5584c509..d11a8eec 100644 --- a/dsSupport/Test/CreatePgTable/CreatePgTable.go +++ b/dsSupport/Test/CreatePgTable/CreatePgTable.go @@ -3,10 +3,12 @@ package main import ( "dsSupport/Utils/CommonUtil" "dsSupport/Utils/DbUtil" + "dsSupport/Utils/PgUtil" "fmt" ) var db = DbUtil.Engine +var pgDb = PgUtil.Engine func main() { //1、读取每一张需要创建的表 @@ -17,7 +19,9 @@ func main() { tableId := list[i]["table_id"].(int64) tableName := list[i]["table_name"].(string) createTableSql := `DROP TABLE IF EXISTS "public"."` + tableName + `";` + "\r\n" - createTableSql += `CREATE TABLE "public"."` + tableName + `" (` + "\r\n" + pgDb.SQL(createTableSql).Execute() + + createTableSql = `CREATE TABLE "public"."` + tableName + `" (` + "\r\n" //2、根据表名，获取相应的创建表的信息 sql = `select * from t_dw_table_field where table_id=?` list2, _ := db.SQL(sql, tableId).Query().List() @@ -35,20 +39,16 @@ func main() { if isNull == 0 { c += " NOT NULL," } else { - if i2 < len(list2)-1 { c += "," - } } createTableSql += `"` + fieldName + `" ` + c + "\r\n" } + createTableSql=createTableSql[0:len(createTableSql)-3] createTableSql += `);` + "\r\n" - //添加生成字段描述信息 - for i2 := range list2 { - fieldName := list2[i2]["field_name"].(string) - comment := list2[i2]["comment"] - if comment != nil && len(comment.(string)) > 0 { - createTableSql += `COMMENT ON COLUMN "public"."` + tableName + `"."` + fieldName + `" IS '` + comment.(string) + `';` + "\r\n" - } + _, err := pgDb.SQL(createTableSql).Execute() + if err != nil { + fmt.Println(createTableSql) + panic(err.Error()) } //主键有哪些 var pks = "" @@ -62,9 +62,12 @@ func main() { //去掉最后一个逗号 pks = pks[0 : len(pks)-1] //添加主键描述 - createTableSql += `-- 设置主键` + "\r\n" - createTableSql += `ALTER TABLE "public"."` + tableName + `" ADD CONSTRAINT "` + tableName + `_dw_pkey" PRIMARY KEY ("` + pks + `");` + "\r\n" + createTableSql = `ALTER TABLE "public"."` + tableName + `" ADD CONSTRAINT "` + tableName + `_dw_pkey" PRIMARY KEY ("` + pks + `");` + "\r\n" + _, err = pgDb.SQL(createTableSql).Execute() + if err != nil { + panic(err.Error()) + } - fmt.Println(createTableSql) + fmt.Println("恭喜，数据仓库中相应表格创建完毕！") } } diff --git a/dsSupport/Test/Elasticsearch/ReadElasticsearchDynamicTable.go b/dsSupport/Test/Elasticsearch/ReadElasticsearchDynamicTable.go deleted file mode 100644 index 52c23421..00000000 --- a/dsSupport/Test/Elasticsearch/ReadElasticsearchDynamicTable.go +++ /dev/null @@ -1,110 +0,0 @@ -package main - -import ( - "context" - "dsSupport/Utils/GpUtil" - "encoding/json" - "fmt" - "github.com/olivere/elastic/v7" - "github.com/tidwall/gjson" - "log" - "os" - "time" -) - -// 深度分页 -// https://www.cnblogs.com/hello-shf/p/11543453.html - -var client elastic.Client -var db = GpUtil.Engine - -func main() { - var host = "http://10.10.14.188:9200/" - esClient, err := elastic.NewClient( - elastic.SetURL(host), - elastic.SetSniff(false), - elastic.SetHealthcheckInterval(10*time.Second), - elastic.SetGzip(true), - elastic.SetErrorLog(log.New(os.Stderr, "ELASTIC ", log.LstdFlags)), - //elastic.SetInfoLog(log.New(os.Stdout, "", log.LstdFlags)), - ) - - if err != nil { - panic(err) - } - - //要同步的索引名称，也就是表名称 - indexName := "org_school" - //取所有 - CTX := context.Background() - result, err := esClient.Scroll().Index(indexName).Size(10).Do(CTX) - if err != nil { - panic(err) - } - //第一次的结果集 - for i := range result.Hits.Hits { - resByte, _ := json.Marshal(result.Hits.Hits[i].Source) - resStr := string(resByte) - value := gjson.Get(resStr, "data_content") - //判断表是不是已存在 - - //如果是第一次的话而且表不存在的情况下，重新创建表结构 - m := make(map[string]interface{}) - json.Unmarshal([]byte(value.String()), &m) - if len(m) > 0 && i == 0 { - CreateTable(indexName, m) - } - } - //继续用的 scoll_id - scrollId := result.ScrollId - - //第一次命中的个数 - var nowCount = int64(len(result.Hits.Hits)) - //总的命中个数 - allCount := result.TotalHits() - - //开始循环 - for { - //如果还有数据没有获取 - if allCount > nowCount { - result, err = esClient.Scroll().ScrollId(scrollId).Do(CTX) - scrollId = result.ScrollId - nowCount += int64(len(result.Hits.Hits)) - - for i := range result.Hits.Hits { - resByte, _ := json.Marshal(result.Hits.Hits[i].Source) - jsonStr := string(resByte) - m := make(map[string]interface{}) - json.Unmarshal([]byte(jsonStr), &m) - for k, v := range m { - fmt.Printf("%v: %v\n", k, v) - } - } - } else { - //没有数据了 - break - } - } -} - -/** -功能：创建一张INDEX对应的表 -*/ -func CreateTable(tableName string, m map[string]interface{}) { - createTableSql := `CREATE TABLE "public"."` + tableName + `" (` + "\r\n" - var c = "" - for k, v := range m { - switch v.(type) { - case float64: - c = "float8" - case string: - c = "varchar(128) " + ` COLLATE "pg_catalog"."default"` - } - createTableSql += `"` + k + `" ` + c + ",\r\n" - } - createTableSql = createTableSql[0 : len(createTableSql)-3] - createTableSql += "\r\n" + `);` + "\r\n" - //创建表 - db.SQL(createTableSql).Execute() - fmt.Println("成功创建表："+tableName+"~") -} diff --git a/dsSupport/Test/ElasticsearchToCsv/ReadElasticsearchToCsv.go b/dsSupport/Test/ElasticsearchToCsv/ReadElasticsearchToCsv.go index 0c872fea..1e49f83b 100644 --- a/dsSupport/Test/ElasticsearchToCsv/ReadElasticsearchToCsv.go +++ b/dsSupport/Test/ElasticsearchToCsv/ReadElasticsearchToCsv.go @@ -2,7 +2,7 @@ package main import ( "context" - "dsSupport/Utils/GpUtil" + "dsSupport/Utils/PgUtil" "encoding/csv" "encoding/json" "fmt" @@ -17,7 +17,7 @@ import ( // https://www.cnblogs.com/hello-shf/p/11543453.html var client elastic.Client -var db = GpUtil.Engine +var db = PgUtil.Engine func main() { var host = "http://10.10.14.188:9200/" @@ -90,9 +90,9 @@ func SaveToCsv(csvFileName string, _map []map[string]interface{}) { // 写入UTF-8 BOM，防止中文乱码 file.WriteString("\xEF\xBB\xBF") w := csv.NewWriter(file) - for i := range _map { - _map[i] - } + //for i := range _map { + // //_map[i] + //} w.Write([]string{"开发者名称", "开发者邮箱", "应用名称"}) // 写文件需要flush，不然缓存满了，后面的就写不进去了，只会写一部分 w.Flush() diff --git a/dsSupport/Test/ReadEsExistMapping/ReadEsExistMapping.go b/dsSupport/Test/ReadEsExistMapping/ReadEsExistMapping.go new file mode 100644 index 00000000..7d6d9054 --- /dev/null +++ b/dsSupport/Test/ReadEsExistMapping/ReadEsExistMapping.go @@ -0,0 +1,89 @@ +package main + +import ( + "context" + "dsSupport/Utils/DbUtil" + "dsSupport/models" + "encoding/json" + "fmt" + "github.com/olivere/elastic/v7" + "github.com/tidwall/gjson" + "log" + "os" + "time" +) + +// 深度分页 +// https://www.cnblogs.com/hello-shf/p/11543453.html + +var client elastic.Client +var db = DbUtil.Engine + +func main() { + var host = "http://10.10.14.188:9200/" + esClient, err := elastic.NewClient( + elastic.SetURL(host), + elastic.SetSniff(false), + elastic.SetHealthcheckInterval(10*time.Second), + elastic.SetGzip(true), + elastic.SetErrorLog(log.New(os.Stderr, "ELASTIC ", log.LstdFlags)), + elastic.SetInfoLog(log.New(os.Stdout, "", log.LstdFlags)), + ) + + if err != nil { + panic(err) + } + + //要同步的索引名称，也就是表名称 + indexName := "org_school" + //取所有 + CTX := context.Background() + result, err := esClient.Scroll().Index(indexName).Size(10).Do(CTX) + if err != nil { + panic(err) + } + //第一次的结果集 + for i := range result.Hits.Hits { + resByte, _ := json.Marshal(result.Hits.Hits[i].Source) + resStr := string(resByte) + value := gjson.Get(resStr, "data_content") + //如果是第一次的话而且表不存在的情况下，重新创建表结构 + m := make(map[string]interface{}) + json.Unmarshal([]byte(value.String()), &m) + if len(m) > 0 && i == 0 { + addFieldData(indexName, m) + break + } + } +} + +/** +功能：创建一张INDEX对应的表 +*/ +func addFieldData(tableName string, m map[string]interface{}) { + //换算table_id + sql := `select table_id from t_dw_table where table_name=?` + list, _ := db.SQL(sql, tableName).Query().List() + tableId := list[0]["table_id"].(int64) + //先删除后插入 + sql = `delete from t_dw_table_field where table_id=?` + db.SQL(sql, tableId).Execute() + //准备插入 + for k, v := range m { + model := new(models.TDwTableField) + model.TableId = int32(tableId) + model.FieldName = k + switch v.(type) { + case float64: + model.DataType = "int" + model.FieldLength = 4 + model.DecimalPointLength = 0 + case string: + model.DataType = "varchar" + model.FieldLength = 128 + model.DecimalPointLength = 0 + } + db.Insert(model) + } + fmt.Println("完成现有Es Mapping的结构反向初始化工作，将手工修改表t_dw_table_field中数据！尤其是主键，一定要设置啊！") +} diff --git a/dsSupport/Utils/GpUtil/GpUtil.go b/dsSupport/Utils/PgUtil/PgUtil.go similarity index 98% rename from dsSupport/Utils/GpUtil/GpUtil.go rename to dsSupport/Utils/PgUtil/PgUtil.go index 7b3e03af..eaac874c 100644 --- a/dsSupport/Utils/GpUtil/GpUtil.go +++ b/dsSupport/Utils/PgUtil/PgUtil.go @@ -1,4 +1,4 @@ -package GpUtil +package PgUtil import ( "dsSupport/Const/ErrorConst" diff --git a/dsSupport/go.mod b/dsSupport/go.mod index c7955465..499c2e9e 100644 --- a/dsSupport/go.mod +++ b/dsSupport/go.mod @@ -27,7 +27,7 @@ require ( github.com/swaggo/gin-swagger v1.2.0 github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tealeg/xlsx v1.0.5 // indirect - github.com/tidwall/gjson v1.6.1 // indirect + github.com/tidwall/gjson v1.6.1 github.com/xormplus/builder v0.0.0-20200331055651-240ff40009be github.com/xormplus/core v0.0.0-20200308074340-f3bce19d5f31 github.com/xormplus/xorm v0.0.0-20200731130200-6811f3bde592 diff --git a/dsSupport/models/t_dw_table.go b/dsSupport/models/t_dw_table.go new file mode 100644 index 00000000..bc01a486 --- /dev/null +++ b/dsSupport/models/t_dw_table.go @@ -0,0 +1,6 @@ +package models + +type TDwTable struct { + TableId int32 `xorm:"not null pk autoincr INT(255)"` + TableName string `xorm:"not null VARCHAR(255)"` +} diff --git a/dsSupport/models/t_dw_table_field.go b/dsSupport/models/t_dw_table_field.go new file mode 100644 index 00000000..7d73919c --- /dev/null +++ b/dsSupport/models/t_dw_table_field.go @@ -0,0 +1,13 @@ +package models + +type TDwTableField struct { + Id int32 `xorm:"not null pk autoincr INT(11)"` + TableId int32 `xorm:"not null comment('哪张表') index INT(11)"` + FieldName string `xorm:"not null comment('字段名称') VARCHAR(255)"` + DataType string `xorm:"not null comment('数据类型') VARCHAR(255)"` + FieldLength int32 `xorm:"not null comment('字段长度') INT(11)"` + DecimalPointLength int32 `xorm:"not null default 0 comment('小数点后长度') INT(11)"` + Comment string `xorm:"default 'NULL' comment('描述') VARCHAR(255)"` + IsPk int32 `xorm:"not null default 0 comment('是否为主键') INT(255)"` + IsNull int32 `xorm:"not null comment('是否可为空') INT(255)"` +}