diff --git a/dsSupport/Test/ElasticsearchToGreenPlum/ElasticsearchToGreenPlum.go b/dsSupport/Test/ElasticsearchToGreenPlum/ElasticsearchToGreenPlum.go index 8cd89d38..e4713f72 100644 --- a/dsSupport/Test/ElasticsearchToGreenPlum/ElasticsearchToGreenPlum.go +++ b/dsSupport/Test/ElasticsearchToGreenPlum/ElasticsearchToGreenPlum.go @@ -30,12 +30,7 @@ var insertStrArray = make([]string, 0) var pkStrArray = make([]string, 0) //缓存表的结构 -type CacheFieldStruct struct { - cacheFieldKeys []string - tableId int -} - -var CacheFieldBean CacheFieldStruct +var cacheFieldKeys []string func main() { //TODO @@ -61,6 +56,10 @@ func main() { indexName := tableList[i]["table_name"].(string) //预热数据表的列名 getFields(int(tableId)) + //清空重新导入 + sql = "truncate table " + indexName + pgDb.SQL(sql).Execute() + // 当前表的主键是什么,目前只支持单业务主键,复合主键的不支持 sql = `select field_name from t_dw_table_field where table_id=? and is_pk=1` list, _ := mysqlDb.SQL(sql, tableId).Query().List() @@ -111,7 +110,7 @@ func main() { //批量执行 batchSave(indexName, pk) } - fmt.Println("索引"+indexName+"同步完成!") + fmt.Println("索引" + indexName + "同步完成!") } } @@ -122,16 +121,15 @@ func main() { */ func getFields(tableId int) { //清空 - CacheFieldBean.cacheFieldKeys = CacheFieldBean.cacheFieldKeys[:] + cacheFieldKeys = cacheFieldKeys[0:0] //填充 sql := `select field_name from t_dw_table_field where table_id=? order by sort_id,field_name` list, _ := mysqlDb.SQL(sql, tableId).Query().List() for i := range list { fieldName := list[i]["field_name"].(string) - CacheFieldBean.cacheFieldKeys = append(CacheFieldBean.cacheFieldKeys, fieldName) + cacheFieldKeys = append(cacheFieldKeys, fieldName) } - CacheFieldBean.tableId = tableId } //后续优化为到达一定的阀值再提交一次,初步定为200一次 @@ -152,7 +150,7 @@ func addRecord(pk string, jsonStr string) { //另一个任务是组装insertStrArray,准备批量提交 var lineSql = "" //根据key从m中拿元素,就是按顺序拿了 - for _, k := range CacheFieldBean.cacheFieldKeys { + for _, k := range cacheFieldKeys { switch mymap[k].(type) { case string: lineSql += "'" + mymap[k].(string) + "'," @@ -190,7 +188,7 @@ func batchSave(tableName string, pkName string) { //插入 sql = `insert into ` + tableName + "(" //根据key从m中拿元素,就是按顺序拿了 - for _, k := range CacheFieldBean.cacheFieldKeys { + for _, k := range cacheFieldKeys { sql += k + "," } sql += "uuid"