|
|
|
@ -32,10 +32,20 @@ type tableStruct struct {
|
|
|
|
|
DataSource string `json:"data_source"`
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
//有同步哪些表,之所以不遍历文件的名称进行上报,是因为需要控制上传的顺序,如果只是文件名,就丢失了顺序
|
|
|
|
|
var sqlDict = []tableStruct{
|
|
|
|
|
//有同步哪些表(增量),之所以不遍历文件的名称进行上报,是因为需要控制上传的顺序,如果只是文件名,就丢失了顺序
|
|
|
|
|
var IncrSqlDict = []tableStruct{
|
|
|
|
|
{TableName: "t_base_organization", PrimaryKey: "org_id", DataSource: "org_school"},
|
|
|
|
|
{TableName: "t_base_class", PrimaryKey: "class_id", DataSource: "org_class"},
|
|
|
|
|
{TableName: "t_base_teacher", PrimaryKey: "teacher_id", DataSource: "user_teacher"},
|
|
|
|
|
{TableName: "t_base_student", PrimaryKey: "student_id", DataSource: "user_student"},
|
|
|
|
|
{TableName: "t_sys_loginperson_log", PrimaryKey: "id", DataSource: "log_login"},
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 全量数据上报
|
|
|
|
|
var FullSqlDict = []tableStruct{
|
|
|
|
|
{TableName: "t_base_organization", PrimaryKey: "org_id", DataSource: "org_school"},
|
|
|
|
|
{TableName: "t_sys_dict", PrimaryKey: "dict_id", DataSource: "sys_dic"},
|
|
|
|
|
{TableName: "t_gov_area", PrimaryKey: "area_code", DataSource: "org_area"},
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 数据上报的结构体
|
|
|
|
@ -67,72 +77,77 @@ func init() {
|
|
|
|
|
os.MkdirAll(progressFilePath, os.ModePerm)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
功能:组织机构上报
|
|
|
|
|
功能:全量数据上报
|
|
|
|
|
作者:黄海
|
|
|
|
|
时间:2020-07-17
|
|
|
|
|
*/
|
|
|
|
|
func InitOrg() {
|
|
|
|
|
//(1)是不是进行过首次上报,如果没有话,需要执行一次组织机构上报
|
|
|
|
|
logName := progressFilePath + "t_base_organization.log"
|
|
|
|
|
//判断文件是不是存在
|
|
|
|
|
if !FileUtil.PathExists(logName) {
|
|
|
|
|
//上报组织机构,读取t_base_organization的SQL脚本
|
|
|
|
|
//SQL内容
|
|
|
|
|
sql := FileUtil.ReadFileContent("./Sql/t_base_organization.sql")
|
|
|
|
|
//组织机构是需要按org_type,area_code排序
|
|
|
|
|
orgSql := sql + " order by t1.org_type,t1.area_code"
|
|
|
|
|
list, _ := db.SQL(orgSql).Query().List()
|
|
|
|
|
*/
|
|
|
|
|
func InitFull() {
|
|
|
|
|
for i := range FullSqlDict {
|
|
|
|
|
//(1)是不是进行过首次上报,如果没有话,需要执行一次组织机构上报
|
|
|
|
|
logName := progressFilePath + FullSqlDict[i].TableName + ".log"
|
|
|
|
|
//判断文件是不是存在
|
|
|
|
|
if !FileUtil.PathExists(logName) {
|
|
|
|
|
//SQL内容
|
|
|
|
|
sql := FileUtil.ReadFileContent("./Sql/" + FullSqlDict[i].TableName + ".sql")
|
|
|
|
|
|
|
|
|
|
var t tableStruct
|
|
|
|
|
for i := range sqlDict {
|
|
|
|
|
if sqlDict[i].TableName == "t_base_organization" {
|
|
|
|
|
t = sqlDict[i]
|
|
|
|
|
break
|
|
|
|
|
var list []map[string]interface{}
|
|
|
|
|
//如果是组织机构表,那么需要变更一下查询的排序条件
|
|
|
|
|
if FullSqlDict[i].TableName == "t_base_organization" {
|
|
|
|
|
//组织机构是需要按org_type,area_code排序
|
|
|
|
|
sql = sql + " order by t1.org_type,t1.area_code"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
var count = 0
|
|
|
|
|
var isFinish = false
|
|
|
|
|
for {
|
|
|
|
|
if isFinish {
|
|
|
|
|
break
|
|
|
|
|
list, _ = db.SQL(sql).Query().List()
|
|
|
|
|
|
|
|
|
|
var count = 0
|
|
|
|
|
var isFinish = false
|
|
|
|
|
for {
|
|
|
|
|
if isFinish {
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
//利用切片分批次上报
|
|
|
|
|
if len(list) > limit {
|
|
|
|
|
success := PostToServer(FullSqlDict[i], list[0:limit]) //0-99不包含100
|
|
|
|
|
if !success {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
count = count + limit
|
|
|
|
|
list = list[limit:]
|
|
|
|
|
} else if len(list) > 0 {
|
|
|
|
|
success := PostToServer(FullSqlDict[i], list)
|
|
|
|
|
if !success {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
count = count + len(list)
|
|
|
|
|
isFinish = true
|
|
|
|
|
} else {
|
|
|
|
|
isFinish = true
|
|
|
|
|
}
|
|
|
|
|
fmt.Println(CommonUtil.GetCurrentTime() + " 同步:成功完成" + FullSqlDict[i].TableName + "初始化上报,本次完成" + CommonUtil.ConvertIntToString(count) + "条!")
|
|
|
|
|
}
|
|
|
|
|
//利用切片分批次上报
|
|
|
|
|
if len(list) > limit {
|
|
|
|
|
success := PostToServer(t, list[0:limit]) //0-99不包含100
|
|
|
|
|
if !success {
|
|
|
|
|
continue
|
|
|
|
|
//对于组织机构进行特殊处理
|
|
|
|
|
if FullSqlDict[i].TableName == "t_base_organization" {
|
|
|
|
|
//记录日志
|
|
|
|
|
maxSql := sql + " order by t1.last_updated_time desc,t1.id_int desc limit 1"
|
|
|
|
|
var l logStruct
|
|
|
|
|
l.IdInt = 0
|
|
|
|
|
l.StartUpdateTs = defaultStartTs
|
|
|
|
|
//取得最后一行的最大值
|
|
|
|
|
list, _ = db.SQL(maxSql).Query().List()
|
|
|
|
|
if len(list) > 0 {
|
|
|
|
|
l.StartUpdateTs = list[len(list)-1]["last_updated_time"].(string)
|
|
|
|
|
l.IdInt = list[len(list)-1]["id_int"].(int64)
|
|
|
|
|
}
|
|
|
|
|
count = count + limit
|
|
|
|
|
list = list[limit:]
|
|
|
|
|
} else if len(list) > 0 {
|
|
|
|
|
success := PostToServer(t, list)
|
|
|
|
|
if !success {
|
|
|
|
|
continue
|
|
|
|
|
jsonBytes, err := json.Marshal(l)
|
|
|
|
|
if err != nil {
|
|
|
|
|
fmt.Println(err.Error())
|
|
|
|
|
}
|
|
|
|
|
count = count + len(list)
|
|
|
|
|
isFinish = true
|
|
|
|
|
FileUtil.WriteContent(logName, string(jsonBytes))
|
|
|
|
|
} else {
|
|
|
|
|
isFinish = true
|
|
|
|
|
FileUtil.WriteContent(logName, "is finished!")
|
|
|
|
|
}
|
|
|
|
|
fmt.Println(CommonUtil.GetCurrentTime() + " 同步:成功完成组织机构初始化上报,本次完成" + CommonUtil.ConvertIntToString(count) + "条!")
|
|
|
|
|
}
|
|
|
|
|
//记录日志
|
|
|
|
|
maxSql:=sql+" order by t1.last_updated_time,t1.id_int desc limit 1"
|
|
|
|
|
var l logStruct
|
|
|
|
|
l.IdInt = 0
|
|
|
|
|
l.StartUpdateTs = defaultStartTs
|
|
|
|
|
//取得最后一行的最大值
|
|
|
|
|
list,_=db.SQL(maxSql).Query().List()
|
|
|
|
|
if len(list) > 0 {
|
|
|
|
|
l.StartUpdateTs = list[len(list)-1]["last_updated_time"].(string)
|
|
|
|
|
l.IdInt = list[len(list)-1]["id_int"].(int64)
|
|
|
|
|
}
|
|
|
|
|
jsonBytes, err := json.Marshal(l)
|
|
|
|
|
if err != nil {
|
|
|
|
|
fmt.Println(err.Error())
|
|
|
|
|
}
|
|
|
|
|
FileUtil.WriteContent(logName, string(jsonBytes))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@ -145,7 +160,8 @@ func DataExchange() {
|
|
|
|
|
//死循环上报中
|
|
|
|
|
for {
|
|
|
|
|
//(1)组织机构上报
|
|
|
|
|
InitOrg()
|
|
|
|
|
InitFull()
|
|
|
|
|
|
|
|
|
|
//(2)本轮上报的数量,如果是0,休息5秒后再继续上传
|
|
|
|
|
postCount := UploadData()
|
|
|
|
|
if postCount == 0 {
|
|
|
|
@ -163,7 +179,7 @@ func DataExchange() {
|
|
|
|
|
func UploadData() int {
|
|
|
|
|
var postCount = 0
|
|
|
|
|
//遍历所有的配置节,进行循环
|
|
|
|
|
for i := range sqlDict {
|
|
|
|
|
for i := range IncrSqlDict {
|
|
|
|
|
count := 0
|
|
|
|
|
//默认的开始时间
|
|
|
|
|
startUpdateTs := defaultStartTs
|
|
|
|
@ -172,7 +188,7 @@ func UploadData() int {
|
|
|
|
|
//查询结果集
|
|
|
|
|
var list []map[string]interface{}
|
|
|
|
|
//表名
|
|
|
|
|
tableName := sqlDict[i].TableName
|
|
|
|
|
tableName := IncrSqlDict[i].TableName
|
|
|
|
|
//日志文件位置
|
|
|
|
|
logName := progressFilePath + tableName + ".log"
|
|
|
|
|
//判断文件是不是存在
|
|
|
|
@ -194,7 +210,7 @@ func UploadData() int {
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
//上报到Http Api--->Body--->Post
|
|
|
|
|
success := PostToServer(sqlDict[i], list)
|
|
|
|
|
success := PostToServer(IncrSqlDict[i], list)
|
|
|
|
|
if !success {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
@ -229,12 +245,18 @@ func PostToServer(t tableStruct, list []map[string]interface{}) bool {
|
|
|
|
|
ps.DataSource = t.DataSource
|
|
|
|
|
ps.AuthToken = ConfigUtil.DataExchangeAuthToken
|
|
|
|
|
ps.SystemId = ConfigUtil.DataExchangeSystemId
|
|
|
|
|
|
|
|
|
|
var dsMap = make([]dataStruct, 0)
|
|
|
|
|
for k := range list {
|
|
|
|
|
var ds dataStruct
|
|
|
|
|
ds.Data, _ = CommonUtil.MapToJson(list[k])
|
|
|
|
|
ds.DataId = list[k][t.PrimaryKey].(string)
|
|
|
|
|
switch list[k][t.PrimaryKey].(type) {
|
|
|
|
|
case int64:
|
|
|
|
|
ds.DataId = CommonUtil.ConvertInt64ToString(list[k][t.PrimaryKey].(int64))
|
|
|
|
|
break
|
|
|
|
|
default:
|
|
|
|
|
ds.DataId = list[k][t.PrimaryKey].(string)
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
ds.DelFlag = list[k]["del_flag"].(int64)
|
|
|
|
|
ds.OrgId = list[k]["bureau_id"].(string)
|
|
|
|
|
dsMap = append(dsMap, ds)
|
|
|
|
@ -245,7 +267,7 @@ func PostToServer(t tableStruct, list []map[string]interface{}) bool {
|
|
|
|
|
msg := string(jsonBytes)
|
|
|
|
|
//提交到汇集中心
|
|
|
|
|
p := httpDo("POST", ConfigUtil.DataExchangeUrl, msg)
|
|
|
|
|
if !p.Success{
|
|
|
|
|
if !p.Success {
|
|
|
|
|
fmt.Println(CommonUtil.GetCurrentTime() + " 同步:上报到数据汇集中心失败,将休息5秒后重试!错误原因:" + p.Message)
|
|
|
|
|
time.Sleep(5 * 1e9)
|
|
|
|
|
}
|
|
|
|
@ -270,7 +292,7 @@ func getRecord(tableName string, lastUpdatedTime string, idInt int64) (string, i
|
|
|
|
|
var rsGt = make([]map[string]interface{}, 0)
|
|
|
|
|
lastUpdatedTime, idInt, rsEq = getRecordEq(eqSql, lastUpdatedTime, idInt, limit)
|
|
|
|
|
|
|
|
|
|
if len(rsEq) < limit {
|
|
|
|
|
if rsEq == nil || len(rsEq) < limit {
|
|
|
|
|
//尝试一次gt操作(组合一下)
|
|
|
|
|
lastUpdatedTime, idInt, rsGt = getRecordGt(gtSql, lastUpdatedTime, idInt, limit-len(rsEq))
|
|
|
|
|
if len(rsGt) > 0 {
|
|
|
|
|