diff --git a/dsBaseRpc/Sql/t_base_organization.sql b/dsBaseRpc/Sql/t_base_organization.sql index 80b28f1a..e71c5250 100644 --- a/dsBaseRpc/Sql/t_base_organization.sql +++ b/dsBaseRpc/Sql/t_base_organization.sql @@ -1,3 +1,4 @@ +select * from ( SELECT cast( t1.`org_id` AS CHAR ( 36 ) charset utf8 ) AS `org_id`, t1.`id_int` AS `id_int`, @@ -32,5 +33,5 @@ SELECT DATE_FORMAT(t1.`create_time`,'%Y/%m/%d %H:%i:%s') AS `create_time`, case t1.`b_use` when -1 then 1 else 0 end AS `del_flag` FROM - `t_base_organization` as t1 - + `t_base_organization` as t1 where t1.b_use=1 +) as t1 \ No newline at end of file diff --git a/dsBaseRpc/Sql/t_base_teacher.sql b/dsBaseRpc/Sql/t_base_teacher.sql index 489f5848..63b76f3b 100644 --- a/dsBaseRpc/Sql/t_base_teacher.sql +++ b/dsBaseRpc/Sql/t_base_teacher.sql @@ -1,3 +1,4 @@ +select * from ( select cast(t1.`person_id` as char(36) charset utf8) AS `teacher_id`, t1.`identity_id` AS `identity_id`, t1.`id_int` AS `id_int`, @@ -36,4 +37,6 @@ case t1.`b_use` when -1 then 1 else 0 end AS `del_flag`, `t_base_organization`.`xxbbm` AS `xxbbm`, `t_base_organization`.`xxbxlxm` AS `xxbxlxm`, `t_base_organization`.`szdcxlxm` AS `szdcxlxm` -from (`t_base_teacher` as t1 join `t_base_organization` on (t1.`org_id` = `t_base_organization`.`org_id`)) \ No newline at end of file +from (`t_base_teacher` as t1 join `t_base_organization` on (t1.`org_id` = `t_base_organization`.`org_id`)) +where t1.identity_id=2 +) as t1 \ No newline at end of file diff --git a/dsBaseRpc/Test/DataExchange/DataExchange.go b/dsBaseRpc/Test/DataExchange/DataExchange.go new file mode 100644 index 00000000..53f89600 --- /dev/null +++ b/dsBaseRpc/Test/DataExchange/DataExchange.go @@ -0,0 +1,160 @@ +package main + +import ( + "bytes" + "crypto/md5" + "dsBaseRpc/Utils/CommonUtil" + "dsBaseRpc/Utils/ConfigUtil" + "dsBaseRpc/Utils/DbUtil" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" +) + +var db = DbUtil.Engine + +//每次获取的条数 +var limit = 200 + +//默认开始时间 +var defaultStartTs = "1970-01-01 00:00:00" + +// 日志文件路径 +var progressFilePath = "/usr/local/SyncDataLogs/" + +//建立与汇集中心的主题映射关系结构 +type tableStruct struct { + TableName string `json:"table_name"` + PrimaryKey string `json:"primary_key"` + DataSource string `json:"data_source"` +} + +//有同步哪些表(增量),之所以不遍历文件的名称进行上报，是因为需要控制上传的顺序，如果只是文件名，就丢失了顺序 +var IncrSqlDict = []tableStruct{ + {TableName: "t_base_organization", PrimaryKey: "org_id", DataSource: "org_school"}, + {TableName: "t_base_class", PrimaryKey: "class_id", DataSource: "org_class"}, + {TableName: "t_base_teacher", PrimaryKey: "teacher_id", DataSource: "user_teacher"}, + {TableName: "t_base_student", PrimaryKey: "student_id", DataSource: "user_student"}, + {TableName: "t_sys_loginperson_log", PrimaryKey: "id_int", DataSource: "log_login"}, +} + +// 全量数据上报 +var FullSqlDict = []tableStruct{ + {TableName: "t_base_organization", PrimaryKey: "org_id", DataSource: "org_school"}, + {TableName: "t_sys_dict", PrimaryKey: "dict_id", DataSource: "sys_dic"}, + {TableName: "t_gov_area", PrimaryKey: "area_code", DataSource: "org_area"}, +} + +// 数据上报的结构体 +type postStruct struct { + AuthToken string `json:"auth_token"` + DataSource string `json:"data_source"` + SystemId string `json:"system_id"` + OrgId string `json:"org_id"` + QueryPage int `json:"query_page"` + QueryTime string `json:"query_time"` +} + +//系统token +var SystemToken = "" + +//是否成功 +var success = false + +// 获取签名用的结构体 +type authStruct struct { + AuthTime string `json:"auth_time"` + SystemId string `json:"system_id"` + SystemToken string `json:"system_token"` +} + +/** +功能：获取系统token +作者：黄海 +时间：2020-07-22 +*/ +func getSystemToken() (bool, string) { + //(1)计算出system_token=MD5.hash(MD5.hash(system_id+auth_time)+system_key) + var as authStruct + as.AuthTime = CommonUtil.GetCurrentTime() + as.SystemId = ConfigUtil.DataExchangeSystemId + //计算 md5 + w := md5.New() + io.WriteString(w, as.SystemId+as.AuthTime) + //将str写入到w中 + md5str := fmt.Sprintf("%x", w.Sum(nil)) + w = md5.New() + io.WriteString(w, md5str+ConfigUtil.DataExchangeSystemKey) + //将str写入到w中 + md5str = fmt.Sprintf("%x", w.Sum(nil)) + //系统token + as.SystemToken = md5str + + //(2)根据system_token换取authToken + jsonBytes, _ := json.Marshal(as) + + p := httpDo("POST", ConfigUtil.DataExchangeSystemAuthUrl, string(jsonBytes)) + if !p.Success { + fmt.Println(CommonUtil.GetCurrentTime() + "获取认证签名失败!") + return false, "获取认证签名失败!" + } + return true, p.Message +} + +type ResultStruct struct { + Message string `json:"message"` + Success bool `json:"success"` +} + +// 基础方法，这里多用于访问webapi，配合上json转换。 +func httpDo(method string, url string, msg string) ResultStruct { + var p ResultStruct + p.Success = false + p.Message = "上报到汇集系统失败！请检查是否SystemToken是有效的，或者有两个及以上客户端同时在上报。" + client := &http.Client{} + body := bytes.NewBuffer([]byte(msg)) + req, err := http.NewRequest(method, + url, + body) + if err != nil { + // handle error + } + req.Header.Set("Content-Type", "application/json;charset=utf-8") + resp, err := client.Do(req) + if err != nil { + fmt.Println(err) + return p + } + defer resp.Body.Close() + resultBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Println(err) + return p + } + json.Unmarshal(resultBody, &p) + return p +} + +func main() { + //获取系统token + success, SystemToken = getSystemToken() + // 访问地址 + url := `http://fort.edusoa.com:7777/v1/dataex/DataexPage` + var a postStruct + a.AuthToken=SystemToken + a.DataSource="org_school" + a.OrgId="-1" + a.QueryPage=0 + a.QueryTime=CommonUtil.GetCurrentTime() + a.SystemId=ConfigUtil.DataExchangeSystemId + + //(2)根据system_token换取authToken + jsonBytes, _ := json.Marshal(a) + p := httpDo("POST", url, string(jsonBytes)) + if !p.Success { + fmt.Println(p) + } + fmt.Println(p) +} diff --git a/dsDataex/go.mod b/dsDataex/go.mod index bd030895..6e9ea645 100644 --- a/dsDataex/go.mod +++ b/dsDataex/go.mod @@ -25,7 +25,8 @@ require ( github.com/go-xorm/cmd/xorm v0.0.0-20190426080617-f87981e709a1 // indirect github.com/golang/protobuf v1.4.2 github.com/imdario/mergo v0.3.9 - github.com/olivere/elastic/v7 v7.0.17 + github.com/mailru/easyjson v0.7.2 // indirect + github.com/olivere/elastic/v7 v7.0.19 github.com/oschwald/geoip2-golang v1.4.0 github.com/pkg/errors v0.9.1 github.com/satori/go.uuid v1.2.0 diff --git a/dsDataex/go.sum b/dsDataex/go.sum index 48223d06..c7b9f882 100644 --- a/dsDataex/go.sum +++ b/dsDataex/go.sum @@ -70,6 +70,7 @@ github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 h1:zV3ejI06 github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/aws/aws-sdk-go v1.14.31/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.31.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.33.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/bndr/gotabulate v1.1.2 h1:yC9izuZEphojb9r+KYL4W9IJKO/ceIO8HDwxMA24U4c= github.com/bndr/gotabulate v1.1.2/go.mod h1:0+8yUgaPTtLRTjf49E8oju7ojpU11YmXyvq1LbPAb3U= @@ -300,6 +301,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -351,6 +353,8 @@ github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonas-p/go-shp v0.1.1/go.mod h1:MRIhyxDQ6VVp0oYeD7yPGr5RSTNScUFKCDsI5DR7PtI= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= @@ -406,6 +410,8 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.2 h1:V9ecaZWDYm7v9uJ15RZD6DajMu5sE0hdep0aoDwT9g4= +github.com/mailru/easyjson v0.7.2/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= @@ -448,6 +454,8 @@ github.com/olivere/elastic v1.0.1 h1:UeafjZg+TifCVPhCJNPof0pUHig6vbXuJEbC/A+Ouo0 github.com/olivere/elastic v6.2.33+incompatible h1:SRPB2w2OhJ7iULftDEHsNPRoL2GLREqPMRalVmbZaEw= github.com/olivere/elastic/v7 v7.0.17 h1:VMHAc164MH85MIOyyyL6AAzIDixsdjIdAfmKotxNxyQ= github.com/olivere/elastic/v7 v7.0.17/go.mod h1:sd6x2HP229aT2+U2261gUUMCD4RVf/Nsso8HxSgcjDs= +github.com/olivere/elastic/v7 v7.0.19 h1:w4F6JpqOISadhYf/n0NR1cNj73xHqh4pzPwD1Gkidts= +github.com/olivere/elastic/v7 v7.0.19/go.mod h1:4Jqt5xvjqpjCqgnTcHwl3j8TLs8mvoOK8NYgo/qEOu4= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -457,6 +465,7 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/oschwald/geoip2-golang v1.4.0 h1:5RlrjCgRyIGDz/mBmPfnAF4h8k0IAcRv9PvrpOfz+Ug= github.com/oschwald/geoip2-golang v1.4.0/go.mod h1:8QwxJvRImBH+Zl6Aa6MaIcs5YdlZSTKtzmPGzQqi9ng= @@ -529,6 +538,7 @@ github.com/sloonz/go-qprintable v0.0.0-20160203160305-775b3a4592d5/go.mod h1:rvs github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.3.4/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak= @@ -621,6 +631,7 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=