bigdata
/
dsMin
6
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

'commit'

Browse Source
master
huanghai 5 years ago
parent f0331e073f
commit b16df2f041
5 changed files with 180 additions and 4 deletions
Show Stats Download Patch File Download Diff File

5
dsBaseRpc/Sql/t_base_organization.sql
Unescape View File

@ -1,3 +1,4 @@
select * from (
SELECT
cast( t1.`org_id` AS CHAR ( 36 ) charset utf8 ) AS `org_id`,
t1.`id_int` AS `id_int`,
@ -32,5 +33,5 @@ SELECT
DATE_FORMAT(t1.`create_time`,'%Y/%m/%d %H:%i:%s') AS `create_time`,
case t1.`b_use` when -1 then 1 else 0 end AS `del_flag`
FROM
`t_base_organization` as t1
`t_base_organization` as t1 where t1.b_use=1
) as t1

5
dsBaseRpc/Sql/t_base_teacher.sql
Unescape View File

@ -1,3 +1,4 @@
select * from (
select cast(t1.`person_id` as char(36) charset utf8) AS `teacher_id`,
t1.`identity_id` AS `identity_id`,
t1.`id_int` AS `id_int`,
@ -36,4 +37,6 @@ case t1.`b_use` when -1 then 1 else 0 end AS `del_flag`,
`t_base_organization`.`xxbbm` AS `xxbbm`,
`t_base_organization`.`xxbxlxm` AS `xxbxlxm`,
`t_base_organization`.`szdcxlxm` AS `szdcxlxm`
from (`t_base_teacher` as t1 join `t_base_organization` on (t1.`org_id` = `t_base_organization`.`org_id`))
from (`t_base_teacher` as t1 join `t_base_organization` on (t1.`org_id` = `t_base_organization`.`org_id`))
where t1.identity_id=2
) as t1

160
dsBaseRpc/Test/DataExchange/DataExchange.go
Unescape View File

@ -0,0 +1,160 @@
package main
import (
"bytes"
"crypto/md5"
"dsBaseRpc/Utils/CommonUtil"
"dsBaseRpc/Utils/ConfigUtil"
"dsBaseRpc/Utils/DbUtil"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
)
var db = DbUtil.Engine
//每次获取的条数
var limit = 200
//默认开始时间
var defaultStartTs = "1970-01-01 00:00:00"
// 日志文件路径
var progressFilePath = "/usr/local/SyncDataLogs/"
//建立与汇集中心的主题映射关系结构
type tableStruct struct {
TableName string `json:"table_name"`
PrimaryKey string `json:"primary_key"`
DataSource string `json:"data_source"`
}
//有同步哪些表(增量),之所以不遍历文件的名称进行上报，是因为需要控制上传的顺序，如果只是文件名，就丢失了顺序
var IncrSqlDict = []tableStruct{
{TableName: "t_base_organization", PrimaryKey: "org_id", DataSource: "org_school"},
{TableName: "t_base_class", PrimaryKey: "class_id", DataSource: "org_class"},
{TableName: "t_base_teacher", PrimaryKey: "teacher_id", DataSource: "user_teacher"},
{TableName: "t_base_student", PrimaryKey: "student_id", DataSource: "user_student"},
{TableName: "t_sys_loginperson_log", PrimaryKey: "id_int", DataSource: "log_login"},
}
// 全量数据上报
var FullSqlDict = []tableStruct{
{TableName: "t_base_organization", PrimaryKey: "org_id", DataSource: "org_school"},
{TableName: "t_sys_dict", PrimaryKey: "dict_id", DataSource: "sys_dic"},
{TableName: "t_gov_area", PrimaryKey: "area_code", DataSource: "org_area"},
}
// 数据上报的结构体
type postStruct struct {
AuthToken string `json:"auth_token"`
DataSource string `json:"data_source"`
SystemId string `json:"system_id"`
OrgId string `json:"org_id"`
QueryPage int `json:"query_page"`
QueryTime string `json:"query_time"`
}
//系统token
var SystemToken = ""
//是否成功
var success = false
// 获取签名用的结构体
type authStruct struct {
AuthTime string `json:"auth_time"`
SystemId string `json:"system_id"`
SystemToken string `json:"system_token"`
}
/**
功能：获取系统token
作者：黄海
时间：2020-07-22
*/
func getSystemToken() (bool, string) {
//(1)计算出system_token=MD5.hash(MD5.hash(system_id+auth_time)+system_key)
var as authStruct
as.AuthTime = CommonUtil.GetCurrentTime()
as.SystemId = ConfigUtil.DataExchangeSystemId
//计算 md5
w := md5.New()
io.WriteString(w, as.SystemId+as.AuthTime)
//将str写入到w中
md5str := fmt.Sprintf("%x", w.Sum(nil))
w = md5.New()
io.WriteString(w, md5str+ConfigUtil.DataExchangeSystemKey)
//将str写入到w中
md5str = fmt.Sprintf("%x", w.Sum(nil))
//系统token
as.SystemToken = md5str
//(2)根据system_token换取authToken
jsonBytes, _ := json.Marshal(as)
p := httpDo("POST", ConfigUtil.DataExchangeSystemAuthUrl, string(jsonBytes))
if !p.Success {
fmt.Println(CommonUtil.GetCurrentTime() + "获取认证签名失败!")
return false, "获取认证签名失败!"
}
return true, p.Message
}
type ResultStruct struct {
Message string `json:"message"`
Success bool `json:"success"`
}
// 基础方法，这里多用于访问webapi，配合上json转换。
func httpDo(method string, url string, msg string) ResultStruct {
var p ResultStruct
p.Success = false
p.Message = "上报到汇集系统失败！请检查是否SystemToken是有效的，或者有两个及以上客户端同时在上报。"
client := &http.Client{}
body := bytes.NewBuffer([]byte(msg))
req, err := http.NewRequest(method,
url,
body)
if err != nil {
// handle error
}
req.Header.Set("Content-Type", "application/json;charset=utf-8")
resp, err := client.Do(req)
if err != nil {
fmt.Println(err)
return p
}
defer resp.Body.Close()
resultBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
fmt.Println(err)
return p
}
json.Unmarshal(resultBody, &p)
return p
}
func main() {
//获取系统token
success, SystemToken = getSystemToken()
// 访问地址
url := `http://fort.edusoa.com:7777/v1/dataex/DataexPage`
var a postStruct
a.AuthToken=SystemToken
a.DataSource="org_school"
a.OrgId="-1"
a.QueryPage=0
a.QueryTime=CommonUtil.GetCurrentTime()
a.SystemId=ConfigUtil.DataExchangeSystemId
//(2)根据system_token换取authToken
jsonBytes, _ := json.Marshal(a)
p := httpDo("POST", url, string(jsonBytes))
if !p.Success {
fmt.Println(p)
}
fmt.Println(p)
}

3
dsDataex/go.mod
Unescape View File

@ -25,7 +25,8 @@ require (
github.com/go-xorm/cmd/xorm v0.0.0-20190426080617-f87981e709a1 // indirect
github.com/golang/protobuf v1.4.2
github.com/imdario/mergo v0.3.9
github.com/olivere/elastic/v7 v7.0.17
github.com/mailru/easyjson v0.7.2 // indirect
github.com/olivere/elastic/v7 v7.0.19
github.com/oschwald/geoip2-golang v1.4.0
github.com/pkg/errors v0.9.1
github.com/satori/go.uuid v1.2.0

11
dsDataex/go.sum
Unescape View File

@ -70,6 +70,7 @@ github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 h1:zV3ejI06
github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
github.com/aws/aws-sdk-go v1.14.31/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
github.com/aws/aws-sdk-go v1.31.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
github.com/aws/aws-sdk-go v1.33.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/bndr/gotabulate v1.1.2 h1:yC9izuZEphojb9r+KYL4W9IJKO/ceIO8HDwxMA24U4c=
github.com/bndr/gotabulate v1.1.2/go.mod h1:0+8yUgaPTtLRTjf49E8oju7ojpU11YmXyvq1LbPAb3U=
@ -300,6 +301,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@ -351,6 +353,8 @@ github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/jonas-p/go-shp v0.1.1/go.mod h1:MRIhyxDQ6VVp0oYeD7yPGr5RSTNScUFKCDsI5DR7PtI=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
@ -406,6 +410,8 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8=
github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
github.com/mailru/easyjson v0.7.2 h1:V9ecaZWDYm7v9uJ15RZD6DajMu5sE0hdep0aoDwT9g4=
github.com/mailru/easyjson v0.7.2/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
@ -448,6 +454,8 @@ github.com/olivere/elastic v1.0.1 h1:UeafjZg+TifCVPhCJNPof0pUHig6vbXuJEbC/A+Ouo0
github.com/olivere/elastic v6.2.33+incompatible h1:SRPB2w2OhJ7iULftDEHsNPRoL2GLREqPMRalVmbZaEw=
github.com/olivere/elastic/v7 v7.0.17 h1:VMHAc164MH85MIOyyyL6AAzIDixsdjIdAfmKotxNxyQ=
github.com/olivere/elastic/v7 v7.0.17/go.mod h1:sd6x2HP229aT2+U2261gUUMCD4RVf/Nsso8HxSgcjDs=
github.com/olivere/elastic/v7 v7.0.19 h1:w4F6JpqOISadhYf/n0NR1cNj73xHqh4pzPwD1Gkidts=
github.com/olivere/elastic/v7 v7.0.19/go.mod h1:4Jqt5xvjqpjCqgnTcHwl3j8TLs8mvoOK8NYgo/qEOu4=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@ -457,6 +465,7 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
github.com/oschwald/geoip2-golang v1.4.0 h1:5RlrjCgRyIGDz/mBmPfnAF4h8k0IAcRv9PvrpOfz+Ug=
github.com/oschwald/geoip2-golang v1.4.0/go.mod h1:8QwxJvRImBH+Zl6Aa6MaIcs5YdlZSTKtzmPGzQqi9ng=
@ -529,6 +538,7 @@ github.com/sloonz/go-qprintable v0.0.0-20160203160305-775b3a4592d5/go.mod h1:rvs
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smartystreets/gunit v1.3.4/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
@ -621,6 +631,7 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=

Write Preview
Loading…
Cancel
Save