diff --git a/Logs/dsBaseRpc.log b/Logs/dsBaseRpc.log new file mode 100644 index 00000000..9db15acf --- /dev/null +++ b/Logs/dsBaseRpc.log @@ -0,0 +1,57 @@ +[Error]2021/03/08 13:48:58 CreateRedisError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:48:58 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:48:58 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:48:58 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:48:58 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:48:58 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:48:58 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:48:58 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:48:58 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:48:58 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:52:21 CreateRedisError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:52:21 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:52:21 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:52:21 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:52:21 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:52:21 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:52:21 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:52:21 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:52:21 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:52:21 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:53:58 CreateRedisError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:53:58 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:53:58 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:53:58 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:53:58 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:53:58 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/08 13:53:58 SqlQueryError dial tcp :0: connectex: The requested address is not valid in its context. +[Error]2021/03/16 14:18:15 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:18:16 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:22:02 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:22:03 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:27:19 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:27:20 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:31:00 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:31:01 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:31:24 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:31:25 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:31:56 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:31:57 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:34:42 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:34:43 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:35:38 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:35:39 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:42:31 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:42:32 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:43:27 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:43:28 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:44:21 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 14:44:22 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 15:41:40 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 15:41:41 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 15:45:12 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 15:45:13 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 15:45:38 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 15:45:39 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 15:58:02 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. +[Error]2021/03/16 15:58:03 SqlQueryError dial tcp 10.10.14.187:32066: connectex: No connection could be made because the target machine actively refused it. diff --git a/dsDataexV1.1/Cache/greenplum/day/gp_school--area_name-total-echarts-20210305.txt b/dsDataexV1.1/Cache/greenplum/day/gp_school--area_name-total-echarts-20210305.txt new file mode 100644 index 00000000..ed8fd87a --- /dev/null +++ b/dsDataexV1.1/Cache/greenplum/day/gp_school--area_name-total-echarts-20210305.txt @@ -0,0 +1 @@ +{"success":true,"message":"数据查询成功","result":"{\"groups\":[[\"南关区\",\"宽城区\",\"朝阳区\",\"二道区\",\"绿园区\",\"双阳区\",\"九台区\",\"农安县\",\"榆树市\",\"德惠市\",\"直属代管\",\"新区代管\",\"经开区代管\",\"净月区代管\",\"汽车区代管\",\"莲花山度假区代管\"]],\"datas\":[[\"61\",\"54\",\"81\",\"72\",\"85\",\"69\",\"39\",\"59\",\"41\",\"93\",\"1\",\"27\",\"53\",\"34\",\"56\",\"3\"]],\"datas2\":[null],\"datas3\":[null],\"datas4\":[null],\"datas5\":[null],\"columns\":null,\"rows\":null}"} \ No newline at end of file diff --git a/dsDataexV1.1/Cache/greenplum/day/gp_school--area_name-total-echarts-20210308.txt b/dsDataexV1.1/Cache/greenplum/day/gp_school--area_name-total-echarts-20210308.txt new file mode 100644 index 00000000..bc024f51 --- /dev/null +++ b/dsDataexV1.1/Cache/greenplum/day/gp_school--area_name-total-echarts-20210308.txt @@ -0,0 +1 @@ +{"success":true,"message":"dial tcp: address 75432: invalid port","result":"{\"groups\":null,\"datas\":null,\"datas2\":null,\"datas3\":null,\"datas4\":null,\"datas5\":null,\"columns\":null,\"rows\":null}"} \ No newline at end of file diff --git a/dsDataexV1.1/Cache/greenplum/day/gp_school--area_name-total-echarts-20210309.txt b/dsDataexV1.1/Cache/greenplum/day/gp_school--area_name-total-echarts-20210309.txt new file mode 100644 index 00000000..bc024f51 --- /dev/null +++ b/dsDataexV1.1/Cache/greenplum/day/gp_school--area_name-total-echarts-20210309.txt @@ -0,0 +1 @@ +{"success":true,"message":"dial tcp: address 75432: invalid port","result":"{\"groups\":null,\"datas\":null,\"datas2\":null,\"datas3\":null,\"datas4\":null,\"datas5\":null,\"columns\":null,\"rows\":null}"} \ No newline at end of file diff --git a/dsDataexV1.1/Config/Config.ini b/dsDataexV1.1/Config/Config.ini index fa778e4f..c030de94 100644 --- a/dsDataexV1.1/Config/Config.ini +++ b/dsDataexV1.1/Config/Config.ini @@ -10,12 +10,22 @@ ip = 127.0.0.1 port = 6379 db = 1 expireTime = 86400 +enable = 0 ;ip = 10.10.14.187 ;port = 18890 ;db = 0 ;expireTime = 86400 +; 文件缓存 +[filecache] +enable = 1 +;cycle: 周期1天, 周期30天 / calendar: 自然日, 自然月 +mode = calendar +path_day = ./Cache/greenplum/day/ +path_month = ./Cache/preenplum/month/ +extension = .txt + [kafka] brokers = 10.10.14.238:9092, KafkaAccessLogTopic = log_dataex @@ -29,6 +39,7 @@ enable = 1 nodes = http://10.10.14.61:9200,http://10.10.14.62:9200,http://10.10.14.63:9200 user = root pwd = dsideal +enable = 1 [greenplum] ip = 10.10.14.206 @@ -36,6 +47,7 @@ port = 5432 database = eduData2019DB user = root pwd = DsideaL147258369 +enable = 1 [project] project_name = dsDataex diff --git a/dsDataexV1.1/MyReport/GPSql/GPSqlOpenapi/GPSqlOpenapi.go b/dsDataexV1.1/MyReport/GPSql/GPSqlOpenapi/GPSqlOpenapi.go index 50a2d41f..8bbedc56 100644 --- a/dsDataexV1.1/MyReport/GPSql/GPSqlOpenapi/GPSqlOpenapi.go +++ b/dsDataexV1.1/MyReport/GPSql/GPSqlOpenapi/GPSqlOpenapi.go @@ -5,6 +5,10 @@ import ( "dsDataex/MyReport/MySwagger" "dsDataex/MyService/Auth/AuthService" "dsDataex/MyService/DataEX/DataexService" + "dsDataex/Utils/ConfigUtil" + "dsDataex/Utils/FileCacheUtil" + "dsDataex/Utils/ValidationUtil" + "fmt" "github.com/gin-gonic/gin" "net/http" "strings" @@ -55,18 +59,33 @@ func QuerySimpleGP(c *gin.Context) { //return } - //TODO:输入参数 SQL注入检测 + // 输入参数 SQL注入检测 if len(input.Query.QueryParam) > 0 { - + result, msg, _ := ValidationUtil.ValidGPSqlInjuect(input.Query.QueryParam) + if result == false { + fmt.Println(msg) + c.JSON(http.StatusOK, MySwagger.DataResult{Success: false, Message: msg}) + return + } } - //TODO: 缓存数据 - if input.QueryCache == 1 || input.QueryCache == 2 { - + // 缓存数据 + var res bool + var msg string + var data string + + if ConfigUtil.FCEnable == 1 && (input.QueryCache == 1 || input.QueryCache == 2) { + if ConfigUtil.FCMode == "calendar" { + // Calendar + res, msg, data = FileCacheUtil.SimpleGPCalendarCache(input) + } else { + // Cycle + res, msg, data = FileCacheUtil.SimpleGPCycleCache(input) + } + } else { + res, msg, data = GPSqlService.QuerySimple(input.Query.QueryID, input.Query.QueryParam, input.QueryGroup, input.QueryCount, input.QueryFormat) } - res, msg, data := GPSqlService.QuerySimple(input.Query.QueryID, input.Query.QueryParam, input.QueryGroup, input.QueryCount, input.QueryFormat) - if res { c.JSON(http.StatusOK, MySwagger.DataResult{ Success: true, diff --git a/dsDataexV1.1/Utils/CommonUtil/CommonUtil.go b/dsDataexV1.1/Utils/CommonUtil/CommonUtil.go index 979ba172..d9fb8b06 100644 --- a/dsDataexV1.1/Utils/CommonUtil/CommonUtil.go +++ b/dsDataexV1.1/Utils/CommonUtil/CommonUtil.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" "time" + "unsafe" ) /** @@ -259,4 +260,15 @@ func ConvertJsonStringToMap(data string) map[string]interface{} { str := []byte(data) json.Unmarshal(str, &m) return m +} + +func ConvertJsonStringToBytes(s string) []byte { + x := (*[2]uintptr)(unsafe.Pointer(&s)) + h := [3]uintptr{x[0], x[1], x[1]} + + return *(*[]byte)(unsafe.Pointer(&h)) +} + +func ConvertBytesToString(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) } \ No newline at end of file diff --git a/dsDataexV1.1/Utils/ConfigUtil/ConfigUtil.go b/dsDataexV1.1/Utils/ConfigUtil/ConfigUtil.go index 605a261a..34f69e4d 100644 --- a/dsDataexV1.1/Utils/ConfigUtil/ConfigUtil.go +++ b/dsDataexV1.1/Utils/ConfigUtil/ConfigUtil.go @@ -21,6 +21,7 @@ var ( GreenPlumUser string GreenPlumPwd string GreenPlumDataBase string + GreenPlumEnable int64 //要发布到的主机信息 DistributeIp string @@ -35,6 +36,14 @@ var ( RedisPort string RedisDb string RedisExpireTime int64 + RedisEnable int64 + + //FILE CACHE配置 + FCEnable int64 + FCMode string + FCPathDay string + FCPathMonth string + FCExtension string //运行时配置 StartPathWindows string @@ -73,6 +82,7 @@ var ( ESUser string ESPWD string + ESEnable int64 //Params 系统参数 ROOT_ORGNAME string @@ -115,6 +125,8 @@ func init() { GreenPlumUser = iniParser.GetString("greenplum", "user") //密码 GreenPlumPwd = iniParser.GetString("greenplum", "pwd") + // 是否启用 + GreenPlumEnable = iniParser.GetInt64("greenplum", "enable") //发布的远端IP DistributeIp = iniParser.GetString("distribute", "ip") @@ -134,6 +146,14 @@ func init() { RedisPort = iniParser.GetString("redis", "port") RedisDb = iniParser.GetString("redis", "db") RedisExpireTime = iniParser.GetInt64("redis", "expireTime") + RedisEnable = iniParser.GetInt64("redis", "enable") + + // 文件缓存 + FCEnable = iniParser.GetInt64("filecache", "enable") + FCMode = iniParser.GetString("filecache", "mode") + FCPathDay = iniParser.GetString("filecache", "path_day") + FCPathMonth = iniParser.GetString("filecache", "path_month") + FCExtension = iniParser.GetString("filecache", "extension") //项目名称 ProjectName = iniParser.GetString("project", "project_name") @@ -157,6 +177,7 @@ func init() { ESNodes = iniParser.GetString("elasticsearch", "nodes") ESUser = iniParser.GetString("elasticsearch", "user") ESPWD = iniParser.GetString("elasticsearch", "pwd") + ESEnable = iniParser.GetInt64("elasticsearch", "enable") ESAddress = strings.Split(ESNodes, ",") diff --git a/dsDataexV1.1/Utils/DbUtil/DbUtil.go b/dsDataexV1.1/Utils/DbUtil/DbUtil.go index 9abb4889..6a9fc04b 100644 --- a/dsDataexV1.1/Utils/DbUtil/DbUtil.go +++ b/dsDataexV1.1/Utils/DbUtil/DbUtil.go @@ -33,7 +33,10 @@ func init() { Engine.SetConnMaxLifetime(time.Second * 30) //调用第一次 - Engine.Ping() + err = Engine.Ping() + if err != nil { + fmt.Println("Mysql Err :", err.Error()) + } //这段代码是黄海后加上的 2020-04-16 //通过定时ping保持鲜活 @@ -41,8 +44,11 @@ func init() { for{ //https://studygolang.com/articles/12617 time.Sleep(time.Duration(600)*time.Second) - Engine.Ping() - fmt.Println("\n[xorm] [info] "+CommonUtil.GetCurrentTime()+" PING DATABASE mysql") + if err != nil { + fmt.Println("\n[xorm] [error] " + CommonUtil.GetCurrentTime() + err.Error()) + } else { + fmt.Println("\n[xorm] [info] "+CommonUtil.GetCurrentTime()+" PING DATABASE mysql") + } } }() diff --git a/dsDataexV1.1/Utils/ES7SqlUtil/ES7SqlUtil.go b/dsDataexV1.1/Utils/ES7SqlUtil/ES7SqlUtil.go index d9e1464d..65baed01 100644 --- a/dsDataexV1.1/Utils/ES7SqlUtil/ES7SqlUtil.go +++ b/dsDataexV1.1/Utils/ES7SqlUtil/ES7SqlUtil.go @@ -22,15 +22,17 @@ func init() { CTX=context.Background() - res, _ := ES7Client.Info() + if ConfigUtil.ESEnable == 1 { + res, _ := ES7Client.Info() - defer res.Body.Close() + defer res.Body.Close() - var result map[string]interface{} + var result map[string]interface{} - json.NewDecoder(res.Body).Decode(&result) + json.NewDecoder(res.Body).Decode(&result) - ServerVersion=(result["version"].(map[string]interface{}))["number"].(string) + ServerVersion=(result["version"].(map[string]interface{}))["number"].(string) + } } func SqlQueryJson(sql string,param []string) map[string]interface{} { diff --git a/dsDataexV1.1/Utils/ES7Util/ES7Util.go b/dsDataexV1.1/Utils/ES7Util/ES7Util.go index ff6c4121..2cd256c9 100644 --- a/dsDataexV1.1/Utils/ES7Util/ES7Util.go +++ b/dsDataexV1.1/Utils/ES7Util/ES7Util.go @@ -34,9 +34,11 @@ func init() { CTX=context.Background() //Ping，获取ES server启动信息 - info, _, _ := ES7Client.Ping(ConfigUtil.ESAddress[0]).Do(CTX) + if ConfigUtil.ESEnable == 1 { + info, _, _ := ES7Client.Ping(ConfigUtil.ESAddress[0]).Do(CTX) - ServerVersion =ES7Client.String() +"【"+ info.Version.Number+"】" + ServerVersion =ES7Client.String() +"【"+ info.Version.Number+"】" + } } /** diff --git a/dsDataexV1.1/Utils/FileCacheUtil/FileCacheUtil.go b/dsDataexV1.1/Utils/FileCacheUtil/FileCacheUtil.go new file mode 100644 index 00000000..ba116f2e --- /dev/null +++ b/dsDataexV1.1/Utils/FileCacheUtil/FileCacheUtil.go @@ -0,0 +1,358 @@ +package FileCacheUtil + +import ( + "dsDataex/MyReport/GPSql/GPSqlService" + "dsDataex/MyReport/MySwagger" + "dsDataex/Utils/ConfigUtil" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "sort" + "strings" + "time" +) + +/** + * @title PathExists + * @Description 文件/路径是否存在 + * @Author wangshuai + * @Date 2021-03-01 + * @Param path string 路径 + * @Return bool 是否存在 + * @Return error 错误信息 + */ +func PathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + + return false, err +} + +/** + * @title WriteFile + * @Description 写入文件 + * @Author wangshuai + * @Date 2021-03-01 + * @Param filename string 文件 + * @Param data []byte 内容 + * @Param perm os.FileMode 模式(0666,0600) + * @Return bool 是否成功 + * @Return error 错误信息 + */ +func WriteFile(filename string, data []byte, perm os.FileMode) (bool, error) { + f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return false, err + } + n, err := f.Write(data) + if err == nil && n < len(data) { + err = io.ErrShortWrite + } + if err1 := f.Close(); err == nil { + err = err1 + } + + return true, err +} + +/** + * @title ReadFile + * @Description 读取文件 + * @Author wangshuai + * @Date 2021-03-01 + * @Param filename string 文件 + * @Param perm os.FileMode 模式(0666,0600) + * @Return bool 是否成功 + * @Return string 内容 + * @Return error 错误信息 + */ +func ReadFile(filename string, perm os.FileMode) (bool, string, error) { + f, err := os.OpenFile(filename, os.O_RDONLY,perm) + defer f.Close() + if err != nil { + return false, "", err + } else { + contentByte, err := ioutil.ReadAll(f) + if err == nil && len(contentByte) > 0 { + return true, string(contentByte), err + } + } + + return false, "", err +} + +/** + * @title ReadDir + * @Description 读取目录所有文件 + * @Author wangshuai + * @Date 2021-03-01 + * @Param path string 路径 + * @Return bool 是否成功 + * @Return []string 文件名列表 + * @Return error 错误信息 + */ +func ReadDir(path string) (bool, []string, error) { + var data []string + dirs, e := ioutil.ReadDir(path) + if e != nil { + return false, nil, e + } + for _, v := range dirs { + data = append(data, v.Name()) + } + + return true, data, e +} + +/** + * @title LatestCacheDate + * @Description 最新缓存文件日期 + * @Author wangshuai + * @Date 2021-03-01 + * @Param path string 路径 + * @Param filenamePrefix string 文件名前缀 + * @Return bool 是否成功 + * @Return string 最新缓存文件日期 + * @Return error 错误信息 + */ +func LatestCacheDate(path, filenamePrefix string) (bool, string, error) { + var data []string + _, dirs, err := ReadDir(path) + if err != nil { + return false, "", err + } + for _, v := range dirs { + prefix := strings.LastIndex(v, filenamePrefix) + start := strings.LastIndex(v, "-") + end := strings.LastIndex(v, ".") + if prefix != -1 && start != -1 && end != -1 { + cacheDate := v[start + 1 : end] + data = append(data, cacheDate) + } + } + + if len(data) > 0 { + sort.Sort(sort.Reverse(sort.StringSlice(data))) + return true, data[0], nil + } else { + return false, "", nil + } +} + +/** + * @title SimpleGPCycleCache + * @Description 周期缓存 + * @Author wangshuai + * @Date 2021-03-04 + * @Param input string 路径 + * @Return bool 是否成功 + * @Return string 提示信息 + * @Return string 缓存内容 + */ +func SimpleGPCycleCache(input MySwagger.QuerySimpleGP) (bool, string, string) { + var queryID string + var queryParam string + var queryGroup string + var queryCount string + var queryFormat string + var timeLayout string = "20060102150405" + var filename string + var filenamePrefix string + var exists bool = false + var res bool + var msg string + var data string + var DataResult MySwagger.DataResult + + strTime := time.Now().Format(timeLayout) + queryID = input.Query.QueryID + queryParam = strings.Join(input.Query.QueryParam , "@") + queryGroup = strings.Join(input.QueryGroup , "@") + queryCount = strings.Join(input.QueryCount , "@") + queryFormat = input.QueryFormat + // 文件名 + filename = ConfigUtil.FCPathDay + queryID + "-" + queryParam + "-" + queryGroup + "-" + queryCount + "-" + queryFormat + "-" + strTime + ConfigUtil.FCExtension + // 文件名前缀 + filenamePrefix = queryID + "-" + queryParam + "-" + queryGroup + "-" + queryCount + "-" + queryFormat + // 最新缓存日期 + result, latestCacheDate, _ := LatestCacheDate(ConfigUtil.FCPathDay, filenamePrefix) + if result == true { + // 转成时间戳 + startUnix, _ := time.Parse(timeLayout, strTime) + endUnix, _ := time.Parse(timeLayout, latestCacheDate) + startTime := startUnix.Unix() + endTime := endUnix.Unix() + // 求相差天数 + timeSub := (startTime - endTime) / 86400 + if input.QueryCache == 1 { // 按日缓存 + //exists, _ = FileCacheUtil.PathExists(filename) + if timeSub < 1 { + exists = true + filename = filenamePrefix + "-" + latestCacheDate + ConfigUtil.FCExtension + } + } else { // 按月缓存 + if timeSub < 30 { + exists = true + filename = filenamePrefix + "-" + latestCacheDate + ConfigUtil.FCExtension + } + } + } + if exists { + // 读取缓存文件 + fmt.Println("开始读取缓存...") + result, cache, _ := ReadFile(filename, 0600) + if result == false { + return false, "读取文件失败, 无法打开", "" + } + fmt.Println("缓存读取成功, 缓存文件: " + filename) + err := json.Unmarshal([]byte(cache), &DataResult) + if err != nil { + fmt.Println(err) + return false, "JSON反序列化失败", "" + } + res, msg, data = DataResult.Success, DataResult.Message, DataResult.Result + return res, msg, data + } else { + res, msg, data = GPSqlService.QuerySimple(input.Query.QueryID, input.Query.QueryParam, input.QueryGroup, input.QueryCount, input.QueryFormat) + fmt.Println("开始写入缓存...") + DataResult = MySwagger.DataResult{ + Success: true, + Message: msg, + Result: data, + } + byteData, err1 := json.Marshal(DataResult) + if err1 != nil { + fmt.Println("JSON ERR:", err1) + return false, "JSON序列化失败", "" + } + // 写入缓存文件 + fmt.Println("filename=", filename, "<<") + result, err := WriteFile(filename, byteData, 0666) + if result == false { + fmt.Println(err) + return false, "写入失败", "" + } + fmt.Println("缓存写入成功, 缓存文件: " + filename) + return res, msg, data + } +} + +/** + * @title SimpleGPCalendarCache + * @Description 自然日/月缓存 + * @Author wangshuai + * @Date 2021-03-04 + * @Param input string 路径 + * @Return bool 是否成功 + * @Return string 提示信息 + * @Return string 缓存内容 + */ +func SimpleGPCalendarCache(input MySwagger.QuerySimpleGP) (bool, string, string) { + var queryID string + var queryParam string + var queryGroup string + var queryCount string + var queryFormat string + var dayTimeLayout string = "20060102" + var monthTimeLayout string = "200601" + var filename string + var exists bool = false + var res bool + var msg string + var data string + var DataResult MySwagger.DataResult + + strDayTime := time.Now().Format(dayTimeLayout) + strMonthTime := time.Now().Format(monthTimeLayout) + queryID = input.Query.QueryID + queryParam = strings.Join(input.Query.QueryParam , "@") + queryGroup = strings.Join(input.QueryGroup , "@") + queryCount = strings.Join(input.QueryCount , "@") + queryFormat = input.QueryFormat + + // 最新缓存日期 + if input.QueryCache == 1 { // 按日缓存 + // 文件名 + filename = ConfigUtil.FCPathDay + queryID + "-" + queryParam + "-" + queryGroup + "-" + queryCount + "-" + queryFormat + "-" + strDayTime + ConfigUtil.FCExtension + } else { // 按月缓存 + // 文件名 + filename = ConfigUtil.FCPathMonth + queryID + "-" + queryParam + "-" + queryGroup + "-" + queryCount + "-" + queryFormat + "-" + strMonthTime + ConfigUtil.FCExtension + } + + exists, _ = PathExists(filename) + if exists { + // 读取缓存文件 + fmt.Println("开始读取缓存...") + result, cache, _ := ReadFile(filename, 0600) + if result == false { + return false, "读取文件失败, 无法打开", "" + } + fmt.Println("缓存读取成功, 缓存文件: " + filename) + err := json.Unmarshal([]byte(cache), &DataResult) + if err != nil { + fmt.Println(err) + return false, "JSON反序列化失败", "" + } + res, msg, data = DataResult.Success, DataResult.Message, DataResult.Result + return res, msg, data + } else { + res, msg, data = GPSqlService.QuerySimple(input.Query.QueryID, input.Query.QueryParam, input.QueryGroup, input.QueryCount, input.QueryFormat) + fmt.Println("开始写入缓存...") + DataResult = MySwagger.DataResult{ + Success: true, + Message: msg, + Result: data, + } + byteData, err1 := json.Marshal(DataResult) + if err1 != nil { + fmt.Println("JSON ERR:", err1) + return false, "JSON序列化失败", "" + } + // 写入缓存文件 + result, err := WriteFile(filename, byteData, 0666) + if result == false { + fmt.Println(err) + return false, "写入失败", "" + } + fmt.Println("缓存写入成功, 缓存文件: " + filename) + return res, msg, data + } +} + +/** + * @title LatestCacheFilename + * @Description 所有缓存文件名 + * @Author wangshuai + * @Date 2021-03-01 + * @Param path string 路径 + * @Param filenamePrefix string 文件名前缀 + * @Return bool 是否成功 + * @Return string 最新缓存文件名 + * @Return error 错误信息 + */ +func LatestCacheFilename(path, filenamePrefix string) (bool, string, error) { + var data []string + _, dirs, err := ReadDir(path) + if err != nil { + return false, "", err + } + for _, v := range dirs { + start := strings.LastIndex(v, "-") + end := strings.LastIndex(v, ".") + if start != -1 && end != -1 { + cacheDate := v[start + 1 : end] + data = append(data, cacheDate) + } + } + sort.Sort(sort.Reverse(sort.StringSlice(data))) + filename := filenamePrefix + "-" + data[0] + ConfigUtil.FCExtension + + return true, filename, err +} \ No newline at end of file diff --git a/dsDataexV1.1/Utils/GPUtil/GPUtil.go b/dsDataexV1.1/Utils/GPUtil/GPUtil.go index 05e78c88..0e01ff53 100644 --- a/dsDataexV1.1/Utils/GPUtil/GPUtil.go +++ b/dsDataexV1.1/Utils/GPUtil/GPUtil.go @@ -41,13 +41,21 @@ func init() { Engine.SetMaxIdleConns(10) Engine.SetConnMaxLifetime(time.Second * 30) - Engine.Ping() + err = Engine.Ping() + if err != nil { + fmt.Println("GreenPlum Err :", err.Error()) + } go func() { for { time.Sleep(time.Duration(600) * time.Second) - Engine.Ping() - fmt.Println("\n[xorm] [info] " + CommonUtil.GetCurrentTime() + " PING DATABASE greenplum") + err = Engine.Ping() + if err != nil { + fmt.Println("\n[xorm] [error] " + CommonUtil.GetCurrentTime() + err.Error()) + } else { + fmt.Println("\n[xorm] [info] " + CommonUtil.GetCurrentTime() + " PING DATABASE greenplum") + } + } }() } diff --git a/dsDataexV1.1/Utils/ValidationUtil/ValidationUtil.go b/dsDataexV1.1/Utils/ValidationUtil/ValidationUtil.go index 04975f26..70165a61 100644 --- a/dsDataexV1.1/Utils/ValidationUtil/ValidationUtil.go +++ b/dsDataexV1.1/Utils/ValidationUtil/ValidationUtil.go @@ -16,8 +16,6 @@ import ( var dics []map[string]interface{} const TIME_LAYOUT = "2020-09-14 15:29:10" -type RegexCheck struct {} - /** * @title ValidESDataContent * @Description 校验ESDataContent @@ -160,6 +158,34 @@ func ValidESDataContentItem(dataContentItem map[string]interface{}, itemName str return true, "ok", nil } +/** + * @title ValidSqlInjuect + * @Description 校验SQL注入 + * @Author wangshuai + * @Date 2021-02-26 + * @Param params []string SQL参数 + * @Return bool 校验是否通过 + * @Return string 校验提示信息 + * @Return error 错误信息 + */ +func ValidGPSqlInjuect(params []string) (bool, string, error) { + flag := true + msg := "" + var err error + for _, v := range params { + // 校验关键字 + matchKeyword, _ := regexp.MatchString("SELECT|INSERT|DELETE|FROM|COUNT\\(|DROP TABLE|TRUNCATE|\\ASC|MID\\(|CHAR\\(|XP_CMDSHELL|EXEC MASTER|NETLOCALGROUP ADMINISTRATORS|\\:|NET USER|\"\"|OR|AND", strings.ToUpper(v)) + // 校验关键字符 + matchCharacter, _ := regexp.MatchString("[-|;|,|\\?|/|\\(|\\)|\\[|\\]|}|{|%|\\@|*|!|']", strings.ToUpper(v)) + if matchKeyword == true || matchCharacter == true { + flag = false + msg = "SQL注入校验不通过" + } + } + + return flag, msg, err +} + /** * @title CheckDic * @Description 字典校验 diff --git a/dsDataexV1.1/main.go b/dsDataexV1.1/main.go index 0d7f1a28..f76509f2 100644 --- a/dsDataexV1.1/main.go +++ b/dsDataexV1.1/main.go @@ -99,7 +99,9 @@ func main() { //var token=MD5Util.MD5V1("TEST_007" + "20200707" + "DSDataex_Token_7ee1f0f76243449f8d75f40fdcc2b93d") //fmt.Println("AccessToken : "+token) - CacheUtil.OrgtreeCacheInit() + if ConfigUtil.RedisEnable == 1 { + CacheUtil.OrgtreeCacheInit() + } GinServerInit()