diff --git a/dsSupport/Utils/CacheUtil/DaoCache.go b/dsSupport/Utils/CacheUtil/DaoCache.go new file mode 100644 index 00000000..77c50d21 --- /dev/null +++ b/dsSupport/Utils/CacheUtil/DaoCache.go @@ -0,0 +1,200 @@ +package CacheUtil + +import ( + "dsDataex/Utils/ErrorConst" + "dsDataex/Utils/CommonUtil" + "dsDataex/Utils/DbUtil" + "dsDataex/Utils/LogUtil" + "dsDataex/Utils/RedisUtil" + "fmt" + "github.com/go-redis/redis/v7" + "reflect" + "strconv" + "strings" + "time" +) + +/** +功能:优先从缓存读取,检查到缺失的,就从数据库读取,然后两个结果集拼接后返回,同时将缺失的缓存补全 +作者:黄海 +时间:2020-02-05 +*/ +func GetListByIds(ids []string, m Selector) []map[string]interface{} { + //1、第一步从缓存读取 + list, notExistsIds := batchReadRedis(ids, m.RedisPrefix) + //2、如果有一些ID在缓存中不存在,那么需要从数据库中查询,然后填充到缓存中 + if notExistsIds != nil && len(notExistsIds) > 0 { + //从数据库中获取缺失的数据 + lostCacheList := fromDb(notExistsIds, m) + //缓存写入 + batchWriteRedis(lostCacheList, m) + + //change by zhangjun 2020-05-19 + //合并两个结果集 + list2, _ := batchReadRedis(notExistsIds, m.RedisPrefix) + for i := 0; i < len(list2); i++ { + list = append(list, list2[i]) + } + } + return list +} + +/** +功能:删除指定ID的缓存 +作者:黄海 +时间:2020-02-21 +*/ +func DeleteCacheByIds(ids []string, m Selector) { + prefix := m.RedisPrefix + for i := 0; i < len(ids); i++ { + id := ids[i] + key := prefix + id + RedisUtil.DEL(key) + } +} + +/*===================以下为内部函数,用于扩展从REDIS缓存中查询===========================*/ +func fromDb(idAry []string, m Selector) []map[string]interface{} { + tableName := m.TableName + pkField := m.PkField + //查询数据库 + ids := strings.Join(idAry, "','") + sqlRaw := fmt.Sprintf(`select * from `+tableName+` where `+pkField+` in ('%s')`, ids) + results, err := DbUtil.Engine.SQL(sqlRaw).Query().List() + if err != nil { + LogUtil.Error(ErrorConst.SqlQueryError, "从数据库中读取数据失败:"+err.Error()) + } + return results +} + +/** +功能:批量写入缓存 +作者:黄海 +时间:2020-02-05 +*/ +func batchWriteRedis(list []map[string]interface{}, m Selector) { + pkField := m.PkField + prefix := m.RedisPrefix + //声明管道 + pipe := RedisUtil.RedisClient.Pipeline() + for i := 0; i < len(list); i++ { + bean := list[i] + var pkSuffix = "" + switch bean[pkField].(type) { + case string: + pkSuffix = bean[pkField].(string) + case int64: + pkSuffix = strconv.FormatInt(reflect.ValueOf(bean[pkField]).Int(), 10) + } + key := prefix + pkSuffix + pipe.HSet(key, bean) + pipe.Expire(key, 24*time.Hour) + } + //执行管道 + //change by zhangjun 2020-05-15 + //pipe.Exec() + _, err := pipe.Exec() + if err != nil { + LogUtil.Error(ErrorConst.SqlQueryError, err.Error()) + } +} + +/** +功能:从REDIS中批量获取指定ID的列表信息 +作者:黄海 +时间:2020-02-05 +*/ +func batchReadRedis(ids []string, prefix string) ([]map[string]interface{}, []string) { + //1、创建管道 + var list []*redis.StringStringMapCmd + pipe := RedisUtil.RedisClient.Pipeline() + for i := 0; i < len(ids); i++ { + list = append(list, pipe.HGetAll(prefix+ids[i])) + } + //2、执行管道 + _, err := pipe.Exec() + if err != nil { + LogUtil.Error(ErrorConst.SqlQueryError, err.Error()) + } + //存在于缓存中的数据 + var existList []map[string]interface{} + //3、输出结果 + //不存在的主键有哪些 + var notExistsIds []string + for i := 0; i < len(list); i++ { + _bean := list[i].Val() + if len(_bean) > 0 { + //将values转成Interface{},其实这个锅 Golang是不背的,因为redis中hset写入的整数,取出来hget也是字符串~ + n := CommonUtil.ConvertMapStringToMapInterface(_bean) + //找到了加到返回值列表中去 + existList = append(existList, n) + } else { + notExistsIds = append(notExistsIds, ids[i]) + } + } + //返回值 + return existList, notExistsIds +} + +/** +功能:对于简单表提供分页的通用SQL组装方法,返回通用分页的SQL语句和获取查询总数的SQL语句 +作者:黄海 +时间:2020-02-21 +*/ +func count(baseSql string, pkField string, args []interface{}) (int32, error) { + //替换掉分页代码,获取数据量总数 + countSql := strings.Replace(baseSql, "SELECT "+pkField+" ", "SELECT count(*) as count ", -1) + var count int32 + //去掉最后的limit+offset + args = args[0 : len(args)-2] + _, err := DbUtil.Engine.SQL(countSql, args...).Get(&count) + if err != nil { + LogUtil.Error(ErrorConst.SqlQueryError, err.Error()) + return 0, err + } + return count, nil +} + +/** +功能:将指定的SQL语句计算分页的SQL +作者:黄海 +时间:2020-03-25 +*/ +func Page(baseSql string, args ...interface{}) ([]map[string]interface{}, int32, error) { + var model Selector + //删除多余的空格 + baseSql=CommonUtil.DeleteExtraSpace(baseSql) + + arr:=strings.Split(baseSql," ") + + for i:=0;i< len(arr)-1;i++{ + if strings.ToLower(arr[i])=="from"{ + model= GetBean(arr[i+1]) + break + } + } + //条件查询语句 + conditionSql := fmt.Sprintf("%s", " limit ? offset ? ") + //分页的语句 + pageSql := fmt.Sprintf("%s %s", baseSql, conditionSql) + //分页数据 + list, err := DbUtil.Engine.SQL(pageSql, args...).Query().List() + + if err != nil { + LogUtil.Error(ErrorConst.SqlQueryError, err.Error()) + return nil, 0, err + } + + //总数 + count,err := count(baseSql, model.PkField, args) + + if err != nil { + LogUtil.Error(ErrorConst.SqlQueryError, err.Error()) + return nil, 0, err + } + //有哪些ids? + var ids = CommonUtil.Ids(list, model.PkField) + //-->缓存层-->数据库-->缓存 + list = GetListByIds(ids, model) + return list, count, nil +} diff --git a/dsSupport/Utils/CacheUtil/OrgtreeCache.go b/dsSupport/Utils/CacheUtil/OrgtreeCache.go new file mode 100644 index 00000000..bf86e433 --- /dev/null +++ b/dsSupport/Utils/CacheUtil/OrgtreeCache.go @@ -0,0 +1,275 @@ +package CacheUtil + +import ( + "dsDataex/Utils/ErrorConst" + "dsDataex/Utils/LogUtil" + "dsDataex/Utils/RedisUtil" + "strings" + "time" +) + +var GBT2260 map[string]string + +var XXBXLXM map[string]string + +var PARENT_IDS []string + +/** + * @Author zhangjun + * @Description 组织机构树缓存初始化,【机构下属学校】递归查询、并缓存 Redis + * @Description TODO:如果性能还不行,使用全局静态变量存储【机构下属学校】!!!!! + * @Date 2020-06-16 10:06 + * @Param + * @return + **/ +func OrgtreeCacheInit(){ + //删除Redis缓存 + RedisUtil.RedisClient.FlushDB() + + //机构树缓存 + sql := "SELECT id from t_dataex_orgtree where delete_flag = -1 and enable_flag = 1 and parent_id = '-1'" + + list, count, _ := Page(sql, 1000,0) + + PARENT_IDS=make([]string,0) + + + if count>0{ + for no:=0;no0{ + Orgtree2Redis(list2,"Dataex_Scope_"+list[no]["id"].(string)) + + PARENT_IDS =append(PARENT_IDS,list[no]["id"].(string)) + + OrgCacheCreate(list[no]["id"].(string),list[no]["id"].(string)) + } + } + } + } + + //行政区划缓存 + sql = "SELECT area_code from t_dataex_gbt2260" + + list,_,_= Page(sql, 5000,0) + + GBT2260=make( map[string]string) + + for no:=0;no< len(list);no++{ + GBT2260[list[no]["area_code"].(string)]=list[no]["area_name"].(string) + } + + XXBXLXM=make( map[string]string) + + XXBXLXM["111"]="幼儿园" + XXBXLXM["119"]="附设幼儿班" + XXBXLXM["211"]="小学" + XXBXLXM["218"]="小学教学点" + XXBXLXM["219"]="附设小学班" + XXBXLXM["221"]="职工小学" + XXBXLXM["222"]="农民小学" + XXBXLXM["228"]="小学班" + XXBXLXM["229"]="扫盲班" + XXBXLXM["311"]="初级中学" + XXBXLXM["312"]="九年一贯制学校" + XXBXLXM["319"]="附设普通初中班" + XXBXLXM["321"]="职业初中" + XXBXLXM["329"]="附设职业初中班" + XXBXLXM["331"]="成人职工初中" + XXBXLXM["332"]="成人农民初中" + XXBXLXM["341"]="完全中学" + XXBXLXM["342"]="高级中学" + XXBXLXM["345"]="十二年一贯制学校" + XXBXLXM["349"]="附设普通高中班" + XXBXLXM["351"]="成人职工高中" + XXBXLXM["352"]="成人农民高中" + XXBXLXM["361"]="中等职业学校" + XXBXLXM["362"]="中等技术学校" + XXBXLXM["363"]="中等师范学校" + XXBXLXM["364"]="成人中等专业学校" + XXBXLXM["365"]="职业高中学校" + XXBXLXM["366"]="技工学校" + XXBXLXM["368"]="附设中职班" + XXBXLXM["369"]="其他机构" + XXBXLXM["371"]="工读学校" + XXBXLXM["411"]="大学" + XXBXLXM["412"]="学院" + XXBXLXM["413"]="独立学院" + XXBXLXM["414"]="高等专科学校" + XXBXLXM["415"]="高等职业学校" + XXBXLXM["419"]="分校、大专班" + XXBXLXM["421"]="职工高校" + XXBXLXM["422"]="农民高校" + XXBXLXM["423"]="管理干部学院" + XXBXLXM["424"]="教育学院" + XXBXLXM["425"]="独立函授学院" + XXBXLXM["426"]="广播电视大学" + XXBXLXM["429"]="其他成人高等教育机构" + XXBXLXM["511"]="盲人学校" + XXBXLXM["512"]="聋人学校" + XXBXLXM["513"]="弱智学校" + XXBXLXM["514"]="特殊教育学校" + XXBXLXM["519"]="附设特教班" + XXBXLXM["911"]="培养研究生的科研机构" + XXBXLXM["921"]="民办的其他高等教育机构" + XXBXLXM["931"]="职工技术培训学校" + XXBXLXM["932"]="农村成人文化技术培训学校" + XXBXLXM["933"]="其他培训机构" +} + +func RefreshOrgtree(){ + + //删除Redis缓存 + RedisUtil.RedisClient.FlushDB() + + //机构树缓存 + sql := "SELECT id from t_dataex_orgtree where delete_flag = -1 and enable_flag = 1 and parent_id = '-1'" + + list, count, _ := Page(sql, 1000,0) + + if count>0{ + for no:=0;no0{ + Orgtree2Redis(list2,"Dataex_Scope_"+list[no]["id"].(string)) + + PARENT_IDS =append(PARENT_IDS,list[no]["id"].(string)) + + OrgCacheCreate(list[no]["id"].(string),list[no]["id"].(string)) + } + } + } + } +} + +/** + * @Author zhangjun + * @Description 刷新机构下属学校缓存 + * @Date 2020-06-16 10:14 + * @Param orgID string 教育局机构ID + * @return + **/ +func OrgCacheCreate(orgID string,scope... string) { + sql := "SELECT id from t_dataex_orgtree where delete_flag = -1 and enable_flag = 1 and parent_id = '" + orgID + "'" + + list, count, _ := Page(sql, 1000,0) + + if count>0{ + for no:=0;no0{ + Orgtree2Redis(list2,"Dataex_Scope_"+list[no]["id"].(string)) + + PARENT_IDS =append(PARENT_IDS,list[no]["id"].(string)) + + var newscope =append( scope ,list[no]["id"].(string)) + + for no:=0;no0{ + return true + } + return false +} + +func OrgtreeGetScope(orgId string) []string { + result,_:= RedisUtil.RedisClient.Get("Dataex_Scope_"+orgId).Result() + + if result != ""{ + + return strings.Split(result,"##") + } + return nil +} \ No newline at end of file diff --git a/dsSupport/Utils/CacheUtil/Selector.go b/dsSupport/Utils/CacheUtil/Selector.go new file mode 100644 index 00000000..6ecf2c66 --- /dev/null +++ b/dsSupport/Utils/CacheUtil/Selector.go @@ -0,0 +1,57 @@ +package CacheUtil + +import ( + "dsDataex/Utils/ErrorConst" + "dsDataex/Utils/CommonUtil" + "dsDataex/Utils/DbUtil" + "dsDataex/Utils/LogUtil" + "dsDataex/Utils/RedisUtil" + "time" +) + +/** +功能:用于将interface转为struct的结构体 +作者:黄海 +时间:2020-02-05 +*/ +type Selector struct { + TableName string + PkField string + RedisPrefix string +} + +/** +功能:注册一个函数,用于提取计算指定数据表对应的相关信息 +作者:黄海 +时间:2020-03-27 +*/ +func GetBean(tableName string) Selector { + var m = new(Selector) + m.TableName = tableName + m.RedisPrefix = CommonUtil.GetSnakeCaseStr(tableName) + ":" + _, m.PkField = GetTablePk(tableName) + return *m +} + +/** +功能:获取指定表的主键 +作者:黄海 +时间:2020-03-26 +*/ +func GetTablePk(tableName string) (bool, string) { + //优先到缓存中查找 + key := "pk_" + tableName + pk, err := RedisUtil.GET(key) + if err != nil { + sql := "select column_name from information_schema.`key_column_usage` where table_name=? and constraint_name='primary'" + list, err := DbUtil.Engine.SQL(sql, tableName).Query().List() + if err != nil || len(list) == 0 { + LogUtil.Error(ErrorConst.SqlQueryError, err.Error()) + return false, "" + } + pk = list[0]["column_name"].(string) + //设置缓存 + RedisUtil.SET(key, pk, 24*7*time.Hour) + } + return true, pk +}