bigdata
/
dsMin
6
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

COPY Utils/CacheUtil

Browse Source
master
wangshuai 5 years ago
parent b3cfeaad09
commit 82b087051f
3 changed files with 532 additions and 0 deletions
Show Stats Download Patch File Download Diff File

200
dsSupport/Utils/CacheUtil/DaoCache.go
Unescape View File

@ -0,0 +1,200 @@
package CacheUtil
import (
"dsDataex/Utils/ErrorConst"
"dsDataex/Utils/CommonUtil"
"dsDataex/Utils/DbUtil"
"dsDataex/Utils/LogUtil"
"dsDataex/Utils/RedisUtil"
"fmt"
"github.com/go-redis/redis/v7"
"reflect"
"strconv"
"strings"
"time"
)
/**
功能：优先从缓存读取，检查到缺失的，就从数据库读取，然后两个结果集拼接后返回，同时将缺失的缓存补全
作者：黄海
时间：2020-02-05
*/
func GetListByIds(ids []string, m Selector) []map[string]interface{} {
//1、第一步从缓存读取
list, notExistsIds := batchReadRedis(ids, m.RedisPrefix)
//2、如果有一些ID在缓存中不存在，那么需要从数据库中查询，然后填充到缓存中
if notExistsIds != nil && len(notExistsIds) > 0 {
//从数据库中获取缺失的数据
lostCacheList := fromDb(notExistsIds, m)
//缓存写入
batchWriteRedis(lostCacheList, m)
//change by zhangjun 2020-05-19
//合并两个结果集
list2, _ := batchReadRedis(notExistsIds, m.RedisPrefix)
for i := 0; i < len(list2); i++ {
list = append(list, list2[i])
}
}
return list
}
/**
功能：删除指定ID的缓存
作者：黄海
时间：2020-02-21
*/
func DeleteCacheByIds(ids []string, m Selector) {
prefix := m.RedisPrefix
for i := 0; i < len(ids); i++ {
id := ids[i]
key := prefix + id
RedisUtil.DEL(key)
}
}
/*===================以下为内部函数，用于扩展从REDIS缓存中查询===========================*/
func fromDb(idAry []string, m Selector) []map[string]interface{} {
tableName := m.TableName
pkField := m.PkField
//查询数据库
ids := strings.Join(idAry, "','")
sqlRaw := fmt.Sprintf(`select * from `+tableName+` where `+pkField+` in ('%s')`, ids)
results, err := DbUtil.Engine.SQL(sqlRaw).Query().List()
if err != nil {
LogUtil.Error(ErrorConst.SqlQueryError, "从数据库中读取数据失败:"+err.Error())
}
return results
}
/**
功能：批量写入缓存
作者：黄海
时间：2020-02-05
*/
func batchWriteRedis(list []map[string]interface{}, m Selector) {
pkField := m.PkField
prefix := m.RedisPrefix
//声明管道
pipe := RedisUtil.RedisClient.Pipeline()
for i := 0; i < len(list); i++ {
bean := list[i]
var pkSuffix = ""
switch bean[pkField].(type) {
case string:
pkSuffix = bean[pkField].(string)
case int64:
pkSuffix = strconv.FormatInt(reflect.ValueOf(bean[pkField]).Int(), 10)
}
key := prefix + pkSuffix
pipe.HSet(key, bean)
pipe.Expire(key, 24*time.Hour)
}
//执行管道
//change by zhangjun 2020-05-15
//pipe.Exec()
_, err := pipe.Exec()
if err != nil {
LogUtil.Error(ErrorConst.SqlQueryError, err.Error())
}
}
/**
功能：从REDIS中批量获取指定ID的列表信息
作者：黄海
时间：2020-02-05
*/
func batchReadRedis(ids []string, prefix string) ([]map[string]interface{}, []string) {
//1、创建管道
var list []*redis.StringStringMapCmd
pipe := RedisUtil.RedisClient.Pipeline()
for i := 0; i < len(ids); i++ {
list = append(list, pipe.HGetAll(prefix+ids[i]))
}
//2、执行管道
_, err := pipe.Exec()
if err != nil {
LogUtil.Error(ErrorConst.SqlQueryError, err.Error())
}
//存在于缓存中的数据
var existList []map[string]interface{}
//3、输出结果
//不存在的主键有哪些
var notExistsIds []string
for i := 0; i < len(list); i++ {
_bean := list[i].Val()
if len(_bean) > 0 {
//将values转成Interface{},其实这个锅 Golang是不背的，因为redis中hset写入的整数，取出来hget也是字符串~
n := CommonUtil.ConvertMapStringToMapInterface(_bean)
//找到了加到返回值列表中去
existList = append(existList, n)
} else {
notExistsIds = append(notExistsIds, ids[i])
}
}
//返回值
return existList, notExistsIds
}
/**
功能：对于简单表提供分页的通用SQL组装方法，返回通用分页的SQL语句和获取查询总数的SQL语句
作者：黄海
时间：2020-02-21
*/
func count(baseSql string, pkField string, args []interface{}) (int32, error) {
//替换掉分页代码，获取数据量总数
countSql := strings.Replace(baseSql, "SELECT "+pkField+" ", "SELECT count(*) as count ", -1)
var count int32
//去掉最后的limit+offset
args = args[0 : len(args)-2]
_, err := DbUtil.Engine.SQL(countSql, args...).Get(&count)
if err != nil {
LogUtil.Error(ErrorConst.SqlQueryError, err.Error())
return 0, err
}
return count, nil
}
/**
功能：将指定的SQL语句计算分页的SQL
作者：黄海
时间：2020-03-25
*/
func Page(baseSql string, args ...interface{}) ([]map[string]interface{}, int32, error) {
var model Selector
//删除多余的空格
baseSql=CommonUtil.DeleteExtraSpace(baseSql)
arr:=strings.Split(baseSql," ")
for i:=0;i< len(arr)-1;i++{
if strings.ToLower(arr[i])=="from"{
model= GetBean(arr[i+1])
break
}
}
//条件查询语句
conditionSql := fmt.Sprintf("%s", " limit ? offset ? ")
//分页的语句
pageSql := fmt.Sprintf("%s %s", baseSql, conditionSql)
//分页数据
list, err := DbUtil.Engine.SQL(pageSql, args...).Query().List()
if err != nil {
LogUtil.Error(ErrorConst.SqlQueryError, err.Error())
return nil, 0, err
}
//总数
count,err := count(baseSql, model.PkField, args)
if err != nil {
LogUtil.Error(ErrorConst.SqlQueryError, err.Error())
return nil, 0, err
}
//有哪些ids?
var ids = CommonUtil.Ids(list, model.PkField)
//-->缓存层-->数据库-->缓存
list = GetListByIds(ids, model)
return list, count, nil
}

275
dsSupport/Utils/CacheUtil/OrgtreeCache.go
Unescape View File

@ -0,0 +1,275 @@
package CacheUtil
import (
"dsDataex/Utils/ErrorConst"
"dsDataex/Utils/LogUtil"
"dsDataex/Utils/RedisUtil"
"strings"
"time"
)
var GBT2260 map[string]string
var XXBXLXM map[string]string
var PARENT_IDS []string
/**
* @Author zhangjun
* @Description 组织机构树缓存初始化，【机构下属学校】递归查询、并缓存 Redis
* @Description TODO：如果性能还不行，使用全局静态变量存储【机构下属学校】！！！！！
* @Date 2020-06-16 10:06
* @Param
* @return
**/
func OrgtreeCacheInit(){
//删除Redis缓存
RedisUtil.RedisClient.FlushDB()
//机构树缓存
sql := "SELECT id from t_dataex_orgtree where delete_flag = -1 and enable_flag = 1 and parent_id = '-1'"
list, count, _ := Page(sql, 1000,0)
PARENT_IDS=make([]string,0)
if count>0{
for no:=0;no<int(count);no++{
//TODO:目前只支持教育局下属学校的数据访问控制，暂不支持其他机构类型（如：大学区、主校分校...）
if list[no]["org_type"].(string)=="1"{
sql = "SELECT id from t_dataex_orgtree where delete_flag = -1 and enable_flag = 1 and parent_id = '"+list[no]["id"].(string)+"'"
list2, count2, _ := Page(sql, 1000,0)
if count2>0{
Orgtree2Redis(list2,"Dataex_Scope_"+list[no]["id"].(string))
PARENT_IDS =append(PARENT_IDS,list[no]["id"].(string))
OrgCacheCreate(list[no]["id"].(string),list[no]["id"].(string))
}
}
}
}
//行政区划缓存
sql = "SELECT area_code from t_dataex_gbt2260"
list,_,_= Page(sql, 5000,0)
GBT2260=make( map[string]string)
for no:=0;no< len(list);no++{
GBT2260[list[no]["area_code"].(string)]=list[no]["area_name"].(string)
}
XXBXLXM=make( map[string]string)
XXBXLXM["111"]="幼儿园"
XXBXLXM["119"]="附设幼儿班"
XXBXLXM["211"]="小学"
XXBXLXM["218"]="小学教学点"
XXBXLXM["219"]="附设小学班"
XXBXLXM["221"]="职工小学"
XXBXLXM["222"]="农民小学"
XXBXLXM["228"]="小学班"
XXBXLXM["229"]="扫盲班"
XXBXLXM["311"]="初级中学"
XXBXLXM["312"]="九年一贯制学校"
XXBXLXM["319"]="附设普通初中班"
XXBXLXM["321"]="职业初中"
XXBXLXM["329"]="附设职业初中班"
XXBXLXM["331"]="成人职工初中"
XXBXLXM["332"]="成人农民初中"
XXBXLXM["341"]="完全中学"
XXBXLXM["342"]="高级中学"
XXBXLXM["345"]="十二年一贯制学校"
XXBXLXM["349"]="附设普通高中班"
XXBXLXM["351"]="成人职工高中"
XXBXLXM["352"]="成人农民高中"
XXBXLXM["361"]="中等职业学校"
XXBXLXM["362"]="中等技术学校"
XXBXLXM["363"]="中等师范学校"
XXBXLXM["364"]="成人中等专业学校"
XXBXLXM["365"]="职业高中学校"
XXBXLXM["366"]="技工学校"
XXBXLXM["368"]="附设中职班"
XXBXLXM["369"]="其他机构"
XXBXLXM["371"]="工读学校"
XXBXLXM["411"]="大学"
XXBXLXM["412"]="学院"
XXBXLXM["413"]="独立学院"
XXBXLXM["414"]="高等专科学校"
XXBXLXM["415"]="高等职业学校"
XXBXLXM["419"]="分校、大专班"
XXBXLXM["421"]="职工高校"
XXBXLXM["422"]="农民高校"
XXBXLXM["423"]="管理干部学院"
XXBXLXM["424"]="教育学院"
XXBXLXM["425"]="独立函授学院"
XXBXLXM["426"]="广播电视大学"
XXBXLXM["429"]="其他成人高等教育机构"
XXBXLXM["511"]="盲人学校"
XXBXLXM["512"]="聋人学校"
XXBXLXM["513"]="弱智学校"
XXBXLXM["514"]="特殊教育学校"
XXBXLXM["519"]="附设特教班"
XXBXLXM["911"]="培养研究生的科研机构"
XXBXLXM["921"]="民办的其他高等教育机构"
XXBXLXM["931"]="职工技术培训学校"
XXBXLXM["932"]="农村成人文化技术培训学校"
XXBXLXM["933"]="其他培训机构"
}
func RefreshOrgtree(){
//删除Redis缓存
RedisUtil.RedisClient.FlushDB()
//机构树缓存
sql := "SELECT id from t_dataex_orgtree where delete_flag = -1 and enable_flag = 1 and parent_id = '-1'"
list, count, _ := Page(sql, 1000,0)
if count>0{
for no:=0;no<int(count);no++{
//TODO:目前只支持教育局下属学校的数据访问控制，暂不支持其他机构类型（如：大学区、主校分校...）
if list[no]["org_type"].(string)=="1"{
sql = "SELECT id from t_dataex_orgtree where delete_flag = -1 and enable_flag = 1 and parent_id = '"+list[no]["id"].(string)+"'"
list2, count2, _ := Page(sql, 1000,0)
if count2>0{
Orgtree2Redis(list2,"Dataex_Scope_"+list[no]["id"].(string))
PARENT_IDS =append(PARENT_IDS,list[no]["id"].(string))
OrgCacheCreate(list[no]["id"].(string),list[no]["id"].(string))
}
}
}
}
}
/**
* @Author zhangjun
* @Description 刷新机构下属学校缓存
* @Date 2020-06-16 10:14
* @Param orgID string 教育局机构ID
* @return
**/
func OrgCacheCreate(orgID string,scope... string) {
sql := "SELECT id from t_dataex_orgtree where delete_flag = -1 and enable_flag = 1 and parent_id = '" + orgID + "'"
list, count, _ := Page(sql, 1000,0)
if count>0{
for no:=0;no<int(count);no++{
//TODO:目前只支持教育局下属学校的数据访问控制，暂不支持其他机构类型（如：大学区、主校分校...）
if list[no]["org_type"].(string)=="1"{
sql = "SELECT id from t_dataex_orgtree where delete_flag = -1 and enable_flag = 1 and parent_id = '"+list[no]["id"].(string)+"'"
list2, count2, _ := Page(sql, 1000,0)
if count2>0{
Orgtree2Redis(list2,"Dataex_Scope_"+list[no]["id"].(string))
PARENT_IDS =append(PARENT_IDS,list[no]["id"].(string))
var newscope =append( scope ,list[no]["id"].(string))
for no:=0;no<len(scope);no++{
Orgtree2Redis(list2,"Dataex_Scope_"+scope[no])
}
OrgCacheCreate(list[no]["id"].(string),newscope...)
}
}
}
}
}
/**
* @Author zhangjun
* @Description
* @Date 2020-06-16 10:49
* @Param list 【】map 机构范围数据【】数组
* @Param cacheName string 缓存名称
* @return
**/
func Orgtree2Redis(list []map[string]interface{}, cacheName string) {
var ids string
//add by zhangjun 2020-08-05
old,_ :=RedisUtil.RedisClient.Get(cacheName).Result()
ids=old
for i := 0; i < len(list); i++ {
bean := list[i]
ids=ids+"##"+bean["id"].(string)
}
//change by zhangjun 2020-08-06
//RedisUtil.RedisClient.Set(cacheName, ids,10*time.Hour)
RedisUtil.RedisClient.Set(cacheName, ids,0 )
}
func Orgtree2Redis2(list []map[string]interface{}, cacheName string) {
//声明管道
pipe := RedisUtil.RedisClient.Pipeline()
for i := 0; i < len(list); i++ {
bean := list[i]
key := cacheName + ":" + bean["id"].(string)
pipe.HSet(key, bean)
pipe.Expire(key, 24*time.Hour)
}
_, err := pipe.Exec()
if err != nil {
LogUtil.Error(ErrorConst.SqlQueryError, err.Error())
}
}
/**
* @Author zhangjun
* @Description
* @Date 2020-06-16 11:21
* @Param
* @return
**/
func OrgtreeCheckScope(provideID string,orgID string)bool{
result,err:= RedisUtil.RedisClient.Get("Dataex_Scope_"+provideID).Result()
if err == nil{
if strings.Contains(result,orgID){
return true
}else {
return false
}
}
return false
}
func OrgtreeCheckScope2(provideID string,orgID string)bool{
count,_:= RedisUtil.RedisClient.Exists("Dataex_Scope_"+provideID+":"+orgID).Result()
if count>0{
return true
}
return false
}
func OrgtreeGetScope(orgId string) []string {
result,_:= RedisUtil.RedisClient.Get("Dataex_Scope_"+orgId).Result()
if result != ""{
return strings.Split(result,"##")
}
return nil
}

57
dsSupport/Utils/CacheUtil/Selector.go
Unescape View File

@ -0,0 +1,57 @@
package CacheUtil
import (
"dsDataex/Utils/ErrorConst"
"dsDataex/Utils/CommonUtil"
"dsDataex/Utils/DbUtil"
"dsDataex/Utils/LogUtil"
"dsDataex/Utils/RedisUtil"
"time"
)
/**
功能：用于将interface转为struct的结构体
作者：黄海
时间：2020-02-05
*/
type Selector struct {
TableName string
PkField string
RedisPrefix string
}
/**
功能：注册一个函数，用于提取计算指定数据表对应的相关信息
作者：黄海
时间：2020-03-27
*/
func GetBean(tableName string) Selector {
var m = new(Selector)
m.TableName = tableName
m.RedisPrefix = CommonUtil.GetSnakeCaseStr(tableName) + ":"
_, m.PkField = GetTablePk(tableName)
return *m
}
/**
功能：获取指定表的主键
作者：黄海
时间：2020-03-26
*/
func GetTablePk(tableName string) (bool, string) {
//优先到缓存中查找
key := "pk_" + tableName
pk, err := RedisUtil.GET(key)
if err != nil {
sql := "select column_name from information_schema.`key_column_usage` where table_name=? and constraint_name='primary'"
list, err := DbUtil.Engine.SQL(sql, tableName).Query().List()
if err != nil || len(list) == 0 {
LogUtil.Error(ErrorConst.SqlQueryError, err.Error())
return false, ""
}
pk = list[0]["column_name"].(string)
//设置缓存
RedisUtil.SET(key, pk, 24*7*time.Hour)
}
return true, pk
}
Write Preview
Loading…
Cancel
Save