From ac9396a5f2cb81bcd940f5abf517b67600973088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E6=B5=B7?= <10402852@qq.com> Date: Tue, 6 Apr 2021 17:17:38 +0800 Subject: [PATCH] commit --- dsTools/Backup/esBackup.go | 10 +- dsTools/Backup/mysqlBackup.go | 14 +- dsTools/Config/Config(10.10.14.220).ini | 7 - dsTools/Config/Config(10.10.14.91).ini | 8 -- dsTools/Config/Config(10.10.6.200).ini | 8 -- dsTools/Config/Obs.ini | 7 + dsTools/Utils/ConfigUtil/ConfigUtil.go | 16 --- dsTools/Utils/ObsUtil/ObsUtil.go | 177 ++++++++++++++++++++++-- 8 files changed, 183 insertions(+), 64 deletions(-) create mode 100644 dsTools/Config/Obs.ini diff --git a/dsTools/Backup/esBackup.go b/dsTools/Backup/esBackup.go index d4943190..3c8b4cc5 100644 --- a/dsTools/Backup/esBackup.go +++ b/dsTools/Backup/esBackup.go @@ -131,7 +131,7 @@ func CreateSnap() { func main() { //杀掉老进程 - command:="ps -ef | grep esBackup | grep -v grep | awk '{print $2}' | xargs kill -9" + command := "ps -ef | grep esBackup | grep -v grep | awk '{print $2}' | xargs kill -9" exec.Command("/bin/bash", "-c", command) //1、创建仓库 @@ -149,7 +149,7 @@ func main() { return } //云存储前缀 - var prefix = ConfigUtil.BackupPrefix + "/" + ConfigUtil.InstallAreaCode + "/" + macAddress + var prefix = ObsUtil.BackupPrefix + "/" + ConfigUtil.InstallAreaCode + "/" + macAddress for _, file := range files { if file.IsDir() { continue @@ -164,18 +164,18 @@ func main() { ObsUtil.UploadFileMultiPart(key, sourceFile) } //删除历史文件 - if sumD.Hours()>24 { + if sumD.Hours() > 24 { sourceFile := dirPath + "/" + file.Name() err := os.Remove(sourceFile) if err != nil { - fmt.Println("删除历史文件"+sourceFile+"失败！") + fmt.Println("删除历史文件" + sourceFile + "失败！") } } } } //3、对云备份的文件进行清理 fmt.Println("5、正在进行云备份文件的清理工作...") - ObsUtil.DeleteExpireFile(prefix+"/es_backup_", ConfigUtil.RemainDays) + ObsUtil.DeleteExpireFile(prefix+"/es_backup_", ObsUtil.RemainDays) //4、提示信息 fmt.Println("恭喜，ES备份成功完成！") } diff --git a/dsTools/Backup/mysqlBackup.go b/dsTools/Backup/mysqlBackup.go index db4712ba..bec53aa7 100644 --- a/dsTools/Backup/mysqlBackup.go +++ b/dsTools/Backup/mysqlBackup.go @@ -15,7 +15,7 @@ import ( func main() { //杀掉老进程 - command:="ps -ef | grep mysqlBackup | grep -v grep | awk '{print $2}' | xargs kill -9" + command := "ps -ef | grep mysqlBackup | grep -v grep | awk '{print $2}' | xargs kill -9" cmd := exec.Command("/bin/bash", "-c", command) //1、执行Shell进行备份数据库 @@ -38,33 +38,33 @@ func main() { return } //云存储前缀 - var prefix = ConfigUtil.BackupPrefix + "/" + ConfigUtil.InstallAreaCode + "/" + macAddress + var prefix = ObsUtil.BackupPrefix + "/" + ConfigUtil.InstallAreaCode + "/" + macAddress for _, file := range files { if file.IsDir() { continue } else { now := time.Now() sumD := now.Sub(file.ModTime()) - if sumD.Hours()<2 && strings.Index(file.Name(), "base_db_") == 0 { - fmt.Println("3、找到刚刚备份的数据库文件:"+file.Name()) + if sumD.Hours() < 2 && strings.Index(file.Name(), "base_db_") == 0 { + fmt.Println("3、找到刚刚备份的数据库文件:" + file.Name()) key := prefix + "/" + file.Name() sourceFile := dirPath + "/" + file.Name() fmt.Println("4、正在上传备份的数据库文件...") ObsUtil.UploadFileMultiPart(key, sourceFile) } //删除历史文件 - if sumD.Hours()>24 { + if sumD.Hours() > 24 { sourceFile := dirPath + "/" + file.Name() err := os.Remove(sourceFile) if err != nil { - fmt.Println("删除历史文件"+sourceFile+"失败！") + fmt.Println("删除历史文件" + sourceFile + "失败！") } } } } //3、对云备份的文件进行清理 fmt.Println("5、正在进行云备份文件的清理工作...") - ObsUtil.DeleteExpireFile(prefix+"/base_db_", ConfigUtil.RemainDays) + ObsUtil.DeleteExpireFile(prefix+"/base_db_", ObsUtil.RemainDays) //4、提示信息 fmt.Println("恭喜，数据库备份成功完成！") } diff --git a/dsTools/Config/Config(10.10.14.220).ini b/dsTools/Config/Config(10.10.14.220).ini index ffb16514..1ab00a08 100644 --- a/dsTools/Config/Config(10.10.14.220).ini +++ b/dsTools/Config/Config(10.10.14.220).ini @@ -26,10 +26,3 @@ code = changchunshi [es] host = http://10.10.14.222:9200 -[obs] -bucket = dsideal -ak = WAFBGJACKDOQZDH1MKZ1 -sk = dlWTUbqgCICaYJG3n0Rot4jXaen2HnfFtMVxiPEo -endpoint = obs.cn-north-1.myhuaweicloud.com -backupPrefix = baseServiceDbBackup -remainDays = 15 \ No newline at end of file diff --git a/dsTools/Config/Config(10.10.14.91).ini b/dsTools/Config/Config(10.10.14.91).ini index eaceac7e..6ba39623 100644 --- a/dsTools/Config/Config(10.10.14.91).ini +++ b/dsTools/Config/Config(10.10.14.91).ini @@ -25,11 +25,3 @@ code = changchunshi [es] host = http://10.10.14.222:9200 - -[obs] -bucket = dsideal -ak = WAFBGJACKDOQZDH1MKZ1 -sk = dlWTUbqgCICaYJG3n0Rot4jXaen2HnfFtMVxiPEo -endpoint = obs.cn-north-1.myhuaweicloud.com -backupPrefix = baseServiceDbBackup -remainDays = 15 \ No newline at end of file diff --git a/dsTools/Config/Config(10.10.6.200).ini b/dsTools/Config/Config(10.10.6.200).ini index 5c5d8136..0440dd02 100644 --- a/dsTools/Config/Config(10.10.6.200).ini +++ b/dsTools/Config/Config(10.10.6.200).ini @@ -25,11 +25,3 @@ code = changchunshi [es] host = http://10.10.6.202:9200 - -[obs] -bucket = dsideal -ak = WAFBGJACKDOQZDH1MKZ1 -sk = dlWTUbqgCICaYJG3n0Rot4jXaen2HnfFtMVxiPEo -endpoint = obs.cn-north-1.myhuaweicloud.com -backupPrefix = baseServiceDbBackup -remainDays = 15 \ No newline at end of file diff --git a/dsTools/Config/Obs.ini b/dsTools/Config/Obs.ini new file mode 100644 index 00000000..02c82659 --- /dev/null +++ b/dsTools/Config/Obs.ini @@ -0,0 +1,7 @@ +[obs] +bucket = dsideal +ak = WAFBGJACKDOQZDH1MKZ1 +sk = dlWTUbqgCICaYJG3n0Rot4jXaen2HnfFtMVxiPEo +endpoint = obs.cn-north-1.myhuaweicloud.com +backupPrefix = baseServiceDbBackup +remainDays = 15 \ No newline at end of file diff --git a/dsTools/Utils/ConfigUtil/ConfigUtil.go b/dsTools/Utils/ConfigUtil/ConfigUtil.go index 1bb211cf..f566a011 100644 --- a/dsTools/Utils/ConfigUtil/ConfigUtil.go +++ b/dsTools/Utils/ConfigUtil/ConfigUtil.go @@ -32,14 +32,6 @@ var ( //Es主机 EsHost string - - //Obs - Bucket string - Ak string - Sk string - Endpoint string - BackupPrefix string - RemainDays int32 ) func init() { @@ -91,14 +83,6 @@ func init() { //Es主机配置 EsHost = iniParser.GetString("es", "host") - - //obs配置 - Ak= iniParser.GetString("obs", "ak") - Sk= iniParser.GetString("obs", "sk") - Bucket= iniParser.GetString("obs", "bucket") - Endpoint= iniParser.GetString("obs", "endpoint") - BackupPrefix= iniParser.GetString("obs", "backupPrefix") - RemainDays=iniParser.GetInt32("obs", "remainDays") } type IniParser struct { diff --git a/dsTools/Utils/ObsUtil/ObsUtil.go b/dsTools/Utils/ObsUtil/ObsUtil.go index d3b90642..bef651be 100644 --- a/dsTools/Utils/ObsUtil/ObsUtil.go +++ b/dsTools/Utils/ObsUtil/ObsUtil.go @@ -1,24 +1,56 @@ package ObsUtil import ( - "dsTools/Utils/ConfigUtil" + "dsTools/Utils/CommonUtil" "dsTools/Utils/obs" "fmt" + "gopkg.in/ini.v1" "os" "time" ) +var ( + //Obs + Bucket string + Ak string + Sk string + Endpoint string + BackupPrefix string + RemainDays int32 +) + +func init() { + var configIniFile = "./Config/Obs.ini" + //判断文件不是存在 + if !CommonUtil.Exists(configIniFile) { + configIniFile = "." + configIniFile + } + + iniParser := IniParser{} + if err := iniParser.Load(configIniFile); err != nil { + fmt.Printf("try load config file[%s] error[%s]\n", configIniFile, err.Error()) + return + } + //obs配置 + Ak = iniParser.GetString("obs", "ak") + Sk = iniParser.GetString("obs", "sk") + Bucket = iniParser.GetString("obs", "bucket") + Endpoint = iniParser.GetString("obs", "endpoint") + BackupPrefix = iniParser.GetString("obs", "backupPrefix") + RemainDays = iniParser.GetInt32("obs", "remainDays") +} + /** 功能：删除指定前缀的过期文件 作者：黄海 时间：2020-04-05 */ -func DeleteExpireFile(prefix string,remainDays int32){ +func DeleteExpireFile(prefix string, remainDays int32) { // 创建ObsClient结构体 - var obsClient, _ = obs.New(ConfigUtil.Ak, ConfigUtil.Sk, ConfigUtil.Endpoint) + var obsClient, _ = obs.New(Ak, Sk, Endpoint) input := &obs.ListObjectsInput{} - input.Bucket = ConfigUtil.Bucket + input.Bucket = Bucket // 设置列举带有prefix前缀的1000个对象 input.MaxKeys = 1000 input.Prefix = prefix @@ -32,14 +64,14 @@ func DeleteExpireFile(prefix string,remainDays int32){ now := time.Now() sumD := now.Sub(val.LastModified) if int32(sumD.Hours()/24) > remainDays { - fmt.Println("发现超时文件，将删除:"+val.Key) + fmt.Println("发现超时文件，将删除:" + val.Key) input := &obs.DeleteObjectInput{} - input.Bucket = ConfigUtil.Bucket + input.Bucket = Bucket input.Key = val.Key obsClient.DeleteObject(input) fmt.Println("已成功删除！" + val.Key) - }else{ - fmt.Println("没有过期，无需删除:"+val.Key) + } else { + fmt.Println("没有过期，无需删除:" + val.Key) } } } else if obsError, ok := err.(obs.ObsError); ok { @@ -54,12 +86,12 @@ func DeleteExpireFile(prefix string,remainDays int32){ 作者：黄海 时间：2020-04-05 */ -func UploadFileMultiPart(key string,sourceFile string){ +func UploadFileMultiPart(key string, sourceFile string) { // 创建ObsClient结构体 - var obsClient, _ = obs.New(ConfigUtil.Ak, ConfigUtil.Sk, ConfigUtil.Endpoint) + var obsClient, _ = obs.New(Ak, Sk, Endpoint) // 初始化分段上传任务 input := &obs.InitiateMultipartUploadInput{} - input.Bucket = ConfigUtil.Bucket + input.Bucket = Bucket input.Key = key output, err := obsClient.InitiateMultipartUpload(input) if err != nil { @@ -91,7 +123,7 @@ func UploadFileMultiPart(key string,sourceFile string){ } go func() { uploadPartInput := &obs.UploadPartInput{} - uploadPartInput.Bucket = ConfigUtil.Bucket + uploadPartInput.Bucket = Bucket uploadPartInput.Key = key uploadPartInput.UploadId = uploadId uploadPartInput.SourceFile = sourceFile @@ -124,7 +156,7 @@ func UploadFileMultiPart(key string,sourceFile string){ } completeMultipartUploadInput := &obs.CompleteMultipartUploadInput{} - completeMultipartUploadInput.Bucket = ConfigUtil.Bucket + completeMultipartUploadInput.Bucket = Bucket completeMultipartUploadInput.Key = key completeMultipartUploadInput.UploadId = uploadId completeMultipartUploadInput.Parts = parts @@ -136,3 +168,122 @@ func UploadFileMultiPart(key string,sourceFile string){ fmt.Printf("RequestId:%s\n", completeMultipartUploadOutput.RequestId) obsClient.Close() } + +type IniParser struct { + confReader *ini.File // config reader +} + +type IniParserError struct { + errorInfo string +} + +func (e *IniParserError) Error() string { return e.errorInfo } + +func (this *IniParser) Load(configFileName string) error { + conf, err := ini.Load(configFileName) + if err != nil { + this.confReader = nil + return err + } + this.confReader = conf + return nil +} + +func (this *IniParser) GetString(section string, key string) string { + if this.confReader == nil { + return "" + } + + s := this.confReader.Section(section) + if s == nil { + return "" + } + + return s.Key(key).String() +} + +func (this *IniParser) GetInt32(section string, key string) int32 { + if this.confReader == nil { + return 0 + } + + s := this.confReader.Section(section) + if s == nil { + return 0 + } + + valueInt, _ := s.Key(key).Int() + + return int32(valueInt) +} + +func (this *IniParser) GetUint32(section string, key string) uint32 { + if this.confReader == nil { + return 0 + } + + s := this.confReader.Section(section) + if s == nil { + return 0 + } + + valueInt, _ := s.Key(key).Uint() + + return uint32(valueInt) +} + +func (this *IniParser) GetInt64(section string, key string) int64 { + if this.confReader == nil { + return 0 + } + + s := this.confReader.Section(section) + if s == nil { + return 0 + } + + valueInt, _ := s.Key(key).Int64() + return valueInt +} + +func (this *IniParser) GetUint64(section string, key string) uint64 { + if this.confReader == nil { + return 0 + } + + s := this.confReader.Section(section) + if s == nil { + return 0 + } + + valueInt, _ := s.Key(key).Uint64() + return valueInt +} + +func (this *IniParser) GetFloat32(section string, key string) float32 { + if this.confReader == nil { + return 0 + } + + s := this.confReader.Section(section) + if s == nil { + return 0 + } + + valueFloat, _ := s.Key(key).Float64() + return float32(valueFloat) +} + +func (this *IniParser) GetFloat64(section string, key string) float64 { + if this.confReader == nil { + return 0 + } + + s := this.confReader.Section(section) + if s == nil { + return 0 + } + + valueFloat, _ := s.Key(key).Float64() + return valueFloat +}