bigdata
/
dsMin
6
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

commit

Browse Source
master
黄海 4 years ago
parent 7635f290fc
commit ac9396a5f2
8 changed files with 183 additions and 64 deletions
Show Stats Download Patch File Download Diff File

10
dsTools/Backup/esBackup.go
Unescape View File

@ -131,7 +131,7 @@ func CreateSnap() {
func main() {
//杀掉老进程
command:="ps -ef | grep esBackup | grep -v grep | awk '{print $2}' | xargs kill -9"
command := "ps -ef | grep esBackup | grep -v grep | awk '{print $2}' | xargs kill -9"
exec.Command("/bin/bash", "-c", command)
//1、创建仓库
@ -149,7 +149,7 @@ func main() {
return
}
//云存储前缀
var prefix = ConfigUtil.BackupPrefix + "/" + ConfigUtil.InstallAreaCode + "/" + macAddress
var prefix = ObsUtil.BackupPrefix + "/" + ConfigUtil.InstallAreaCode + "/" + macAddress
for _, file := range files {
if file.IsDir() {
continue
@ -164,18 +164,18 @@ func main() {
ObsUtil.UploadFileMultiPart(key, sourceFile)
}
//删除历史文件
if sumD.Hours()>24 {
if sumD.Hours() > 24 {
sourceFile := dirPath + "/" + file.Name()
err := os.Remove(sourceFile)
if err != nil {
fmt.Println("删除历史文件"+sourceFile+"失败！")
fmt.Println("删除历史文件" + sourceFile + "失败！")
}
}
}
}
//3、对云备份的文件进行清理
fmt.Println("5、正在进行云备份文件的清理工作...")
ObsUtil.DeleteExpireFile(prefix+"/es_backup_", ConfigUtil.RemainDays)
ObsUtil.DeleteExpireFile(prefix+"/es_backup_", ObsUtil.RemainDays)
//4、提示信息
fmt.Println("恭喜，ES备份成功完成！")
}

14
dsTools/Backup/mysqlBackup.go
Unescape View File

@ -15,7 +15,7 @@ import (
func main() {
//杀掉老进程
command:="ps -ef | grep mysqlBackup | grep -v grep | awk '{print $2}' | xargs kill -9"
command := "ps -ef | grep mysqlBackup | grep -v grep | awk '{print $2}' | xargs kill -9"
cmd := exec.Command("/bin/bash", "-c", command)
//1、执行Shell进行备份数据库
@ -38,33 +38,33 @@ func main() {
return
}
//云存储前缀
var prefix = ConfigUtil.BackupPrefix + "/" + ConfigUtil.InstallAreaCode + "/" + macAddress
var prefix = ObsUtil.BackupPrefix + "/" + ConfigUtil.InstallAreaCode + "/" + macAddress
for _, file := range files {
if file.IsDir() {
continue
} else {
now := time.Now()
sumD := now.Sub(file.ModTime())
if sumD.Hours()<2 && strings.Index(file.Name(), "base_db_") == 0 {
fmt.Println("3、找到刚刚备份的数据库文件:"+file.Name())
if sumD.Hours() < 2 && strings.Index(file.Name(), "base_db_") == 0 {
fmt.Println("3、找到刚刚备份的数据库文件:" + file.Name())
key := prefix + "/" + file.Name()
sourceFile := dirPath + "/" + file.Name()
fmt.Println("4、正在上传备份的数据库文件...")
ObsUtil.UploadFileMultiPart(key, sourceFile)
}
//删除历史文件
if sumD.Hours()>24 {
if sumD.Hours() > 24 {
sourceFile := dirPath + "/" + file.Name()
err := os.Remove(sourceFile)
if err != nil {
fmt.Println("删除历史文件"+sourceFile+"失败！")
fmt.Println("删除历史文件" + sourceFile + "失败！")
}
}
}
}
//3、对云备份的文件进行清理
fmt.Println("5、正在进行云备份文件的清理工作...")
ObsUtil.DeleteExpireFile(prefix+"/base_db_", ConfigUtil.RemainDays)
ObsUtil.DeleteExpireFile(prefix+"/base_db_", ObsUtil.RemainDays)
//4、提示信息
fmt.Println("恭喜，数据库备份成功完成！")
}

7
dsTools/Config/Config(10.10.14.220).ini
Unescape View File

@ -26,10 +26,3 @@ code = changchunshi
[es]
host = http://10.10.14.222:9200
[obs]
bucket = dsideal
ak = WAFBGJACKDOQZDH1MKZ1
sk = dlWTUbqgCICaYJG3n0Rot4jXaen2HnfFtMVxiPEo
endpoint = obs.cn-north-1.myhuaweicloud.com
backupPrefix = baseServiceDbBackup
remainDays = 15

8
dsTools/Config/Config(10.10.14.91).ini
Unescape View File

@ -25,11 +25,3 @@ code = changchunshi
[es]
host = http://10.10.14.222:9200
[obs]
bucket = dsideal
ak = WAFBGJACKDOQZDH1MKZ1
sk = dlWTUbqgCICaYJG3n0Rot4jXaen2HnfFtMVxiPEo
endpoint = obs.cn-north-1.myhuaweicloud.com
backupPrefix = baseServiceDbBackup
remainDays = 15

8
dsTools/Config/Config(10.10.6.200).ini
Unescape View File

@ -25,11 +25,3 @@ code = changchunshi
[es]
host = http://10.10.6.202:9200
[obs]
bucket = dsideal
ak = WAFBGJACKDOQZDH1MKZ1
sk = dlWTUbqgCICaYJG3n0Rot4jXaen2HnfFtMVxiPEo
endpoint = obs.cn-north-1.myhuaweicloud.com
backupPrefix = baseServiceDbBackup
remainDays = 15

7
dsTools/Config/Obs.ini
Unescape View File

@ -0,0 +1,7 @@
[obs]
bucket = dsideal
ak = WAFBGJACKDOQZDH1MKZ1
sk = dlWTUbqgCICaYJG3n0Rot4jXaen2HnfFtMVxiPEo
endpoint = obs.cn-north-1.myhuaweicloud.com
backupPrefix = baseServiceDbBackup
remainDays = 15

16
dsTools/Utils/ConfigUtil/ConfigUtil.go
Unescape View File

@ -32,14 +32,6 @@ var (
//Es主机
EsHost string
//Obs
Bucket string
Ak string
Sk string
Endpoint string
BackupPrefix string
RemainDays int32
)
func init() {
@ -91,14 +83,6 @@ func init() {
//Es主机配置
EsHost = iniParser.GetString("es", "host")
//obs配置
Ak= iniParser.GetString("obs", "ak")
Sk= iniParser.GetString("obs", "sk")
Bucket= iniParser.GetString("obs", "bucket")
Endpoint= iniParser.GetString("obs", "endpoint")
BackupPrefix= iniParser.GetString("obs", "backupPrefix")
RemainDays=iniParser.GetInt32("obs", "remainDays")
}
type IniParser struct {

177
dsTools/Utils/ObsUtil/ObsUtil.go
Unescape View File

@ -1,24 +1,56 @@
package ObsUtil
import (
"dsTools/Utils/ConfigUtil"
"dsTools/Utils/CommonUtil"
"dsTools/Utils/obs"
"fmt"
"gopkg.in/ini.v1"
"os"
"time"
)
var (
//Obs
Bucket string
Ak string
Sk string
Endpoint string
BackupPrefix string
RemainDays int32
)
func init() {
var configIniFile = "./Config/Obs.ini"
//判断文件不是存在
if !CommonUtil.Exists(configIniFile) {
configIniFile = "." + configIniFile
}
iniParser := IniParser{}
if err := iniParser.Load(configIniFile); err != nil {
fmt.Printf("try load config file[%s] error[%s]\n", configIniFile, err.Error())
return
}
//obs配置
Ak = iniParser.GetString("obs", "ak")
Sk = iniParser.GetString("obs", "sk")
Bucket = iniParser.GetString("obs", "bucket")
Endpoint = iniParser.GetString("obs", "endpoint")
BackupPrefix = iniParser.GetString("obs", "backupPrefix")
RemainDays = iniParser.GetInt32("obs", "remainDays")
}
/**
功能：删除指定前缀的过期文件
作者：黄海
时间：2020-04-05
*/
func DeleteExpireFile(prefix string,remainDays int32){
func DeleteExpireFile(prefix string, remainDays int32) {
// 创建ObsClient结构体
var obsClient, _ = obs.New(ConfigUtil.Ak, ConfigUtil.Sk, ConfigUtil.Endpoint)
var obsClient, _ = obs.New(Ak, Sk, Endpoint)
input := &obs.ListObjectsInput{}
input.Bucket = ConfigUtil.Bucket
input.Bucket = Bucket
// 设置列举带有prefix前缀的1000个对象
input.MaxKeys = 1000
input.Prefix = prefix
@ -32,14 +64,14 @@ func DeleteExpireFile(prefix string,remainDays int32){
now := time.Now()
sumD := now.Sub(val.LastModified)
if int32(sumD.Hours()/24) > remainDays {
fmt.Println("发现超时文件，将删除:"+val.Key)
fmt.Println("发现超时文件，将删除:" + val.Key)
input := &obs.DeleteObjectInput{}
input.Bucket = ConfigUtil.Bucket
input.Bucket = Bucket
input.Key = val.Key
obsClient.DeleteObject(input)
fmt.Println("已成功删除！" + val.Key)
}else{
fmt.Println("没有过期，无需删除:"+val.Key)
} else {
fmt.Println("没有过期，无需删除:" + val.Key)
}
}
} else if obsError, ok := err.(obs.ObsError); ok {
@ -54,12 +86,12 @@ func DeleteExpireFile(prefix string,remainDays int32){
作者：黄海
时间：2020-04-05
*/
func UploadFileMultiPart(key string,sourceFile string){
func UploadFileMultiPart(key string, sourceFile string) {
// 创建ObsClient结构体
var obsClient, _ = obs.New(ConfigUtil.Ak, ConfigUtil.Sk, ConfigUtil.Endpoint)
var obsClient, _ = obs.New(Ak, Sk, Endpoint)
// 初始化分段上传任务
input := &obs.InitiateMultipartUploadInput{}
input.Bucket = ConfigUtil.Bucket
input.Bucket = Bucket
input.Key = key
output, err := obsClient.InitiateMultipartUpload(input)
if err != nil {
@ -91,7 +123,7 @@ func UploadFileMultiPart(key string,sourceFile string){
}
go func() {
uploadPartInput := &obs.UploadPartInput{}
uploadPartInput.Bucket = ConfigUtil.Bucket
uploadPartInput.Bucket = Bucket
uploadPartInput.Key = key
uploadPartInput.UploadId = uploadId
uploadPartInput.SourceFile = sourceFile
@ -124,7 +156,7 @@ func UploadFileMultiPart(key string,sourceFile string){
}
completeMultipartUploadInput := &obs.CompleteMultipartUploadInput{}
completeMultipartUploadInput.Bucket = ConfigUtil.Bucket
completeMultipartUploadInput.Bucket = Bucket
completeMultipartUploadInput.Key = key
completeMultipartUploadInput.UploadId = uploadId
completeMultipartUploadInput.Parts = parts
@ -136,3 +168,122 @@ func UploadFileMultiPart(key string,sourceFile string){
fmt.Printf("RequestId:%s\n", completeMultipartUploadOutput.RequestId)
obsClient.Close()
}
type IniParser struct {
confReader *ini.File // config reader
}
type IniParserError struct {
errorInfo string
}
func (e *IniParserError) Error() string { return e.errorInfo }
func (this *IniParser) Load(configFileName string) error {
conf, err := ini.Load(configFileName)
if err != nil {
this.confReader = nil
return err
}
this.confReader = conf
return nil
}
func (this *IniParser) GetString(section string, key string) string {
if this.confReader == nil {
return ""
}
s := this.confReader.Section(section)
if s == nil {
return ""
}
return s.Key(key).String()
}
func (this *IniParser) GetInt32(section string, key string) int32 {
if this.confReader == nil {
return 0
}
s := this.confReader.Section(section)
if s == nil {
return 0
}
valueInt, _ := s.Key(key).Int()
return int32(valueInt)
}
func (this *IniParser) GetUint32(section string, key string) uint32 {
if this.confReader == nil {
return 0
}
s := this.confReader.Section(section)
if s == nil {
return 0
}
valueInt, _ := s.Key(key).Uint()
return uint32(valueInt)
}
func (this *IniParser) GetInt64(section string, key string) int64 {
if this.confReader == nil {
return 0
}
s := this.confReader.Section(section)
if s == nil {
return 0
}
valueInt, _ := s.Key(key).Int64()
return valueInt
}
func (this *IniParser) GetUint64(section string, key string) uint64 {
if this.confReader == nil {
return 0
}
s := this.confReader.Section(section)
if s == nil {
return 0
}
valueInt, _ := s.Key(key).Uint64()
return valueInt
}
func (this *IniParser) GetFloat32(section string, key string) float32 {
if this.confReader == nil {
return 0
}
s := this.confReader.Section(section)
if s == nil {
return 0
}
valueFloat, _ := s.Key(key).Float64()
return float32(valueFloat)
}
func (this *IniParser) GetFloat64(section string, key string) float64 {
if this.confReader == nil {
return 0
}
s := this.confReader.Section(section)
if s == nil {
return 0
}
valueFloat, _ := s.Key(key).Float64()
return valueFloat
}

Write Preview
Loading…
Cancel
Save