|
|
package ObsUtil
|
|
|
|
|
|
import (
|
|
|
"dsTools/Utils/CommonUtil"
|
|
|
"dsTools/Utils/obs"
|
|
|
"fmt"
|
|
|
"gopkg.in/ini.v1"
|
|
|
"os"
|
|
|
"time"
|
|
|
)
|
|
|
|
|
|
var (
|
|
|
//Obs
|
|
|
Bucket string
|
|
|
Ak string
|
|
|
Sk string
|
|
|
Endpoint string
|
|
|
BackupPrefix string
|
|
|
RemainDays int32
|
|
|
)
|
|
|
|
|
|
func init() {
|
|
|
var configIniFile = "./Config/Obs.ini"
|
|
|
//判断文件不是存在
|
|
|
if !CommonUtil.Exists(configIniFile) {
|
|
|
configIniFile = "." + configIniFile
|
|
|
}
|
|
|
|
|
|
iniParser := IniParser{}
|
|
|
if err := iniParser.Load(configIniFile); err != nil {
|
|
|
fmt.Printf("try load config file[%s] error[%s]\n", configIniFile, err.Error())
|
|
|
return
|
|
|
}
|
|
|
//obs配置
|
|
|
Ak = iniParser.GetString("obs", "ak")
|
|
|
Sk = iniParser.GetString("obs", "sk")
|
|
|
Bucket = iniParser.GetString("obs", "bucket")
|
|
|
Endpoint = iniParser.GetString("obs", "endpoint")
|
|
|
BackupPrefix = iniParser.GetString("obs", "backupPrefix")
|
|
|
RemainDays = iniParser.GetInt32("obs", "remainDays")
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
功能：删除指定前缀的过期文件
|
|
|
作者：黄海
|
|
|
时间：2020-04-05
|
|
|
*/
|
|
|
func DeleteExpireFile(prefix string, remainDays int32) {
|
|
|
// 创建ObsClient结构体
|
|
|
var obsClient, _ = obs.New(Ak, Sk, Endpoint)
|
|
|
|
|
|
input := &obs.ListObjectsInput{}
|
|
|
input.Bucket = Bucket
|
|
|
// 设置列举带有prefix前缀的1000个对象
|
|
|
input.MaxKeys = 1000
|
|
|
input.Prefix = prefix
|
|
|
|
|
|
output, err := obsClient.ListObjects(input)
|
|
|
if err == nil {
|
|
|
for index, val := range output.Contents {
|
|
|
fmt.Printf("Content[%d]-OwnerId:%s, ETag:%s, Key:%s, LastModified:%s, Size:%d, StorageClass:%s\n",
|
|
|
index, val.Owner.ID, val.ETag, val.Key, val.LastModified, val.Size, val.StorageClass)
|
|
|
//判断是否过期
|
|
|
now := time.Now()
|
|
|
sumD := now.Sub(val.LastModified)
|
|
|
if int32(sumD.Hours()/24) > remainDays {
|
|
|
fmt.Println("发现超时文件，将删除:" + val.Key)
|
|
|
input := &obs.DeleteObjectInput{}
|
|
|
input.Bucket = Bucket
|
|
|
input.Key = val.Key
|
|
|
obsClient.DeleteObject(input)
|
|
|
fmt.Println("已成功删除！" + val.Key)
|
|
|
} else {
|
|
|
fmt.Println("没有过期，无需删除:" + val.Key)
|
|
|
}
|
|
|
}
|
|
|
} else if obsError, ok := err.(obs.ObsError); ok {
|
|
|
fmt.Printf("Code:%s\n", obsError.Code)
|
|
|
fmt.Printf("Message:%s\n", obsError.Message)
|
|
|
}
|
|
|
obsClient.Close()
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
功能：上传本地文件到华为云存储
|
|
|
作者：黄海
|
|
|
时间：2020-04-05
|
|
|
*/
|
|
|
func UploadFileMultiPart(key string, sourceFile string) {
|
|
|
// 创建ObsClient结构体
|
|
|
var obsClient, _ = obs.New(Ak, Sk, Endpoint)
|
|
|
// 初始化分段上传任务
|
|
|
input := &obs.InitiateMultipartUploadInput{}
|
|
|
input.Bucket = Bucket
|
|
|
input.Key = key
|
|
|
output, err := obsClient.InitiateMultipartUpload(input)
|
|
|
if err != nil {
|
|
|
panic(err)
|
|
|
}
|
|
|
uploadId := output.UploadId
|
|
|
fmt.Printf("UploadId:%s\n", uploadId)
|
|
|
fmt.Println()
|
|
|
// 每段上传100MB
|
|
|
var partSize int64 = 100 * 1024 * 1024
|
|
|
stat, err := os.Stat(sourceFile)
|
|
|
if err != nil {
|
|
|
panic(err)
|
|
|
}
|
|
|
fileSize := stat.Size()
|
|
|
// 计算需要上传的段数
|
|
|
partCount := int(fileSize / partSize)
|
|
|
if fileSize%partSize != 0 {
|
|
|
partCount++
|
|
|
}
|
|
|
// 执行并发上传段
|
|
|
partChan := make(chan obs.Part, 5)
|
|
|
for i := 0; i < partCount; i++ {
|
|
|
partNumber := i + 1
|
|
|
offset := int64(i) * partSize
|
|
|
currPartSize := partSize
|
|
|
if i+1 == partCount {
|
|
|
currPartSize = fileSize - offset
|
|
|
}
|
|
|
go func() {
|
|
|
uploadPartInput := &obs.UploadPartInput{}
|
|
|
uploadPartInput.Bucket = Bucket
|
|
|
uploadPartInput.Key = key
|
|
|
uploadPartInput.UploadId = uploadId
|
|
|
uploadPartInput.SourceFile = sourceFile
|
|
|
// 分段号
|
|
|
uploadPartInput.PartNumber = partNumber
|
|
|
// 分段在文件中的起始位置
|
|
|
uploadPartInput.Offset = offset
|
|
|
// 分段大小
|
|
|
uploadPartInput.PartSize = currPartSize
|
|
|
uploadPartInputOutput, err := obsClient.UploadPart(uploadPartInput)
|
|
|
if err != nil {
|
|
|
panic(err)
|
|
|
}
|
|
|
fmt.Printf("%d finished\n", partNumber)
|
|
|
partChan <- obs.Part{ETag: uploadPartInputOutput.ETag, PartNumber: uploadPartInputOutput.PartNumber}
|
|
|
}()
|
|
|
}
|
|
|
parts := make([]obs.Part, 0, partCount)
|
|
|
// 等待上传完成
|
|
|
for {
|
|
|
part, ok := <-partChan
|
|
|
if !ok {
|
|
|
break
|
|
|
}
|
|
|
parts = append(parts, part)
|
|
|
|
|
|
if len(parts) == partCount {
|
|
|
close(partChan)
|
|
|
}
|
|
|
}
|
|
|
|
|
|
completeMultipartUploadInput := &obs.CompleteMultipartUploadInput{}
|
|
|
completeMultipartUploadInput.Bucket = Bucket
|
|
|
completeMultipartUploadInput.Key = key
|
|
|
completeMultipartUploadInput.UploadId = uploadId
|
|
|
completeMultipartUploadInput.Parts = parts
|
|
|
// 合并段
|
|
|
completeMultipartUploadOutput, err := obsClient.CompleteMultipartUpload(completeMultipartUploadInput)
|
|
|
if err != nil {
|
|
|
panic(err)
|
|
|
}
|
|
|
fmt.Printf("RequestId:%s\n", completeMultipartUploadOutput.RequestId)
|
|
|
obsClient.Close()
|
|
|
}
|
|
|
|
|
|
type IniParser struct {
|
|
|
confReader *ini.File // config reader
|
|
|
}
|
|
|
|
|
|
type IniParserError struct {
|
|
|
errorInfo string
|
|
|
}
|
|
|
|
|
|
func (e *IniParserError) Error() string { return e.errorInfo }
|
|
|
|
|
|
func (this *IniParser) Load(configFileName string) error {
|
|
|
conf, err := ini.Load(configFileName)
|
|
|
if err != nil {
|
|
|
this.confReader = nil
|
|
|
return err
|
|
|
}
|
|
|
this.confReader = conf
|
|
|
return nil
|
|
|
}
|
|
|
|
|
|
func (this *IniParser) GetString(section string, key string) string {
|
|
|
if this.confReader == nil {
|
|
|
return ""
|
|
|
}
|
|
|
|
|
|
s := this.confReader.Section(section)
|
|
|
if s == nil {
|
|
|
return ""
|
|
|
}
|
|
|
|
|
|
return s.Key(key).String()
|
|
|
}
|
|
|
|
|
|
func (this *IniParser) GetInt32(section string, key string) int32 {
|
|
|
if this.confReader == nil {
|
|
|
return 0
|
|
|
}
|
|
|
|
|
|
s := this.confReader.Section(section)
|
|
|
if s == nil {
|
|
|
return 0
|
|
|
}
|
|
|
|
|
|
valueInt, _ := s.Key(key).Int()
|
|
|
|
|
|
return int32(valueInt)
|
|
|
}
|
|
|
|
|
|
func (this *IniParser) GetUint32(section string, key string) uint32 {
|
|
|
if this.confReader == nil {
|
|
|
return 0
|
|
|
}
|
|
|
|
|
|
s := this.confReader.Section(section)
|
|
|
if s == nil {
|
|
|
return 0
|
|
|
}
|
|
|
|
|
|
valueInt, _ := s.Key(key).Uint()
|
|
|
|
|
|
return uint32(valueInt)
|
|
|
}
|
|
|
|
|
|
func (this *IniParser) GetInt64(section string, key string) int64 {
|
|
|
if this.confReader == nil {
|
|
|
return 0
|
|
|
}
|
|
|
|
|
|
s := this.confReader.Section(section)
|
|
|
if s == nil {
|
|
|
return 0
|
|
|
}
|
|
|
|
|
|
valueInt, _ := s.Key(key).Int64()
|
|
|
return valueInt
|
|
|
}
|
|
|
|
|
|
func (this *IniParser) GetUint64(section string, key string) uint64 {
|
|
|
if this.confReader == nil {
|
|
|
return 0
|
|
|
}
|
|
|
|
|
|
s := this.confReader.Section(section)
|
|
|
if s == nil {
|
|
|
return 0
|
|
|
}
|
|
|
|
|
|
valueInt, _ := s.Key(key).Uint64()
|
|
|
return valueInt
|
|
|
}
|
|
|
|
|
|
func (this *IniParser) GetFloat32(section string, key string) float32 {
|
|
|
if this.confReader == nil {
|
|
|
return 0
|
|
|
}
|
|
|
|
|
|
s := this.confReader.Section(section)
|
|
|
if s == nil {
|
|
|
return 0
|
|
|
}
|
|
|
|
|
|
valueFloat, _ := s.Key(key).Float64()
|
|
|
return float32(valueFloat)
|
|
|
}
|
|
|
|
|
|
func (this *IniParser) GetFloat64(section string, key string) float64 {
|
|
|
if this.confReader == nil {
|
|
|
return 0
|
|
|
}
|
|
|
|
|
|
s := this.confReader.Section(section)
|
|
|
if s == nil {
|
|
|
return 0
|
|
|
}
|
|
|
|
|
|
valueFloat, _ := s.Key(key).Float64()
|
|
|
return valueFloat
|
|
|
}
|
