dsMin/dsTools/Backup/esBackup.go

package main

import (
	"crypto/tls"
	"dsTools/Utils/CommonUtil"
	"dsTools/Utils/ConfigUtil"
	"dsTools/Utils/NetUtil"
	"dsTools/Utils/ObsUtil"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"
)

/*
vi /usr/local/elasticsearch/config/elasticsearch.yml
在最后面追加上三行配置

http.cors.enabled: true
http.cors.allow-origin: "*"
path.repo: ["/usr/local/esBackup"]

mkdir -p /usr/local/esBackup
chown -R elasticsearch /usr/local/esBackup

然后重新启动一下elasticsearch,让配置生效。
*/
/**
功能：获取指定仓库的信息
作者：黄海
时间：2020-04-05
*/
func GetRepository(repositoryName string) {
	client := &http.Client{}
	req, _ := http.NewRequest("GET", ConfigUtil.EsHost+"/_snapshot/"+repositoryName, nil)
	resp, _ := client.Do(req)
	body, _ := ioutil.ReadAll(resp.Body)
	fmt.Printf(string(body))
}

/**
功能:创建快照仓库
作者：黄海
时间：2020-04-05
*/
func CreateRepository(repositoryName string, repositoryPath string) {
	data := `
		{
		"type": "fs",
		"settings": {
					"location": "` + repositoryPath + `"
					}
		}`
	url := ConfigUtil.EsHost + "/_snapshot/" + repositoryName
	payload := strings.NewReader(data)
	req, _ := http.NewRequest("PUT", url, payload)
	req.Header.Add("Content-Type", "application/json")
	res, _ := http.DefaultClient.Do(req)
	defer res.Body.Close()
	body, _ := ioutil.ReadAll(res.Body)
	fmt.Println(res)
	fmt.Println(string(body))
}

/**
功能:设置http的超时时间
作者：黄海
时间:2020-04-05
*/
func TimeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(net, addr string) (c net.Conn, err error) {
	return func(netw, addr string) (net.Conn, error) {
		conn, err := net.DialTimeout(netw, addr, cTimeout)
		if err != nil {
			return nil, err
		}
		conn.SetDeadline(time.Now().Add(rwTimeout))
		return conn, nil
	}
}

/**
功能：创建快照仓库
作者：黄海
时间：2020-04-05
*/
func CreateSnap() {
	//超时时间设置
	connectTimeout := 5 * time.Second
	readWriteTimeout := 2 * time.Hour

	c := http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
			Dial:            TimeoutDialer(connectTimeout, readWriteTimeout),
		},
	}
	data := `{
  				"ignore_unavailable": true,
  				"include_global_state": false
			}`
	fileName := strings.Replace(CommonUtil.GetCurrentTime(), " ", "", -1)
	fileName = strings.Replace(fileName, "-", "", -1)
	fileName = strings.Replace(fileName, ":", "", -1)
	request, _ := http.NewRequest("POST", ConfigUtil.EsHost+"/_snapshot/esBackup/snapshot_"+fileName+"?wait_for_completion=true", strings.NewReader(data))
	//post数据并接收http响应
	//增加header选项
	request.Header.Set("Content-Type", "application/json")
	resp, err := c.Do(request)
	if err != nil {
		fmt.Printf("post data error:%v\n", err)
	} else {
		fmt.Println("post a data successful.")
		respBody, _ := ioutil.ReadAll(resp.Body)
		fmt.Printf("response data:%v\n", string(respBody))
	}
	//调用shell进行打包备份
	command := `/usr/local/dsMin/dsTools/Shell/BackupEsSnapshot.sh`
	cmd := exec.Command("/bin/bash", "-c", command)
	output, err := cmd.Output()
	if err != nil {
		fmt.Printf("Execute Shell:%s failed with error:%s", command, err.Error())
		return
	}
	fmt.Printf("Execute Shell:%s finished with output:\n%s", command, string(output))

}

func main() {
	//杀掉老进程
	command := "ps -ef | grep esBackup | grep -v grep | awk '{print $2}' | xargs kill -9"
	exec.Command("/bin/bash", "-c", command)

	//1、创建仓库
	CreateRepository("esBackup", "/usr/local/esBackup")
	//2、es产生快照
	fmt.Println("1、正在生成快照...")
	CreateSnap()
	//3、上传数据库备份到云存储
	fmt.Println("2、正在将备份文件上传到云存储...")
	macAddress := NetUtil.GetMacAddrs()[0]
	dirPath := "/usr/local/Backup"
	//此目录下，最近2小时内生成的文件，并且以es_backup_开头的找出来，应该只有1个
	files, err := ioutil.ReadDir(dirPath) //读取目录下文件
	if err != nil {
		return
	}
	//云存储前缀
	var prefix = ObsUtil.BackupPrefix + "/" + ConfigUtil.InstallAreaCode + "/" + macAddress
	for _, file := range files {
		if file.IsDir() {
			continue
		} else {
			now := time.Now()
			sumD := now.Sub(file.ModTime())
			if sumD.Hours() < 2 && strings.Index(file.Name(), "es_backup_") == 0 {
				fmt.Println("3、找到刚刚备份的es文件:" + file.Name())
				key := prefix + "/" + file.Name()
				sourceFile := dirPath + "/" + file.Name()
				fmt.Println("4、正在上传备份的es文件...")
				ObsUtil.UploadFileMultiPart(key, sourceFile)
			}
			//删除历史文件
			if sumD.Hours() > 24 {
				sourceFile := dirPath + "/" + file.Name()
				err := os.Remove(sourceFile)
				if err != nil {
					fmt.Println("删除历史文件" + sourceFile + "失败！")
				}
			}
		}
	}
	//3、对云备份的文件进行清理
	fmt.Println("5、正在进行云备份文件的清理工作...")
	ObsUtil.DeleteExpireFile(prefix+"/es_backup_", ObsUtil.RemainDays)
	//4、提示信息
	fmt.Println("恭喜，ES备份成功完成！")
}