You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

100 lines
2.4 KiB

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

package main
import (
"context"
"dsSupport/Utils/PgUtil"
"encoding/csv"
"encoding/json"
"fmt"
"github.com/olivere/elastic/v7"
"github.com/tidwall/gjson"
"log"
"os"
"time"
)
// 深度分页
// https://www.cnblogs.com/hello-shf/p/11543453.html
var client elastic.Client
var db = PgUtil.Engine
func main() {
var host = "http://10.10.14.188:9200/"
esClient, err := elastic.NewClient(
elastic.SetURL(host),
elastic.SetSniff(false),
elastic.SetHealthcheckInterval(10*time.Second),
elastic.SetGzip(true),
elastic.SetErrorLog(log.New(os.Stderr, "ELASTIC ", log.LstdFlags)),
elastic.SetInfoLog(log.New(os.Stdout, "", log.LstdFlags)),
)
if err != nil {
panic(err)
}
//要同步的索引名称,也就是表名称
indexName := "org_school"
//取所有
CTX := context.Background()
result, err := esClient.Scroll().Index(indexName).Size(10).Do(CTX)
if err != nil {
panic(err)
}
//第一次的结果集
for i := range result.Hits.Hits {
resByte, _ := json.Marshal(result.Hits.Hits[i].Source)
resStr := string(resByte)
value := gjson.Get(resStr, "data_content")
fmt.Println(value.String())
}
//继续用的 scoll_id
scrollId := result.ScrollId
//第一次命中的个数
var nowCount = int64(len(result.Hits.Hits))
//总的命中个数
allCount := result.TotalHits()
//开始循环
for {
//如果还有数据没有获取
if allCount > nowCount {
result, err = esClient.Scroll().ScrollId(scrollId).Do(CTX)
scrollId = result.ScrollId
nowCount += int64(len(result.Hits.Hits))
for i := range result.Hits.Hits {
resByte, _ := json.Marshal(result.Hits.Hits[i].Source)
jsonStr := string(resByte)
m := make(map[string]interface{})
json.Unmarshal([]byte(jsonStr), &m)
for k, v := range m {
fmt.Printf("%v: %v\n", k, v)
}
}
} else {
//没有数据了
break
}
}
}
func SaveToCsv(csvFileName string, _map []map[string]interface{}) {
file, err := os.OpenFile(csvFileName, os.O_CREATE|os.O_RDWR, 0644)
if err != nil {
fmt.Println("open file is failed, err: ", err)
}
defer file.Close()
// 写入UTF-8 BOM防止中文乱码
file.WriteString("\xEF\xBB\xBF")
w := csv.NewWriter(file)
//for i := range _map {
// //_map[i]
//}
w.Write([]string{"开发者名称", "开发者邮箱", "应用名称"})
// 写文件需要flush不然缓存满了后面的就写不进去了只会写一部分
w.Flush()
}