huanghai
/
QingLong
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

'commit'

Browse Source
main
黄海 6 months ago
parent a0c0fb9fd4
commit 75e02afd4d
5 changed files with 24 additions and 198 deletions
Show Stats Download Patch File Download Diff File

49
YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/MySQLBackup.java → YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/BackupUtil.java
Unescape View File

@ -1,22 +1,17 @@
package com.dsideal.YunXiaoTools;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.ZipUtil;
import com.dsideal.YunXiaoTools.Utils.ObsUtil;
import com.jfinal.kit.PropKit;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
public class MySQLBackup {
public class BackupUtil {
//数据库连接串
public static String jdbcUrl;
//数据库用户名
@ -144,31 +139,29 @@ public class MySQLBackup {
}
}
public static void main(String[] args) {
public static void doAction() {
BackupUtil backup = new BackupUtil();
String sourceFile = backup.backup();
try {
MySQLBackup backup = new MySQLBackup();
String sourceFile = backup.backup();
//压缩成ZIP文件
String zipFile = sourceFile.replace(".sql", ".zip");
ZipUtil.zip(sourceFile, zipFile);
//压缩成ZIP文件
String zipFile = sourceFile.replace(".sql", ".zip");
ZipUtil.zip(sourceFile, zipFile);
//文件名称
String fileName = new File(zipFile).getName();
// 上传到华为云OBS
String key = "HuangHai/YunXiao/" + fileName;
ObsUtil.uploadToObs(zipFile, key);
//文件名称
String fileName = new File(zipFile).getName();
// 上传到华为云OBS
String key = "HuangHai/YunXiao/" + fileName;
ObsUtil.uploadToObs(zipFile, key);
//清理此路径下旧的文件
String url = "https://dsideal.obs.cn-north-1.myhuaweicloud.com/" + key;
System.out.println(url);
//清理
FileUtil.del(sourceFile);
FileUtil.del(zipFile);
}
//清理此路径下旧的文件
String url = "https://dsideal.obs.cn-north-1.myhuaweicloud.com/" + key;
System.out.println(url);
//清理
FileUtil.del(sourceFile);
FileUtil.del(zipFile);
} catch (Exception e) {
e.printStackTrace();
System.err.println("备份失败: " + e.getMessage());
}
public static void main(String[] args) {
doAction();
}
}

5
YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Controller/ReadDataController.java
Unescape View File

@ -1,6 +1,5 @@
package com.dsideal.YunXiaoTools.Controller;
import com.dsideal.YunXiaoTools.Utils.ReadDataUtil;
import com.jfinal.aop.Before;
import com.jfinal.core.Controller;
import com.jfinal.ext.interceptor.GET;
@ -16,11 +15,11 @@ public class ReadDataController extends Controller {
// http://10.10.21.20:9008/readMysql
@Before({GET.class})
public void readMysql() throws IOException {
String url=ReadDataUtil.doAction();
//String url=ReadDataUtil.doAction();
Kv kv=Kv.create();
kv.set("success",true);
kv.set("message","获取并上传数据到OBS成功！");
kv.set("url",url);
//kv.set("url",url);
renderJson(kv);
}
}

3
YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Cron4j/LingDianTask.java
Unescape View File

@ -1,7 +1,6 @@
package com.dsideal.YunXiaoTools.Cron4j;
import com.dsideal.YunXiaoTools.Start;
import com.dsideal.YunXiaoTools.Utils.ReadDataUtil;
import com.jfinal.plugin.cron4j.ITask;
import lombok.SneakyThrows;
@ -11,7 +10,7 @@ public class LingDianTask implements ITask {
public void run() {
//如果是读取侧
if(Start.DEPLOY_ID ==1){
ReadDataUtil.doAction();
// ReadDataUtil.doAction();
}
}

97
YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/CsvExportUtil.java
Unescape View File

@ -1,97 +0,0 @@
package com.dsideal.YunXiaoTools.Utils;
import com.jfinal.plugin.activerecord.Db;
import com.jfinal.plugin.activerecord.Record;
import com.opencsv.CSVWriter;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.util.List;
public class CsvExportUtil {
private static final int BATCH_SIZE = 1000;
private static final String NULL_REPLACEMENT = "NULL"; // 使用NULL文本
/**
* 进度回调接口
*/
public interface ProcessCallback {
void onProgress(long processed, long total);
}
public void exportToCsv(String sql, String filePath, ProcessCallback callback) {
// 获取总记录数
long totalCount = Db.queryLong("select count(*) from (select * " + sql + ") t");
long processedCount = 0;
try (BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(filePath), 8192);
CSVWriter writer = new CSVWriter(bufferedWriter,
'\t', // 使用Tab分隔符
CSVWriter.NO_QUOTE_CHARACTER, // 不使用引号
CSVWriter.DEFAULT_ESCAPE_CHARACTER,
CSVWriter.DEFAULT_LINE_END)) {
boolean isFirst = true;
int page = 1;
while (true) {
// 分页查询数据
List<Record> records = Db.paginate(page, BATCH_SIZE, "select *", sql).getList();
if (records.isEmpty()) {
break;
}
// 第一次写入表头
if (isFirst) {
String[] headers = records.getFirst().getColumnNames();
writer.writeNext(headers, false); // false表示不加引号
isFirst = false;
}
// 写入数据
for (Record record : records) {
String[] row = new String[record.getColumnNames().length];
int i = 0;
for (String column : record.getColumnNames()) {
Object value = record.get(column);
row[i++] = formatValue(value);
}
writer.writeNext(row, false); // false表示不加引号
}
// 更新进度
processedCount += records.size();
if (callback != null) {
callback.onProgress(processedCount, totalCount);
}
// 定期刷新缓冲区
bufferedWriter.flush();
page++;
}
} catch (IOException e) {
throw new RuntimeException("导出CSV文件失败", e);
}
}
/**
* 格式化值
*/
private String formatValue(Object value) {
if (value == null) {
return NULL_REPLACEMENT;
}
String strValue = value.toString();
// 如果数据中包含Tab，替换为空格
return strValue.replace("\t", " ");
}
/**
* 不需要进度回调的重载方法
*/
public void exportToCsv(String sql, String filePath) {
exportToCsv(sql, filePath, null);
}
}

68
YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/ReadDataUtil.java
Unescape View File

@ -1,68 +0,0 @@
package com.dsideal.YunXiaoTools.Utils;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.io.FileUtil;
import com.jfinal.kit.PathKit;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
public class ReadDataUtil {
public static String doAction() throws IOException {
// 创建临时目录
Path tempDir = Files.createTempDirectory("csv_export_");
List<String> csvFiles = new ArrayList<>();
// 读取指定的表列表，生成csv文件
String content=FileUtil.readUtf8String(PathKit.getRootClassPath()+"/syncTables.txt");
for (String table : content.split("\r\n")) {
System.out.println("正在生成" + table + "的csv文件");
// 在临时目录中创建CSV文件
String csvPath = tempDir.resolve(table + ".csv").toString();
csvFiles.add(csvPath);
// 导出CSV
CsvExportUtil exporter = new CsvExportUtil();
String sql = " from `" + table + "`";
//带进度生成
exporter.exportToCsv(sql, csvPath, (processed, total) -> {
double percent = (double) processed / total * 100;
System.out.printf("导出进度：%.2f%% (%d/%d)%n",
percent, processed, total);
});
}
// 创建ZIP文件
String timestamp = DateUtil.format(DateUtil.date(), "yyyyMMddHHmmss");
String zipPath = tempDir.resolve(timestamp + ".zip").toString();
try (ZipOutputStream zos = new ZipOutputStream(Files.newOutputStream(Path.of(zipPath)))) {
for (String csvFile : csvFiles) {
File fZip = new File(csvFile);
ZipEntry zipEntry = new ZipEntry(fZip.getName());
zos.putNextEntry(zipEntry);
Files.copy(fZip.toPath(), zos);
zos.closeEntry();
}
}
// 上传到华为云OBS
String key="HuangHai/YunXiao/"+ timestamp + ".zip";
ObsUtil.uploadToObs(zipPath,key);
// 清理临时文件
csvFiles.forEach(csv -> FileUtil.del(csv));
FileUtil.del(zipPath);
FileUtil.del(tempDir.toString());
//清理此路径下旧的文件
return "https://dsideal.obs.cn-north-1.myhuaweicloud.com/"+key;
}
}
Write Preview
Loading…
Cancel
Save