diff --git a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/MySQLBackup.java b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/BackupUtil.java similarity index 79% rename from YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/MySQLBackup.java rename to YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/BackupUtil.java index 6667d44e..abf418c8 100644 --- a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/MySQLBackup.java +++ b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/BackupUtil.java @@ -1,22 +1,17 @@ package com.dsideal.YunXiaoTools; -import cn.hutool.core.date.DateUtil; import cn.hutool.core.io.FileUtil; import cn.hutool.core.util.ZipUtil; import com.dsideal.YunXiaoTools.Utils.ObsUtil; import com.jfinal.kit.PropKit; import java.io.*; -import java.nio.file.Files; -import java.nio.file.Path; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; -public class MySQLBackup { +public class BackupUtil { //数据库连接串 public static String jdbcUrl; //数据库用户名 @@ -144,31 +139,29 @@ public class MySQLBackup { } } - public static void main(String[] args) { + public static void doAction() { + BackupUtil backup = new BackupUtil(); + String sourceFile = backup.backup(); - try { - MySQLBackup backup = new MySQLBackup(); - String sourceFile = backup.backup(); + //压缩成ZIP文件 + String zipFile = sourceFile.replace(".sql", ".zip"); + ZipUtil.zip(sourceFile, zipFile); - //压缩成ZIP文件 - String zipFile = sourceFile.replace(".sql", ".zip"); - ZipUtil.zip(sourceFile, zipFile); + //文件名称 + String fileName = new File(zipFile).getName(); + // 上传到华为云OBS + String key = "HuangHai/YunXiao/" + fileName; + ObsUtil.uploadToObs(zipFile, key); - //文件名称 - String fileName = new File(zipFile).getName(); - // 上传到华为云OBS - String key = "HuangHai/YunXiao/" + fileName; - ObsUtil.uploadToObs(zipFile, key); + //清理此路径下旧的文件 + String url = "https://dsideal.obs.cn-north-1.myhuaweicloud.com/" + key; + System.out.println(url); + //清理 + FileUtil.del(sourceFile); + FileUtil.del(zipFile); + } - //清理此路径下旧的文件 - String url = "https://dsideal.obs.cn-north-1.myhuaweicloud.com/" + key; - System.out.println(url); - //清理 - FileUtil.del(sourceFile); - FileUtil.del(zipFile); - } catch (Exception e) { - e.printStackTrace(); - System.err.println("备份失败: " + e.getMessage()); - } + public static void main(String[] args) { + doAction(); } } \ No newline at end of file diff --git a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Controller/ReadDataController.java b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Controller/ReadDataController.java index 2131814f..d3d3fd8d 100644 --- a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Controller/ReadDataController.java +++ b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Controller/ReadDataController.java @@ -1,6 +1,5 @@ package com.dsideal.YunXiaoTools.Controller; -import com.dsideal.YunXiaoTools.Utils.ReadDataUtil; import com.jfinal.aop.Before; import com.jfinal.core.Controller; import com.jfinal.ext.interceptor.GET; @@ -16,11 +15,11 @@ public class ReadDataController extends Controller { // http://10.10.21.20:9008/readMysql @Before({GET.class}) public void readMysql() throws IOException { - String url=ReadDataUtil.doAction(); + //String url=ReadDataUtil.doAction(); Kv kv=Kv.create(); kv.set("success",true); kv.set("message","获取并上传数据到OBS成功！"); - kv.set("url",url); + //kv.set("url",url); renderJson(kv); } } diff --git a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Cron4j/LingDianTask.java b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Cron4j/LingDianTask.java index 0ebe0317..59fb4b29 100644 --- a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Cron4j/LingDianTask.java +++ b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Cron4j/LingDianTask.java @@ -1,7 +1,6 @@ package com.dsideal.YunXiaoTools.Cron4j; import com.dsideal.YunXiaoTools.Start; -import com.dsideal.YunXiaoTools.Utils.ReadDataUtil; import com.jfinal.plugin.cron4j.ITask; import lombok.SneakyThrows; @@ -11,7 +10,7 @@ public class LingDianTask implements ITask { public void run() { //如果是读取侧 if(Start.DEPLOY_ID ==1){ - ReadDataUtil.doAction(); + // ReadDataUtil.doAction(); } } diff --git a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/CsvExportUtil.java b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/CsvExportUtil.java deleted file mode 100644 index 451c6dec..00000000 --- a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/CsvExportUtil.java +++ /dev/null @@ -1,97 +0,0 @@ -package com.dsideal.YunXiaoTools.Utils; - -import com.jfinal.plugin.activerecord.Db; -import com.jfinal.plugin.activerecord.Record; -import com.opencsv.CSVWriter; - -import java.io.BufferedWriter; -import java.io.FileWriter; -import java.io.IOException; -import java.util.List; - -public class CsvExportUtil { - private static final int BATCH_SIZE = 1000; - private static final String NULL_REPLACEMENT = "NULL"; // 使用NULL文本 - /** - * 进度回调接口 - */ - public interface ProcessCallback { - void onProgress(long processed, long total); - } - - public void exportToCsv(String sql, String filePath, ProcessCallback callback) { - // 获取总记录数 - long totalCount = Db.queryLong("select count(*) from (select * " + sql + ") t"); - long processedCount = 0; - - try (BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(filePath), 8192); - CSVWriter writer = new CSVWriter(bufferedWriter, - '\t', // 使用Tab分隔符 - CSVWriter.NO_QUOTE_CHARACTER, // 不使用引号 - CSVWriter.DEFAULT_ESCAPE_CHARACTER, - CSVWriter.DEFAULT_LINE_END)) { - - boolean isFirst = true; - int page = 1; - - while (true) { - // 分页查询数据 - List records = Db.paginate(page, BATCH_SIZE, "select *", sql).getList(); - if (records.isEmpty()) { - break; - } - - // 第一次写入表头 - if (isFirst) { - String[] headers = records.getFirst().getColumnNames(); - writer.writeNext(headers, false); // false表示不加引号 - isFirst = false; - } - - // 写入数据 - for (Record record : records) { - String[] row = new String[record.getColumnNames().length]; - int i = 0; - for (String column : record.getColumnNames()) { - Object value = record.get(column); - row[i++] = formatValue(value); - } - writer.writeNext(row, false); // false表示不加引号 - } - - // 更新进度 - processedCount += records.size(); - if (callback != null) { - callback.onProgress(processedCount, totalCount); - } - - // 定期刷新缓冲区 - bufferedWriter.flush(); - - page++; - } - - } catch (IOException e) { - throw new RuntimeException("导出CSV文件失败", e); - } - } - - /** - * 格式化值 - */ - private String formatValue(Object value) { - if (value == null) { - return NULL_REPLACEMENT; - } - String strValue = value.toString(); - // 如果数据中包含Tab，替换为空格 - return strValue.replace("\t", " "); - } - - /** - * 不需要进度回调的重载方法 - */ - public void exportToCsv(String sql, String filePath) { - exportToCsv(sql, filePath, null); - } -} \ No newline at end of file diff --git a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/ReadDataUtil.java b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/ReadDataUtil.java deleted file mode 100644 index e316cc22..00000000 --- a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/ReadDataUtil.java +++ /dev/null @@ -1,68 +0,0 @@ -package com.dsideal.YunXiaoTools.Utils; - -import cn.hutool.core.date.DateUtil; -import cn.hutool.core.io.FileUtil; -import com.jfinal.kit.PathKit; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -public class ReadDataUtil { - - public static String doAction() throws IOException { - // 创建临时目录 - Path tempDir = Files.createTempDirectory("csv_export_"); - List csvFiles = new ArrayList<>(); - - // 读取指定的表列表，生成csv文件 - String content=FileUtil.readUtf8String(PathKit.getRootClassPath()+"/syncTables.txt"); - for (String table : content.split("\r\n")) { - System.out.println("正在生成" + table + "的csv文件"); - - // 在临时目录中创建CSV文件 - String csvPath = tempDir.resolve(table + ".csv").toString(); - csvFiles.add(csvPath); - - // 导出CSV - CsvExportUtil exporter = new CsvExportUtil(); - String sql = " from `" + table + "`"; - - //带进度生成 - exporter.exportToCsv(sql, csvPath, (processed, total) -> { - double percent = (double) processed / total * 100; - System.out.printf("导出进度：%.2f%% (%d/%d)%n", - percent, processed, total); - }); - } - - // 创建ZIP文件 - String timestamp = DateUtil.format(DateUtil.date(), "yyyyMMddHHmmss"); - String zipPath = tempDir.resolve(timestamp + ".zip").toString(); - - try (ZipOutputStream zos = new ZipOutputStream(Files.newOutputStream(Path.of(zipPath)))) { - for (String csvFile : csvFiles) { - File fZip = new File(csvFile); - ZipEntry zipEntry = new ZipEntry(fZip.getName()); - zos.putNextEntry(zipEntry); - Files.copy(fZip.toPath(), zos); - zos.closeEntry(); - } - } - - // 上传到华为云OBS - String key="HuangHai/YunXiao/"+ timestamp + ".zip"; - ObsUtil.uploadToObs(zipPath,key); - // 清理临时文件 - csvFiles.forEach(csv -> FileUtil.del(csv)); - FileUtil.del(zipPath); - FileUtil.del(tempDir.toString()); - //清理此路径下旧的文件 - return "https://dsideal.obs.cn-north-1.myhuaweicloud.com/"+key; - } -}