diff --git a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Cron4j/LingDianTask.java b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Cron4j/LingDianTask.java new file mode 100644 index 00000000..e6330790 --- /dev/null +++ b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Cron4j/LingDianTask.java @@ -0,0 +1,21 @@ +package com.dsideal.YunXiaoTools.Cron4j; + +import com.dsideal.YunXiaoTools.Start; +import com.dsideal.YunXiaoTools.Utils.ReadDataUtil; +import com.jfinal.plugin.cron4j.ITask; +import lombok.SneakyThrows; + +//0点任务 +public class LingDianTask implements ITask { + @SneakyThrows + public void run() { + //如果是读取侧 + if(Start.isMysql==1){ + ReadDataUtil.doAction(); + } + } + + public void stop() { + // 这里的代码会在 task 被关闭前调用 + } +} diff --git a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Start.java b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Start.java index e35245d5..81a1532d 100644 --- a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Start.java +++ b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Start.java @@ -1,5 +1,6 @@ package com.dsideal.YunXiaoTools; +import cn.hutool.core.date.DateUtil; import cn.hutool.core.io.FileUtil; import cn.hutool.core.text.csv.CsvUtil; @@ -8,12 +9,19 @@ import com.jfinal.kit.PropKit; import com.jfinal.plugin.activerecord.ActiveRecordPlugin; import com.jfinal.plugin.activerecord.dialect.MysqlDialect; import com.jfinal.plugin.activerecord.dialect.PostgreSqlDialect; +import com.jfinal.plugin.cron4j.Cron4jPlugin; import com.jfinal.plugin.hikaricp.HikariCpPlugin; import com.jfinal.server.undertow.UndertowServer; import com.jfinal.template.Engine; import lombok.SneakyThrows; import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; public class Start extends JFinalConfig { @@ -102,7 +110,7 @@ public class Start extends JFinalConfig { me.add(arp); //加载定时器 - //me.add(new Cron4jPlugin(PropKit.use("cron4j.properties"))); + me.add(new Cron4jPlugin(PropKit.use("cron4j.properties"))); } /** @@ -128,34 +136,9 @@ public class Start extends JFinalConfig { @Override public void onStart() { //打印 启动Logo - String logoFile; - if (isMysql == 1) { - logoFile = "mysqlLogo.txt"; - } else { - logoFile = "postgresqlLogo.txt"; - } + String logoFile = (isMysql == 1) ? "mysqlLogo.txt" : "postgresqlLogo.txt"; String path = Start.class.getClassLoader().getResource(logoFile).getPath(); File file = new File(path); System.out.println(FileUtil.readUtf8String(file)); - - //读取指定的表列表，生成csv文件 - String[] tables = {"excel_学前幼儿入园总量_86df0e5a25", "excel_义务教育在校生总量_2a7d8ec9ec"}; - for (int i = 0; i < tables.length; i++) { - System.out.println("正在生成" + tables[i] + "的csv文件"); - StreamingCsvExporter exporter = new StreamingCsvExporter(); - String sql = " from `" + tables[i] + "`"; - - String filePath = "c:/" + tables[i] + ".csv"; - - // 方式1：带进度回调 - exporter.exportToCsv(sql, filePath, (processed, total) -> { - double percent = (double) processed / total * 100; - System.out.printf("导出进度：%.2f%% (%d/%d)%n", - percent, processed, total); - }); - - // 方式2：不需要进度回调 - exporter.exportToCsv(sql, filePath); - } } } diff --git a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/StreamingCsvExporter.java b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/CsvExportUtil.java similarity index 94% rename from YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/StreamingCsvExporter.java rename to YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/CsvExportUtil.java index ef78e8cf..451c6dec 100644 --- a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/StreamingCsvExporter.java +++ b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/CsvExportUtil.java @@ -1,4 +1,4 @@ -package com.dsideal.YunXiaoTools; +package com.dsideal.YunXiaoTools.Utils; import com.jfinal.plugin.activerecord.Db; import com.jfinal.plugin.activerecord.Record; @@ -9,7 +9,7 @@ import java.io.FileWriter; import java.io.IOException; import java.util.List; -public class StreamingCsvExporter { +public class CsvExportUtil { private static final int BATCH_SIZE = 1000; private static final String NULL_REPLACEMENT = "NULL"; // 使用NULL文本 /** diff --git a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/ObsUtil.java b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/ObsUtil.java new file mode 100644 index 00000000..195f49b1 --- /dev/null +++ b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/ObsUtil.java @@ -0,0 +1,40 @@ +package com.dsideal.YunXiaoTools.Utils; + +import com.jfinal.kit.PropKit; +import com.obs.services.ObsClient; +import com.obs.services.model.PutObjectResult; + +import java.io.File; + +public class ObsUtil { + /** + * 上传文件到华为云OBS + */ + public static void uploadToObs(String localFile, String objectKey) { + // 从配置文件中读取OBS配置 + String endPoint = PropKit.get("obs_endpoint"); + String ak = PropKit.get("obs_accessKeyId"); + String sk = PropKit.get("obs_accessKeySecret"); + String bucketName = PropKit.get("obs_bucket_name"); + + // 创建ObsClient实例 + ObsClient obsClient = new ObsClient(ak, sk, endPoint); + + try { + // 上传文件 + PutObjectResult res=obsClient.putObject(bucketName, objectKey, new File(localFile)); + System.out.println(res.getBucketName()); + System.out.println(res.getObjectKey()); + System.out.println("文件已上传到OBS: " + objectKey); + } catch (Exception e) { + System.err.println("上传文件到OBS失败: " + e.getMessage()); + e.printStackTrace(); + } finally { + try { + obsClient.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } +} diff --git a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/ReadDataUtil.java b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/ReadDataUtil.java new file mode 100644 index 00000000..867f54ed --- /dev/null +++ b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/ReadDataUtil.java @@ -0,0 +1,64 @@ +package com.dsideal.YunXiaoTools.Utils; + +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.io.FileUtil; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +public class ReadDataUtil { + + public static void doAction() throws IOException { + // 创建临时目录 + Path tempDir = Files.createTempDirectory("csv_export_"); + List csvFiles = new ArrayList<>(); + + // 读取指定的表列表，生成csv文件 + String[] tables = {"excel_学前幼儿入园总量_86df0e5a25", "excel_义务教育在校生总量_2a7d8ec9ec"}; + for (String table : tables) { + System.out.println("正在生成" + table + "的csv文件"); + + // 在临时目录中创建CSV文件 + String csvPath = tempDir.resolve(table + ".csv").toString(); + csvFiles.add(csvPath); + + // 导出CSV + CsvExportUtil exporter = new CsvExportUtil(); + String sql = " from `" + table + "`"; + + //带进度生成 + exporter.exportToCsv(sql, csvPath, (processed, total) -> { + double percent = (double) processed / total * 100; + System.out.printf("导出进度：%.2f%% (%d/%d)%n", + percent, processed, total); + }); + } + + // 创建ZIP文件 + String timestamp = DateUtil.format(DateUtil.date(), "yyyyMMddHHmmss"); + String zipPath = tempDir.resolve(timestamp + ".zip").toString(); + + try (ZipOutputStream zos = new ZipOutputStream(Files.newOutputStream(Path.of(zipPath)))) { + for (String csvFile : csvFiles) { + File fZip = new File(csvFile); + ZipEntry zipEntry = new ZipEntry(fZip.getName()); + zos.putNextEntry(zipEntry); + Files.copy(fZip.toPath(), zos); + zos.closeEntry(); + } + } + + // 上传到华为云OBS + ObsUtil.uploadToObs(zipPath,"HuangHai/YunXiao/"+ timestamp + ".zip"); + // 清理临时文件 + csvFiles.forEach(csv -> FileUtil.del(csv)); + FileUtil.del(zipPath); + FileUtil.del(tempDir.toString()); + } +} diff --git a/YunXiaoTools/src/main/resources/application.properties b/YunXiaoTools/src/main/resources/application.properties index 3231e855..098199a1 100644 --- a/YunXiaoTools/src/main/resources/application.properties +++ b/YunXiaoTools/src/main/resources/application.properties @@ -13,4 +13,10 @@ postgresql.jdbcUrl=jdbc:postgresql://10.10.14.71:5432/szjz_db?reWriteBatchedInse # 当前是读取方还是写入方 isMysql=1 +# 华为云OBS配置 +#OBS(华为云云存储) +obs_accessKeyId=WAFBGJACKDOQZDH1MKZ1 +obs_accessKeySecret=dlWTUbqgCICaYJG3n0Rot4jXaen2HnfFtMVxiPEo +obs_endpoint=obs.cn-north-1.myhuaweicloud.com +obs_bucket_name=dsideal diff --git a/YunXiaoTools/src/main/resources/cron4j.properties b/YunXiaoTools/src/main/resources/cron4j.properties new file mode 100644 index 00000000..2c851d6e --- /dev/null +++ b/YunXiaoTools/src/main/resources/cron4j.properties @@ -0,0 +1,19 @@ +# cron 表达式由五部分组成：分 时 天 月 周 # 分 ：从 0 到 59 # 时 ：从 0 到 23 # 天 ：从 1 到 31，字母 L 可以表示月的最后一天 # 月 ：从 1 到 12，可以别名：jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov" and "dec" +# 周 ：从 0 到 6，0 表示周日，6 表示周六，可以使用别名： "sun", "mon", "tue", "wed", "thu", "fri" and "sat" +# +# 数字 n：表示一个具体的时间点，例如 5 * * * * 表示 5 分这个时间点时执行 +# 逗号 , ：表示指定多个数值，例如 3,5 * * * * 表示 3 和 5 分这两个时间点执行 +# 减号 -：表示范围，例如 1-3 * * * * 表示 1 分、2 分再到 3 分这三个时间点执行 +# 星号 *：表示每一个时间点，例如 * * * * * 表示每分钟执行 +# 除号 /：表示指定一个值的增加幅度。例如 n/m表示从 n 开始，每次增加 m 的时间点执行 +# cron4j 集成cron 只有 分 时 天 月 周 没有秒 +cron4j=LingDianTask +# 每天凌晨0点执行一次 +LingDianTask.cron=0 0 * * * +LingDianTask.class=com.dsideal.YunXiaoTools.Cron4j.LingDianTask +LingDianTask.daemon=true +LingDianTask.enable=true + + + + diff --git a/YunXiaoTools/target/classes/application.properties b/YunXiaoTools/target/classes/application.properties index 3231e855..098199a1 100644 --- a/YunXiaoTools/target/classes/application.properties +++ b/YunXiaoTools/target/classes/application.properties @@ -13,4 +13,10 @@ postgresql.jdbcUrl=jdbc:postgresql://10.10.14.71:5432/szjz_db?reWriteBatchedInse # 当前是读取方还是写入方 isMysql=1 +# 华为云OBS配置 +#OBS(华为云云存储) +obs_accessKeyId=WAFBGJACKDOQZDH1MKZ1 +obs_accessKeySecret=dlWTUbqgCICaYJG3n0Rot4jXaen2HnfFtMVxiPEo +obs_endpoint=obs.cn-north-1.myhuaweicloud.com +obs_bucket_name=dsideal