huanghai
/
QingLong
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

'commit'

Browse Source
main
黄海 6 months ago
parent e28c448bc7
commit 6dd49551bb
11 changed files with 12867 additions and 43 deletions
Show Stats Download Patch File Download Diff File

3521
WebRoot/csv/excel_义务教育在校生总量_2a7d8ec9ec.csv
View File

File diff suppressed because it is too large Load Diff

9184
WebRoot/csv/excel_学前幼儿入园总量_86df0e5a25.csv
View File

File diff suppressed because it is too large Load Diff

5
YunXiaoTools/pom.xml
Unescape View File

@ -45,6 +45,11 @@
<artifactId>cos</artifactId>
<version>2022.2</version>
</dependency>
<dependency>
<groupId>com.opencsv</groupId>
<artifactId>opencsv</artifactId>
<version>5.7.1</version> <!-- 可以使用最新版本 -->
</dependency>
<dependency>
<groupId>de.ruedigermoeller</groupId>
<artifactId>fst</artifactId>

24
YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Start.java
Unescape View File

@ -2,6 +2,7 @@ package com.dsideal.YunXiaoTools;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.text.csv.CsvUtil;
import com.jfinal.config.*;
import com.jfinal.kit.PropKit;
import com.jfinal.plugin.activerecord.ActiveRecordPlugin;
@ -59,9 +60,6 @@ public class Start extends JFinalConfig {
@Override
public void configPlugin(Plugins me) {
//是不是从Mysql读取数据库,还是到PostgreSql写入到数据库
String jdbcUrl, user, password, driverClassName;
if (isMysql == 1) {
jdbcUrl = PropKit.get("mysql.jdbcUrl");
@ -139,5 +137,25 @@ public class Start extends JFinalConfig {
String path = Start.class.getClassLoader().getResource(logoFile).getPath();
File file = new File(path);
System.out.println(FileUtil.readUtf8String(file));
//读取指定的表列表，生成csv文件
String[] tables = {"excel_学前幼儿入园总量_86df0e5a25", "excel_义务教育在校生总量_2a7d8ec9ec"};
for (int i = 0; i < tables.length; i++) {
System.out.println("正在生成" + tables[i] + "的csv文件");
StreamingCsvExporter exporter = new StreamingCsvExporter();
String sql = " from `" + tables[i] + "`";
String filePath = "c:/" + tables[i] + ".csv";
// 方式1：带进度回调
exporter.exportToCsv(sql, filePath, (processed, total) -> {
double percent = (double) processed / total * 100;
System.out.printf("导出进度：%.2f%% (%d/%d)%n",
percent, processed, total);
});
// 方式2：不需要进度回调
exporter.exportToCsv(sql, filePath);
}
}
}

97
YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/StreamingCsvExporter.java
Unescape View File

@ -0,0 +1,97 @@
package com.dsideal.YunXiaoTools;
import com.jfinal.plugin.activerecord.Db;
import com.jfinal.plugin.activerecord.Record;
import com.opencsv.CSVWriter;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.util.List;
public class StreamingCsvExporter {
private static final int BATCH_SIZE = 1000;
private static final String NULL_REPLACEMENT = "NULL"; // 使用NULL文本
/**
* 进度回调接口
*/
public interface ProcessCallback {
void onProgress(long processed, long total);
}
public void exportToCsv(String sql, String filePath, ProcessCallback callback) {
// 获取总记录数
long totalCount = Db.queryLong("select count(*) from (select * " + sql + ") t");
long processedCount = 0;
try (BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(filePath), 8192);
CSVWriter writer = new CSVWriter(bufferedWriter,
'\t', // 使用Tab分隔符
CSVWriter.NO_QUOTE_CHARACTER, // 不使用引号
CSVWriter.DEFAULT_ESCAPE_CHARACTER,
CSVWriter.DEFAULT_LINE_END)) {
boolean isFirst = true;
int page = 1;
while (true) {
// 分页查询数据
List<Record> records = Db.paginate(page, BATCH_SIZE, "select *", sql).getList();
if (records.isEmpty()) {
break;
}
// 第一次写入表头
if (isFirst) {
String[] headers = records.getFirst().getColumnNames();
writer.writeNext(headers, false); // false表示不加引号
isFirst = false;
}
// 写入数据
for (Record record : records) {
String[] row = new String[record.getColumnNames().length];
int i = 0;
for (String column : record.getColumnNames()) {
Object value = record.get(column);
row[i++] = formatValue(value);
}
writer.writeNext(row, false); // false表示不加引号
}
// 更新进度
processedCount += records.size();
if (callback != null) {
callback.onProgress(processedCount, totalCount);
}
// 定期刷新缓冲区
bufferedWriter.flush();
page++;
}
} catch (IOException e) {
throw new RuntimeException("导出CSV文件失败", e);
}
}
/**
* 格式化值
*/
private String formatValue(Object value) {
if (value == null) {
return NULL_REPLACEMENT;
}
String strValue = value.toString();
// 如果数据中包含Tab，替换为空格
return strValue.replace("\t", " ");
}
/**
* 不需要进度回调的重载方法
*/
public void exportToCsv(String sql, String filePath) {
exportToCsv(sql, filePath, null);
}
}

11
YunXiaoTools/src/main/resources/application.properties
Unescape View File

@ -1,9 +1,10 @@
# 数据库信息
mysql.driverClassName=org.postgresql.Driver
mysql.user=postgres
mysql.password=DsideaL147258369
mysql.jdbcUrl=jdbc:postgresql://10.10.14.71:5432/szjz_db?reWriteBatchedInserts=true
# Mysql数据库信息
mysql.driverClassName=com.mysql.cj.jdbc.Driver
mysql.user=root
mysql.password=Password123@mysql
mysql.jdbcUrl=jdbc:mysql://10.10.14.203:3306/dataease?rewriteBatchedStatements=true&useUnicode=true&zeroDateTimeBehavior=CONVERT_TO_NULL&useSSL=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&autoReconnect=true&failOverReadOnly=false
# Postgresql数据库信息
postgresql.driverClassName=org.postgresql.Driver
postgresql.user=postgres
postgresql.password=DsideaL147258369

11
YunXiaoTools/target/classes/application.properties
Unescape View File

@ -1,9 +1,10 @@
# 数据库信息
mysql.driverClassName=org.postgresql.Driver
mysql.user=postgres
mysql.password=DsideaL147258369
mysql.jdbcUrl=jdbc:postgresql://10.10.14.71:5432/szjz_db?reWriteBatchedInserts=true
# Mysql数据库信息
mysql.driverClassName=com.mysql.cj.jdbc.Driver
mysql.user=root
mysql.password=Password123@mysql
mysql.jdbcUrl=jdbc:mysql://10.10.14.203:3306/dataease?rewriteBatchedStatements=true&useUnicode=true&zeroDateTimeBehavior=CONVERT_TO_NULL&useSSL=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&autoReconnect=true&failOverReadOnly=false
# Postgresql数据库信息
postgresql.driverClassName=org.postgresql.Driver
postgresql.user=postgres
postgresql.password=DsideaL147258369

14
YunXiaoTools/target/classes/mysqlLogo.txt
Unescape View File

@ -0,0 +1,14 @@
$$$$$$$\ $$\ $$$$$$$\ $$\
$$ __$$\ $$ |$$ __$$\ $$ |
$$ | $$ | $$$$$$\ $$$$$$\ $$$$$$$ |$$ | $$ | $$$$$$\ $$$$$$\ $$$$$$\
$$$$$$$ |$$ __$$\ \____$$\ $$ __$$ |$$ | $$ | \____$$\\_$$ _| \____$$\
$$ __$$< $$$$$$$$ | $$$$$$$ |$$ / $$ |$$ | $$ | $$$$$$$ | $$ | $$$$$$$ |
$$ | $$ |$$ ____|$$ __$$ |$$ | $$ |$$ | $$ |$$ __$$ | $$ |$$\ $$ __$$ |
$$ | $$ |\$$$$$$$\ \$$$$$$$ |\$$$$$$$ |$$$$$$$ |\$$$$$$$ | \$$$$ |\$$$$$$$ |
\__| \__| \_______| \_______| \_______|\_______/ \_______| \____/ \_______|
power by http://patorjk.com/software/taag/

13
YunXiaoTools/target/classes/postgresqlLogo.txt
Unescape View File

@ -0,0 +1,13 @@
__ __ __ __ _______ __
| \ _ | \ | \ | \ | \ | \
| $$ / \ | $$ ______ \$$ _| $$_ ______ | $$$$$$$\ ______ _| $$_ ______
| $$/ $\| $$ / \ | \| $$ \ / \ | $$ | $$ | \| $$ \ | \
| $$ $$$\ $$| $$$$$$\| $$ \$$$$$$ | $$$$$$\| $$ | $$ \$$$$$$\\$$$$$$ \$$$$$$\
| $$ $$\$$\$$| $$ \$$| $$ | $$ __ | $$ $$| $$ | $$ / $$ | $$ __ / $$
| $$$$ \$$$$| $$ | $$ | $$| \| $$$$$$$$| $$__/ $$| $$$$$$$ | $$| \| $$$$$$$
| $$$ \$$$| $$ | $$ \$$ $$ \$$ \| $$ $$ \$$ $$ \$$ $$ \$$ $$
\$$ \$$ \$$ \$$ \$$$$ \$$$$$$$ \$$$$$$$ \$$$$$$$ \$$$$ \$$$$$$$
power by http://patorjk.com/software/taag/

14
YunXiaoTools/target/classes/readLogo.txt
Unescape View File

@ -1,14 +0,0 @@
$$\ $$\ $$\ $$\ $$\ $$$$$$$\ $$\ $$$$$$$\ $$\
\$$\ $$ | $$ | $$ |\__| $$ __$$\ $$ |$$ __$$\ $$ |
\$$\ $$ /$$\ $$\ $$$$$$$\ \$$\ $$ |$$\ $$$$$$\ $$$$$$\ $$ | $$ | $$$$$$\ $$$$$$\ $$$$$$$ |$$ | $$ | $$$$$$\ $$$$$$\ $$$$$$\
\$$$$ / $$ | $$ |$$ __$$\ \$$$$ / $$ | \____$$\ $$ __$$\ $$$$$$$ |$$ __$$\ \____$$\ $$ __$$ |$$ | $$ | \____$$\\_$$ _| \____$$\
\$$ / $$ | $$ |$$ | $$ | $$ $$< $$ | $$$$$$$ |$$ / $$ | $$ __$$< $$$$$$$$ | $$$$$$$ |$$ / $$ |$$ | $$ | $$$$$$$ | $$ | $$$$$$$ |
$$ | $$ | $$ |$$ | $$ |$$ /\$$\ $$ |$$ __$$ |$$ | $$ | $$ | $$ |$$ ____|$$ __$$ |$$ | $$ |$$ | $$ |$$ __$$ | $$ |$$\ $$ __$$ |
$$ | \$$$$$$ |$$ | $$ |$$ / $$ |$$ |\$$$$$$$ |\$$$$$$ | $$ | $$ |\$$$$$$$\ \$$$$$$$ |\$$$$$$$ |$$$$$$$ |\$$$$$$$ | \$$$$ |\$$$$$$$ |
\__| \______/ \__| \__|\__| \__|\__| \_______| \______/ \__| \__| \_______| \_______| \_______|\_______/ \_______| \____/ \_______|
power by http://patorjk.com/software/taag/

16
YunXiaoTools/target/classes/writeLogo.txt
Unescape View File

@ -1,16 +0,0 @@
__ __ __ __ __ __ __ __ __ _______ __
| \ / \ | \ | \| \ | \ _ | \ | \ | \ | \ | \
\$$\ / $$__ __ _______ | $$ | $$ \$$ ______ ______ | $$ / \ | $$ ______ \$$ _| $$_ ______ | $$$$$$$\ ______ _| $$_ ______
\$$\/ $$| \ | \| \ \$$\/ $$| \ | \ / \ | $$/ $\| $$ / \ | \| $$ \ / \ | $$ | $$ | \| $$ \ | \
\$$ $$ | $$ | $$| $$$$$$$\ >$$ $$ | $$ \$$$$$$\| $$$$$$\ | $$ $$$\ $$| $$$$$$\| $$ \$$$$$$ | $$$$$$\| $$ | $$ \$$$$$$\\$$$$$$ \$$$$$$\
\$$$$ | $$ | $$| $$ | $$ / $$$$\ | $$ / $$| $$ | $$ | $$ $$\$$\$$| $$ \$$| $$ | $$ __ | $$ $$| $$ | $$ / $$ | $$ __ / $$
| $$ | $$__/ $$| $$ | $$| $$ \$$\| $$| $$$$$$$| $$__/ $$ | $$$$ \$$$$| $$ | $$ | $$| \| $$$$$$$$| $$__/ $$| $$$$$$$ | $$| \| $$$$$$$
| $$ \$$ $$| $$ | $$| $$ | $$| $$ \$$ $$ \$$ $$ | $$$ \$$$| $$ | $$ \$$ $$ \$$ \| $$ $$ \$$ $$ \$$ $$ \$$ $$
\$$ \$$$$$$ \$$ \$$ \$$ \$$ \$$ \$$$$$$$ \$$$$$$ \$$ \$$ \$$ \$$ \$$$$ \$$$$$$$ \$$$$$$$ \$$$$$$$ \$$$$ \$$$$$$$
power by http://patorjk.com/software/taag/
Write Preview
Loading…
Cancel
Save