@ -5,6 +5,7 @@ import com.dsideal.base.Util.LocalMysqlConnectUtil;
import com.jfinal.plugin.activerecord.Db ;
import com.jfinal.plugin.activerecord.Db ;
import com.jfinal.plugin.activerecord.Record ;
import com.jfinal.plugin.activerecord.Record ;
import com.dsideal.base.Util.CallDeepSeek ;
import com.dsideal.base.Util.CallDeepSeek ;
import java.util.List ;
import java.util.List ;
import java.util.Map ;
import java.util.Map ;
import java.util.Set ;
import java.util.Set ;
@ -13,7 +14,9 @@ import java.util.ArrayList;
import java.util.concurrent.CountDownLatch ;
import java.util.concurrent.CountDownLatch ;
import java.text.SimpleDateFormat ;
import java.text.SimpleDateFormat ;
import java.util.Date ;
import java.util.Date ;
import cn.hutool.core.io.FileUtil ;
import cn.hutool.core.io.FileUtil ;
import java.io.File ;
import java.io.File ;
public class TestMax32K {
public class TestMax32K {
@ -21,27 +24,27 @@ public class TestMax32K {
public static void main ( String [ ] args ) {
public static void main ( String [ ] args ) {
LocalMysqlConnectUtil . Init ( ) ;
LocalMysqlConnectUtil . Init ( ) ;
// 直接调用生成综合报告
// 直接调用生成综合报告
String report = generateComprehensiveReport ( ) ;
String report = generateComprehensiveReport ( ) ;
System . out . println ( "\n=== 最终报告 ===" ) ;
System . out . println ( "\n=== 最终报告 ===" ) ;
System . out . println ( report ) ;
System . out . println ( report ) ;
}
}
/ * *
/ * *
* 分 割 过 大 的 单 表 数 据
* 分 割 过 大 的 单 表 数 据
* /
* /
private static List < String > splitLargeTable ( S tring tableName , S et< String > fieldNames ,
private static List < String > splitLargeTable ( S et< String > fieldNames ,
List < Record > allTableData , int maxSize ) {
List < Record > allTableData , int maxSize ) {
List < String > chunks = new ArrayList < > ( ) ;
List < String > chunks = new ArrayList < > ( ) ;
StringBuilder currentTableChunk = new StringBuilder ( ) ;
StringBuilder currentTableChunk = new StringBuilder ( ) ;
for ( Record dataRecord : allTableData ) {
for ( Record dataRecord : allTableData ) {
Map < String , Object > columns = dataRecord . getColumns ( ) ;
Map < String , Object > columns = dataRecord . getColumns ( ) ;
StringBuilder rowData = new StringBuilder ( ) ;
StringBuilder rowData = new StringBuilder ( ) ;
rowData . append ( "[" ) ;
rowData . append ( "[" ) ;
boolean first = true ;
boolean first = true ;
for ( String fieldName : fieldNames ) {
for ( String fieldName : fieldNames ) {
if ( ! first ) rowData . append ( "," ) ;
if ( ! first ) rowData . append ( "," ) ;
@ -54,7 +57,7 @@ public class TestMax32K {
first = false ;
first = false ;
}
}
rowData . append ( "]\n" ) ;
rowData . append ( "]\n" ) ;
// 检查是否超过限制
// 检查是否超过限制
if ( currentTableChunk . length ( ) + rowData . length ( ) > maxSize ) {
if ( currentTableChunk . length ( ) + rowData . length ( ) > maxSize ) {
if ( currentTableChunk . length ( ) > 0 ) {
if ( currentTableChunk . length ( ) > 0 ) {
@ -64,59 +67,50 @@ public class TestMax32K {
}
}
currentTableChunk . append ( rowData ) ;
currentTableChunk . append ( rowData ) ;
}
}
if ( currentTableChunk . length ( ) > 0 ) {
if ( currentTableChunk . length ( ) > 0 ) {
chunks . add ( currentTableChunk . toString ( ) ) ;
chunks . add ( currentTableChunk . toString ( ) ) ;
}
}
return chunks ;
return chunks ;
}
}
/ * *
* 获 取 分 块 数 据 的 方 法 ( 可 供 其 他 类 调 用 )
* /
public static String [ ] getDataChunks ( ) {
// 这里可以将main方法中的逻辑提取出来, 返回分块数据
// 为了简化,这里只是示例
return new String [ ] { "示例数据块1" , "示例数据块2" } ;
}
public static String generateComprehensiveReport ( ) {
public static String generateComprehensiveReport ( ) {
String [ ] regions = { "文山州" , "楚雄州" } ;
String [ ] regions = { "文山州" , "楚雄州" } ;
String sql = "select table_name as TABLE_NAME from core_dataset_table where dataset_group_id in (select id from core_dataset_group where pid='1036317909951057920')" ;
String sql = "select table_name as TABLE_NAME from core_dataset_table where dataset_group_id in (select id from core_dataset_group where pid='1036317909951057920')" ;
List < Record > tableList = Db . use ( DataEaseModel . DB_NAME ) . find ( sql ) ;
List < Record > tableList = Db . use ( DataEaseModel . DB_NAME ) . find ( sql ) ;
// 获取分块数据
// 获取分块数据
String [ ] dataChunks = getDataChunks ( regions , tableList ) ;
String [ ] dataChunks = getDataChunks ( regions , tableList ) ;
List < String > chunkAnalyses = new ArrayList < > ( ) ;
List < String > chunkAnalyses = new ArrayList < > ( ) ;
System . out . println ( "开始分析 " + dataChunks . length + " 个数据块..." ) ;
System . out . println ( "开始分析 " + dataChunks . length + " 个数据块..." ) ;
// 第一阶段:流式分析各个数据块
// 第一阶段:流式分析各个数据块
for ( int i = 0 ; i < dataChunks . length ; i + + ) {
for ( int i = 0 ; i < dataChunks . length ; i + + ) {
final int chunkIndex = i ;
final int chunkIndex = i ;
final StringBuilder chunkResult = new StringBuilder ( ) ;
final StringBuilder chunkResult = new StringBuilder ( ) ;
final CountDownLatch latch = new CountDownLatch ( 1 ) ;
final CountDownLatch latch = new CountDownLatch ( 1 ) ;
String prompt = "请对以下教育数据进行简要分析, 重点关注关键指标和趋势, 控制在500字以内: \n" + dataChunks [ i ] ;
String prompt = "请对以下教育数据进行简要分析, 重点关注关键指标和趋势, 控制在500字以内: \n" + dataChunks [ i ] ;
System . out . println ( "\n=== 正在分析第 " + ( i + 1 ) + " 个数据块 ===" ) ;
System . out . println ( "\n=== 正在分析第 " + ( i + 1 ) + " 个数据块 ===" ) ;
CallDeepSeek . callDeepSeekStream ( prompt , new CallDeepSeek . SSEListener ( ) {
CallDeepSeek . callDeepSeekStream ( prompt , new CallDeepSeek . SSEListener ( ) {
@Override
@Override
public void onData ( String data ) {
public void onData ( String data ) {
System . out . print ( data ) ;
System . out . print ( data ) ;
chunkResult . append ( data ) ;
chunkResult . append ( data ) ;
}
}
@Override
@Override
public void onComplete ( String fullResponse ) {
public void onComplete ( String fullResponse ) {
System . out . println ( "\n--- 第 " + ( chunkIndex + 1 ) + " 个数据块分析完成 ---\n" ) ;
System . out . println ( "\n--- 第 " + ( chunkIndex + 1 ) + " 个数据块分析完成 ---\n" ) ;
chunkAnalyses . add ( chunkResult . toString ( ) ) ;
chunkAnalyses . add ( chunkResult . toString ( ) ) ;
latch . countDown ( ) ;
latch . countDown ( ) ;
}
}
@Override
@Override
public void onError ( String error ) {
public void onError ( String error ) {
System . err . println ( "分析第 " + ( chunkIndex + 1 ) + " 个数据块时出错: " + error ) ;
System . err . println ( "分析第 " + ( chunkIndex + 1 ) + " 个数据块时出错: " + error ) ;
@ -124,7 +118,7 @@ public class TestMax32K {
latch . countDown ( ) ;
latch . countDown ( ) ;
}
}
} ) ;
} ) ;
try {
try {
// 等待当前块分析完成
// 等待当前块分析完成
latch . await ( ) ;
latch . await ( ) ;
@ -133,47 +127,47 @@ public class TestMax32K {
System . err . println ( "等待分析结果时被中断: " + e . getMessage ( ) ) ;
System . err . println ( "等待分析结果时被中断: " + e . getMessage ( ) ) ;
}
}
}
}
// 第二阶段:流式生成综合报告
// 第二阶段:流式生成综合报告
System . out . println ( "\n=== 开始生成综合分析报告 ===" ) ;
System . out . println ( "\n=== 开始生成综合分析报告 ===" ) ;
StringBuilder combinedAnalysis = new StringBuilder ( ) ;
StringBuilder combinedAnalysis = new StringBuilder ( ) ;
combinedAnalysis . append ( "基于以下分块分析结果, 请生成一份完整的教育数据综合分析报告( 3000字以内) : \n\n" ) ;
combinedAnalysis . append ( "基于以下分块分析结果, 请生成一份完整的教育数据综合分析报告( 3000字以内) : \n\n" ) ;
for ( int i = 0 ; i < chunkAnalyses . size ( ) ; i + + ) {
for ( int i = 0 ; i < chunkAnalyses . size ( ) ; i + + ) {
combinedAnalysis . append ( "数据块" ) . append ( i + 1 ) . append ( "分析:\n" ) ;
combinedAnalysis . append ( "数据块" ) . append ( i + 1 ) . append ( "分析:\n" ) ;
combinedAnalysis . append ( chunkAnalyses . get ( i ) ) . append ( "\n\n" ) ;
combinedAnalysis . append ( chunkAnalyses . get ( i ) ) . append ( "\n\n" ) ;
}
}
final StringBuilder finalReport = new StringBuilder ( ) ;
final StringBuilder finalReport = new StringBuilder ( ) ;
final CountDownLatch finalLatch = new CountDownLatch ( 1 ) ;
final CountDownLatch finalLatch = new CountDownLatch ( 1 ) ;
CallDeepSeek . callDeepSeekStream ( combinedAnalysis . toString ( ) , new CallDeepSeek . SSEListener ( ) {
CallDeepSeek . callDeepSeekStream ( combinedAnalysis . toString ( ) , new CallDeepSeek . SSEListener ( ) {
@Override
@Override
public void onData ( String data ) {
public void onData ( String data ) {
System . out . print ( data ) ;
System . out . print ( data ) ;
finalReport . append ( data ) ;
finalReport . append ( data ) ;
}
}
@Override
@Override
public void onComplete ( String fullResponse ) {
public void onComplete ( String fullResponse ) {
System . out . println ( "\n\n=== 综合分析报告生成完成 ===" ) ;
System . out . println ( "\n\n=== 综合分析报告生成完成 ===" ) ;
// 保存报告到文件
// 保存报告到文件
try {
try {
String timestamp = new SimpleDateFormat ( "yyyyMMdd_HHmmss" ) . format ( new Date ( ) ) ;
String timestamp = new SimpleDateFormat ( "yyyyMMdd_HHmmss" ) . format ( new Date ( ) ) ;
String fileName = "教育数据综合分析报告_" + timestamp + ".txt" ;
String fileName = "教育数据综合分析报告_" + timestamp + ".txt" ;
String filePath = "WebRoot/upload/" + fileName ;
String filePath = "WebRoot/upload/" + fileName ;
FileUtil . writeString ( finalReport . toString ( ) , new File ( filePath ) , "UTF-8" ) ;
FileUtil . writeString ( finalReport . toString ( ) , new File ( filePath ) , "UTF-8" ) ;
System . out . println ( "报告已保存到: " + filePath ) ;
System . out . println ( "报告已保存到: " + filePath ) ;
} catch ( Exception e ) {
} catch ( Exception e ) {
System . err . println ( "保存报告时出错: " + e . getMessage ( ) ) ;
System . err . println ( "保存报告时出错: " + e . getMessage ( ) ) ;
}
}
finalLatch . countDown ( ) ;
finalLatch . countDown ( ) ;
}
}
@Override
@Override
public void onError ( String error ) {
public void onError ( String error ) {
System . err . println ( "生成综合报告时出错: " + error ) ;
System . err . println ( "生成综合报告时出错: " + error ) ;
@ -181,41 +175,41 @@ public class TestMax32K {
finalLatch . countDown ( ) ;
finalLatch . countDown ( ) ;
}
}
} ) ;
} ) ;
try {
try {
finalLatch . await ( ) ;
finalLatch . await ( ) ;
} catch ( InterruptedException e ) {
} catch ( InterruptedException e ) {
System . err . println ( "等待最终报告时被中断: " + e . getMessage ( ) ) ;
System . err . println ( "等待最终报告时被中断: " + e . getMessage ( ) ) ;
}
}
return finalReport . toString ( ) ;
return finalReport . toString ( ) ;
}
}
/ * *
/ * *
* 提 取 数 据 分 块 逻 辑 为 独 立 方 法
* 提 取 数 据 分 块 逻 辑 为 独 立 方 法
* /
* /
private static String [ ] getDataChunks ( String [ ] regions , List < Record > tableList ) {
private static String [ ] getDataChunks ( String [ ] regions , List < Record > tableList ) {
List < String > dataChunks = new ArrayList < > ( ) ;
List < String > dataChunks = new ArrayList < > ( ) ;
StringBuilder currentChunk = new StringBuilder ( ) ;
StringBuilder currentChunk = new StringBuilder ( ) ;
String header = "数据说明: 以下是云南省教育数据的压缩格式\n" +
String header = "数据说明: 以下是云南省教育数据的压缩格式\n" +
"格式: 表名 -> 字段列表 -> 数据行(数组格式)\n" +
"格式: 表名 -> 字段列表 -> 数据行(数组格式)\n" +
"地区范围: " + String . join ( "," , regions ) + "\n\n" ;
"地区范围: " + String . join ( "," , regions ) + "\n\n" ;
currentChunk . append ( header ) ;
currentChunk . append ( header ) ;
// 遍历所有相关数据表
// 遍历所有相关数据表
for ( Record record : tableList ) {
for ( Record record : tableList ) {
String tableName = record . getStr ( "TABLE_NAME" ) ;
String tableName = record . getStr ( "TABLE_NAME" ) ;
// 为当前表收集所有数据
// 为当前表收集所有数据
List < Record > allTableData = new ArrayList < > ( ) ;
List < Record > allTableData = new ArrayList < > ( ) ;
Set < String > fieldNames = new LinkedHashSet < > ( ) ;
Set < String > fieldNames = new LinkedHashSet < > ( ) ;
// 为每个地区收集数据
// 为每个地区收集数据
for ( String region : regions ) {
for ( String region : regions ) {
String sql = "select * from `" + tableName + "` where `行政区划`=?" ;
String sql = "select * from `" + tableName + "` where `行政区划`=?" ;
List < Record > listContent = Db . use ( DataEaseModel . DB_NAME ) . find ( sql , region ) ;
List < Record > listContent = Db . use ( DataEaseModel . DB_NAME ) . find ( sql , region ) ;
if ( ! listContent . isEmpty ( ) ) {
if ( ! listContent . isEmpty ( ) ) {
allTableData . addAll ( listContent ) ;
allTableData . addAll ( listContent ) ;
// 收集字段名(使用第一条记录的字段结构)
// 收集字段名(使用第一条记录的字段结构)
@ -224,18 +218,18 @@ public class TestMax32K {
}
}
}
}
}
}
if ( ! allTableData . isEmpty ( ) ) {
if ( ! allTableData . isEmpty ( ) ) {
// 构建当前表的完整数据块
// 构建当前表的完整数据块
StringBuilder tableData = new StringBuilder ( ) ;
StringBuilder tableData = new StringBuilder ( ) ;
tableData . append ( "\n表: " ) . append ( tableName ) . append ( "\n" ) ;
tableData . append ( "\n表: " ) . append ( tableName ) . append ( "\n" ) ;
tableData . append ( "字段: " ) . append ( String . join ( "," , fieldNames ) ) . append ( "\n" ) ;
tableData . append ( "字段: " ) . append ( String . join ( "," , fieldNames ) ) . append ( "\n" ) ;
// 输出压缩格式的数据
// 输出压缩格式的数据
for ( Record dataRecord : allTableData ) {
for ( Record dataRecord : allTableData ) {
Map < String , Object > columns = dataRecord . getColumns ( ) ;
Map < String , Object > columns = dataRecord . getColumns ( ) ;
tableData . append ( "[" ) ;
tableData . append ( "[" ) ;
boolean first = true ;
boolean first = true ;
for ( String fieldName : fieldNames ) {
for ( String fieldName : fieldNames ) {
if ( ! first ) tableData . append ( "," ) ;
if ( ! first ) tableData . append ( "," ) ;
@ -247,10 +241,10 @@ public class TestMax32K {
}
}
first = false ;
first = false ;
}
}
tableData . append ( "]\n" ) ;
tableData . append ( "]\n" ) ;
}
}
// 检查是否需要分块
// 检查是否需要分块
String tableDataStr = tableData . toString ( ) ;
String tableDataStr = tableData . toString ( ) ;
if ( currentChunk . length ( ) + tableDataStr . length ( ) > MAX_CHUNK_SIZE ) {
if ( currentChunk . length ( ) + tableDataStr . length ( ) > MAX_CHUNK_SIZE ) {
@ -260,10 +254,10 @@ public class TestMax32K {
currentChunk = new StringBuilder ( ) ;
currentChunk = new StringBuilder ( ) ;
currentChunk . append ( header ) ;
currentChunk . append ( header ) ;
}
}
// 如果单个表数据超过限制,需要进一步分割
// 如果单个表数据超过限制,需要进一步分割
if ( tableDataStr . length ( ) > MAX_CHUNK_SIZE - header . length ( ) ) {
if ( tableDataStr . length ( ) > MAX_CHUNK_SIZE - header . length ( ) ) {
List < String > tableChunks = splitLargeTable ( tableName, fieldNames, allTableData , MAX_CHUNK_SIZE - header . length ( ) ) ;
List < String > tableChunks = splitLargeTable ( fieldNames, allTableData , MAX_CHUNK_SIZE - header . length ( ) ) ;
for ( int i = 0 ; i < tableChunks . size ( ) ; i + + ) {
for ( int i = 0 ; i < tableChunks . size ( ) ; i + + ) {
StringBuilder chunkBuilder = new StringBuilder ( ) ;
StringBuilder chunkBuilder = new StringBuilder ( ) ;
chunkBuilder . append ( header ) ;
chunkBuilder . append ( header ) ;
@ -280,12 +274,12 @@ public class TestMax32K {
}
}
}
}
}
}
// 添加最后一个块
// 添加最后一个块
if ( currentChunk . length ( ) > header . length ( ) ) {
if ( currentChunk . length ( ) > header . length ( ) ) {
dataChunks . add ( currentChunk . toString ( ) ) ;
dataChunks . add ( currentChunk . toString ( ) ) ;
}
}
return dataChunks . toArray ( new String [ 0 ] ) ;
return dataChunks . toArray ( new String [ 0 ] ) ;
}
}
}
}