'commit'

src/main/java/com/dsideal/base/AI/TestMax32K.java

@@ -0,0 +1,189 @@
+package com.dsideal.base.AI;
+
+import com.dsideal.base.DataEase.Model.DataEaseModel;
+import com.dsideal.base.Util.LocalMysqlConnectUtil;
+import com.jfinal.plugin.activerecord.Db;
+import com.jfinal.plugin.activerecord.Record;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.LinkedHashSet;
+import java.util.ArrayList;
+
+public class TestMax32K {
+    private static final int MAX_CHUNK_SIZE = 30000; // 30K字符限制
+
+    public static void main(String[] args) {
+        LocalMysqlConnectUtil.Init();
+
+        String[] regions = {"文山州", "楚雄州"};
+        String sql = "select table_name as TABLE_NAME from core_dataset_table where dataset_group_id in (select id from core_dataset_group where pid='1036317909951057920')";
+        List<Record> tableList = Db.use(DataEaseModel.DB_NAME).find(sql);
+        
+        // 使用字符串数组存储分块数据
+        List<String> dataChunks = new ArrayList<>();
+        StringBuilder currentChunk = new StringBuilder();
+        
+        // 添加数据说明头部
+        String header = "数据说明: 以下是云南省教育数据的压缩格式\n" +
+                       "格式: 表名 -> 字段列表 -> 数据行(数组格式)\n" +
+                       "地区范围: " + String.join(",", regions) + "\n\n";
+        currentChunk.append(header);
+        
+        // 遍历所有相关数据表
+        for (Record record : tableList) {
+            String tableName = record.getStr("TABLE_NAME");
+            
+            // 为当前表收集所有数据
+            List<Record> allTableData = new ArrayList<>();
+            Set<String> fieldNames = new LinkedHashSet<>();
+            
+            // 为每个地区收集数据
+            for (String region : regions) {
+                sql = "select * from `" + tableName + "` where `行政区划`=?";
+                List<Record> listContent = Db.use(DataEaseModel.DB_NAME).find(sql, region);
+                
+                if (!listContent.isEmpty()) {
+                    allTableData.addAll(listContent);
+                    // 收集字段名（使用第一条记录的字段结构）
+                    if (fieldNames.isEmpty()) {
+                        fieldNames.addAll(listContent.get(0).getColumns().keySet());
+                    }
+                }
+            }
+            
+            if (!allTableData.isEmpty()) {
+                // 构建当前表的完整数据块
+                StringBuilder tableData = new StringBuilder();
+                tableData.append("\n表: ").append(tableName).append("\n");
+                tableData.append("字段: ").append(String.join(",", fieldNames)).append("\n");
+                
+                // 输出压缩格式的数据
+                for (Record dataRecord : allTableData) {
+                    Map<String, Object> columns = dataRecord.getColumns();
+                    tableData.append("[");
+                    
+                    boolean first = true;
+                    for (String fieldName : fieldNames) {
+                        if (!first) tableData.append(",");
+                        Object value = columns.get(fieldName);
+                        if (value instanceof String) {
+                            tableData.append("\"").append(value).append("\"");
+                        } else {
+                            tableData.append(value);
+                        }
+                        first = false;
+                    }
+                    
+                    tableData.append("]\n");
+                }
+                
+                // 检查是否需要分块
+                String tableDataStr = tableData.toString();
+                if (currentChunk.length() + tableDataStr.length() > MAX_CHUNK_SIZE) {
+                    // 当前块已满，保存并开始新块
+                    if (currentChunk.length() > header.length()) {
+                        dataChunks.add(currentChunk.toString());
+                        currentChunk = new StringBuilder();
+                        currentChunk.append(header);
+                    }
+                    
+                    // 如果单个表数据超过限制，需要进一步分割
+                    if (tableDataStr.length() > MAX_CHUNK_SIZE - header.length()) {
+                        List<String> tableChunks = splitLargeTable(tableName, fieldNames, allTableData, MAX_CHUNK_SIZE - header.length());
+                        for (int i = 0; i < tableChunks.size(); i++) {
+                            StringBuilder chunkBuilder = new StringBuilder();
+                            chunkBuilder.append(header);
+                            chunkBuilder.append("\n[续] 表: ").append(tableName).append(" (第").append(i + 1).append("部分)\n");
+                            chunkBuilder.append("字段: ").append(String.join(",", fieldNames)).append("\n");
+                            chunkBuilder.append(tableChunks.get(i));
+                            dataChunks.add(chunkBuilder.toString());
+                        }
+                    } else {
+                        currentChunk.append(tableDataStr);
+                    }
+                } else {
+                    currentChunk.append(tableDataStr);
+                }
+            }
+        }
+        
+        // 添加最后一个块
+        if (currentChunk.length() > header.length()) {
+            dataChunks.add(currentChunk.toString());
+        }
+        
+        // 输出分块结果统计
+        System.out.println("总共分成 " + dataChunks.size() + " 个数据块:");
+        for (int i = 0; i < dataChunks.size(); i++) {
+            String chunk = dataChunks.get(i);
+            System.out.println("数据块 " + (i + 1) + " 长度: " + chunk.length() + " 字符");
+        }
+        
+        // 返回分块数据数组供后续使用
+        String[] chunksArray = dataChunks.toArray(new String[0]);
+        
+        // 示例：如何使用分块数据
+        System.out.println("\n=== 可以这样使用分块数据 ===");
+        for (int i = 0; i < chunksArray.length; i++) {
+            System.out.println("处理第 " + (i + 1) + " 个数据块...");
+            // 这里可以调用DeepSeek API处理每个块
+            // String result = CallDeepSeek.callDeepSeek(chunksArray[i]);
+            System.out.println("块 " + (i + 1) + " 内容预览: " + 
+                chunksArray[i].substring(0, Math.min(200, chunksArray[i].length())) + "...");
+        }
+    }
+    
+    /**
+     * 分割过大的单表数据
+     */
+    private static List<String> splitLargeTable(String tableName, Set<String> fieldNames, 
+                                               List<Record> allTableData, int maxSize) {
+        List<String> chunks = new ArrayList<>();
+        StringBuilder currentTableChunk = new StringBuilder();
+        
+        for (Record dataRecord : allTableData) {
+            Map<String, Object> columns = dataRecord.getColumns();
+            StringBuilder rowData = new StringBuilder();
+            rowData.append("[");
+            
+            boolean first = true;
+            for (String fieldName : fieldNames) {
+                if (!first) rowData.append(",");
+                Object value = columns.get(fieldName);
+                if (value instanceof String) {
+                    rowData.append("\"").append(value).append("\"");
+                } else {
+                    rowData.append(value);
+                }
+                first = false;
+            }
+            rowData.append("]\n");
+            
+            // 检查是否超过限制
+            if (currentTableChunk.length() + rowData.length() > maxSize) {
+                if (currentTableChunk.length() > 0) {
+                    chunks.add(currentTableChunk.toString());
+                    currentTableChunk = new StringBuilder();
+                }
+            }
+            currentTableChunk.append(rowData);
+        }
+        
+        if (currentTableChunk.length() > 0) {
+            chunks.add(currentTableChunk.toString());
+        }
+        
+        return chunks;
+    }
+    
+    /**
+     * 获取分块数据的方法（可供其他类调用）
+     */
+    public static String[] getDataChunks() {
+        // 这里可以将main方法中的逻辑提取出来，返回分块数据
+        // 为了简化，这里只是示例
+        return new String[]{"示例数据块1", "示例数据块2"};
+    }
+}

sse-page.html

@@ -0,0 +1,296 @@
+<!DOCTYPE html>
+<html lang="zh-CN">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>SSE数据生成页面</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            max-width: 800px;
+            margin: 0 auto;
+            padding: 20px;
+            background-color: #f5f5f5;
+        }
+        
+        .container {
+            background: white;
+            padding: 30px;
+            border-radius: 8px;
+            box-shadow: 0 2px 10px rgba(0,0,0,0.1);
+        }
+        
+        h1 {
+            color: #333;
+            text-align: center;
+            margin-bottom: 30px;
+        }
+        
+        .button-container {
+            text-align: center;
+            margin-bottom: 30px;
+        }
+        
+        #generateBtn {
+            background-color: #007bff;
+            color: white;
+            border: none;
+            padding: 12px 30px;
+            font-size: 16px;
+            border-radius: 5px;
+            cursor: pointer;
+            transition: background-color 0.3s;
+        }
+        
+        #generateBtn:hover {
+            background-color: #0056b3;
+        }
+        
+        #generateBtn:disabled {
+            background-color: #6c757d;
+            cursor: not-allowed;
+        }
+        
+        .status {
+            text-align: center;
+            margin-bottom: 20px;
+            font-weight: bold;
+        }
+        
+        .status.connecting {
+            color: #ffc107;
+        }
+        
+        .status.connected {
+            color: #28a745;
+        }
+        
+        .status.error {
+            color: #dc3545;
+        }
+        
+        .status.completed {
+            color: #17a2b8;
+        }
+        
+        .data-area {
+            border: 1px solid #ddd;
+            border-radius: 5px;
+            padding: 15px;
+            min-height: 300px;
+            max-height: 400px;
+            overflow-y: auto;
+            background-color: #f8f9fa;
+            font-family: 'Courier New', monospace;
+            font-size: 14px;
+            line-height: 1.4;
+            white-space: pre-wrap;
+            word-wrap: break-word;
+        }
+        
+        .download-section {
+            margin-top: 20px;
+            padding: 15px;
+            background-color: #e9ecef;
+            border-radius: 5px;
+        }
+        
+        .download-links {
+            margin-top: 10px;
+        }
+        
+        .download-link {
+            display: inline-block;
+            margin: 5px 10px 5px 0;
+            padding: 8px 15px;
+            background-color: #28a745;
+            color: white;
+            text-decoration: none;
+            border-radius: 4px;
+            transition: background-color 0.3s;
+        }
+        
+        .download-link:hover {
+            background-color: #218838;
+            text-decoration: none;
+            color: white;
+        }
+        
+        .clear-btn {
+            background-color: #6c757d;
+            color: white;
+            border: none;
+            padding: 8px 15px;
+            border-radius: 4px;
+            cursor: pointer;
+            margin-top: 10px;
+        }
+        
+        .clear-btn:hover {
+            background-color: #545b62;
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <h1>SSE数据生成器</h1>
+        
+        <div class="button-container">
+            <button id="generateBtn" onclick="startGeneration()">生成</button>
+        </div>
+        
+        <div id="status" class="status">准备就绪</div>
+        
+        <div class="data-area" id="dataArea">等待数据...</div>
+        
+        <button class="clear-btn" onclick="clearData()">清空数据</button>
+        
+        <div class="download-section">
+            <h3>下载文件：</h3>
+            <div id="downloadLinks" class="download-links">
+                <span style="color: #6c757d;">暂无可下载文件</span>
+            </div>
+        </div>
+    </div>
+
+    <script>
+        let eventSource = null;
+        let downloadUrls = [];
+        
+        function startGeneration() {
+            const generateBtn = document.getElementById('generateBtn');
+            const statusDiv = document.getElementById('status');
+            const dataArea = document.getElementById('dataArea');
+            
+            // 禁用按钮
+            generateBtn.disabled = true;
+            generateBtn.textContent = '生成中...';
+            
+            // 清空之前的数据
+            dataArea.textContent = '';
+            downloadUrls = [];
+            updateDownloadLinks();
+            
+            // 更新状态
+            statusDiv.textContent = '正在连接...';
+            statusDiv.className = 'status connecting';
+            
+            // 创建SSE连接
+            // 注意：请将下面的URL替换为你的实际接口地址
+            eventSource = new EventSource('http://xxx/a.action');
+            
+            eventSource.onopen = function(event) {
+                statusDiv.textContent = '连接成功，等待数据...';
+                statusDiv.className = 'status connected';
+            };
+            
+            eventSource.onmessage = function(event) {
+                const data = event.data;
+                
+                // 在数据区域显示接收到的数据
+                dataArea.textContent += data + '\n';
+                
+                // 自动滚动到底部
+                dataArea.scrollTop = dataArea.scrollHeight;
+                
+                // 检查是否包含下载链接
+                checkForDownloadLinks(data);
+            };
+            
+            eventSource.onerror = function(event) {
+                console.error('SSE连接错误:', event);
+                statusDiv.textContent = '连接错误或已断开';
+                statusDiv.className = 'status error';
+                
+                // 重新启用按钮
+                generateBtn.disabled = false;
+                generateBtn.textContent = '生成';
+                
+                // 关闭连接
+                if (eventSource) {
+                    eventSource.close();
+                    eventSource = null;
+                }
+            };
+            
+            // 监听自定义事件（如果服务器发送特定事件类型）
+            eventSource.addEventListener('complete', function(event) {
+                statusDiv.textContent = '数据生成完成';
+                statusDiv.className = 'status completed';
+                
+                // 重新启用按钮
+                generateBtn.disabled = false;
+                generateBtn.textContent = '生成';
+                
+                // 关闭连接
+                eventSource.close();
+                eventSource = null;
+            });
+        }
+        
+        function checkForDownloadLinks(data) {
+            // 使用正则表达式匹配URL模式
+            const urlPattern = /https?:\/\/[^\s]+\.(docx|pdf|xlsx|txt|zip|rar)/gi;
+            const matches = data.match(urlPattern);
+            
+            if (matches) {
+                matches.forEach(url => {
+                    if (!downloadUrls.includes(url)) {
+                        downloadUrls.push(url);
+                    }
+                });
+                updateDownloadLinks();
+            }
+        }
+        
+        function updateDownloadLinks() {
+            const downloadLinksDiv = document.getElementById('downloadLinks');
+            
+            if (downloadUrls.length === 0) {
+                downloadLinksDiv.innerHTML = '<span style="color: #6c757d;">暂无可下载文件</span>';
+                return;
+            }
+            
+            let linksHtml = '';
+            downloadUrls.forEach((url, index) => {
+                const fileName = url.split('/').pop() || `文件${index + 1}`;
+                linksHtml += `<a href="${url}" class="download-link" target="_blank" download>${fileName}</a>`;
+            });
+            
+            downloadLinksDiv.innerHTML = linksHtml;
+        }
+        
+        function clearData() {
+            const dataArea = document.getElementById('dataArea');
+            dataArea.textContent = '等待数据...';
+            
+            // 可选：也清空下载链接
+            // downloadUrls = [];
+            // updateDownloadLinks();
+        }
+        
+        // 页面卸载时关闭SSE连接
+        window.addEventListener('beforeunload', function() {
+            if (eventSource) {
+                eventSource.close();
+            }
+        });
+        
+        // 停止生成功能（可选）
+        function stopGeneration() {
+            if (eventSource) {
+                eventSource.close();
+                eventSource = null;
+                
+                const generateBtn = document.getElementById('generateBtn');
+                const statusDiv = document.getElementById('status');
+                
+                generateBtn.disabled = false;
+                generateBtn.textContent = '生成';
+                statusDiv.textContent = '已停止';
+                statusDiv.className = 'status error';
+            }
+        }
+    </script>
+</body>
+</html>