huanghai
/
dsProject
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

数据治理-元数据采集

Browse Source 
modify by Kalman.CHENG ☆ At 2024-06-24
main
chengminglong 4 weeks ago
parent bebee721d3
commit d43db4d172
7 changed files with 325 additions and 13 deletions
Show Stats Download Patch File Download Diff File

46
dsBase/src/main/java/com/dsideal/Base/MetaData/Controller/MetaDataController.java
Unescape View File

@ -1,5 +1,6 @@
package com.dsideal.Base.MetaData.Controller;
import cn.hutool.core.util.ZipUtil;
import com.alibaba.fastjson.JSONObject;
import com.dsideal.Base.Interceptor.EmptyInterface;
import com.dsideal.Base.Interceptor.IsNumericInterface;
@ -19,7 +20,6 @@ import org.apache.poi.hssf.usermodel.*;
import org.apache.poi.ss.usermodel.IndexedColors;
import java.io.File;
import java.net.URISyntaxException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
@ -122,7 +122,7 @@ public class MetaDataController extends Controller {
//数字：长度；
//1：String；2：int；3：date；
String[] columns = {
"column_name@字段名@1@1@50@1", "data_type@数据类型@1@0@0@1", "max_length@长度@1@2@0@2", "is_nullable@是否为空@1@3@0@2",
"column_name@字段名@1@1@50@1", "data_type@数据类型@1@0@0@1", "max_length@长度@0@2@0@2", "is_nullable@是否为空@1@3@0@2",
"primary_key@是否主键@1@3@0@2", "column_default@默认值@0@0@0@1", "column_comment@描述@1@1@1024@1", "column_create_time@创建时间@1@0@0@3",
"column_update_time@更新时间@0@0@0@3", "development_manager@负责人@1@1@200@1"
};
@ -954,4 +954,46 @@ public class MetaDataController extends Controller {
renderJson(result);
}
/**
* 功能：【MetaData-Task-Collect-10】元数据采集-生成脚本
* 作者：Kalman.CHENG ☆
* 日期：2025/6/23 10:26
* 备注：
**/
@Before({GET.class})
@EmptyInterface({"task_id"})
@IsNumericInterface({"task_id"})
public void generateSqlScript(int task_id, int supplier_id) throws Exception {
String supplier_ids = "";
if (supplier_id == 0) {
List<Record> supplier_list = metaDataModel.getSupplierListByTaskId(task_id);
for (int i = 0; i < supplier_list.size(); i++) {
supplier_ids += supplier_list.get(i).getInt("supplier_id") + ",";
}
if (!supplier_ids.isEmpty()) {
supplier_ids = supplier_ids.substring(0, supplier_ids.length() - 1);
}
} else {
supplier_ids = String.valueOf(supplier_id);
}
JSONObject result = metaDataModel.generateSqlScriptByTaskInfo(task_id, supplier_ids);
if (!result.getBoolean("success")) {
renderJson(result);
return;
}
String taskFilePath = result.getString("taskFilePath");
// 压缩文件
String filePath = taskFilePath.substring(0, taskFilePath.length() - 1) + ".zip";
ZipUtil.zip(taskFilePath, filePath, true);
// 下载zip文件
ImportExcelController.downloadFile(getResponse(), filePath);
renderNull();
}
}

254
dsBase/src/main/java/com/dsideal/Base/MetaData/Model/MetaDataModel.java
Unescape View File

@ -7,6 +7,7 @@ import com.dsideal.Base.Util.CommonUtil;
import com.dsideal.Base.Util.IpUtil;
import com.dsideal.Base.Util.JwtUtil;
import com.dsideal.Base.Util.PkUtil;
import com.dsideal.Config.PropKit;
import com.jfinal.plugin.activerecord.Db;
import com.jfinal.plugin.activerecord.Page;
import com.jfinal.plugin.activerecord.Record;
@ -21,6 +22,8 @@ import org.apache.poi.ss.usermodel.DataValidationHelper;
import org.apache.poi.ss.util.CellRangeAddressList;
import javax.servlet.http.HttpServletRequest;
import java.io.File;
import java.io.FileWriter;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
@ -412,16 +415,23 @@ public class MetaDataModel {
errorList.add(errorMessage);
}
} else if (columnVerifyTypes[j] == 2) {
errorMessage = "文件中sheet【" + table_name + "】的第" + (i + 3) + "行，列【" + columnCNs[j] + "】数据不合格，应该为大于0的整数！";
//2：验证数字
try {
int dataValueInt = Integer.parseInt(dataValue);
if (dataValueInt <= 0) {
// varchar、char 需要验证 必填长度
String columnDataType = dataList.get(i).get("data_type") + "";
if (columnDataType.equals("VARCHAR") || columnDataType.equals("CHAR")) {
errorMessage = "文件中sheet【" + table_name + "】的第" + (i + 3) + "行，列【" + columnCNs[j] + "】数据不合格，应该为大于0的整数！";
// 2：验证数字
try {
int dataValueInt = Integer.parseInt(dataValue);
if (dataValueInt <= 0) {
errorList.add(errorMessage);
}
} catch (Exception e) {
System.out.println(e.getMessage());
errorList.add(errorMessage);
}
} catch (Exception e) {
System.out.println(e.getMessage());
errorList.add(errorMessage);
} else {
// 不需要填的时候 处理现有填写数据为空
dataList.get(i).put(columnEns[j], null);
}
} else if (columnVerifyTypes[j] == 3) {
errorMessage = "文件中sheet【" + table_name + "】的第" + (i + 3) + "行，列【" + columnCNs[j] + "】数据不合格，应从下拉列表中选择！";
@ -1120,4 +1130,232 @@ public class MetaDataModel {
return result;
}
/**
* 功能：【MetaData-Task-Collect-10】元数据采集-生成脚本
* 作者：Kalman.CHENG ☆
* 日期：2025/6/23 11:40
* 备注：
**/
public JSONObject generateSqlScriptByTaskInfo(int task_id, String supplier_ids) throws Exception {
JSONObject result = new JSONObject();
List<Record> taskList = Db.find("select * from t_metadata_collect_task where is_deleted = 0 and id = " + task_id + ";");
if (taskList.isEmpty()) {
result.put("success", false);
result.put("message", "未查询到[task_id]为" + task_id + "的采集任务信息！");
return result;
}
String metaDataSqlScriptExportTemplate = CommonUtil.getClassPath() +
PropKit.get("excel.MetaDataSqlScriptExportSuffix").replace("\\", "/");
String taskFilePath = metaDataSqlScriptExportTemplate + taskList.getFirst().getStr("task_name") + System.currentTimeMillis() + File.separator;
File taskFileDir = new File(taskFilePath);
if (!taskFileDir.exists()) {
taskFileDir.mkdirs();
}
HashMap<String, String> supplierNameMap = new HashMap<>();
HashMap<String, String> systemNameMap = new HashMap<>();
HashMap<String, List<String>> supplierSystemIdsMap = new HashMap<>();
String selectTaskCollectSupplierSystemSQL = " SELECT distinct t1.supplier_id, t2.supplier_name, t1.system_id, t3.system_name " +
" FROM t_metadata_collect_task_table t1, t_metadata_supplier t2, t_metadata_system t3 " +
" WHERE t1.is_deleted = 0 AND t1.supplier_id = t2.id AND t1.system_id = t3.id and t1.task_id = " + task_id + " and t1.supplier_id in (" + supplier_ids + ")";
List<Record> selectTaskCollectSupplierSystemResult = Db.find(selectTaskCollectSupplierSystemSQL);
for (Record record : selectTaskCollectSupplierSystemResult) {
String supplier_id = record.getInt("supplier_id") + "";
String system_id = record.getInt("system_id") + "";
supplierNameMap.put(record.get("supplier_id") + "", record.getStr("supplier_name"));
systemNameMap.put(record.get("system_id") + "", record.getStr("system_name"));
List<String> list = new ArrayList<>();
if (supplierSystemIdsMap.get(supplier_id) != null) {
list = supplierSystemIdsMap.get(supplier_id);
}
list.add(system_id);
supplierSystemIdsMap.put(supplier_id, list);
}
HashMap<String, List<Record>> systemTableListMap = new HashMap<>();
String selectSystemTableListSQL = " SELECT " +
" system_id, task_id, id as table_id, table_name, table_description " +
" FROM t_metadata_collect_task_table " +
" WHERE is_deleted = 0 and task_id = " + task_id + " and supplier_id in (" + supplier_ids + ")";
List<Record> selectSystemTableList = Db.find(selectSystemTableListSQL);
for (Record record : selectSystemTableList) {
String system_id = record.getInt("system_id") + "";
List<Record> list = new ArrayList<>();
if (systemTableListMap.get(system_id) != null) {
list = systemTableListMap.get(system_id);
}
list.add(record);
systemTableListMap.put(system_id, list);
}
HashMap<String, List<Record>> tableColumnListMap = new HashMap<>();
String selectTableColumnListSQL = " SELECT " +
" task_id,system_id,table_id,id as column_id,column_name,data_type,max_length,is_nullable,primary_key,column_default,column_comment,column_create_time,column_update_time,development_manager " +
" FROM t_metadata_collect_task_column " +
" WHERE is_deleted = 0 and task_id = " + task_id + " and supplier_id in (" + supplier_ids + ")";
List<Record> selectTableColumnList = Db.find(selectTableColumnListSQL);
for (Record record : selectTableColumnList) {
String table_id = record.getInt("table_id") + "";
List<Record> list = new ArrayList<>();
if (tableColumnListMap.get(table_id) != null) {
list = tableColumnListMap.get(table_id);
}
list.add(record);
tableColumnListMap.put(table_id, list);
}
// 遍历supplier, system, table 生成脚本
Iterator<Map.Entry<String, List<String>>> iterator = supplierSystemIdsMap.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, List<String>> entry = iterator.next();
String supplier_id = entry.getKey();
String supplier_name = supplierNameMap.getOrDefault(supplier_id, "");
String taskSupplierFilePath = taskFilePath + supplier_name + File.separator;
File taskSupplierFileDir = new File(taskSupplierFilePath);
if (!taskSupplierFileDir.exists()) {
taskSupplierFileDir.mkdirs();
}
List<String> systemList = entry.getValue();
for (String system_id : systemList) {
String system_name = systemNameMap.getOrDefault(system_id, "");
List<Record> tableList = systemTableListMap.containsKey(system_id) ? systemTableListMap.get(system_id) : new ArrayList<>();
List<String> tableSqlList = new ArrayList<>();
// 遍历table
for (Record record : tableList) {
String table_id = record.getInt("table_id") + "";
String table_name = record.getStr("table_name");
String table_description = record.getStr("table_description");
String createTableSQL = " CREATE TABLE IF NOT EXISTS `" + table_name + "` (\n";
List<Record> columnList = tableColumnListMap.containsKey(table_id) ? tableColumnListMap.get(table_id) : new ArrayList<>();
if (columnList.size() > 0) {
String primaryKeyColumnId = "";
Record primaryRecord = null;
for (Record columnRecord : columnList) {
if (columnRecord.getInt("primary_key") == 1) {
primaryKeyColumnId = columnRecord.getInt("column_id") + "";
primaryRecord = columnRecord;
break;
}
}
if (primaryRecord != null) {
String column_name = primaryRecord.getStr("column_name");
String data_type = primaryRecord.getStr("data_type");
int max_length = primaryRecord.getInt("max_length");
int is_nullable = primaryRecord.getInt("is_nullable");
int primary_key = primaryRecord.getInt("primary_key");
String column_default = primaryRecord.getStr("column_default");
String column_comment = primaryRecord.getStr("column_comment");
createTableSQL += " `" + column_name + "` ";
if ("INT".equals(data_type) || "BIGINT".equals(data_type) || "DECIMAL".equals(data_type)) {
// INT、BIGINT、DECIMAL
if ("INT".equals(data_type)) {
createTableSQL += "INT(11) ";
} else if ("BIGINT".equals(data_type)) {
createTableSQL += "BIGINT(20) ";
} else {
createTableSQL += "DECIMAL(20,6) ";
}
} else if ("VARCHAR".equals(data_type) || "CHAR".equals(data_type)) {
// VARCHAR、CHAR
createTableSQL += data_type + "(" + max_length + ") ";
} else {
// TEXT、DATE、DATETIME、BOOLEAN、FLOAT、DOUBLE
createTableSQL += data_type + " ";
}
createTableSQL += "NOT NULL ";
if (column_comment != null || !"null".equals(column_comment) || !"NULL".equals(column_comment) || !"".equals(column_comment)) {
createTableSQL += "COMMENT '" + column_comment + "', \n";
}
}
// 遍历非主键字段
for (Record columnRecord : columnList) {
String column_id = columnRecord.getStr("column_id");
if (column_id.equals(primaryKeyColumnId)) {
continue;
}
String column_name = columnRecord.getStr("column_name");
String data_type = columnRecord.getStr("data_type");
int max_length = columnRecord.getInt("max_length");
int is_nullable = columnRecord.getInt("is_nullable");
String column_default = columnRecord.getStr("column_default");
String column_comment = columnRecord.getStr("column_comment");
createTableSQL += " `" + column_name + "` ";
if ("INT".equals(data_type) || "BIGINT".equals(data_type) || "DECIMAL".equals(data_type)) {
// INT、BIGINT、DECIMAL
if ("INT".equals(data_type)) {
createTableSQL += "INT(11) ";
} else if ("BIGINT".equals(data_type)) {
createTableSQL += "BIGINT(20) ";
} else {
createTableSQL += "DECIMAL(20,6) ";
}
} else if ("VARCHAR".equals(data_type) || "CHAR".equals(data_type)) {
// VARCHAR、CHAR
createTableSQL += data_type + "(" + max_length + ") ";
} else {
// TEXT、DATE、DATETIME、BOOLEAN、FLOAT、DOUBLE
createTableSQL += data_type + " ";
}
if (is_nullable == 1) {
createTableSQL += "NOT NULL ";
} else {
createTableSQL += "NULL ";
}
if (column_default != null && !"null".equals(column_default) && !"NULL".equals(column_default) && !"".equals(column_default)) {
if ("INT".equals(data_type) || "BIGINT".equals(data_type) || "DECIMAL".equals(data_type) ||
"FLOAT".equals(data_type) || "DOUBLE".equals(data_type) || "BOOLEAN".equals(data_type)) {
createTableSQL += "DEFAULT " + column_default + " ";
} else if ("VARCHAR".equals(data_type) || "CHAR".equals(data_type) || "TEXT".equals(data_type)) {
createTableSQL += "DEFAULT '" + column_default + "' ";
} else if("DATE".equals(data_type) || "DATETIME".equals(data_type)) {
if (column_default.contains("()")) {
createTableSQL += "DEFAULT " + column_default + " ";
} else {
createTableSQL += "DEFAULT '" + column_default + "' ";
}
}
}
if (column_comment != null || !"null".equals(column_comment) || !"NULL".equals(column_comment) || !"".equals(column_comment)) {
createTableSQL += "COMMENT '" + column_comment + "', \n";
}
}
createTableSQL += " PRIMARY KEY (`" + primaryRecord.get("column_name") + "`) \n";
createTableSQL += ") ENGINE = InnoDB CHARACTER SET='utf8' COLLATE='utf8_general_ci' COMMENT='" + (table_description == null ? "" : table_description) + "' ROW_FORMAT=COMPACT;";
tableSqlList.add(createTableSQL);
}
}
// 生成文件
String systemWriterStr = "";
for (String sql : tableSqlList) {
systemWriterStr += sql + "\n\n\n";
}
File systemFile = new File(taskSupplierFilePath + system_name + ".sql");
try {
FileWriter systemWriter = new FileWriter(systemFile);
systemWriter.write(systemWriterStr);
systemWriter.close();
} catch (Exception e) {
System.out.println("写入sql文件出错！");
result.put("success", false);
result.put("message", "写入sql文件出错！");
return result;
}
}
}
result.put("success", true);
result.put("message", "SQL文件生成成功！");
result.put("taskFilePath", taskFilePath);
return result;
}
}

28
dsBase/src/main/java/com/dsideal/Base/MetaData/Utils/ImportExcelController.java
Unescape View File

@ -318,4 +318,32 @@ public class ImportExcelController extends Controller {
}
}
/**
* 功能：下载zip文件
* 作者：Kalman.CHENG ☆
* 日期：2025/6/24 8:23
* 备注：
**/
public static void downloadFile(HttpServletResponse response, String filePath) throws Exception {
File file = new File(filePath);
String fileName = file.getName();
// 读到流中
InputStream inputStream = new FileInputStream(filePath);
// 设置输出的格式
response.reset();
response.setContentType("application/octet-stream");
response.setHeader("Content-Disposition", "attachment; filename=\"" + URLEncoder.encode(fileName, "UTF-8") + "\"; fileName*=UTF-8''" + URLEncoder.encode(fileName, "UTF-8"));
// 循环取出流中的数据
byte[] b = new byte[100];
int len;
try {
while ((len = inputStream.read(b)) > 0) {
response.getOutputStream().write(b, 0, len);
}
inputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}

BIN
dsBase/src/main/resources/ExcelMetaDataImportTemplate/metadataTemplate.xls
View File

Binary file not shown.

4
dsBase/src/main/resources/Sql/metadata.sql
Unescape View File

@ -161,7 +161,7 @@
SELECT
t1.*, t2.supplier_name
FROM t_metadata_collect_task_supplier t1, t_metadata_supplier t2
WHERE t1.is_deleted = 0 and t2.is_deleted and t1.task_id = ? and t1.supplier_id = t2.id
WHERE t1.is_deleted = 0 and t2.is_deleted = 0 and t1.task_id = ? and t1.supplier_id = t2.id
#end
-- 根据任务ID、供应商ID获取table_list
@ -213,7 +213,7 @@
#end
-- 删除元数据采集任务，字段信息
-- 删除元数据采集任务，表信息
#sql("deleteTaskTableById")
UPDATE t_metadata_collect_task_table
SET person_id = ?, is_deleted = 1, update_time = now()

2
dsBase/src/main/resources/application_dev.yaml
Unescape View File

@ -35,3 +35,5 @@ excel:
ExcelImportTemplatePathSuffix: /ExcelImportTemplate/
# 元数据导入模板配置路径 add by Kalman.CHENG ☆ at 2025-06-09
ExcelMetaDataImportTemplatePathSuffix: /ExcelMetaDataImportTemplate/
# 元数据采集任务SQL脚本导出路径 add by Kalman.CHENG ☆ At 2025-06-23
MetaDataSqlScriptExportSuffix: /metaDataSqlScriptExport/

4
dsBase/src/main/resources/application_pro.yaml
Unescape View File

@ -34,4 +34,6 @@ excel:
# 导入excel 的模板配置路径
ExcelImportTemplatePathSuffix: /ExcelImportTemplate/
# 元数据导入模板配置路径 add by Kalman.CHENG ☆ at 2025-06-09
ExcelMetaDataImportTemplatePathSuffix: /ExcelMetaDataImportTemplate/
ExcelMetaDataImportTemplatePathSuffix: /ExcelMetaDataImportTemplate/
# 元数据采集任务SQL脚本导出路径 add by Kalman.CHENG ☆ At 2025-06-23
MetaDataSqlScriptExportSuffix: /metaDataSqlScriptExport/
Write Preview
Loading…
Cancel
Save