main
HuangHai 5 months ago
parent 3c69bfcab6
commit 2e916f8a03

@ -1,7 +1,3 @@
-- ----------------------------
-- Table structure for t_crawler_lesson
-- ----------------------------
DROP TABLE IF EXISTS "public"."t_crawler_lesson";
CREATE TABLE "public"."t_crawler_lesson" (
"lesson_id" varchar(255) COLLATE "pg_catalog"."default",
"lesson_name" varchar(255) COLLATE "pg_catalog"."default",
@ -13,8 +9,6 @@ CREATE TABLE "public"."t_crawler_lesson" (
"id" varchar(255) COLLATE "pg_catalog"."default" NOT NULL,
"book_id" varchar(255) COLLATE "pg_catalog"."default",
"scheme_id" varchar(255) COLLATE "pg_catalog"."default",
"subject_id" varchar(255) COLLATE "pg_catalog"."default",
"stage_id" varchar(255) COLLATE "pg_catalog"."default",
"preview_count" int4 DEFAULT 0,
"learning_person_count" int4 DEFAULT 0,
"learning_person_times" int4 DEFAULT 0,
@ -23,7 +17,8 @@ CREATE TABLE "public"."t_crawler_lesson" (
"node_type" int2 DEFAULT 1,
"gather_regionc" varchar(255) COLLATE "pg_catalog"."default",
"school_running_type" varchar(255) COLLATE "pg_catalog"."default",
"stage_name" varchar(255) COLLATE "pg_catalog"."default"
"stage_name" varchar(255) COLLATE "pg_catalog"."default",
"subject_name" varchar(255) COLLATE "pg_catalog"."default"
)
;
COMMENT ON COLUMN "public"."t_crawler_lesson"."lesson_id" IS '课程ID';
@ -45,6 +40,5 @@ COMMENT ON COLUMN "public"."t_crawler_lesson"."node_type" IS '节点类型 1
COMMENT ON COLUMN "public"."t_crawler_lesson"."gather_regionc" IS '行政区域名称';
COMMENT ON COLUMN "public"."t_crawler_lesson"."school_running_type" IS '学校类型';
COMMENT ON COLUMN "public"."t_crawler_lesson"."stage_name" IS '学段名称';
COMMENT ON TABLE "public"."t_crawler_lesson" IS '课程resource资源表';
COMMENT ON COLUMN "public"."t_crawler_lesson"."subject_name" IS '学科名称';
COMMENT ON TABLE "public"."t_crawler_lesson" IS '课程resource资源表';

@ -7,70 +7,6 @@ from Text2Sql.Util.PostgreSQLUtil import PostgreSQLUtil
from Text2Sql.Util.SaveToExcel import save_to_excel
from Text2Sql.Util.Text2SqlUtil import *
def generate_bar_chart_from_result(result, x_column: str, y_columns: list, output_file: str = "bar_chart.html"):
"""
根据结果集生成柱状图并保存为 HTML 文件
参数:
result: 结果集通常是一个列表字典格式
x_column (str): 作为 X 轴的列名
y_columns (list): 作为 Y 轴的列名列表
output_file (str): 输出 HTML 文件名默认 "bar_chart.html"
"""
try:
# 如果结果集为空,直接返回
if not result:
raise ValueError("结果集为空,无法生成图表!")
# 检查列名是否存在
if x_column not in result[0]:
raise ValueError(f"X 轴列名 '{x_column}' 不存在!")
for col in y_columns:
if col not in result[0]:
raise ValueError(f"Y 轴列名 '{col}' 不存在!")
# 提取 X 轴和 Y 轴数据
x_axis_data = [row[x_column] for row in result]
series_data = [
(col, [row[col] for row in result])
for col in y_columns
]
print("X 轴数据:", x_axis_data)
print("Y 轴数据:", series_data)
# 创建柱状图
bar = Bar()
bar.add_xaxis(x_axis_data)
for name, data in series_data:
bar.add_yaxis(name, data)
# 设置全局配置
bar.set_global_opts(
title_opts=opts.TitleOpts(title="动态柱状图"),
xaxis_opts=opts.AxisOpts(name=x_column),
yaxis_opts=opts.AxisOpts(name=""),
legend_opts=opts.LegendOpts(is_show=True)
)
# 确保目标目录存在
if not os.path.exists(os.path.dirname(output_file)):
os.makedirs(os.path.dirname(output_file))
# 保存为 HTML 文件
bar.render(output_file)
print(f"柱状图已保存为 {output_file}")
# 检查文件是否存在
if os.path.exists(output_file):
print("文件生成成功!")
else:
print("文件生成失败!")
except Exception as e:
print(f"生成柱状图时发生错误: {e}")
'''
经验
1尽量使用宽表少用关联越少越好
@ -82,7 +18,7 @@ if __name__ == "__main__":
# 开始训练
print("开始训练...")
# 打开CreateTable.sql文件内容
# 打开AreaSchoolLesson.sql文件内容
with open("Sql/AreaSchoolLesson.sql", "r", encoding="utf-8") as file:
ddl = file.read()
# 训练数据
@ -117,9 +53,7 @@ if __name__ == "__main__":
common_prompt = '''
返回的信息要求
1行政区划为NULL 或者是空字符的不参加统计
2目标数据库是Postgresql 16,注意字段名称不要有二义性问题
3使用stage_name描述学段,一定不要使用stage_id
4使用subject_name描述科目,一定不要使用subject_id
2目标数据库是Postgresql 16
'''
question = question + common_prompt
# 开始查询
@ -132,13 +66,6 @@ if __name__ == "__main__":
with PostgreSQLUtil() as db:
_data = db.execute_query(sql)
# 生成柱状图
# generate_bar_chart_from_result(
# _data,
# x_column="科目",
# y_columns=["课程数量"],
# output_file="d:/lesson_bar_chart.html"
# )
filename = "d:/导出信息.xlsx"
save_to_excel(_data, filename)
os.startfile(filename)

@ -1,7 +1,5 @@
package com.dsideal.QingLong.Cron4j;
import Tools.Crawler.Util.BookLesson;
import Tools.Crawler.Util.KnowledgeLesson;
import com.jfinal.plugin.cron4j.ITask;
import lombok.SneakyThrows;
@ -11,13 +9,6 @@ public class LingDianTask implements ITask {
public void run() {
// 这里放被执行的调试任务代码
Tools.FillSubject.syncSubject();
//爬去云校数据
//同步章节目录
BookLesson.Start();
//同步知识点
KnowledgeLesson.Start();
//打补丁
BookLesson.fixPatch();
}
public void stop() {

Loading…
Cancel
Save