From 1285645ee6282321180e11d2addcb52ae5edc59c Mon Sep 17 00:00:00 2001 From: HuangHai <10402852@qq.com> Date: Thu, 27 Feb 2025 15:48:13 +0800 Subject: [PATCH] 'commit' --- AI/SyncData/config/db_config.py | 3 ++ AI/SyncData/publish.py | 21 +++++----- AI/SyncData/readme.txt | 33 +++++++++++++++- AI/Text2Sql/top10_chart.html | 10 ++--- src/main/java/Tools/Crawler/SyncDataEase.java | 39 +++++++++++++++++++ src/main/resources/Sql/DataEase.sql | 12 ++++++ 6 files changed, 103 insertions(+), 15 deletions(-) diff --git a/AI/SyncData/config/db_config.py b/AI/SyncData/config/db_config.py index b71476dc..7aa55021 100644 --- a/AI/SyncData/config/db_config.py +++ b/AI/SyncData/config/db_config.py @@ -36,6 +36,9 @@ else: CH_CONFIG = { 'host': '10.10.14.250', 'port': 9000, + 'user': 'default', + 'password': 'DsideaL147258369', + 'database': 'default', 'settings': { 'max_insert_block_size': 100000, 'async_insert': 1, diff --git a/AI/SyncData/publish.py b/AI/SyncData/publish.py index 7dd537d3..d9580801 100644 --- a/AI/SyncData/publish.py +++ b/AI/SyncData/publish.py @@ -46,10 +46,13 @@ def execute_remote_command(ssh, command): def main(): # SFTP 连接信息 - hostname = '10.10.14.250' - port = 22 + #hostname = '10.10.14.250' + hostname = 'www.wmarkj.com' + #port = 22 + port = 27250 username = 'root' - password = 'dsideal' + #password = 'dsideal' + password='DsideaL4r5t6y7u' remote_path = '/usr/local/SyncData' venv_path = '/usr/local/SyncData/venv' # 虚拟环境路径 @@ -87,14 +90,14 @@ def main(): execute_remote_command(ssh, cmd) # 构建镜像 - cmd = 'cd /usr/local/SyncData && docker build --no-cache --build-arg PYTHON_VERSION=3.10 --build-arg APP_ENV=prod -t data-sync-service:1.0 .' - execute_remote_command(ssh, cmd) - print("构建镜像完成") + #cmd = 'cd /usr/local/SyncData && docker build --no-cache --build-arg PYTHON_VERSION=3.10 --build-arg APP_ENV=prod -t data-sync-service:1.0 .' + #execute_remote_command(ssh, cmd) + #print("构建镜像完成") # 启动镜像 - cmd = 'docker run -d --rm data-sync-service:1.0' - execute_remote_command(ssh, cmd) - print("部署完成") + #cmd = 'docker run -d --rm data-sync-service:1.0' + #execute_remote_command(ssh, cmd) + #print("部署完成") finally: # 关闭 SFTP 和 SSH 连接 sftp.close() diff --git a/AI/SyncData/readme.txt b/AI/SyncData/readme.txt index 03505ece..e6608bb9 100644 --- a/AI/SyncData/readme.txt +++ b/AI/SyncData/readme.txt @@ -9,4 +9,35 @@ docker run -it --rm data-sync-service:1.0 # 模拟生产环境 export ENV=prod -python C2_SyncData.py \ No newline at end of file +python C2_SyncData.py + +# 获取镜像 +docker pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/clickhouse/clickhouse-server:24.8.7.41 +docker tag swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/clickhouse/clickhouse-server:24.8.7.41 docker.io/clickhouse/clickhouse-server:24.8.7.41 + +# 启动容器 +docker stop clickhouse && docker rm clickhouse +docker run -d --name clickhouse -e CLICKHOUSE_USER='default' -e CLICKHOUSE_PASSWORD='DsideaL147258369' --ulimit nofile=262144:262144 -p 8123:8123 -p 9000:9000 -p 9009:9009 --privileged=true -v /usr/local/clickhouse/log:/var/log/clickhouse-server -v /usr/local/clickhouse/data:/var/lib/clickhouse docker.io/clickhouse/clickhouse-server:24.8.7.41 + +------------------------------------下面是防火墙配置-------------------- +# 1. 启用并启动防火墙 +sudo systemctl enable --now firewalld + +# 2. 创建信任区域并添加白名单IP（完全开放） +sudo firewall-cmd --permanent --new-zone=trusted-ip +sudo firewall-cmd --permanent --zone=trusted-ip --add-source=124.235.206.60 +sudo firewall-cmd --permanent --zone=trusted-ip --set-target=ACCEPT + +# 3. 设置默认区域规则（其他IP只开放27251） +sudo firewall-cmd --set-default-zone=public +sudo firewall-cmd --permanent --zone=public --add-port=27251/tcp + +# 4. 应用配置 +sudo firewall-cmd --reload + +# 5. 验证配置 +sudo firewall-cmd --zone=trusted-ip --list-all # 应显示ACCEPT和正确IP +sudo firewall-cmd --zone=public --list-ports # 应显示27251/tcp + + + diff --git a/AI/Text2Sql/top10_chart.html b/AI/Text2Sql/top10_chart.html index f3fbac3c..b5e9abd7 100644 --- a/AI/Text2Sql/top10_chart.html +++ b/AI/Text2Sql/top10_chart.html @@ -8,11 +8,11 @@ -
+
diff --git a/src/main/java/Tools/Crawler/SyncDataEase.java b/src/main/java/Tools/Crawler/SyncDataEase.java index a1522508..11aeb7d0 100644 --- a/src/main/java/Tools/Crawler/SyncDataEase.java +++ b/src/main/java/Tools/Crawler/SyncDataEase.java @@ -48,6 +48,13 @@ public class SyncDataEase { ZhangJieFuGaiDetail(); //知识点覆盖率 ZhiShiDianFuGaiDetail(); + + //以下是县区的统计 + + //课程数量 + Xq_KeChengShuLiang(); + //参与学校数量 + Xq_CanYuXueXiao(); } /** @@ -410,4 +417,36 @@ public class SyncDataEase { } print("更新知识点覆盖率详情成功！"); } + + //------------------------下面是县区的大屏统计分析部分-------------------- + + /** + * 课程建设统计 + */ + public static void Xq_KeChengShuLiang() { + SqlPara sqlPara = Db.getSqlPara("DataEase.Xq_KeChengShuLiang"); + List list = Db.find(sqlPara); + String sql="update `excel_数据_78904d5d99` set `数值`=? where `行政区划`=? and `分类`='直播课程'"; + for (Record record : list) { + int cnt = record.getInt("cnt"); + String gather_regionc = record.getStr("gather_regionc"); + Db.use(MYSQL_DB).update(sql, cnt, gather_regionc); + } + print("更新县区课程建设统计成功！"); + } + + /** + * 参与学校统计 + */ + public static void Xq_CanYuXueXiao() { + SqlPara sqlPara = Db.getSqlPara("DataEase.Xq_CanYuXueXiao"); + List list = Db.find(sqlPara); + String sql="update `excel_数据_78904d5d99` set `数值`=? where `行政区划`=? and `分类`='参与学校数量'"; + for (Record record : list) { + int cnt = record.getInt("cnt"); + String gather_regionc = record.getStr("gather_regionc"); + Db.use(MYSQL_DB).update(sql, cnt, gather_regionc); + } + print("更新县区参与学校统计成功！"); + } } diff --git a/src/main/resources/Sql/DataEase.sql b/src/main/resources/Sql/DataEase.sql index 9369771e..497a3f25 100644 --- a/src/main/resources/Sql/DataEase.sql +++ b/src/main/resources/Sql/DataEase.sql @@ -305,4 +305,16 @@ ORDER BY insert into `excel_学段应用分布_c747031012`(`学段`,`学科`,`一级节点数`,`一级节点覆盖率`) values(?,?,?,?); #end + --------------------------------- 下面是县区的数据统计-------------------------------- + -- 课程建设数量 + #sql("Xq_KeChengShuLiang") + SELECT T1.gather_regionc,COUNT(DISTINCT T1.lesson_id) as cnt FROM + t_crawler_lesson_school T1 INNER JOIN t_crawler_lesson T2 ON T1.lesson_id=T2.lesson_id + WHERE T2.node_type=1 AND T2.subject_id IN ('213','241','214','264','215','225','228','224','313','314','341','316','317','318','321','320','364','325','322','324','413','414','416','441','417','418','448','421','420','425','422','424') and T1.gather_regionc<>'' GROUP BY T1.gather_regionc; + #end + -- 参与学校数量 + #sql("Xq_CanYuXueXiao") + SELECT T1.gather_regionc,COUNT(DISTINCT T1.organization_name) as cnt + FROM t_crawler_lesson_school T1 INNER JOIN t_crawler_lesson T2 ON T1.lesson_id=T2.lesson_id WHERE T1.match_type=2 AND T2.subject_id IN ('213','241','214','264','215','225','228','224','313','314','341','316','317','318','321','320','364','325','322','324','413','414','416','441','417','418','448','421','420','425','422','424') and T1.gather_regionc<>'' GROUP BY gather_regionc; + #end #end \ No newline at end of file