huanghai
/
QingLong
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

'commit'

Browse Source
main
HuangHai 5 months ago
parent 3c570b7432
commit 1285645ee6
6 changed files with 103 additions and 15 deletions
Show Stats Download Patch File Download Diff File

3
AI/SyncData/config/db_config.py
Unescape View File

@ -36,6 +36,9 @@ else:
CH_CONFIG = {
'host': '10.10.14.250',
'port': 9000,
'user': 'default',
'password': 'DsideaL147258369',
'database': 'default',
'settings': {
'max_insert_block_size': 100000,
'async_insert': 1,

21
AI/SyncData/publish.py
Unescape View File

@ -46,10 +46,13 @@ def execute_remote_command(ssh, command):
def main():
# SFTP 连接信息
hostname = '10.10.14.250'
port = 22
#hostname = '10.10.14.250'
hostname = 'www.wmarkj.com'
#port = 22
port = 27250
username = 'root'
password = 'dsideal'
#password = 'dsideal'
password='DsideaL4r5t6y7u'
remote_path = '/usr/local/SyncData'
venv_path = '/usr/local/SyncData/venv' # 虚拟环境路径
@ -87,14 +90,14 @@ def main():
execute_remote_command(ssh, cmd)
# 构建镜像
cmd = 'cd /usr/local/SyncData && docker build --no-cache --build-arg PYTHON_VERSION=3.10 --build-arg APP_ENV=prod -t data-sync-service:1.0 .'
execute_remote_command(ssh, cmd)
print("构建镜像完成")
#cmd = 'cd /usr/local/SyncData && docker build --no-cache --build-arg PYTHON_VERSION=3.10 --build-arg APP_ENV=prod -t data-sync-service:1.0 .'
#execute_remote_command(ssh, cmd)
#print("构建镜像完成")
# 启动镜像
cmd = 'docker run -d --rm data-sync-service:1.0'
execute_remote_command(ssh, cmd)
print("部署完成")
#cmd = 'docker run -d --rm data-sync-service:1.0'
#execute_remote_command(ssh, cmd)
#print("部署完成")
finally:
# 关闭 SFTP 和 SSH 连接
sftp.close()

33
AI/SyncData/readme.txt
Unescape View File

@ -9,4 +9,35 @@ docker run -it --rm data-sync-service:1.0
# 模拟生产环境
export ENV=prod
python C2_SyncData.py
python C2_SyncData.py
# 获取镜像
docker pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/clickhouse/clickhouse-server:24.8.7.41
docker tag swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/clickhouse/clickhouse-server:24.8.7.41 docker.io/clickhouse/clickhouse-server:24.8.7.41
# 启动容器
docker stop clickhouse && docker rm clickhouse
docker run -d --name clickhouse -e CLICKHOUSE_USER='default' -e CLICKHOUSE_PASSWORD='DsideaL147258369' --ulimit nofile=262144:262144 -p 8123:8123 -p 9000:9000 -p 9009:9009 --privileged=true -v /usr/local/clickhouse/log:/var/log/clickhouse-server -v /usr/local/clickhouse/data:/var/lib/clickhouse docker.io/clickhouse/clickhouse-server:24.8.7.41
------------------------------------下面是防火墙配置--------------------
# 1. 启用并启动防火墙
sudo systemctl enable --now firewalld
# 2. 创建信任区域并添加白名单IP（完全开放）
sudo firewall-cmd --permanent --new-zone=trusted-ip
sudo firewall-cmd --permanent --zone=trusted-ip --add-source=124.235.206.60
sudo firewall-cmd --permanent --zone=trusted-ip --set-target=ACCEPT
# 3. 设置默认区域规则（其他IP只开放27251）
sudo firewall-cmd --set-default-zone=public
sudo firewall-cmd --permanent --zone=public --add-port=27251/tcp
# 4. 应用配置
sudo firewall-cmd --reload
# 5. 验证配置
sudo firewall-cmd --zone=trusted-ip --list-all # 应显示ACCEPT和正确IP
sudo firewall-cmd --zone=public --list-ports # 应显示27251/tcp

10
AI/Text2Sql/top10_chart.html
Unescape View File

@ -8,11 +8,11 @@
</head>
<body >
<div id="f0dc5418f1844bb99ff48a343422c11f" class="chart-container" style="width:1200px; height:600px; "></div>
<div id="30dbf1d9b9bf4c76a79725d385eb5254" class="chart-container" style="width:1200px; height:600px; "></div>
<script>
var chart_f0dc5418f1844bb99ff48a343422c11f = echarts.init(
document.getElementById('f0dc5418f1844bb99ff48a343422c11f'), 'white', {renderer: 'canvas'});
var option_f0dc5418f1844bb99ff48a343422c11f = {
var chart_30dbf1d9b9bf4c76a79725d385eb5254 = echarts.init(
document.getElementById('30dbf1d9b9bf4c76a79725d385eb5254'), 'white', {renderer: 'canvas'});
var option_30dbf1d9b9bf4c76a79725d385eb5254 = {
"animation": true,
"animationThreshold": 2000,
"animationDuration": 1000,
@ -267,7 +267,7 @@
}
]
};
chart_f0dc5418f1844bb99ff48a343422c11f.setOption(option_f0dc5418f1844bb99ff48a343422c11f);
chart_30dbf1d9b9bf4c76a79725d385eb5254.setOption(option_30dbf1d9b9bf4c76a79725d385eb5254);
</script>
</body>
</html>

39
src/main/java/Tools/Crawler/SyncDataEase.java
Unescape View File

@ -48,6 +48,13 @@ public class SyncDataEase {
ZhangJieFuGaiDetail();
//知识点覆盖率
ZhiShiDianFuGaiDetail();
//以下是县区的统计
//课程数量
Xq_KeChengShuLiang();
//参与学校数量
Xq_CanYuXueXiao();
}
/**
@ -410,4 +417,36 @@ public class SyncDataEase {
}
print("更新知识点覆盖率详情成功！");
}
//------------------------下面是县区的大屏统计分析部分--------------------
/**
* 课程建设统计
*/
public static void Xq_KeChengShuLiang() {
SqlPara sqlPara = Db.getSqlPara("DataEase.Xq_KeChengShuLiang");
List<Record> list = Db.find(sqlPara);
String sql="update `excel_数据_78904d5d99` set `数值`=? where `行政区划`=? and `分类`='直播课程'";
for (Record record : list) {
int cnt = record.getInt("cnt");
String gather_regionc = record.getStr("gather_regionc");
Db.use(MYSQL_DB).update(sql, cnt, gather_regionc);
}
print("更新县区课程建设统计成功！");
}
/**
* 参与学校统计
*/
public static void Xq_CanYuXueXiao() {
SqlPara sqlPara = Db.getSqlPara("DataEase.Xq_CanYuXueXiao");
List<Record> list = Db.find(sqlPara);
String sql="update `excel_数据_78904d5d99` set `数值`=? where `行政区划`=? and `分类`='参与学校数量'";
for (Record record : list) {
int cnt = record.getInt("cnt");
String gather_regionc = record.getStr("gather_regionc");
Db.use(MYSQL_DB).update(sql, cnt, gather_regionc);
}
print("更新县区参与学校统计成功！");
}
}

12
src/main/resources/Sql/DataEase.sql
Unescape View File

@ -305,4 +305,16 @@ ORDER BY
insert into `excel_学段应用分布_c747031012`(`学段`,`学科`,`一级节点数`,`一级节点覆盖率`)
values(?,?,?,?);
#end
--------------------------------- 下面是县区的数据统计--------------------------------
-- 课程建设数量
#sql("Xq_KeChengShuLiang")
SELECT T1.gather_regionc,COUNT(DISTINCT T1.lesson_id) as cnt FROM
t_crawler_lesson_school T1 INNER JOIN t_crawler_lesson T2 ON T1.lesson_id=T2.lesson_id
WHERE T2.node_type=1 AND T2.subject_id IN ('213','241','214','264','215','225','228','224','313','314','341','316','317','318','321','320','364','325','322','324','413','414','416','441','417','418','448','421','420','425','422','424') and T1.gather_regionc<>'' GROUP BY T1.gather_regionc;
#end
-- 参与学校数量
#sql("Xq_CanYuXueXiao")
SELECT T1.gather_regionc,COUNT(DISTINCT T1.organization_name) as cnt
FROM t_crawler_lesson_school T1 INNER JOIN t_crawler_lesson T2 ON T1.lesson_id=T2.lesson_id WHERE T1.match_type=2 AND T2.subject_id IN ('213','241','214','264','215','225','228','224','313','314','341','316','317','318','321','320','364','325','322','324','413','414','416','441','417','418','448','421','420','425','422','424') and T1.gather_regionc<>'' GROUP BY gather_regionc;
#end
#end
Write Preview
Loading…
Cancel
Save