huanghai
/
QingLong
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

'commit'

Browse Source
main
黄海 6 months ago
parent acfb2dd0aa
commit 88c7263c4f
3 changed files with 108 additions and 56 deletions
Show Stats Download Patch File Download Diff File

37
src/main/java/Tools/Crawler/CrawlerInit.java
Unescape View File

@ -0,0 +1,37 @@
package Tools.Crawler;
import com.dsideal.QingLong.Start;
import com.jfinal.kit.PropKit;
import com.jfinal.plugin.activerecord.ActiveRecordPlugin;
import com.jfinal.plugin.activerecord.CaseInsensitiveContainerFactory;
import com.jfinal.plugin.activerecord.dialect.PostgreSqlDialect;
import com.jfinal.plugin.hikaricp.HikariCpPlugin;
import java.io.File;
public class CrawlerInit {
public static void Init(){
// 初始化数据库连接
PropKit.use("application.properties");
HikariCpPlugin hp = new HikariCpPlugin(PropKit.get("jdbcUrl"), PropKit.get("user"),
PropKit.get("password").trim(), PropKit.get("driverClassName"));
hp.start();
ActiveRecordPlugin arp = new ActiveRecordPlugin(hp);
arp.setContainerFactory(new CaseInsensitiveContainerFactory(true));
arp.setDialect(new PostgreSqlDialect());
String basePath = Start.class.getClassLoader().getResource(".").getPath();
File sqlDir = new File(basePath + "/Sql");
File[] sqlFiles = sqlDir.listFiles();
File[] var10 = sqlFiles != null ? sqlFiles : new File[0];
int var11 = var10.length;
for (int i = 0; i < var11; ++i) {
File sqlFile = var10[i];
if (sqlFile.getName().indexOf(".sql") > 0) {
arp.addSqlTemplate("/Sql/" + sqlFile.getName());
}
}
arp.start();
}
}

23
src/main/java/Tools/Crawler/YunXiaoBook.java
Unescape View File

@ -567,28 +567,7 @@ public class YunXiaoBook {
}
public static void main(String[] args) {
// 初始化数据库连接
PropKit.use("application.properties");
HikariCpPlugin hp = new HikariCpPlugin(PropKit.get("jdbcUrl"), PropKit.get("user"),
PropKit.get("password").trim(), PropKit.get("driverClassName"));
hp.start();
ActiveRecordPlugin arp = new ActiveRecordPlugin(hp);
arp.setContainerFactory(new CaseInsensitiveContainerFactory(true));
arp.setDialect(new PostgreSqlDialect());
String basePath = Start.class.getClassLoader().getResource(".").getPath();
File sqlDir = new File(basePath + "/Sql");
File[] sqlFiles = sqlDir.listFiles();
File[] var10 = sqlFiles != null ? sqlFiles : new File[0];
int var11 = var10.length;
for (int i = 0; i < var11; ++i) {
File sqlFile = var10[i];
if (sqlFile.getName().indexOf(".sql") > 0) {
arp.addSqlTemplate("/Sql/" + sqlFile.getName());
}
}
arp.start();
CrawlerInit.Init();
//爬虫开始
Start();
}

104
src/main/java/Tools/Crawler/YunXiaoKnowledge.java
Unescape View File

@ -2,23 +2,17 @@ package Tools.Crawler;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.dsideal.QingLong.Start;
import com.jfinal.kit.PropKit;
import com.jfinal.plugin.activerecord.ActiveRecordPlugin;
import com.jfinal.plugin.activerecord.CaseInsensitiveContainerFactory;
import com.jfinal.plugin.activerecord.Db;
import com.jfinal.plugin.activerecord.Record;
import com.jfinal.plugin.activerecord.dialect.PostgreSqlDialect;
import com.jfinal.plugin.hikaricp.HikariCpPlugin;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
public class YunXiaoKnowledge {
public static YunXiaoBook yx = new YunXiaoBook();
public static List<Record> lessonList = new ArrayList<>();
public static int batchSize = 100;
// 从数据库加载学段和学科映射
private static Map<String, String> loadStageMap() {
@ -83,29 +77,56 @@ public class YunXiaoKnowledge {
}
}
public static void main(String[] args) {
// 初始化数据库连接
PropKit.use("application.properties");
HikariCpPlugin hp = new HikariCpPlugin(PropKit.get("jdbcUrl"), PropKit.get("user"),
PropKit.get("password").trim(), PropKit.get("driverClassName"));
hp.start();
ActiveRecordPlugin arp = new ActiveRecordPlugin(hp);
arp.setContainerFactory(new CaseInsensitiveContainerFactory(true));
arp.setDialect(new PostgreSqlDialect());
String basePath = Start.class.getClassLoader().getResource(".").getPath();
File sqlDir = new File(basePath + "/Sql");
File[] sqlFiles = sqlDir.listFiles();
File[] var10 = sqlFiles != null ? sqlFiles : new File[0];
int var11 = var10.length;
for (int i = 0; i < var11; ++i) {
File sqlFile = var10[i];
if (sqlFile.getName().indexOf(".sql") > 0) {
arp.addSqlTemplate("/Sql/" + sqlFile.getName());
}
public static void collectLesson(String nodeId, String stageCode, String subjectCode) {
String url = "https://yx.ccsjy.cn/api/cloud-school/v1/cloudLesson/getOnDemandLessonPage";
JSONObject argBook = new JSONObject();
argBook.put("nodeId", nodeId);
argBook.put("nodeType", 2);
argBook.put("pageNum", 1);
argBook.put("pageSize", 10);
argBook.put("sortType", 2);
argBook.put("stageCode", stageCode);
argBook.put("subjectCode", subjectCode);
String respBook = YunXiaoBook.doRequestWithRetry(url, argBook.toString(), false, 3);
JSONArray jsonArrSource = JSONObject.parseObject(respBook).getJSONObject("data").getJSONArray("rows");
for (int m = 0; m < jsonArrSource.size(); m++) {
JSONObject jsonSource = jsonArrSource.getJSONObject(m);
// 解析日期时间字符串
String publishTimeStr = jsonSource.getString("publishTime");
LocalDateTime dateTime = LocalDateTime.parse(publishTimeStr,
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
LocalDate date = dateTime.toLocalDate();
Record record = new Record()
.set("lesson_id", jsonSource.getString("lessonId"))
.set("lesson_name", jsonSource.getString("lessonName"))
.set("node_id", nodeId)
.set("teacher_id", jsonSource.getString("teacherId"))
.set("teacher_school_id", jsonSource.getString("teacherSchoolId"))
.set("teacher_school_name", jsonSource.getString("teacherSchoolName"))
.set("teacher_name", jsonSource.getString("teacherName"))
.set("book_id", null)
.set("scheme_id", null)
.set("subject_id", subjectCode)
.set("grade_code", jsonSource.getString("gradeCode"))
.set("publish_time", date)
.set("stage_id", stageCode)
//增加学习次数
.set("preview_count", jsonSource.getIntValue("previewCount"))
.set("learning_person_count", jsonSource.getIntValue("learningPersonCount"))
.set("learning_person_times", jsonSource.getIntValue("learningPersonTimes"))
.set("id", UUID.randomUUID().toString())
.set("node_type", 2);//知识点类型
lessonList.add(record);
}
arp.start();
}
public static void main(String[] args) {
CrawlerInit.Init();
String sql = "truncate table t_crawler_structure_knowledge";
Db.update(sql);
@ -158,6 +179,21 @@ public class YunXiaoKnowledge {
subjectMap.get(subjectCode));
}
}
//遍历所有的知识点
sql = "select * from t_crawler_structure_knowledge";
List<Record> listKnowledge = Db.find(sql);
for (Record record : listKnowledge) {
String stage_code = record.getStr("stage_code");
String subject_code = record.getStr("subject_code");
String nodeId = record.getStr("value");
collectLesson(nodeId, stage_code, subject_code);
}
//先删除掉知识点关联过来的课程数据
sql = "delete from t_crawler_lesson where node_type=2";
Db.update(sql);
//批量保存
Db.batchSave("t_crawler_lesson", lessonList, batchSize);
System.out.println("数据处理完成!");
}
}

Write Preview
Loading…
Cancel
Save