'commit'

src/main/java/Tools/Crawler/YunXiao.java

@@ -39,8 +39,6 @@ public class YunXiao {
 
 
     public static void main(String[] args) {
-
-
         PropKit.use("application.properties");
         HikariCpPlugin hp = new HikariCpPlugin(PropKit.get("jdbcUrl"), PropKit.get("user"),
                 PropKit.get("password").trim(), PropKit.get("driverClassName"));
@@ -52,17 +50,17 @@ public class YunXiao {
         arp.setDialect(new PostgreSqlDialect());
         arp.start();
 
-
-        String truncateSql = "truncate table t_crawler_subject";
-        Db.update(truncateSql);
-        truncateSql = "truncate table t_crawler_scheme";
-        Db.update(truncateSql);
-        truncateSql = "truncate table t_crawler_book";
-        Db.update(truncateSql);
-        truncateSql = "truncate table t_crawler_structure";
-        Db.update(truncateSql);
-        truncateSql = "truncate table t_crawler_resource";
-        Db.update(truncateSql);
+        //清空爬虫表
+        String truncatSql = "truncate table t_crawler_subject";
+        Db.update(truncatSql);
+        truncatSql = "truncate table t_crawler_scheme";
+        Db.update(truncatSql);
+        truncatSql = "truncate table t_crawler_book";
+        Db.update(truncatSql);
+        truncatSql = "truncate table t_crawler_structure";
+        Db.update(truncatSql);
+        truncatSql = "truncate table t_crawler_resource";
+        Db.update(truncatSql);
 
         Map<String, String> map = new HashMap<>();
         map.put("1", "学前");
@@ -73,8 +71,6 @@ public class YunXiao {
         System.out.println("开始爬取数据！");
 
         map.forEach((key, value) -> {
-            String sql = "";
-
             String respSubject = Util.doGet("https://yx.ccsjy.cn/api/business/v1/subject/list/" + key);
             JSONObject jsonObj = JSONObject.parseObject(respSubject);
             JSONArray jsonArr = jsonObj.getJSONObject("data").getJSONArray("rows");
@@ -83,10 +79,9 @@ public class YunXiao {
                 String subjectCode = jsonSubject.getString("subjectCode");
                 String subjectName = jsonSubject.getString("subjectName");
 
-                sql = "insert into t_crawler_subject (subject_id,subject_name,stage_id) values (?,?,?)";
+                String sql = "insert into t_crawler_subject (subject_id,subject_name,stage_id) values (?,?,?)";
                 Db.update(sql, subjectCode, subjectName, key);
 
-
                 JSONObject argScheme = new JSONObject();
                 argScheme.put("subjectCode", subjectCode);
                 argScheme.put("systemId", 1);
@@ -117,16 +112,15 @@ public class YunXiao {
                         String businessBookId = jsonBook.getString("businessBookId");
                         String bookName = jsonBook.getString("bookName");
 
-
                         sql = "insert into t_crawler_book (book_id,book_name,scheme_id,subject_id,stage_id,id) values (?,?,?,?,?,?)";
                         Db.update(sql, businessBookId, bookName, businessEditionId, subjectCode, key, UUID.randomUUID().toString());
 
                         try {
                             Thread.sleep(1000);
-                            System.out.println("===休息1秒！===");
                         } catch (InterruptedException e) {
-                            e.printStackTrace();
+                            throw new RuntimeException(e);
                         }
+                        System.out.println("===休息1秒！===");
 
 
                         JSONObject argTree = new JSONObject();
@@ -140,23 +134,18 @@ public class YunXiao {
                         JSONArray jsonArrTree = jsonObjTree.getJSONObject("data").getJSONArray("tree");
 
                         tempTree = new JSONArray();
-
                         traverseTree(jsonArrTree);
 
-
                         for (int n = 0; n < tempTree.size(); n++) {
                             JSONObject jsonTree = tempTree.getJSONObject(n);
-
                             String nodeId = jsonTree.getString("nodeId");
                             String nodeName = jsonTree.getString("nodeName");
                             Boolean isLeaf = jsonTree.getBoolean("isLeaf");
                             String parentValue = jsonTree.getString("parentValue");
-
                             int is_leaf = 0;
                             if (isLeaf) {
                                 is_leaf = 1;
                             }
-
                             sql = "insert into t_crawler_structure (node_id,node_name,parent_id,is_leaf,book_id,scheme_id,subject_id,stage_id,id) values (?,?,?,?,?,?,?,?,?)";
                             Db.update(sql, nodeId, nodeName, parentValue, is_leaf, businessBookId, businessEditionId, subjectCode, key, UUID.randomUUID().toString());
 
@@ -191,20 +180,12 @@ public class YunXiao {
 
                                 sql = "insert into t_crawler_resource (lesson_id,lesson_name,node_id,teacher_id,teacher_name,teacher_school_id,teacher_school_name,id,book_id,scheme_id,subject_id,stage_id) values (?,?,?,?,?,?,?,?,?,?,?,?)";
                                 Db.update(sql, lessonId, lessonName, nodeId, teacherSchoolId, teacherSchoolName, teacherId, teacherName, UUID.randomUUID().toString(), businessBookId, businessEditionId, subjectCode, key);
-
                             }
-
-
                         }
-
-
                     }
-
-
                 }
             }
         });
-
         System.out.println("爬取数据完成！");
     }
 }