# pip install pymysql
# pip install requests beautifulsoup4

# 查看结果
# select * from t_dm_area where province_id='FD61813E-70A1-42AB-9A8E-141ED4D47B98' order by level_id;

import time

import pymysql
import requests
from bs4 import BeautifulSoup
import re

if __name__ == '__main__':
    # 遍历 mysql数据库，然后开启爬虫
    # 建立数据库连接
    conn = pymysql.connect(
        host='10.10.14.203',  # 主机名（或IP地址）
        port=3306,  # 端口号，默认为3306
        user='root',  # 用户名
        password='Password123@mysql',  # 密码
        charset='utf8mb4'  # 设置字符编码
    )

    # 创建游标对象
    cursor = conn.cursor()
    # 选择数据库
    conn.select_db("ds_db")
    # 执行查询操作
    cursor.execute(
        "SELECT id,full_name FROM t_dm_area where province_id='FD61813E-70A1-42AB-9A8E-141ED4D47B98' order by level_id")

    # 获取查询结果，返回元组
    result: tuple = cursor.fetchall()

    for e in result:
        id = e[0]
        area_name = e[1]
        url = "https://baike.baidu.com/item/" + area_name + "?fromModule=lemma_search-box"

        print(url)
        # 发送HTTP GET请求
        response = requests.get(url)
        # 检查请求是否成功
        if response.status_code == 200:
            # 使用BeautifulSoup解析HTML内容
            soup = BeautifulSoup(response.text, 'html.parser')
            # 假设我们要抓取的是<h1>标签中的文字
            # 你可以根据需要修改选择器来抓取不同的内容
            specific_divs = soup.select('div.para_YYuCh.summary_nfAdr.MARK_MODULE')
            # 遍历找到的所有特定div标签，并打印它们的文本内容
            for div in specific_divs:
                text = div.get_text(strip=True)  # 使用get_text()方法获取文本，并去除
                # 使用正则表达式移除所有形如[数字]和[数字-数字]的字符串
                cleaned_text = re.sub(r'\[\d+(?:-\d+)?\]', '', text)
                sql = "update t_dm_area set memo=%s where id=%s"
                cursor.execute(sql, (cleaned_text, id))
                conn.commit()
                print("更新"+area_name+"数据成功")
                break
        else:
            print('Failed to retrieve the webpage')

        time.sleep(2)
    # 关闭游标和连接
    cursor.close()
    conn.close()
    print("结束")