From 69e8e833e6c6299d75cb0312118cb5b0f4782dac Mon Sep 17 00:00:00 2001
From: HuangHai <10402852@qq.com>
Date: Mon, 14 Jul 2025 15:41:30 +0800
Subject: [PATCH] 'commit'
---
.../Test/Test/Logs/article_bfc50bb7d7.html | 162 ++++++++++++++++++
dsLightRag/Test/TestCrawl.py | 148 ++++++++++++++--
2 files changed, 295 insertions(+), 15 deletions(-)
create mode 100644 dsLightRag/Test/Test/Logs/article_bfc50bb7d7.html
diff --git a/dsLightRag/Test/Test/Logs/article_bfc50bb7d7.html b/dsLightRag/Test/Test/Logs/article_bfc50bb7d7.html
new file mode 100644
index 00000000..cd460649
--- /dev/null
+++ b/dsLightRag/Test/Test/Logs/article_bfc50bb7d7.html
@@ -0,0 +1,162 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+:
+,
+。
+
+
+
+视频
+小程序
+
+
+赞
+,轻点两下取消赞
+在看
+,轻点两下取消在看
+
+
+
+
+
+
+
+
+
+
+
diff --git a/dsLightRag/Test/TestCrawl.py b/dsLightRag/Test/TestCrawl.py
index 7a9a4e6d..63d2eb14 100644
--- a/dsLightRag/Test/TestCrawl.py
+++ b/dsLightRag/Test/TestCrawl.py
@@ -7,20 +7,10 @@ import datetime
import logging
import random
import re
+import os
import requests
-# 1、安装Firefox软件【最新】
-# https://www.firefox.com.cn/download/#product-desktop-release
-
-# 2、下载geckodriver驱动【最新】
-# https://splinter-docs-zh-cn.readthedocs.io/zh/latest/drivers/firefox.html
-# https://github.com/mozilla/geckodriver/releases
-
-# 3、Python爬虫实战系列:微信公众号文章爬取的5种技术方案总结及代码示例!
-# 方案5:微信公众号后台引用链接方式爬取
-# https://blog.csdn.net/Python_trys/article/details/146506009
-
"""
# 查看selenium版本
pip show selenium
@@ -43,17 +33,25 @@ import json
if __name__ == '__main__':
# 定义一个空的字典,存放cookies内容
cookies = {}
- # 设置headers
+ # 设置headers - 使用微信内置浏览器的User-Agent
header = {
"HOST": "mp.weixin.qq.com",
- "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0"
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36 QBCore/4.0.1301.400 QQBrowser/9.0.2524.400 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2875.116 Safari/537.36 NetType/WIFI MicroMessenger/7.0.20.1781(0x6700143B) WindowsWechat(0x63010200)",
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+ "Accept-Encoding": "gzip, deflate, br",
+ "Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.5;q=0.4",
+ "Connection": "keep-alive"
}
# 用webdriver启动谷歌浏览器
logging.info("启动浏览器,打开微信公众号登录界面")
options = Options()
- options.add_argument('-headless') # 无头参数
+ # options.add_argument('-headless') # 无头参数,调试时可以注释掉
+
+ # 设置微信内置浏览器的User-Agent
+ options.add_argument('--user-agent=Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36 QBCore/4.0.1301.400 QQBrowser/9.0.2524.400 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2875.116 Safari/537.36 NetType/WIFI MicroMessenger/7.0.20.1781(0x6700143B) WindowsWechat(0x63010200)')
+
service = ChromeService(executable_path=r"C:\Windows\System32\chromedriver.exe")
- driver = webdriver.Chrome(service=service)
+ driver = webdriver.Chrome(service=service, options=options)
# 打开微信公众号登录页面
driver.get('https://mp.weixin.qq.com/')
# 等待5秒钟
@@ -143,10 +141,130 @@ if __name__ == '__main__':
'publish_time': datetime.datetime.fromtimestamp(int(item.get("update_time"))).strftime('%Y-%m-%d %H:%M:%S'),
'collection_time': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
}
+ print("new_article:", new_article)
logging.info("new_article:", new_article)
article_urls.append(item.get('link'))
time.sleep(1)
+ # 确保Logs目录存在
+ logs_dir = "./Test/Logs"
+ if not os.path.exists(logs_dir):
+ os.makedirs(logs_dir)
+
for article_url in article_urls:
print("正在爬取文章:" + article_url)
+ try:
+ # 使用requests直接获取文章内容,模拟微信环境
+ wechat_headers = {
+ "User-Agent": "Mozilla/5.0 (Linux; Android 10; MI 8 Build/QKQ1.190828.002; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/83.0.4103.101 Mobile Safari/537.36 XWEB/1768 MMWEBSDK/20210302 MMWEBID/6253 MicroMessenger/8.0.2.1860(0x28000234) Process/toolsmp WeChat/arm64 Weixin NetType/WIFI Language/zh_CN ABI/arm64",
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+ "Accept-Encoding": "gzip, deflate",
+ "Accept-Language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7",
+ "X-Requested-With": "com.tencent.mm",
+ "Referer": "https://mp.weixin.qq.com/"
+ }
+
+ # 使用selenium打开文章链接,设置请求头
+ driver.execute_cdp_cmd('Network.setExtraHTTPHeaders', {'headers': wechat_headers})
+ driver.get(article_url)
+ # 增加等待时间,确保页面完全加载
+ time.sleep(5)
+
+ # 检查是否需要登录
+ if "请在微信客户端中打开链接" in driver.page_source or "请在微信中打开此链接" in driver.page_source:
+ print(f"文章需要在微信中打开,尝试使用requests直接获取:{article_url}")
+ # 尝试使用requests直接获取
+ response = requests.get(article_url, headers=wechat_headers, cookies=cookies)
+ if "请在微信客户端中打开链接" in response.text or "请在微信中打开此链接" in response.text:
+ print(f"使用requests仍然无法获取,跳过此文章:{article_url}")
+ continue
+ else:
+ # 保存获取到的HTML内容
+ filename = f"article_{article_url.split('sn=')[1][:10] if 'sn=' in article_url else 'unknown'}"
+ save_path = f"{logs_dir}/{filename}.html"
+ with open(save_path, "w", encoding="utf-8") as f:
+ f.write(response.text)
+ print(f"已保存文章HTML内容:{save_path}")
+ continue
+
+ # 使用更可靠的选择器查找标题和内容
+ try:
+ # 尝试多种可能的标题选择器
+ title_selectors = [
+ '//h1[@class="rich_media_title"]',
+ '//h1[@id="activity-name"]',
+ '//h2[@class="rich_media_title"]',
+ '//div[@class="rich_media_content"]//h1',
+ '//div[@id="js_article"]//h1'
+ ]
+
+ title = None
+ for selector in title_selectors:
+ try:
+ title_element = driver.find_element('xpath', selector)
+ title = title_element.text.strip()
+ if title:
+ break
+ except:
+ continue
+
+ if not title:
+ # 如果所有选择器都失败,尝试从页面标题获取
+ title = driver.title.replace(" - 微信公众号", "").strip()
+
+ # 尝试多种可能的内容选择器
+ content_selectors = [
+ '//div[@class="rich_media_content"]',
+ '//div[@id="js_content"]',
+ '//div[@class="rich_media_wrp"]'
+ ]
+
+ content = None
+ for selector in content_selectors:
+ try:
+ content_element = driver.find_element('xpath', selector)
+ content = content_element.text.strip()
+ if content:
+ break
+ except:
+ continue
+
+ if not content:
+ # 如果无法获取内容,至少保存页面源码
+ content = "无法提取正文内容,保存页面源码:\n" + driver.page_source
+
+ # 创建文件名(使用标题,但去除不合法的文件名字符)
+ if not title:
+ title = "未知标题_" + article_url.split("sn=")[1][:10] if "sn=" in article_url else "未知标题"
+
+ filename = re.sub(r'[\\/:*?"<>|]', '_', title)
+
+ # 保存文章内容到文件
+ save_path = f"{logs_dir}/{filename}.txt"
+ with open(save_path, "w", encoding="utf-8") as f:
+ f.write(f"标题:{title}\n\n")
+ f.write(f"链接:{article_url}\n\n")
+ f.write(f"内容:\n{content}")
+
+ print(f"文章《{title}》保存成功:{save_path}")
+
+ except Exception as e:
+ print(f"提取文章内容失败:{str(e)}")
+ # 保存页面源码以便分析
+ error_filename = "error_" + article_url.split("sn=")[1][:10] if "sn=" in article_url else "error_page"
+ error_path = f"{logs_dir}/{error_filename}.html"
+ with open(error_path, "w", encoding="utf-8") as f:
+ f.write(driver.page_source)
+ print(f"已保存页面源码到:{error_path}")
+
+ # 避免频繁请求被封
+ time.sleep(random.uniform(3, 7))
+
+ except Exception as e:
+ print(f"爬取文章失败:{article_url},错误信息:{str(e)}")
+ continue
+
+ # 关闭浏览器
+ driver.quit()
+ print("所有文章爬取完成!")