234 lines
8.7 KiB
Python
234 lines
8.7 KiB
Python
|
# 请先安装必要的库: pip install tqdm requests
|
|||
|
|
import os
|
|||
|
|
import json
|
|||
|
|
import threading
|
|||
|
|
import requests
|
|||
|
|
from tqdm import tqdm
|
|||
|
|
|
|||
|
|
|
|||
|
|
def fetch_lesson_data(page_num=1, page_size=10, search_keyword=""):
|
|||
|
|
"""
|
|||
|
|
发送POST请求获取云课堂课程数据
|
|||
|
|
|
|||
|
|
参数:
|
|||
|
|
page_num: 页码，默认为1
|
|||
|
|
page_size: 每页数量，默认为10
|
|||
|
|
search_keyword: 搜索关键词，默认为空
|
|||
|
|
|
|||
|
|
返回:
|
|||
|
|
解析后的JSON数据，如果请求失败则返回None
|
|||
|
|
"""
|
|||
|
|
# API URL
|
|||
|
|
url = "https://yx.ccsjy.cn/api/cloud-school/v1/cloudLesson/getOnDemandLessonPage"
|
|||
|
|
|
|||
|
|
# 请求参数
|
|||
|
|
payload = {
|
|||
|
|
"pageNum": page_num,
|
|||
|
|
"pageSize": page_size,
|
|||
|
|
"businessBookId": "3029115d4afa424f8160adb04bd10e6a",
|
|||
|
|
"businessEditionId": "19F93E4A7C9B4B589EB001FBFEE6230A",
|
|||
|
|
"excellentFlag": "",
|
|||
|
|
"nodeId": None,
|
|||
|
|
"nodeType": 1,
|
|||
|
|
"stageCode": "3",
|
|||
|
|
"subjectCode": "314",
|
|||
|
|
"sortType": 2,
|
|||
|
|
"source": "",
|
|||
|
|
"searchKeyword": search_keyword
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
# 发送POST请求
|
|||
|
|
response = requests.post(url, json=payload)
|
|||
|
|
|
|||
|
|
# 检查响应状态
|
|||
|
|
if response.status_code == 200:
|
|||
|
|
# 解析JSON响应
|
|||
|
|
data = response.json()
|
|||
|
|
return data
|
|||
|
|
else:
|
|||
|
|
print(f"请求失败，状态码: {response.status_code}")
|
|||
|
|
return None
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f"请求异常: {str(e)}")
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
# 创建多线程下载类
|
|||
|
|
class MultiThreadDownloader:
|
|||
|
|
def __init__(self, url, file_path, num_threads=4):
|
|||
|
|
self.url = url
|
|||
|
|
self.file_path = file_path
|
|||
|
|
self.num_threads = num_threads
|
|||
|
|
self.total_size = 0
|
|||
|
|
self.progress = None
|
|||
|
|
|
|||
|
|
def get_file_size(self):
|
|||
|
|
response = requests.head(self.url, allow_redirects=True)
|
|||
|
|
if 'content-length' in response.headers:
|
|||
|
|
self.total_size = int(response.headers['content-length'])
|
|||
|
|
return True
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
def download_chunk(self, start, end, thread_id):
|
|||
|
|
headers = {'Range': f'bytes={start}-{end}'}
|
|||
|
|
response = requests.get(self.url, headers=headers, stream=True)
|
|||
|
|
|
|||
|
|
chunk_size = 1024
|
|||
|
|
with open(f'{self.file_path}.part{thread_id}', 'wb') as f:
|
|||
|
|
for data in response.iter_content(chunk_size=chunk_size):
|
|||
|
|
f.write(data)
|
|||
|
|
if self.progress:
|
|||
|
|
self.progress.update(len(data))
|
|||
|
|
|
|||
|
|
def merge_chunks(self):
|
|||
|
|
with open(self.file_path, 'wb') as f:
|
|||
|
|
for i in range(self.num_threads):
|
|||
|
|
part_file = f'{self.file_path}.part{i}'
|
|||
|
|
if os.path.exists(part_file):
|
|||
|
|
with open(part_file, 'rb') as pf:
|
|||
|
|
f.write(pf.read())
|
|||
|
|
os.remove(part_file)
|
|||
|
|
|
|||
|
|
def start(self):
|
|||
|
|
# 确保目录存在
|
|||
|
|
os.makedirs(os.path.dirname(self.file_path), exist_ok=True)
|
|||
|
|
|
|||
|
|
if not self.get_file_size():
|
|||
|
|
print(f"无法获取文件大小，使用单线程下载: {self.url}")
|
|||
|
|
# 回退到单线程下载
|
|||
|
|
try:
|
|||
|
|
response = requests.get(self.url, stream=True)
|
|||
|
|
with open(self.file_path, 'wb') as f:
|
|||
|
|
total_size = int(response.headers.get('content-length', 0))
|
|||
|
|
with tqdm(total=total_size, unit='B', unit_scale=True, desc=self.file_path.split('\\')[-1]) as pbar:
|
|||
|
|
for data in response.iter_content(chunk_size=1024):
|
|||
|
|
f.write(data)
|
|||
|
|
pbar.update(len(data))
|
|||
|
|
return True
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f"单线程下载失败: {str(e)}")
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
# 多线程下载
|
|||
|
|
chunk_size = self.total_size // self.num_threads
|
|||
|
|
self.progress = tqdm(total=self.total_size, unit='B', unit_scale=True, desc=self.file_path.split('\\')[-1])
|
|||
|
|
|
|||
|
|
threads = []
|
|||
|
|
for i in range(self.num_threads):
|
|||
|
|
start = i * chunk_size
|
|||
|
|
end = self.total_size - 1 if i == self.num_threads - 1 else (i + 1) * chunk_size - 1
|
|||
|
|
thread = threading.Thread(target=self.download_chunk, args=(start, end, i))
|
|||
|
|
threads.append(thread)
|
|||
|
|
thread.start()
|
|||
|
|
|
|||
|
|
# 等待所有线程完成
|
|||
|
|
for thread in threads:
|
|||
|
|
thread.join()
|
|||
|
|
|
|||
|
|
self.progress.close()
|
|||
|
|
self.merge_chunks()
|
|||
|
|
return True
|
|||
|
|
|
|||
|
|
|
|||
|
|
# 测试函数
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
# 创建下载目录
|
|||
|
|
down_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "Down")
|
|||
|
|
os.makedirs(down_dir, exist_ok=True)
|
|||
|
|
|
|||
|
|
# 获取总记录数
|
|||
|
|
first_page = fetch_lesson_data(page_num=1, page_size=1)
|
|||
|
|
if first_page:
|
|||
|
|
total = first_page['data']['total']
|
|||
|
|
page_size = 10 # 每页获取10条
|
|||
|
|
total_pages = (total + page_size - 1) // page_size
|
|||
|
|
print(f"获取成功，共{total}条记录")
|
|||
|
|
|
|||
|
|
# 遍历所有页面
|
|||
|
|
all_lessons = []
|
|||
|
|
for page_num in range(1, total_pages + 1):
|
|||
|
|
print(f"正在获取第{page_num}/{total_pages}页数据...")
|
|||
|
|
result = fetch_lesson_data(page_num=page_num, page_size=page_size)
|
|||
|
|
if result and result['data']['rows']:
|
|||
|
|
all_lessons.extend(result['data']['rows'])
|
|||
|
|
else:
|
|||
|
|
print(f"第{page_num}页数据获取失败")
|
|||
|
|
|
|||
|
|
print(f"成功获取{len(all_lessons)}条课程数据")
|
|||
|
|
|
|||
|
|
# 准备保存下载信息
|
|||
|
|
download_info = []
|
|||
|
|
success_count = 0
|
|||
|
|
fail_count = 0
|
|||
|
|
|
|||
|
|
# 遍历所有课程
|
|||
|
|
for idx, lesson in enumerate(all_lessons, 1):
|
|||
|
|
lesson_id = lesson.get('lessonId')
|
|||
|
|
lesson_name = lesson.get('lessonName')
|
|||
|
|
print(f"\n处理课程 {idx}/{len(all_lessons)}: {lesson_name}")
|
|||
|
|
|
|||
|
|
# 提取视频URL
|
|||
|
|
lesson_resources = lesson.get('lessonResources', [])
|
|||
|
|
video_url = None
|
|||
|
|
if lesson_resources:
|
|||
|
|
# 假设第一个资源是视频
|
|||
|
|
video_url ="https://ccschool.edusoa.com/"+ lesson_resources[0].get('fileUrl', None)
|
|||
|
|
safe_name=lesson_resources[0].get('fileUrl', None).split('/')[-1]
|
|||
|
|
|
|||
|
|
if video_url:
|
|||
|
|
# 生成安全的文件名
|
|||
|
|
file_path = os.path.join(down_dir, f"{safe_name}")
|
|||
|
|
|
|||
|
|
# 检查文件是否已存在
|
|||
|
|
if os.path.exists(file_path):
|
|||
|
|
print(f"文件已存在，跳过下载: {file_path}")
|
|||
|
|
download_info.append({
|
|||
|
|
"course_name": lesson_name,
|
|||
|
|
"course_id": lesson_id,
|
|||
|
|
"file_path": file_path,
|
|||
|
|
"status": "已存在"
|
|||
|
|
})
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
# 开始下载
|
|||
|
|
print(f"开始下载视频: {video_url}")
|
|||
|
|
downloader = MultiThreadDownloader(video_url, file_path)
|
|||
|
|
if downloader.start():
|
|||
|
|
print(f"视频下载成功: {file_path}")
|
|||
|
|
download_info.append({
|
|||
|
|
"course_name": lesson_name,
|
|||
|
|
"course_id": lesson_id,
|
|||
|
|
"file_path": file_path,
|
|||
|
|
"status": "成功"
|
|||
|
|
})
|
|||
|
|
success_count += 1
|
|||
|
|
else:
|
|||
|
|
print(f"视频下载失败: {video_url}")
|
|||
|
|
download_info.append({
|
|||
|
|
"course_name": lesson_name,
|
|||
|
|
"course_id": lesson_id,
|
|||
|
|
"file_path": file_path,
|
|||
|
|
"status": "失败"
|
|||
|
|
})
|
|||
|
|
fail_count += 1
|
|||
|
|
else:
|
|||
|
|
print("未找到视频URL")
|
|||
|
|
download_info.append({
|
|||
|
|
"course_name": lesson_name,
|
|||
|
|
"course_id": lesson_id,
|
|||
|
|
"file_path": "",
|
|||
|
|
"status": "无视频URL"
|
|||
|
|
})
|
|||
|
|
fail_count += 1
|
|||
|
|
|
|||
|
|
# 每5个课程保存一次下载信息
|
|||
|
|
if idx % 5 == 0 or idx == len(all_lessons):
|
|||
|
|
info_file = os.path.join(down_dir, "download_info.json")
|
|||
|
|
with open(info_file, 'w', encoding='utf-8') as f:
|
|||
|
|
json.dump(download_info, f, ensure_ascii=False, indent=2)
|
|||
|
|
print(f"已保存下载信息到: {info_file}")
|
|||
|
|
|
|||
|
|
print(f"\n下载完成！成功: {success_count}, 失败: {fail_count}")
|
|||
|
|
else:
|
|||
|
|
print("获取课程数据失败")
|