|
|
# pip install esdk-obs-python --trusted-host pypi.org
|
|
|
# pip install tqdm
|
|
|
# python.exe -m pip install --upgrade pip
|
|
|
# 引入模块
|
|
|
import os
|
|
|
from datetime import datetime
|
|
|
from obs import ObsClient
|
|
|
import time
|
|
|
import requests
|
|
|
from subprocess import call
|
|
|
import urllib.parse
|
|
|
|
|
|
# 替换为你的华为云AK/SK
|
|
|
ak = "WAFBGJACKDOQZDH1MKZ1"
|
|
|
sk = "dlWTUbqgCICaYJG3n0Rot4jXaen2HnfFtMVxiPEo"
|
|
|
server = "https://obs.cn-north-1.myhuaweicloud.com"
|
|
|
bucketName = "dsideal"
|
|
|
prefix = "HuangHai/HuiZhiKeJi/"
|
|
|
localPath = "D:/课程备份/"
|
|
|
UrlPrefix = "https://dsideal.obs.cn-north-1.myhuaweicloud.com/HuangHai/HuiZhiKeJi/"
|
|
|
|
|
|
|
|
|
def getTime():
|
|
|
# 获取当前时间
|
|
|
now = datetime.now()
|
|
|
# 格式化到秒
|
|
|
current_time_to_seconds = now.strftime("%H:%M:%S")
|
|
|
return current_time_to_seconds
|
|
|
|
|
|
|
|
|
def idmDownloader(task_url, folder_path, file_name):
|
|
|
"""
|
|
|
IDM下载器
|
|
|
:param task_url: 下载任务地址
|
|
|
:param folder_path: 存放文件夹
|
|
|
:param file_name: 文件名
|
|
|
:return:
|
|
|
"""
|
|
|
# IDM安装目录
|
|
|
idm_engine = "D:\\IDM\\IDMan.exe"
|
|
|
# 将任务添加至队列
|
|
|
call([idm_engine, '/d', task_url, '/p', folder_path, '/f', file_name, '/a', '/q'])
|
|
|
# 开始任务队列
|
|
|
call([idm_engine, '/s'])
|
|
|
|
|
|
|
|
|
# 执行一次
|
|
|
def solve():
|
|
|
# 创建obsClient实例
|
|
|
obsClient = ObsClient(access_key_id=ak, secret_access_key=sk, server=server)
|
|
|
# 使用访问OBS
|
|
|
|
|
|
max_keys = 1000
|
|
|
mark = None
|
|
|
resp = None
|
|
|
# 创建一个文件列表
|
|
|
# 检查文件是否存在
|
|
|
idm_url_txt = 'idm_url.txt'
|
|
|
if os.path.exists(idm_url_txt):
|
|
|
# 文件存在，先删除它
|
|
|
os.remove(idm_url_txt)
|
|
|
print(f"已删除文件: {idm_url_txt}")
|
|
|
|
|
|
flag = False
|
|
|
while True:
|
|
|
fileList = obsClient.listObjects(bucketName=bucketName, prefix=prefix, max_keys=max_keys, marker=mark)
|
|
|
for file in fileList.body.contents:
|
|
|
fullName = file.key.replace(prefix, "")
|
|
|
array = fullName.split("/")
|
|
|
fileName = array[-1]
|
|
|
dirStr = ''
|
|
|
for i in range(len(array) - 1):
|
|
|
dirStr += array[i] + "/"
|
|
|
|
|
|
if dirStr == '' or fileName == '':
|
|
|
continue
|
|
|
localDirStr = localPath + dirStr
|
|
|
if not os.path.isdir(localDirStr):
|
|
|
os.makedirs(localDirStr, exist_ok=True)
|
|
|
|
|
|
downloadPath = localDirStr + fileName
|
|
|
# 下载对象
|
|
|
resp = obsClient.getObject(bucketName=bucketName, objectKey=file.key)
|
|
|
|
|
|
# 如果已存在，但是，与云存储中文件大小不一样大，那么删除掉
|
|
|
if os.path.isfile(downloadPath):
|
|
|
localFileSize = os.path.getsize(downloadPath)
|
|
|
if file.size != localFileSize:
|
|
|
os.remove(downloadPath)
|
|
|
print(getTime() + " " + "发现文件大小不一致，删除本地文件：" + downloadPath)
|
|
|
|
|
|
if not os.path.isfile(downloadPath):
|
|
|
flag = True
|
|
|
print(
|
|
|
getTime() + " 下载文件：" + fileName + ",文件大小：" + str(
|
|
|
"{:.2f}".format(file.size / 1024 / 1024)) + "MB")
|
|
|
|
|
|
# 发送HTTP GET请求
|
|
|
url_to_encode = dirStr + fileName
|
|
|
encoded_url = urllib.parse.quote(url_to_encode, safe='')
|
|
|
url = "https://dsideal.obs.cn-north-1.myhuaweicloud.com/HuangHai/HuiZhiKeJi/" + encoded_url
|
|
|
|
|
|
# 创建一个下载文件列表
|
|
|
with open(idm_url_txt, 'a', encoding='utf-8') as file:
|
|
|
file.write(url + "\n")
|
|
|
else:
|
|
|
print(
|
|
|
getTime() + " 相同的文件：" + fileName + ",文件大小：" + str(
|
|
|
"{:.2f}".format(file.size / 1024 / 1024)) + "MB，跳过...")
|
|
|
|
|
|
# 是否已经完成了所有任务
|
|
|
if resp != None and resp.body.is_truncated is True:
|
|
|
mark = resp.body.next_marker
|
|
|
else:
|
|
|
break
|
|
|
# 关闭obsClient
|
|
|
obsClient.close()
|
|
|
|
|
|
if flag:
|
|
|
result = []
|
|
|
with open(idm_url_txt, encoding='utf-8') as f:
|
|
|
for line in f:
|
|
|
result.append(line.strip('\n').split(',')[0])
|
|
|
|
|
|
for i in range(len(result)):
|
|
|
url = result[i] # 取出每次要下载的链接
|
|
|
url = url.replace(prefix, '')
|
|
|
# 获取文件所在的目录
|
|
|
directory = os.path.dirname(localPath + url)
|
|
|
# 获取文件名（包括后缀）
|
|
|
file_name = os.path.basename(localPath + url)
|
|
|
idmDownloader(url, directory, file_name) # 添加进IDM中下载
|
|
|
|
|
|
print(getTime() + " 同步完成")
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
while True:
|
|
|
solve()
|
|
|
time.sleep(3600 * 2) # 每2小时执行一次同步操作
|
