From 1b51d00f09c29b49392bcd1d239d86a1b1a0eb98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E6=B5=B7?= <10402852@qq.com> Date: Fri, 10 Jan 2025 14:22:09 +0800 Subject: [PATCH] 'commit' --- .../Service/MysqlBackupService.java | 2 +- .../Service/MysqlRestoreService.java | 44 ++-- .../Utils/MultiThreadDownloader.java | 214 ++++++++++++++++++ .../src/main/resources/application.properties | 3 +- .../target/classes/application.properties | 3 +- 5 files changed, 242 insertions(+), 24 deletions(-) create mode 100644 YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/MultiThreadDownloader.java diff --git a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Service/MysqlBackupService.java b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Service/MysqlBackupService.java index b32bbeb6..df4f5236 100644 --- a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Service/MysqlBackupService.java +++ b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Service/MysqlBackupService.java @@ -101,6 +101,6 @@ public class MysqlBackupService { FileUtil.del(sourceFile); FileUtil.del(zipFile); //返回文件路径 - return "https://dsideal.obs.cn-north-1.myhuaweicloud.com/" + key; + return PropKit.get("obs_url_prefix") + key; } } \ No newline at end of file diff --git a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Service/MysqlRestoreService.java b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Service/MysqlRestoreService.java index 65a965a1..7d52fd62 100644 --- a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Service/MysqlRestoreService.java +++ b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Service/MysqlRestoreService.java @@ -1,10 +1,12 @@ package com.dsideal.YunXiaoTools.Service; import com.dsideal.YunXiaoTools.Utils.CommonUtil; +import com.dsideal.YunXiaoTools.Utils.MultiThreadDownloader; import com.jfinal.kit.PropKit; import com.obs.services.ObsClient; import java.io.*; +import java.util.UUID; import java.util.zip.*; public class MysqlRestoreService { @@ -33,11 +35,12 @@ public class MysqlRestoreService { */ public void restore(String obsKey) { String tempDir = System.getProperty("java.io.tmpdir"); - String zipFile = tempDir + obsKey; String sqlFile = null; + String zipFile = tempDir + obsKey.split("/")[2]; try { // 1. 从OBS下载ZIP文件 - downloadFromObs(obsKey, zipFile); + downloadFromObs(obsKey, tempDir); + System.out.println("文件下载完成: " + zipFile); // 2. 解压ZIP文件 @@ -64,24 +67,25 @@ public class MysqlRestoreService { /** * 从OBS下载文件 */ - private void downloadFromObs(String obsKey, String localFile) { - // 从配置文件中读取OBS配置 - String endPoint = PropKit.get("obs_endpoint"); - String ak = PropKit.get("obs_accessKeyId"); - String sk = PropKit.get("obs_accessKeySecret"); - String bucketName = PropKit.get("obs_bucket_name"); - // 创建ObsClient实例 - ObsClient obsClient = new ObsClient(ak, sk, endPoint); + private void downloadFromObs(String obsKey, String saveDir) { + String fileUrl = PropKit.get("obs_url_prefix") + obsKey; + //使用多线程下载办法下载文件并保存到localFile中 + int threadCount = 5; // 线程数 + MultiThreadDownloader downloader = new MultiThreadDownloader(fileUrl, saveDir, threadCount); try { - obsClient.getObject(bucketName, obsKey, localFile); - } catch (Exception e) { - throw new RuntimeException("从OBS下载文件失败: " + e.getMessage(), e); - } finally { - try { - obsClient.close(); - } catch (Exception e) { - e.printStackTrace(); - } + // 在另一个线程中启动下载 + Thread downloadThread = new Thread(() -> { + try { + downloader.download(); + } catch (Exception e) { + e.printStackTrace(); + } + }); + downloadThread.start(); + // 等待下载完成 + downloadThread.join(); + } catch (InterruptedException e) { + e.printStackTrace(); } } @@ -147,7 +151,6 @@ public class MysqlRestoreService { bos.write(buffer, 0, len); } } - // 读取错误输出 StringBuilder errorOutput = new StringBuilder(); try (BufferedReader reader = new BufferedReader( @@ -160,7 +163,6 @@ public class MysqlRestoreService { } } } - // 等待命令执行完成 int exitCode = process.waitFor(); if (exitCode != 0) { diff --git a/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/MultiThreadDownloader.java b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/MultiThreadDownloader.java new file mode 100644 index 00000000..5161e5fa --- /dev/null +++ b/YunXiaoTools/src/main/java/com/dsideal/YunXiaoTools/Utils/MultiThreadDownloader.java @@ -0,0 +1,214 @@ +package com.dsideal.YunXiaoTools.Utils; + +import java.io.*; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.concurrent.*; +import java.nio.file.*; + +public class MultiThreadDownloader { + private final String fileUrl; + private final String saveDir; + private final int threadCount; + private final long[] progress; + private volatile boolean isPaused = false; + private volatile boolean isCanceled = false; + + public MultiThreadDownloader(String fileUrl, String saveDir, int threadCount) { + this.fileUrl = fileUrl; + this.saveDir = saveDir; + this.threadCount = threadCount; + this.progress = new long[threadCount]; + } + + /** + * 开始下载 + */ + public void download() throws Exception { + // 获取文件大小 + HttpURLConnection conn = (HttpURLConnection) new URL(fileUrl).openConnection(); + conn.setRequestMethod("GET"); + long fileSize = conn.getContentLengthLong(); + String fileName = getFileName(conn); + conn.disconnect(); + + System.out.println("文件大小: " + formatFileSize(fileSize)); + + // 创建临时文件 + String tempDir = saveDir + File.separator + fileName + ".downloading"; + Files.createDirectories(Paths.get(tempDir)); + + // 计算每个线程下载的块大小 + long blockSize = fileSize / threadCount; + CountDownLatch latch = new CountDownLatch(threadCount); + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + + // 启动下载线程 + for (int i = 0; i < threadCount; i++) { + long startPos = i * blockSize; + long endPos = (i == threadCount - 1) ? fileSize - 1 : (i + 1) * blockSize - 1; + String tempFile = tempDir + File.separator + "part_" + i; + + executor.execute(new DownloadTask(i, startPos, endPos, tempFile, latch)); + } + + // 启动进度显示线程 + ScheduledExecutorService progressExecutor = Executors.newSingleThreadScheduledExecutor(); + progressExecutor.scheduleAtFixedRate(() -> { + if (!isPaused && !isCanceled) { + showProgress(fileSize); + } + }, 1, 1, TimeUnit.SECONDS); + + // 等待所有线程完成 + latch.await(); + executor.shutdown(); + progressExecutor.shutdown(); + + if (!isCanceled) { + // 合并文件 + mergeFiles(tempDir, saveDir + File.separator + fileName, threadCount); + // 删除临时文件 + deleteDirectory(tempDir); + System.out.println("\n下载完成: " + saveDir + File.separator + fileName); + } + } + + private class DownloadTask implements Runnable { + private final int threadId; + private final long startPos; + private final long endPos; + private final String tempFile; + private final CountDownLatch latch; + + public DownloadTask(int threadId, long startPos, long endPos, + String tempFile, CountDownLatch latch) { + this.threadId = threadId; + this.startPos = startPos; + this.endPos = endPos; + this.tempFile = tempFile; + this.latch = latch; + } + + @Override + public void run() { + try { + // 检查断点续传 + long downloadedSize = 0; + File temp = new File(tempFile); + if (temp.exists()) { + downloadedSize = temp.length(); + progress[threadId] = downloadedSize; + } + + if (downloadedSize < (endPos - startPos + 1)) { + HttpURLConnection conn = (HttpURLConnection) new URL(fileUrl).openConnection(); + conn.setRequestMethod("GET"); + conn.setRequestProperty("Range", + "bytes=" + (startPos + downloadedSize) + "-" + endPos); + + try (InputStream in = conn.getInputStream(); + RandomAccessFile out = new RandomAccessFile(tempFile, "rw")) { + + out.seek(downloadedSize); + byte[] buffer = new byte[8192]; + int len; + + while ((len = in.read(buffer)) != -1 && !isCanceled) { + if (!isPaused) { + out.write(buffer, 0, len); + progress[threadId] += len; + } else { + Thread.sleep(1000); // 暂停时休眠 + } + } + } + + conn.disconnect(); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + latch.countDown(); + } + } + } + + /** + * 显示下载进度 + */ + private void showProgress(long fileSize) { + long downloaded = 0; + for (long p : progress) { + downloaded += p; + } + + double progress = (double) downloaded / fileSize * 100; + String speed = formatFileSize(downloaded / 1024) + "/s"; + + System.out.printf("\r下载进度: %.2f%% [%s]", progress, speed); + } + + /** + * 合并文件 + */ + private void mergeFiles(String tempDir, String targetFile, int parts) throws IOException { + try (RandomAccessFile target = new RandomAccessFile(targetFile, "rw")) { + byte[] buffer = new byte[8192]; + for (int i = 0; i < parts; i++) { + String partFile = tempDir + File.separator + "part_" + i; + try (RandomAccessFile source = new RandomAccessFile(partFile, "r")) { + int len; + while ((len = source.read(buffer)) != -1) { + target.write(buffer, 0, len); + } + } + } + } + } + + // 工具方法 + private String getFileName(HttpURLConnection conn) { + String fileName = fileUrl.substring(fileUrl.lastIndexOf("/") + 1); + String disposition = conn.getHeaderField("Content-Disposition"); + if (disposition != null) { + int index = disposition.indexOf("filename="); + if (index > 0) { + fileName = disposition.substring(index + 9).replace("\"", ""); + } + } + return fileName; + } + + private String formatFileSize(long size) { + if (size < 1024) return size + " B"; + if (size < 1024 * 1024) return String.format("%.2f KB", size / 1024.0); + if (size < 1024 * 1024 * 1024) return String.format("%.2f MB", size / (1024.0 * 1024)); + return String.format("%.2f GB", size / (1024.0 * 1024 * 1024)); + } + + private void deleteDirectory(String dir) { + Path directory = Paths.get(dir); + try { + Files.walk(directory) + .sorted(java.util.Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } catch (IOException e) { + e.printStackTrace(); + } + } + + // 控制方法 + public void pause() { + isPaused = true; + } + + public void resume() { + isPaused = false; + } + + public void cancel() { + isCanceled = true; + } +} \ No newline at end of file diff --git a/YunXiaoTools/src/main/resources/application.properties b/YunXiaoTools/src/main/resources/application.properties index 571afe3c..82ab339f 100644 --- a/YunXiaoTools/src/main/resources/application.properties +++ b/YunXiaoTools/src/main/resources/application.properties @@ -25,4 +25,5 @@ obs_accessKeyId=WAFBGJACKDOQZDH1MKZ1 obs_accessKeySecret=dlWTUbqgCICaYJG3n0Rot4jXaen2HnfFtMVxiPEo obs_endpoint=obs.cn-north-1.myhuaweicloud.com obs_bucket_name=dsideal - +# 华为云下载的前缀 +obs_url_prefix=https://dsideal.obs.cn-north-1.myhuaweicloud.com/ diff --git a/YunXiaoTools/target/classes/application.properties b/YunXiaoTools/target/classes/application.properties index 571afe3c..82ab339f 100644 --- a/YunXiaoTools/target/classes/application.properties +++ b/YunXiaoTools/target/classes/application.properties @@ -25,4 +25,5 @@ obs_accessKeyId=WAFBGJACKDOQZDH1MKZ1 obs_accessKeySecret=dlWTUbqgCICaYJG3n0Rot4jXaen2HnfFtMVxiPEo obs_endpoint=obs.cn-north-1.myhuaweicloud.com obs_bucket_name=dsideal - +# 华为云下载的前缀 +obs_url_prefix=https://dsideal.obs.cn-north-1.myhuaweicloud.com/