Minio大文件分片上傳

一、引入依賴

		<dependency><groupId>io.minio</groupId><artifactId>minio</artifactId><version>8.3.3</version></dependency>

二、自定義Minio客戶端

package com.gstanzer.video.controller;import com.google.common.collect.Multimap;
import io.minio.*;
import io.minio.errors.*;
import io.minio.messages.Part;import java.io.IOException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;public class CustomMinioClient extends MinioClient {/*** 繼承父類*/public CustomMinioClient(MinioClient client) {super(client);}/*** 初始化分片上傳即獲取uploadId*/public String initMultiPartUpload(String bucket, String region, String object, Multimap<String, String> headers, Multimap<String, String> extraQueryParams) throws IOException, InvalidKeyException, NoSuchAlgorithmException, InsufficientDataException, ServerException, InternalException, XmlParserException, InvalidResponseException, ErrorResponseException {CreateMultipartUploadResponse response = this.createMultipartUpload(bucket, region, object, headers, extraQueryParams);return response.result().uploadId();}/*** 上傳單個分片*/public UploadPartResponse uploadMultiPart(String bucket, String region, String object, Object data,long length,String uploadId,int partNumber,Multimap<String, String> headers,Multimap<String, String> extraQueryParams) throws IOException, InvalidKeyException, NoSuchAlgorithmException, InsufficientDataException, ServerException, InternalException, XmlParserException, InvalidResponseException, ErrorResponseException {return this.uploadPart(bucket, region, object, data, length, uploadId, partNumber, headers, extraQueryParams);}/*** 合并分片*/public ObjectWriteResponse mergeMultipartUpload(String bucketName, String region, String objectName, String uploadId, Part[] parts, Multimap<String, String> extraHeaders, Multimap<String, String> extraQueryParams) throws IOException, NoSuchAlgorithmException, InsufficientDataException, ServerException, InternalException, XmlParserException, InvalidResponseException, ErrorResponseException, ServerException, InvalidKeyException {return this.completeMultipartUpload(bucketName, region, objectName, uploadId, parts, extraHeaders, extraQueryParams);}public void cancelMultipartUpload(String bucketName, String region, String objectName, String uploadId, Multimap<String, String> extraHeaders, Multimap<String, String> extraQueryParams) throws ServerException, InsufficientDataException, ErrorResponseException, NoSuchAlgorithmException, IOException, InvalidKeyException, XmlParserException, InvalidResponseException, InternalException {this.abortMultipartUpload(bucketName, region, objectName, uploadId, extraHeaders, extraQueryParams);}/*** 查詢當前上傳后的分片信息*/public ListPartsResponse listMultipart(String bucketName, String region, String objectName, Integer maxParts, Integer partNumberMarker, String uploadId, Multimap<String, String> extraHeaders, Multimap<String, String> extraQueryParams) throws NoSuchAlgorithmException, InsufficientDataException, IOException, InvalidKeyException, ServerException, XmlParserException, ErrorResponseException, InternalException, InvalidResponseException {return this.listParts(bucketName, region, objectName, maxParts, partNumberMarker, uploadId, extraHeaders, extraQueryParams);}
}

三、分片上傳核心完整代碼

1.實體類

package com.gstanzer.video.form;import lombok.Data;/*** @author: tangbingbing* @date: 2025/6/4 08:27*/
@Data
public class UploadPartForm {// 分片文件路徑拼接字符串(比如:E:\\scdx\\test\\test.doc.part1,E:\\scdx\\test\\test.doc.part2)private String partFilePaths;// 分片文件urlprivate String partFileUrl;// 上傳idprivate String uploadId;// 文件類型(word類型:application/msword,安卓安裝包類型:application/vnd.android.package-archive)private String contentType;
}

2.控制層

package com.gstanzer.video.controller;import com.alibaba.fastjson.JSON;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Lists;
import com.gstanzer.video.form.UploadPartForm;
import com.gstanzer.video.swagger.ApiForBackEndInVideo;
import io.minio.*;
import io.minio.http.Method;
import io.minio.messages.Part;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.FileEntity;
import org.apache.http.impl.client.HttpClients;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;import javax.annotation.PostConstruct;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;@Slf4j
@RestController
@RequestMapping("/minio")
public class MinioController {private CustomMinioClient minioClient;@Value("${minio.endpoint}")private String endpoint;@Value("${minio.bucketName}")private String bucketName;@Value("${minio.access-key}")private String accessKey;@Value("${minio.secret-key}")private String secretKey;@PostConstructpublic void init() throws Exception {minioClient = new CustomMinioClient(MinioClient.builder().endpoint(endpoint).credentials(accessKey, secretKey).build());boolean found = minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());if (!found) {log.info("Not found minio bucket: {}, auto create it now", bucketName);minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build());found = minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());log.info("Auto create minio bucket: {} {}", bucketName, found);} else {log.info("Found minio bucket: {}", bucketName);}}/*** 第一步,簡單用一個10MB出頭的文件,按5MB分片大小進行分片*/public static void main(String[] args) throws Exception {long CHUNK_SIZE = 15 * 1024 * 1024;// 將文件分片存儲String filePath = "E:\\scdx\\app\\xfgj_v3.0.88.apk";File file = new File(filePath);long fileSize = file.length();int chunkCount = (int) Math.ceil((double) fileSize / CHUNK_SIZE);log.info("Created: " + chunkCount + " chunks");try (FileInputStream fis = new FileInputStream(file)) {byte[] buffer = new byte[(int) CHUNK_SIZE];for (int i = 0; i < chunkCount; i++) {String chunkFileName = filePath + ".part" + (i + 1);try (FileOutputStream fos = new FileOutputStream(chunkFileName)) {int bytesRead = fis.read(buffer);fos.write(buffer, 0, bytesRead);log.info("Created: " + chunkFileName + " (" + bytesRead + " bytes)");}}} catch (IOException e) {e.printStackTrace();}}/*** 第二步,申請一個大文件上傳*/@ApiForBackEndInVideo@PostMapping("/uploadLargeFile/apply")public ResponseEntity<Map> applyUploadPsiResult2Minio(@RequestParam("largeFileName") String largeFileName,@RequestParam("chunkCount") Integer chunkCount) {String uploadId = getUploadId(largeFileName, null);Map<String, Object> map = new HashMap<>();map.put("uploadId", uploadId);Map<String, String> reqParams = new HashMap<>();reqParams.put("uploadId", uploadId);List<String> uploadUrlList = new ArrayList<>();for (int i = 1; i <= chunkCount; i++) {reqParams.put("partNumber", String.valueOf(i));String uploadUrl = getPresignedObjectUrl(largeFileName, reqParams);uploadUrlList.add(uploadUrl);}map.put("chunkUploadUrls", uploadUrlList);return ResponseEntity.ok(map);}/*** 準備分片上傳時,在此先獲取上傳任務id*/private String getUploadId(String objectName, String contentType){log.info("tip message: 通過 <{}-{}-{}> 開始初始化<分片上傳>數據", objectName, contentType, bucketName);if (StringUtils.isBlank(contentType)) {contentType = "application/octet-stream";}HashMultimap<String, String> headers = HashMultimap.create();headers.put("Content-Type", contentType);try {return minioClient.initMultiPartUpload(bucketName, null, objectName, headers, null);} catch (Exception e) {throw new RuntimeException("獲取uploadId失敗", e);}}/*** 請求分片上傳的url*/private String getPresignedObjectUrl(String fileName, Map<String, String> reqParams) {try {String presignedObjectUrl = minioClient.getPresignedObjectUrl(GetPresignedObjectUrlArgs.builder().method(Method.PUT).bucket(bucketName).object(fileName).expiry(1, TimeUnit.DAYS).extraQueryParams(reqParams).build());return presignedObjectUrl;} catch (Exception e) {throw new RuntimeException("getPresignedObjectUrl failed", e);}}/*** 第三步,每個分片逐個通過簽名后的分片上傳url,進行上傳,未合并之前都可以重復覆蓋上傳* 在video服務的控制臺通過如下命令將分片下載到video服務中* wget http://100.86.2.1:80/xfaq/xfgj_v3.0.88.apk.part1*/@ApiForBackEndInVideo@PostMapping("/uploadLargeFile/uploadPart")public ResponseEntity<String> applyUploadPsiResult2Minio(@RequestBody UploadPartForm form) {String partFilePaths = form.getPartFilePaths();String partFileUrl = form.getPartFileUrl();try {// 第一步得到的分片文件List<String> chunkFilePaths = Lists.newArrayList(partFilePaths.split(","));// 第二步得到的上傳url信息Map<String, Object> uploadIdMap = new HashMap<>();uploadIdMap.put("uploadId", form.getUploadId());List<String> chunkUploadUrls = Lists.newArrayList(partFileUrl.split(","));uploadIdMap.put("chunkUploadIds", chunkUploadUrls);// 基于此,分片直接上傳到minio,不走服務端,省去一次網絡IO開銷HttpClient httpClient = HttpClients.createDefault();for (int i = 0; i < chunkUploadUrls.size(); i++) {// PUT直接上傳到minioString chunkUploadId = chunkUploadUrls.get(i);HttpPut httpPut = new HttpPut(chunkUploadId);httpPut.setHeader("Content-Type",form.getContentType());File chunkFile = new File(chunkFilePaths.get(i));FileEntity chunkFileEntity = new FileEntity(chunkFile);httpPut.setEntity(chunkFileEntity);HttpResponse chunkUploadResp = httpClient.execute(httpPut);log.info("[分片" + (i + 1) + "]上傳響應:" + JSON.toJSONString(chunkUploadResp));httpPut.releaseConnection();}return ResponseEntity.ok("上傳成功!");} catch (IOException e) {e.printStackTrace();throw new RuntimeException(e);}}public static void main3(String[] args) throws Exception {// 第一步得到的分片文件List<String> chunkFilePaths = Lists.newArrayList("E:\\scdx\\test\\test.doc.part1","E:\\scdx\\test\\test.doc.part2");// 第二步得到的上傳url信息Map<String, Object> uploadIdMap = new HashMap<>();uploadIdMap.put("uploadId", "YTJlOWZhMmEtM2I3My00MmIyLWE0YjgtMDFkYjQzMzIyNmVhLjFmYzQ2M2ViLTI2YmYtNDZjMi04M2ZlLWJjMThjOTk0MWU4OHgxNzQ4OTM3NDE3MjI4ODA1MTAw");List<String> chunkUploadUrls = Lists.newArrayList("http://127.0.0.1:9000/xfaq/test.doc?uploadId=YTJlOWZhMmEtM2I3My00MmIyLWE0YjgtMDFkYjQzMzIyNmVhLjFmYzQ2M2ViLTI2YmYtNDZjMi04M2ZlLWJjMThjOTk0MWU4OHgxNzQ4OTM3NDE3MjI4ODA1MTAw&partNumber=1&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=minioadmin%2F20250603%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20250603T075657Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=649af2001e096f422479e139efc4aab51301061c026f40c0685cd20edbb28211","http://127.0.0.1:9000/xfaq/test.doc?uploadId=YTJlOWZhMmEtM2I3My00MmIyLWE0YjgtMDFkYjQzMzIyNmVhLjFmYzQ2M2ViLTI2YmYtNDZjMi04M2ZlLWJjMThjOTk0MWU4OHgxNzQ4OTM3NDE3MjI4ODA1MTAw&partNumber=2&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=minioadmin%2F20250603%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20250603T075657Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=eedd5431a6e4ba0ebda1c2b227f7543eed77d245b3d86c6d5d505d27f61cd823");uploadIdMap.put("chunkUploadIds", chunkUploadUrls);// 基于此,分片直接上傳到minio,不走服務端,省去一次網絡IO開銷HttpClient httpClient = HttpClients.createDefault();for (int i = 0; i < chunkUploadUrls.size(); i++) {// PUT直接上傳到minioString chunkUploadId = chunkUploadUrls.get(i);HttpPut httpPut = new HttpPut(chunkUploadId);httpPut.setHeader("Content-Type","application/msword");File chunkFile = new File(chunkFilePaths.get(i));FileEntity chunkFileEntity = new FileEntity(chunkFile);httpPut.setEntity(chunkFileEntity);HttpResponse chunkUploadResp = httpClient.execute(httpPut);log.info("[分片" + (i + 1) + "]上傳響應:" + JSON.toJSONString(chunkUploadResp));httpPut.releaseConnection();}}/*** 第四步,合并分片*/@ApiForBackEndInVideo@PostMapping("/uploadLargeFile/merge")public ResponseEntity<Boolean> applyUploadPsiResult2Minio(@RequestParam("largeFileName") String largeFileName,@RequestParam("uploadId") String uploadId) {boolean mergeResult = mergeMultipartUpload(largeFileName, uploadId);return ResponseEntity.ok(mergeResult);}/*** 分片上傳完后合并* @param objectName 文件名稱* @param uploadId* @return*/public boolean mergeMultipartUpload(String objectName, String uploadId) {try {log.info("ready to merge <" + objectName + " - " + uploadId + " - " + bucketName + ">");// 查詢上傳后的分片數據ListPartsResponse partResult = minioClient.listMultipart(bucketName, null, objectName, 1000, 0, uploadId, null, null);int chunkCount = partResult.result().partList().size();Part[] parts = new Part[chunkCount];int partNumber = 1;for (Part part : partResult.result().partList()) {parts[partNumber - 1] = new Part(partNumber, part.etag());partNumber++;}// 合并分片ObjectWriteResponse objectWriteResponse = minioClient.mergeMultipartUpload(bucketName, null, objectName, uploadId, parts, null, null);log.info("mergeMultipartUpload resp: {}", objectWriteResponse);} catch (Exception e) {log.error("合并失敗:", e);return false;}return true;}
}

本文來自互聯網用戶投稿,該文觀點僅代表作者本人,不代表本站立場。本站僅提供信息存儲空間服務,不擁有所有權,不承擔相關法律責任。
如若轉載,請注明出處:http://www.pswp.cn/bicheng/87635.shtml
繁體地址,請注明出處:http://hk.pswp.cn/bicheng/87635.shtml
英文地址,請注明出處:http://en.pswp.cn/bicheng/87635.shtml

如若內容造成侵權/違法違規/事實不符,請聯系多彩編程網進行投訴反饋email:809451989@qq.com,一經查實,立即刪除!

相關文章

Jenkins 插件深度應用:讓你的CI/CD流水線如虎添翼 [特殊字符]

Jenkins 插件深度應用&#xff1a;讓你的CI/CD流水線如虎添翼 &#x1f680; 嘿&#xff0c;各位開發小伙伴&#xff01;今天咱們來聊聊Jenkins的插件生態系統。如果說Jenkins是一臺強大的引擎&#xff0c;那插件就是讓這臺引擎發揮最大威力的各種零部件。準備好了嗎&#xff1…

密碼學(斯坦福)

密碼學筆記 \huge{密碼學筆記} 密碼學筆記 斯坦福大學密碼學的課程筆記 課程網址&#xff1a;https://www.bilibili.com/video/BV1Rf421o79E/?spm_id_from333.337.search-card.all.click&vd_source5cc05a038b81f6faca188e7cf00484f6 概述 密碼學的使用背景 安全信息保護…

代碼隨想錄算法訓練營第四十六天|動態規劃part13

647. 回文子串 題目鏈接&#xff1a;647. 回文子串 - 力扣&#xff08;LeetCode&#xff09; 文章講解&#xff1a;代碼隨想錄 思路&#xff1a; 以dp【i】表示以s【i】結尾的回文子串的個數&#xff0c;發現遞推公式推導不出來此路不通 以dp【i】【j】表示s【i】到s【j】的回…

基于四種機器學習算法的球隊數據分析預測系統的設計與實現

文章目錄 有需要本項目的代碼或文檔以及全部資源&#xff0c;或者部署調試可以私信博主項目介紹項目展示隨機森林模型XGBoost模型邏輯回歸模型catboost模型每文一語 有需要本項目的代碼或文檔以及全部資源&#xff0c;或者部署調試可以私信博主 項目介紹 本項目旨在設計與實現…

http、SSL、TLS、https、證書

一、基礎概念 1.HTTP HTTP (超文本傳輸協議) 是一種用于客戶端和服務器之間傳輸超媒體文檔的應用層協議&#xff0c;是萬維網的基礎。 簡而言之&#xff1a;一種獲取和發送信息的標準協議 2.SSL 安全套接字層&#xff08;SSL&#xff09;是一種通信協議或一組規則&#xf…

在 C++ 中,判斷 `std::string` 是否為空字符串

在 C 中&#xff0c;判斷 std::string 是否為空字符串有多種方法&#xff0c;以下是最常用的幾種方式及其區別&#xff1a; 1. 使用 empty() 方法&#xff08;推薦&#xff09; #include <string>std::string s; if (s.empty()) {// s 是空字符串 }特性&#xff1a; 時間…

【Harmony】鴻蒙企業應用詳解

【HarmonyOS】鴻蒙企業應用詳解 一、前言 1、應用類型定義速覽&#xff1a; HarmonyOS目前針對應用分為三種類型&#xff1a;普通應用&#xff0c;游戲應用&#xff0c;企業應用。 而企業應用又分為&#xff0c;企業普通應用和設備管理應用MDM&#xff08;Mobile Device Man…

Linux云計算基礎篇(8)

VIM 高級特性插入模式按 i 進入插入模式。按 o 在當前行下方插入空行并進入插入模式。按 O 在當前行上方插入空行并進入插入模式。命令模式:set nu 顯示行號。:set nonu 取消顯示行號。:100 光標跳轉到第 100 行。G 光標跳轉到文件最后一行。gg 光標跳轉到文件第一行。30G 跳轉…

Linux進程單例模式運行

Linux進程單例模式運行 #include <iostream> #include <stdlib.h> #include <unistd.h> #include <string.h> #include <stdio.h> #include <sys/types.h> #include <sys/stat.h> #include <fcntl.h>int write_pid(const cha…

【Web 后端】部署服務到服務器

文章目錄 前言一、如何啟動服務二、掛載和開機啟動服務1. 配置systemctl 服務2. 創建server用戶3. 啟動服務 總結 前言 如果你的后端服務寫好了如果部署到你的服務器呢&#xff0c;本次通過fastapi寫的服務實例&#xff0c;示范如何部署到服務器&#xff0c;并做服務管理。 一…

國產MCU學習Day5——CW32F030C8T6:窗口看門狗功能全解析

每日更新教程&#xff0c;評論區答疑解惑&#xff0c;小白也能變大神&#xff01;" 目錄 一.窗口看門狗&#xff08;WWDG&#xff09;簡介 二.窗口看門狗寄存器列表 三.窗口看門狗復位案例 一.窗口看門狗&#xff08;WWDG&#xff09;簡介 CW32F030C8T6 內部集成窗口看…

2025年文件加密軟件分享:守護數字世界的核心防線

在數字化時代&#xff0c;數據已成為個人與企業的寶貴資產&#xff0c;文件加密軟件通過復雜的算法&#xff0c;確保信息在存儲、傳輸與共享過程中的保密性、完整性與可用性。一、文件加密軟件的核心原理文件加密軟件算法以其高效性與安全性廣泛應用&#xff0c;通過對文件數據…

node.js下載教程

1.項目環境文檔 語雀 2.nvm安裝教程與nvm常見命令,超詳細!-阿里云開發者社區 C:\Windows\System32>nvm -v 1.2.2 C:\Windows\System32>nvm list available Error retrieving "http://npm.taobao.org/mirrors/node/index.json": HTTP Status 404 C:\Window…

(AI如何解決問題)在一個項目,跳轉到外部html頁面,頁面布局

問題描述目前&#xff0c;ERP后臺有很多跳轉外部鏈接的地方&#xff0c;會直接打開一個tab顯示。因為有些頁面是適配手機屏幕顯示&#xff0c;放在瀏覽器會超級大。不美觀&#xff0c;因此提出優化。修改前&#xff1a;修改后&#xff1a;思考過程1、先看下代碼&#xff1a;log…

網絡通信協議與虛擬網絡技術相關整理(上)

#作者&#xff1a;程宏斌 文章目錄 tcp協議udp協議arp協議icmp協議dhcp協議BGP協議OSPF協議BGP vs OSPF 對比表VLAN&#xff08;Virtual LAN&#xff09;VXLAN&#xff08;Virtual Extensible LAN&#xff09;IPIP&#xff08;IP-in-IP&#xff09;vxlan/vlan/ipip網橋/veth網…

物聯網軟件層面的核心技術體系

物聯網軟件層面的核心技術體系 物聯網(IoT)軟件技術棧是一個多層次的復雜體系&#xff0c;涵蓋從設備端到云平臺的完整解決方案。以下是物聯網軟件層面的關鍵技術分類及詳細說明&#xff1a; 一、設備端軟件技術 1. 嵌入式操作系統 實時操作系統(RTOS)&#xff1a; FreeRTO…

GreatSQL通過偽裝從庫回放Binlog文件

GreatSQL通過偽裝從庫回放Binlog文件 一、適用場景說明 1、主庫誤操作恢復 利用 Binlog 在其他實例解析、回放&#xff0c;根據gtid只回放到指定位點。 2、網絡隔離環境同步 備份恢復后可以拉去主庫Binlog文件至新實例同步增量數據。 3、備份恢復遇到Binlog文件過大處理 恢復實…

MVC 架構設計模式

在現代軟件開發中&#xff0c;架構設計決定了一個項目的可維護性與可擴展性。MVC&#xff08;Model-View-Controller&#xff09;作為經典的分層設計模式&#xff0c;廣泛應用于 Web 系統、前端應用乃至移動端開發中。本文不僅介紹 MVC 的核心思想和機制&#xff0c;還將結合具…

(18)python+playwright自動化測試鼠標拖拽-上

1.簡介 本文主要介紹兩個在測試過程中可能會用到的功能&#xff1a;在selenium中介紹了Actions類中的拖拽操作和Actions類中的劃取字段操作。例如&#xff1a;需要在一堆log字符中隨機劃取一段文字&#xff0c;然后右鍵選擇摘取功能。playwright同樣可以實現元素的拖拽和釋放的…

Android 網絡全棧攻略(四)—— TCPIP 協議族與 HTTPS 協議

Android 網絡全棧攻略系列文章&#xff1a; Android 網絡全棧攻略&#xff08;一&#xff09;—— HTTP 協議基礎 Android 網絡全棧攻略&#xff08;二&#xff09;—— 編碼、加密、哈希、序列化與字符集 Android 網絡全棧攻略&#xff08;三&#xff09;—— 登錄與授權 Andr…