java對大文件分片上傳

這里記錄一下，Java對大文件的切分，和后端接口分片上傳的實現邏輯
正常，前后端分離的項目其實是前端去切分文件，后端接口接收到切分后的分片文件去合并，這里都用java來記錄一下。
特別說明：我這里用的是zip包的上傳，里面是音頻文件，如果你的文件是單個文件，切分和合并文件邏輯都是一樣的，只是不用后續的解壓。
因為是測試代碼，所以部分代碼規范不嚴謹

1.文件的切分

public static void chunkFile(){//每片的大小，這里是10Mint TENMB = 10485760;//要切分的大文件和切分后的文件放到目錄String PATH = "D:\\test\\fenpian\\";try {File file = new File(PATH, "55.zip");RandomAccessFile accessFile = new RandomAccessFile(file, "r");// 文件的大小long size = FileUtil.size(file);int chunkSize = (int) Math.ceil((double) size / TENMB);for (int i = 0; i < chunkSize; i++) {// 文件操作的指針位置long filePointer = accessFile.getFilePointer();byte[] bytes;if (i == chunkSize - 1) {int len = (int) (size - filePointer);bytes = new byte[len];accessFile.read(bytes, 0, bytes.length);} else {bytes = new byte[TENMB];accessFile.read(bytes, 0, bytes.length);}FileUtil.writeBytes(bytes, new File(PATH, String.valueOf(i) + ".zip"));}} catch (FileNotFoundException e) {e.printStackTrace();} catch (IOException e) {e.printStackTrace();}}

55.zip切分成了10個小的zip包

2.Spring boot分片上傳接口

controller層

	@Resourceprivate FileUploadService fileUploadService;@RequestMapping(value = "/upload")public String upload(MultipartFileParam fileParam) throws IOException {try{return fileUploadService.fileUpload(fileParam);}catch (Exception e){e.printStackTrace();return "error";}}

service層

public interface FileUploadService {String fileUpload(MultipartFileParam fileParam) throws IOException;}

service實現層

@Slf4j
@Service
public class FileUploadServiceImpl implements FileUploadService {//合并后的文件的父目錄private String FILE_UPLOAD_DIR = "D:\\test\\fenpian";//分片文件大小private Integer CHUNK_SIZE = 10485760;/*** 分片上傳* @param fileParam* @return* @throws IOException*/private String chunkUpload(MultipartFileParam fileParam) throws IOException {// 是否為最后一片boolean lastFlag = false;int currentChunk = fileParam.getChunk();int totalChunk = fileParam.getTotalChunk();long totalSize = fileParam.getTotalSize();String fileName = fileParam.getName();String fileMd5 = fileParam.getMd5();MultipartFile multipartFile = fileParam.getFile();String parentDir = FILE_UPLOAD_DIR + File.separator + fileMd5 + File.separator;String tempFileName = fileName + "_tmp";// 寫入到臨時文件File tmpFile = tmpFile(parentDir, tempFileName, multipartFile, currentChunk, totalSize, fileMd5);// 檢測是否為最后一個分片（這里吧每個分片數據放到了一張表里，后續可以改為用redis記錄）FileChunkRecordExample example = new FileChunkRecordExample();example.createCriteria().andMd5EqualTo(fileMd5);long count = fileChunkRecordMapper.countByExample(example);if (count == totalChunk) {lastFlag = true;}if (lastFlag) {// 檢查md5是否一致log.info("是否最后一個分片:{}","是");if (!checkFileMd5(tmpFile, fileMd5)) {cleanUp(tmpFile, fileMd5);throw new RuntimeException("文件md5檢測不符合要求, 請檢查!");}System.out.println("開始重命名....");File newFile = renameFile(tmpFile, fileName);//解析文件數據 -解壓縮unzipSystem.out.println("開始解壓縮....");File zipFile = ZipUtil.unzip(newFile);//得到壓縮包內所有文件System.out.println("遍歷zipFile.....");File[] files = zipFile.listFiles();System.out.println("打印fileName.....");//解析文件，處理業務數據for (File file : files) {System.out.println(file.getName());}log.info("所有文件上傳完成, 時間是:{}, 文件名稱是:{}", DateUtil.now(), fileName);//所有數據都處理完成后，刪除文件和數據庫記錄cleanUp(new File(parentDir + fileName),fileMd5);}else{log.info("是否最后一個分片:{}","否");}return "success";}private File tmpFile(String parentDir, String tempFileName, MultipartFile file,int currentChunk, long totalSize, String fileMd5) throws IOException {log.info("開始上傳文件, 時間是:{}, 文件名稱是:{}", DateUtil.now(), tempFileName);long position = (currentChunk - 1) * CHUNK_SIZE;File tmpDir = new File(parentDir);File tmpFile = new File(parentDir, tempFileName);if (!tmpDir.exists()) {tmpDir.mkdirs();}RandomAccessFile tempRaf = new RandomAccessFile(tmpFile, "rw");if (tempRaf.length() == 0) {tempRaf.setLength(totalSize);}// 寫入該分片數據FileChannel fc = tempRaf.getChannel();MappedByteBuffer map = fc.map(FileChannel.MapMode.READ_WRITE, position, file.getSize());map.put(file.getBytes());clean(map);fc.close();tempRaf.close();// 記錄已經完成的分片FileChunkRecord fileChunkRecord = new FileChunkRecord();fileChunkRecord.setMd5(fileMd5);fileChunkRecord.setUploadStatus(1);fileChunkRecord.setChunk(currentChunk);fileChunkRecordMapper.insert(fileChunkRecord);log.info("分片文件上傳完成, 時間是:{}, 文件名稱是:{}", DateUtil.now(), tempFileName);return tmpFile;}private void cleanUp(File file, String md5) {if (file.exists()) {file.delete();}// 刪除上傳記錄FileChunkRecordExample example = new FileChunkRecordExample();example.createCriteria().andMd5EqualTo(md5);fileChunkRecordMapper.deleteByExample(example);}/*** 最后一片接受完后執行* @param toBeRenamed* @param toFileNewName* @return*/private File renameFile(File toBeRenamed, String toFileNewName) {// 檢查要重命名的文件是否存在，是否是文件if (!toBeRenamed.exists() || toBeRenamed.isDirectory()) {log.info("File does not exist: " + toBeRenamed.getName());throw new RuntimeException("File does not exist");}String parentPath = toBeRenamed.getParent();File newFile = new File(parentPath + File.separatorChar + toFileNewName);// 如果存在, 先刪除if (newFile.exists()) {newFile.delete();}toBeRenamed.renameTo(newFile);return newFile;}private static void clean(MappedByteBuffer map) {try {Method getCleanerMethod = map.getClass().getMethod("cleaner");Cleaner.create(map, null);getCleanerMethod.setAccessible(true);Cleaner cleaner = (Cleaner) getCleanerMethod.invoke(map);cleaner.clean();} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {e.printStackTrace();}}/*** 文件md5值檢查，最后一片文件合并后執行，* @param file    所有分片文件合并后的文件（正常情況下md5應該和前端傳過來的大文件的md5一致）* @param fileMd5 大文件的md5值* @return* @throws IOException*/private boolean checkFileMd5(File file, String fileMd5) throws IOException {FileInputStream fis = new FileInputStream(file);String checkMd5 = DigestUtils.md5DigestAsHex(fis).toUpperCase();fis.close();if (checkMd5.equals(fileMd5.toUpperCase())) {return true;}return false;}/*** 不分片* @param fileParam* @return*/private String singleUpload(MultipartFileParam fileParam) {MultipartFile file = fileParam.getFile();File baseFile = new File(FILE_UPLOAD_DIR);if (!baseFile.exists()) {baseFile.mkdirs();}try {file.transferTo(new File(baseFile, fileParam.getName()));Date now = new Date();FileRecord fileRecord = new FileRecord();String filePath = FILE_UPLOAD_DIR + File.separator + fileParam.getName();long size = FileUtil.size(new File(filePath));String sizeStr = size / (1024 * 1024) + "Mb";fileRecord.setFileName(fileParam.getName()).setFilePath(filePath).setUploadStatus(1).setFileMd5(fileParam.getMd5()).setCreateTime(now).setUpdateTime(now).setFileSize(sizeStr);//fileRecordMapper.insert(fileRecord);} catch (IOException e) {log.error("單獨上傳文件錯誤, 問題是:{}, 時間是:{}", e.getMessage(), DateUtil.now());}return "success";}}

后端入參實體類

package com.server.controller.bigFileUpload;import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import lombok.experimental.Accessors;
import org.springframework.web.multipart.MultipartFile;/*** @description:* @date: created in 2021/10/6* @modified:*/
@Getter
@Setter
@ToString
@Accessors(chain = true)
public class MultipartFileParam {/*** 是否分片*/private boolean chunkFlag;/*** 當前為第幾塊分片*/private int chunk;/*** 總分片數量*/private int totalChunk;/*** 文件總大小, 單位是byte*/private long totalSize;/*** 文件名*/private String name;/*** 文件*/private MultipartFile file;/*** md5值*/private String md5;}

合并后的文件放到了文件mdf的文件夾內
在這里插入圖片描述