在进行视频、压缩包等大文件上传时,我们有时会遇到上传速度过慢、上传到一半失败等问题。这时我们可以将一个大文件切成若干个小文件依次上传,这样不仅可以看到上传进度,当上传到一半失败时也可以继承上一次的上传进度,而避免了每次都要从头上传。
大体思路就是前端根据文件名生成MD5编码,再将大文件按100M一块切成若干小片(Minio允许最小分片为5M),给每一片赋上序号(currentPiece)后依次调用接口上传。后端拿到数据后在redis里创建一个list存放已上传的片,再全部上传完成后由Minio进行合并并将地址返回给前端。
文件上传接口
/*** 大文件切片上传*/@PostMapping("/uploadFileChunk")public ResultMoudel uploadFileChunk( @RequestBody MultipartFileWrapper fileChunk ){try{Date date = new Date();String md5=fileChunk.getMd5();BoundListOperations files= redisTemplate.boundListOps(md5);List<Integer> list=files.range(0,-1);//拿出所有值if (CollectionUtils.isEmpty(list)){//如果是新上传的 创建redis对象files.expire(12, TimeUnit.HOURS);files.leftPush(null);}//如果已上传过 跳过if (list.contains(fileChunk.getCurrentPiece())){return new ResultMoudel().success("");}BigFile bigFile= ossTemplate.putBigFile(fileChunk);//将结果存入redisfiles.leftPush(fileChunk.getCurrentPiece());if (!"continue".equals(bigFile.getTransFlag())) {//文件传输完毕, 删除当前keyredisTemplate.delete(md5);return new ResultMoudel<>().success(bigFile.getLink());}Date date1 = new Date();System.out.println(Thread.currentThread()+"文件上传花费了" + (date1.getTime() - date.getTime()));} catch (Exception exception) {//文件exception.printStackTrace();return new ResultMoudel().error(fileChunk.getCurrentPiece());}return new ResultMoudel().success("");}
对接Minio的文件上传实现
public BigFile putBigFile(MultipartFileWrapper bigPartFile) {try {InputStream inputStream = bigPartFile.getCurrentFile().getInputStream();Map headers = new HashMap();PutObjectArgs putObjectArgs = (PutObjectArgs)((Builder)((Builder)((Builder)PutObjectArgs.builder().object(bigPartFile.getMd5().concat("/") + bigPartFile.getCurrentPiece())).headers(headers)).bucket(this.ossProperties.getTempBucketName())).stream(inputStream, (long)inputStream.available(), -1L).contentType("application/octet-stream").build();this.minioClient.putObject(putObjectArgs);if (this.isFinish(bigPartFile.getMd5(), bigPartFile.getTotalPiece())) {BigFile bigFile = this.mergeFile(bigPartFile.getTotalPiece(), bigPartFile.getMd5(), bigPartFile.getFileName());this.deleteTempFile(bigPartFile.getTotalPiece(), bigPartFile.getMd5());return bigFile;} else {return BigFile.builder().transFlag("continue").build();}} catch (Throwable var6) {throw var6;}}private boolean isFinish(String md5, int totalPieces) throws Exception {Iterable<Result<Item>> results = this.minioClient.listObjects((ListObjectsArgs)((io.minio.ListObjectsArgs.Builder)ListObjectsArgs.builder().bucket(this.ossProperties.getTempBucketName())).prefix(md5.concat("/")).build());Set<String> objectNames = Sets.newHashSet();Iterator var5 = results.iterator();while(var5.hasNext()) {Result<Item> item = (Result)var5.next();objectNames.add(((Item)item.get()).objectName());}return objectNames.size() == totalPieces;}private void deleteTempFile(int totalPieces, String md5) throws ServerException, InsufficientDataException, ErrorResponseException, IOException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException {try {List<DeleteObject> delObjects = (List)Stream.iterate(0, (i) -> {return i + 1;}).limit((long)totalPieces).map((i) -> {return new DeleteObject(md5.concat("/").concat(Integer.toString(i)));}).collect(Collectors.toList());Iterable<Result<DeleteError>> results = this.minioClient.removeObjects((RemoveObjectsArgs)((io.minio.RemoveObjectsArgs.Builder)RemoveObjectsArgs.builder().bucket(this.ossProperties.getTempBucketName())).objects(delObjects).build());Iterator var5 = results.iterator();while(var5.hasNext()) {Result<DeleteError> result = (Result)var5.next();DeleteError error = (DeleteError)result.get();System.out.println("delete files " + error.objectName() + " " + error.message());}} catch (Throwable var8) {throw var8;}}
查看上传进度条
@PostMapping("selectChunks")public ResultMoudel selectChunks(@RequestParam String md5){BoundListOperations files=redisTemplate.boundListOps(md5);List range = files.range(0, -1);return new ResultMoudel().success(range.stream().filter(Objects::nonNull));}