Spring Cloud + Vue前后端分离-第9章大文件断点续与极速秒传

源代码在GitHub - 629y/course: Spring Cloud + Vue前后端分离-在线课程

Spring Cloud + Vue前后端分离-第9章大文件断点续与极速秒传

作为一个视频网站，一个文件小则几十M，大则上G，上传一个大文件受网络影响很大，文件越大，上传失败率越高

我们要完善文件上传功能，支持断点续传，当文件上传到一半，网络断了，下次再上传时，只上传剩下的部分，这就是断点续传。

极速秒传，听起来很高深，原理其实很简单，就是开始上传前，先检查一下文件是否上传过了，如果已上传过，直接弹出提示，极速秒传成功。

9-1 分片传输的试探

一个文件有10M，每个分片定义为1M，那这个文件就会被分为10片进行传输。

分片上传

1.大文件断点续传与极速秒传：增加big-file组件，分片传输测试成功

把file.vue复制一份重命名big-file.vue

big-file.vue

section.vue

big-file.vue

分片合并

1.大文件断点续传与极速秒传：分片合并测试成功

UploadController.java

9-2 分片上传功能开发1

文件分片管理的模型设计

1.大文件断点续传与极速秒传：文件表增加分片相关字段

all.sql

generatorConfig.xml

分片数据传输

1.大文件断点续传与极速秒传：分片相关的数据传输

经验分享：对于一些不涉及安全性的数据，可以交由前端来计算，这样可以减轻服务端的压力

FileDto.java

big-file.vue

UploadController.java

文件标识的设计

1.大文件断点续传与极速秒传：使用文件生成md5签名，作为文件标识

数据库设计：上传第一个分片时，插入一条记录，上传之后的分片，都是更新记录，只需更新shard_index字段，当然也包括updated_at

数据库设计：shard_index和shard_total的值相等时，表示所有的分片都已上传完成。

数据库设计：不管上传第几个分片，传输的key值是一样的，key跟文件相关，跟分片无关。

big-file.vue

MD5信息摘要算法，常用于密码加密传输与存储，会生成128位二制数，每4个二进制可以转为1个16进制数字，结果是32个16进制数字

UploadController.java

md5.js

var KEY = "!@#QWERT";
/** Configurable variables. You may need to tweak these to be compatible with* the server-side, but the defaults work in most cases.*/
var hexcase = 0;  /* hex output format. 0 - lowercase; 1 - uppercase        */
var b64pad  = ""; /* base-64 pad character. "=" for strict RFC compliance   */
var chrsz   = 8;  /* bits per input character. 8 - ASCII; 16 - Unicode      *//** These are the functions you'll usually want to call* They take string arguments and return either hex or base-64 encoded strings*/
function hex_md5(s){ return binl2hex(core_md5(str2binl(s), s.length * chrsz));}
function b64_md5(s){ return binl2b64(core_md5(str2binl(s), s.length * chrsz));}
function str_md5(s){ return binl2str(core_md5(str2binl(s), s.length * chrsz));}
function hex_hmac_md5(key, data) { return binl2hex(core_hmac_md5(key, data)); }
function b64_hmac_md5(key, data) { return binl2b64(core_hmac_md5(key, data)); }
function str_hmac_md5(key, data) { return binl2str(core_hmac_md5(key, data)); }/** Perform a simple self-test to see if the VM is working*/
function md5_vm_test()
{return hex_md5("abc") == "900150983cd24fb0d6963f7d28e17f72";
}/** Calculate the MD5 of an array of little-endian words, and a bit length*/
function core_md5(x, len)
{/* append padding */x[len >> 5] |= 0x80 << ((len) % 32);x[(((len + 64) >>> 9) << 4) + 14] = len;var a =  1732584193;var b = -271733879;var c = -1732584194;var d =  271733878;for(var i = 0; i < x.length; i += 16){var olda = a;var oldb = b;var oldc = c;var oldd = d;a = md5_ff(a, b, c, d, x[i+ 0], 7 , -680876936);d = md5_ff(d, a, b, c, x[i+ 1], 12, -389564586);c = md5_ff(c, d, a, b, x[i+ 2], 17,  606105819);b = md5_ff(b, c, d, a, x[i+ 3], 22, -1044525330);a = md5_ff(a, b, c, d, x[i+ 4], 7 , -176418897);d = md5_ff(d, a, b, c, x[i+ 5], 12,  1200080426);c = md5_ff(c, d, a, b, x[i+ 6], 17, -1473231341);b = md5_ff(b, c, d, a, x[i+ 7], 22, -45705983);a = md5_ff(a, b, c, d, x[i+ 8], 7 ,  1770035416);d = md5_ff(d, a, b, c, x[i+ 9], 12, -1958414417);c = md5_ff(c, d, a, b, x[i+10], 17, -42063);b = md5_ff(b, c, d, a, x[i+11], 22, -1990404162);a = md5_ff(a, b, c, d, x[i+12], 7 ,  1804603682);d = md5_ff(d, a, b, c, x[i+13], 12, -40341101);c = md5_ff(c, d, a, b, x[i+14], 17, -1502002290);b = md5_ff(b, c, d, a, x[i+15], 22,  1236535329);a = md5_gg(a, b, c, d, x[i+ 1], 5 , -165796510);d = md5_gg(d, a, b, c, x[i+ 6], 9 , -1069501632);c = md5_gg(c, d, a, b, x[i+11], 14,  643717713);b = md5_gg(b, c, d, a, x[i+ 0], 20, -373897302);a = md5_gg(a, b, c, d, x[i+ 5], 5 , -701558691);d = md5_gg(d, a, b, c, x[i+10], 9 ,  38016083);c = md5_gg(c, d, a, b, x[i+15], 14, -660478335);b = md5_gg(b, c, d, a, x[i+ 4], 20, -405537848);a = md5_gg(a, b, c, d, x[i+ 9], 5 ,  568446438);d = md5_gg(d, a, b, c, x[i+14], 9 , -1019803690);c = md5_gg(c, d, a, b, x[i+ 3], 14, -187363961);b = md5_gg(b, c, d, a, x[i+ 8], 20,  1163531501);a = md5_gg(a, b, c, d, x[i+13], 5 , -1444681467);d = md5_gg(d, a, b, c, x[i+ 2], 9 , -51403784);c = md5_gg(c, d, a, b, x[i+ 7], 14,  1735328473);b = md5_gg(b, c, d, a, x[i+12], 20, -1926607734);a = md5_hh(a, b, c, d, x[i+ 5], 4 , -378558);d = md5_hh(d, a, b, c, x[i+ 8], 11, -2022574463);c = md5_hh(c, d, a, b, x[i+11], 16,  1839030562);b = md5_hh(b, c, d, a, x[i+14], 23, -35309556);a = md5_hh(a, b, c, d, x[i+ 1], 4 , -1530992060);d = md5_hh(d, a, b, c, x[i+ 4], 11,  1272893353);c = md5_hh(c, d, a, b, x[i+ 7], 16, -155497632);b = md5_hh(b, c, d, a, x[i+10], 23, -1094730640);a = md5_hh(a, b, c, d, x[i+13], 4 ,  681279174);d = md5_hh(d, a, b, c, x[i+ 0], 11, -358537222);c = md5_hh(c, d, a, b, x[i+ 3], 16, -722521979);b = md5_hh(b, c, d, a, x[i+ 6], 23,  76029189);a = md5_hh(a, b, c, d, x[i+ 9], 4 , -640364487);d = md5_hh(d, a, b, c, x[i+12], 11, -421815835);c = md5_hh(c, d, a, b, x[i+15], 16,  530742520);b = md5_hh(b, c, d, a, x[i+ 2], 23, -995338651);a = md5_ii(a, b, c, d, x[i+ 0], 6 , -198630844);d = md5_ii(d, a, b, c, x[i+ 7], 10,  1126891415);c = md5_ii(c, d, a, b, x[i+14], 15, -1416354905);b = md5_ii(b, c, d, a, x[i+ 5], 21, -57434055);a = md5_ii(a, b, c, d, x[i+12], 6 ,  1700485571);d = md5_ii(d, a, b, c, x[i+ 3], 10, -1894986606);c = md5_ii(c, d, a, b, x[i+10], 15, -1051523);b = md5_ii(b, c, d, a, x[i+ 1], 21, -2054922799);a = md5_ii(a, b, c, d, x[i+ 8], 6 ,  1873313359);d = md5_ii(d, a, b, c, x[i+15], 10, -30611744);c = md5_ii(c, d, a, b, x[i+ 6], 15, -1560198380);b = md5_ii(b, c, d, a, x[i+13], 21,  1309151649);a = md5_ii(a, b, c, d, x[i+ 4], 6 , -145523070);d = md5_ii(d, a, b, c, x[i+11], 10, -1120210379);c = md5_ii(c, d, a, b, x[i+ 2], 15,  718787259);b = md5_ii(b, c, d, a, x[i+ 9], 21, -343485551);a = safe_add(a, olda);b = safe_add(b, oldb);c = safe_add(c, oldc);d = safe_add(d, oldd);}return Array(a, b, c, d);}/** These functions implement the four basic operations the algorithm uses.*/
function md5_cmn(q, a, b, x, s, t)
{return safe_add(bit_rol(safe_add(safe_add(a, q), safe_add(x, t)), s),b);
}
function md5_ff(a, b, c, d, x, s, t)
{return md5_cmn((b & c) | ((~b) & d), a, b, x, s, t);
}
function md5_gg(a, b, c, d, x, s, t)
{return md5_cmn((b & d) | (c & (~d)), a, b, x, s, t);
}
function md5_hh(a, b, c, d, x, s, t)
{return md5_cmn(b ^ c ^ d, a, b, x, s, t);
}
function md5_ii(a, b, c, d, x, s, t)
{return md5_cmn(c ^ (b | (~d)), a, b, x, s, t);
}/** Calculate the HMAC-MD5, of a key and some data*/
function core_hmac_md5(key, data)
{var bkey = str2binl(key);if(bkey.length > 16) bkey = core_md5(bkey, key.length * chrsz);var ipad = Array(16), opad = Array(16);for(var i = 0; i < 16; i++){ipad[i] = bkey[i] ^ 0x36363636;opad[i] = bkey[i] ^ 0x5C5C5C5C;}var hash = core_md5(ipad.concat(str2binl(data)), 512 + data.length * chrsz);return core_md5(opad.concat(hash), 512 + 128);
}/** Add integers, wrapping at 2^32. This uses 16-bit operations internally* to work around bugs in some JS interpreters.*/
function safe_add(x, y)
{var lsw = (x & 0xFFFF) + (y & 0xFFFF);var msw = (x >> 16) + (y >> 16) + (lsw >> 16);return (msw << 16) | (lsw & 0xFFFF);
}/** Bitwise rotate a 32-bit number to the left.*/
function bit_rol(num, cnt)
{return (num << cnt) | (num >>> (32 - cnt));
}/** Convert a string to an array of little-endian words* If chrsz is ASCII, characters >255 have their hi-byte silently ignored.*/
function str2binl(str)
{var bin = Array();var mask = (1 << chrsz) - 1;for(var i = 0; i < str.length * chrsz; i += chrsz)bin[i>>5] |= (str.charCodeAt(i / chrsz) & mask) << (i%32);return bin;
}/** Convert an array of little-endian words to a string*/
function binl2str(bin)
{var str = "";var mask = (1 << chrsz) - 1;for(var i = 0; i < bin.length * 32; i += chrsz)str += String.fromCharCode((bin[i>>5] >>> (i % 32)) & mask);return str;
}/** Convert an array of little-endian words to a hex string.*/
function binl2hex(binarray)
{var hex_tab = hexcase ? "0123456789ABCDEF" : "0123456789abcdef";var str = "";for(var i = 0; i < binarray.length * 4; i++){str += hex_tab.charAt((binarray[i>>2] >> ((i%4)*8+4)) & 0xF) +hex_tab.charAt((binarray[i>>2] >> ((i%4)*8  )) & 0xF);}return str;
}/** Convert an array of little-endian words to a base-64 string*/
function binl2b64(binarray)
{var tab = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";var str = "";for(var i = 0; i < binarray.length * 4; i += 3){var triplet = (((binarray[i   >> 2] >> 8 * ( i   %4)) & 0xFF) << 16)| (((binarray[i+1 >> 2] >> 8 * ((i+1)%4)) & 0xFF) << 8 )|  ((binarray[i+2 >> 2] >> 8 * ((i+2)%4)) & 0xFF);for(var j = 0; j < 4; j++){if(i * 8 + j * 6 > binarray.length * 32) str += b64pad;else str += tab.charAt((triplet >> 6*(3-j)) & 0x3F);}}return str;
}

index.html

big-file.vue

tool.js

1.大文件断点续传与极速秒传：根据文件标识来新增或修改文件记录，修改分片索引从1开始

实际使用场景中，一个文件太大，当第一次上传一半，失败了。第二次再上传时，有可能隔了一天或更久。

FileService.java

big-file.vue

此时是干净的，我们进行测试

完成，没有多余的数据

9-3 分片上传功能开发2

分片数据格式转换，统一系列前后端数据交互传参

1.大文件断点续传与极速秒传：分片数据改成base64 进行传输，后端统一用RequestBody 接收数据

big-file.vue

FileDto.java

Base64ToMultipartFile.java

package com.course.server.util;import org.springframework.web.multipart.MultipartFile;
import sun.misc.BASE64Decoder;import java.io.*;public class Base64ToMultipartFile implements MultipartFile {private final byte[] imgContent;private final String header;public Base64ToMultipartFile(byte[] imgContent, String header) {this.imgContent = imgContent;this.header = header.split(";")[0];}@Overridepublic String getName() {// TODO - implementation depends on your requirementsreturn System.currentTimeMillis() + Math.random() + "." + header.split("/")[1];}@Overridepublic String getOriginalFilename() {// TODO - implementation depends on your requirementsreturn System.currentTimeMillis() + (int) Math.random() * 10000 + "." + header.split("/")[1];}@Overridepublic String getContentType() {// TODO - implementation depends on your requirementsreturn header.split(":")[1];}@Overridepublic boolean isEmpty() {return imgContent == null || imgContent.length == 0;}@Overridepublic long getSize() {return imgContent.length;}@Overridepublic byte[] getBytes() throws IOException {return imgContent;}@Overridepublic InputStream getInputStream() throws IOException {return new ByteArrayInputStream(imgContent);}@Overridepublic void transferTo(File dest) throws IOException, IllegalStateException {new FileOutputStream(dest).write(imgContent);}public static MultipartFile base64ToMultipart(String base64) {try {String[] baseStrs = base64.split(",");BASE64Decoder decoder = new BASE64Decoder();byte[] b = new byte[0];b = decoder.decodeBuffer(baseStrs[1]);for(int i = 0; i < b.length; ++i) {if (b[i] < 0) {b[i] += 256;}}return new Base64ToMultipartFile(b, baseStrs[0]);} catch (IOException e) {e.printStackTrace();return null;}}
}

UploadController.java

测试

base64格式：文件说明+“,”+文件内容

将文件转成base64 进行传输，优势是变成了常见的字符串类型，劣势是传输的数据会变大。

idea 的控制台默认大小是1M，可通过在设置中搜索console，修改默认配置

LogAspect.java

我们其它的业务表中，不要用shard字段，否则也会不打印日志

设置分片文件名

1.大文件断点续传与极速秒传：设置分片文件名,以分片索引结尾

小提示：当字符串拼接个数太多时，不要使用+，而是使用StringBuffer

UploadController.java

测试

这个日志可以删去，太长了

big-file.vue

支持连续上传多个分片

1.大文件断点续传与极速秒传：重构文件上传代码，将上传功能提取成一个单独的upload方法，使用递归调用upload方法，实现连续上传多个分片功能

在上传结束时，做个判断，如果shardIndex == shardTotal ,表示所有分片都上传完成了，如果shardIndex < shardTotal,继续上传下一个分片

重构

我们把生成的方法剪切到后面

把方法都放到下面来

递归，不断的重复做某一件事（上传分片），直到某个条件成立 (shardIndex == shardTotal) ,退出重复做的事。初学者慎用，容易陷入无限递归，跳不出来。

测试

9-4 分片合并功能开发

按分片顺序合并成原始文件

1.大文件断点续传与极速秒传：按分片顺序合并成原始文件

UploadController.java

package com.course.file.controller.admin;import com.course.server.dto.FileDto;
import com.course.server.dto.ResponseDto;
import com.course.server.enums.FileUseEnum;
import com.course.server.service.FileService;
import com.course.server.util.Base64ToMultipartFile;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;import javax.annotation.Resource;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;@RequestMapping("/admin")
@RestController
public class UploadController {private static final Logger LOG = LoggerFactory.getLogger(UploadController.class);public static final String BUSINESS_NAME= "文件上传";//@Value,注入属性值@Value("${file.domain}")private String FILE_DOMAIN;@Value("${file.path}")private String FILE_PATH;@Resourceprivate FileService fileService;@RequestMapping("/upload")public ResponseDto upload(@RequestBody FileDto fileDto) throws Exception {LOG.info("上传文件开始");String use = fileDto.getUse();String key = fileDto.getKey();String suffix = fileDto.getSuffix();String shardBase64 = fileDto.getShard();MultipartFile shard = Base64ToMultipartFile.base64ToMultipart(shardBase64);//保存文件到本地FileUseEnum useEnum = FileUseEnum.getByCode(use);//如果文件夹不存在则创建String dir = useEnum.name().toLowerCase();File fullDir = new File(FILE_PATH + dir);if(!fullDir.exists()){fullDir.mkdirs();}//String path = dir + File.separator + key + "." + suffix + "." +fileDto.getShardIndex();//小提示：当字符串拼接个数太多时，不要使用+，而是使用StringBufferString path = new StringBuffer(dir).append(File.separator).append(key).append(".").append(suffix).toString();//course/6sfSqfOwzmik4A4icMYuUe.mp4String localPath = new StringBuffer(path).append(".").append(fileDto.getShardIndex()).toString();//course/6sfSqfOwzmik4A4icMYuUe.mp4.1String fullPath =FILE_PATH + localPath;File dest = new File(fullPath);shard.transferTo(dest);LOG.info(dest.getAbsolutePath());LOG.info("保存文件记录开始");fileDto.setPath(path);fileService.save(fileDto);ResponseDto responseDto = new ResponseDto();fileDto.setPath(FILE_DOMAIN + path);responseDto.setContent(fileDto);if (fileDto.getShardIndex() == fileDto.getShardTotal()){this.merge(fileDto);}return responseDto;}public void merge(FileDto fileDto) throws Exception {LOG.info("合并分片开始");String path = fileDto.getPath();//全路径 http://127.0.0.1:9000/file/f/course/6sfSqfOwzmik4A4icMYuUe.mp4//merge里面的方法，只需要相对路径path = path.replace(FILE_DOMAIN,"");//course/6sfSqfOwzmik4A4icMYuUe.mp4Integer shardTotal = fileDto.getShardTotal();File newFile = new File(FILE_PATH + path);FileOutputStream outputStream = new FileOutputStream(newFile, true);//文件追加写入FileInputStream fileInputStream = null;//分片文件byte[] byt = new byte[10 * 1024 * 1024];int len;try {for (int i = 0; i < shardTotal; i++) {//读取第i个分片fileInputStream = new FileInputStream(FILE_PATH + path + "." + (i + 1));course/6sfSqfOwzmik4A4icMYuUe.mp4.1while ((len = fileInputStream.read(byt)) != -1) {outputStream.write(byt, 0, len);}}} catch (IOException e) {LOG.error("分片合并异常", e);}finally {try{if(fileInputStream != null){fileInputStream.close();}outputStream.close();LOG.info("io流关闭");}catch (Exception e){LOG.info("io流关闭",e);}}LOG.info("合并分片结束");}
}

merge里面的方法，只需要相对路径

保存到本地的分片名是xxx.mp4.1,但是我们返给前端和保存进数据库的path 都用xxx.mp4

测试

合并完成后删除多余分片

1.大文件断点续传与极速秒传：合并成原始文件后，删除所有分片

UploadController.java

big-file.vue

测试

9-5 分片检查与极速秒传

上传分片之前先进行分片检查

1.大文件断点续传与极速秒传：上传分片之前先进行分片检查，支持断点续传

开始上传之前，先检查一下当前的文件是否上传过了（数据库是否有记录），如果上传过了，已经传到第几个分片了。

FileService.java

UploadController.java

big-file.vue

测试

全初始化再次测试上传一半的结果

big-file.vue

也就是说现在只能上传第一个、第二个分片

只请求了两次，现在把代码恢复

再次测试

确实是从中间开始继续上传

极速秒传

1.大文件断点续传与极速秒传：支持极速秒传

big-file.vue

UploadController.java

测试

上传成功，但没有点击保存

再次上传，即为文件极速秒传成功，换一个也是可以

增加上传进度条

1.大文件断点续传与极速秒传：增加上传进度条组件

文件被分成四个分片，所以每个分片上传完，进度条就加25%

可以做成vue 组件，也可以做成js工具类，做成vue 组件的话，就只能在vue框架中使用，做成js 工具类的话，可以在jsp页面，thymeleaf模板页面、angular等前端框架中直接使用。

progress.js

Progress = {show: function (width) {let _this = this;_this.width = width;if ($("#progress-div").length > 0) {$(".progress").attr("data-percent", width);$(".progress-bar").width(width + "%");} else {let progressDiv = "<div id=\"progress-div\" style=\"z-index: 10011;\n" +"    position: fixed;\n" +"    padding: 10px;\n" +"    margin: 0px 0px 0px -150px;\n" +"    width: 300px;\n" +"    top: 40%;\n" +"    left: 50%;\n" +"    text-align: center;\n" +"    height: 45px;\n" +"    color: rgb(0, 0, 0);\n" +"    border: 3px solid rgb(170, 170, 170);\n" +"    background-color: rgb(255, 255, 255);\n" +"    cursor: wait;\"><div class=\"progress pos-rel\" data-percent=\"" + width + "%\"><div class=\"progress-bar\"></div></div></div>";$("#progress-div").remove();$("body").append(progressDiv);// 背景遮罩$("body").append($("<div id=\"progress-overlay\" style=\"z-index: 10010;\n" +"  border: none;\n" +"  margin: 0px;\n" +"  padding: 0px;\n" +"  width: 100%;\n" +"  height: 100%;\n" +"  top: 0px;\n" +"  left: 0px;\n" +"  background-color: rgb(0, 0, 0);\n" +"  opacity: 0.6;\n" +"  cursor: wait;\n" +"  position: fixed;\"></div>"));$(".progress-bar").width(width + "%");}},hide: function () {setTimeout(function () {$("#progress-div").remove();//progress-div：显示的进度条；//progress-overlay：背景遮罩，将页面所有的元素挡住，不可点击任何地方，常用于防止重复提交$("#progress-overlay").remove();}, 1000);//上传到100%时，应该关闭进度条，但是一关闭，100%这个值就会一闪而过，所以为了用户体验，故意延迟1秒}
};

progress-div：显示的进度条；
progress-overlay：背景遮罩，将页面所有的元素挡住，不可点击任何地方，常用于防止重复提交

上传到100%时，应该关闭进度条，但是一关闭，100%这个值就会一闪而过，所以为了用户体验，故意延迟1秒

big-file.vue

做成js全局工具类，可以直接使用，不需要import