架构图如下
原生js，并发请求
就是提前把片切好，然后每个片一次性传给服务器
<div class="container">
<div class="item">
<h3>大文件上传</h3>
<section class="upload_box" id="upload7">
<input type="file" class="upload_inp">
<div class="upload_button_box">
<button class="upload_button select">上传图片</button>
</div>
<div class="upload_progress">
<div class="value"></div>
</div>
</section>
</div>
</div>
需要 MD5 加密框架 spark-md5.min.js
// 大文件上传 接口步骤 查看已经上传过的切片(upload_already) -> 上传每个切片(upload_chunk) -> 合并切片(upload_merge)
(function (){
let upload = document.querySelector("#upload7"),
upload_inp = upload.querySelector('.upload_inp'), //file input框
upload_button_select = upload.querySelector('.upload_button.select'),
upload_progress = upload.querySelector('.upload_progress'),
upload_progress_value = upload_progress.querySelector('.value')
// 生成 HASH 对象
const changeBuffer = file => {
return new Promise(resolve => {
let fileReader = new FileReader();
fileReader.readAsArrayBuffer(file);
fileReader.onload = ev => {
let buffer = ev.target.result,
spark = new SparkMD5.ArrayBuffer(),
HASH,
suffix;
spark.append(buffer);
HASH = spark.end();
suffix = /\.([a-zA-Z0-9]+)$/.exec(file.name)[1];
resolve({
buffer,
HASH,
suffix,
filename: `${HASH}.${suffix}`
});
};
});
};
// 上传成功的处理
const clear = () => {
upload_button_select.classList.remove('loading');
upload_progress.style.display = 'none';
upload_progress_value.style.width = '0%';
}
// 监听文件被选择
upload_inp.addEventListener('change', async function() {
let file = upload_inp.files[0]
if(!file) return;
upload_button_select.classList.add('loading');
upload_progress.style.display = 'block';
let already = [],
data = null,
{
HASH,
suffix
} = await changeBuffer(file) // 获取文件的 HASH
// 获取已经上传的切片
try {
data = await instance.get('/upload_already', {
params: {
HASH
}
});
if(+data.code === 0) {
already = data.fileList;
}
} catch (err) {}
// 开始切片 [固定数量 & 固定文件大小]
let max = 1024 * 100, // 固定每个文件大小为 100KB
count = Math.ceil(file.size / max),
index = 0, // 用作循环
chunks = []; // 存片
if (count > 100) {
max = file.size / 100;
count = 100;
}
while (index < count) {
chunks.push({
file: file.slice(index * max, (index + 1) * max), // index*max,(index+1)*max
filename: `${HASH}_${index+1}.${suffix}`
})
index++;
}
// 上传成功的处理
index = 0;
const complete = async () => {
// 控制进度条
index++;
// 进度条
upload_progress_value.style.width = `${index/count*100}%`;
// 当所有切片都上传成功，我们合并切片
if(index < count) return;
upload_progress_value.style.width = `100%`;
try {
data = await instance.post('/upload_merge', {
HASH,
count
}, {
headers: {
'Content-Type': 'application/x-www-form-urlencoded'
}
})
if(+data.code === 0) {
alert(`上传成功，访问链接：${data.servicePath}`)
clear();
return;
}
throw data.codeText;
} catch (e) {
console.log('切片合并失败，请您稍后重试~~',e)
clear();
}
}
// 切片完成，把每个切片上传到服务器
chunks.forEach(chunk => {
// 已经上传的无需再上传
if(already.length && already.includes(chunk.filename)) {
complete();
return;
}
let fm = new FormData;
fm.append('file',chunk.file);
fm.append('filename',chunk.filename);
instance.post('/upload_chunk', fm).then(data => {
// 每个片传成功就执行一次该方法
if(+data.code === 0) {
complete();
return;
}
return Promise.reject(data.codeText)
}).catch(() => {
alert('当前切片上传失败，请您稍后再试~~')
clear()
})
})
})
// 点击上传按钮
upload_button_select.addEventListener('click',function() {
upload_inp.click();
})
})();
断续上传，一个一个上传
用了上面的并发请求，发现同时请求100个的文件到第50个的时候nginx就给断了，为了解决这个问题，优化了一下代码，当上一个请求成功了再请求下一个。用的是Vue
async changeFile(e){
let files = e.target.files;
await this.changeMaxFile(files)
}
async changeMaxFile(files) {
this.loadingMaxItem = ElLoading.service({
text: '大文件正在处理中，请稍等...'
})
let file = files[0]
const num = 100; // 切片的个数
let data = null,
{
HASH,
suffix
} = await this.changeBuffer(file) // 获取文件的 HASH
this.loadingMaxItem.setText(`处理完成，正在上传中...`)
// 开始切片 [固定数量 & 固定文件大小]
let max = 1024 * 100, // 固定每个文件大小为 100KB
count = Math.ceil(file.size / max),
index = 0, // 用作循环
chunks = []; // 存片
if (count > num) {
max = file.size / num;
count = num;
}
while (index < count) {
chunks.push({
file: file.slice(index * max, (index + 1) * max), // index*max,(index+1)*max
filename: `${HASH}_${index+1}.${suffix}`
})
index++;
}
// 逐个上传切片
let successCount = 0
for (let i = 0; i < chunks.length; i++) {
const currentFile = chunks[i]
try {
let fm = new FormData();
fm.append('file', currentFile.file);
fm.append('chunk', i); // 第几个切片
fm.append('task', HASH);
// 上传切片
await this.$axios.post(this.$urls.upload.uploadAccept, fm, {
timeout: 10*60*60
});
successCount++
this.loadingMaxItem.setText(`大文件正在上传中，已完成${Math.ceil(successCount/count*100)}%`)
} catch (e) {
this.loadingMaxItem.close()
this.$tips.error({text:'当前切片上传失败，请您稍后再试~~'})
return
}
}
// 当所有切片都上传成功，我们合并切片
if(successCount === count){
try {
let url = this.$tools.getURL(this.$urls.upload.uploadComplete,{
target_filename: file.name,
task: HASH,
count,
})
data = await this.$axios.get(url)
this.loadingMaxItem.setText(`大文件正在上传中，已完成100%`)
if(data?.file_url) {
let file_url = data.file_url
alert(`您的访问链接为${file_url}`)
}
} catch (e) {
this.loadingMaxItem.close()
this.$tips.error({text:'切片合并失败，请您稍后重试~~'})
console.log('切片合并失败，请您稍后重试~~',e)
}
}
},
后端代码
用的是python，链接：https://www.cnblogs.com/Zzbj/p/16373953.html
💬 COMMENT