img

js大文件切片断续上传

2022-10-13 0条评论 4.4k次阅读 JavaScript

学会大文件切片,面试时直接涨薪5K!


架构图如下

image.png

原生js,并发请求

就是提前把片切好,然后每个片一次性传给服务器

 <div class="container">
        <div class="item">
            <h3>大文件上传</h3>
            <section class="upload_box" id="upload7">
                <input type="file" class="upload_inp">
                <div class="upload_button_box">
                    <button class="upload_button select">上传图片</button>
                </div>
                <div class="upload_progress">
                    <div class="value"></div>
                </div>
            </section>
        </div>
    </div>

需要 MD5 加密框架 spark-md5.min.js

// 大文件上传 接口步骤 查看已经上传过的切片(upload_already) -> 上传每个切片(upload_chunk) -> 合并切片(upload_merge)
(function (){
    let upload = document.querySelector("#upload7"),
        upload_inp = upload.querySelector('.upload_inp'), //file input框
        upload_button_select = upload.querySelector('.upload_button.select'),
        upload_progress = upload.querySelector('.upload_progress'),
        upload_progress_value = upload_progress.querySelector('.value')

    // 生成 HASH 对象
    const changeBuffer = file => {
        return new Promise(resolve => {
            let fileReader = new FileReader();
            fileReader.readAsArrayBuffer(file);
            fileReader.onload = ev => {
                let buffer = ev.target.result,
                    spark = new SparkMD5.ArrayBuffer(),
                    HASH,
                    suffix;
                spark.append(buffer);
                HASH = spark.end();
                suffix = /\.([a-zA-Z0-9]+)$/.exec(file.name)[1];
                resolve({
                    buffer,
                    HASH,
                    suffix,
                    filename: `${HASH}.${suffix}`
                });
            };
        });
    };

    // 上传成功的处理
    const clear = () => {
        upload_button_select.classList.remove('loading');
        upload_progress.style.display = 'none';
        upload_progress_value.style.width = '0%';
    }

    // 监听文件被选择
    upload_inp.addEventListener('change', async function() {
        let file = upload_inp.files[0]
        if(!file) return;
        upload_button_select.classList.add('loading');
        upload_progress.style.display = 'block';

        let already = [],
            data = null,
            {
                HASH,
                suffix
            } = await changeBuffer(file) // 获取文件的 HASH

        // 获取已经上传的切片
        try {
            data = await instance.get('/upload_already', {
                params: {
                    HASH
                }
            });
            if(+data.code === 0) {
                already = data.fileList;
            }
        } catch (err) {}

        // 开始切片 [固定数量 & 固定文件大小]
        let max = 1024 * 100, // 固定每个文件大小为 100KB
            count = Math.ceil(file.size / max),
            index = 0, // 用作循环
            chunks = []; // 存片
        if (count > 100) {
            max = file.size / 100;
            count = 100;
        }

        while (index < count) {
            chunks.push({
                file: file.slice(index * max, (index + 1) * max), // index*max,(index+1)*max
                filename: `${HASH}_${index+1}.${suffix}`
            })
            index++;
        }
        // 上传成功的处理
        index = 0;
        const complete = async () => {
            // 控制进度条
            index++;
            // 进度条
            upload_progress_value.style.width = `${index/count*100}%`;

            // 当所有切片都上传成功,我们合并切片
            if(index < count) return;
            upload_progress_value.style.width = `100%`;

            try {
                data = await instance.post('/upload_merge', {
                    HASH,
                    count
                }, {
                    headers: {
                        'Content-Type': 'application/x-www-form-urlencoded'
                    }
                })

                if(+data.code === 0) {
                    alert(`上传成功,访问链接:${data.servicePath}`)
                    clear();
                    return;
                }
                throw data.codeText;
            } catch (e) {
                console.log('切片合并失败,请您稍后重试~~',e)
                clear();
            }
        }

        // 切片完成,把每个切片上传到服务器
        chunks.forEach(chunk => {
            // 已经上传的无需再上传
            if(already.length && already.includes(chunk.filename)) {
                complete();
                return;
            }
            let fm = new FormData;
            fm.append('file',chunk.file);
            fm.append('filename',chunk.filename);
            instance.post('/upload_chunk', fm).then(data => {
                // 每个片传成功就执行一次该方法
                if(+data.code === 0) {
                    complete();
                    return;
                }
                return Promise.reject(data.codeText)
            }).catch(() => {
                alert('当前切片上传失败,请您稍后再试~~')
                clear()
            })
        })
    })

    // 点击上传按钮
    upload_button_select.addEventListener('click',function() {
        upload_inp.click();
    })

})();

断续上传,一个一个上传

用了上面的并发请求,发现同时请求100个的文件到第50个的时候nginx就给断了,为了解决这个问题,优化了一下代码,当上一个请求成功了再请求下一个。用的是Vue

22231122221.gif

async changeFile(e){
  let files = e.target.files;
  await this.changeMaxFile(files)
}
async changeMaxFile(files) {
  this.loadingMaxItem = ElLoading.service({
    text: '大文件正在处理中,请稍等...'
  })
  let file = files[0]
  const num = 100; // 切片的个数
  let data = null,
    {
      HASH,
      suffix
    } = await this.changeBuffer(file) // 获取文件的 HASH
  this.loadingMaxItem.setText(`处理完成,正在上传中...`)
  // 开始切片 [固定数量 & 固定文件大小]
  let max = 1024 * 100, // 固定每个文件大小为 100KB
    count = Math.ceil(file.size / max),
    index = 0, // 用作循环
    chunks = []; // 存片
  if (count > num) {
    max = file.size / num;
    count = num;
  }

  while (index < count) {
    chunks.push({
      file: file.slice(index * max, (index + 1) * max), // index*max,(index+1)*max
      filename: `${HASH}_${index+1}.${suffix}`
    })
    index++;
  }

  // 逐个上传切片
  let successCount = 0
  for (let i = 0; i < chunks.length; i++) {
    const currentFile = chunks[i]
    try {
      let fm = new FormData();
      fm.append('file', currentFile.file);
      fm.append('chunk', i); // 第几个切片
      fm.append('task', HASH);

      // 上传切片
      await this.$axios.post(this.$urls.upload.uploadAccept, fm, {
        timeout: 10*60*60
      });
      successCount++
      this.loadingMaxItem.setText(`大文件正在上传中,已完成${Math.ceil(successCount/count*100)}%`)
    } catch (e) {
      this.loadingMaxItem.close()
      this.$tips.error({text:'当前切片上传失败,请您稍后再试~~'})
      return
    }
  }

  // 当所有切片都上传成功,我们合并切片
  if(successCount === count){
    try {
      let url = this.$tools.getURL(this.$urls.upload.uploadComplete,{
        target_filename: file.name,
        task: HASH,
        count,
      })

      data = await this.$axios.get(url)
      this.loadingMaxItem.setText(`大文件正在上传中,已完成100%`)
      if(data?.file_url) {
        let file_url = data.file_url
        alert(`您的访问链接为${file_url}`)
      }
    } catch (e) {
      this.loadingMaxItem.close()
      this.$tips.error({text:'切片合并失败,请您稍后重试~~'})
      console.log('切片合并失败,请您稍后重试~~',e)
    }
  }
},

后端代码

用的是python,链接:https://www.cnblogs.com/Zzbj/p/16373953.html

💬 COMMENT


🦄 支持markdown语法

👋友