前端大文件分片上传和上传流程的封装-天翼云开发者社区

Web 应用程序中，文件上传是一个常见的需求。尤其是在处理大文件时，传统的一次性上传方式可能会遇到性能和稳定性问题。为了解决这些问题，我们可以采用文件分片上传的方式，并利用 Web Workers 进行多线程优化，以提高上传效率和性能。本文将介绍如何利用 JavaScript 实现文件分片上传，并封装整个上传流程，使其更加可控和灵活。

文件分片和多线程优化

首先，让我们来看一下文件分片和多线程优化的实现。

文件分片

文件分片是将大文件拆分成多个小块进行上传的一种策略。这样做的好处是可以降低单个请求的负载，提高上传的稳定性。在我们的实现中，我们将使用 JavaScript 对文件进行分片，具体步骤如下：

首先，我们根据指定的分片大小将文件拆分成多个 Blob 对象。
然后，针对每个 Blob 对象，我们计算其哈希值，用于后续校验和验证。

多线程优化

为了提高计算哈希值的效率，我们可以利用浏览器的多线程能力，使用 Web Workers 进行并行计算。Web Workers 是 JavaScript 中的一种多线程解决方案，可以在后台执行脚本，不会影响页面的性能和响应速度。在我们的实现中，我们会根据系统硬件并发数量创建对应数量的 Web Worker，并将任务分配给它们，以加速文件分片的哈希值计算。

utils.js

export function blobHash(blob) {
  return new Promise((resolve) => {
    const spark = new SparkMD5.ArrayBuffer();
    const reader = new FileReader();
    reader.onload = (e) => {
      const bytes = e.target.result;
      spark.append(bytes);
      const hash = spark.end();
      resolve(hash);
    };
    reader.readAsArrayBuffer(blob);
  });
}

bigFileWorker.js

import { blobHash } from "./utils.js";
import "../lib/spark-md5.min.js";

onmessage = async (e) => {
  const { blobList } = e.data;
  const proms = [];
  for (let i = 0; i < blobList.length; i++) {
    proms.push(blobHash(blobList[i]));
  }
  const hashList = await Promise.all(proms);
  postMessage(hashList);
};

bigFile.js

import { blobHash } from "./utils.js";
const THREAD_COUNT = navigator.hardwareConcurrency || 4;

// 计算出文件分片对象列表
export function fileCreateChunks(
  file,
  { chunkSize, isMultithreading = false }
) {
  return new Promise(async (resolve) => {
    if (typeof Worker === "undefined") {
      isMultithreading = false;
    }
    const allBlobList = [];
    for (let i = 0; i < file.size; i += chunkSize) {
      allBlobList.push(file.slice(i, i + chunkSize));
    }
    if (!isMultithreading) {
      const results = [];
      let count = 0;
      for (let i = 0; i < allBlobList.length; i++) {
        const blob = allBlobList[i];
        blobHash(blob).then((hash) => {
          results[i] = {
            blob: blob,
            index: i,
            start: i * chunkSize,
            end: Math.min(i * chunkSize + chunkSize, file.size),
            hash: hash,
          };
          count++;
          if (count >= allBlobList.length) {
            resolve(results);
          }
        });
      }
    } else {
      const results = [];
      const threadChunkCount = Math.ceil(allBlobList.length / THREAD_COUNT); // 每个线程需要分配的分片数量
      let finishedCount = 0;
      const thread_count = Math.min(THREAD_COUNT, allBlobList.length); // 要防止线程多开
      // 创建线程并且分配任务
      for (let i = 0; i < thread_count; i++) {
        const startIndex = i * threadChunkCount;
        const endIndex = Math.min(
          startIndex + threadChunkCount,
          allBlobList.length
        );
        const blobList = [];
        for (let j = startIndex; j < endIndex; j++) {
          blobList.push(allBlobList[j]);
        }
        const worker = new Worker("./js/bigFileWorker.js", {
          type: "module",
        });
        worker.postMessage({
          blobList,
        });
        worker.onmessage = (e) => {
          let k = 0;
          while (k < e.data.length) {
            const hash = e.data[k];
            results[startIndex + k] = {
              blob: blobList[k],
              index: startIndex + k,
              start: (startIndex + k) * chunkSize,
              end: Math.min(
                (startIndex + k) * chunkSize + chunkSize,
                file.size
              ),
              hash,
            };
            k++;
          }
          worker.terminate();
          finishedCount++;
          if (finishedCount >= thread_count) {
            resolve(results);
          }
        };
      }
    }
  });
}

封装文件分片上传流程

接下来，让我们来详细介绍如何封装文件分片上传流程，使其更加灵活和易用。

文件分片上传类设计

我们设计了一个名为 FileChunkUploader 的类，用于管理文件分片上传的整个流程。该类接受一系列配置参数，并提供了一系列生命周期函数和控制方法，以方便用户在上传过程中进行处理。

fileChunkUploader.js

import { fileCreateChunks } from "./bigFile.js";

export class FileChunkUploader {
  constructor({
    file, // 文件
    chunkSize = 2 * 1024 * 1024, // 分片大小
    isSplitFileMultithreading = true, // 切片文件是否多线程
    beforeSplitChunk = () => {}, // 开始分片前
    splitChunkDone = () => {}, // 分片完成
    beforeChunkUploadStart = () => {}, // 开始上传前，可能需要到后端去请求该文件已经被上传的情况
    uploadFunc = () => {}, // 进行上传的函数，由外部进行控制如何上传
    onChunkUploadSuccess = () => {}, // 分块上传成功后的回调
    onChunkUploadError = () => {}, // 分块上传错误的时候的回调
    onAllChunkUploadDone = () => {}, // 所有分块上传都成功了的回调
    // onPauseUpload = () => {}, // 当暂停上传的时候的回调
  }) {
    this.file = file;
    this.chunkSize = chunkSize;
    this.isSplitFileMultithreading = isSplitFileMultithreading;
    this.beforeSplitChunk = beforeSplitChunk;
    this.splitChunkDone = splitChunkDone;
    this.beforeChunkUploadStart = beforeChunkUploadStart;
    this.uploadFunc = uploadFunc;
    this.onChunkUploadSuccess = onChunkUploadSuccess;
    this.onChunkUploadError = onChunkUploadError;
    this.onAllChunkUploadDone = onAllChunkUploadDone;
    // this.onPauseUpload = onPauseUpload;

    this._chunkList = []; // 分片列表
    this._fileHash = ""; // 用来存储整个文件hash
    this._currentChunkIndex = 0; // 上传的文件索引

    this._isUploading = false; // 是否上传中的标记
    this._waitResolve = null; // 暂停之后等待的resolve
  }

  // 开始运行的函数
  async start() {
    // 设置当前上传索引
    const setCurrentChunkIndex = (i) => {
      // 设置的索引范围必须控制在分片数量范围内
      if (i < 0) {
        i = 0;
      } else if (i > this._chunkList.length - 1) {
        i = this._chunkList.length - 1;
      }
      this._currentChunkIndex = i;
    };

    const waitForUpload = () => {
      return new Promise((resolve) => {
        this._waitResolve = resolve;
      });
    };

    await this.beforeSplitChunk(); // 必须等待执行完
    // 先做切片
    this._chunkList = await fileCreateChunks(this.file, {
      chunkSize: this.chunkSize,
      isMultithreading: this.isSplitFileMultithreading,
    });
    this._fileHash = SparkMD5.hash(
      this._chunkList.map((chunk) => chunk.hash).join("")
    );
    await this.splitChunkDone(); // 完成分片
    // 开始上传分片前，可能需要到后端去请求该文件已经被上传的情况
    // 这里需要提供给用户可以给对象设置上传从哪个分片开始的方法
    await this.beforeChunkUploadStart({ setCurrentChunkIndex });
    // 进行分片上传了
    this._isUploading = true; // 标记为正在进行上传分片
    while (this._currentChunkIndex < this._chunkList.length) {
      if (this._isUploading) {
        try {
          // 上传分片
          await this.uploadFunc({
            currentChunkIndex: this._currentChunkIndex,
            chunk: this._chunkList[this._currentChunkIndex],
          });
          await this.onChunkUploadSuccess({
            currentChunkIndex: this._currentChunkIndex,
            allChunkLength: this._chunkList.length,
            chunk: this._chunkList[this._currentChunkIndex],
          });
          this._currentChunkIndex++;
        } catch (err) {
          console.error(err); // 打印错误
          await this.onChunkUploadError({
            err: err,
            pauseUpload: this.pauseUpload.bind(this),
          });
        }
      } else {
        // 暂停等待
        await waitForUpload(); // 这里索引不用加
      }
    }
    await this.onAllChunkUploadDone();
  }

  // 取消上传
  cancelUpload() {
    this._isUploading = false;
    this._currentChunkIndex = 0;
    this._chunkList = [];
  }

  // 暂停函数，不能暂停分片，只能暂停上传的任务
  pauseUpload() {
    this._isUploading = false;
  }

  // 重新上传
  resumeUpload() {
    this._isUploading = true;
    // 继续resolve出去，继续上传分片
    if (this._waitResolve) {
      this._waitResolve();
      this._waitResolve = null;
    }
  }
}

主要功能和方法

start() 方法：该方法是上传的入口函数，负责调用生命周期函数和控制上传流程。
cancelUpload() 方法：用于取消上传任务，清空分片列表和上传索引。
pauseUpload() 方法：用于暂停上传任务。
resumeUpload() 方法：用于恢复上传任务。

通过封装文件分片上传流程，我们使得文件上传的整个过程变得清晰易懂，同时还提供了灵活的控制和回调机制，方便用户在不同阶段进行处理。

测试代码如下：

import { fileCreateChunks } from "./bigFile.js";
import { FileChunkUploader } from "./fileChunkUploader.js";

const inp = document.querySelector("input");
inp.onchange = async (e) => {
  const file = e.target.files[0];
  if (!file) return;
  console.log(file);

  const timeThread1 = new Date();
  const chunksWithMultithreading = await fileCreateChunks(file, {
    chunkSize: 2 * 1024 * 1024,
    isMultithreading: true,
  });
  const timeThread2 = new Date();
  console.log("时间 with multithread：", timeThread2 - timeThread1);
  console.log(chunksWithMultithreading);
  console.log(
    Array.isArray(chunksWithMultithreading),
    chunksWithMultithreading.length
  );

  const time1 = new Date();
  const chunks = await fileCreateChunks(file, {
    chunkSize: 2 * 1024 * 1024,
    isMultithreading: false,
  });
  const time2 = new Date();
  console.log("时间：", time2 - time1);
  console.log(chunks);
  let isIndex3Error = true;
  const fileChunkUploader = new FileChunkUploader({
    file,
    beforeSplitChunk: () => {
      return new Promise((resolve) => {
        console.log("🚀🚀🚀分片前🚀🚀🚀");
        resolve();
      });
    },
    splitChunkDone: () => {
      console.log("🚀🚀🚀分片完成🚀🚀🚀");
    },
    uploadFunc: ({ currentChunkIndex, chunk }) => {
      return new Promise((resolve, reject) => {
        setTimeout(() => {
          if (currentChunkIndex === 2 && isIndex3Error) {
            reject();
          }
          console.log("🚀 upload", currentChunkIndex, chunk);
          resolve();
        }, 2000);
      });
    },
    onChunkUploadSuccess: ({ currentChunkIndex, allChunkLength }) => {
      console.log(
        "🎉 upload success，进度",
        `${currentChunkIndex + 1} / ${allChunkLength}`
      );
    },
    onChunkUploadError: ({ pauseUpload }) => {
      return new Promise((resolve, reject) => {
        pauseUpload();
        isIndex3Error = false;
        resolve();
      });
    },
    onAllChunkUploadDone: () => {
      console.log("✔✔✔全部完成");
    },
  });
  window.fileChunkUploader = fileChunkUploader;
  await fileChunkUploader.start();
  console.log(fileChunkUploader);
};

测试结果：

如果文件比较大，多线程计算会明显更快

总结

本文介绍了如何利用 JavaScript 实现文件分片上传，并通过 Web Workers 进行多线程优化，以提高上传效率和性能。同时，我们还封装了文件分片上传流程，使其更加可控和灵活。希望本文对你理解文件上传的优化和封装有所帮助，也希望能够对你的实际项目中的文件上传功能有所启发。

export function blobHash(blob) { return new Promise((resolve) => { const spark = new SparkMD5.ArrayBuffer(); const reader = new FileReader(); reader.onload = (e) => { const bytes = e.target.result; spark.append(bytes); const hash = spark.end(); resolve(hash); }; reader.readAsArrayBuffer(blob); }); }

import { blobHash } from "./utils.js"; import "../lib/spark-md5.min.js"; onmessage = async (e) => { const { blobList } = e.data; const proms = []; for (let i = 0; i < blobList.length; i++) { proms.push(blobHash(blobList[i])); } const hashList = await Promise.all(proms); postMessage(hashList); };

import { blobHash } from "./utils.js"; const THREAD_COUNT = navigator.hardwareConcurrency || 4; // 计算出文件分片对象列表 export function fileCreateChunks( file, { chunkSize, isMultithreading = false } ) { return new Promise(async (resolve) => { if (typeof Worker === "undefined") { isMultithreading = false; } const allBlobList = []; for (let i = 0; i < file.size; i += chunkSize) { allBlobList.push(file.slice(i, i + chunkSize)); } if (!isMultithreading) { const results = []; let count = 0; for (let i = 0; i < allBlobList.length; i++) { const blob = allBlobList[i]; blobHash(blob).then((hash) => { results[i] = { blob: blob, index: i, start: i * chunkSize, end: Math.min(i * chunkSize + chunkSize, file.size), hash: hash, }; count++; if (count >= allBlobList.length) { resolve(results); } }); } } else { const results = []; const threadChunkCount = Math.ceil(allBlobList.length / THREAD_COUNT); // 每个线程需要分配的分片数量 let finishedCount = 0; const thread_count = Math.min(THREAD_COUNT, allBlobList.length); // 要防止线程多开 // 创建线程并且分配任务 for (let i = 0; i < thread_count; i++) { const startIndex = i * threadChunkCount; const endIndex = Math.min( startIndex + threadChunkCount, allBlobList.length ); const blobList = []; for (let j = startIndex; j < endIndex; j++) { blobList.push(allBlobList[j]); } const worker = new Worker("./js/bigFileWorker.js", { type: "module", }); worker.postMessage({ blobList, }); worker.onmessage = (e) => { let k = 0; while (k < e.data.length) { const hash = e.data[k]; results[startIndex + k] = { blob: blobList[k], index: startIndex + k, start: (startIndex + k) * chunkSize, end: Math.min( (startIndex + k) * chunkSize + chunkSize, file.size ), hash, }; k++; } worker.terminate(); finishedCount++; if (finishedCount >= thread_count) { resolve(results); } }; } } }); }

import { fileCreateChunks } from "./bigFile.js"; export class FileChunkUploader { constructor({ file, // 文件 chunkSize = 2 * 1024 * 1024, // 分片大小 isSplitFileMultithreading = true, // 切片文件是否多线程 beforeSplitChunk = () => {}, // 开始分片前 splitChunkDone = () => {}, // 分片完成 beforeChunkUploadStart = () => {}, // 开始上传前，可能需要到后端去请求该文件已经被上传的情况 uploadFunc = () => {}, // 进行上传的函数，由外部进行控制如何上传 onChunkUploadSuccess = () => {}, // 分块上传成功后的回调 onChunkUploadError = () => {}, // 分块上传错误的时候的回调 onAllChunkUploadDone = () => {}, // 所有分块上传都成功了的回调 // onPauseUpload = () => {}, // 当暂停上传的时候的回调 }) { this.file = file; this.chunkSize = chunkSize; this.isSplitFileMultithreading = isSplitFileMultithreading; this.beforeSplitChunk = beforeSplitChunk; this.splitChunkDone = splitChunkDone; this.beforeChunkUploadStart = beforeChunkUploadStart; this.uploadFunc = uploadFunc; this.onChunkUploadSuccess = onChunkUploadSuccess; this.onChunkUploadError = onChunkUploadError; this.onAllChunkUploadDone = onAllChunkUploadDone; // this.onPauseUpload = onPauseUpload; this._chunkList = []; // 分片列表 this._fileHash = ""; // 用来存储整个文件hash this._currentChunkIndex = 0; // 上传的文件索引 this._isUploading = false; // 是否上传中的标记 this._waitResolve = null; // 暂停之后等待的resolve } // 开始运行的函数 async start() { // 设置当前上传索引 const setCurrentChunkIndex = (i) => { // 设置的索引范围必须控制在分片数量范围内 if (i < 0) { i = 0; } else if (i > this._chunkList.length - 1) { i = this._chunkList.length - 1; } this._currentChunkIndex = i; }; const waitForUpload = () => { return new Promise((resolve) => { this._waitResolve = resolve; }); }; await this.beforeSplitChunk(); // 必须等待执行完 // 先做切片 this._chunkList = await fileCreateChunks(this.file, { chunkSize: this.chunkSize, isMultithreading: this.isSplitFileMultithreading, }); this._fileHash = SparkMD5.hash( this._chunkList.map((chunk) => chunk.hash).join("") ); await this.splitChunkDone(); // 完成分片 // 开始上传分片前，可能需要到后端去请求该文件已经被上传的情况 // 这里需要提供给用户可以给对象设置上传从哪个分片开始的方法 await this.beforeChunkUploadStart({ setCurrentChunkIndex }); // 进行分片上传了 this._isUploading = true; // 标记为正在进行上传分片 while (this._currentChunkIndex < this._chunkList.length) { if (this._isUploading) { try { // 上传分片 await this.uploadFunc({ currentChunkIndex: this._currentChunkIndex, chunk: this._chunkList[this._currentChunkIndex], }); await this.onChunkUploadSuccess({ currentChunkIndex: this._currentChunkIndex, allChunkLength: this._chunkList.length, chunk: this._chunkList[this._currentChunkIndex], }); this._currentChunkIndex++; } catch (err) { console.error(err); // 打印错误 await this.onChunkUploadError({ err: err, pauseUpload: this.pauseUpload.bind(this), }); } } else { // 暂停等待 await waitForUpload(); // 这里索引不用加 } } await this.onAllChunkUploadDone(); } // 取消上传 cancelUpload() { this._isUploading = false; this._currentChunkIndex = 0; this._chunkList = []; } // 暂停函数，不能暂停分片，只能暂停上传的任务 pauseUpload() { this._isUploading = false; } // 重新上传 resumeUpload() { this._isUploading = true; // 继续resolve出去，继续上传分片 if (this._waitResolve) { this._waitResolve(); this._waitResolve = null; } } }

import { fileCreateChunks } from "./bigFile.js"; import { FileChunkUploader } from "./fileChunkUploader.js"; const inp = document.querySelector("input"); inp.onchange = async (e) => { const file = e.target.files[0]; if (!file) return; console.log(file); const timeThread1 = new Date(); const chunksWithMultithreading = await fileCreateChunks(file, { chunkSize: 2 * 1024 * 1024, isMultithreading: true, }); const timeThread2 = new Date(); console.log("时间 with multithread：", timeThread2 - timeThread1); console.log(chunksWithMultithreading); console.log( Array.isArray(chunksWithMultithreading), chunksWithMultithreading.length ); const time1 = new Date(); const chunks = await fileCreateChunks(file, { chunkSize: 2 * 1024 * 1024, isMultithreading: false, }); const time2 = new Date(); console.log("时间：", time2 - time1); console.log(chunks); let isIndex3Error = true; const fileChunkUploader = new FileChunkUploader({ file, beforeSplitChunk: () => { return new Promise((resolve) => { console.log("🚀🚀🚀分片前🚀🚀🚀"); resolve(); }); }, splitChunkDone: () => { console.log("🚀🚀🚀分片完成🚀🚀🚀"); }, uploadFunc: ({ currentChunkIndex, chunk }) => { return new Promise((resolve, reject) => { setTimeout(() => { if (currentChunkIndex === 2 && isIndex3Error) { reject(); } console.log("🚀 upload", currentChunkIndex, chunk); resolve(); }, 2000); }); }, onChunkUploadSuccess: ({ currentChunkIndex, allChunkLength }) => { console.log( "🎉 upload success，进度", `${currentChunkIndex + 1} / ${allChunkLength}` ); }, onChunkUploadError: ({ pauseUpload }) => { return new Promise((resolve, reject) => { pauseUpload(); isIndex3Error = false; resolve(); }); }, onAllChunkUploadDone: () => { console.log("✔✔✔全部完成"); }, }); window.fileChunkUploader = fileChunkUploader; await fileChunkUploader.start(); console.log(fileChunkUploader); };

智算服务

应用商城

合作伙伴

开发者

支持与服务

了解天翼云

前端大文件分片上传和上传流程的封装

文件分片和多线程优化

文件分片

多线程优化

封装文件分片上传流程

文件分片上传类设计

主要功能和方法

总结

前端大文件分片上传和上传流程的封装

文件分片和多线程优化

文件分片

多线程优化

封装文件分片上传流程

文件分片上传类设计

主要功能和方法

总结

活动

智算服务

应用商城

合作伙伴

开发者

支持与服务

了解天翼云

前端大文件分片上传和上传流程的封装

文件分片和多线程优化

文件分片

多线程优化

封装文件分片上传流程

文件分片上传类设计

主要功能和方法

总结

前端大文件分片上传和上传流程的封装

文件分片和多线程优化

文件分片

多线程优化

封装文件分片上传流程

文件分片上传类设计

主要功能和方法

总结