fix(fe/worker): improve uploader robustness and upload files one by one

src/client/web/src/worker/chunk_uploader.ts

@@ -1,5 +1,10 @@
 import { FilesClient } from "../client/files";
-import { IFilesClient, Response, isFatalErr } from "../client";
+import {
+  IFilesClient,
+  Response,
+  isFatalErr,
+  UploadStatusResp,
+} from "../client";
 import { UploadStatus, UploadState } from "./interface";
 
 // TODO: get settings from server
@@ -8,8 +13,9 @@ const defaultChunkLen = 1024 * 1024 * 1;
 const speedDownRatio = 0.5;
 const speedUpRatio = 1.1;
 const chunkLimit = 1024 * 1024 * 50; // 50MB
-const createRetryLimit = 512;
+const createRetryLimit = 1024;
 const uploadRetryLimit = 1024;
+const readRetryLimit = 8;
 const backoffMax = 2000;
 
 export interface ReaderResult {
@@ -96,7 +102,7 @@ export class ChunkUploader {
       }
     );
 
-    for (let i = 0; i < 3; i++) {
+    for (let i = 0; i < readRetryLimit; i++) {
       try {
         const chunkRightPos =
           uploaded + this.chunkLen > file.size
@@ -124,12 +130,22 @@ export class ChunkUploader {
     }
 
     try {
-      const uploadResp = await this.client.uploadChunk(
+      let uploadResp: Response<UploadStatusResp> = undefined;
+      for (let i = 0; i < uploadRetryLimit; i++) {
+        uploadResp = await this.client.uploadChunk(
           filePath,
           result.chunk,
           uploaded
         );
 
+        if (uploadResp.status === 200) {
+          break;
+        } else if (uploadResp.status !== 429) {
+          break;
+        }
+        await this.backOff();
+      }
+
       if (uploadResp.status === 200 && uploadResp.data != null) {
         this.chunkLen = Math.ceil(this.chunkLen * speedUpRatio);
         return {
@@ -152,7 +168,18 @@ export class ChunkUploader {
       this.chunkLen = Math.ceil(this.chunkLen * speedDownRatio);
       await this.backOff();
 
-      const uploadStatusResp = await this.client.uploadStatus(filePath);
+      let uploadStatusResp: Response<UploadStatusResp> = undefined;
+      for (let i = 0; i < uploadRetryLimit; i++) {
+        uploadStatusResp = await this.client.uploadStatus(filePath);
+
+        if (uploadStatusResp.status === 200) {
+          break;
+        } else if (uploadStatusResp.status !== 429) {
+          break;
+        }
+        await this.backOff();
+      }
+
       return uploadStatusResp.status === 200
         ? {
             filePath,

src/client/web/src/worker/upload_mgr.ts

@@ -15,15 +15,12 @@ import {
 import { errUploadMgr } from "../common/errors";
 import { ErrorLogger } from "../common/log_error";
 
-const win: Window = self as any;
-
 export interface IWorker {
   onmessage: (event: MessageEvent) => void;
   postMessage: (event: FileWorkerReq) => void;
 }
 
 export class UploadMgr {
-  private idx = 0;
   private cycle: number = 500;
   private intervalID: number;
   private worker: IWorker;
@@ -42,15 +39,10 @@ export class UploadMgr {
     if (this.infos.size === 0) {
       return;
     }
-    if (this.idx > 10000) {
-      this.idx = 0;
-    }
 
-    const start = this.idx % this.infos.size;
     const infos = this.infos.valueSeq().toArray();
     for (let i = 0; i < this.infos.size; i++) {
-      const pos = (start + i) % this.infos.size;
+      const info = infos[i];
-      const info = infos[pos];
 
       if (
         info.state === UploadState.Ready ||
@@ -72,8 +64,6 @@ export class UploadMgr {
         break;
       }
     }
-
-    this.idx++;
   };
 
   _setInfos = (infos: OrderedMap<string, UploadEntry>) => {