doesnm/quickshare
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(fe/worker): improve uploader robustness and upload files one by one

Browse source
This commit is contained in:
hexxa 2022-01-09 18:34:00 +08:00 • committed by Hexxa
parent 4e9a53574f
commit 46ccddc5f8
2 changed files with 37 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

37
src/client/web/src/worker/chunk_uploader.ts
View file

@ -1,5 +1,10 @@
import { FilesClient } from "../client/files";
import { IFilesClient, Response, isFatalErr } from "../client";
import {
IFilesClient,
Response,
isFatalErr,
UploadStatusResp,
} from "../client";
import { UploadStatus, UploadState } from "./interface";
// TODO: get settings from server
@ -8,8 +13,9 @@ const defaultChunkLen = 1024 * 1024 * 1;
const speedDownRatio = 0.5;
const speedUpRatio = 1.1;
const chunkLimit = 1024 * 1024 * 50; // 50MB
const createRetryLimit = 512;
const createRetryLimit = 1024;
const uploadRetryLimit = 1024;
const readRetryLimit = 8;
const backoffMax = 2000;
export interface ReaderResult {
@ -96,7 +102,7 @@ export class ChunkUploader {
}
);
for (let i = 0; i < 3; i++) {
for (let i = 0; i < readRetryLimit; i++) {
try {
const chunkRightPos =
uploaded + this.chunkLen > file.size
@ -124,12 +130,22 @@ export class ChunkUploader {
}
try {
const uploadResp = await this.client.uploadChunk(
let uploadResp: Response<UploadStatusResp> = undefined;
for (let i = 0; i < uploadRetryLimit; i++) {
uploadResp = await this.client.uploadChunk(
filePath,
result.chunk,
uploaded
);
if (uploadResp.status === 200) {
break;
} else if (uploadResp.status !== 429) {
break;
}
await this.backOff();
}
if (uploadResp.status === 200 && uploadResp.data != null) {
this.chunkLen = Math.ceil(this.chunkLen * speedUpRatio);
return {
@ -152,7 +168,18 @@ export class ChunkUploader {
this.chunkLen = Math.ceil(this.chunkLen * speedDownRatio);
await this.backOff();
const uploadStatusResp = await this.client.uploadStatus(filePath);
let uploadStatusResp: Response<UploadStatusResp> = undefined;
for (let i = 0; i < uploadRetryLimit; i++) {
uploadStatusResp = await this.client.uploadStatus(filePath);
if (uploadStatusResp.status === 200) {
break;
} else if (uploadStatusResp.status !== 429) {
break;
}
await this.backOff();
}
return uploadStatusResp.status === 200
? {
filePath,

12
src/client/web/src/worker/upload_mgr.ts
View file

@ -15,15 +15,12 @@ import {
import { errUploadMgr } from "../common/errors";
import { ErrorLogger } from "../common/log_error";
const win: Window = self as any;
export interface IWorker {
onmessage: (event: MessageEvent) => void;
postMessage: (event: FileWorkerReq) => void;
}
export class UploadMgr {
private idx = 0;
private cycle: number = 500;
private intervalID: number;
private worker: IWorker;
@ -42,15 +39,10 @@ export class UploadMgr {
if (this.infos.size === 0) {
return;
}
if (this.idx > 10000) {
this.idx = 0;
}
const start = this.idx % this.infos.size;
const infos = this.infos.valueSeq().toArray();
for (let i = 0; i < this.infos.size; i++) {
const pos = (start + i) % this.infos.size;
const info = infos[pos];
const info = infos[i];
if (
info.state === UploadState.Ready ||
@ -72,8 +64,6 @@ export class UploadMgr {
break;
}
}
this.idx++;
};
_setInfos = (infos: OrderedMap<string, UploadEntry>) => {