feat(client/worker): support web work in frontend with fallback (#31)
* feat(client/web): add upload mgr * feat(uploader): move uploader to worker * feat(upload_mgr): use native worker for uploading * fix(upload_mgr): fix uploading stop not working * chore(client/web): cleanups * test(upload.worker): add unit test for upload.worker * feat(worker): add foreground upload worker * chore(uploader): turn down the speedup
This commit is contained in:
Hexxa
GitHub
parent
67c07cc81f
commit
31e4850344
27 changed files with 1192 additions and 265 deletions
|
|
@ -29,6 +29,7 @@
|
|||
"@types/deep-diff": "^1.0.0",
|
||||
"@types/jest": "^24.0.12",
|
||||
"assert": "^2.0.0",
|
||||
"babel-loader": "^8.2.2",
|
||||
"css-loader": "^2.1.1",
|
||||
"deep-diff": "^1.0.2",
|
||||
"html-webpack-plugin": "^4.0.0-beta.5",
|
||||
|
|
@ -38,14 +39,16 @@
|
|||
"terser-webpack-plugin": "^1.3.0",
|
||||
"ts-jest": "^26.4.4",
|
||||
"ts-loader": "^6.0.0",
|
||||
"ts-mockito": "^2.6.1",
|
||||
"ts-node": "^8.2.0",
|
||||
"tslint": "^5.16.0",
|
||||
"typescript": "^3.4.3",
|
||||
"typescript": "^4.1.3",
|
||||
"uglifyjs-webpack-plugin": "^2.1.3",
|
||||
"webpack": "^5.0.0-rc.6",
|
||||
"webpack-bundle-analyzer": "^3.3.2",
|
||||
"webpack-cli": "^4.2.0",
|
||||
"webpack-merge": "^4.2.1"
|
||||
"webpack-merge": "^4.2.1",
|
||||
"worker-loader": "^3.0.7"
|
||||
},
|
||||
"dependencies": {
|
||||
"@types/axios": "^0.14.0",
|
||||
|
|
@ -62,7 +65,8 @@
|
|||
"react-copy-to-clipboard": "^5.0.1",
|
||||
"react-dom": "^16.8.6",
|
||||
"react-svg": "^8.0.6",
|
||||
"throttle-debounce": "^2.1.0"
|
||||
"throttle-debounce": "^2.1.0",
|
||||
"worker-loader": "^3.0.7"
|
||||
},
|
||||
"jest": {
|
||||
"testMatch": [
|
||||
|
|
|
|||
|
|
@ -1,14 +1,36 @@
|
|||
import {
|
||||
BaseClient,
|
||||
FatalErrResp,
|
||||
Response,
|
||||
UploadStatusResp,
|
||||
ListResp,
|
||||
ListUploadingsResp,
|
||||
} from "./";
|
||||
|
||||
const filePathQuery = "fp";
|
||||
const listDirQuery = "dp";
|
||||
// TODO: get timeout from server
|
||||
|
||||
function translateResp(resp: Response<any>): Response<any> {
|
||||
if (resp.status === 500) {
|
||||
if (
|
||||
(resp.data == null || resp.data === "") ||
|
||||
(
|
||||
resp.data.error != null &&
|
||||
!resp.data.error.includes("fail to lock the file") &&
|
||||
!resp.data.error.includes("offset != uploaded") &&
|
||||
!resp.data.error.includes("i/o timeout")
|
||||
)
|
||||
) {
|
||||
return FatalErrResp(resp.statusText);
|
||||
}
|
||||
} else if (resp.status === 404) {
|
||||
return FatalErrResp(resp.statusText);
|
||||
}
|
||||
|
||||
return resp;
|
||||
}
|
||||
|
||||
export class FilesClient extends BaseClient {
|
||||
constructor(url: string) {
|
||||
super(url);
|
||||
|
|
@ -79,7 +101,13 @@ export class FilesClient extends BaseClient {
|
|||
content,
|
||||
offset,
|
||||
},
|
||||
});
|
||||
})
|
||||
.then((resp) => {
|
||||
return translateResp(resp);
|
||||
})
|
||||
.catch((e) => {
|
||||
return FatalErrResp(`unknow uploadStatus error ${e.toString()}`);
|
||||
});
|
||||
};
|
||||
|
||||
uploadStatus = (filePath: string): Promise<Response<UploadStatusResp>> => {
|
||||
|
|
@ -89,7 +117,13 @@ export class FilesClient extends BaseClient {
|
|||
params: {
|
||||
[filePathQuery]: filePath,
|
||||
},
|
||||
});
|
||||
})
|
||||
.then((resp) => {
|
||||
return translateResp(resp);
|
||||
})
|
||||
.catch((e) => {
|
||||
return FatalErrResp(`unknow uploadStatus error ${e.toString()}`);
|
||||
});
|
||||
};
|
||||
|
||||
list = (dirPath: string): Promise<Response<ListResp>> => {
|
||||
|
|
@ -101,4 +135,21 @@ export class FilesClient extends BaseClient {
|
|||
},
|
||||
});
|
||||
};
|
||||
|
||||
listUploadings = (): Promise<Response<ListUploadingsResp>> => {
|
||||
return this.do({
|
||||
method: "get",
|
||||
url: `${this.url}/v1/fs/uploadings`,
|
||||
});
|
||||
};
|
||||
|
||||
deleteUploading = (filePath: string): Promise<Response> => {
|
||||
return this.do({
|
||||
method: "delete",
|
||||
url: `${this.url}/v1/fs/uploadings`,
|
||||
params: {
|
||||
[filePathQuery]: filePath,
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import {
|
|||
Response,
|
||||
UploadStatusResp,
|
||||
ListResp,
|
||||
ListUploadingsResp,
|
||||
} from "./";
|
||||
|
||||
export class FilesClient {
|
||||
|
|
@ -18,6 +19,8 @@ export class FilesClient {
|
|||
private uploadStatusMockResps: Array<Promise<Response<UploadStatusResp>>>;
|
||||
private uploadStatusMockRespID: number = 0;
|
||||
private listMockResp: Promise<Response<ListResp>>;
|
||||
private listUploadingsMockResp: Promise<Response<ListUploadingsResp>>;
|
||||
private deleteUploadingMockResp: Promise<Response>;
|
||||
|
||||
constructor(url: string) {
|
||||
this.url = url;
|
||||
|
|
@ -55,6 +58,14 @@ export class FilesClient {
|
|||
this.listMockResp = resp;
|
||||
};
|
||||
|
||||
listUploadingsMock = (resp: Promise<Response<ListUploadingsResp>>) => {
|
||||
this.listUploadingsMockResp = resp;
|
||||
}
|
||||
|
||||
deleteUploadingMock = (resp: Promise<Response>) => {
|
||||
this.deleteUploadingMockResp = resp;
|
||||
}
|
||||
|
||||
create = (filePath: string, fileSize: number): Promise<Response> => {
|
||||
if (this.createMockRespID < this.createMockResps.length) {
|
||||
return this.createMockResps[this.createMockRespID++];
|
||||
|
|
@ -99,4 +110,12 @@ export class FilesClient {
|
|||
list = (dirPath: string): Promise<Response<ListResp>> => {
|
||||
return this.listMockResp;
|
||||
};
|
||||
|
||||
listUploadings = (): Promise<Response<ListUploadingsResp>> => {
|
||||
return this.listUploadingsMockResp;
|
||||
};
|
||||
|
||||
deleteUploading = (filePath:string): Promise<Response<Response>> => {
|
||||
return this.deleteUploadingMockResp;
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,11 +20,21 @@ export interface ListResp {
|
|||
metadatas: MetadataResp[];
|
||||
}
|
||||
|
||||
export interface UploadInfo {
|
||||
realFilePath: string;
|
||||
size: number;
|
||||
uploaded: number; // TODO: use string instead
|
||||
}
|
||||
|
||||
export interface ListUploadingsResp {
|
||||
uploadInfos: UploadInfo[];
|
||||
}
|
||||
|
||||
export interface IUsersClient {
|
||||
login: (user: string, pwd: string) => Promise<Response>
|
||||
logout: () => Promise<Response>
|
||||
isAuthed: () => Promise<Response>
|
||||
setPwd: (oldPwd: string, newPwd: string) => Promise<Response>
|
||||
login: (user: string, pwd: string) => Promise<Response>;
|
||||
logout: () => Promise<Response>;
|
||||
isAuthed: () => Promise<Response>;
|
||||
setPwd: (oldPwd: string, newPwd: string) => Promise<Response>;
|
||||
}
|
||||
|
||||
export interface IFilesClient {
|
||||
|
|
@ -40,6 +50,8 @@ export interface IFilesClient {
|
|||
) => Promise<Response<UploadStatusResp>>;
|
||||
uploadStatus: (filePath: string) => Promise<Response<UploadStatusResp>>;
|
||||
list: (dirPath: string) => Promise<Response<ListResp>>;
|
||||
listUploadings: () => Promise<Response<ListUploadingsResp>>;
|
||||
deleteUploading: (filePath: string) => Promise<Response>;
|
||||
}
|
||||
|
||||
export interface Response<T = any> {
|
||||
|
|
@ -61,7 +73,7 @@ export const EmptyBodyResp: Response<any> = {
|
|||
statusText: "Empty Response Body",
|
||||
};
|
||||
|
||||
export const UnknownErrResp = (errMsg: string): Response<any> => {
|
||||
export const FatalErrResp = (errMsg: string): Response<any> => {
|
||||
return {
|
||||
status: 600,
|
||||
data: {},
|
||||
|
|
@ -69,6 +81,10 @@ export const UnknownErrResp = (errMsg: string): Response<any> => {
|
|||
};
|
||||
};
|
||||
|
||||
export const isFatalErr = (resp: Response<any>): boolean => {
|
||||
return resp.status === 600;
|
||||
};
|
||||
|
||||
export class BaseClient {
|
||||
protected url: string;
|
||||
|
||||
|
|
@ -95,11 +111,15 @@ export class BaseClient {
|
|||
})
|
||||
.catch((e) => {
|
||||
const errMsg = e.toString();
|
||||
|
||||
if (errMsg.includes("ERR_EMPTY")) {
|
||||
// this means connection is eliminated by server because of timeout.
|
||||
// this means connection is eliminated by server, it may be caused by timeout.
|
||||
resolve(EmptyBodyResp);
|
||||
} else if (e.response != null) {
|
||||
resolve(e.response);
|
||||
} else {
|
||||
resolve(UnknownErrResp(errMsg));
|
||||
// TODO: check e.request to get more friendly error message
|
||||
resolve(FatalErrResp(errMsg));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,151 +0,0 @@
|
|||
import { IFilesClient } from "../client";
|
||||
import { FilesClient } from "../client/files";
|
||||
import { Response, UnknownErrResp, UploadStatusResp } from "./";
|
||||
|
||||
// TODO: get settings from server
|
||||
const defaultChunkLen = 1024 * 1024 * 30; // 15MB/s
|
||||
const speedDownRatio = 0.5;
|
||||
const speedUpRatio = 1.1;
|
||||
const retryLimit = 4;
|
||||
|
||||
export class FileUploader {
|
||||
private reader = new FileReader();
|
||||
private client: IFilesClient = new FilesClient("");
|
||||
private chunkLen: number = defaultChunkLen;
|
||||
private file: File;
|
||||
private offset: number;
|
||||
private filePath: string;
|
||||
private errMsg: string | null;
|
||||
private progressCb: (filePath: string, progress: number) => void;
|
||||
|
||||
constructor(
|
||||
file: File,
|
||||
filePath: string,
|
||||
progressCb?: (filePath: string, progress: number) => void
|
||||
) {
|
||||
this.file = file;
|
||||
this.filePath = filePath;
|
||||
this.progressCb = progressCb;
|
||||
this.offset = 0;
|
||||
}
|
||||
|
||||
err = (): string | null => {
|
||||
return this.errMsg;
|
||||
};
|
||||
|
||||
setClient = (client: IFilesClient) => {
|
||||
this.client = client;
|
||||
};
|
||||
|
||||
create = async (filePath: string, fileSize: number): Promise<Response> => {
|
||||
return this.client.create(filePath, fileSize);
|
||||
};
|
||||
|
||||
uploadChunk = async (
|
||||
filePath: string,
|
||||
base64Chunk: string,
|
||||
offset: number
|
||||
): Promise<Response<UploadStatusResp>> => {
|
||||
return this.client.uploadChunk(filePath, base64Chunk, offset);
|
||||
};
|
||||
|
||||
uploadStatus = async (
|
||||
filePath: string
|
||||
): Promise<Response<UploadStatusResp>> => {
|
||||
return this.client.uploadStatus(filePath);
|
||||
};
|
||||
|
||||
start = async (): Promise<boolean> => {
|
||||
let resp: Response;
|
||||
for (let i = 0; i < retryLimit; i++) {
|
||||
resp = await this.create(this.filePath, this.file.size);
|
||||
if (resp.status === 200) {
|
||||
return await this.upload();
|
||||
}
|
||||
}
|
||||
|
||||
this.errMsg = `failed to create ${this.filePath}: status=${resp.statusText}`;
|
||||
return false;
|
||||
};
|
||||
|
||||
upload = async (): Promise<boolean> => {
|
||||
while (
|
||||
this.chunkLen > 0 &&
|
||||
this.offset >= 0 &&
|
||||
this.offset < this.file.size
|
||||
) {
|
||||
const uploadPromise = new Promise<Response<UploadStatusResp>>(
|
||||
(resolve: (resp: Response<UploadStatusResp>) => void) => {
|
||||
this.reader.onerror = (ev: ProgressEvent<FileReader>) => {
|
||||
resolve(UnknownErrResp(this.reader.error.toString()));
|
||||
};
|
||||
|
||||
this.reader.onloadend = (ev: ProgressEvent<FileReader>) => {
|
||||
const dataURL = ev.target.result as string; // readAsDataURL
|
||||
const base64Chunk = dataURL.slice(dataURL.indexOf(",") + 1);
|
||||
this.uploadChunk(this.filePath, base64Chunk, this.offset)
|
||||
.then((resp: Response<UploadStatusResp>) => {
|
||||
resolve(resp);
|
||||
})
|
||||
.catch((e) => {
|
||||
resolve(UnknownErrResp(e.toString()));
|
||||
});
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
const chunkRightPos =
|
||||
this.offset + this.chunkLen > this.file.size
|
||||
? this.file.size
|
||||
: this.offset + this.chunkLen;
|
||||
const blob = this.file.slice(this.offset, chunkRightPos);
|
||||
this.reader.readAsDataURL(blob);
|
||||
|
||||
const uploadResp = await uploadPromise;
|
||||
|
||||
if (uploadResp.status === 200 && uploadResp.data != null) {
|
||||
this.offset = uploadResp.data.uploaded;
|
||||
this.chunkLen = Math.ceil(this.chunkLen * speedUpRatio);
|
||||
} else {
|
||||
this.errMsg = uploadResp.statusText;
|
||||
this.chunkLen = Math.ceil(this.chunkLen * speedDownRatio);
|
||||
|
||||
let uploadStatusResp: Response<UploadStatusResp> = undefined;
|
||||
try {
|
||||
uploadStatusResp = await this.uploadStatus(this.filePath);
|
||||
} catch (e) {
|
||||
if (uploadStatusResp == null) {
|
||||
this.errMsg = `${this.errMsg}; unknown error: empty uploadStatus response`;
|
||||
break;
|
||||
} else if (uploadStatusResp.status === 500) {
|
||||
if (
|
||||
!uploadStatusResp.statusText.includes("fail to lock the file") &&
|
||||
uploadStatusResp.statusText !== ""
|
||||
) {
|
||||
this.errMsg = `${this.errMsg}; unknown error: ${uploadStatusResp.statusText}`;
|
||||
break;
|
||||
}
|
||||
} else if (uploadStatusResp.status === 600) {
|
||||
this.errMsg = `${this.errMsg}; unknown error: ${uploadStatusResp.statusText}`;
|
||||
break;
|
||||
} else {
|
||||
// ignore error and retry
|
||||
}
|
||||
}
|
||||
|
||||
if (uploadStatusResp.status === 200) {
|
||||
this.offset = uploadStatusResp.data.uploaded;
|
||||
}
|
||||
}
|
||||
|
||||
if (this.progressCb != null) {
|
||||
this.progressCb(this.filePath, Math.ceil(this.offset / this.file.size));
|
||||
}
|
||||
}
|
||||
|
||||
if (this.chunkLen === 0) {
|
||||
this.errMsg = "network is bad, please retry later.";
|
||||
}
|
||||
return this.offset === this.file.size;
|
||||
};
|
||||
}
|
||||
|
|
@ -1,9 +1,15 @@
|
|||
import { init } from "../core_state";
|
||||
import { mock, instance } from "ts-mockito";
|
||||
|
||||
import { initWithWorker } from "../core_state";
|
||||
import { Updater } from "../auth_pane";
|
||||
import { MockUsersClient } from "../../client/users_mock";
|
||||
import { Response } from "../../client";
|
||||
import { MockWorker } from "../../worker/interface";
|
||||
|
||||
describe("AuthPane", () => {
|
||||
const mockWorkerClass = mock(MockWorker);
|
||||
const mockWorker = instance(mockWorkerClass);
|
||||
|
||||
const makePromise = (ret: any): Promise<any> => {
|
||||
return new Promise<any>((resolve) => {
|
||||
resolve(ret);
|
||||
|
|
@ -44,7 +50,7 @@ describe("AuthPane", () => {
|
|||
client.isAuthedMock(makeNumberResponse(tc.isAuthedStatus));
|
||||
client.setPwdMock(makeNumberResponse(tc.setPwdStatus));
|
||||
|
||||
const coreState = init();
|
||||
const coreState = initWithWorker(mockWorker);
|
||||
Updater.setClient(client);
|
||||
Updater.init(coreState.panel.authPane);
|
||||
await Updater.initIsAuthed();
|
||||
|
|
|
|||
|
|
@ -1,13 +1,18 @@
|
|||
import { List, Map } from "immutable";
|
||||
import { mock, instance } from "ts-mockito";
|
||||
|
||||
import { init } from "../core_state";
|
||||
import { initWithWorker } from "../core_state";
|
||||
import { makePromise, makeNumberResponse } from "../../test/helpers";
|
||||
import { Updater } from "../browser";
|
||||
import { MockUsersClient } from "../../client/users_mock";
|
||||
import { FilesClient } from "../../client/files_mock";
|
||||
import { MetadataResp } from "../../client";
|
||||
import { MockWorker } from "../../worker/interface";
|
||||
|
||||
describe("Browser", () => {
|
||||
const mockWorkerClass = mock(MockWorker);
|
||||
const mockWorker = instance(mockWorkerClass);
|
||||
|
||||
test("Updater: setPwd", async () => {
|
||||
const tests = [
|
||||
{
|
||||
|
|
@ -43,7 +48,7 @@ describe("Browser", () => {
|
|||
filesClient.listMock(makePromise(tc.listResp));
|
||||
Updater.setClients(usersClient, filesClient);
|
||||
|
||||
const coreState = init();
|
||||
const coreState = initWithWorker(mockWorker);
|
||||
Updater.init(coreState.panel.browser);
|
||||
await Updater.setItems(List<string>(tc.filePath.split("/")));
|
||||
const newState = Updater.setBrowser(coreState);
|
||||
|
|
@ -105,7 +110,7 @@ describe("Browser", () => {
|
|||
filesClient.deleteMock(makeNumberResponse(200));
|
||||
Updater.setClients(usersClient, filesClient);
|
||||
|
||||
const coreState = init();
|
||||
const coreState = initWithWorker(mockWorker);
|
||||
Updater.init(coreState.panel.browser);
|
||||
await Updater.delete(
|
||||
List<string>(tc.dirPath.split("/")),
|
||||
|
|
@ -166,7 +171,7 @@ describe("Browser", () => {
|
|||
filesClient.moveMock(makeNumberResponse(200));
|
||||
Updater.setClients(usersClient, filesClient);
|
||||
|
||||
const coreState = init();
|
||||
const coreState = initWithWorker(mockWorker);
|
||||
Updater.init(coreState.panel.browser);
|
||||
await Updater.moveHere(
|
||||
tc.dirPath1,
|
||||
|
|
|
|||
|
|
@ -1,13 +1,20 @@
|
|||
import * as React from "react";
|
||||
import * as ReactDOM from "react-dom";
|
||||
import { List, Map } from "immutable";
|
||||
import * as Filesize from "filesize";
|
||||
import FileSize from "filesize";
|
||||
|
||||
import { ICoreState } from "./core_state";
|
||||
import { IUsersClient, IFilesClient, MetadataResp } from "../client";
|
||||
import {
|
||||
IUsersClient,
|
||||
IFilesClient,
|
||||
MetadataResp,
|
||||
UploadInfo,
|
||||
} from "../client";
|
||||
import { FilesClient } from "../client/files";
|
||||
import { UsersClient } from "../client/users";
|
||||
import { FileUploader } from "../client/uploader";
|
||||
import { UploadMgr } from "../worker/upload_mgr";
|
||||
import { UploadEntry } from "../worker/interface";
|
||||
// import { FileUploader } from "../worker/uploader";
|
||||
|
||||
export const uploadCheckCycle = 1000;
|
||||
|
||||
|
|
@ -22,6 +29,7 @@ export interface Item {
|
|||
export interface Props {
|
||||
dirPath: List<string>;
|
||||
items: List<MetadataResp>;
|
||||
uploadings: List<UploadInfo>;
|
||||
|
||||
uploadFiles: List<File>;
|
||||
uploadValue: string;
|
||||
|
|
@ -46,9 +54,23 @@ export class Updater {
|
|||
Updater.filesClient = filesClient;
|
||||
}
|
||||
|
||||
static setUploadings = (infos: Map<string, UploadEntry>) => {
|
||||
Updater.props.uploadings = List<UploadInfo>(
|
||||
infos.valueSeq().map(
|
||||
(v: UploadEntry): UploadInfo => {
|
||||
return {
|
||||
realFilePath: v.filePath,
|
||||
size: v.size,
|
||||
uploaded: v.uploaded,
|
||||
};
|
||||
}
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
static setItems = async (dirParts: List<string>): Promise<void> => {
|
||||
let dirPath = dirParts.join("/");
|
||||
let listResp = await Updater.filesClient.list(dirPath);
|
||||
const dirPath = dirParts.join("/");
|
||||
const listResp = await Updater.filesClient.list(dirPath);
|
||||
|
||||
Updater.props.dirPath = dirParts;
|
||||
Updater.props.items =
|
||||
|
|
@ -57,6 +79,26 @@ export class Updater {
|
|||
: Updater.props.items;
|
||||
};
|
||||
|
||||
static refreshUploadings = async (): Promise<boolean> => {
|
||||
const luResp = await Updater.filesClient.listUploadings();
|
||||
|
||||
Updater.props.uploadings =
|
||||
luResp.status === 200
|
||||
? List<UploadInfo>(luResp.data.uploadInfos)
|
||||
: Updater.props.uploadings;
|
||||
return luResp.status === 200;
|
||||
};
|
||||
|
||||
static deleteUploading = async (filePath: string): Promise<boolean> => {
|
||||
UploadMgr.delete(filePath);
|
||||
const resp = await Updater.filesClient.deleteUploading(filePath);
|
||||
return resp.status === 200;
|
||||
};
|
||||
|
||||
static stopUploading = (filePath: string) => {
|
||||
UploadMgr.stop(filePath);
|
||||
};
|
||||
|
||||
static mkDir = async (dirPath: string): Promise<void> => {
|
||||
let resp = await Updater.filesClient.mkdir(dirPath);
|
||||
if (resp.status !== 200) {
|
||||
|
|
@ -120,16 +162,11 @@ export class Updater {
|
|||
};
|
||||
|
||||
static addUploadFiles = (fileList: FileList, len: number) => {
|
||||
let newUploads = List<File>([]);
|
||||
for (let i = 0; i < len; i++) {
|
||||
newUploads = newUploads.push(fileList.item(i));
|
||||
// do not wait for the promise
|
||||
UploadMgr.add(fileList[i], fileList[i].name);
|
||||
}
|
||||
|
||||
Updater.props.uploadFiles = Updater.props.uploadFiles.concat(newUploads);
|
||||
};
|
||||
|
||||
static setUploadFiles = (uploadFiles: List<File>) => {
|
||||
Updater.props.uploadFiles = uploadFiles;
|
||||
Updater.setUploadings(UploadMgr.list());
|
||||
};
|
||||
|
||||
static setBrowser = (prevState: ICoreState): ICoreState => {
|
||||
|
|
@ -154,7 +191,6 @@ export class Browser extends React.Component<Props, State, {}> {
|
|||
private uploadInput: Element | Text;
|
||||
private assignInput: (input: Element) => void;
|
||||
private onClickUpload: () => void;
|
||||
private uploading: boolean;
|
||||
|
||||
constructor(p: Props) {
|
||||
super(p);
|
||||
|
|
@ -176,45 +212,20 @@ export class Browser extends React.Component<Props, State, {}> {
|
|||
this.uploadInput = ReactDOM.findDOMNode(input);
|
||||
};
|
||||
this.onClickUpload = () => {
|
||||
// TODO: check if the re-upload file is same as previous upload
|
||||
const uploadInput = this.uploadInput as HTMLButtonElement;
|
||||
uploadInput.click();
|
||||
};
|
||||
|
||||
Updater.setItems(p.dirPath).then(() => {
|
||||
this.update(Updater.setBrowser);
|
||||
});
|
||||
|
||||
setInterval(this.pollUploads, uploadCheckCycle);
|
||||
}
|
||||
|
||||
pollUploads = () => {
|
||||
if (this.props.uploadFiles.size > 0 && !this.uploading) {
|
||||
this.uploading = true;
|
||||
const file = this.props.uploadFiles.get(0);
|
||||
Updater.setUploadFiles(this.props.uploadFiles.slice(1));
|
||||
this.update(Updater.setBrowser);
|
||||
|
||||
const uploader = new FileUploader(
|
||||
file,
|
||||
`${this.props.dirPath.join("/")}/${file.name}`,
|
||||
this.updateProgress
|
||||
);
|
||||
|
||||
uploader.start().then((ok: boolean) => {
|
||||
Updater.setItems(this.props.dirPath).then(() => {
|
||||
this.update(Updater.setBrowser);
|
||||
});
|
||||
if (!ok) {
|
||||
alert(`upload failed: ${uploader.err()}`);
|
||||
}
|
||||
this.uploading = false;
|
||||
UploadMgr.setStatusCb(this.updateProgress);
|
||||
Updater.setItems(p.dirPath)
|
||||
.then(() => {
|
||||
return Updater.refreshUploadings();
|
||||
})
|
||||
.then((_: boolean) => {
|
||||
this.update(Updater.setBrowser);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
updateProgress = (filePath: string, progress: number) => {
|
||||
// update uploading progress in the core state
|
||||
};
|
||||
}
|
||||
|
||||
showPane = () => {
|
||||
this.setState({ show: !this.state.show });
|
||||
|
|
@ -231,12 +242,6 @@ export class Browser extends React.Component<Props, State, {}> {
|
|||
onInputChange = (ev: React.ChangeEvent<HTMLInputElement>) => {
|
||||
this.setState({ inputValue: ev.target.value });
|
||||
};
|
||||
|
||||
addUploadFile = (event: React.ChangeEvent<HTMLInputElement>) => {
|
||||
Updater.addUploadFiles(event.target.files, event.target.files.length);
|
||||
this.update(Updater.setBrowser);
|
||||
};
|
||||
|
||||
select = (itemName: string) => {
|
||||
const selectedItems = this.state.selectedItems.has(itemName)
|
||||
? this.state.selectedItems.delete(itemName)
|
||||
|
|
@ -248,6 +253,18 @@ export class Browser extends React.Component<Props, State, {}> {
|
|||
});
|
||||
};
|
||||
|
||||
addUploadFile = (event: React.ChangeEvent<HTMLInputElement>) => {
|
||||
Updater.addUploadFiles(event.target.files, event.target.files.length);
|
||||
this.update(Updater.setBrowser);
|
||||
};
|
||||
|
||||
updateProgress = (infos: Map<string, UploadEntry>) => {
|
||||
Updater.setUploadings(infos);
|
||||
Updater.setItems(this.props.dirPath).then(() => {
|
||||
this.update(Updater.setBrowser);
|
||||
});
|
||||
};
|
||||
|
||||
onMkDir = () => {
|
||||
Updater.mkDir(this.state.inputValue)
|
||||
.then(() => {
|
||||
|
|
@ -259,6 +276,24 @@ export class Browser extends React.Component<Props, State, {}> {
|
|||
});
|
||||
};
|
||||
|
||||
deleteUploading = (filePath: string) => {
|
||||
Updater.deleteUploading(filePath)
|
||||
.then((ok: boolean) => {
|
||||
if (!ok) {
|
||||
alert(`Failed to delete uploading ${filePath}`);
|
||||
}
|
||||
return Updater.refreshUploadings();
|
||||
})
|
||||
.then(() => {
|
||||
this.update(Updater.setBrowser);
|
||||
});
|
||||
};
|
||||
|
||||
stopUploading = (filePath: string) => {
|
||||
Updater.stopUploading(filePath);
|
||||
this.update(Updater.setBrowser);
|
||||
};
|
||||
|
||||
delete = () => {
|
||||
if (this.props.dirPath.join("/") !== this.state.selectedSrc) {
|
||||
alert("please select file or folder to delete at first");
|
||||
|
|
@ -460,7 +495,10 @@ export class Browser extends React.Component<Props, State, {}> {
|
|||
: `${dirPath}/${item.name}`;
|
||||
|
||||
return item.isDir ? (
|
||||
<tr key={item.name} className={`${isSelected ? "white0-bg selected" : ""}`}>
|
||||
<tr
|
||||
key={item.name}
|
||||
className={`${isSelected ? "white0-bg selected" : ""}`}
|
||||
>
|
||||
<td className="padding-l-l" style={{ width: "3rem" }}>
|
||||
<span className="dot yellow0-bg"></span>
|
||||
</td>
|
||||
|
|
@ -485,7 +523,10 @@ export class Browser extends React.Component<Props, State, {}> {
|
|||
</td>
|
||||
</tr>
|
||||
) : (
|
||||
<tr key={item.name} className={`${isSelected ? "white0-bg selected" : ""}`}>
|
||||
<tr
|
||||
key={item.name}
|
||||
className={`${isSelected ? "white0-bg selected" : ""}`}
|
||||
>
|
||||
<td className="padding-l-l" style={{ width: "3rem" }}>
|
||||
<span className="dot green0-bg"></span>
|
||||
</td>
|
||||
|
|
@ -498,7 +539,7 @@ export class Browser extends React.Component<Props, State, {}> {
|
|||
{item.name}
|
||||
</a>
|
||||
</td>
|
||||
<td>{Filesize(item.size, {round: 0})}</td>
|
||||
<td>{FileSize(item.size, { round: 0 })}</td>
|
||||
<td>{item.modTime.slice(0, item.modTime.indexOf("T"))}</td>
|
||||
|
||||
<td>
|
||||
|
|
@ -514,6 +555,38 @@ export class Browser extends React.Component<Props, State, {}> {
|
|||
);
|
||||
});
|
||||
|
||||
const uploadingList = this.props.uploadings.map((uploading: UploadInfo) => {
|
||||
const pathParts = uploading.realFilePath.split("/");
|
||||
const fileName = pathParts[pathParts.length - 1];
|
||||
|
||||
return (
|
||||
<tr key={fileName}>
|
||||
<td className="padding-l-l" style={{ width: "3rem" }}>
|
||||
<span className="dot blue0-bg"></span>
|
||||
</td>
|
||||
<td>
|
||||
<span className="item-name pointer">{fileName}</span>
|
||||
</td>
|
||||
<td>{FileSize(uploading.uploaded, { round: 0 })}</td>
|
||||
<td>{FileSize(uploading.size, { round: 0 })}</td>
|
||||
<td>
|
||||
<button
|
||||
onClick={() => this.stopUploading(uploading.realFilePath)}
|
||||
className="white-font margin-m"
|
||||
>
|
||||
Stop
|
||||
</button>
|
||||
<button
|
||||
onClick={() => this.deleteUploading(uploading.realFilePath)}
|
||||
className="white-font margin-m"
|
||||
>
|
||||
Delete
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
});
|
||||
|
||||
return (
|
||||
<div>
|
||||
<div id="op-bar" className="op-bar">
|
||||
|
|
@ -522,6 +595,31 @@ export class Browser extends React.Component<Props, State, {}> {
|
|||
|
||||
<div id="item-list" className="">
|
||||
<div className="margin-b-l">{breadcrumb}</div>
|
||||
|
||||
<table>
|
||||
<thead style={{ fontWeight: "bold" }}>
|
||||
<tr>
|
||||
<td className="padding-l-l" style={{ width: "3rem" }}>
|
||||
<span className="dot black-bg"></span>
|
||||
</td>
|
||||
<td>Name</td>
|
||||
<td>Uploaded</td>
|
||||
<td>Size</td>
|
||||
<td>Action</td>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>{uploadingList}</tbody>
|
||||
<tfoot>
|
||||
<tr>
|
||||
<td></td>
|
||||
<td></td>
|
||||
<td></td>
|
||||
<td></td>
|
||||
<td></td>
|
||||
</tr>
|
||||
</tfoot>
|
||||
</table>
|
||||
|
||||
<table>
|
||||
<thead style={{ fontWeight: "bold" }}>
|
||||
<tr>
|
||||
|
|
|
|||
|
|
@ -1,8 +1,12 @@
|
|||
import { List } from "immutable";
|
||||
|
||||
import BgWorker from "../worker/upload.bg.worker";
|
||||
import { FgWorker } from "../worker/upload.fgworker";
|
||||
|
||||
import { Props as PanelProps } from "./panel";
|
||||
import { Item } from "./browser";
|
||||
|
||||
import { UploadInfo } from "../client";
|
||||
import { UploadMgr, IWorker } from "../worker/upload_mgr";
|
||||
|
||||
export interface IContext {
|
||||
update: (targetStatePatch: any) => void;
|
||||
|
|
@ -13,7 +17,23 @@ export interface ICoreState {
|
|||
panel: PanelProps;
|
||||
}
|
||||
|
||||
export function initWithWorker(worker: IWorker): ICoreState {
|
||||
UploadMgr.init(worker);
|
||||
return initState();
|
||||
}
|
||||
|
||||
export function init(): ICoreState {
|
||||
const scripts = Array.from(document.querySelectorAll("script"));
|
||||
if (!Worker) {
|
||||
alert("web worker is not supported");
|
||||
}
|
||||
|
||||
const worker = new BgWorker();
|
||||
UploadMgr.init(worker);
|
||||
return initState();
|
||||
}
|
||||
|
||||
export function initState(): ICoreState {
|
||||
return {
|
||||
ctx: null,
|
||||
panel: {
|
||||
|
|
@ -24,6 +44,7 @@ export function init(): ICoreState {
|
|||
browser: {
|
||||
dirPath: List<string>(["."]),
|
||||
items: List<Item>([]),
|
||||
uploadings: List<UploadInfo>([]),
|
||||
uploadValue: "",
|
||||
uploadFiles: List<File>([]),
|
||||
},
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ export class Panel extends React.Component<Props, State, {}> {
|
|||
<Browser
|
||||
dirPath={this.props.browser.dirPath}
|
||||
items={this.props.browser.items}
|
||||
uploadings={this.props.browser.uploadings}
|
||||
update={this.update}
|
||||
uploadFiles={this.props.browser.uploadFiles}
|
||||
uploadValue={this.props.browser.uploadValue}
|
||||
|
|
|
|||
7
src/client/web/src/typings/custom.d.ts
vendored
Normal file
7
src/client/web/src/typings/custom.d.ts
vendored
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
declare module "worker-loader!*" {
|
||||
class UploadWorker extends Worker {
|
||||
constructor();
|
||||
}
|
||||
|
||||
export = UploadWorker;
|
||||
}
|
||||
97
src/client/web/src/worker/__test__/upload.worker.test.ts
Normal file
97
src/client/web/src/worker/__test__/upload.worker.test.ts
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
import { mock, instance, when } from "ts-mockito";
|
||||
|
||||
import { UploadWorker } from "../upload.baseworker";
|
||||
import { FileUploader } from "../uploader";
|
||||
import { FileWorkerResp, UploadEntry, syncReqKind } from "../interface";
|
||||
|
||||
describe("upload.worker", () => {
|
||||
const content = ["123456"];
|
||||
const filePath = "mock/file";
|
||||
const blob = new Blob(content);
|
||||
const fileSize = blob.size;
|
||||
const file = new File(content, filePath);
|
||||
|
||||
const makeEntry = (filePath: string, runnable: boolean): UploadEntry => {
|
||||
return {
|
||||
file: file,
|
||||
filePath,
|
||||
size: fileSize,
|
||||
uploaded: 0,
|
||||
runnable,
|
||||
err: "",
|
||||
};
|
||||
};
|
||||
|
||||
test("onMsg:syncReqKind: filter list and start uploading correct file", async () => {
|
||||
const mockUploaderClass = mock(FileUploader);
|
||||
when(mockUploaderClass.start()).thenCall(
|
||||
(): Promise<boolean> => {
|
||||
return new Promise((resolve) => resolve(true));
|
||||
}
|
||||
);
|
||||
when(mockUploaderClass.stop()).thenCall(() => {});
|
||||
|
||||
let currentUploader: FileUploader = undefined;
|
||||
let uploaderFile: File = undefined;
|
||||
let uploaderFilePath: string = undefined;
|
||||
let uploaderStopFilePath: string = undefined;
|
||||
|
||||
interface TestCase {
|
||||
infos: Array<UploadEntry>;
|
||||
expectedUploadingFile: string;
|
||||
expectedUploaderStartInput: string;
|
||||
currentFilePath: string;
|
||||
}
|
||||
|
||||
const tcs: Array<TestCase> = [
|
||||
{
|
||||
infos: [makeEntry("file1", true), makeEntry("file2", true)],
|
||||
expectedUploadingFile: "file1",
|
||||
expectedUploaderStartInput: "file1",
|
||||
currentFilePath: "",
|
||||
},
|
||||
{
|
||||
infos: [makeEntry("file1", false), makeEntry("file2", true)],
|
||||
expectedUploadingFile: "file2",
|
||||
expectedUploaderStartInput: "file2",
|
||||
currentFilePath: "",
|
||||
},
|
||||
{
|
||||
infos: [makeEntry("file1", true), makeEntry("file0", true)],
|
||||
expectedUploadingFile: "file1",
|
||||
expectedUploaderStartInput: "file1",
|
||||
currentFilePath: "file0",
|
||||
},
|
||||
];
|
||||
|
||||
for (let i = 0; i < tcs.length; i++) {
|
||||
const uploadWorker = new UploadWorker();
|
||||
uploadWorker.sendEvent = (_: FileWorkerResp) => {};
|
||||
uploadWorker.makeUploader = (
|
||||
file: File,
|
||||
filePath: string
|
||||
): FileUploader => {
|
||||
uploaderFile = file;
|
||||
uploaderFilePath = filePath;
|
||||
currentUploader = instance(mockUploaderClass);
|
||||
return currentUploader;
|
||||
};
|
||||
|
||||
if (tcs[i].currentFilePath !== "") {
|
||||
uploadWorker.setFilePath(tcs[i].currentFilePath);
|
||||
}
|
||||
const req = {
|
||||
kind: syncReqKind,
|
||||
infos: tcs[i].infos,
|
||||
};
|
||||
|
||||
uploadWorker.onMsg(
|
||||
new MessageEvent("worker", {
|
||||
data: req,
|
||||
})
|
||||
);
|
||||
expect(uploadWorker.getFilePath()).toEqual(tcs[i].expectedUploadingFile);
|
||||
expect(uploaderFilePath).toEqual(tcs[i].expectedUploaderStartInput);
|
||||
}
|
||||
});
|
||||
});
|
||||
141
src/client/web/src/worker/__test__/upload_mgr.test.ts
Normal file
141
src/client/web/src/worker/__test__/upload_mgr.test.ts
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
import { Map } from "immutable";
|
||||
import { mock, instance, when, anything } from "ts-mockito";
|
||||
|
||||
import { FilesClient } from "../../client/files_mock";
|
||||
import { makePromise } from "../../test/helpers";
|
||||
|
||||
import { UploadMgr } from "../upload_mgr";
|
||||
|
||||
import {
|
||||
FileWorkerReq,
|
||||
UploadEntry,
|
||||
uploadInfoKind,
|
||||
syncReqKind,
|
||||
SyncReq,
|
||||
} from "../interface";
|
||||
|
||||
function arraytoMap(infos: Array<UploadEntry>): Map<string, UploadEntry> {
|
||||
let map = Map<string, UploadEntry>();
|
||||
infos.forEach((info) => {
|
||||
map = map.set(info.filePath, info);
|
||||
});
|
||||
return map;
|
||||
}
|
||||
|
||||
const delay = (ms: number): Promise<void> => {
|
||||
return new Promise<void>((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
};
|
||||
|
||||
describe("UploadMgr", () => {
|
||||
const content = ["123456"];
|
||||
const filePath = "mock/file";
|
||||
const blob = new Blob(content);
|
||||
const fileSize = blob.size;
|
||||
const file = new File(content, filePath);
|
||||
const makeInfo = (filePath: string, runnable: boolean): UploadEntry => {
|
||||
return {
|
||||
file: file,
|
||||
filePath: filePath,
|
||||
size: fileSize,
|
||||
uploaded: 0,
|
||||
runnable,
|
||||
err: "",
|
||||
};
|
||||
};
|
||||
|
||||
test("test init and respHandler: pick up tasks and remove them after done", async () => {
|
||||
interface TestCase {
|
||||
inputInfos: Array<UploadEntry>;
|
||||
expectedInfos: Array<UploadEntry>;
|
||||
}
|
||||
|
||||
class MockWorker {
|
||||
constructor() {}
|
||||
onmessage = (event: MessageEvent): void => {};
|
||||
postMessage = (req: FileWorkerReq): void => {
|
||||
switch (req.kind) {
|
||||
case syncReqKind:
|
||||
const syncReq = req as SyncReq;
|
||||
// find the first qualified task
|
||||
const infoArray = syncReq.infos;
|
||||
for (let i = 0; i < infoArray.length; i++) {
|
||||
if (
|
||||
infoArray[i].runnable &&
|
||||
infoArray[i].uploaded < infoArray[i].size
|
||||
) {
|
||||
this.onmessage(
|
||||
new MessageEvent("worker", {
|
||||
data: {
|
||||
kind: uploadInfoKind,
|
||||
filePath: infoArray[i].filePath,
|
||||
uploaded: infoArray[i].size,
|
||||
runnable: true,
|
||||
err: "",
|
||||
},
|
||||
})
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
default:
|
||||
throw Error(
|
||||
`unknown worker request ${req.kind} ${req.kind === syncReqKind}`
|
||||
);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
const tcs: Array<TestCase> = [
|
||||
{
|
||||
inputInfos: [
|
||||
makeInfo("path1/file1", true),
|
||||
makeInfo("path2/file1", true),
|
||||
],
|
||||
expectedInfos: [],
|
||||
},
|
||||
{
|
||||
inputInfos: [
|
||||
makeInfo("path1/file1", true),
|
||||
makeInfo("path2/file1", false),
|
||||
],
|
||||
expectedInfos: [makeInfo("path2/file1", false)],
|
||||
},
|
||||
{
|
||||
inputInfos: [
|
||||
makeInfo("path1/file1", false),
|
||||
makeInfo("path2/file1", true),
|
||||
],
|
||||
expectedInfos: [
|
||||
makeInfo("path1/file1", false),
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const worker = new MockWorker();
|
||||
UploadMgr.setCycle(100);
|
||||
|
||||
for (let i = 0; i < tcs.length; i++) {
|
||||
const infoMap = arraytoMap(tcs[i].inputInfos);
|
||||
UploadMgr._setInfos(infoMap);
|
||||
|
||||
UploadMgr.init(worker);
|
||||
// polling needs several rounds to finish all the tasks
|
||||
await delay(tcs.length * UploadMgr.getCycle() + 1000);
|
||||
// TODO: find a better way to wait
|
||||
const gotInfos = UploadMgr.list();
|
||||
|
||||
const expectedInfoMap = arraytoMap(tcs[i].expectedInfos);
|
||||
gotInfos.keySeq().forEach((filePath) => {
|
||||
expect(gotInfos.get(filePath)).toEqual(expectedInfoMap.get(filePath));
|
||||
});
|
||||
expectedInfoMap.keySeq().forEach((filePath) => {
|
||||
expect(expectedInfoMap.get(filePath)).toEqual(gotInfos.get(filePath));
|
||||
});
|
||||
|
||||
UploadMgr.destory();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { FileUploader } from "../uploader";
|
||||
import { FilesClient } from "../files_mock";
|
||||
import { FilesClient } from "../../client/files_mock";
|
||||
import { makePromise } from "../../test/helpers";
|
||||
|
||||
describe("Uploader", () => {
|
||||
|
|
@ -37,12 +37,14 @@ describe("Uploader", () => {
|
|||
status: number;
|
||||
uploaded: number;
|
||||
}
|
||||
|
||||
interface TestCase {
|
||||
createResps: Array<number>;
|
||||
uploadChunkResps: Array<any>;
|
||||
uploadStatusResps: Array<any>;
|
||||
result: boolean;
|
||||
}
|
||||
|
||||
test("test start and upload method", async () => {
|
||||
const testCases: Array<TestCase> = [
|
||||
{
|
||||
|
|
@ -83,7 +85,14 @@ describe("Uploader", () => {
|
|||
},
|
||||
{
|
||||
// fail twice
|
||||
createResps: [500, 500, 500, 200],
|
||||
createResps: [500, 500],
|
||||
uploadChunkResps: [],
|
||||
uploadStatusResps: [],
|
||||
result: false,
|
||||
},
|
||||
{
|
||||
// fail twice
|
||||
createResps: [500, 200],
|
||||
uploadChunkResps: [
|
||||
{ status: 200, uploaded: 0 },
|
||||
{ status: 500, uploaded: 1 },
|
||||
|
|
@ -115,7 +124,13 @@ describe("Uploader", () => {
|
|||
|
||||
for (let i = 0; i < testCases.length; i++) {
|
||||
const tc = testCases[i];
|
||||
const uploader = new FileUploader(file, filePath);
|
||||
const mockCb = (
|
||||
filePath: string,
|
||||
uploaded: number,
|
||||
done: boolean,
|
||||
err: string
|
||||
):void => {};
|
||||
const uploader = new FileUploader(file, filePath, mockCb);
|
||||
const mockClient = new FilesClient("");
|
||||
|
||||
const createResps = tc.createResps.map((resp) => makeCreateResp(resp));
|
||||
49
src/client/web/src/worker/interface.ts
Normal file
49
src/client/web/src/worker/interface.ts
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
export interface UploadEntry {
|
||||
file: File;
|
||||
filePath: string;
|
||||
size: number;
|
||||
uploaded: number;
|
||||
runnable: boolean;
|
||||
err: string;
|
||||
}
|
||||
|
||||
export type eventKind = SyncReqKind | ErrKind | UploadInfoKind;
|
||||
export interface WorkerEvent {
|
||||
kind: eventKind;
|
||||
}
|
||||
|
||||
export type SyncReqKind = "worker.req.sync";
|
||||
export const syncReqKind: SyncReqKind = "worker.req.sync";
|
||||
|
||||
export interface SyncReq extends WorkerEvent {
|
||||
kind: SyncReqKind;
|
||||
infos: Array<UploadEntry>;
|
||||
}
|
||||
|
||||
export type FileWorkerReq = SyncReq;
|
||||
|
||||
export type ErrKind = "worker.resp.err";
|
||||
export const errKind: ErrKind = "worker.resp.err";
|
||||
export interface ErrResp extends WorkerEvent {
|
||||
kind: ErrKind;
|
||||
err: string;
|
||||
}
|
||||
|
||||
// caller should combine uploaded and done to see if the upload is successfully finished
|
||||
export type UploadInfoKind = "worker.resp.info";
|
||||
export const uploadInfoKind: UploadInfoKind = "worker.resp.info";
|
||||
export interface UploadInfoResp extends WorkerEvent {
|
||||
kind: UploadInfoKind;
|
||||
filePath: string;
|
||||
uploaded: number;
|
||||
runnable: boolean;
|
||||
err: string;
|
||||
}
|
||||
|
||||
export type FileWorkerResp = ErrResp | UploadInfoResp;
|
||||
|
||||
export class MockWorker {
|
||||
constructor() {}
|
||||
onmessage = (event: MessageEvent): void => {};
|
||||
postMessage = (event: FileWorkerReq): void => {};
|
||||
}
|
||||
93
src/client/web/src/worker/upload.baseworker.ts
Normal file
93
src/client/web/src/worker/upload.baseworker.ts
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
import { FileUploader } from "./uploader";
|
||||
import {
|
||||
FileWorkerReq,
|
||||
syncReqKind,
|
||||
SyncReq,
|
||||
errKind,
|
||||
ErrResp,
|
||||
uploadInfoKind,
|
||||
UploadInfoResp,
|
||||
FileWorkerResp,
|
||||
} from "./interface";
|
||||
|
||||
export class UploadWorker {
|
||||
private file: File = undefined;
|
||||
private filePath: string = undefined;
|
||||
private uploader: FileUploader = undefined;
|
||||
sendEvent = (resp: FileWorkerResp):void => {
|
||||
// TODO: make this abstract
|
||||
throw new Error("not implemented");
|
||||
};
|
||||
makeUploader = (file: File, filePath: string): FileUploader => {
|
||||
return new FileUploader(file, filePath, this.onCb);
|
||||
};
|
||||
startUploader = (file: File, filePath: string) => {
|
||||
this.file = file;
|
||||
this.filePath = filePath;
|
||||
this.uploader = this.makeUploader(file, filePath);
|
||||
this.uploader.start();
|
||||
};
|
||||
stopUploader = () => {
|
||||
if (this.uploader != null) {
|
||||
this.uploader.stop();
|
||||
}
|
||||
};
|
||||
getFilePath = (): string => {
|
||||
return this.filePath;
|
||||
};
|
||||
|
||||
setFilePath = (fp: string) => {
|
||||
this.filePath = fp;
|
||||
};
|
||||
|
||||
constructor() {}
|
||||
|
||||
onMsg = (event: MessageEvent) => {
|
||||
const req = event.data as FileWorkerReq;
|
||||
switch (req.kind) {
|
||||
case syncReqKind:
|
||||
// find the first qualified task
|
||||
const syncReq = req as SyncReq;
|
||||
const infoArray = syncReq.infos;
|
||||
for (let i = 0; i < infoArray.length; i++) {
|
||||
if (
|
||||
infoArray[i].runnable &&
|
||||
infoArray[i].uploaded < infoArray[i].size
|
||||
) {
|
||||
if (infoArray[i].filePath !== this.filePath) {
|
||||
this.stopUploader();
|
||||
this.startUploader(infoArray[i].file, infoArray[i].filePath);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
default:
|
||||
console.log(`unknown worker request(${JSON.stringify(req)})`);
|
||||
}
|
||||
};
|
||||
|
||||
onError = (ev: ErrorEvent) => {
|
||||
const errResp: ErrResp = {
|
||||
kind: errKind,
|
||||
err: ev.error,
|
||||
};
|
||||
this.sendEvent(errResp);
|
||||
};
|
||||
|
||||
onCb = (
|
||||
filePath: string,
|
||||
uploaded: number,
|
||||
runnable: boolean,
|
||||
err: string
|
||||
): void => {
|
||||
const uploadInfoResp: UploadInfoResp = {
|
||||
kind: uploadInfoKind,
|
||||
filePath,
|
||||
uploaded,
|
||||
runnable,
|
||||
err,
|
||||
};
|
||||
this.sendEvent(uploadInfoResp);
|
||||
};
|
||||
}
|
||||
20
src/client/web/src/worker/upload.bg.worker.ts
Normal file
20
src/client/web/src/worker/upload.bg.worker.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import { UploadWorker } from "./upload.baseworker";
|
||||
import { FileWorkerResp } from "./interface";
|
||||
|
||||
const ctx: Worker = self as any;
|
||||
|
||||
class BgWorker extends UploadWorker {
|
||||
constructor() {
|
||||
super();
|
||||
}
|
||||
|
||||
sendEvent = (resp: FileWorkerResp): void => {
|
||||
ctx.postMessage(resp);
|
||||
};
|
||||
}
|
||||
|
||||
const worker = new BgWorker();
|
||||
ctx.addEventListener("message", worker.onMsg);
|
||||
ctx.addEventListener("error", worker.onError);
|
||||
|
||||
export default null as any;
|
||||
27
src/client/web/src/worker/upload.fgworker.ts
Normal file
27
src/client/web/src/worker/upload.fgworker.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
import { UploadWorker } from "./upload.baseworker";
|
||||
import { FileWorkerReq, FileWorkerResp } from "./interface";
|
||||
|
||||
export class FgWorker extends UploadWorker {
|
||||
constructor() {
|
||||
super();
|
||||
}
|
||||
|
||||
// provide interfaces for non-worker mode
|
||||
onmessage = (event: MessageEvent): void => {};
|
||||
|
||||
sendEvent = (resp: FileWorkerResp) => {
|
||||
this.onmessage(
|
||||
new MessageEvent("worker", {
|
||||
data: resp,
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
postMessage = (req: FileWorkerReq): void => {
|
||||
this.onMsg(
|
||||
new MessageEvent("worker", {
|
||||
data: req,
|
||||
})
|
||||
);
|
||||
};
|
||||
}
|
||||
145
src/client/web/src/worker/upload_mgr.ts
Normal file
145
src/client/web/src/worker/upload_mgr.ts
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
import { Map } from "immutable";
|
||||
|
||||
import {
|
||||
FileWorkerReq,
|
||||
FileWorkerResp,
|
||||
UploadInfoResp,
|
||||
ErrResp,
|
||||
UploadEntry,
|
||||
syncReqKind,
|
||||
errKind,
|
||||
uploadInfoKind,
|
||||
} from "./interface";
|
||||
|
||||
const win = self as any;
|
||||
|
||||
export interface IWorker {
|
||||
onmessage: (event: MessageEvent) => void;
|
||||
postMessage: (event: FileWorkerReq) => void;
|
||||
}
|
||||
|
||||
export class UploadMgr {
|
||||
private static infos = Map<string, UploadEntry>();
|
||||
private static worker: IWorker;
|
||||
private static intervalID: NodeJS.Timeout;
|
||||
private static cycle: number = 500;
|
||||
private static statusCb = (infos: Map<string, UploadEntry>):void => {};
|
||||
|
||||
static _setInfos = (infos: Map<string, UploadEntry>) => {
|
||||
UploadMgr.infos = infos;
|
||||
};
|
||||
|
||||
static setCycle = (ms: number) => {
|
||||
UploadMgr.cycle = ms;
|
||||
};
|
||||
|
||||
static getCycle = (): number => {
|
||||
return UploadMgr.cycle;
|
||||
};
|
||||
|
||||
static setStatusCb = (cb: (infos: Map<string, UploadEntry>) => void) => {
|
||||
UploadMgr.statusCb = cb;
|
||||
}
|
||||
|
||||
static init = (worker: IWorker) => {
|
||||
UploadMgr.worker = worker;
|
||||
// TODO: fallback to normal if Web Worker is not available
|
||||
UploadMgr.worker.onmessage = UploadMgr.respHandler;
|
||||
|
||||
const syncing = () => {
|
||||
UploadMgr.worker.postMessage({
|
||||
kind: syncReqKind,
|
||||
infos: UploadMgr.infos.valueSeq().toArray(),
|
||||
});
|
||||
};
|
||||
UploadMgr.intervalID = win.setInterval(syncing, UploadMgr.cycle);
|
||||
};
|
||||
|
||||
static destory = () => {
|
||||
win.clearInterval(UploadMgr.intervalID);
|
||||
};
|
||||
|
||||
static add = (file: File, filePath: string) => {
|
||||
const entry = UploadMgr.infos.get(filePath);
|
||||
if (entry == null) {
|
||||
// new uploading
|
||||
UploadMgr.infos = UploadMgr.infos.set(filePath, {
|
||||
file: file,
|
||||
filePath: filePath,
|
||||
size: file.size,
|
||||
uploaded: 0,
|
||||
runnable: true,
|
||||
err: "",
|
||||
});
|
||||
} else {
|
||||
// restart the uploading
|
||||
UploadMgr.infos = UploadMgr.infos.set(filePath, {
|
||||
...entry,
|
||||
runnable: true,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
static stop = (filePath: string) => {
|
||||
const entry = UploadMgr.infos.get(filePath);
|
||||
if (entry != null) {
|
||||
UploadMgr.infos = UploadMgr.infos.set(filePath, {
|
||||
...entry,
|
||||
runnable: false,
|
||||
});
|
||||
console.log("stopped", filePath);
|
||||
} else {
|
||||
alert(`failed to stop uploading ${filePath}: not found`);
|
||||
}
|
||||
};
|
||||
|
||||
static delete = (filePath: string) => {
|
||||
UploadMgr.stop(filePath);
|
||||
UploadMgr.infos = UploadMgr.infos.delete(filePath);
|
||||
};
|
||||
|
||||
static list = (): Map<string, UploadEntry> => {
|
||||
return UploadMgr.infos;
|
||||
};
|
||||
|
||||
static respHandler = (event: MessageEvent) => {
|
||||
const resp = event.data as FileWorkerResp;
|
||||
|
||||
switch (resp.kind) {
|
||||
case errKind:
|
||||
// TODO: refine this
|
||||
const errResp = resp as ErrResp;
|
||||
console.error(`respHandler: ${errResp}`);
|
||||
break;
|
||||
case uploadInfoKind:
|
||||
const infoResp = resp as UploadInfoResp;
|
||||
const entry = UploadMgr.infos.get(infoResp.filePath);
|
||||
|
||||
if (entry != null) {
|
||||
if (infoResp.uploaded === entry.size) {
|
||||
UploadMgr.infos = UploadMgr.infos.delete(infoResp.filePath);
|
||||
} else {
|
||||
UploadMgr.infos = UploadMgr.infos.set(infoResp.filePath, {
|
||||
...entry,
|
||||
uploaded: infoResp.uploaded,
|
||||
runnable: infoResp.runnable,
|
||||
err: infoResp.err,
|
||||
});
|
||||
}
|
||||
|
||||
// call back to update the info
|
||||
UploadMgr.statusCb(UploadMgr.infos);
|
||||
} else {
|
||||
// TODO: refine this
|
||||
console.error(
|
||||
`respHandler: fail to found: file(${
|
||||
infoResp.filePath
|
||||
}) infos(${UploadMgr.infos.toObject()})`
|
||||
);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
console.error(`respHandler: response kind not found: ${resp}`);
|
||||
}
|
||||
};
|
||||
}
|
||||
176
src/client/web/src/worker/uploader.ts
Normal file
176
src/client/web/src/worker/uploader.ts
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
import { FilesClient } from "../client/files";
|
||||
import { IFilesClient, Response, isFatalErr } from "../client";
|
||||
|
||||
// TODO: get settings from server
|
||||
// TODO: move chunk copying to worker
|
||||
const defaultChunkLen = 1024 * 1024 * 1;
|
||||
const speedDownRatio = 0.5;
|
||||
const speedUpRatio = 1.05;
|
||||
const createRetryLimit = 2;
|
||||
const uploadRetryLimit = 1024;
|
||||
|
||||
export interface IFileUploader {
|
||||
stop: () => void;
|
||||
err: () => string | null;
|
||||
setClient: (client: IFilesClient) => void;
|
||||
start: () => Promise<boolean>;
|
||||
upload: () => Promise<boolean>;
|
||||
}
|
||||
|
||||
export interface ReaderResult {
|
||||
chunk?: string;
|
||||
err?: Error;
|
||||
}
|
||||
|
||||
export class FileUploader {
|
||||
private reader = new FileReader();
|
||||
private client: IFilesClient = new FilesClient("");
|
||||
|
||||
private file: File;
|
||||
private filePath: string;
|
||||
private chunkLen: number = defaultChunkLen;
|
||||
private offset: number = 0;
|
||||
|
||||
private errMsgs: string[] = new Array<string>();
|
||||
private isOn: boolean = true;
|
||||
private progressCb: (
|
||||
filePath: string,
|
||||
uploaded: number,
|
||||
runnable: boolean,
|
||||
err: string
|
||||
) => void;
|
||||
|
||||
constructor(
|
||||
file: File,
|
||||
filePath: string,
|
||||
progressCb: (
|
||||
filePath: string,
|
||||
uploaded: number,
|
||||
runnable: boolean,
|
||||
err: string
|
||||
) => void
|
||||
) {
|
||||
this.file = file;
|
||||
this.filePath = filePath;
|
||||
this.progressCb = progressCb;
|
||||
}
|
||||
|
||||
getOffset = (): number => {
|
||||
return this.offset;
|
||||
};
|
||||
|
||||
stop = () => {
|
||||
this.isOn = false;
|
||||
};
|
||||
|
||||
err = (): string | null => {
|
||||
return this.errMsgs.length === 0 ? null : this.errMsgs.reverse().join(";");
|
||||
};
|
||||
|
||||
setClient = (client: IFilesClient) => {
|
||||
this.client = client;
|
||||
};
|
||||
|
||||
start = async (): Promise<boolean> => {
|
||||
let resp: Response;
|
||||
|
||||
for (let i = 0; i < createRetryLimit; i++) {
|
||||
try {
|
||||
resp = await this.client.create(this.filePath, this.file.size);
|
||||
if (resp.status === 200 || resp.status === 304) {
|
||||
return await this.upload();
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
this.errMsgs.push(
|
||||
`failed to create ${this.filePath}: status=${resp.statusText}`
|
||||
);
|
||||
return false;
|
||||
};
|
||||
|
||||
upload = async (): Promise<boolean> => {
|
||||
while (
|
||||
this.chunkLen > 0 &&
|
||||
this.offset >= 0 &&
|
||||
this.offset < this.file.size &&
|
||||
this.isOn
|
||||
) {
|
||||
const readerPromise = new Promise<ReaderResult>(
|
||||
(resolve: (result: ReaderResult) => void) => {
|
||||
this.reader.onerror = (_: ProgressEvent<FileReader>) => {
|
||||
resolve({ err: this.reader.error });
|
||||
};
|
||||
|
||||
this.reader.onloadend = (ev: ProgressEvent<FileReader>) => {
|
||||
const dataURL = ev.target.result as string; // readAsDataURL
|
||||
const base64Chunk = dataURL.slice(dataURL.indexOf(",") + 1); // remove prefix
|
||||
resolve({ chunk: base64Chunk });
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
const chunkRightPos =
|
||||
this.offset + this.chunkLen > this.file.size
|
||||
? this.file.size
|
||||
: this.offset + this.chunkLen;
|
||||
const blob = this.file.slice(this.offset, chunkRightPos);
|
||||
this.reader.readAsDataURL(blob);
|
||||
|
||||
const result = await readerPromise;
|
||||
if (result.err != null) {
|
||||
this.errMsgs.push(result.err.toString());
|
||||
break;
|
||||
}
|
||||
|
||||
try {
|
||||
const uploadResp = await this.client.uploadChunk(
|
||||
this.filePath,
|
||||
result.chunk,
|
||||
this.offset
|
||||
);
|
||||
|
||||
if (uploadResp.status === 200 && uploadResp.data != null) {
|
||||
this.offset = uploadResp.data.uploaded;
|
||||
this.chunkLen = Math.ceil(this.chunkLen * speedUpRatio);
|
||||
} else if (isFatalErr(uploadResp)) {
|
||||
this.errMsgs.push(`failed to upload chunk: ${uploadResp.statusText}`);
|
||||
break;
|
||||
} else {
|
||||
// this.errMsgs.push(uploadResp.statusText);
|
||||
this.chunkLen = Math.ceil(this.chunkLen * speedDownRatio);
|
||||
|
||||
const uploadStatusResp = await this.client.uploadStatus(
|
||||
this.filePath
|
||||
);
|
||||
if (uploadStatusResp.status === 200) {
|
||||
this.offset = uploadStatusResp.data.uploaded;
|
||||
} else if (isFatalErr(uploadStatusResp)) {
|
||||
this.errMsgs.push(
|
||||
`failed to get upload status: ${uploadStatusResp.statusText}`
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
this.errMsgs.push(e.toString());
|
||||
break;
|
||||
}
|
||||
|
||||
this.progressCb(this.filePath, this.offset, true, this.err());
|
||||
}
|
||||
|
||||
if (this.chunkLen === 0) {
|
||||
this.errMsgs.push(
|
||||
"the network condition may be poor, please retry later."
|
||||
);
|
||||
} else if (!this.isOn) {
|
||||
this.errMsgs.push("uploading is stopped");
|
||||
}
|
||||
|
||||
this.progressCb(this.filePath, this.offset, false, this.err());
|
||||
return this.offset === this.file.size;
|
||||
};
|
||||
}
|
||||
|
|
@ -6,8 +6,10 @@
|
|||
"module": "commonjs",
|
||||
"target": "es5",
|
||||
"jsx": "react",
|
||||
"lib": ["es2015", "dom"]
|
||||
"esModuleInterop": true,
|
||||
"allowJs": true,
|
||||
"lib": ["es5", "dom", "scripthost", "es2015.symbol"]
|
||||
},
|
||||
"include": ["./src/**/*", "webpack.config.js", "webpack..js"],
|
||||
"exclude": ["**/*.test.ts", "**/*.test.tsx"]
|
||||
"include": ["./src/**/*"],
|
||||
"exclude": ["**/*.test.ts*"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,8 +5,13 @@ const dev = require("./webpack.dev.js");
|
|||
|
||||
module.exports = merge(dev, {
|
||||
entry: "./src/app.tsx",
|
||||
// entry: [
|
||||
// "./src/app.tsx",
|
||||
// "./src/worker/uploader.worker.ts",
|
||||
// ],
|
||||
context: `${__dirname}`,
|
||||
output: {
|
||||
globalObject: "this",
|
||||
path: `${__dirname}/../../../public/static`,
|
||||
chunkFilename: "[name].bundle.js",
|
||||
filename: "[name].bundle.js"
|
||||
|
|
|
|||
|
|
@ -9,11 +9,20 @@ const BundleAnalyzerPlugin = require("webpack-bundle-analyzer")
|
|||
module.exports = {
|
||||
module: {
|
||||
rules: [
|
||||
{
|
||||
test: /\.worker\.ts$/,
|
||||
use: {
|
||||
loader: "worker-loader",
|
||||
options: {
|
||||
// inline: "fallback",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
test: /\.ts|tsx$/,
|
||||
loader: "ts-loader",
|
||||
include: [path.resolve(__dirname, "src")],
|
||||
exclude: /\.test\.(ts|tsx)$/
|
||||
exclude: [/node_modules/, /\.test\.(ts|tsx)$/],
|
||||
},
|
||||
{
|
||||
test: /\.css$/,
|
||||
|
|
@ -22,15 +31,15 @@ module.exports = {
|
|||
{
|
||||
loader: "css-loader",
|
||||
options: {
|
||||
url: false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
url: false,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
resolve: {
|
||||
extensions: [".ts", ".tsx", ".js", ".json"]
|
||||
extensions: [".ts", ".tsx", ".js", ".json"],
|
||||
},
|
||||
plugins: [
|
||||
// new BundleAnalyzerPlugin()
|
||||
|
|
@ -38,7 +47,7 @@ module.exports = {
|
|||
externals: {
|
||||
react: "React",
|
||||
"react-dom": "ReactDOM",
|
||||
immutable: "Immutable"
|
||||
immutable: "Immutable",
|
||||
},
|
||||
optimization: {
|
||||
minimizer: [new TerserPlugin()],
|
||||
|
|
@ -48,16 +57,21 @@ module.exports = {
|
|||
cacheGroups: {
|
||||
default: {
|
||||
name: "main",
|
||||
filename: "[name].bundle.js"
|
||||
filename: "[name].bundle.js",
|
||||
},
|
||||
commons: {
|
||||
test: /[\\/]node_modules[\\/]/,
|
||||
name: "vendors",
|
||||
test: /[\\/]node_modules[\\/]/,
|
||||
chunks: "all",
|
||||
minChunks: 2,
|
||||
reuseExistingChunk: true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
reuseExistingChunk: true,
|
||||
},
|
||||
// worker: {
|
||||
// name: "worker",
|
||||
// test: /[\\/]worker[\\/]/,
|
||||
// filename: "[name].bundle.js"
|
||||
// }
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -118,7 +118,11 @@ func (h *FileHandlers) Create(c *gin.Context) {
|
|||
locker.Exec(func() {
|
||||
err := h.deps.FS().Create(tmpFilePath)
|
||||
if err != nil {
|
||||
c.JSON(q.ErrResp(c, 500, err))
|
||||
if os.IsExist(err) {
|
||||
c.JSON(q.ErrResp(c, 304, err))
|
||||
} else {
|
||||
c.JSON(q.ErrResp(c, 500, err))
|
||||
}
|
||||
return
|
||||
}
|
||||
err = h.uploadMgr.AddInfo(userName, req.Path, tmpFilePath, req.FileSize)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue