2021-08-10 12:29:57 +00:00
|
|
|
import { File, getLocalFiles } from '../fileService';
|
|
|
|
import { Collection } from '../collectionService';
|
|
|
|
import { SetFiles } from 'pages/gallery';
|
|
|
|
import { ComlinkWorker, getDedicatedCryptoWorker } from 'utils/crypto';
|
|
|
|
import {
|
|
|
|
sortFilesIntoCollections,
|
|
|
|
sortFiles,
|
|
|
|
removeUnneccessaryFileProps,
|
|
|
|
} from 'utils/file';
|
|
|
|
import { logError } from 'utils/sentry';
|
|
|
|
import localForage from 'utils/storage/localForage';
|
|
|
|
import { ParsedMetaDataJSON, parseMetadataJSON } from './metadataService';
|
|
|
|
import { segregateFiles } from 'utils/upload';
|
|
|
|
import { ProgressUpdater } from 'components/pages/gallery/Upload';
|
|
|
|
import uploader from './uploader';
|
2021-08-11 05:26:40 +00:00
|
|
|
import UIService from './uiService';
|
|
|
|
import UploadService from './uploadService';
|
2021-08-10 12:29:57 +00:00
|
|
|
|
|
|
|
const MAX_CONCURRENT_UPLOADS = 4;
|
|
|
|
const FILE_UPLOAD_COMPLETED = 100;
|
|
|
|
|
|
|
|
export enum FileUploadResults {
|
|
|
|
FAILED = -1,
|
|
|
|
SKIPPED = -2,
|
|
|
|
UNSUPPORTED = -3,
|
|
|
|
BLOCKED = -4,
|
|
|
|
UPLOADED = 100,
|
|
|
|
}
|
|
|
|
|
|
|
|
export interface UploadURL {
|
|
|
|
url: string;
|
|
|
|
objectKey: string;
|
|
|
|
}
|
|
|
|
|
|
|
|
export interface FileWithCollection {
|
|
|
|
file: globalThis.File;
|
|
|
|
collection: Collection;
|
|
|
|
}
|
|
|
|
|
|
|
|
export enum UPLOAD_STAGES {
|
|
|
|
START,
|
|
|
|
READING_GOOGLE_METADATA_FILES,
|
|
|
|
UPLOADING,
|
|
|
|
FINISH,
|
|
|
|
}
|
|
|
|
|
|
|
|
class UploadManager {
|
|
|
|
private cryptoWorkers = new Array<ComlinkWorker>(MAX_CONCURRENT_UPLOADS);
|
|
|
|
private metadataMap: Map<string, ParsedMetaDataJSON>;
|
|
|
|
private filesToBeUploaded: FileWithCollection[];
|
|
|
|
private failedFiles: FileWithCollection[];
|
|
|
|
private existingFilesCollectionWise: Map<number, File[]>;
|
|
|
|
private existingFiles: File[];
|
|
|
|
private setFiles: SetFiles;
|
|
|
|
|
2021-08-11 05:24:07 +00:00
|
|
|
public initUploader(progressUpdater: ProgressUpdater, setFiles: SetFiles) {
|
2021-08-11 05:26:40 +00:00
|
|
|
UIService.init(progressUpdater);
|
2021-08-10 12:29:57 +00:00
|
|
|
this.setFiles = setFiles;
|
|
|
|
}
|
|
|
|
|
|
|
|
private async init() {
|
2021-08-11 05:24:07 +00:00
|
|
|
this.filesToBeUploaded = [];
|
2021-08-10 12:29:57 +00:00
|
|
|
this.failedFiles = [];
|
|
|
|
this.metadataMap = new Map<string, ParsedMetaDataJSON>();
|
|
|
|
this.existingFiles = await getLocalFiles();
|
|
|
|
this.existingFilesCollectionWise = sortFilesIntoCollections(
|
|
|
|
this.existingFiles,
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
public async queueFilesForUpload(
|
|
|
|
filesWithCollectionToUpload: FileWithCollection[],
|
|
|
|
) {
|
|
|
|
try {
|
|
|
|
await this.init();
|
|
|
|
|
|
|
|
const { metadataFiles, mediaFiles } = segregateFiles(
|
|
|
|
filesWithCollectionToUpload,
|
|
|
|
);
|
|
|
|
if (metadataFiles.length) {
|
2021-08-11 05:26:40 +00:00
|
|
|
UIService.setUploadStage(
|
2021-08-10 12:29:57 +00:00
|
|
|
UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES,
|
|
|
|
);
|
2021-08-11 04:46:06 +00:00
|
|
|
await this.seedMetadataMap(metadataFiles);
|
2021-08-10 12:29:57 +00:00
|
|
|
}
|
|
|
|
if (mediaFiles.length) {
|
2021-08-11 05:26:40 +00:00
|
|
|
UIService.setUploadStage(UPLOAD_STAGES.START);
|
2021-08-10 12:29:57 +00:00
|
|
|
await this.uploadMediaFiles(mediaFiles);
|
|
|
|
}
|
2021-08-11 05:26:40 +00:00
|
|
|
UIService.setUploadStage(UPLOAD_STAGES.FINISH);
|
|
|
|
UIService.setPercentComplete(FILE_UPLOAD_COMPLETED);
|
2021-08-10 12:29:57 +00:00
|
|
|
} catch (e) {
|
|
|
|
logError(e, 'uploading failed with error');
|
|
|
|
throw e;
|
|
|
|
} finally {
|
|
|
|
for (let i = 0; i < MAX_CONCURRENT_UPLOADS; i++) {
|
|
|
|
this.cryptoWorkers[i]?.worker.terminate();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-08-11 04:46:06 +00:00
|
|
|
private async seedMetadataMap(metadataFiles: globalThis.File[]) {
|
2021-08-11 05:26:40 +00:00
|
|
|
UIService.reset(metadataFiles.length);
|
2021-08-10 12:29:57 +00:00
|
|
|
|
|
|
|
for (const rawFile of metadataFiles) {
|
|
|
|
const parsedMetaDataJSON = await parseMetadataJSON(rawFile);
|
|
|
|
this.metadataMap.set(parsedMetaDataJSON.title, parsedMetaDataJSON);
|
2021-08-11 05:26:40 +00:00
|
|
|
UIService.increaseFileUploaded();
|
2021-08-10 12:29:57 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private async uploadMediaFiles(mediaFiles: FileWithCollection[]) {
|
|
|
|
this.filesToBeUploaded.push(...mediaFiles);
|
2021-08-11 05:26:40 +00:00
|
|
|
UIService.reset(mediaFiles.length);
|
2021-08-10 12:29:57 +00:00
|
|
|
|
2021-08-11 05:26:40 +00:00
|
|
|
UploadService.init(mediaFiles.length, this.metadataMap);
|
2021-08-10 12:29:57 +00:00
|
|
|
|
2021-08-11 05:26:40 +00:00
|
|
|
UIService.setUploadStage(UPLOAD_STAGES.UPLOADING);
|
2021-08-10 12:29:57 +00:00
|
|
|
|
|
|
|
const uploadProcesses = [];
|
|
|
|
for (
|
|
|
|
let i = 0;
|
2021-08-11 05:24:07 +00:00
|
|
|
i < MAX_CONCURRENT_UPLOADS && this.filesToBeUploaded.length > 0;
|
2021-08-10 12:29:57 +00:00
|
|
|
i++
|
|
|
|
) {
|
|
|
|
this.cryptoWorkers[i] = getDedicatedCryptoWorker();
|
|
|
|
uploadProcesses.push(
|
|
|
|
this.uploadNextFileInQueue(
|
|
|
|
await new this.cryptoWorkers[i].comlink(),
|
|
|
|
new FileReader(),
|
|
|
|
),
|
|
|
|
);
|
|
|
|
}
|
|
|
|
await Promise.all(uploadProcesses);
|
|
|
|
}
|
|
|
|
|
|
|
|
private async uploadNextFileInQueue(worker: any, fileReader: FileReader) {
|
2021-08-11 04:46:06 +00:00
|
|
|
while (this.filesToBeUploaded.length > 0) {
|
2021-08-10 12:29:57 +00:00
|
|
|
const fileWithCollection = this.filesToBeUploaded.pop();
|
|
|
|
const { fileUploadResult, file } = await uploader(
|
|
|
|
worker,
|
|
|
|
fileReader,
|
|
|
|
fileWithCollection,
|
|
|
|
this.existingFilesCollectionWise,
|
|
|
|
);
|
|
|
|
|
|
|
|
if (fileUploadResult === FileUploadResults.UPLOADED) {
|
|
|
|
this.existingFiles.push(file);
|
|
|
|
this.existingFiles = sortFiles(this.existingFiles);
|
|
|
|
await localForage.setItem(
|
|
|
|
'files',
|
|
|
|
removeUnneccessaryFileProps(this.existingFiles),
|
|
|
|
);
|
|
|
|
this.setFiles(this.existingFiles);
|
|
|
|
}
|
|
|
|
|
2021-08-11 05:26:40 +00:00
|
|
|
UIService.moveFileToResultList(fileWithCollection.file.name);
|
2021-08-10 12:29:57 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
async retryFailedFiles() {
|
|
|
|
await this.queueFilesForUpload(this.failedFiles);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
export default new UploadManager();
|