2021-08-10 12:29:57 +00:00
|
|
|
import { File, getLocalFiles } from '../fileService';
|
2021-08-11 07:11:57 +00:00
|
|
|
import { Collection, getLocalCollections } from '../collectionService';
|
2021-08-10 12:29:57 +00:00
|
|
|
import { SetFiles } from 'pages/gallery';
|
|
|
|
import { ComlinkWorker, getDedicatedCryptoWorker } from 'utils/crypto';
|
|
|
|
import {
|
|
|
|
sortFilesIntoCollections,
|
|
|
|
sortFiles,
|
2021-08-13 05:15:40 +00:00
|
|
|
removeUnnecessaryFileProps,
|
2021-08-10 12:29:57 +00:00
|
|
|
} from 'utils/file';
|
|
|
|
import { logError } from 'utils/sentry';
|
|
|
|
import localForage from 'utils/storage/localForage';
|
2021-08-11 07:11:57 +00:00
|
|
|
import {
|
2021-08-13 05:10:03 +00:00
|
|
|
getMetadataMapKey,
|
2021-08-11 07:11:57 +00:00
|
|
|
ParsedMetaDataJSON,
|
|
|
|
parseMetadataJSON,
|
|
|
|
} from './metadataService';
|
2021-08-10 12:29:57 +00:00
|
|
|
import { segregateFiles } from 'utils/upload';
|
|
|
|
import { ProgressUpdater } from 'components/pages/gallery/Upload';
|
|
|
|
import uploader from './uploader';
|
2021-08-11 05:26:40 +00:00
|
|
|
import UIService from './uiService';
|
|
|
|
import UploadService from './uploadService';
|
2021-08-13 05:30:40 +00:00
|
|
|
import { CustomError } from 'utils/common/errorUtil';
|
2021-08-10 12:29:57 +00:00
|
|
|
|
|
|
|
const MAX_CONCURRENT_UPLOADS = 4;
|
|
|
|
const FILE_UPLOAD_COMPLETED = 100;
|
|
|
|
|
|
|
|
export enum FileUploadResults {
|
|
|
|
FAILED = -1,
|
|
|
|
SKIPPED = -2,
|
|
|
|
UNSUPPORTED = -3,
|
|
|
|
BLOCKED = -4,
|
2021-08-29 07:26:24 +00:00
|
|
|
TOO_LARGE = -5,
|
2021-08-10 12:29:57 +00:00
|
|
|
UPLOADED = 100,
|
|
|
|
}
|
|
|
|
|
|
|
|
export interface FileWithCollection {
|
|
|
|
file: globalThis.File;
|
2021-08-13 05:30:40 +00:00
|
|
|
collectionID?: number;
|
2021-08-13 05:56:31 +00:00
|
|
|
collection?: Collection;
|
2021-08-10 12:29:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
export enum UPLOAD_STAGES {
|
|
|
|
START,
|
|
|
|
READING_GOOGLE_METADATA_FILES,
|
|
|
|
UPLOADING,
|
|
|
|
FINISH,
|
|
|
|
}
|
|
|
|
|
2021-08-11 07:11:57 +00:00
|
|
|
export type MetadataMap = Map<string, ParsedMetaDataJSON>;
|
|
|
|
|
2021-08-10 12:29:57 +00:00
|
|
|
class UploadManager {
|
|
|
|
private cryptoWorkers = new Array<ComlinkWorker>(MAX_CONCURRENT_UPLOADS);
|
2021-08-11 07:11:57 +00:00
|
|
|
private metadataMap: MetadataMap;
|
2021-08-10 12:29:57 +00:00
|
|
|
private filesToBeUploaded: FileWithCollection[];
|
|
|
|
private failedFiles: FileWithCollection[];
|
|
|
|
private existingFilesCollectionWise: Map<number, File[]>;
|
|
|
|
private existingFiles: File[];
|
|
|
|
private setFiles: SetFiles;
|
2021-08-11 07:11:57 +00:00
|
|
|
private collections: Map<number, Collection>;
|
2021-08-11 05:24:07 +00:00
|
|
|
public initUploader(progressUpdater: ProgressUpdater, setFiles: SetFiles) {
|
2021-08-11 05:26:40 +00:00
|
|
|
UIService.init(progressUpdater);
|
2021-08-10 12:29:57 +00:00
|
|
|
this.setFiles = setFiles;
|
|
|
|
}
|
|
|
|
|
2021-08-11 07:11:57 +00:00
|
|
|
private async init(newCollections?: Collection[]) {
|
2021-08-11 05:24:07 +00:00
|
|
|
this.filesToBeUploaded = [];
|
2021-08-10 12:29:57 +00:00
|
|
|
this.failedFiles = [];
|
|
|
|
this.metadataMap = new Map<string, ParsedMetaDataJSON>();
|
|
|
|
this.existingFiles = await getLocalFiles();
|
|
|
|
this.existingFilesCollectionWise = sortFilesIntoCollections(
|
2021-08-13 03:38:02 +00:00
|
|
|
this.existingFiles
|
2021-08-10 12:29:57 +00:00
|
|
|
);
|
2021-08-11 07:11:57 +00:00
|
|
|
const collections = await getLocalCollections();
|
|
|
|
if (newCollections) {
|
|
|
|
collections.push(...newCollections);
|
|
|
|
}
|
|
|
|
this.collections = new Map(
|
2021-08-13 03:38:02 +00:00
|
|
|
collections.map((collection) => [collection.id, collection])
|
2021-08-11 07:11:57 +00:00
|
|
|
);
|
2021-08-10 12:29:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
public async queueFilesForUpload(
|
2021-08-13 05:15:40 +00:00
|
|
|
fileWithCollectionToBeUploaded: FileWithCollection[],
|
2021-08-13 03:38:02 +00:00
|
|
|
newCreatedCollections?: Collection[]
|
2021-08-10 12:29:57 +00:00
|
|
|
) {
|
|
|
|
try {
|
2021-08-12 08:25:17 +00:00
|
|
|
await this.init(newCreatedCollections);
|
2021-08-10 12:29:57 +00:00
|
|
|
const { metadataFiles, mediaFiles } = segregateFiles(
|
2021-08-13 05:15:40 +00:00
|
|
|
fileWithCollectionToBeUploaded
|
2021-08-10 12:29:57 +00:00
|
|
|
);
|
|
|
|
if (metadataFiles.length) {
|
2021-08-11 05:26:40 +00:00
|
|
|
UIService.setUploadStage(
|
2021-08-13 03:38:02 +00:00
|
|
|
UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES
|
2021-08-10 12:29:57 +00:00
|
|
|
);
|
2021-08-11 04:46:06 +00:00
|
|
|
await this.seedMetadataMap(metadataFiles);
|
2021-08-10 12:29:57 +00:00
|
|
|
}
|
|
|
|
if (mediaFiles.length) {
|
2021-08-11 05:26:40 +00:00
|
|
|
UIService.setUploadStage(UPLOAD_STAGES.START);
|
2021-08-10 12:29:57 +00:00
|
|
|
await this.uploadMediaFiles(mediaFiles);
|
|
|
|
}
|
2021-08-11 05:26:40 +00:00
|
|
|
UIService.setUploadStage(UPLOAD_STAGES.FINISH);
|
|
|
|
UIService.setPercentComplete(FILE_UPLOAD_COMPLETED);
|
2021-08-10 12:29:57 +00:00
|
|
|
} catch (e) {
|
|
|
|
logError(e, 'uploading failed with error');
|
|
|
|
throw e;
|
|
|
|
} finally {
|
|
|
|
for (let i = 0; i < MAX_CONCURRENT_UPLOADS; i++) {
|
|
|
|
this.cryptoWorkers[i]?.worker.terminate();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-08-11 07:11:57 +00:00
|
|
|
private async seedMetadataMap(metadataFiles: FileWithCollection[]) {
|
2021-08-12 15:36:44 +00:00
|
|
|
try {
|
|
|
|
UIService.reset(metadataFiles.length);
|
2021-08-10 12:29:57 +00:00
|
|
|
|
2021-08-12 15:36:44 +00:00
|
|
|
for (const fileWithCollection of metadataFiles) {
|
|
|
|
const parsedMetaDataJSONWithTitle = await parseMetadataJSON(
|
2021-08-13 03:38:02 +00:00
|
|
|
fileWithCollection.file
|
2021-08-11 12:12:28 +00:00
|
|
|
);
|
2021-08-12 15:36:44 +00:00
|
|
|
if (parsedMetaDataJSONWithTitle) {
|
2021-08-13 05:03:21 +00:00
|
|
|
const { title, parsedMetaDataJSON } =
|
2021-08-12 15:36:44 +00:00
|
|
|
parsedMetaDataJSONWithTitle;
|
|
|
|
this.metadataMap.set(
|
2021-08-13 05:10:03 +00:00
|
|
|
getMetadataMapKey(
|
|
|
|
fileWithCollection.collectionID,
|
|
|
|
title
|
|
|
|
),
|
2021-08-13 03:38:02 +00:00
|
|
|
parsedMetaDataJSON
|
2021-08-12 15:36:44 +00:00
|
|
|
);
|
|
|
|
UIService.increaseFileUploaded();
|
|
|
|
}
|
2021-08-11 12:12:28 +00:00
|
|
|
}
|
2021-08-12 15:36:44 +00:00
|
|
|
} catch (e) {
|
2021-08-13 05:15:40 +00:00
|
|
|
logError(e, 'error seeding MetadataMap');
|
|
|
|
// silently ignore the error
|
2021-08-10 12:29:57 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private async uploadMediaFiles(mediaFiles: FileWithCollection[]) {
|
|
|
|
this.filesToBeUploaded.push(...mediaFiles);
|
2021-08-11 05:26:40 +00:00
|
|
|
UIService.reset(mediaFiles.length);
|
2021-08-10 12:29:57 +00:00
|
|
|
|
2021-08-11 06:10:53 +00:00
|
|
|
await UploadService.init(mediaFiles.length, this.metadataMap);
|
2021-08-10 12:29:57 +00:00
|
|
|
|
2021-08-11 05:26:40 +00:00
|
|
|
UIService.setUploadStage(UPLOAD_STAGES.UPLOADING);
|
2021-08-10 12:29:57 +00:00
|
|
|
|
|
|
|
const uploadProcesses = [];
|
|
|
|
for (
|
|
|
|
let i = 0;
|
2021-08-11 05:24:07 +00:00
|
|
|
i < MAX_CONCURRENT_UPLOADS && this.filesToBeUploaded.length > 0;
|
2021-08-10 12:29:57 +00:00
|
|
|
i++
|
|
|
|
) {
|
2021-08-13 05:30:40 +00:00
|
|
|
const cryptoWorker = getDedicatedCryptoWorker();
|
|
|
|
if (!cryptoWorker) {
|
|
|
|
throw Error(CustomError.FAILED_TO_LOAD_WEB_WORKER);
|
|
|
|
}
|
|
|
|
this.cryptoWorkers[i] = cryptoWorker;
|
2021-08-10 12:29:57 +00:00
|
|
|
uploadProcesses.push(
|
|
|
|
this.uploadNextFileInQueue(
|
2021-08-18 05:38:20 +00:00
|
|
|
await new this.cryptoWorkers[i].comlink()
|
2021-08-13 03:38:02 +00:00
|
|
|
)
|
2021-08-10 12:29:57 +00:00
|
|
|
);
|
|
|
|
}
|
|
|
|
await Promise.all(uploadProcesses);
|
|
|
|
}
|
|
|
|
|
2021-08-18 05:38:20 +00:00
|
|
|
private async uploadNextFileInQueue(worker: any) {
|
2021-08-11 04:46:06 +00:00
|
|
|
while (this.filesToBeUploaded.length > 0) {
|
2021-08-10 12:29:57 +00:00
|
|
|
const fileWithCollection = this.filesToBeUploaded.pop();
|
2021-08-12 15:36:44 +00:00
|
|
|
const existingFilesInCollection =
|
|
|
|
this.existingFilesCollectionWise.get(
|
2021-08-13 03:38:02 +00:00
|
|
|
fileWithCollection.collectionID
|
2021-08-13 04:47:55 +00:00
|
|
|
) ?? [];
|
2021-08-12 15:36:44 +00:00
|
|
|
const collection = this.collections.get(
|
2021-08-13 03:38:02 +00:00
|
|
|
fileWithCollection.collectionID
|
2021-08-12 15:36:44 +00:00
|
|
|
);
|
2021-08-13 05:30:40 +00:00
|
|
|
fileWithCollection.collection = collection;
|
2021-08-10 12:29:57 +00:00
|
|
|
const { fileUploadResult, file } = await uploader(
|
|
|
|
worker,
|
2021-08-12 15:36:44 +00:00
|
|
|
existingFilesInCollection,
|
2021-08-13 05:30:40 +00:00
|
|
|
fileWithCollection
|
2021-08-10 12:29:57 +00:00
|
|
|
);
|
|
|
|
|
|
|
|
if (fileUploadResult === FileUploadResults.UPLOADED) {
|
|
|
|
this.existingFiles.push(file);
|
|
|
|
this.existingFiles = sortFiles(this.existingFiles);
|
|
|
|
await localForage.setItem(
|
|
|
|
'files',
|
2021-08-13 05:15:40 +00:00
|
|
|
removeUnnecessaryFileProps(this.existingFiles)
|
2021-08-10 12:29:57 +00:00
|
|
|
);
|
|
|
|
this.setFiles(this.existingFiles);
|
2021-08-18 15:11:11 +00:00
|
|
|
if (!this.existingFilesCollectionWise.has(file.collectionID)) {
|
|
|
|
this.existingFilesCollectionWise.set(file.collectionID, []);
|
|
|
|
}
|
|
|
|
this.existingFilesCollectionWise
|
|
|
|
.get(file.collectionID)
|
|
|
|
.push(file);
|
2021-08-10 12:29:57 +00:00
|
|
|
}
|
2021-08-11 05:55:18 +00:00
|
|
|
if (
|
|
|
|
fileUploadResult === FileUploadResults.BLOCKED ||
|
2021-08-12 08:25:17 +00:00
|
|
|
fileUploadResult === FileUploadResults.FAILED
|
2021-08-11 05:55:18 +00:00
|
|
|
) {
|
|
|
|
this.failedFiles.push(fileWithCollection);
|
|
|
|
}
|
2021-08-10 12:29:57 +00:00
|
|
|
|
2021-08-11 05:26:40 +00:00
|
|
|
UIService.moveFileToResultList(fileWithCollection.file.name);
|
2021-08-10 12:29:57 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
async retryFailedFiles() {
|
|
|
|
await this.queueFilesForUpload(this.failedFiles);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
export default new UploadManager();
|