From b0b54661c2a6188567b3c333f0034f2ca0141500 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 24 Jun 2022 11:11:21 +0530 Subject: [PATCH 1/5] move upload strategy to constants --- src/components/WatchFolder/index.tsx | 2 +- src/components/pages/gallery/Upload.tsx | 7 +------ src/constants/upload/index.ts | 5 +++++ src/services/watchFolder/watchFolderService.ts | 2 +- src/types/watchFolder/index.ts | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/components/WatchFolder/index.tsx b/src/components/WatchFolder/index.tsx index 0c9f90e6b..8bbed4a25 100644 --- a/src/components/WatchFolder/index.tsx +++ b/src/components/WatchFolder/index.tsx @@ -9,7 +9,7 @@ import constants from 'utils/strings/constants'; import DialogBoxBase from 'components/DialogBox/base'; import DialogTitleWithCloseButton from 'components/DialogBox/titleWithCloseButton'; import UploadStrategyChoiceModal from 'components/pages/gallery/UploadStrategyChoiceModal'; -import { UPLOAD_STRATEGY } from 'components/pages/gallery/Upload'; +import { UPLOAD_STRATEGY } from 'constants/upload'; interface Iprops { open: boolean; diff --git a/src/components/pages/gallery/Upload.tsx b/src/components/pages/gallery/Upload.tsx index 86d9451a1..5e57c1806 100644 --- a/src/components/pages/gallery/Upload.tsx +++ b/src/components/pages/gallery/Upload.tsx @@ -33,7 +33,7 @@ import { SegregatedFinishedUploads, InProgressUpload, } from 'types/upload/ui'; -import { UPLOAD_STAGES } from 'constants/upload'; +import { UPLOAD_STAGES, UPLOAD_STRATEGY } from 'constants/upload'; const FIRST_ALBUM_NAME = 'My First Album'; @@ -59,11 +59,6 @@ interface Props { showSessionExpiredMessage: () => void; } -export enum UPLOAD_STRATEGY { - SINGLE_COLLECTION, - COLLECTION_PER_FOLDER, -} - export enum DESKTOP_UPLOAD_TYPE { FILES = 'files', FOLDERS = 'folders', diff --git a/src/constants/upload/index.ts b/src/constants/upload/index.ts index 2c9d7a4e6..709122682 100644 --- a/src/constants/upload/index.ts +++ b/src/constants/upload/index.ts @@ -31,6 +31,11 @@ export enum UPLOAD_STAGES { FINISH, } +export enum UPLOAD_STRATEGY { + SINGLE_COLLECTION, + COLLECTION_PER_FOLDER, +} + export enum UPLOAD_RESULT { FAILED, ALREADY_UPLOADED, diff --git a/src/services/watchFolder/watchFolderService.ts b/src/services/watchFolder/watchFolderService.ts index 61999ee6a..717b6daf2 100644 --- a/src/services/watchFolder/watchFolderService.ts +++ b/src/services/watchFolder/watchFolderService.ts @@ -16,8 +16,8 @@ import { diskFileRemovedCallback, diskFolderRemovedCallback, } from './watchFolderEventHandlers'; -import { UPLOAD_STRATEGY } from 'components/pages/gallery/Upload'; import { getParentFolderName } from './utils'; +import { UPLOAD_STRATEGY } from 'constants/upload'; class watchFolderService { private ElectronAPIs: ElectronAPIsInterface; diff --git a/src/types/watchFolder/index.ts b/src/types/watchFolder/index.ts index 1c02e4b44..97fa96dce 100644 --- a/src/types/watchFolder/index.ts +++ b/src/types/watchFolder/index.ts @@ -1,4 +1,4 @@ -import { UPLOAD_STRATEGY } from 'components/pages/gallery/Upload'; +import { UPLOAD_STRATEGY } from 'constants/upload'; import { ElectronFile } from 'types/upload'; export interface WatchMapping { From ff04c9c89c72cf91a63e6c25e4c5926b02a02dce Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 24 Jun 2022 11:30:15 +0530 Subject: [PATCH 2/5] simply filePaths handling --- src/services/upload/livePhotoService.ts | 14 ++++---------- src/services/upload/uploadManager.ts | 23 +++++++++++------------ src/types/upload/index.ts | 4 ++-- src/utils/file/index.ts | 8 ++++---- 4 files changed, 21 insertions(+), 28 deletions(-) diff --git a/src/services/upload/livePhotoService.ts b/src/services/upload/livePhotoService.ts index 740c4da2a..1b2000cc1 100644 --- a/src/services/upload/livePhotoService.ts +++ b/src/services/upload/livePhotoService.ts @@ -1,7 +1,6 @@ import { FILE_TYPE } from 'constants/file'; import { LIVE_PHOTO_ASSET_SIZE_LIMIT } from 'constants/upload'; import { encodeMotionPhoto } from 'services/motionPhotoService'; -import { FileMagicMetadata } from 'types/file'; import { ElectronFile, FileTypeInfo, @@ -62,12 +61,8 @@ export function getLivePhotoMetadata( }; } -export function getLivePhotoMagicMetadata( - imageFile: FileWithCollection -): FileMagicMetadata['data'] { - return { - filePaths: [getLivePhotoName((imageFile.file as any).path as string)], - }; +export function getLivePhotoFilePath(imageAsset: Asset): string { + return getLivePhotoName((imageAsset.file as any).path); } export function getLivePhotoSize(livePhotoAssets: LivePhotoAssets) { @@ -198,12 +193,11 @@ export function clusterLivePhotoFiles(mediaFiles: FileWithCollection[]) { imageAsset.metadata, videoAsset.metadata ); - const livePhotoMagicMetadata: FileMagicMetadata['data'] = - getLivePhotoMagicMetadata(firstMediaFile); + const livePhotoPath = getLivePhotoFilePath(imageAsset); uploadService.setFileMetadataAndFileTypeInfo(livePhotoLocalID, { fileTypeInfo: { ...livePhotoFileTypeInfo }, metadata: { ...livePhotoMetadata }, - magicMetadata: { ...livePhotoMagicMetadata }, + filePath: livePhotoPath, }); index += 2; } else { diff --git a/src/services/upload/uploadManager.ts b/src/services/upload/uploadManager.ts index 051a455e8..cd548d885 100644 --- a/src/services/upload/uploadManager.ts +++ b/src/services/upload/uploadManager.ts @@ -6,7 +6,7 @@ import { sortFiles, preservePhotoswipeProps, decryptFile, - changeFilePaths, + appendFilePath, } from 'utils/file'; import { logError } from 'utils/sentry'; import { getMetadataJSONMapKey, parseMetadataJSON } from './metadataService'; @@ -20,7 +20,7 @@ import UIService from './uiService'; import UploadService from './uploadService'; import { CustomError } from 'utils/error'; import { Collection } from 'types/collection'; -import { EnteFile, FileMagicMetadata } from 'types/file'; +import { EnteFile } from 'types/file'; import { FileWithCollection, MetadataAndFileTypeInfo, @@ -230,7 +230,7 @@ class UploadManager { UIService.reset(mediaFiles.length); for (const { file, localID, collectionID } of mediaFiles) { try { - const { fileTypeInfo, metadata, magicMetadata } = + const { fileTypeInfo, metadata, filePath } = await (async () => { if (file.size >= MAX_FILE_SIZE_SUPPORTED) { logUploadInfo( @@ -260,10 +260,8 @@ class UploadManager { collectionID, fileTypeInfo )) || null; - const magicMetadata = { - filePaths: [(file as any).path as string], - } as FileMagicMetadata['data']; - return { fileTypeInfo, metadata, magicMetadata }; + const filePath = (file as any).path as string; + return { fileTypeInfo, metadata, filePath }; })(); logUploadInfo( @@ -274,7 +272,7 @@ class UploadManager { this.metadataAndFileTypeInfoMap.set(localID, { fileTypeInfo: fileTypeInfo && { ...fileTypeInfo }, metadata: metadata && { ...metadata }, - magicMetadata: magicMetadata && { ...magicMetadata }, + filePath: filePath, }); UIService.increaseFileUploaded(); } catch (e) { @@ -341,14 +339,15 @@ class UploadManager { this.existingFiles, fileWithCollection ); - const filePaths = UploadService.getFileMetadataAndFileTypeInfo( + const filePath = UploadService.getFileMetadataAndFileTypeInfo( fileWithCollection.localID - ).magicMetadata.filePaths; - await changeFilePaths( + ).filePath; + + await appendFilePath( fileUploadResult, uploadedFile, fileWithCollection.collection.key, - filePaths + filePath ); UIService.moveFileToResultList( fileWithCollection.localID, diff --git a/src/types/upload/index.ts b/src/types/upload/index.ts index e473f419f..f42d5ead6 100644 --- a/src/types/upload/index.ts +++ b/src/types/upload/index.ts @@ -1,6 +1,6 @@ import { FILE_TYPE } from 'constants/file'; import { Collection } from 'types/collection'; -import { fileAttribute, FileMagicMetadata } from 'types/file'; +import { fileAttribute } from 'types/file'; export interface DataStream { stream: ReadableStream; @@ -92,7 +92,7 @@ export interface FileWithCollection extends UploadAsset { export interface MetadataAndFileTypeInfo { metadata: Metadata; fileTypeInfo: FileTypeInfo; - magicMetadata: FileMagicMetadata['data']; + filePath: string; } export type MetadataAndFileTypeInfoMap = Map; diff --git a/src/utils/file/index.ts b/src/utils/file/index.ts index ceed4ba2f..c768f6831 100644 --- a/src/utils/file/index.ts +++ b/src/utils/file/index.ts @@ -454,23 +454,23 @@ export async function changeFileName(file: EnteFile, editedName: string) { return file; } -export const changeFilePaths = async ( +export const appendFilePath = async ( fileUploadResult: UPLOAD_RESULT, file: EnteFile, collectionKey: string, - filePaths: string[] + filePath: string ) => { if ( fileUploadResult === UPLOAD_RESULT.UPLOADED || fileUploadResult === UPLOAD_RESULT.ALREADY_UPLOADED || fileUploadResult === UPLOAD_RESULT.UPLOADED_WITH_STATIC_THUMBNAIL ) { - let mergedMetadataFilePaths = [...filePaths]; + let mergedMetadataFilePaths = [filePath]; if (file.magicMetadata?.data.filePaths?.length > 0) { mergedMetadataFilePaths = [ ...new Set([ ...file.magicMetadata.data.filePaths, - ...filePaths, + ...mergedMetadataFilePaths, ]), ]; } From e022f5ef891f14825e0bfbe44c0d5384789351d5 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 24 Jun 2022 11:43:34 +0530 Subject: [PATCH 3/5] move file path upadting logic to post Upload action --- src/services/upload/uploadManager.ts | 39 +++++++++++++----------- src/utils/file/index.ts | 45 ++++++++++------------------ 2 files changed, 38 insertions(+), 46 deletions(-) diff --git a/src/services/upload/uploadManager.ts b/src/services/upload/uploadManager.ts index 0781bc3d9..2dd948c1b 100644 --- a/src/services/upload/uploadManager.ts +++ b/src/services/upload/uploadManager.ts @@ -6,7 +6,7 @@ import { sortFiles, preservePhotoswipeProps, decryptFile, - appendFilePath, + appendNewFilePath, } from 'utils/file'; import { logError } from 'utils/sentry'; import { getMetadataJSONMapKey, parseMetadataJSON } from './metadataService'; @@ -338,26 +338,18 @@ class UploadManager { this.existingFiles, fileWithCollection ); - const filePath = UploadService.getFileMetadataAndFileTypeInfo( - fileWithCollection.localID - ).filePath; - await appendFilePath( - fileUploadResult, - uploadedFile, - fileWithCollection.collection.key, - filePath - ); - UIService.moveFileToResultList( - fileWithCollection.localID, - fileUploadResult - ); - UploadService.reducePendingUploadCount(); - await this.postUploadTask( + const finalUploadResult = await this.postUploadTask( fileUploadResult, uploadedFile, fileWithCollection ); + + UIService.moveFileToResultList( + fileWithCollection.localID, + finalUploadResult + ); + UploadService.reducePendingUploadCount(); } } @@ -403,14 +395,16 @@ class UploadManager { fileWithCollection, uploadedFile ); + await this.updateFilePaths(decryptedFile, fileWithCollection); } + return fileUploadResult; } catch (e) { logError(e, 'failed to do post file upload action'); logUploadInfo( `failed to do post file upload action -> ${e.message} ${(e as Error).stack}` ); - throw e; + return UPLOAD_RESULT.FAILED; } } @@ -456,6 +450,17 @@ class UploadManager { } } + private async updateFilePaths( + decryptedFile: EnteFile, + fileWithCollection: FileWithCollection + ) { + const filePath = UploadService.getFileMetadataAndFileTypeInfo( + fileWithCollection.localID + ).filePath; + + await appendNewFilePath(decryptedFile, filePath); + } + async retryFailedFiles() { await this.queueFilesForUpload(this.failedFiles, [ ...this.collections.values(), diff --git a/src/utils/file/index.ts b/src/utils/file/index.ts index c768f6831..49f1c9589 100644 --- a/src/utils/file/index.ts +++ b/src/utils/file/index.ts @@ -26,7 +26,6 @@ import ffmpegService from 'services/ffmpeg/ffmpegService'; import { NEW_FILE_MAGIC_METADATA, VISIBILITY_STATE } from 'types/magicMetadata'; import { IsArchived, updateMagicMetadataProps } from 'utils/magicMetadata'; import { ARCHIVE_SECTION, TRASH_SECTION } from 'constants/collection'; -import { UPLOAD_RESULT } from 'constants/upload'; import { updateFileMagicMetadata } from 'services/fileService'; export function downloadAsFile(filename: string, content: string) { const file = new Blob([content], { @@ -454,35 +453,23 @@ export async function changeFileName(file: EnteFile, editedName: string) { return file; } -export const appendFilePath = async ( - fileUploadResult: UPLOAD_RESULT, - file: EnteFile, - collectionKey: string, - filePath: string -) => { - if ( - fileUploadResult === UPLOAD_RESULT.UPLOADED || - fileUploadResult === UPLOAD_RESULT.ALREADY_UPLOADED || - fileUploadResult === UPLOAD_RESULT.UPLOADED_WITH_STATIC_THUMBNAIL - ) { - let mergedMetadataFilePaths = [filePath]; - if (file.magicMetadata?.data.filePaths?.length > 0) { - mergedMetadataFilePaths = [ - ...new Set([ - ...file.magicMetadata.data.filePaths, - ...mergedMetadataFilePaths, - ]), - ]; - } - file.key = await getFileKey(file, collectionKey); - const updatedMagicMetadata = await updateMagicMetadataProps( - file.magicMetadata ?? NEW_FILE_MAGIC_METADATA, - file.key, - { filePaths: mergedMetadataFilePaths } - ); - file.magicMetadata = updatedMagicMetadata; - await updateFileMagicMetadata([file]); +export const appendNewFilePath = async (file: EnteFile, filePath: string) => { + let mergedMetadataFilePaths = [filePath]; + if (file.magicMetadata?.data.filePaths?.length > 0) { + mergedMetadataFilePaths = [ + ...new Set([ + ...file.magicMetadata.data.filePaths, + ...mergedMetadataFilePaths, + ]), + ]; } + const updatedMagicMetadata = await updateMagicMetadataProps( + file.magicMetadata ?? NEW_FILE_MAGIC_METADATA, + file.key, + { filePaths: mergedMetadataFilePaths } + ); + file.magicMetadata = updatedMagicMetadata; + await updateFileMagicMetadata([file]); }; export function isSharedFile(file: EnteFile) { From 7b584774cdef78337a889caae126c7c31239db8c Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 24 Jun 2022 11:45:32 +0530 Subject: [PATCH 4/5] move api call to update path to uploadManager --- src/services/upload/uploadManager.ts | 9 +++++++-- src/utils/file/index.ts | 5 +++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/services/upload/uploadManager.ts b/src/services/upload/uploadManager.ts index 2dd948c1b..709237c2f 100644 --- a/src/services/upload/uploadManager.ts +++ b/src/services/upload/uploadManager.ts @@ -1,4 +1,8 @@ -import { getLocalFiles, setLocalFiles } from '../fileService'; +import { + getLocalFiles, + setLocalFiles, + updateFileMagicMetadata, +} from '../fileService'; import { SetFiles } from 'types/gallery'; import { getDedicatedCryptoWorker } from 'utils/crypto'; import { @@ -458,7 +462,8 @@ class UploadManager { fileWithCollection.localID ).filePath; - await appendNewFilePath(decryptedFile, filePath); + const updatedFile = await appendNewFilePath(decryptedFile, filePath); + await updateFileMagicMetadata([updatedFile]); } async retryFailedFiles() { diff --git a/src/utils/file/index.ts b/src/utils/file/index.ts index 49f1c9589..c009d1ab9 100644 --- a/src/utils/file/index.ts +++ b/src/utils/file/index.ts @@ -26,7 +26,7 @@ import ffmpegService from 'services/ffmpeg/ffmpegService'; import { NEW_FILE_MAGIC_METADATA, VISIBILITY_STATE } from 'types/magicMetadata'; import { IsArchived, updateMagicMetadataProps } from 'utils/magicMetadata'; import { ARCHIVE_SECTION, TRASH_SECTION } from 'constants/collection'; -import { updateFileMagicMetadata } from 'services/fileService'; + export function downloadAsFile(filename: string, content: string) { const file = new Blob([content], { type: 'text/plain', @@ -469,7 +469,8 @@ export const appendNewFilePath = async (file: EnteFile, filePath: string) => { { filePaths: mergedMetadataFilePaths } ); file.magicMetadata = updatedMagicMetadata; - await updateFileMagicMetadata([file]); + + return file; }; export function isSharedFile(file: EnteFile) { From 0a1076c584d0f04c317bcf2df8c7035fee455911 Mon Sep 17 00:00:00 2001 From: Rushikesh Tote Date: Fri, 24 Jun 2022 12:27:01 +0530 Subject: [PATCH 5/5] fix update file paths when already uploaded --- src/services/upload/uploadManager.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/services/upload/uploadManager.ts b/src/services/upload/uploadManager.ts index 709237c2f..276410dc5 100644 --- a/src/services/upload/uploadManager.ts +++ b/src/services/upload/uploadManager.ts @@ -378,6 +378,10 @@ class UploadManager { uploadedFile ); } + await this.updateFilePaths( + uploadedFile, + fileWithCollection + ); break; case UPLOAD_RESULT.ADDED_SYMLINK: decryptedFile = uploadedFile;