Merge branch 'cancel-upload-without-reload' into watch

src/components/Upload/Uploader.tsx

@@ -17,7 +17,11 @@ import isElectron from 'is-electron';
 import { CustomError } from 'utils/error';
 import { Collection } from 'types/collection';
 import { SetLoading, SetFiles } from 'types/gallery';
-import { AnalysisResult, ElectronFile, FileWithCollection } from 'types/upload';
+import {
+    ImportSuggestion,
+    ElectronFile,
+    FileWithCollection,
+} from 'types/upload';
 import { isCanvasBlocked } from 'utils/upload/isCanvasBlocked';
 import { downloadApp, waitAndRun } from 'utils/common';
 import watchFolderService from 'services/watchFolder/watchFolderService';
@@ -30,15 +34,19 @@ import {
     InProgressUpload,
 } from 'types/upload/ui';
 import {
-    NULL_ANALYSIS_RESULT,
+    DEFAULT_IMPORT_SUGGESTION,
     UPLOAD_STAGES,
     UPLOAD_STRATEGY,
-    UPLOAD_TYPE,
+    PICKED_UPLOAD_TYPE,
 } from 'constants/upload';
 import importService from 'services/importService';
 import { getDownloadAppMessage } from 'utils/ui';
 import UploadTypeSelector from './UploadTypeSelector';
-import { analyseUploadFiles, getCollectionWiseFiles } from 'utils/upload';
+import {
+    getImportSuggestion,
+    groupFilesBasedOnParentFolder,
+} from 'utils/upload';
+import { getUserOwnedCollections } from 'utils/collection';
 
 const FIRST_ALBUM_NAME = 'My First Album';
 
@@ -58,8 +66,8 @@ interface Props {
     showSessionExpiredMessage: () => void;
     showUploadFilesDialog: () => void;
     showUploadDirsDialog: () => void;
-    folderSelectorFiles: File[];
+    webFolderSelectorFiles: File[];
-    fileSelectorFiles: File[];
+    webFileSelectorFiles: File[];
     dragAndDropFiles: File[];
 }
 
@@ -80,15 +88,17 @@ export default function Uploader(props: Props) {
     const [hasLivePhotos, setHasLivePhotos] = useState(false);
 
     const [choiceModalView, setChoiceModalView] = useState(false);
-    const [analysisResult, setAnalysisResult] =
+    const [importSuggestion, setImportSuggestion] = useState<ImportSuggestion>(
-        useState<AnalysisResult>(NULL_ANALYSIS_RESULT);
+        DEFAULT_IMPORT_SUGGESTION
+    );
     const appContext = useContext(AppContext);
     const galleryContext = useContext(GalleryContext);
 
     const toUploadFiles = useRef<File[] | ElectronFile[]>(null);
     const isPendingDesktopUpload = useRef(false);
     const pendingDesktopUploadCollectionName = useRef<string>('');
-    const uploadType = useRef<UPLOAD_TYPE>(null);
+    // This is set when the user choses a type to upload from the upload type selector dialog
+    const pickedUploadType = useRef<PICKED_UPLOAD_TYPE>(null);
     const zipPaths = useRef<string[]>(null);
     const previousUploadPromise = useRef<Promise<void>>(null);
     const [electronFiles, setElectronFiles] = useState<ElectronFile[]>(null);
@@ -130,6 +140,9 @@ export default function Uploader(props: Props) {
         }
     }, []);
 
+    // this handles the change of selectorFiles changes on web when user selects
+    // files for upload through the opened file/folder selector or dragAndDrop them
+    //  the webFiles state is update which triggers the upload of those files
     useEffect(() => {
         if (appContext.watchFolderView) {
             // if watch folder dialog is open don't catch the dropped file
@@ -137,22 +150,22 @@ export default function Uploader(props: Props) {
             return;
         }
         if (
-            uploadType.current === UPLOAD_TYPE.FOLDERS &&
+            pickedUploadType.current === PICKED_UPLOAD_TYPE.FOLDERS &&
-            props.folderSelectorFiles?.length > 0
+            props.webFolderSelectorFiles?.length > 0
         ) {
-            setWebFiles(props.folderSelectorFiles);
+            setWebFiles(props.webFolderSelectorFiles);
         } else if (
-            uploadType.current === UPLOAD_TYPE.FILES &&
+            pickedUploadType.current === PICKED_UPLOAD_TYPE.FILES &&
-            props.fileSelectorFiles?.length > 0
+            props.webFileSelectorFiles?.length > 0
         ) {
-            setWebFiles(props.fileSelectorFiles);
+            setWebFiles(props.webFileSelectorFiles);
         } else if (props.dragAndDropFiles?.length > 0) {
             setWebFiles(props.dragAndDropFiles);
         }
     }, [
         props.dragAndDropFiles,
-        props.fileSelectorFiles,
+        props.webFileSelectorFiles,
-        props.folderSelectorFiles,
+        props.webFolderSelectorFiles,
     ]);
 
     useEffect(() => {
@@ -198,14 +211,14 @@ export default function Uploader(props: Props) {
                 toUploadFiles.current = electronFiles;
                 setElectronFiles([]);
             }
-            const analysisResult = analyseUploadFiles(
+            const importSuggestion = getImportSuggestion(
-                uploadType.current,
+                pickedUploadType.current,
                 toUploadFiles.current
             );
-            setAnalysisResult(analysisResult);
+            setImportSuggestion(importSuggestion);
 
             handleCollectionCreationAndUpload(
-                analysisResult,
+                importSuggestion,
                 props.isFirstUpload
             );
             props.setLoading(false);
@@ -213,14 +226,14 @@ export default function Uploader(props: Props) {
     }, [webFiles, appContext.sharedFiles, electronFiles]);
 
     const resumeDesktopUpload = async (
-        type: UPLOAD_TYPE,
+        type: PICKED_UPLOAD_TYPE,
         electronFiles: ElectronFile[],
         collectionName: string
     ) => {
         if (electronFiles && electronFiles?.length > 0) {
             isPendingDesktopUpload.current = true;
             pendingDesktopUploadCollectionName.current = collectionName;
-            uploadType.current = type;
+            pickedUploadType.current = type;
             setElectronFiles(electronFiles);
         }
     };
@@ -250,21 +263,29 @@ export default function Uploader(props: Props) {
             await preUploadAction();
             const filesWithCollectionToUpload: FileWithCollection[] = [];
             const collections: Collection[] = [];
-            let collectionWiseFiles = new Map<
+            let collectionNameToFilesMap = new Map<
                 string,
                 (File | ElectronFile)[]
             >();
             if (strategy === UPLOAD_STRATEGY.SINGLE_COLLECTION) {
-                collectionWiseFiles.set(collectionName, toUploadFiles.current);
+                collectionNameToFilesMap.set(
+                    collectionName,
+                    toUploadFiles.current
+                );
             } else {
-                collectionWiseFiles = getCollectionWiseFiles(
+                collectionNameToFilesMap = groupFilesBasedOnParentFolder(
                     toUploadFiles.current
                 );
             }
             try {
-                const existingCollection = await syncCollections();
+                const existingCollection = getUserOwnedCollections(
+                    await syncCollections()
+                );
                 let index = 0;
-                for (const [collectionName, files] of collectionWiseFiles) {
+                for (const [
+                    collectionName,
+                    files,
+                ] of collectionNameToFilesMap) {
                     const collection = await createAlbum(
                         collectionName,
                         existingCollection
@@ -332,22 +353,26 @@ export default function Uploader(props: Props) {
                 await ImportService.setToUploadCollection(collections);
                 if (zipPaths.current) {
                     await ImportService.setToUploadFiles(
-                        UPLOAD_TYPE.ZIPS,
+                        PICKED_UPLOAD_TYPE.ZIPS,
                         zipPaths.current
                     );
                     zipPaths.current = null;
                 }
                 await ImportService.setToUploadFiles(
-                    UPLOAD_TYPE.FILES,
+                    PICKED_UPLOAD_TYPE.FILES,
                     filesWithCollectionToUploadIn.map(
                         ({ file }) => (file as ElectronFile).path
                     )
                 );
             }
+            const shouldCloseUploadProgress =
                 await uploadManager.queueFilesForUpload(
                     filesWithCollectionToUploadIn,
                     collections
                 );
+            if (shouldCloseUploadProgress) {
+                closeUploadProgress();
+            }
             if (isElectron()) {
                 if (watchFolderService.isUploadRunning()) {
                     await watchFolderService.allFileUploadsDone(
@@ -370,8 +395,13 @@ export default function Uploader(props: Props) {
 
     const retryFailed = async () => {
         try {
+            const filesWithCollections =
+                await uploadManager.getFailedFilesWithCollections();
             await preUploadAction();
-            await uploadManager.retryFailedFiles();
+            await uploadManager.queueFilesForUpload(
+                filesWithCollections.files,
+                filesWithCollections.collections
+            );
         } catch (err) {
             showUserFacingError(err.message);
             closeUploadProgress();
@@ -435,7 +465,7 @@ export default function Uploader(props: Props) {
     };
 
     const handleCollectionCreationAndUpload = (
-        analysisResult: AnalysisResult,
+        importSuggestion: ImportSuggestion,
         isFirstUpload: boolean
     ) => {
         if (isPendingDesktopUpload.current) {
@@ -452,21 +482,22 @@ export default function Uploader(props: Props) {
             }
             return;
         }
-        if (isElectron() && uploadType.current === UPLOAD_TYPE.ZIPS) {
+        if (
+            isElectron() &&
+            pickedUploadType.current === PICKED_UPLOAD_TYPE.ZIPS
+        ) {
             uploadFilesToNewCollections(UPLOAD_STRATEGY.COLLECTION_PER_FOLDER);
             return;
         }
-        if (isFirstUpload && !analysisResult.suggestedCollectionName) {
+        if (isFirstUpload && !importSuggestion.rootFolderName) {
-            analysisResult.suggestedCollectionName = FIRST_ALBUM_NAME;
+            importSuggestion.rootFolderName = FIRST_ALBUM_NAME;
         }
         let showNextModal = () => {};
-        if (analysisResult.multipleFolders) {
+        if (importSuggestion.hasNestedFolders) {
             showNextModal = () => setChoiceModalView(true);
         } else {
             showNextModal = () =>
-                uploadToSingleNewCollection(
+                uploadToSingleNewCollection(importSuggestion.rootFolderName);
-                    analysisResult.suggestedCollectionName
-                );
         }
         props.setCollectionSelectorAttributes({
             callback: uploadFilesToExistingCollection,
@@ -474,12 +505,12 @@ export default function Uploader(props: Props) {
             title: constants.UPLOAD_TO_COLLECTION,
         });
     };
-    const handleDesktopUpload = async (type: UPLOAD_TYPE) => {
+    const handleDesktopUpload = async (type: PICKED_UPLOAD_TYPE) => {
         let files: ElectronFile[];
-        uploadType.current = type;
+        pickedUploadType.current = type;
-        if (type === UPLOAD_TYPE.FILES) {
+        if (type === PICKED_UPLOAD_TYPE.FILES) {
             files = await ImportService.showUploadFilesDialog();
-        } else if (type === UPLOAD_TYPE.FOLDERS) {
+        } else if (type === PICKED_UPLOAD_TYPE.FOLDERS) {
             files = await ImportService.showUploadDirsDialog();
         } else {
             const response = await ImportService.showUploadZipDialog();
@@ -492,18 +523,18 @@ export default function Uploader(props: Props) {
         }
     };
 
-    const handleWebUpload = async (type: UPLOAD_TYPE) => {
+    const handleWebUpload = async (type: PICKED_UPLOAD_TYPE) => {
-        uploadType.current = type;
+        pickedUploadType.current = type;
-        if (type === UPLOAD_TYPE.FILES) {
+        if (type === PICKED_UPLOAD_TYPE.FILES) {
             props.showUploadFilesDialog();
-        } else if (type === UPLOAD_TYPE.FOLDERS) {
+        } else if (type === PICKED_UPLOAD_TYPE.FOLDERS) {
             props.showUploadDirsDialog();
         } else {
             appContext.setDialogMessage(getDownloadAppMessage());
         }
     };
 
-    const cancelUploads = async () => {
+    const cancelUploads = () => {
         uploadManager.cancelRunningUpload();
     };
 
@@ -515,9 +546,9 @@ export default function Uploader(props: Props) {
         }
     };
 
-    const handleFileUpload = handleUpload(UPLOAD_TYPE.FILES);
+    const handleFileUpload = handleUpload(PICKED_UPLOAD_TYPE.FILES);
-    const handleFolderUpload = handleUpload(UPLOAD_TYPE.FOLDERS);
+    const handleFolderUpload = handleUpload(PICKED_UPLOAD_TYPE.FOLDERS);
-    const handleZipUpload = handleUpload(UPLOAD_TYPE.ZIPS);
+    const handleZipUpload = handleUpload(PICKED_UPLOAD_TYPE.ZIPS);
 
     return (
         <>
@@ -525,9 +556,7 @@ export default function Uploader(props: Props) {
                 open={choiceModalView}
                 onClose={() => setChoiceModalView(false)}
                 uploadToSingleCollection={() =>
-                    uploadToSingleNewCollection(
+                    uploadToSingleNewCollection(importSuggestion.rootFolderName)
-                        analysisResult.suggestedCollectionName
-                    )
                 }
                 uploadToMultipleCollection={() =>
                     uploadFilesToNewCollections(

src/constants/export/index.ts

@@ -1,4 +1,4 @@
-export const METADATA_FOLDER_NAME = 'metadata';
+export const ENTE_METADATA_FOLDER = 'metadata';
 
 export enum ExportNotification {
     START = 'export started',

src/constants/upload/index.ts

@@ -1,6 +1,10 @@
 import { ENCRYPTION_CHUNK_SIZE } from 'constants/crypto';
 import { FILE_TYPE } from 'constants/file';
-import { Location, ParsedExtractedMetadata } from 'types/upload';
+import {
+    ImportSuggestion,
+    Location,
+    ParsedExtractedMetadata,
+} from 'types/upload';
 
 // list of format that were missed by type-detection for some files.
 export const FORMAT_MISSED_BY_FILE_TYPE_LIB = [
@@ -28,8 +32,8 @@ export enum UPLOAD_STAGES {
     READING_GOOGLE_METADATA_FILES,
     EXTRACTING_METADATA,
     UPLOADING,
+    CANCELLING,
     FINISH,
-    PAUSING,
 }
 
 export enum UPLOAD_STRATEGY {
@@ -50,7 +54,7 @@ export enum UPLOAD_RESULT {
     CANCELLED,
 }
 
-export enum UPLOAD_TYPE {
+export enum PICKED_UPLOAD_TYPE {
     FILES = 'files',
     FOLDERS = 'folders',
     ZIPS = 'zips',
@@ -69,9 +73,9 @@ export const A_SEC_IN_MICROSECONDS = 1e6;
 
 export const USE_CF_PROXY = false;
 
-export const NULL_ANALYSIS_RESULT = {
+export const DEFAULT_IMPORT_SUGGESTION: ImportSuggestion = {
-    suggestedCollectionName: '',
+    rootFolderName: '',
-    multipleFolders: false,
+    hasNestedFolders: false,
 };
 
 export const BLACK_THUMBNAIL_BASE64 =

src/pages/gallery/index.tsx

@@ -161,14 +161,14 @@ export default function Gallery() {
         disabled: uploadInProgress,
     });
     const {
-        selectedFiles: fileSelectorFiles,
+        selectedFiles: webFileSelectorFiles,
         open: openFileSelector,
         getInputProps: getFileSelectorInputProps,
     } = useFileInput({
         directory: false,
     });
     const {
-        selectedFiles: folderSelectorFiles,
+        selectedFiles: webFolderSelectorFiles,
         open: openFolderSelector,
         getInputProps: getFolderSelectorInputProps,
     } = useFileInput({
@@ -672,8 +672,8 @@ export default function Gallery() {
                     setUploadInProgress={setUploadInProgress}
                     setFiles={setFiles}
                     isFirstUpload={hasNonEmptyCollections(collectionSummaries)}
-                    fileSelectorFiles={fileSelectorFiles}
+                    webFileSelectorFiles={webFileSelectorFiles}
-                    folderSelectorFiles={folderSelectorFiles}
+                    webFolderSelectorFiles={webFolderSelectorFiles}
                     dragAndDropFiles={dragAndDropFiles}
                     uploadTypeSelectorView={uploadTypeSelectorView}
                     showUploadFilesDialog={openFileSelector}

src/services/importService.ts

@@ -1,4 +1,4 @@
-import { UPLOAD_TYPE } from 'constants/upload';
+import { PICKED_UPLOAD_TYPE } from 'constants/upload';
 import { Collection } from 'types/collection';
 import { ElectronAPIs } from 'types/electron';
 import { ElectronFile, FileWithCollection } from 'types/upload';
@@ -8,7 +8,7 @@ import { logError } from 'utils/sentry';
 interface PendingUploads {
     files: ElectronFile[];
     collectionName: string;
-    type: UPLOAD_TYPE;
+    type: PICKED_UPLOAD_TYPE;
 }
 
 interface selectZipResult {
@@ -88,7 +88,7 @@ class ImportService {
     }
 
     async setToUploadFiles(
-        type: UPLOAD_TYPE.FILES | UPLOAD_TYPE.ZIPS,
+        type: PICKED_UPLOAD_TYPE.FILES | PICKED_UPLOAD_TYPE.ZIPS,
         filePaths: string[]
     ) {
         if (this.allElectronAPIsExist) {
@@ -117,14 +117,14 @@ class ImportService {
                     );
                 }
             }
-            this.setToUploadFiles(UPLOAD_TYPE.FILES, filePaths);
+            this.setToUploadFiles(PICKED_UPLOAD_TYPE.FILES, filePaths);
         }
     }
     cancelRemainingUploads() {
         if (this.allElectronAPIsExist) {
             this.ElectronAPIs.setToUploadCollection(null);
-            this.ElectronAPIs.setToUploadFiles(UPLOAD_TYPE.ZIPS, []);
+            this.ElectronAPIs.setToUploadFiles(PICKED_UPLOAD_TYPE.ZIPS, []);
-            this.ElectronAPIs.setToUploadFiles(UPLOAD_TYPE.FILES, []);
+            this.ElectronAPIs.setToUploadFiles(PICKED_UPLOAD_TYPE.FILES, []);
         }
     }
 }

src/services/upload/uiService.ts

@@ -14,6 +14,7 @@ import {
 import { CustomError } from 'utils/error';
 import uploadCancelService from './uploadCancelService';
 
+const REQUEST_TIMEOUT_TIME = 30 * 1000; // 30 sec;
 class UIService {
     private perFileProgress: number;
     private filesUploaded: number;
@@ -75,7 +76,19 @@ class UIService {
         this.updateProgressBarUI();
     }
 
-    updateProgressBarUI() {
+    hasFilesInResultList() {
+        const finishedUploadsList = segregatedFinishedUploadsToList(
+            this.finishedUploads
+        );
+        for (const x of finishedUploadsList.values()) {
+            if (x.length > 0) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private updateProgressBarUI() {
         const {
             setPercentComplete,
             setUploadCounter,
@@ -102,10 +115,10 @@ class UIService {
 
         setPercentComplete(percentComplete);
         setInProgressUploads(
-            this.convertInProgressUploadsToList(this.inProgressUploads)
+            convertInProgressUploadsToList(this.inProgressUploads)
         );
         setFinishedUploads(
-            this.segregatedFinishedUploadsToList(this.finishedUploads)
+            segregatedFinishedUploadsToList(this.finishedUploads)
         );
     }
 
@@ -115,20 +128,18 @@ class UIService {
         index = 0
     ) {
         const cancel: { exec: Canceler } = { exec: () => {} };
+        const cancelTimedOutRequest = () =>
+            cancel.exec(CustomError.REQUEST_TIMEOUT);
+
+        const cancelCancelledUploadRequest = () =>
+            cancel.exec(CustomError.UPLOAD_CANCELLED);
+
         let timeout = null;
         const resetTimeout = () => {
             if (timeout) {
                 clearTimeout(timeout);
             }
-            timeout = setTimeout(
+            timeout = setTimeout(cancelTimedOutRequest, REQUEST_TIMEOUT_TIME);
-                () => cancel.exec(CustomError.REQUEST_TIMEOUT),
-                30 * 1000
-            );
-        };
-        const cancelIfUploadPaused = () => {
-            if (uploadCancelService.isUploadCancelationRequested()) {
-                cancel.exec(CustomError.UPLOAD_CANCELLED);
-            }
         };
         return {
             cancel,
@@ -149,12 +160,17 @@ class UIService {
                 } else {
                     resetTimeout();
                 }
-                cancelIfUploadPaused();
+                if (uploadCancelService.isUploadCancelationRequested()) {
+                    cancelCancelledUploadRequest();
+                }
             },
         };
     }
+}
 
-    convertInProgressUploadsToList(inProgressUploads) {
+export default new UIService();
+
+function convertInProgressUploadsToList(inProgressUploads) {
     return [...inProgressUploads.entries()].map(
         ([localFileID, progress]) =>
             ({
@@ -164,9 +180,8 @@ class UIService {
     );
 }
 
-    segregatedFinishedUploadsToList(finishedUploads: FinishedUploads) {
+function segregatedFinishedUploadsToList(finishedUploads: FinishedUploads) {
-        const segregatedFinishedUploads =
+    const segregatedFinishedUploads = new Map() as SegregatedFinishedUploads;
-            new Map() as SegregatedFinishedUploads;
     for (const [localID, result] of finishedUploads) {
         if (!segregatedFinishedUploads.has(result)) {
             segregatedFinishedUploads.set(result, []);
@@ -175,6 +190,3 @@ class UIService {
     }
     return segregatedFinishedUploads;
 }
-}
-
-export default new UIService();

src/services/upload/uploadCancelService.ts

@@ -1,22 +1,22 @@
 interface UploadCancelStatus {
-    val: boolean;
+    value: boolean;
 }
 
 class UploadCancelService {
     private shouldUploadBeCancelled: UploadCancelStatus = {
-        val: false,
+        value: false,
     };
 
     reset() {
-        this.shouldUploadBeCancelled.val = false;
+        this.shouldUploadBeCancelled.value = false;
     }
 
     requestUploadCancelation() {
-        this.shouldUploadBeCancelled.val = true;
+        this.shouldUploadBeCancelled.value = true;
     }
 
     isUploadCancelationRequested(): boolean {
-        return this.shouldUploadBeCancelled.val;
+        return this.shouldUploadBeCancelled.value;
     }
 }
 

src/services/upload/uploadHttpClient.ts

@@ -92,14 +92,16 @@ class UploadHttpClient {
         progressTracker
     ): Promise<string> {
         try {
-            await retryHTTPCall(() =>
+            await retryHTTPCall(
+                () =>
                     HTTPService.put(
                         fileUploadURL.url,
                         file,
                         null,
                         null,
                         progressTracker
-                )
+                    ),
+                handleUploadError
             );
             return fileUploadURL.objectKey;
         } catch (e) {
@@ -127,7 +129,9 @@ class UploadHttpClient {
             );
             return fileUploadURL.objectKey;
         } catch (e) {
+            if (e.message !== CustomError.UPLOAD_CANCELLED) {
                 logError(e, 'putFile to dataStore failed ');
+            }
             throw e;
         }
     }
@@ -152,10 +156,12 @@ class UploadHttpClient {
                     throw err;
                 }
                 return resp;
-            });
+            }, handleUploadError);
             return response.headers.etag as string;
         } catch (e) {
+            if (e.message !== CustomError.UPLOAD_CANCELLED) {
                 logError(e, 'put filePart failed');
+            }
             throw e;
         }
     }

src/services/upload/uploadManager.ts

@@ -1,16 +1,12 @@
-import {
+import { getLocalFiles, updateFileMagicMetadata } from '../fileService';
-    getLocalFiles,
-    setLocalFiles,
-    updateFileMagicMetadata,
-} from '../fileService';
 import { SetFiles } from 'types/gallery';
 import { getDedicatedCryptoWorker } from 'utils/crypto';
 import {
-    groupFilesBasedOnCollectionID,
     sortFiles,
     preservePhotoswipeProps,
     decryptFile,
     appendNewFilePath,
+    getUserOwnedNonTrashedFiles,
 } from 'utils/file';
 import { logError } from 'utils/sentry';
 import { getMetadataJSONMapKey, parseMetadataJSON } from './metadataService';
@@ -58,8 +54,8 @@ class UploadManager {
     private filesToBeUploaded: FileWithCollection[];
     private remainingFiles: FileWithCollection[] = [];
     private failedFiles: FileWithCollection[];
-    private collectionToExistingFilesMap: Map<number, EnteFile[]>;
     private existingFiles: EnteFile[];
+    private userOwnedNonTrashedExistingFiles: EnteFile[];
     private setFiles: SetFiles;
     private collections: Map<number, Collection>;
     private uploadInProgress: boolean;
@@ -85,12 +81,13 @@ class UploadManager {
     prepareForNewUpload() {
         this.resetState();
         UIService.reset();
+        uploadCancelService.reset();
         UIService.setUploadStage(UPLOAD_STAGES.START);
     }
 
     async updateExistingFilesAndCollections(collections: Collection[]) {
         this.existingFiles = await getLocalFiles();
-        this.collectionToExistingFilesMap = groupFilesBasedOnCollectionID(
+        this.userOwnedNonTrashedExistingFiles = getUserOwnedNonTrashedFiles(
             this.existingFiles
         );
         this.collections = new Map(
@@ -203,6 +200,16 @@ class UploadManager {
             }
             this.uploadInProgress = false;
         }
+        try {
+            if (!UIService.hasFilesInResultList()) {
+                return true;
+            } else {
+                return false;
+            }
+        } catch (e) {
+            logError(e, ' failed to return shouldCloseProgressBar');
+            return false;
+        }
     }
 
     private async parseMetadataJSONFiles(metadataFiles: FileWithCollection[]) {
@@ -216,7 +223,6 @@ class UploadManager {
                     if (uploadCancelService.isUploadCancelationRequested()) {
                         throw Error(CustomError.UPLOAD_CANCELLED);
                     }
-
                     addLogLine(
                         `parsing metadata json file ${getFileNameSize(file)}`
                     );
@@ -242,8 +248,8 @@ class UploadManager {
                     if (e.message === CustomError.UPLOAD_CANCELLED) {
                         throw e;
                     } else {
+                        // and don't break for subsequent files just log and move on
                         logError(e, 'parsing failed for a file');
-                        // and don't break for subsequent files
                     }
                     addLogLine(
                         `failed to parse metadata json file ${getFileNameSize(
@@ -257,7 +263,6 @@ class UploadManager {
                 logError(e, 'error seeding MetadataMap');
             }
             throw e;
-            // silently ignore the error
         }
     }
 
@@ -292,8 +297,8 @@ class UploadManager {
                     if (e.message === CustomError.UPLOAD_CANCELLED) {
                         throw e;
                     } else {
+                        // and don't break for subsequent files just log and move on
                         logError(e, 'extractFileTypeAndMetadata failed');
-                        // and don't break for subsequent files
                     }
                     addLogLine(
                         `metadata extraction failed ${getFileNameSize(
@@ -393,14 +398,11 @@ class UploadManager {
             }
             let fileWithCollection = this.filesToBeUploaded.pop();
             const { collectionID } = fileWithCollection;
-            const collectionExistingFiles =
-                this.collectionToExistingFilesMap.get(collectionID) ?? [];
             const collection = this.collections.get(collectionID);
             fileWithCollection = { ...fileWithCollection, collection };
             const { fileUploadResult, uploadedFile } = await uploader(
                 worker,
-                collectionExistingFiles,
+                this.userOwnedNonTrashedExistingFiles,
-                this.existingFiles,
                 fileWithCollection
             );
 
@@ -420,7 +422,7 @@ class UploadManager {
 
     async postUploadTask(
         fileUploadResult: UPLOAD_RESULT,
-        uploadedFile: EnteFile,
+        uploadedFile: EnteFile | null,
         fileWithCollection: FileWithCollection
     ) {
         try {
@@ -461,8 +463,14 @@ class UploadManager {
                 default:
                     throw Error('Invalid Upload Result' + fileUploadResult);
             }
-            if (decryptedFile) {
+            if (
-                await this.updateExistingFiles(decryptedFile);
+                [
+                    UPLOAD_RESULT.ADDED_SYMLINK,
+                    UPLOAD_RESULT.UPLOADED,
+                    UPLOAD_RESULT.UPLOADED_WITH_STATIC_THUMBNAIL,
+                ].includes(fileUploadResult)
+            ) {
+                this.updateExistingFiles(decryptedFile);
                 await this.updateFilePaths(decryptedFile, fileWithCollection);
             }
             return fileUploadResult;
@@ -489,35 +497,28 @@ class UploadManager {
     }
 
     public cancelRunningUpload() {
-        UIService.setUploadStage(UPLOAD_STAGES.PAUSING);
+        UIService.setUploadStage(UPLOAD_STAGES.CANCELLING);
         uploadCancelService.requestUploadCancelation();
     }
 
-    async retryFailedFiles() {
+    async getFailedFilesWithCollections() {
-        await this.queueFilesForUpload(this.failedFiles, [
+        return {
-            ...this.collections.values(),
+            files: this.failedFiles,
-        ]);
+            collections: [...this.collections.values()],
+        };
     }
 
-    private updateExistingFileToCollectionMap(decryptedFile: EnteFile) {
+    private updateExistingFiles(decryptedFile: EnteFile) {
-        if (
+        if (!decryptedFile) {
-            !this.collectionToExistingFilesMap.has(decryptedFile.collectionID)
+            throw Error("decrypted file can't be undefined");
-        ) {
-            this.collectionToExistingFilesMap.set(
-                decryptedFile.collectionID,
-                []
-            );
         }
-        this.collectionToExistingFilesMap
+        this.userOwnedNonTrashedExistingFiles.push(decryptedFile);
-            .get(decryptedFile.collectionID)
+        this.updateUIFiles(decryptedFile);
-            .push(decryptedFile);
     }
 
-    private async updateExistingFiles(decryptedFile: EnteFile) {
+    private updateUIFiles(decryptedFile: EnteFile) {
         this.existingFiles.push(decryptedFile);
-        this.updateExistingFileToCollectionMap(decryptedFile);
         this.existingFiles = sortFiles(this.existingFiles);
-        await setLocalFiles(this.existingFiles);
         this.setFiles(preservePhotoswipeProps(this.existingFiles));
     }
 

src/services/upload/uploader.ts

@@ -1,14 +1,14 @@
 import { EnteFile } from 'types/file';
 import { handleUploadError, CustomError } from 'utils/error';
 import { logError } from 'utils/sentry';
-import { findMatchingExistingFile } from 'utils/upload';
+import { findMatchingExistingFiles } from 'utils/upload';
 import UploadHttpClient from './uploadHttpClient';
 import UIService from './uiService';
 import UploadService from './uploadService';
 import { FILE_TYPE } from 'constants/file';
 import { UPLOAD_RESULT, MAX_FILE_SIZE_SUPPORTED } from 'constants/upload';
 import { FileWithCollection, BackupedFile, UploadFile } from 'types/upload';
-import { addLogLine } from 'utils/logging';
+import { addLocalLog, addLogLine } from 'utils/logging';
 import { convertBytesToHumanReadable } from 'utils/file/size';
 import { sleep } from 'utils/common';
 import { addToCollection } from 'services/collectionService';
@@ -21,7 +21,6 @@ interface UploadResponse {
 
 export default async function uploader(
     worker: any,
-    existingFilesInCollection: EnteFile[],
     existingFiles: EnteFile[],
     fileWithCollection: FileWithCollection
 ): Promise<UploadResponse> {
@@ -47,18 +46,35 @@ export default async function uploader(
             throw Error(CustomError.NO_METADATA);
         }
 
-        const existingFile = findMatchingExistingFile(existingFiles, metadata);
+        const matchingExistingFiles = findMatchingExistingFiles(
-        if (existingFile) {
+            existingFiles,
-            if (existingFile.collectionID === collection.id) {
+            metadata
+        );
+        addLocalLog(
+            () =>
+                `matchedFileList: ${matchingExistingFiles
+                    .map((f) => `${f.id}-${f.metadata.title}`)
+                    .join(',')}`
+        );
+        if (matchingExistingFiles?.length) {
+            const matchingExistingFilesCollectionIDs =
+                matchingExistingFiles.map((e) => e.collectionID);
+            addLocalLog(
+                () =>
+                    `matched file collectionIDs:${matchingExistingFilesCollectionIDs}
+                       and collectionID:${collection.id}`
+            );
+            if (matchingExistingFilesCollectionIDs.includes(collection.id)) {
                 addLogLine(
                     `file already present in the collection , skipped upload for  ${fileNameSize}`
                 );
                 return { fileUploadResult: UPLOAD_RESULT.ALREADY_UPLOADED };
             } else {
                 addLogLine(
-                    `same file in other collection found for  ${fileNameSize}`
+                    `same file in ${matchingExistingFilesCollectionIDs.length} collection found for  ${fileNameSize}`
                 );
-                const resultFile = Object.assign({}, existingFile);
+                // any of the matching file can used to add a symlink
+                const resultFile = Object.assign({}, matchingExistingFiles[0]);
                 resultFile.collectionID = collection.id;
                 await addToCollection(collection, [resultFile]);
                 return {

src/types/upload/index.ts

@@ -144,7 +144,8 @@ export interface ParsedExtractedMetadata {
     creationTime: number;
 }
 
-export interface AnalysisResult {
+// This is used to prompt the user the make upload strategy choice
-    suggestedCollectionName: string;
+export interface ImportSuggestion {
-    multipleFolders: boolean;
+    rootFolderName: string;
+    hasNestedFolders: boolean;
 }

src/utils/collection/index.ts

@@ -218,3 +218,11 @@ export const shouldShowOptions = (type: CollectionSummaryType) => {
 export const shouldBeShownOnCollectionBar = (type: CollectionSummaryType) => {
     return !HIDE_FROM_COLLECTION_BAR_TYPES.has(type);
 };
+
+export const getUserOwnedCollections = (collections: Collection[]) => {
+    const user: User = getData(LS_KEYS.USER);
+    if (!user?.id) {
+        throw Error('user missing');
+    }
+    return collections.filter((collection) => collection.owner.id === user.id);
+};

src/utils/error/index.ts

@@ -83,6 +83,7 @@ export function handleUploadError(error): Error {
         case CustomError.SUBSCRIPTION_EXPIRED:
         case CustomError.STORAGE_QUOTA_EXCEEDED:
         case CustomError.SESSION_EXPIRED:
+        case CustomError.UPLOAD_CANCELLED:
             throw parsedError;
     }
     return parsedError;

src/utils/export/index.ts

@@ -6,7 +6,7 @@ import { EnteFile } from 'types/file';
 
 import { Metadata } from 'types/upload';
 import { formatDate, splitFilenameAndExtension } from 'utils/file';
-import { METADATA_FOLDER_NAME } from 'constants/export';
+import { ENTE_METADATA_FOLDER } from 'constants/export';
 
 export const getExportRecordFileUID = (file: EnteFile) =>
     `${file.id}_${file.collectionID}_${file.updationTime}`;
@@ -179,7 +179,7 @@ export const getUniqueCollectionFolderPath = (
 };
 
 export const getMetadataFolderPath = (collectionFolderPath: string) =>
-    `${collectionFolderPath}/${METADATA_FOLDER_NAME}`;
+    `${collectionFolderPath}/${ENTE_METADATA_FOLDER}`;
 
 export const getUniqueFileSaveName = (
     collectionPath: string,
@@ -211,7 +211,7 @@ export const getOldFileSaveName = (filename: string, fileID: number) =>
 export const getFileMetadataSavePath = (
     collectionFolderPath: string,
     fileSaveName: string
-) => `${collectionFolderPath}/${METADATA_FOLDER_NAME}/${fileSaveName}.json`;
+) => `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${fileSaveName}.json`;
 
 export const getFileSavePath = (
     collectionFolderPath: string,
@@ -235,6 +235,6 @@ export const getOldFileMetadataSavePath = (
     collectionFolderPath: string,
     file: EnteFile
 ) =>
-    `${collectionFolderPath}/${METADATA_FOLDER_NAME}/${
+    `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${
         file.id
     }_${oldSanitizeName(file.metadata.title)}.json`;

src/utils/file/index.ts

@@ -534,3 +534,11 @@ export const createTypedObjectURL = async (blob: Blob, fileName: string) => {
     const type = await getFileType(new File([blob], fileName));
     return URL.createObjectURL(new Blob([blob], { type: type.mimeType }));
 };
+
+export const getUserOwnedNonTrashedFiles = (files: EnteFile[]) => {
+    const user: User = getData(LS_KEYS.USER);
+    if (!user?.id) {
+        throw Error('user missing');
+    }
+    return files.filter((file) => file.isTrashed || file.ownerID === user.id);
+};

src/utils/logging/index.ts

@@ -4,12 +4,21 @@ import { formatDateTime } from 'utils/time';
 import { saveLogLine, getLogs } from 'utils/storage';
 
 export function addLogLine(log: string) {
+    if (!process.env.NEXT_PUBLIC_SENTRY_ENV) {
+        console.log(log);
+    }
     saveLogLine({
         timestamp: Date.now(),
         logLine: log,
     });
 }
 
+export const addLocalLog = (getLog: () => string) => {
+    if (!process.env.NEXT_PUBLIC_SENTRY_ENV) {
+        console.log(getLog());
+    }
+};
+
 export function getDebugLogs() {
     return getLogs().map(
         (log) => `[${formatDateTime(log.timestamp)}] ${log.logLine}`

src/utils/strings/englishConstants.tsx

@@ -110,8 +110,8 @@ const englishConstants = {
         2: 'Reading file metadata',
         3: (fileCounter) =>
             `${fileCounter.finished} / ${fileCounter.total} files backed up`,
-        4: 'Backup complete',
+        4: 'Cancelling remaining uploads',
-        5: 'Pausing remaining uploads',
+        5: 'Backup complete',
     },
     UPLOADING_FILES: 'File upload',
     FILE_NOT_UPLOADED_LIST: 'The following files were not uploaded',

src/utils/upload/index.ts

@@ -1,5 +1,5 @@
 import {
-    AnalysisResult,
+    ImportSuggestion,
     ElectronFile,
     FileWithCollection,
     Metadata,
@@ -7,45 +7,27 @@ import {
 import { EnteFile } from 'types/file';
 import {
     A_SEC_IN_MICROSECONDS,
-    NULL_ANALYSIS_RESULT,
+    DEFAULT_IMPORT_SUGGESTION,
-    UPLOAD_TYPE,
+    PICKED_UPLOAD_TYPE,
 } from 'constants/upload';
 import { FILE_TYPE } from 'constants/file';
-import { METADATA_FOLDER_NAME } from 'constants/export';
+import { ENTE_METADATA_FOLDER } from 'constants/export';
 import isElectron from 'is-electron';
 
 const TYPE_JSON = 'json';
 const DEDUPE_COLLECTION = new Set(['icloud library', 'icloudlibrary']);
 
-export function findMatchingExistingFile(
+export function findMatchingExistingFiles(
     existingFiles: EnteFile[],
     newFileMetadata: Metadata
-): EnteFile {
+): EnteFile[] {
+    const matchingFiles: EnteFile[] = [];
     for (const existingFile of existingFiles) {
         if (areFilesSame(existingFile.metadata, newFileMetadata)) {
-            return existingFile;
+            matchingFiles.push(existingFile);
         }
     }
-    return null;
+    return matchingFiles;
-}
-
-export function findSameFileInOtherCollection(
-    existingFiles: EnteFile[],
-    newFileMetadata: Metadata
-) {
-    if (!hasFileHash(newFileMetadata)) {
-        return null;
-    }
-
-    for (const existingFile of existingFiles) {
-        if (
-            hasFileHash(existingFile.metadata) &&
-            areFilesWithFileHashSame(existingFile.metadata, newFileMetadata)
-        ) {
-            return existingFile;
-        }
-    }
-    return null;
 }
 
 export function shouldDedupeAcrossCollection(collectionName: string): boolean {
@@ -128,12 +110,12 @@ export function areFileWithCollectionsSame(
     return firstFile.localID === secondFile.localID;
 }
 
-export function analyseUploadFiles(
+export function getImportSuggestion(
-    uploadType: UPLOAD_TYPE,
+    uploadType: PICKED_UPLOAD_TYPE,
     toUploadFiles: File[] | ElectronFile[]
-): AnalysisResult {
+): ImportSuggestion {
-    if (isElectron() && uploadType === UPLOAD_TYPE.FILES) {
+    if (isElectron() && uploadType === PICKED_UPLOAD_TYPE.FILES) {
-        return NULL_ANALYSIS_RESULT;
+        return DEFAULT_IMPORT_SUGGESTION;
     }
 
     const paths: string[] = toUploadFiles.map((file) => file['path']);
@@ -161,27 +143,49 @@ export function analyseUploadFiles(
         }
     }
     return {
-        suggestedCollectionName: commonPathPrefix || null,
+        rootFolderName: commonPathPrefix || null,
-        multipleFolders: firstFileFolder !== lastFileFolder,
+        hasNestedFolders: firstFileFolder !== lastFileFolder,
     };
 }
 
-export function getCollectionWiseFiles(toUploadFiles: File[] | ElectronFile[]) {
+// This function groups files that are that have the same parent folder into collections
-    const collectionWiseFiles = new Map<string, (File | ElectronFile)[]>();
+// For Example, for user files have a directory structure like this
+//              a
+//            / |  \
+//           b  j   c
+//          /|\    /  \
+//         e f g   h  i
+//
+// The files will grouped into 3 collections.
+// [a => [j],
+// b => [e,f,g],
+// c => [h, i]]
+export function groupFilesBasedOnParentFolder(
+    toUploadFiles: File[] | ElectronFile[]
+) {
+    const collectionNameToFilesMap = new Map<string, (File | ElectronFile)[]>();
     for (const file of toUploadFiles) {
         const filePath = file['path'] as string;
 
         let folderPath = filePath.substring(0, filePath.lastIndexOf('/'));
-        if (folderPath.endsWith(METADATA_FOLDER_NAME)) {
+        // If the parent folder of a file is "metadata"
+        // we consider it to be part of the parent folder
+        // For Eg,For FileList  -> [a/x.png, a/metadata/x.png.json]
+        // they will both we grouped into the collection "a"
+        // This is cluster the metadata json files in the same collection as the file it is for
+        if (folderPath.endsWith(ENTE_METADATA_FOLDER)) {
             folderPath = folderPath.substring(0, folderPath.lastIndexOf('/'));
         }
         const folderName = folderPath.substring(
             folderPath.lastIndexOf('/') + 1
         );
-        if (!collectionWiseFiles.has(folderName)) {
+        if (!folderName?.length) {
-            collectionWiseFiles.set(folderName, []);
+            throw Error("folderName can't be null");
         }
-        collectionWiseFiles.get(folderName).push(file);
+        if (!collectionNameToFilesMap.has(folderName)) {
+            collectionNameToFilesMap.set(folderName, []);
         }
-    return collectionWiseFiles;
+        collectionNameToFilesMap.get(folderName).push(file);
+    }
+    return collectionNameToFilesMap;
 }