Start using it

web/apps/photos/src/components/Upload/Uploader.tsx

@@ -31,7 +31,7 @@ import {
     SetLoading,
     UploadTypeSelectorIntent,
 } from "types/gallery";
-import { ElectronFile, FileWithCollection } from "types/upload";
+import { ElectronFile, FileWithCollection, type FileWithCollection2 } from "types/upload";
 import {
     InProgressUpload,
     SegregatedFinishedUploads,
@@ -432,7 +432,7 @@ export default function Uploader(props: Props) {
                 `upload file to an new collections strategy:${strategy} ,collectionName:${collectionName}`,
             );
             await preCollectionCreationAction();
-            let filesWithCollectionToUpload: FileWithCollection[] = [];
+            let filesWithCollectionToUpload: FileWithCollection2[] = [];
             const collections: Collection[] = [];
             let collectionNameToFilesMap = new Map<
                 string,
@@ -487,7 +487,7 @@ export default function Uploader(props: Props) {
                 });
                 throw e;
             }
-            await waitInQueueAndUploadFiles(
+            await waitInQueueAndUploadFiles2(
                 filesWithCollectionToUpload,
                 collections,
             );
@@ -515,6 +515,24 @@ export default function Uploader(props: Props) {
         await currentUploadPromise.current;
     };
 
+    const waitInQueueAndUploadFiles2 = async (
+        filesWithCollectionToUploadIn: FileWithCollection2[],
+        collections: Collection[],
+        uploaderName?: string,
+    ) => {
+        const currentPromise = currentUploadPromise.current;
+        currentUploadPromise.current = waitAndRun(
+            currentPromise,
+            async () =>
+                await uploadFiles2(
+                    filesWithCollectionToUploadIn,
+                    collections,
+                    uploaderName,
+                ),
+        );
+        await currentUploadPromise.current;
+    };
+
     const preUploadAction = async () => {
         uploadManager.prepareForNewUpload();
         setUploadProgressView(true);
@@ -541,7 +559,6 @@ export default function Uploader(props: Props) {
                 !watcher.isUploadRunning()
             ) {
                 await setToUploadCollection(collections);
-                // TODO (MR): What happens when we have both?
                 if (zipPaths.current) {
                     await electron.setPendingUploadFiles(
                         "zips",
@@ -585,6 +602,63 @@ export default function Uploader(props: Props) {
         }
     };
 
+    const uploadFiles2 = async (
+        filesWithCollectionToUploadIn: FileWithCollection2[],
+        collections: Collection[],
+        uploaderName?: string,
+    ) => {
+        try {
+            log.info("uploadFiles called");
+            preUploadAction();
+            if (
+                electron &&
+                !isPendingDesktopUpload.current &&
+                !watcher.isUploadRunning()
+            ) {
+                await setToUploadCollection(collections);
+                if (zipPaths.current) {
+                    await electron.setPendingUploadFiles(
+                        "zips",
+                        zipPaths.current,
+                    );
+                    zipPaths.current = null;
+                }
+                await electron.setPendingUploadFiles(
+                    "files",
+                    filesWithCollectionToUploadIn.map(
+                        ({ file }) => (file as ElectronFile).path,
+                    ),
+                );
+            }
+            const shouldCloseUploadProgress =
+                await uploadManager.queueFilesForUpload2(
+                    filesWithCollectionToUploadIn,
+                    collections,
+                    uploaderName,
+                );
+            if (shouldCloseUploadProgress) {
+                closeUploadProgress();
+            }
+            if (isElectron()) {
+                if (watcher.isUploadRunning()) {
+                    await watcher.allFileUploadsDone(
+                        filesWithCollectionToUploadIn,
+                        collections,
+                    );
+                } else if (watcher.isSyncPaused()) {
+                    // resume the service after user upload is done
+                    watcher.resumePausedSync();
+                }
+            }
+        } catch (e) {
+            log.error("failed to upload files", e);
+            showUserFacingError(e.message);
+            closeUploadProgress();
+        } finally {
+            postUploadAction();
+        }
+    };
+
     const retryFailed = async () => {
         try {
             log.info("user retrying failed  upload");

web/apps/photos/src/services/upload/uploadManager.ts

@@ -26,6 +26,7 @@ import {
     ParsedMetadataJSON,
     ParsedMetadataJSONMap,
     PublicUploadProps,
+    type FileWithCollection2,
 } from "types/upload";
 import { ProgressUpdater } from "types/upload/ui";
 import { decryptFile, getUserOwnedFiles, sortFiles } from "utils/file";
@@ -204,6 +205,97 @@ class UploadManager {
         }
     }
 
+    public async queueFilesForUpload2(
+        filesWithCollectionToUploadIn: FileWithCollection2[],
+        collections: Collection[],
+        uploaderName?: string,
+    ) {
+        try {
+            if (this.uploadInProgress) {
+                throw Error("can't run multiple uploads at once");
+            }
+            this.uploadInProgress = true;
+            await this.updateExistingFilesAndCollections(collections);
+            this.uploaderName = uploaderName;
+            log.info(
+                `received ${filesWithCollectionToUploadIn.length} files to upload`,
+            );
+            uiService.setFilenames(
+                new Map<number, string>(
+                    filesWithCollectionToUploadIn.map((mediaFile) => [
+                        mediaFile.localID,
+                        UploadService.getAssetName(mediaFile),
+                    ]),
+                ),
+            );
+            const { metadataJSONFiles, mediaFiles } =
+                segregateMetadataAndMediaFiles(filesWithCollectionToUploadIn);
+            log.info(`has ${metadataJSONFiles.length} metadata json files`);
+            log.info(`has ${mediaFiles.length} media files`);
+            if (metadataJSONFiles.length) {
+                UIService.setUploadStage(
+                    UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES,
+                );
+                await this.parseMetadataJSONFiles(metadataJSONFiles);
+
+                UploadService.setParsedMetadataJSONMap(
+                    this.parsedMetadataJSONMap,
+                );
+            }
+            if (mediaFiles.length) {
+                log.info(`clusterLivePhotoFiles started`);
+                const analysedMediaFiles =
+                    await UploadService.clusterLivePhotoFiles(mediaFiles);
+                log.info(`clusterLivePhotoFiles ended`);
+                log.info(
+                    `got live photos: ${
+                        mediaFiles.length !== analysedMediaFiles.length
+                    }`,
+                );
+                uiService.setFilenames(
+                    new Map<number, string>(
+                        analysedMediaFiles.map((mediaFile) => [
+                            mediaFile.localID,
+                            UploadService.getAssetName(mediaFile),
+                        ]),
+                    ),
+                );
+
+                UIService.setHasLivePhoto(
+                    mediaFiles.length !== analysedMediaFiles.length,
+                );
+
+                await this.uploadMediaFiles(analysedMediaFiles);
+            }
+        } catch (e) {
+            if (e.message === CustomError.UPLOAD_CANCELLED) {
+                if (isElectron()) {
+                    this.remainingFiles = [];
+                    await cancelRemainingUploads();
+                }
+            } else {
+                log.error("uploading failed with error", e);
+                throw e;
+            }
+        } finally {
+            UIService.setUploadStage(UPLOAD_STAGES.FINISH);
+            for (let i = 0; i < MAX_CONCURRENT_UPLOADS; i++) {
+                this.cryptoWorkers[i]?.terminate();
+            }
+            this.uploadInProgress = false;
+        }
+        try {
+            if (!UIService.hasFilesInResultList()) {
+                return true;
+            } else {
+                return false;
+            }
+        } catch (e) {
+            log.error(" failed to return shouldCloseProgressBar", e);
+            return false;
+        }
+    }
+
     private async parseMetadataJSONFiles(metadataFiles: FileWithCollection[]) {
         try {
             log.info(`parseMetadataJSONFiles function executed `);

web/apps/photos/src/services/watch.ts

@@ -11,12 +11,17 @@ import type {
     FolderWatch,
     FolderWatchSyncedFile,
 } from "@/next/types/ipc";
+import { ensureString } from "@/utils/ensure";
 import { UPLOAD_RESULT } from "constants/upload";
 import debounce from "debounce";
 import uploadManager from "services/upload/uploadManager";
 import { Collection } from "types/collection";
 import { EncryptedEnteFile } from "types/file";
-import { ElectronFile, FileWithCollection } from "types/upload";
+import {
+    ElectronFile,
+    FileWithCollection,
+    type FileWithCollection2,
+} from "types/upload";
 import { groupFilesBasedOnCollectionID } from "utils/file";
 import { isHiddenFile } from "utils/upload";
 import { removeFromCollection } from "./collectionService";
@@ -367,7 +372,7 @@ class FolderWatcher {
      * {@link upload} get uploaded.
      */
     async allFileUploadsDone(
-        filesWithCollection: FileWithCollection[],
+        filesWithCollection: FileWithCollection2[],
         collections: Collection[],
     ) {
         const electron = ensureElectron();
@@ -411,18 +416,20 @@ class FolderWatcher {
         this.debouncedRunNextEvent();
     }
 
-    private parseAllFileUploadsDone(filesWithCollection: FileWithCollection[]) {
+    private parseAllFileUploadsDone(
+        filesWithCollection: FileWithCollection2[],
+    ) {
         const syncedFiles: FolderWatch["syncedFiles"] = [];
         const ignoredFiles: FolderWatch["ignoredFiles"] = [];
 
         for (const fileWithCollection of filesWithCollection) {
             if (fileWithCollection.isLivePhoto) {
-                const imagePath = (
-                    fileWithCollection.livePhotoAssets.image as ElectronFile
-                ).path;
-                const videoPath = (
-                    fileWithCollection.livePhotoAssets.video as ElectronFile
-                ).path;
+                const imagePath = ensureString(
+                    fileWithCollection.livePhotoAssets.image,
+                );
+                const videoPath = ensureString(
+                    fileWithCollection.livePhotoAssets.video,
+                );
 
                 if (
                     this.filePathToUploadedFileIDMap.has(imagePath) &&
@@ -468,7 +475,7 @@ class FolderWatcher {
                 this.filePathToUploadedFileIDMap.delete(imagePath);
                 this.filePathToUploadedFileIDMap.delete(videoPath);
             } else {
-                const filePath = (fileWithCollection.file as ElectronFile).path;
+                const filePath = ensureString(fileWithCollection.file);
 
                 if (this.filePathToUploadedFileIDMap.has(filePath)) {
                     const file = {

web/packages/utils/ensure.ts

@@ -5,3 +5,12 @@ export const ensure = <T>(v: T | undefined): T => {
     if (v === undefined) throw new Error("Required value was not found");
     return v;
 };
+
+/**
+ * Throw an exception if the given value is not a string.
+ */
+export const ensureString = (v: unknown): string => {
+    if (typeof v != "string")
+        throw new Error(`Expected a string, instead found ${String(v)}`);
+    return v;
+};