From 2c62f983a8318fbb5910a217854a33b16e19c708 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 29 Apr 2024 20:35:00 +0530 Subject: [PATCH] wipx --- .../photos/src/components/Upload/Uploader.tsx | 357 ++++++++---------- .../src/services/upload/uploadManager.ts | 18 +- web/apps/photos/src/services/watch.ts | 59 +-- 3 files changed, 197 insertions(+), 237 deletions(-) diff --git a/web/apps/photos/src/components/Upload/Uploader.tsx b/web/apps/photos/src/components/Upload/Uploader.tsx index f3a6968af..dd90bb98c 100644 --- a/web/apps/photos/src/components/Upload/Uploader.tsx +++ b/web/apps/photos/src/components/Upload/Uploader.tsx @@ -21,12 +21,12 @@ import { savePublicCollectionUploaderName, } from "services/publicCollectionService"; import type { - FileWithCollection, InProgressUpload, SegregatedFinishedUploads, UploadCounter, UploadFileNames, UploadItem, + UploadItemWithCollection, } from "services/upload/uploadManager"; import uploadManager from "services/upload/uploadManager"; import watcher from "services/watch"; @@ -86,6 +86,7 @@ interface Props { } export default function Uploader({ + isFirstUpload, dragAndDropFiles, openFileSelector, fileSelectorFiles, @@ -162,12 +163,14 @@ export default function Uploader({ * {@link desktopFiles}, {@link desktopFilePaths} and * {@link desktopZipEntries}. * + * Augment each {@link UploadItem} with its "path" (relative path or name in + * the case of {@link webFiles}, absolute path in the case of + * {@link desktopFiles}, {@link desktopFilePaths}, and the path within the + * zip file for {@link desktopZipEntries}). + * * See the documentation of {@link UploadItem} for more details. */ - const uploadItems = useRef([]); - - // TODO(MR): temp, doesn't have zips - const fileOrPathsToUpload = useRef<(File | string)[]>([]); + const uploadItemsAndPaths = useRef<[UploadItem, string][]>([]); /** * If true, then the next upload we'll be processing was initiated by our @@ -301,15 +304,15 @@ export default function Uploader({ // Trigger an upload when any of the dependencies change. useEffect(() => { - const itemAndPaths = [ + const allItemAndPaths = [ /* TODO(MR): ElectronFile | use webkitRelativePath || name here */ - webFiles.map((f) => [f, f["path"]]), + webFiles.map((f) => [f, f["path"] ?? f.name]), desktopFiles.map((fp) => [fp, fp.path]), desktopFilePaths.map((p) => [p, p]), desktopZipEntries.map((ze) => [ze, ze[1]]), - ].flat(); + ].flat() as [UploadItem, string][]; - if (itemAndPaths.length == 0) return; + if (allItemAndPaths.length == 0) return; if (uploadManager.isUploadRunning()) { if (watcher.isUploadRunning()) { @@ -333,42 +336,93 @@ export default function Uploader({ setDesktopZipEntries([]); // Remove hidden files (files whose names begins with a "."). - const prunedItemAndPaths = itemAndPaths.filter( + const prunedItemAndPaths = allItemAndPaths.filter( + // eslint-disable-next-line @typescript-eslint/no-unused-vars ([_, p]) => !basename(p).startsWith("."), ); - uploadItems.current = prunedItemAndPaths.map(([i]) => i); - fileOrPathsToUpload.current = uploadItems.current - .map((i) => { - if (typeof i == "string" || i instanceof File) return i; - if (Array.isArray(i)) return undefined; - return i.file; - }) - .filter((x) => x); - uploadItems.current = []; - if (fileOrPathsToUpload.current.length === 0) { + uploadItemsAndPaths.current = prunedItemAndPaths; + if (uploadItemsAndPaths.current.length === 0) { props.setLoading(false); return; } const importSuggestion = getImportSuggestion( pickedUploadType.current, + // eslint-disable-next-line @typescript-eslint/no-unused-vars prunedItemAndPaths.map(([_, p]) => p), ); setImportSuggestion(importSuggestion); log.debug(() => "Uploader invoked:"); - log.debug(() => fileOrPathsToUpload.current); + log.debug(() => uploadItemsAndPaths.current); log.debug(() => importSuggestion); - handleCollectionCreationAndUpload( - importSuggestion, - props.isFirstUpload, - pickedUploadType.current, - publicCollectionGalleryContext.accessedThroughSharedURL, - ); + const _pickedUploadType = pickedUploadType.current; pickedUploadType.current = null; props.setLoading(false); + + (async () => { + if (publicCollectionGalleryContext.accessedThroughSharedURL) { + const uploaderName = await getPublicCollectionUploaderName( + getPublicCollectionUID( + publicCollectionGalleryContext.token, + ), + ); + uploaderNameRef.current = uploaderName; + showUserNameInputDialog(); + return; + } + + if (isPendingDesktopUpload.current) { + isPendingDesktopUpload.current = false; + if (pendingDesktopUploadCollectionName.current) { + uploadFilesToNewCollections( + "root", + pendingDesktopUploadCollectionName.current, + ); + pendingDesktopUploadCollectionName.current = null; + } else { + uploadFilesToNewCollections("parent"); + } + return; + } + + if (electron && _pickedUploadType === PICKED_UPLOAD_TYPE.ZIPS) { + uploadFilesToNewCollections("parent"); + return; + } + + if (isFirstUpload && !importSuggestion.rootFolderName) { + importSuggestion.rootFolderName = FIRST_ALBUM_NAME; + } + + if (isDragAndDrop.current) { + isDragAndDrop.current = false; + if ( + props.activeCollection && + props.activeCollection.owner.id === galleryContext.user?.id + ) { + uploadFilesToExistingCollection(props.activeCollection); + return; + } + } + + let showNextModal = () => {}; + if (importSuggestion.hasNestedFolders) { + showNextModal = () => setChoiceModalView(true); + } else { + showNextModal = () => + showCollectionCreateModal(importSuggestion.rootFolderName); + } + + props.setCollectionSelectorAttributes({ + callback: uploadFilesToExistingCollection, + onCancel: handleCollectionSelectorCancel, + showNextModal, + intent: CollectionSelectorIntent.upload, + }); + })(); }, [webFiles, desktopFiles, desktopFilePaths, desktopZipEntries]); const preCollectionCreationAction = async () => { @@ -382,100 +436,78 @@ export default function Uploader({ collection: Collection, uploaderName?: string, ) => { - try { - log.info( - `Uploading files existing collection id ${collection.id} (${collection.name})`, - ); - await preCollectionCreationAction(); - const filesWithCollectionToUpload = fileOrPathsToUpload.current.map( - (fileOrPath, index) => ({ - fileOrPath, - localID: index, - collectionID: collection.id, - }), - ); - await waitInQueueAndUploadFiles( - filesWithCollectionToUpload, - [collection], - uploaderName, - ); - } catch (e) { - log.error("Failed to upload files to existing collection", e); - } + await preCollectionCreationAction(); + const uploadItemsWithCollection = uploadItemsAndPaths.current.map( + ([uploadItem], index) => ({ + uploadItem, + localID: index, + collectionID: collection.id, + }), + ); + await waitInQueueAndUploadFiles( + uploadItemsWithCollection, + [collection], + uploaderName, + ); + uploadItemsAndPaths.current = null; }; const uploadFilesToNewCollections = async ( mapping: CollectionMapping, collectionName?: string, ) => { - try { - log.info( - `Uploading files to collection using ${mapping} mapping (${collectionName ?? ""})`, + await preCollectionCreationAction(); + let uploadItemsWithCollection: UploadItemWithCollection[] = []; + const collections: Collection[] = []; + let collectionNameToUploadItems = new Map(); + if (mapping == "root") { + collectionNameToUploadItems.set( + collectionName, + uploadItemsAndPaths.current.map(([i]) => i), ); - await preCollectionCreationAction(); - let filesWithCollectionToUpload: FileWithCollection[] = []; - const collections: Collection[] = []; - let collectionNameToFileOrPaths = new Map< - string, - (File | string)[] - >(); - if (mapping == "root") { - collectionNameToFileOrPaths.set( - collectionName, - fileOrPathsToUpload.current, - ); - } else { - collectionNameToFileOrPaths = groupFilesBasedOnParentFolder( - fileOrPathsToUpload.current, - ); - } - try { - const existingCollections = await getLatestCollections(); - let index = 0; - for (const [ - collectionName, - fileOrPaths, - ] of collectionNameToFileOrPaths) { - const collection = await getOrCreateAlbum( - collectionName, - existingCollections, - ); - collections.push(collection); - props.setCollections([ - ...existingCollections, - ...collections, - ]); - filesWithCollectionToUpload = [ - ...filesWithCollectionToUpload, - ...fileOrPaths.map((fileOrPath) => ({ - localID: index++, - collectionID: collection.id, - fileOrPath, - })), - ]; - } - } catch (e) { - closeUploadProgress(); - log.error("Failed to create album", e); - appContext.setDialogMessage({ - title: t("ERROR"), - close: { variant: "critical" }, - content: t("CREATE_ALBUM_FAILED"), - }); - throw e; - } - await waitInQueueAndUploadFiles( - filesWithCollectionToUpload, - collections, + } else { + collectionNameToUploadItems = groupFilesBasedOnParentFolder( + uploadItemsAndPaths.current, ); - fileOrPathsToUpload.current = null; - } catch (e) { - log.error("Failed to upload files to new collections", e); } + try { + const existingCollections = await getLatestCollections(); + let index = 0; + for (const [ + collectionName, + fileOrPaths, + ] of collectionNameToUploadItems) { + const collection = await getOrCreateAlbum( + collectionName, + existingCollections, + ); + collections.push(collection); + props.setCollections([...existingCollections, ...collections]); + uploadItemsWithCollection = [ + ...uploadItemsWithCollection, + ...fileOrPaths.map((fileOrPath) => ({ + localID: index++, + collectionID: collection.id, + fileOrPath, + })), + ]; + } + } catch (e) { + closeUploadProgress(); + log.error("Failed to create album", e); + appContext.setDialogMessage({ + title: t("ERROR"), + close: { variant: "critical" }, + content: t("CREATE_ALBUM_FAILED"), + }); + throw e; + } + await waitInQueueAndUploadFiles(uploadItemsWithCollection, collections); + uploadItemsAndPaths.current = null; }; const waitInQueueAndUploadFiles = async ( - filesWithCollectionToUploadIn: FileWithCollection[], + uploadItemsWithCollection: UploadItemWithCollection[], collections: Collection[], uploaderName?: string, ) => { @@ -484,7 +516,7 @@ export default function Uploader({ currentPromise, async () => await uploadFiles( - filesWithCollectionToUploadIn, + uploadItemsWithCollection, collections, uploaderName, ), @@ -505,7 +537,7 @@ export default function Uploader({ } const uploadFiles = async ( - filesWithCollectionToUploadIn: FileWithCollection[], + uploadItemsWithCollection: UploadItemWithCollection[], collections: Collection[], uploaderName?: string, ) => { @@ -519,11 +551,13 @@ export default function Uploader({ setPendingUploads( electron, collections, - filesWithCollectionToUploadIn, + uploadItemsWithCollection + .map(({ uploadItem }) => uploadItem) + .filter((x) => x), ); } const wereFilesProcessed = await uploadManager.uploadFiles( - filesWithCollectionToUploadIn, + uploadItemsWithCollection, collections, uploaderName, ); @@ -531,11 +565,12 @@ export default function Uploader({ if (isElectron()) { if (watcher.isUploadRunning()) { await watcher.allFileUploadsDone( - filesWithCollectionToUploadIn, + uploadItemsWithCollection, collections, ); } else if (watcher.isSyncPaused()) { - // resume the service after user upload is done + // Resume folder watch after the user upload that + // interrupted it is done. watcher.resumePausedSync(); } } @@ -610,78 +645,6 @@ export default function Uploader({ }); }; - const handleCollectionCreationAndUpload = async ( - importSuggestion: ImportSuggestion, - isFirstUpload: boolean, - pickedUploadType: PICKED_UPLOAD_TYPE, - accessedThroughSharedURL?: boolean, - ) => { - try { - if (accessedThroughSharedURL) { - const uploaderName = await getPublicCollectionUploaderName( - getPublicCollectionUID( - publicCollectionGalleryContext.token, - ), - ); - uploaderNameRef.current = uploaderName; - showUserNameInputDialog(); - return; - } - - if (isPendingDesktopUpload.current) { - isPendingDesktopUpload.current = false; - if (pendingDesktopUploadCollectionName.current) { - uploadFilesToNewCollections( - "root", - pendingDesktopUploadCollectionName.current, - ); - pendingDesktopUploadCollectionName.current = null; - } else { - uploadFilesToNewCollections("parent"); - } - return; - } - - if (isElectron() && pickedUploadType === PICKED_UPLOAD_TYPE.ZIPS) { - uploadFilesToNewCollections("parent"); - return; - } - - if (isFirstUpload && !importSuggestion.rootFolderName) { - importSuggestion.rootFolderName = FIRST_ALBUM_NAME; - } - - if (isDragAndDrop.current) { - isDragAndDrop.current = false; - if ( - props.activeCollection && - props.activeCollection.owner.id === galleryContext.user?.id - ) { - uploadFilesToExistingCollection(props.activeCollection); - return; - } - } - - let showNextModal = () => {}; - if (importSuggestion.hasNestedFolders) { - showNextModal = () => setChoiceModalView(true); - } else { - showNextModal = () => - showCollectionCreateModal(importSuggestion.rootFolderName); - } - - props.setCollectionSelectorAttributes({ - callback: uploadFilesToExistingCollection, - onCancel: handleCollectionSelectorCancel, - showNextModal, - intent: CollectionSelectorIntent.upload, - }); - } catch (e) { - // TODO(MR): Why? - log.warn("Ignoring error in handleCollectionCreationAndUpload", e); - } - }; - const cancelUploads = () => { uploadManager.cancelRunningUpload(); }; @@ -784,7 +747,7 @@ export default function Uploader({ open={userNameInputDialogView} onClose={handleUserNameInputDialogClose} onNameSubmit={handlePublicUpload} - toUploadFilesCount={fileOrPathsToUpload.current?.length} + toUploadFilesCount={uploadItemsAndPaths.current?.length} uploaderName={uploaderNameRef.current} /> @@ -884,16 +847,12 @@ function getImportSuggestion( // [a => [j], // b => [e,f,g], // c => [h, i]] -const groupFilesBasedOnParentFolder = (fileOrPaths: (File | string)[]) => { - const result = new Map(); - for (const fileOrPath of fileOrPaths) { - const filePath = - /* TODO(MR): ElectronFile */ - typeof fileOrPath == "string" - ? fileOrPath - : (fileOrPath["path"] as string); - - let folderPath = filePath.substring(0, filePath.lastIndexOf("/")); +const groupFilesBasedOnParentFolder = ( + uploadItemsAndPaths: [UploadItem, string][], +) => { + const result = new Map(); + for (const [uploadItem, pathOrName] of uploadItemsAndPaths) { + let folderPath = pathOrName.substring(0, pathOrName.lastIndexOf("/")); // If the parent folder of a file is "metadata" // we consider it to be part of the parent folder // For Eg,For FileList -> [a/x.png, a/metadata/x.png.json] @@ -907,7 +866,7 @@ const groupFilesBasedOnParentFolder = (fileOrPaths: (File | string)[]) => { ); if (!folderName) throw Error("Unexpected empty folder name"); if (!result.has(folderName)) result.set(folderName, []); - result.get(folderName).push(fileOrPath); + result.get(folderName).push(uploadItem); } return result; }; diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index 9a8cb6c6d..00741843c 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -109,17 +109,17 @@ const maxConcurrentUploads = 4; */ export type UploadItem = File | FileAndPath | string | ZipEntry; -export interface FileWithCollection { +export interface UploadItemWithCollection { localID: number; collectionID: number; isLivePhoto?: boolean; - fileOrPath?: File | string; + uploadItem?: UploadItem; livePhotoAssets?: LivePhotoAssets; } export interface LivePhotoAssets { - image: File | string; - video: File | string; + image: UploadItem; + video: UploadItem; } export interface PublicUploadProps { @@ -419,7 +419,7 @@ class UploadManager { * @returns `true` if at least one file was processed */ public async uploadFiles( - filesWithCollectionToUploadIn: FileWithCollection[], + filesWithCollectionToUploadIn: UploadItemWithCollection[], collections: Collection[], uploaderName?: string, ) { @@ -735,8 +735,8 @@ export default new UploadManager(); * As files progress through stages, they get more and more bits tacked on to * them. These types document the journey. * - * - The input is {@link FileWithCollection}. This can either be a new - * {@link FileWithCollection}, in which case it'll only have a + * - The input is {@link UploadItemWithCollection}. This can either be a new + * {@link UploadItemWithCollection}, in which case it'll only have a * {@link localID}, {@link collectionID} and a {@link fileOrPath}. Or it could * be a retry, in which case it'll not have a {@link fileOrPath} but instead * will have data from a previous stage (concretely, it'll just be a @@ -772,9 +772,9 @@ type FileWithCollectionIDAndName = { }; const makeFileWithCollectionIDAndName = ( - f: FileWithCollection, + f: UploadItemWithCollection, ): FileWithCollectionIDAndName => { - const fileOrPath = f.fileOrPath; + const fileOrPath = f.uploadItem; /* TODO(MR): ElectronFile */ if (!(fileOrPath instanceof File || typeof fileOrPath == "string")) throw new Error(`Unexpected file ${f}`); diff --git a/web/apps/photos/src/services/watch.ts b/web/apps/photos/src/services/watch.ts index 4de5881aa..82d3b2f4e 100644 --- a/web/apps/photos/src/services/watch.ts +++ b/web/apps/photos/src/services/watch.ts @@ -15,7 +15,7 @@ import { ensureString } from "@/utils/ensure"; import { UPLOAD_RESULT } from "constants/upload"; import debounce from "debounce"; import uploadManager, { - type FileWithCollection, + type UploadItemWithCollection, } from "services/upload/uploadManager"; import { Collection } from "types/collection"; import { EncryptedEnteFile } from "types/file"; @@ -317,16 +317,17 @@ class FolderWatcher { } /** - * Callback invoked by the uploader whenever a file we requested to + * Callback invoked by the uploader whenever a item we requested to * {@link upload} gets uploaded. */ async onFileUpload( fileUploadResult: UPLOAD_RESULT, - fileWithCollection: FileWithCollection, + item: UploadItemWithCollection, file: EncryptedEnteFile, ) { - // The files we get here will have fileWithCollection.file as a string, - // not as a File or a ElectronFile + // Re the usage of ensureString: For desktop watch, the only possibility + // for a UploadItem is for it to be a string (the absolute path to a + // file on disk). if ( [ UPLOAD_RESULT.ADDED_SYMLINK, @@ -335,18 +336,18 @@ class FolderWatcher { UPLOAD_RESULT.ALREADY_UPLOADED, ].includes(fileUploadResult) ) { - if (fileWithCollection.isLivePhoto) { + if (item.isLivePhoto) { this.uploadedFileForPath.set( - ensureString(fileWithCollection.livePhotoAssets.image), + ensureString(item.livePhotoAssets.image), file, ); this.uploadedFileForPath.set( - ensureString(fileWithCollection.livePhotoAssets.video), + ensureString(item.livePhotoAssets.video), file, ); } else { this.uploadedFileForPath.set( - ensureString(fileWithCollection.fileOrPath), + ensureString(item.uploadItem), file, ); } @@ -355,17 +356,15 @@ class FolderWatcher { fileUploadResult, ) ) { - if (fileWithCollection.isLivePhoto) { + if (item.isLivePhoto) { this.unUploadableFilePaths.add( - ensureString(fileWithCollection.livePhotoAssets.image), + ensureString(item.livePhotoAssets.image), ); this.unUploadableFilePaths.add( - ensureString(fileWithCollection.livePhotoAssets.video), + ensureString(item.livePhotoAssets.video), ); } else { - this.unUploadableFilePaths.add( - ensureString(fileWithCollection.fileOrPath), - ); + this.unUploadableFilePaths.add(ensureString(item.uploadItem)); } } } @@ -375,7 +374,7 @@ class FolderWatcher { * {@link upload} get uploaded. */ async allFileUploadsDone( - filesWithCollection: FileWithCollection[], + uploadItemsWithCollection: UploadItemWithCollection[], collections: Collection[], ) { const electron = ensureElectron(); @@ -384,14 +383,15 @@ class FolderWatcher { log.debug(() => JSON.stringify({ f: "watch/allFileUploadsDone", - filesWithCollection, + uploadItemsWithCollection, collections, watch, }), ); - const { syncedFiles, ignoredFiles } = - this.deduceSyncedAndIgnored(filesWithCollection); + const { syncedFiles, ignoredFiles } = this.deduceSyncedAndIgnored( + uploadItemsWithCollection, + ); if (syncedFiles.length > 0) await electron.watch.updateSyncedFiles( @@ -411,7 +411,9 @@ class FolderWatcher { this.debouncedRunNextEvent(); } - private deduceSyncedAndIgnored(filesWithCollection: FileWithCollection[]) { + private deduceSyncedAndIgnored( + uploadItemsWithCollection: UploadItemWithCollection[], + ) { const syncedFiles: FolderWatch["syncedFiles"] = []; const ignoredFiles: FolderWatch["ignoredFiles"] = []; @@ -430,14 +432,13 @@ class FolderWatcher { this.unUploadableFilePaths.delete(path); }; - for (const fileWithCollection of filesWithCollection) { - if (fileWithCollection.isLivePhoto) { - const imagePath = ensureString( - fileWithCollection.livePhotoAssets.image, - ); - const videoPath = ensureString( - fileWithCollection.livePhotoAssets.video, - ); + for (const item of uploadItemsWithCollection) { + // Re the usage of ensureString: For desktop watch, the only + // possibility for a UploadItem is for it to be a string (the + // absolute path to a file on disk). + if (item.isLivePhoto) { + const imagePath = ensureString(item.livePhotoAssets.image); + const videoPath = ensureString(item.livePhotoAssets.video); const imageFile = this.uploadedFileForPath.get(imagePath); const videoFile = this.uploadedFileForPath.get(videoPath); @@ -453,7 +454,7 @@ class FolderWatcher { markIgnored(videoPath); } } else { - const path = ensureString(fileWithCollection.fileOrPath); + const path = ensureString(item.uploadItem); const file = this.uploadedFileForPath.get(path); if (file) { markSynced(file, path);