WIP 1

desktop/src/main/fs.ts

@@ -27,8 +27,3 @@ export const fsIsDir = async (dirPath: string) => {
     const stat = await fs.stat(dirPath);
     return stat.isDirectory();
 };
-
-export const fsLsFiles = async (dirPath: string) =>
-    (await fs.readdir(dirPath, { withFileTypes: true }))
-        .filter((e) => e.isFile())
-        .map((e) => e.name);

desktop/src/main/ipc.ts

@@ -20,7 +20,7 @@ import {
 import {
     fsExists,
     fsIsDir,
-    fsLsFiles,
+    fsListFiles,
     fsMkdirIfNeeded,
     fsReadTextFile,
     fsRename,
@@ -135,7 +135,7 @@ export const attachIPCHandlers = () => {
 
     ipcMain.handle("fsIsDir", (_, dirPath: string) => fsIsDir(dirPath));
 
-    ipcMain.handle("fsLsFiles", (_, dirPath: string) => fsLsFiles(dirPath));
+    ipcMain.handle("fsListFiles", (_, dirPath: string) => fsListFiles(dirPath));
 
     // - Conversion
 

desktop/src/main/services/watch.ts

@@ -2,6 +2,32 @@ import type { FSWatcher } from "chokidar";
 import ElectronLog from "electron-log";
 import { FolderWatch, WatchStoreType } from "../../types/ipc";
 import { watchStore } from "../stores/watch.store";
+import path from "node:path";
+import fs from "node:fs/promises";
+
+/**
+ * Return the paths of all the files under the given directory (recursive).
+ *
+ * This function walks the directory tree starting at {@link dirPath}, and
+ * returns a list of the absolute paths of all the files that exist therein. It
+ * will recursively traverse into nested directories, and return the absolute
+ * paths of the files there too.
+ *
+ * The returned paths are guaranteed to use POSIX separators ('/').
+ */
+export const findFiles = async (dirPath: string) => {
+    const items = await fs.readdir(dirPath, { withFileTypes: true })
+    let paths: string[] = [];
+    for (const item of items) {
+        const itemPath = path.posix.join(dirPath, item.name);
+        if (item.isFile()) {
+            paths.push(itemPath)
+        } else if (item.isDirectory()) {
+            paths = [...paths, ...await findFiles(itemPath)]
+        }
+    }
+    return paths
+}
 
 export const addWatchMapping = async (
     watcher: FSWatcher,

desktop/src/preload.ts

@@ -121,8 +121,8 @@ const fsWriteFile = (path: string, contents: string): Promise<void> =>
 const fsIsDir = (dirPath: string): Promise<boolean> =>
     ipcRenderer.invoke("fsIsDir", dirPath);
 
-const fsLsFiles = (dirPath: string): Promise<boolean> =>
+const fsListFiles = (dirPath: string): Promise<string[]> =>
-    ipcRenderer.invoke("fsLsFiles", dirPath);
+    ipcRenderer.invoke("fsListFiles", dirPath);
 
 // - AUDIT below this
 
@@ -325,7 +325,7 @@ contextBridge.exposeInMainWorld("electron", {
         readTextFile: fsReadTextFile,
         writeFile: fsWriteFile,
         isDir: fsIsDir,
-        lsFiles: fsLsFiles,
+        listFiles: fsListFiles,
     },
 
     // - Conversion

web/apps/photos/src/services/watch.ts

@@ -12,19 +12,43 @@ import uploadManager from "services/upload/uploadManager";
 import { Collection } from "types/collection";
 import { EncryptedEnteFile } from "types/file";
 import { ElectronFile, FileWithCollection } from "types/upload";
-import {
+import { WatchMapping, WatchMappingSyncedFile } from "types/watchFolder";
-    EventQueueItem,
-    WatchMapping,
-    WatchMappingSyncedFile,
-} from "types/watchFolder";
 import { groupFilesBasedOnCollectionID } from "utils/file";
 import { isSystemFile } from "utils/upload";
 import { removeFromCollection } from "./collectionService";
 import { getLocalFiles } from "./fileService";
 
+/**
+ * A file system event encapsulates a change that has occurred on disk that
+ * needs us to take some action within Ente to synchronize with the user's
+ * (Ente) albums.
+ *
+ * Events get added in two ways:
+ *
+ * - When the app starts, it reads the current state of files on disk and
+ *   compares that with its last known state to determine what all events it
+ *   missed. This is easier than it sounds as we have only two events, add and
+ *   remove.
+ *
+ * - When the app is running, it gets live notifications from our file system
+ *   watcher (from the Node.js layer) about changes that have happened on disk,
+ *   which the app then enqueues onto the event queue if they pertain to the
+ *   files we're interested in.
+ */
+interface FSEvent {
+    /** The action to take */
+    action: "upload" | "trash";
+    /** The path of the root folder corresponding to the {@link FolderWatch}. */
+    folderPath: string;
+    /** If applicable, the name of the (Ente) collection the file belongs to. */
+    collectionName?: string;
+    /** The absolute path to the file under consideration. */
+    filePath: string;
+}
+
 class WatchFolderService {
-    private eventQueue: EventQueueItem[] = [];
+    private eventQueue: FSEvent[] = [];
-    private currentEvent: EventQueueItem;
+    private currentEvent: FSEvent;
     private currentlySyncedMapping: WatchMapping;
     private trashingDirQueue: string[] = [];
     private isEventRunning: boolean = false;
@@ -94,6 +118,7 @@ class WatchFolderService {
 
     pushEvent(event: EventQueueItem) {
         this.eventQueue.push(event);
+        log.info("FS event", event);
         this.debouncedRunNextEvent();
     }
 
@@ -566,55 +591,41 @@ const getParentFolderName = (filePath: string) => {
 };
 
 async function diskFileAddedCallback(file: ElectronFile) {
-    try {
+    const collectionNameAndFolderPath =
-        const collectionNameAndFolderPath =
+        await watchFolderService.getCollectionNameAndFolderPath(file.path);
-            await watchFolderService.getCollectionNameAndFolderPath(file.path);
 
-        if (!collectionNameAndFolderPath) {
+    if (!collectionNameAndFolderPath) {
-            return;
+        return;
-        }
-
-        const { collectionName, folderPath } = collectionNameAndFolderPath;
-
-        const event: EventQueueItem = {
-            type: "upload",
-            collectionName,
-            folderPath,
-            files: [file],
-        };
-        watchFolderService.pushEvent(event);
-        log.info(
-            `added (upload) to event queue, collectionName:${event.collectionName} folderPath:${event.folderPath}, filesCount: ${event.files.length}`,
-        );
-    } catch (e) {
-        log.error("error while calling diskFileAddedCallback", e);
     }
+
+    const { collectionName, folderPath } = collectionNameAndFolderPath;
+
+    const event: EventQueueItem = {
+        type: "upload",
+        collectionName,
+        folderPath,
+        path: file.path,
+    };
+    watchFolderService.pushEvent(event);
 }
 
 async function diskFileRemovedCallback(filePath: string) {
-    try {
+    const collectionNameAndFolderPath =
-        const collectionNameAndFolderPath =
+        await watchFolderService.getCollectionNameAndFolderPath(filePath);
-            await watchFolderService.getCollectionNameAndFolderPath(filePath);
 
-        if (!collectionNameAndFolderPath) {
+    if (!collectionNameAndFolderPath) {
-            return;
+        return;
-        }
-
-        const { collectionName, folderPath } = collectionNameAndFolderPath;
-
-        const event: EventQueueItem = {
-            type: "trash",
-            collectionName,
-            folderPath,
-            paths: [filePath],
-        };
-        watchFolderService.pushEvent(event);
-        log.info(
-            `added (trash) to event queue collectionName:${event.collectionName} folderPath:${event.folderPath} , pathsCount: ${event.paths.length}`,
-        );
-    } catch (e) {
-        log.error("error while calling diskFileRemovedCallback", e);
     }
+
+    const { collectionName, folderPath } = collectionNameAndFolderPath;
+
+    const event: EventQueueItem = {
+        type: "trash",
+        collectionName,
+        folderPath,
+        path: filePath,
+    };
+    watchFolderService.pushEvent(event);
 }
 
 async function diskFolderRemovedCallback(folderPath: string) {
@@ -682,34 +693,51 @@ const syncWithDisk = async (
     const events: EventQueueItem[] = [];
 
     for (const mapping of activeMappings) {
-        const files = await electron.getDirFiles(mapping.folderPath);
+        const folderPath = mapping.folderPath;
 
-        const filesToUpload = getValidFilesToUpload(files, mapping);
+        const paths = (await electron.fs.listFiles(folderPath))
+            // Filter out hidden files (files whose names begins with a dot)
+            .filter((n) => !n.startsWith("."))
+            // Prepend folderPath to get the full path
+            .map((f) => `${folderPath}/${f}`);
 
-        for (const file of filesToUpload)
+        // Files that are on disk but not yet synced.
+        const pathsToUpload = paths.filter(
+            (path) => !isSyncedOrIgnoredPath(path, mapping),
+        );
+
+        for (const path of pathsToUpload)
             events.push({
                 type: "upload",
-                collectionName: getCollectionNameForMapping(mapping, file.path),
+                collectionName: getCollectionNameForMapping(mapping, path),
-                folderPath: mapping.folderPath,
+                folderPath,
-                files: [file],
+                filePath: path,
             });
 
-        const filesToRemove = mapping.syncedFiles.filter((file) => {
+        // Synced files that are no longer on disk
-            return !files.find((f) => f.path === file.path);
+        const pathsToRemove = mapping.syncedFiles.filter(
-        });
+            (file) => !paths.includes(file.path),
+        );
 
-        for (const file of filesToRemove)
+        for (const path of pathsToRemove)
             events.push({
                 type: "trash",
-                collectionName: getCollectionNameForMapping(mapping, file.path),
+                collectionName: getCollectionNameForMapping(mapping, path),
                 folderPath: mapping.folderPath,
-                paths: [file.path],
+                filePath: path,
             });
     }
 
     return { events, nonExistentFolderPaths };
 };
 
+function isSyncedOrIgnoredPath(path: string, mapping: WatchMapping) {
+    return (
+        mapping.ignoredFiles.includes(path) ||
+        mapping.syncedFiles.find((f) => f.path === path)
+    );
+}
+
 const getCollectionNameForMapping = (
     mapping: WatchMapping,
     filePath: string,

web/apps/photos/src/types/watchFolder/index.ts

@@ -1,5 +1,4 @@
 import { UPLOAD_STRATEGY } from "constants/upload";
-import { ElectronFile } from "types/upload";
 
 export interface WatchMappingSyncedFile {
     path: string;
@@ -14,11 +13,3 @@ export interface WatchMapping {
     syncedFiles: WatchMappingSyncedFile[];
     ignoredFiles: string[];
 }
-
-export interface EventQueueItem {
-    type: "upload" | "trash";
-    folderPath: string;
-    collectionName?: string;
-    paths?: string[];
-    files?: ElectronFile[];
-}

web/packages/next/types/ipc.ts

@@ -207,7 +207,7 @@ export interface Electron {
         isDir: (dirPath: string) => Promise<boolean>;
 
         /**
-         * Return a list of the names of the files in the given directory.
+         * Return a list of the file names of the files in the given directory.
          *
          * Note:
          *
@@ -216,7 +216,7 @@ export interface Electron {
          *
          * - It will return only the names of files, not directories.
          */
-        lsFiles: (dirPath: string) => Promise<string>;
+        listFiles: (dirPath: string) => Promise<string[]>;
     };
 
     /*