implemented delete sync functionality

src/components/ExportModal.tsx

@@ -225,7 +225,7 @@ export default function ExportModal(props: Props) {
     const startExport = () => {
         try {
             verifyExportFolderExists();
-            exportService.runExport();
+            exportService.scheduleExport();
         } catch (e) {
             if (e.message !== CustomError.EXPORT_FOLDER_DOES_NOT_EXIST) {
                 logError(e, 'startExport failed');

src/services/exportService.ts

@@ -13,9 +13,11 @@ import {
     getOldFileMetadataSavePath,
     getExportedFiles,
     getMetadataFolderPath,
-    getCollectionsRenamedAfterLastExport,
+    getRenamedCollections,
-    convertIDPathObjectToMap,
+    getDeletedExportedFiles,
-    convertMapToIDPathObject,
+    convertCollectionIDPathObjectToMap,
+    convertFileIDPathObjectToMap,
+    getDeletedExportedCollections,
 } from 'utils/export';
 import { retryAsyncFunction } from 'utils/network';
 import { logError } from 'utils/sentry';
@@ -43,7 +45,9 @@ import {
     ExportProgress,
     ExportRecord,
     ExportRecordV1,
+    ExportRecordV2,
     ExportUIUpdaters,
+    ExportedFilePaths,
     FileExportStats,
 } from 'types/export';
 import { User } from 'types/user';
@@ -52,7 +56,6 @@ import { ExportStage } from 'constants/export';
 import { ElectronAPIs } from 'types/electron';
 import { CustomError } from 'utils/error';
 import { addLogLine } from 'utils/logging';
-import { t } from 'i18next';
 import { eventBus, Events } from './events';
 import { getCollectionNameMap } from 'utils/collection';
 
@@ -119,7 +122,7 @@ class ExportService {
                 addLogLine('continuous export already enabled');
                 return;
             }
-            this.continuousExportEventHandler = this.runExport;
+            this.continuousExportEventHandler = this.scheduleExport;
             this.continuousExportEventHandler();
             eventBus.addListener(
                 Events.LOCAL_FILES_UPDATED,
@@ -166,6 +169,23 @@ class ExportService {
         }
     };
 
+    async preExport() {
+        this.stopExport = false;
+        this.exportInProgress = true;
+        await this.uiUpdater.updateExportStage(ExportStage.INPROGRESS);
+        this.updateExportProgress({
+            success: 0,
+            failed: 0,
+            total: 0,
+        });
+    }
+
+    async postExport() {
+        await this.uiUpdater.updateExportStage(ExportStage.FINISHED);
+        await this.uiUpdater.updateLastExportTime(Date.now());
+        this.uiUpdater.updateFileExportStats(await this.getFileExportStats());
+    }
+
     async stopRunningExport() {
         try {
             this.stopExport = true;
@@ -176,44 +196,35 @@ class ExportService {
         }
     }
 
-    runExport = async () => {
+    scheduleExport = async () => {
         try {
             if (this.exportInProgress) {
                 addLogLine('export in progress, scheduling re-run');
-                this.electronAPIs.sendNotification(
-                    t('EXPORT_NOTIFICATION.IN_PROGRESS')
-                );
                 this.reRunNeeded = true;
                 return;
             }
             try {
-                addLogLine('starting export');
+                await this.preExport();
-                this.exportInProgress = true;
+                addLogLine('export started');
-                await this.uiUpdater.updateExportStage(ExportStage.INPROGRESS);
+                await this.runExport();
-                this.updateExportProgress({
-                    success: 0,
-                    failed: 0,
-                    total: 0,
-                });
-                await this.exportFiles();
                 addLogLine('export completed');
             } finally {
                 this.exportInProgress = false;
                 if (this.reRunNeeded) {
                     this.reRunNeeded = false;
                     addLogLine('re-running export');
-                    setTimeout(this.runExport, 0);
+                    setTimeout(this.scheduleExport, 0);
                 }
                 await this.postExport();
             }
         } catch (e) {
             if (e.message !== CustomError.EXPORT_FOLDER_DOES_NOT_EXIST) {
-                logError(e, 'runExport failed');
+                logError(e, 'scheduleExport failed');
             }
         }
     };
 
-    private async exportFiles() {
+    private async runExport() {
         try {
             const exportDir = getData(LS_KEYS.EXPORT)?.folder;
             if (!exportDir) {
@@ -241,67 +252,124 @@ class ExportService {
             if (this.checkAllElectronAPIsExists()) {
                 await this.migrateExport(
                     exportDir,
-                    collections,
+                    userCollections,
                     userPersonalFiles
                 );
             }
             const exportRecord = await this.getExportRecord(exportDir);
 
+            const collectionIDPathMap = convertCollectionIDPathObjectToMap(
+                exportRecord.exportedCollectionPaths
+            );
+            const collectionIDNameMap = getCollectionNameMap(collections);
+            const renamedCollections = getRenamedCollections(
+                userCollections,
+                exportRecord
+            );
+
+            if (
+                renamedCollections?.length > 0 &&
+                this.checkAllElectronAPIsExists()
+            ) {
+                this.collectionRenamer(
+                    exportDir,
+                    collectionIDPathMap,
+                    renamedCollections
+                );
+            }
+
+            const deletedExportedCollections = getDeletedExportedCollections(
+                userCollections,
+                exportRecord
+            );
+
+            if (deletedExportedCollections?.length > 0) {
+                await this.collectionRemover(
+                    deletedExportedCollections,
+                    exportDir
+                );
+            }
+
             const filesToExport = getUnExportedFiles(
                 userPersonalFiles,
                 exportRecord
             );
 
-            addLogLine(
+            if (filesToExport?.length > 0) {
-                `exportFiles: filesToExportCount: ${filesToExport?.length}, userPersonalFileCount: ${userPersonalFiles?.length}`
+                await this.fileExporter(
-            );
+                    filesToExport,
-
+                    collectionIDNameMap,
-            const collectionIDPathMap = convertIDPathObjectToMap(
+                    collectionIDPathMap,
-                exportRecord.exportedCollectionPaths
+                    exportDir
-            );
+                );
-            const collectionIDNameMap = getCollectionNameMap(collections);
+            }
-            const renamedCollections = getCollectionsRenamedAfterLastExport(
+            const removedFileUIDs = getDeletedExportedFiles(
-                userCollections,
+                userPersonalFiles,
                 exportRecord
             );
-            await this.fileExporter(
+
-                filesToExport,
+            if (removedFileUIDs?.length) {
-                collectionIDNameMap,
+                await this.fileRemover(removedFileUIDs, exportDir);
-                renamedCollections,
+            }
-                collectionIDPathMap,
-                exportDir
-            );
         } catch (e) {
-            logError(e, 'exportFiles failed');
+            logError(e, 'runExport failed');
+        }
+    }
+
+    async collectionRenamer(
+        exportFolder: string,
+        collectionIDPathMap: Map<number, string>,
+        renamedCollections: Collection[]
+    ) {
+        for (const collection of renamedCollections) {
+            const oldCollectionFolderPath = collectionIDPathMap.get(
+                collection.id
+            );
+
+            const newCollectionFolderPath = getUniqueCollectionFolderPath(
+                exportFolder,
+                collection.id,
+                collection.name
+            );
+            await this.electronAPIs.checkExistsAndRename(
+                oldCollectionFolderPath,
+                newCollectionFolderPath
+            );
+
+            await this.addCollectionExportedRecord(
+                exportFolder,
+                collection.id,
+                newCollectionFolderPath
+            );
+            collectionIDPathMap.set(collection.id, newCollectionFolderPath);
+        }
+    }
+
+    async collectionRemover(
+        deletedExportedCollectionIDs: number[],
+        exportFolder: string
+    ) {
+        const exportRecord = await this.getExportRecord(exportFolder);
+        const collectionIDPathMap = convertCollectionIDPathObjectToMap(
+            exportRecord.exportedCollectionPaths
+        );
+        for (const collectionID of deletedExportedCollectionIDs) {
+            const collectionFolderPath = collectionIDPathMap.get(collectionID);
+            await this.electronAPIs.removeFolder(collectionFolderPath);
+            await this.removeCollectionExportedRecord(
+                exportFolder,
+                collectionID
+            );
         }
     }
 
     async fileExporter(
         files: EnteFile[],
         collectionIDNameMap: Map<number, string>,
-        renamedCollections: Collection[],
         collectionIDPathMap: Map<number, string>,
         exportDir: string
     ): Promise<void> {
         try {
-            if (
-                renamedCollections?.length &&
-                this.checkAllElectronAPIsExists()
-            ) {
-                await this.renameCollectionFolders(
-                    renamedCollections,
-                    exportDir,
-                    collectionIDPathMap
-                );
-            }
-            if (!files?.length) {
-                this.electronAPIs.sendNotification(
-                    t('EXPORT_NOTIFICATION.UP_TO_DATE')
-                );
-                return;
-            }
-            this.stopExport = false;
-            this.electronAPIs.sendNotification(t('EXPORT_NOTIFICATION.START'));
             let success = 0;
             let failed = 0;
             this.updateExportProgress({
@@ -351,21 +419,57 @@ class ExportService {
                     total: files.length,
                 });
             }
-            if (!this.stopExport) {
-                this.electronAPIs.sendNotification(
-                    t('EXPORT_NOTIFICATION.FINISH')
-                );
-            }
         } catch (e) {
             logError(e, 'fileExporter failed');
             throw e;
         }
     }
 
-    async postExport() {
+    async fileRemover(
-        await this.uiUpdater.updateExportStage(ExportStage.FINISHED);
+        removedFileUIDs: string[],
-        await this.uiUpdater.updateLastExportTime(Date.now());
+        exportDir: string
-        this.uiUpdater.updateFileExportStats(await this.getFileExportStats());
+    ): Promise<void> {
+        try {
+            let success = 0;
+            let failed = 0;
+            this.updateExportProgress({
+                success,
+                failed,
+                total: removedFileUIDs.length,
+            });
+            const exportRecord = await this.getExportRecord(exportDir);
+            const fileIDPathMap = convertFileIDPathObjectToMap(
+                exportRecord.exportedFilePaths
+            );
+            for (const fileUID of removedFileUIDs) {
+                if (this.stopExport) {
+                    break;
+                }
+                try {
+                    const filePath = fileIDPathMap.get(fileUID);
+                    await this.removeFile(filePath);
+                    await this.removeFileExportedRecord(exportDir, fileUID);
+                    success++;
+                } catch (e) {
+                    failed++;
+                    logError(e, 'remove failed for a file');
+                    if (
+                        e.message ===
+                        CustomError.ADD_FILE_EXPORTED_RECORD_FAILED
+                    ) {
+                        throw e;
+                    }
+                }
+                this.updateExportProgress({
+                    success,
+                    failed,
+                    total: removedFileUIDs.length,
+                });
+            }
+        } catch (e) {
+            logError(e, 'fileRemover failed');
+            throw e;
+        }
     }
 
     async addFileExportedRecord(
@@ -376,10 +480,6 @@ class ExportService {
         try {
             const fileUID = getExportRecordFileUID(file);
             const exportRecord = await this.getExportRecord(folder);
-            if (!exportRecord.exportedFiles) {
-                exportRecord.exportedFiles = [];
-            }
-            exportRecord.exportedFiles.push(fileUID);
             if (!exportRecord.exportedFilePaths) {
                 exportRecord.exportedFilePaths = {};
             }
@@ -399,16 +499,53 @@ class ExportService {
         collectionID: number,
         collectionFolderPath: string
     ) {
-        const exportRecord = await this.getExportRecord(folder);
+        try {
-        if (!exportRecord?.exportedCollectionPaths) {
+            const exportRecord = await this.getExportRecord(folder);
-            exportRecord.exportedCollectionPaths = {};
+            if (!exportRecord?.exportedCollectionPaths) {
-        }
+                exportRecord.exportedCollectionPaths = {};
-        exportRecord.exportedCollectionPaths = {
+            }
-            ...exportRecord.exportedCollectionPaths,
+            exportRecord.exportedCollectionPaths = {
-            [collectionID]: collectionFolderPath,
+                ...exportRecord.exportedCollectionPaths,
-        };
+                [collectionID]: collectionFolderPath,
+            };
 
-        await this.updateExportRecord(exportRecord, folder);
+            await this.updateExportRecord(exportRecord, folder);
+        } catch (e) {
+            logError(e, 'addCollectionExportedRecord failed');
+            throw Error(CustomError.ADD_FILE_EXPORTED_RECORD_FAILED);
+        }
+    }
+
+    async removeCollectionExportedRecord(folder: string, collectionID: number) {
+        try {
+            const exportRecord = await this.getExportRecord(folder);
+
+            exportRecord.exportedCollectionPaths = Object.fromEntries(
+                Object.entries(exportRecord.exportedCollectionPaths).filter(
+                    ([key]) => key === collectionID.toString()
+                )
+            );
+
+            await this.updateExportRecord(exportRecord, folder);
+        } catch (e) {
+            logError(e, 'removeCollectionExportedRecord failed');
+            throw Error(CustomError.ADD_FILE_EXPORTED_RECORD_FAILED);
+        }
+    }
+
+    async removeFileExportedRecord(folder: string, fileUID: string) {
+        try {
+            const exportRecord = await this.getExportRecord(folder);
+            exportRecord.exportedFilePaths = Object.fromEntries(
+                Object.entries(exportRecord.exportedFilePaths).filter(
+                    ([key]) => key === fileUID
+                )
+            );
+            await this.updateExportRecord(exportRecord, folder);
+        } catch (e) {
+            logError(e, 'removeFileExportedRecord failed');
+            throw Error(CustomError.ADD_FILE_EXPORTED_RECORD_FAILED);
+        }
     }
 
     async updateExportRecord(newData: Partial<ExportRecord>, folder?: string) {
@@ -620,6 +757,10 @@ class ExportService {
         );
     }
 
+    private async removeFile(filePath: string) {
+        await this.electronAPIs.removeFile(filePath);
+    }
+
     isExportInProgress = () => {
         return this.exportInProgress;
     };
@@ -669,7 +810,7 @@ class ExportService {
             });
         }
         if (currentVersion === 2) {
-            await this.addExportedFilePathsProperty(
+            await this.updateExportedFilesToExportedFilePathsProperty(
                 getExportedFiles(files, exportRecord)
             );
             currentVersion++;
@@ -772,12 +913,15 @@ class ExportService {
         await this.updateExportRecord(exportRecord);
     }
 
-    private async addExportedFilePathsProperty(exportedFiles: EnteFile[]) {
+    private async updateExportedFilesToExportedFilePathsProperty(
-        const exportRecord = await this.getExportRecord();
+        exportedFiles: EnteFile[]
-        const exportedFilePaths = new Map<number, string>();
+    ) {
+        const exportRecord =
+            (await this.getExportRecord()) as unknown as ExportRecordV2;
+        let exportedFilePaths: ExportedFilePaths;
         const usedFilePaths = new Set<string>();
-        const exportedCollectionPaths = convertIDPathObjectToMap(
+        const exportedCollectionPaths = convertCollectionIDPathObjectToMap(
-            exportRecord?.exportedCollectionPaths
+            exportRecord.exportedCollectionPaths
         );
         for (const file of exportedFiles) {
             const collectionPath = exportedCollectionPaths.get(
@@ -789,11 +933,19 @@ class ExportService {
                 usedFilePaths
             );
             const filePath = getFileSavePath(collectionPath, fileSaveName);
-            exportedFilePaths.set(file.id, filePath);
+            exportedFilePaths = {
+                ...exportedFilePaths,
+                [getExportRecordFileUID(file)]: filePath,
+            };
         }
-        exportRecord.exportedFilePaths =
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
-            convertMapToIDPathObject(exportedFilePaths);
+        const { exportedFiles: _, ...rest } = exportRecord;
-        await this.updateExportRecord(exportRecord);
+        const updatedExportRecord: ExportRecord = {
+            ...rest,
+            exportedFilePaths,
+        };
+
+        await this.updateExportRecord(updatedExportRecord);
     }
 }
 export default new ExportService();

src/types/electron/index.ts

@@ -90,4 +90,6 @@ export interface ElectronAPIs {
     logRendererProcessMemoryUsage: (message: string) => Promise<void>;
     registerForegroundEventListener: (onForeground: () => void) => void;
     openDirectory: (dirPath: string) => Promise<void>;
+    removeFile: (filePath: string) => Promise<void>;
+    removeFolder: (folderPath: string) => Promise<void>;
 }

src/types/export/index.ts

@@ -5,10 +5,13 @@ export interface ExportProgress {
     failed: number;
     total: number;
 }
-export interface ExportedEntityPaths {
+export interface ExportedCollectionPaths {
     [ID: number]: string;
 }
 
+export interface ExportedFilePaths {
+    [ID: string]: string;
+}
 export interface FileExportStats {
     totalCount: number;
     pendingCount: number;
@@ -22,16 +25,23 @@ export interface ExportRecordV1 {
     queuedFiles?: string[];
     exportedFiles?: string[];
     failedFiles?: string[];
-    exportedCollectionPaths?: ExportedEntityPaths;
+    exportedCollectionPaths?: ExportedCollectionPaths;
+}
+
+export interface ExportRecordV2 {
+    version: number;
+    stage: ExportStage;
+    lastAttemptTimestamp: number;
+    exportedFiles: string[];
+    exportedCollectionPaths: ExportedCollectionPaths;
 }
 
 export interface ExportRecord {
     version: number;
     stage: ExportStage;
     lastAttemptTimestamp: number;
-    exportedFiles: string[];
+    exportedCollectionPaths: ExportedCollectionPaths;
-    exportedCollectionPaths: ExportedEntityPaths;
+    exportedFilePaths: ExportedFilePaths;
-    exportedFilePaths: ExportedEntityPaths;
 }
 
 export interface ExportSettings {

src/utils/export/index.ts

@@ -1,6 +1,10 @@
 import { Collection } from 'types/collection';
 import exportService from 'services/exportService';
-import { ExportRecord, ExportedEntityPaths } from 'types/export';
+import {
+    ExportRecord,
+    ExportedCollectionPaths,
+    ExportedFilePaths,
+} from 'types/export';
 
 import { EnteFile } from 'types/file';
 
@@ -30,31 +34,31 @@ export const getCollectionsCreatedAfterLastExport = (
     });
     return unExportedCollections;
 };
-export const convertIDPathObjectToMap = (
+export const convertCollectionIDPathObjectToMap = (
-    exportedEntityPaths: ExportedEntityPaths
+    exportedCollectionPaths: ExportedCollectionPaths
-) => {
+): Map<number, string> => {
     return new Map<number, string>(
-        Object.entries(exportedEntityPaths ?? {}).map((e) => {
+        Object.entries(exportedCollectionPaths ?? {}).map((e) => {
             return [Number(e[0]), String(e[1])];
         })
     );
 };
 
-export const convertMapToIDPathObject = (
+export const convertFileIDPathObjectToMap = (
-    exportedEntityPaths: Map<number, string>
+    exportedFilePaths: ExportedFilePaths
-) => {
+): Map<string, string> => {
-    const exportedEntityPathsObject: ExportedEntityPaths = {};
+    return new Map<string, string>(
-    exportedEntityPaths.forEach((value, key) => {
+        Object.entries(exportedFilePaths ?? {}).map((e) => {
-        exportedEntityPathsObject[key] = value;
+            return [String(e[0]), String(e[1])];
-    });
+        })
-    return exportedEntityPathsObject;
+    );
 };
 
-export const getCollectionsRenamedAfterLastExport = (
+export const getRenamedCollections = (
     collections: Collection[],
     exportRecord: ExportRecord
 ) => {
-    const collectionIDPathMap = convertIDPathObjectToMap(
+    const collectionIDPathMap = convertCollectionIDPathObjectToMap(
         exportRecord.exportedCollectionPaths
     );
     const renamedCollections = collections.filter((collection) => {
@@ -76,11 +80,31 @@ export const getCollectionsRenamedAfterLastExport = (
     return renamedCollections;
 };
 
+export const getDeletedExportedCollections = (
+    collections: Collection[],
+    exportRecord: ExportRecord
+) => {
+    const presentCollections = new Set(
+        collections?.map((collection) => collection.id)
+    );
+    const deletedExportedCollections = Object.keys(
+        exportRecord?.exportedCollectionPaths
+    )
+        .map(Number)
+        .filter((collectionID) => {
+            if (!presentCollections.has(collectionID)) {
+                return true;
+            }
+            return false;
+        });
+    return deletedExportedCollections;
+};
+
 export const getUnExportedFiles = (
     allFiles: EnteFile[],
     exportRecord: ExportRecord
 ) => {
-    const exportedFiles = new Set(exportRecord?.exportedFiles);
+    const exportedFiles = new Set(Object.keys(exportRecord?.exportedFilePaths));
     const unExportedFiles = allFiles.filter((file) => {
         if (!exportedFiles.has(getExportRecordFileUID(file))) {
             return true;
@@ -94,7 +118,9 @@ export const getExportedFiles = (
     allFiles: EnteFile[],
     exportRecord: ExportRecord
 ) => {
-    const exportedFileIds = new Set(exportRecord?.exportedFiles);
+    const exportedFileIds = new Set(
+        Object.keys(exportRecord?.exportedFilePaths)
+    );
     const exportedFiles = allFiles.filter((file) => {
         if (exportedFileIds.has(getExportRecordFileUID(file))) {
             return true;
@@ -104,6 +130,24 @@ export const getExportedFiles = (
     return exportedFiles;
 };
 
+export const getDeletedExportedFiles = (
+    allFiles: EnteFile[],
+    exportRecord: ExportRecord
+): string[] => {
+    const presentFileUIDs = new Set(
+        allFiles?.map((file) => getExportRecordFileUID(file))
+    );
+    const deletedExportedFiles = Object.keys(
+        exportRecord?.exportedFilePaths
+    ).filter((fileUID) => {
+        if (!presentFileUIDs.has(fileUID)) {
+            return true;
+        }
+        return false;
+    });
+    return deletedExportedFiles;
+};
+
 export const getGoogleLikeMetadataFile = (
     fileSaveName: string,
     file: EnteFile