implemented delete sync functionality

2023-04-25 11:56:45 +05:30 · 2023-04-25 11:56:45 +05:30 · 575c61ff93
parent e6ac93cf70
commit 575c61ff93
5 changed files with 322 additions and 114 deletions
--- a/src/components/ExportModal.tsx
+++ b/src/components/ExportModal.tsx
@ -225,7 +225,7 @@ export default function ExportModal(props: Props) {
    const startExport = () => {
        try {
            verifyExportFolderExists();
-            exportService.runExport();
+            exportService.scheduleExport();
        } catch (e) {
            if (e.message !== CustomError.EXPORT_FOLDER_DOES_NOT_EXIST) {
                logError(e, 'startExport failed');
--- a/src/services/exportService.ts
+++ b/src/services/exportService.ts
@ -13,9 +13,11 @@ import {
    getOldFileMetadataSavePath,
    getExportedFiles,
    getMetadataFolderPath,
-    getCollectionsRenamedAfterLastExport,
-    convertIDPathObjectToMap,
-    convertMapToIDPathObject,
+    getRenamedCollections,
+    getDeletedExportedFiles,
+    convertCollectionIDPathObjectToMap,
+    convertFileIDPathObjectToMap,
+    getDeletedExportedCollections,
 } from 'utils/export';
 import { retryAsyncFunction } from 'utils/network';
 import { logError } from 'utils/sentry';
@ -43,7 +45,9 @@ import {
    ExportProgress,
    ExportRecord,
    ExportRecordV1,
+    ExportRecordV2,
    ExportUIUpdaters,
+    ExportedFilePaths,
    FileExportStats,
 } from 'types/export';
 import { User } from 'types/user';
@ -52,7 +56,6 @@ import { ExportStage } from 'constants/export';
 import { ElectronAPIs } from 'types/electron';
 import { CustomError } from 'utils/error';
 import { addLogLine } from 'utils/logging';
-import { t } from 'i18next';
 import { eventBus, Events } from './events';
 import { getCollectionNameMap } from 'utils/collection';

@ -119,7 +122,7 @@ class ExportService {
                addLogLine('continuous export already enabled');
                return;
            }
-            this.continuousExportEventHandler = this.runExport;
+            this.continuousExportEventHandler = this.scheduleExport;
            this.continuousExportEventHandler();
            eventBus.addListener(
                Events.LOCAL_FILES_UPDATED,
@ -166,6 +169,23 @@ class ExportService {
        }
    };

+    async preExport() {
+        this.stopExport = false;
+        this.exportInProgress = true;
+        await this.uiUpdater.updateExportStage(ExportStage.INPROGRESS);
+        this.updateExportProgress({
+            success: 0,
+            failed: 0,
+            total: 0,
+        });
+    }
+
+    async postExport() {
+        await this.uiUpdater.updateExportStage(ExportStage.FINISHED);
+        await this.uiUpdater.updateLastExportTime(Date.now());
+        this.uiUpdater.updateFileExportStats(await this.getFileExportStats());
+    }
+
    async stopRunningExport() {
        try {
            this.stopExport = true;
@ -176,44 +196,35 @@ class ExportService {
        }
    }

-    runExport = async () => {
+    scheduleExport = async () => {
        try {
            if (this.exportInProgress) {
                addLogLine('export in progress, scheduling re-run');
-                this.electronAPIs.sendNotification(
-                    t('EXPORT_NOTIFICATION.IN_PROGRESS')
-                );
                this.reRunNeeded = true;
                return;
            }
            try {
-                addLogLine('starting export');
-                this.exportInProgress = true;
-                await this.uiUpdater.updateExportStage(ExportStage.INPROGRESS);
-                this.updateExportProgress({
-                    success: 0,
-                    failed: 0,
-                    total: 0,
-                });
-                await this.exportFiles();
+                await this.preExport();
+                addLogLine('export started');
+                await this.runExport();
                addLogLine('export completed');
            } finally {
                this.exportInProgress = false;
                if (this.reRunNeeded) {
                    this.reRunNeeded = false;
                    addLogLine('re-running export');
-                    setTimeout(this.runExport, 0);
+                    setTimeout(this.scheduleExport, 0);
                }
                await this.postExport();
            }
        } catch (e) {
            if (e.message !== CustomError.EXPORT_FOLDER_DOES_NOT_EXIST) {
-                logError(e, 'runExport failed');
+                logError(e, 'scheduleExport failed');
            }
        }
    };

-    private async exportFiles() {
+    private async runExport() {
        try {
            const exportDir = getData(LS_KEYS.EXPORT)?.folder;
            if (!exportDir) {
@ -241,67 +252,124 @@ class ExportService {
            if (this.checkAllElectronAPIsExists()) {
                await this.migrateExport(
                    exportDir,
-                    collections,
+                    userCollections,
                    userPersonalFiles
                );
            }
            const exportRecord = await this.getExportRecord(exportDir);

+            const collectionIDPathMap = convertCollectionIDPathObjectToMap(
+                exportRecord.exportedCollectionPaths
+            );
+            const collectionIDNameMap = getCollectionNameMap(collections);
+            const renamedCollections = getRenamedCollections(
+                userCollections,
+                exportRecord
+            );
+
+            if (
+                renamedCollections?.length > 0 &&
+                this.checkAllElectronAPIsExists()
+            ) {
+                this.collectionRenamer(
+                    exportDir,
+                    collectionIDPathMap,
+                    renamedCollections
+                );
+            }
+
+            const deletedExportedCollections = getDeletedExportedCollections(
+                userCollections,
+                exportRecord
+            );
+
+            if (deletedExportedCollections?.length > 0) {
+                await this.collectionRemover(
+                    deletedExportedCollections,
+                    exportDir
+                );
+            }
+
            const filesToExport = getUnExportedFiles(
                userPersonalFiles,
                exportRecord
            );

-            addLogLine(
-                `exportFiles: filesToExportCount: ${filesToExport?.length}, userPersonalFileCount: ${userPersonalFiles?.length}`
-            );
-
-            const collectionIDPathMap = convertIDPathObjectToMap(
-                exportRecord.exportedCollectionPaths
-            );
-            const collectionIDNameMap = getCollectionNameMap(collections);
-            const renamedCollections = getCollectionsRenamedAfterLastExport(
-                userCollections,
+            if (filesToExport?.length > 0) {
+                await this.fileExporter(
+                    filesToExport,
+                    collectionIDNameMap,
+                    collectionIDPathMap,
+                    exportDir
+                );
+            }
+            const removedFileUIDs = getDeletedExportedFiles(
+                userPersonalFiles,
                exportRecord
            );
-            await this.fileExporter(
-                filesToExport,
-                collectionIDNameMap,
-                renamedCollections,
-                collectionIDPathMap,
-                exportDir
-            );
+
+            if (removedFileUIDs?.length) {
+                await this.fileRemover(removedFileUIDs, exportDir);
+            }
        } catch (e) {
-            logError(e, 'exportFiles failed');
+            logError(e, 'runExport failed');
+        }
+    }
+
+    async collectionRenamer(
+        exportFolder: string,
+        collectionIDPathMap: Map<number, string>,
+        renamedCollections: Collection[]
+    ) {
+        for (const collection of renamedCollections) {
+            const oldCollectionFolderPath = collectionIDPathMap.get(
+                collection.id
+            );
+
+            const newCollectionFolderPath = getUniqueCollectionFolderPath(
+                exportFolder,
+                collection.id,
+                collection.name
+            );
+            await this.electronAPIs.checkExistsAndRename(
+                oldCollectionFolderPath,
+                newCollectionFolderPath
+            );
+
+            await this.addCollectionExportedRecord(
+                exportFolder,
+                collection.id,
+                newCollectionFolderPath
+            );
+            collectionIDPathMap.set(collection.id, newCollectionFolderPath);
+        }
+    }
+
+    async collectionRemover(
+        deletedExportedCollectionIDs: number[],
+        exportFolder: string
+    ) {
+        const exportRecord = await this.getExportRecord(exportFolder);
+        const collectionIDPathMap = convertCollectionIDPathObjectToMap(
+            exportRecord.exportedCollectionPaths
+        );
+        for (const collectionID of deletedExportedCollectionIDs) {
+            const collectionFolderPath = collectionIDPathMap.get(collectionID);
+            await this.electronAPIs.removeFolder(collectionFolderPath);
+            await this.removeCollectionExportedRecord(
+                exportFolder,
+                collectionID
+            );
        }
    }

    async fileExporter(
        files: EnteFile[],
        collectionIDNameMap: Map<number, string>,
-        renamedCollections: Collection[],
        collectionIDPathMap: Map<number, string>,
        exportDir: string
    ): Promise<void> {
        try {
-            if (
-                renamedCollections?.length &&
-                this.checkAllElectronAPIsExists()
-            ) {
-                await this.renameCollectionFolders(
-                    renamedCollections,
-                    exportDir,
-                    collectionIDPathMap
-                );
-            }
-            if (!files?.length) {
-                this.electronAPIs.sendNotification(
-                    t('EXPORT_NOTIFICATION.UP_TO_DATE')
-                );
-                return;
-            }
-            this.stopExport = false;
-            this.electronAPIs.sendNotification(t('EXPORT_NOTIFICATION.START'));
            let success = 0;
            let failed = 0;
            this.updateExportProgress({
@ -351,21 +419,57 @@ class ExportService {
                    total: files.length,
                });
            }
-            if (!this.stopExport) {
-                this.electronAPIs.sendNotification(
-                    t('EXPORT_NOTIFICATION.FINISH')
-                );
-            }
        } catch (e) {
            logError(e, 'fileExporter failed');
            throw e;
        }
    }

-    async postExport() {
-        await this.uiUpdater.updateExportStage(ExportStage.FINISHED);
-        await this.uiUpdater.updateLastExportTime(Date.now());
-        this.uiUpdater.updateFileExportStats(await this.getFileExportStats());
+    async fileRemover(
+        removedFileUIDs: string[],
+        exportDir: string
+    ): Promise<void> {
+        try {
+            let success = 0;
+            let failed = 0;
+            this.updateExportProgress({
+                success,
+                failed,
+                total: removedFileUIDs.length,
+            });
+            const exportRecord = await this.getExportRecord(exportDir);
+            const fileIDPathMap = convertFileIDPathObjectToMap(
+                exportRecord.exportedFilePaths
+            );
+            for (const fileUID of removedFileUIDs) {
+                if (this.stopExport) {
+                    break;
+                }
+                try {
+                    const filePath = fileIDPathMap.get(fileUID);
+                    await this.removeFile(filePath);
+                    await this.removeFileExportedRecord(exportDir, fileUID);
+                    success++;
+                } catch (e) {
+                    failed++;
+                    logError(e, 'remove failed for a file');
+                    if (
+                        e.message ===
+                        CustomError.ADD_FILE_EXPORTED_RECORD_FAILED
+                    ) {
+                        throw e;
+                    }
+                }
+                this.updateExportProgress({
+                    success,
+                    failed,
+                    total: removedFileUIDs.length,
+                });
+            }
+        } catch (e) {
+            logError(e, 'fileRemover failed');
+            throw e;
+        }
    }

    async addFileExportedRecord(
@ -376,10 +480,6 @@ class ExportService {
        try {
            const fileUID = getExportRecordFileUID(file);
            const exportRecord = await this.getExportRecord(folder);
-            if (!exportRecord.exportedFiles) {
-                exportRecord.exportedFiles = [];
-            }
-            exportRecord.exportedFiles.push(fileUID);
            if (!exportRecord.exportedFilePaths) {
                exportRecord.exportedFilePaths = {};
            }
@ -399,16 +499,53 @@ class ExportService {
        collectionID: number,
        collectionFolderPath: string
    ) {
-        const exportRecord = await this.getExportRecord(folder);
-        if (!exportRecord?.exportedCollectionPaths) {
-            exportRecord.exportedCollectionPaths = {};
-        }
-        exportRecord.exportedCollectionPaths = {
-            ...exportRecord.exportedCollectionPaths,
-            [collectionID]: collectionFolderPath,
-        };
+        try {
+            const exportRecord = await this.getExportRecord(folder);
+            if (!exportRecord?.exportedCollectionPaths) {
+                exportRecord.exportedCollectionPaths = {};
+            }
+            exportRecord.exportedCollectionPaths = {
+                ...exportRecord.exportedCollectionPaths,
+                [collectionID]: collectionFolderPath,
+            };

-        await this.updateExportRecord(exportRecord, folder);
+            await this.updateExportRecord(exportRecord, folder);
+        } catch (e) {
+            logError(e, 'addCollectionExportedRecord failed');
+            throw Error(CustomError.ADD_FILE_EXPORTED_RECORD_FAILED);
+        }
+    }
+
+    async removeCollectionExportedRecord(folder: string, collectionID: number) {
+        try {
+            const exportRecord = await this.getExportRecord(folder);
+
+            exportRecord.exportedCollectionPaths = Object.fromEntries(
+                Object.entries(exportRecord.exportedCollectionPaths).filter(
+                    ([key]) => key === collectionID.toString()
+                )
+            );
+
+            await this.updateExportRecord(exportRecord, folder);
+        } catch (e) {
+            logError(e, 'removeCollectionExportedRecord failed');
+            throw Error(CustomError.ADD_FILE_EXPORTED_RECORD_FAILED);
+        }
+    }
+
+    async removeFileExportedRecord(folder: string, fileUID: string) {
+        try {
+            const exportRecord = await this.getExportRecord(folder);
+            exportRecord.exportedFilePaths = Object.fromEntries(
+                Object.entries(exportRecord.exportedFilePaths).filter(
+                    ([key]) => key === fileUID
+                )
+            );
+            await this.updateExportRecord(exportRecord, folder);
+        } catch (e) {
+            logError(e, 'removeFileExportedRecord failed');
+            throw Error(CustomError.ADD_FILE_EXPORTED_RECORD_FAILED);
+        }
    }

    async updateExportRecord(newData: Partial<ExportRecord>, folder?: string) {
@ -620,6 +757,10 @@ class ExportService {
        );
    }

+    private async removeFile(filePath: string) {
+        await this.electronAPIs.removeFile(filePath);
+    }
+
    isExportInProgress = () => {
        return this.exportInProgress;
    };
@ -669,7 +810,7 @@ class ExportService {
            });
        }
        if (currentVersion === 2) {
-            await this.addExportedFilePathsProperty(
+            await this.updateExportedFilesToExportedFilePathsProperty(
                getExportedFiles(files, exportRecord)
            );
            currentVersion++;
@ -772,12 +913,15 @@ class ExportService {
        await this.updateExportRecord(exportRecord);
    }

-    private async addExportedFilePathsProperty(exportedFiles: EnteFile[]) {
-        const exportRecord = await this.getExportRecord();
-        const exportedFilePaths = new Map<number, string>();
+    private async updateExportedFilesToExportedFilePathsProperty(
+        exportedFiles: EnteFile[]
+    ) {
+        const exportRecord =
+            (await this.getExportRecord()) as unknown as ExportRecordV2;
+        let exportedFilePaths: ExportedFilePaths;
        const usedFilePaths = new Set<string>();
-        const exportedCollectionPaths = convertIDPathObjectToMap(
-            exportRecord?.exportedCollectionPaths
+        const exportedCollectionPaths = convertCollectionIDPathObjectToMap(
+            exportRecord.exportedCollectionPaths
        );
        for (const file of exportedFiles) {
            const collectionPath = exportedCollectionPaths.get(
@ -789,11 +933,19 @@ class ExportService {
                usedFilePaths
            );
            const filePath = getFileSavePath(collectionPath, fileSaveName);
-            exportedFilePaths.set(file.id, filePath);
+            exportedFilePaths = {
+                ...exportedFilePaths,
+                [getExportRecordFileUID(file)]: filePath,
+            };
        }
-        exportRecord.exportedFilePaths =
-            convertMapToIDPathObject(exportedFilePaths);
-        await this.updateExportRecord(exportRecord);
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        const { exportedFiles: _, ...rest } = exportRecord;
+        const updatedExportRecord: ExportRecord = {
+            ...rest,
+            exportedFilePaths,
+        };
+
+        await this.updateExportRecord(updatedExportRecord);
    }
 }
 export default new ExportService();
--- a/src/types/electron/index.ts
+++ b/src/types/electron/index.ts
@ -90,4 +90,6 @@ export interface ElectronAPIs {
    logRendererProcessMemoryUsage: (message: string) => Promise<void>;
    registerForegroundEventListener: (onForeground: () => void) => void;
    openDirectory: (dirPath: string) => Promise<void>;
+    removeFile: (filePath: string) => Promise<void>;
+    removeFolder: (folderPath: string) => Promise<void>;
 }
--- a/src/types/export/index.ts
+++ b/src/types/export/index.ts
@ -5,10 +5,13 @@ export interface ExportProgress {
    failed: number;
    total: number;
 }
-export interface ExportedEntityPaths {
+export interface ExportedCollectionPaths {
    [ID: number]: string;
 }

+export interface ExportedFilePaths {
+    [ID: string]: string;
+}
 export interface FileExportStats {
    totalCount: number;
    pendingCount: number;
@ -22,16 +25,23 @@ export interface ExportRecordV1 {
    queuedFiles?: string[];
    exportedFiles?: string[];
    failedFiles?: string[];
-    exportedCollectionPaths?: ExportedEntityPaths;
+    exportedCollectionPaths?: ExportedCollectionPaths;
+}
+
+export interface ExportRecordV2 {
+    version: number;
+    stage: ExportStage;
+    lastAttemptTimestamp: number;
+    exportedFiles: string[];
+    exportedCollectionPaths: ExportedCollectionPaths;
 }

 export interface ExportRecord {
    version: number;
    stage: ExportStage;
    lastAttemptTimestamp: number;
-    exportedFiles: string[];
-    exportedCollectionPaths: ExportedEntityPaths;
-    exportedFilePaths: ExportedEntityPaths;
+    exportedCollectionPaths: ExportedCollectionPaths;
+    exportedFilePaths: ExportedFilePaths;
 }

 export interface ExportSettings {
--- a/src/utils/export/index.ts
+++ b/src/utils/export/index.ts
@ -1,6 +1,10 @@
 import { Collection } from 'types/collection';
 import exportService from 'services/exportService';
-import { ExportRecord, ExportedEntityPaths } from 'types/export';
+import {
+    ExportRecord,
+    ExportedCollectionPaths,
+    ExportedFilePaths,
+} from 'types/export';

 import { EnteFile } from 'types/file';

@ -30,31 +34,31 @@ export const getCollectionsCreatedAfterLastExport = (
    });
    return unExportedCollections;
 };
-export const convertIDPathObjectToMap = (
-    exportedEntityPaths: ExportedEntityPaths
-) => {
+export const convertCollectionIDPathObjectToMap = (
+    exportedCollectionPaths: ExportedCollectionPaths
+): Map<number, string> => {
    return new Map<number, string>(
-        Object.entries(exportedEntityPaths ?? {}).map((e) => {
+        Object.entries(exportedCollectionPaths ?? {}).map((e) => {
            return [Number(e[0]), String(e[1])];
        })
    );
 };

-export const convertMapToIDPathObject = (
-    exportedEntityPaths: Map<number, string>
-) => {
-    const exportedEntityPathsObject: ExportedEntityPaths = {};
-    exportedEntityPaths.forEach((value, key) => {
-        exportedEntityPathsObject[key] = value;
-    });
-    return exportedEntityPathsObject;
+export const convertFileIDPathObjectToMap = (
+    exportedFilePaths: ExportedFilePaths
+): Map<string, string> => {
+    return new Map<string, string>(
+        Object.entries(exportedFilePaths ?? {}).map((e) => {
+            return [String(e[0]), String(e[1])];
+        })
+    );
 };

-export const getCollectionsRenamedAfterLastExport = (
+export const getRenamedCollections = (
    collections: Collection[],
    exportRecord: ExportRecord
 ) => {
-    const collectionIDPathMap = convertIDPathObjectToMap(
+    const collectionIDPathMap = convertCollectionIDPathObjectToMap(
        exportRecord.exportedCollectionPaths
    );
    const renamedCollections = collections.filter((collection) => {
@ -76,11 +80,31 @@ export const getCollectionsRenamedAfterLastExport = (
    return renamedCollections;
 };

+export const getDeletedExportedCollections = (
+    collections: Collection[],
+    exportRecord: ExportRecord
+) => {
+    const presentCollections = new Set(
+        collections?.map((collection) => collection.id)
+    );
+    const deletedExportedCollections = Object.keys(
+        exportRecord?.exportedCollectionPaths
+    )
+        .map(Number)
+        .filter((collectionID) => {
+            if (!presentCollections.has(collectionID)) {
+                return true;
+            }
+            return false;
+        });
+    return deletedExportedCollections;
+};
+
 export const getUnExportedFiles = (
    allFiles: EnteFile[],
    exportRecord: ExportRecord
 ) => {
-    const exportedFiles = new Set(exportRecord?.exportedFiles);
+    const exportedFiles = new Set(Object.keys(exportRecord?.exportedFilePaths));
    const unExportedFiles = allFiles.filter((file) => {
        if (!exportedFiles.has(getExportRecordFileUID(file))) {
            return true;
@ -94,7 +118,9 @@ export const getExportedFiles = (
    allFiles: EnteFile[],
    exportRecord: ExportRecord
 ) => {
-    const exportedFileIds = new Set(exportRecord?.exportedFiles);
+    const exportedFileIds = new Set(
+        Object.keys(exportRecord?.exportedFilePaths)
+    );
    const exportedFiles = allFiles.filter((file) => {
        if (exportedFileIds.has(getExportRecordFileUID(file))) {
            return true;
@ -104,6 +130,24 @@ export const getExportedFiles = (
    return exportedFiles;
 };

+export const getDeletedExportedFiles = (
+    allFiles: EnteFile[],
+    exportRecord: ExportRecord
+): string[] => {
+    const presentFileUIDs = new Set(
+        allFiles?.map((file) => getExportRecordFileUID(file))
+    );
+    const deletedExportedFiles = Object.keys(
+        exportRecord?.exportedFilePaths
+    ).filter((fileUID) => {
+        if (!presentFileUIDs.has(fileUID)) {
+            return true;
+        }
+        return false;
+    });
+    return deletedExportedFiles;
+};
+
 export const getGoogleLikeMetadataFile = (
    fileSaveName: string,
    file: EnteFile