Export redesign (#993)

2023-03-28 18:51:47 +05:30 · 2023-03-28 18:51:47 +05:30 · 74ea904bfe
parent 4209014d25 455bce552a
commit 74ea904bfe
8 changed files with 196 additions and 184 deletions
--- a/public/locales/en/translation.json
+++ b/public/locales/en/translation.json
@ -300,7 +300,6 @@
    "EXPORT_DATA": "Export data",
    "SELECT_FOLDER": "Select folder",
    "DESTINATION": "Destination",
-    "TOTAL_FILE_COUNT": "Total file count",
    "START": "Start",
    "EXPORT_IN_PROGRESS": "Export in progress...",
    "PAUSE": "Pause",
@ -577,5 +576,8 @@
        "FINISH": "Export finished",
        "UP_TO_DATE": "No new files to export"
    },
-    "CONTINUOUS_EXPORT": "Sync continuously"
+    "CONTINUOUS_EXPORT": "Sync continuously",
+    "TOTAL_ITEMS": "Total items",
+    "PENDING_ITEMS": "Pending items",
+    "EXPORT_STARTING": "Export starting..."
 }
--- a/src/components/ExportFinished.tsx
+++ b/src/components/ExportFinished.tsx
@ -7,14 +7,13 @@ import {
 } from '@mui/material';
 import React from 'react';
 import { t } from 'i18next';
-import { ExportStats } from 'types/export';
 import { formatDateTime } from 'utils/time/format';
 import { SpaceBetweenFlex } from './Container';

 interface Props {
+    pendingFileCount: number;
    onHide: () => void;
    lastExportTime: number;
-    exportStats: ExportStats;
    startExport: () => void;
 }

@ -22,8 +21,14 @@ export default function ExportFinished(props: Props) {
    return (
        <>
            <DialogContent>
-                <Stack spacing={2.5} pr={2}>
-                    <SpaceBetweenFlex>
+                <Stack pr={2}>
+                    <SpaceBetweenFlex minHeight={'48px'}>
+                        <Typography color={'text.secondary'}>
+                            {t('PENDING_ITEMS')}
+                        </Typography>
+                        <Typography>{props.pendingFileCount}</Typography>
+                    </SpaceBetweenFlex>
+                    <SpaceBetweenFlex minHeight={'48px'}>
                        <Typography color="text.secondary">
                            {t('LAST_EXPORT_TIME')}
                        </Typography>
@ -31,20 +36,6 @@ export default function ExportFinished(props: Props) {
                            {formatDateTime(props.lastExportTime)}
                        </Typography>
                    </SpaceBetweenFlex>
-                    <SpaceBetweenFlex>
-                        <Typography color="text.secondary">
-                            {t('SUCCESSFULLY_EXPORTED_FILES')}
-                        </Typography>
-                        <Typography>{props.exportStats.success}</Typography>
-                    </SpaceBetweenFlex>
-                    {props.exportStats.failed > 0 && (
-                        <SpaceBetweenFlex>
-                            <Typography color="text.secondary">
-                                {t('FAILED_EXPORTED_FILES')}
-                            </Typography>
-                            <Typography>{props.exportStats.failed}</Typography>
-                        </SpaceBetweenFlex>
-                    )}
                </Stack>
            </DialogContent>
            <DialogActions>
--- a/src/components/ExportInProgress.tsx
+++ b/src/components/ExportInProgress.tsx
@ -27,28 +27,37 @@ interface Props {
 }

 export default function ExportInProgress(props: Props) {
+    const isLoading = props.exportProgress.total === 0;
    return (
        <>
            <DialogContent>
                <VerticallyCentered>
                    <Box mb={1.5}>
-                        <Trans
-                            i18nKey={'EXPORT_PROGRESS'}
-                            components={{
-                                a: <ComfySpan />,
-                            }}
-                            values={{
-                                progress: props.exportProgress,
-                            }}
-                        />
+                        {isLoading ? (
+                            t('EXPORT_STARTING')
+                        ) : (
+                            <Trans
+                                i18nKey={'EXPORT_PROGRESS'}
+                                components={{
+                                    a: <ComfySpan />,
+                                }}
+                                values={{
+                                    progress: props.exportProgress,
+                                }}
+                            />
+                        )}
                    </Box>
                    <FlexWrapper px={1}>
                        <ProgressBar
                            style={{ width: '100%' }}
-                            now={Math.round(
-                                (props.exportProgress.current * 100) /
-                                    props.exportProgress.total
-                            )}
+                            now={
+                                isLoading
+                                    ? 100
+                                    : Math.round(
+                                          (props.exportProgress.current * 100) /
+                                              props.exportProgress.total
+                                      )
+                            }
                            animated
                            variant="upload-progress-bar"
                        />
--- a/src/components/ExportModal.tsx
+++ b/src/components/ExportModal.tsx
@ -1,7 +1,7 @@
 import isElectron from 'is-electron';
 import React, { useEffect, useState, useContext } from 'react';
 import exportService from 'services/exportService';
-import { ExportProgress, ExportSettings, ExportStats } from 'types/export';
+import { ExportProgress, ExportSettings, FileExportStats } from 'types/export';
 import {
    Box,
    Button,
@ -28,9 +28,8 @@ import { OverflowMenuOption } from './OverflowMenu/option';
 import { AppContext } from 'pages/_app';
 import { getExportDirectoryDoesNotExistMessage } from 'utils/ui';
 import { t } from 'i18next';
-import { getTotalFileCount } from 'utils/file';
-import { eventBus, Events } from 'services/events';
 import LinkButton from './pages/gallery/LinkButton';
+import { CustomError } from 'utils/error';

 const ExportFolderPathContainer = styled(LinkButton)`
    width: 262px;
@ -51,14 +50,13 @@ export default function ExportModal(props: Props) {
    const [exportStage, setExportStage] = useState(ExportStage.INIT);
    const [exportFolder, setExportFolder] = useState('');
    const [continuousExport, setContinuousExport] = useState(false);
-    const [totalFileCount, setTotalFileCount] = useState(0);
    const [exportProgress, setExportProgress] = useState<ExportProgress>({
        current: 0,
        total: 0,
    });
-    const [exportStats, setExportStats] = useState<ExportStats>({
-        failed: 0,
-        success: 0,
+    const [fileExportStats, setFileExportStats] = useState<FileExportStats>({
+        totalCount: 0,
+        pendingCount: 0,
    });
    const [lastExportTime, setLastExportTime] = useState(0);

@ -73,16 +71,19 @@ export default function ExportModal(props: Props) {
            const exportSettings: ExportSettings = getData(LS_KEYS.EXPORT);
            setExportFolder(exportSettings?.folder);
            setContinuousExport(exportSettings?.continuousExport);
-            const localFileUpdateHandler = async () => {
-                setTotalFileCount(await getTotalFileCount());
-            };
-            localFileUpdateHandler();
-            eventBus.on(Events.LOCAL_FILES_UPDATED, localFileUpdateHandler);
+            syncFileCounts();
        } catch (e) {
            logError(e, 'error in exportModal');
        }
    }, []);

+    useEffect(() => {
+        if (!props.show) {
+            return;
+        }
+        syncFileCounts();
+    }, [props.show]);
+
    useEffect(() => {
        try {
            if (continuousExport) {
@ -102,12 +103,13 @@ export default function ExportModal(props: Props) {
        const main = async () => {
            try {
                const exportInfo = await exportService.getExportRecord();
-                setExportStage(exportInfo?.stage ?? ExportStage.INIT);
-                setLastExportTime(exportInfo?.lastAttemptTimestamp);
-                setExportStats({
-                    success: exportInfo?.exportedFiles?.length ?? 0,
-                    failed: exportInfo?.failedFiles?.length ?? 0,
-                });
+                if (exportInfo?.stage) {
+                    setExportStage(exportInfo?.stage);
+                }
+                if (exportInfo?.lastAttemptTimestamp) {
+                    setLastExportTime(exportInfo?.lastAttemptTimestamp);
+                }
+                await syncFileCounts();
                if (exportInfo?.stage === ExportStage.INPROGRESS) {
                    await startExport();
                }
@ -155,7 +157,7 @@ export default function ExportModal(props: Props) {

    // ======================
    // HELPER FUNCTIONS
-    // =========================
+    // =======================

    const preExportRun = async () => {
        const exportFolder = getData(LS_KEYS.EXPORT)?.folder;
@ -164,7 +166,7 @@ export default function ExportModal(props: Props) {
            appContext.setDialogMessage(
                getExportDirectoryDoesNotExistMessage()
            );
-            return;
+            throw Error(CustomError.EXPORT_FOLDER_DOES_NOT_EXIST);
        }
        await updateExportStage(ExportStage.INPROGRESS);
    };
@ -172,14 +174,16 @@ export default function ExportModal(props: Props) {
    const postExportRun = async () => {
        await updateExportStage(ExportStage.FINISHED);
        await updateExportTime(Date.now());
-        await syncExportStatsWithRecord();
+        await syncFileCounts();
    };

-    const syncExportStatsWithRecord = async () => {
-        const exportRecord = await exportService.getExportRecord();
-        const failed = exportRecord?.failedFiles?.length ?? 0;
-        const success = exportRecord?.exportedFiles?.length ?? 0;
-        setExportStats({ failed, success });
+    const syncFileCounts = async () => {
+        try {
+            const fileExportStats = await exportService.getFileExportStats();
+            setFileExportStats(fileExportStats);
+        } catch (e) {
+            logError(e, 'error updating file counts');
+        }
    };

    // =============
@ -205,24 +209,13 @@ export default function ExportModal(props: Props) {
    const startExport = async () => {
        try {
            await preExportRun();
-            const exportRecord = await exportService.getExportRecord();
-            const totalFileCount = await getTotalFileCount();
-            const exportedFileCount = exportRecord.exportedFiles?.length ?? 0;
-            setExportProgress({
-                current: exportedFileCount,
-                total: totalFileCount,
-            });
-
-            const updateExportStatsWithOffset = (current: number) =>
-                setExportProgress({
-                    current: exportedFileCount + current,
-                    total: totalFileCount,
-                });
-            await exportService.exportFiles(updateExportStatsWithOffset);
-
+            setExportProgress({ current: 0, total: 0 });
+            await exportService.exportFiles(setExportProgress);
            await postExportRun();
        } catch (e) {
-            logError(e, 'startExport failed');
+            if (e.message !== CustomError.EXPORT_FOLDER_DOES_NOT_EXIST) {
+                logError(e, 'startExport failed');
+            }
        }
    };

@ -235,35 +228,6 @@ export default function ExportModal(props: Props) {
        }
    };

-    const ExportDynamicContent = () => {
-        switch (exportStage) {
-            case ExportStage.INIT:
-                return <ExportInit startExport={startExport} />;
-
-            case ExportStage.INPROGRESS:
-                return (
-                    <ExportInProgress
-                        exportStage={exportStage}
-                        exportProgress={exportProgress}
-                        stopExport={stopExport}
-                        closeExportDialog={props.onHide}
-                    />
-                );
-            case ExportStage.FINISHED:
-                return (
-                    <ExportFinished
-                        onHide={props.onHide}
-                        lastExportTime={lastExportTime}
-                        exportStats={exportStats}
-                        startExport={startExport}
-                    />
-                );
-
-            default:
-                return <></>;
-        }
-    };
-
    return (
        <Dialog open={props.show} onClose={props.onHide} maxWidth="xs">
            <DialogTitleWithCloseButton onClose={props.onHide}>
@ -276,14 +240,29 @@ export default function ExportModal(props: Props) {
                    exportStage={exportStage}
                    openExportDirectory={handleOpenExportDirectoryClick}
                />
-                <TotalFileCount totalFileCount={totalFileCount} />
                <ContinuousExport
                    continuousExport={continuousExport}
                    toggleContinuousExport={toggleContinuousExport}
                />
+                <SpaceBetweenFlex minHeight={'48px'} pr={'16px'}>
+                    <Typography color="text.secondary">
+                        {t('TOTAL_ITEMS')}
+                    </Typography>
+                    <Typography color="text.secondary">
+                        {fileExportStats.totalCount}
+                    </Typography>
+                </SpaceBetweenFlex>
            </DialogContent>
            <Divider />
-            <ExportDynamicContent />
+            <ExportDynamicContent
+                exportStage={exportStage}
+                startExport={startExport}
+                stopExport={stopExport}
+                onHide={props.onHide}
+                lastExportTime={lastExportTime}
+                pendingFileCount={fileExportStats.pendingCount}
+                exportProgress={exportProgress}
+            />
        </Dialog>
    );
 }
@ -328,17 +307,6 @@ function ExportDirectory({
    );
 }

-function TotalFileCount({ totalFileCount }) {
-    return (
-        <SpaceBetweenFlex minHeight={'40px'} pr={2}>
-            <Typography color={'text.secondary'}>
-                {t('TOTAL_FILE_COUNT')}{' '}
-            </Typography>
-            <Typography>{totalFileCount}</Typography>
-        </SpaceBetweenFlex>
-    );
-}
-
 function ExportDirectoryOption({ changeExportDirectory }) {
    return (
        <OverflowMenu
@ -360,7 +328,7 @@ function ExportDirectoryOption({ changeExportDirectory }) {

 function ContinuousExport({ continuousExport, toggleContinuousExport }) {
    return (
-        <SpaceBetweenFlex minHeight={'40px'}>
+        <SpaceBetweenFlex minHeight={'48px'}>
            <Typography color="text.secondary">
                {t('CONTINUOUS_EXPORT')}
            </Typography>
@ -374,3 +342,48 @@ function ContinuousExport({ continuousExport, toggleContinuousExport }) {
        </SpaceBetweenFlex>
    );
 }
+
+const ExportDynamicContent = ({
+    exportStage,
+    startExport,
+    stopExport,
+    onHide,
+    lastExportTime,
+    pendingFileCount,
+    exportProgress,
+}: {
+    exportStage: ExportStage;
+    startExport: () => void;
+    stopExport: () => void;
+    onHide: () => void;
+    lastExportTime: number;
+    pendingFileCount: number;
+    exportProgress: ExportProgress;
+}) => {
+    switch (exportStage) {
+        case ExportStage.INIT:
+            return <ExportInit startExport={startExport} />;
+
+        case ExportStage.INPROGRESS:
+            return (
+                <ExportInProgress
+                    exportStage={exportStage}
+                    exportProgress={exportProgress}
+                    stopExport={stopExport}
+                    closeExportDialog={onHide}
+                />
+            );
+        case ExportStage.FINISHED:
+            return (
+                <ExportFinished
+                    onHide={onHide}
+                    lastExportTime={lastExportTime}
+                    pendingFileCount={pendingFileCount}
+                    startExport={startExport}
+                />
+            );
+
+        default:
+            return <></>;
+    }
+};
--- a/src/services/exportService.ts
+++ b/src/services/exportService.ts
@ -31,6 +31,7 @@ import { decodeMotionPhoto } from './motionPhotoService';
 import {
    generateStreamFromArrayBuffer,
    getFileExtension,
+    getPersonalFiles,
    mergeMetadata,
 } from 'utils/file';

@ -40,8 +41,10 @@ import { Collection } from 'types/collection';
 import {
    CollectionIDNameMap,
    CollectionIDPathMap,
+    ExportProgress,
    ExportRecord,
    ExportRecordV1,
+    FileExportStats,
 } from 'types/export';
 import { User } from 'types/user';
 import { FILE_TYPE, TYPE_JPEG, TYPE_JPG } from 'constants/file';
@ -64,7 +67,7 @@ class ExportService {
    private stopExport: boolean = false;
    private allElectronAPIsExist: boolean = false;
    private fileReader: FileReader = null;
-    private continuousExportEventListener: () => void;
+    private continuousExportEventHandler: () => void;

    constructor() {
        this.electronAPIs = runningInBrowser() && window['ElectronAPIs'];
@ -93,24 +96,35 @@ class ExportService {
        }
    }

-    enableContinuousExport(startExport: () => void) {
+    enableContinuousExport(startExport: () => Promise<void>) {
        try {
-            if (this.continuousExportEventListener) {
+            if (this.continuousExportEventHandler) {
                addLogLine('continuous export already enabled');
                return;
            }
-            startExport();
-            this.continuousExportEventListener = () => {
-                addLogLine('continuous export triggered');
-                if (this.exportInProgress) {
-                    addLogLine('export in progress, skipping');
-                    return;
+            const reRunNeeded = { current: false };
+            this.continuousExportEventHandler = async () => {
+                try {
+                    addLogLine('continuous export triggered');
+                    if (this.exportInProgress) {
+                        addLogLine('export in progress, scheduling re-run');
+                        reRunNeeded.current = true;
+                        return;
+                    }
+                    await startExport();
+                    if (reRunNeeded.current) {
+                        reRunNeeded.current = false;
+                        addLogLine('re-running export');
+                        setTimeout(this.continuousExportEventHandler, 0);
+                    }
+                } catch (e) {
+                    logError(e, 'continuous export failed');
                }
-                startExport();
            };
+            this.continuousExportEventHandler();
            eventBus.addListener(
                Events.LOCAL_FILES_UPDATED,
-                this.continuousExportEventListener
+                this.continuousExportEventHandler
            );
        } catch (e) {
            logError(e, 'failed to enableContinuousExport ');
@ -120,26 +134,44 @@ class ExportService {

    disableContinuousExport() {
        try {
-            if (!this.continuousExportEventListener) {
+            if (!this.continuousExportEventHandler) {
                addLogLine('continuous export already disabled');
                return;
            }
            eventBus.removeListener(
                Events.LOCAL_FILES_UPDATED,
-                this.continuousExportEventListener
+                this.continuousExportEventHandler
            );
-            this.continuousExportEventListener = null;
+            this.continuousExportEventHandler = null;
        } catch (e) {
            logError(e, 'failed to disableContinuousExport');
            throw e;
        }
    }

+    getFileExportStats = async (): Promise<FileExportStats> => {
+        try {
+            const exportRecord = await this.getExportRecord();
+            const userPersonalFiles = await getPersonalFiles();
+            const unExportedFiles = getUnExportedFiles(
+                userPersonalFiles,
+                exportRecord
+            );
+            return {
+                totalCount: userPersonalFiles.length,
+                pendingCount: unExportedFiles.length,
+            };
+        } catch (e) {
+            logError(e, 'getUpdateFileLists failed');
+            throw e;
+        }
+    };
+
    stopRunningExport() {
        this.stopExport = true;
    }

-    async exportFiles(updateProgress: (current: number) => void) {
+    async exportFiles(updateProgress: (progress: ExportProgress) => void) {
        try {
            // eslint-disable-next-line @typescript-eslint/no-misused-promises
            if (this.exportInProgress) {
@ -219,7 +251,7 @@ class ExportService {
        collectionIDNameMap: CollectionIDNameMap,
        renamedCollections: Collection[],
        collectionIDPathMap: CollectionIDPathMap,
-        updateProgress: (current: number) => void,
+        updateProgress: (progress: ExportProgress) => void,
        exportDir: string
    ): Promise<void> {
        try {
@ -241,8 +273,8 @@ class ExportService {
            }
            this.stopExport = false;
            this.electronAPIs.sendNotification(t('EXPORT_NOTIFICATION.START'));
-
-            for (const [index, file] of files.entries()) {
+            let success = 0;
+            for (const file of files) {
                if (this.stopExport) {
                    break;
                }
@ -264,6 +296,8 @@ class ExportService {
                        file,
                        RecordType.SUCCESS
                    );
+                    success++;
+                    updateProgress({ current: success, total: files.length });
                } catch (e) {
                    logError(e, 'export failed for a file');
                    if (
@ -278,7 +312,6 @@ class ExportService {
                        RecordType.FAILED
                    );
                }
-                updateProgress(index + 1);
            }
            if (!this.stopExport) {
                this.electronAPIs.sendNotification(
@ -304,20 +337,8 @@ class ExportService {
                    exportRecord.exportedFiles = [];
                }
                exportRecord.exportedFiles.push(fileUID);
-                exportRecord.failedFiles &&
-                    (exportRecord.failedFiles = exportRecord.failedFiles.filter(
-                        (FailedFileUID) => FailedFileUID !== fileUID
-                    ));
-            } else {
-                if (!exportRecord.failedFiles) {
-                    exportRecord.failedFiles = [];
-                }
-                if (!exportRecord.failedFiles.find((x) => x === fileUID)) {
-                    exportRecord.failedFiles.push(fileUID);
-                }
            }
            exportRecord.exportedFiles = dedupe(exportRecord.exportedFiles);
-            exportRecord.failedFiles = dedupe(exportRecord.failedFiles);
            await this.updateExportRecord(exportRecord, folder);
        } catch (e) {
            logError(e, 'addFileExportedRecord failed');
@ -375,20 +396,16 @@ class ExportService {
                folder = getData(LS_KEYS.EXPORT)?.folder;
            }
            if (!folder) {
-                throw Error(CustomError.NO_EXPORT_FOLDER_SELECTED);
+                return null;
            }
            const exportFolderExists = this.exists(folder);
            if (!exportFolderExists) {
-                throw Error(CustomError.EXPORT_FOLDER_DOES_NOT_EXIST);
+                return null;
            }
            const recordFile = await this.electronAPIs.getExportRecord(
                `${folder}/${EXPORT_RECORD_FILE_NAME}`
            );
-            if (recordFile) {
-                return JSON.parse(recordFile);
-            } else {
-                return {} as ExportRecord;
-            }
+            return JSON.parse(recordFile);
        } catch (e) {
            logError(e, 'export Record JSON parsing failed ');
            throw e;
@ -675,6 +692,9 @@ class ExportService {
        if (exportRecord?.progress) {
            exportRecord.progress = undefined;
        }
+        if (exportRecord?.failedFiles) {
+            exportRecord.failedFiles = undefined;
+        }
        await this.updateExportRecord(exportRecord);
    }
 }
--- a/src/types/export/index.ts
+++ b/src/types/export/index.ts
@ -9,9 +9,9 @@ export interface ExportProgress {
 export interface ExportedCollectionPaths {
    [collectionID: number]: string;
 }
-export interface ExportStats {
-    failed: number;
-    success: number;
+export interface FileExportStats {
+    totalCount: number;
+    pendingCount: number;
 }

 export interface ExportRecordV1 {
@ -30,7 +30,6 @@ export interface ExportRecord {
    stage: ExportStage;
    lastAttemptTimestamp: number;
    exportedFiles: string[];
-    failedFiles: string[];
    exportedCollectionPaths: ExportedCollectionPaths;
 }

--- a/src/utils/export/index.ts
+++ b/src/utils/export/index.ts
@ -93,20 +93,6 @@ export const getExportedFiles = (
    return exportedFiles;
 };

-export const getExportFailedFiles = (
-    allFiles: EnteFile[],
-    exportRecord: ExportRecord
-) => {
-    const failedFiles = new Set(exportRecord?.failedFiles);
-    const filesToExport = allFiles.filter((file) => {
-        if (failedFiles.has(getExportRecordFileUID(file))) {
-            return true;
-        }
-        return false;
-    });
-    return filesToExport;
-};
-
 export const dedupe = (files: string[]) => {
    const fileSet = new Set(files);
    return Array.from(fileSet);
--- a/src/utils/file/index.ts
+++ b/src/utils/file/index.ts
@ -580,19 +580,11 @@ export function getLatestVersionFiles(files: EnteFile[]) {
    );
 }

-export function getUserPersonalFiles(files: EnteFile[]) {
+export async function getPersonalFiles() {
+    const files = await getLocalFiles();
    const user: User = getData(LS_KEYS.USER);
    if (!user?.id) {
        throw Error('user missing');
    }
    return files.filter((file) => file.ownerID === user.id);
 }
-
-export const getTotalFileCount = async () => {
-    try {
-        const userPersonalFiles = getUserPersonalFiles(await getLocalFiles());
-        return userPersonalFiles.length;
-    } catch (e) {
-        logError(e, 'updateTotalFileCount failed');
-    }
-};