Export redesign (#993)

public/locales/en/translation.json

@@ -300,7 +300,6 @@
     "EXPORT_DATA": "Export data",
     "SELECT_FOLDER": "Select folder",
     "DESTINATION": "Destination",
-    "TOTAL_FILE_COUNT": "Total file count",
     "START": "Start",
     "EXPORT_IN_PROGRESS": "Export in progress...",
     "PAUSE": "Pause",
@@ -577,5 +576,8 @@
         "FINISH": "Export finished",
         "UP_TO_DATE": "No new files to export"
     },
-    "CONTINUOUS_EXPORT": "Sync continuously"
+    "CONTINUOUS_EXPORT": "Sync continuously",
+    "TOTAL_ITEMS": "Total items",
+    "PENDING_ITEMS": "Pending items",
+    "EXPORT_STARTING": "Export starting..."
 }

src/components/ExportFinished.tsx

@@ -7,14 +7,13 @@ import {
 } from '@mui/material';
 import React from 'react';
 import { t } from 'i18next';
-import { ExportStats } from 'types/export';
 import { formatDateTime } from 'utils/time/format';
 import { SpaceBetweenFlex } from './Container';
 
 interface Props {
+    pendingFileCount: number;
     onHide: () => void;
     lastExportTime: number;
-    exportStats: ExportStats;
     startExport: () => void;
 }
 
@@ -22,8 +21,14 @@ export default function ExportFinished(props: Props) {
     return (
         <>
             <DialogContent>
-                <Stack spacing={2.5} pr={2}>
+                <Stack pr={2}>
-                    <SpaceBetweenFlex>
+                    <SpaceBetweenFlex minHeight={'48px'}>
+                        <Typography color={'text.secondary'}>
+                            {t('PENDING_ITEMS')}
+                        </Typography>
+                        <Typography>{props.pendingFileCount}</Typography>
+                    </SpaceBetweenFlex>
+                    <SpaceBetweenFlex minHeight={'48px'}>
                         <Typography color="text.secondary">
                             {t('LAST_EXPORT_TIME')}
                         </Typography>
@@ -31,20 +36,6 @@ export default function ExportFinished(props: Props) {
                             {formatDateTime(props.lastExportTime)}
                         </Typography>
                     </SpaceBetweenFlex>
-                    <SpaceBetweenFlex>
-                        <Typography color="text.secondary">
-                            {t('SUCCESSFULLY_EXPORTED_FILES')}
-                        </Typography>
-                        <Typography>{props.exportStats.success}</Typography>
-                    </SpaceBetweenFlex>
-                    {props.exportStats.failed > 0 && (
-                        <SpaceBetweenFlex>
-                            <Typography color="text.secondary">
-                                {t('FAILED_EXPORTED_FILES')}
-                            </Typography>
-                            <Typography>{props.exportStats.failed}</Typography>
-                        </SpaceBetweenFlex>
-                    )}
                 </Stack>
             </DialogContent>
             <DialogActions>

src/components/ExportInProgress.tsx

@@ -27,28 +27,37 @@ interface Props {
 }
 
 export default function ExportInProgress(props: Props) {
+    const isLoading = props.exportProgress.total === 0;
     return (
         <>
             <DialogContent>
                 <VerticallyCentered>
                     <Box mb={1.5}>
-                        <Trans
+                        {isLoading ? (
-                            i18nKey={'EXPORT_PROGRESS'}
+                            t('EXPORT_STARTING')
-                            components={{
+                        ) : (
-                                a: <ComfySpan />,
+                            <Trans
-                            }}
+                                i18nKey={'EXPORT_PROGRESS'}
-                            values={{
+                                components={{
-                                progress: props.exportProgress,
+                                    a: <ComfySpan />,
-                            }}
+                                }}
-                        />
+                                values={{
+                                    progress: props.exportProgress,
+                                }}
+                            />
+                        )}
                     </Box>
                     <FlexWrapper px={1}>
                         <ProgressBar
                             style={{ width: '100%' }}
-                            now={Math.round(
+                            now={
-                                (props.exportProgress.current * 100) /
+                                isLoading
-                                    props.exportProgress.total
+                                    ? 100
-                            )}
+                                    : Math.round(
+                                          (props.exportProgress.current * 100) /
+                                              props.exportProgress.total
+                                      )
+                            }
                             animated
                             variant="upload-progress-bar"
                         />

src/components/ExportModal.tsx

@@ -1,7 +1,7 @@
 import isElectron from 'is-electron';
 import React, { useEffect, useState, useContext } from 'react';
 import exportService from 'services/exportService';
-import { ExportProgress, ExportSettings, ExportStats } from 'types/export';
+import { ExportProgress, ExportSettings, FileExportStats } from 'types/export';
 import {
     Box,
     Button,
@@ -28,9 +28,8 @@ import { OverflowMenuOption } from './OverflowMenu/option';
 import { AppContext } from 'pages/_app';
 import { getExportDirectoryDoesNotExistMessage } from 'utils/ui';
 import { t } from 'i18next';
-import { getTotalFileCount } from 'utils/file';
-import { eventBus, Events } from 'services/events';
 import LinkButton from './pages/gallery/LinkButton';
+import { CustomError } from 'utils/error';
 
 const ExportFolderPathContainer = styled(LinkButton)`
     width: 262px;
@@ -51,14 +50,13 @@ export default function ExportModal(props: Props) {
     const [exportStage, setExportStage] = useState(ExportStage.INIT);
     const [exportFolder, setExportFolder] = useState('');
     const [continuousExport, setContinuousExport] = useState(false);
-    const [totalFileCount, setTotalFileCount] = useState(0);
     const [exportProgress, setExportProgress] = useState<ExportProgress>({
         current: 0,
         total: 0,
     });
-    const [exportStats, setExportStats] = useState<ExportStats>({
+    const [fileExportStats, setFileExportStats] = useState<FileExportStats>({
-        failed: 0,
+        totalCount: 0,
-        success: 0,
+        pendingCount: 0,
     });
     const [lastExportTime, setLastExportTime] = useState(0);
 
@@ -73,16 +71,19 @@ export default function ExportModal(props: Props) {
             const exportSettings: ExportSettings = getData(LS_KEYS.EXPORT);
             setExportFolder(exportSettings?.folder);
             setContinuousExport(exportSettings?.continuousExport);
-            const localFileUpdateHandler = async () => {
+            syncFileCounts();
-                setTotalFileCount(await getTotalFileCount());
-            };
-            localFileUpdateHandler();
-            eventBus.on(Events.LOCAL_FILES_UPDATED, localFileUpdateHandler);
         } catch (e) {
             logError(e, 'error in exportModal');
         }
     }, []);
 
+    useEffect(() => {
+        if (!props.show) {
+            return;
+        }
+        syncFileCounts();
+    }, [props.show]);
+
     useEffect(() => {
         try {
             if (continuousExport) {
@@ -102,12 +103,13 @@ export default function ExportModal(props: Props) {
         const main = async () => {
             try {
                 const exportInfo = await exportService.getExportRecord();
-                setExportStage(exportInfo?.stage ?? ExportStage.INIT);
+                if (exportInfo?.stage) {
-                setLastExportTime(exportInfo?.lastAttemptTimestamp);
+                    setExportStage(exportInfo?.stage);
-                setExportStats({
+                }
-                    success: exportInfo?.exportedFiles?.length ?? 0,
+                if (exportInfo?.lastAttemptTimestamp) {
-                    failed: exportInfo?.failedFiles?.length ?? 0,
+                    setLastExportTime(exportInfo?.lastAttemptTimestamp);
-                });
+                }
+                await syncFileCounts();
                 if (exportInfo?.stage === ExportStage.INPROGRESS) {
                     await startExport();
                 }
@@ -155,7 +157,7 @@ export default function ExportModal(props: Props) {
 
     // ======================
     // HELPER FUNCTIONS
-    // =========================
+    // =======================
 
     const preExportRun = async () => {
         const exportFolder = getData(LS_KEYS.EXPORT)?.folder;
@@ -164,7 +166,7 @@ export default function ExportModal(props: Props) {
             appContext.setDialogMessage(
                 getExportDirectoryDoesNotExistMessage()
             );
-            return;
+            throw Error(CustomError.EXPORT_FOLDER_DOES_NOT_EXIST);
         }
         await updateExportStage(ExportStage.INPROGRESS);
     };
@@ -172,14 +174,16 @@ export default function ExportModal(props: Props) {
     const postExportRun = async () => {
         await updateExportStage(ExportStage.FINISHED);
         await updateExportTime(Date.now());
-        await syncExportStatsWithRecord();
+        await syncFileCounts();
     };
 
-    const syncExportStatsWithRecord = async () => {
+    const syncFileCounts = async () => {
-        const exportRecord = await exportService.getExportRecord();
+        try {
-        const failed = exportRecord?.failedFiles?.length ?? 0;
+            const fileExportStats = await exportService.getFileExportStats();
-        const success = exportRecord?.exportedFiles?.length ?? 0;
+            setFileExportStats(fileExportStats);
-        setExportStats({ failed, success });
+        } catch (e) {
+            logError(e, 'error updating file counts');
+        }
     };
 
     // =============
@@ -205,24 +209,13 @@ export default function ExportModal(props: Props) {
     const startExport = async () => {
         try {
             await preExportRun();
-            const exportRecord = await exportService.getExportRecord();
+            setExportProgress({ current: 0, total: 0 });
-            const totalFileCount = await getTotalFileCount();
+            await exportService.exportFiles(setExportProgress);
-            const exportedFileCount = exportRecord.exportedFiles?.length ?? 0;
-            setExportProgress({
-                current: exportedFileCount,
-                total: totalFileCount,
-            });
-
-            const updateExportStatsWithOffset = (current: number) =>
-                setExportProgress({
-                    current: exportedFileCount + current,
-                    total: totalFileCount,
-                });
-            await exportService.exportFiles(updateExportStatsWithOffset);
-
             await postExportRun();
         } catch (e) {
-            logError(e, 'startExport failed');
+            if (e.message !== CustomError.EXPORT_FOLDER_DOES_NOT_EXIST) {
+                logError(e, 'startExport failed');
+            }
         }
     };
 
@@ -235,35 +228,6 @@ export default function ExportModal(props: Props) {
         }
     };
 
-    const ExportDynamicContent = () => {
-        switch (exportStage) {
-            case ExportStage.INIT:
-                return <ExportInit startExport={startExport} />;
-
-            case ExportStage.INPROGRESS:
-                return (
-                    <ExportInProgress
-                        exportStage={exportStage}
-                        exportProgress={exportProgress}
-                        stopExport={stopExport}
-                        closeExportDialog={props.onHide}
-                    />
-                );
-            case ExportStage.FINISHED:
-                return (
-                    <ExportFinished
-                        onHide={props.onHide}
-                        lastExportTime={lastExportTime}
-                        exportStats={exportStats}
-                        startExport={startExport}
-                    />
-                );
-
-            default:
-                return <></>;
-        }
-    };
-
     return (
         <Dialog open={props.show} onClose={props.onHide} maxWidth="xs">
             <DialogTitleWithCloseButton onClose={props.onHide}>
@@ -276,14 +240,29 @@ export default function ExportModal(props: Props) {
                     exportStage={exportStage}
                     openExportDirectory={handleOpenExportDirectoryClick}
                 />
-                <TotalFileCount totalFileCount={totalFileCount} />
                 <ContinuousExport
                     continuousExport={continuousExport}
                     toggleContinuousExport={toggleContinuousExport}
                 />
+                <SpaceBetweenFlex minHeight={'48px'} pr={'16px'}>
+                    <Typography color="text.secondary">
+                        {t('TOTAL_ITEMS')}
+                    </Typography>
+                    <Typography color="text.secondary">
+                        {fileExportStats.totalCount}
+                    </Typography>
+                </SpaceBetweenFlex>
             </DialogContent>
             <Divider />
-            <ExportDynamicContent />
+            <ExportDynamicContent
+                exportStage={exportStage}
+                startExport={startExport}
+                stopExport={stopExport}
+                onHide={props.onHide}
+                lastExportTime={lastExportTime}
+                pendingFileCount={fileExportStats.pendingCount}
+                exportProgress={exportProgress}
+            />
         </Dialog>
     );
 }
@@ -328,17 +307,6 @@ function ExportDirectory({
     );
 }
 
-function TotalFileCount({ totalFileCount }) {
-    return (
-        <SpaceBetweenFlex minHeight={'40px'} pr={2}>
-            <Typography color={'text.secondary'}>
-                {t('TOTAL_FILE_COUNT')}{' '}
-            </Typography>
-            <Typography>{totalFileCount}</Typography>
-        </SpaceBetweenFlex>
-    );
-}
-
 function ExportDirectoryOption({ changeExportDirectory }) {
     return (
         <OverflowMenu
@@ -360,7 +328,7 @@ function ExportDirectoryOption({ changeExportDirectory }) {
 
 function ContinuousExport({ continuousExport, toggleContinuousExport }) {
     return (
-        <SpaceBetweenFlex minHeight={'40px'}>
+        <SpaceBetweenFlex minHeight={'48px'}>
             <Typography color="text.secondary">
                 {t('CONTINUOUS_EXPORT')}
             </Typography>
@@ -374,3 +342,48 @@ function ContinuousExport({ continuousExport, toggleContinuousExport }) {
         </SpaceBetweenFlex>
     );
 }
+
+const ExportDynamicContent = ({
+    exportStage,
+    startExport,
+    stopExport,
+    onHide,
+    lastExportTime,
+    pendingFileCount,
+    exportProgress,
+}: {
+    exportStage: ExportStage;
+    startExport: () => void;
+    stopExport: () => void;
+    onHide: () => void;
+    lastExportTime: number;
+    pendingFileCount: number;
+    exportProgress: ExportProgress;
+}) => {
+    switch (exportStage) {
+        case ExportStage.INIT:
+            return <ExportInit startExport={startExport} />;
+
+        case ExportStage.INPROGRESS:
+            return (
+                <ExportInProgress
+                    exportStage={exportStage}
+                    exportProgress={exportProgress}
+                    stopExport={stopExport}
+                    closeExportDialog={onHide}
+                />
+            );
+        case ExportStage.FINISHED:
+            return (
+                <ExportFinished
+                    onHide={onHide}
+                    lastExportTime={lastExportTime}
+                    pendingFileCount={pendingFileCount}
+                    startExport={startExport}
+                />
+            );
+
+        default:
+            return <></>;
+    }
+};

src/services/exportService.ts

@@ -31,6 +31,7 @@ import { decodeMotionPhoto } from './motionPhotoService';
 import {
     generateStreamFromArrayBuffer,
     getFileExtension,
+    getPersonalFiles,
     mergeMetadata,
 } from 'utils/file';
 
@@ -40,8 +41,10 @@ import { Collection } from 'types/collection';
 import {
     CollectionIDNameMap,
     CollectionIDPathMap,
+    ExportProgress,
     ExportRecord,
     ExportRecordV1,
+    FileExportStats,
 } from 'types/export';
 import { User } from 'types/user';
 import { FILE_TYPE, TYPE_JPEG, TYPE_JPG } from 'constants/file';
@@ -64,7 +67,7 @@ class ExportService {
     private stopExport: boolean = false;
     private allElectronAPIsExist: boolean = false;
     private fileReader: FileReader = null;
-    private continuousExportEventListener: () => void;
+    private continuousExportEventHandler: () => void;
 
     constructor() {
         this.electronAPIs = runningInBrowser() && window['ElectronAPIs'];
@@ -93,24 +96,35 @@ class ExportService {
         }
     }
 
-    enableContinuousExport(startExport: () => void) {
+    enableContinuousExport(startExport: () => Promise<void>) {
         try {
-            if (this.continuousExportEventListener) {
+            if (this.continuousExportEventHandler) {
                 addLogLine('continuous export already enabled');
                 return;
             }
-            startExport();
+            const reRunNeeded = { current: false };
-            this.continuousExportEventListener = () => {
+            this.continuousExportEventHandler = async () => {
-                addLogLine('continuous export triggered');
+                try {
-                if (this.exportInProgress) {
+                    addLogLine('continuous export triggered');
-                    addLogLine('export in progress, skipping');
+                    if (this.exportInProgress) {
-                    return;
+                        addLogLine('export in progress, scheduling re-run');
+                        reRunNeeded.current = true;
+                        return;
+                    }
+                    await startExport();
+                    if (reRunNeeded.current) {
+                        reRunNeeded.current = false;
+                        addLogLine('re-running export');
+                        setTimeout(this.continuousExportEventHandler, 0);
+                    }
+                } catch (e) {
+                    logError(e, 'continuous export failed');
                 }
-                startExport();
             };
+            this.continuousExportEventHandler();
             eventBus.addListener(
                 Events.LOCAL_FILES_UPDATED,
-                this.continuousExportEventListener
+                this.continuousExportEventHandler
             );
         } catch (e) {
             logError(e, 'failed to enableContinuousExport ');
@@ -120,26 +134,44 @@ class ExportService {
 
     disableContinuousExport() {
         try {
-            if (!this.continuousExportEventListener) {
+            if (!this.continuousExportEventHandler) {
                 addLogLine('continuous export already disabled');
                 return;
             }
             eventBus.removeListener(
                 Events.LOCAL_FILES_UPDATED,
-                this.continuousExportEventListener
+                this.continuousExportEventHandler
             );
-            this.continuousExportEventListener = null;
+            this.continuousExportEventHandler = null;
         } catch (e) {
             logError(e, 'failed to disableContinuousExport');
             throw e;
         }
     }
 
+    getFileExportStats = async (): Promise<FileExportStats> => {
+        try {
+            const exportRecord = await this.getExportRecord();
+            const userPersonalFiles = await getPersonalFiles();
+            const unExportedFiles = getUnExportedFiles(
+                userPersonalFiles,
+                exportRecord
+            );
+            return {
+                totalCount: userPersonalFiles.length,
+                pendingCount: unExportedFiles.length,
+            };
+        } catch (e) {
+            logError(e, 'getUpdateFileLists failed');
+            throw e;
+        }
+    };
+
     stopRunningExport() {
         this.stopExport = true;
     }
 
-    async exportFiles(updateProgress: (current: number) => void) {
+    async exportFiles(updateProgress: (progress: ExportProgress) => void) {
         try {
             // eslint-disable-next-line @typescript-eslint/no-misused-promises
             if (this.exportInProgress) {
@@ -219,7 +251,7 @@ class ExportService {
         collectionIDNameMap: CollectionIDNameMap,
         renamedCollections: Collection[],
         collectionIDPathMap: CollectionIDPathMap,
-        updateProgress: (current: number) => void,
+        updateProgress: (progress: ExportProgress) => void,
         exportDir: string
     ): Promise<void> {
         try {
@@ -241,8 +273,8 @@ class ExportService {
             }
             this.stopExport = false;
             this.electronAPIs.sendNotification(t('EXPORT_NOTIFICATION.START'));
-
+            let success = 0;
-            for (const [index, file] of files.entries()) {
+            for (const file of files) {
                 if (this.stopExport) {
                     break;
                 }
@@ -264,6 +296,8 @@ class ExportService {
                         file,
                         RecordType.SUCCESS
                     );
+                    success++;
+                    updateProgress({ current: success, total: files.length });
                 } catch (e) {
                     logError(e, 'export failed for a file');
                     if (
@@ -278,7 +312,6 @@ class ExportService {
                         RecordType.FAILED
                     );
                 }
-                updateProgress(index + 1);
             }
             if (!this.stopExport) {
                 this.electronAPIs.sendNotification(
@@ -304,20 +337,8 @@ class ExportService {
                     exportRecord.exportedFiles = [];
                 }
                 exportRecord.exportedFiles.push(fileUID);
-                exportRecord.failedFiles &&
-                    (exportRecord.failedFiles = exportRecord.failedFiles.filter(
-                        (FailedFileUID) => FailedFileUID !== fileUID
-                    ));
-            } else {
-                if (!exportRecord.failedFiles) {
-                    exportRecord.failedFiles = [];
-                }
-                if (!exportRecord.failedFiles.find((x) => x === fileUID)) {
-                    exportRecord.failedFiles.push(fileUID);
-                }
             }
             exportRecord.exportedFiles = dedupe(exportRecord.exportedFiles);
-            exportRecord.failedFiles = dedupe(exportRecord.failedFiles);
             await this.updateExportRecord(exportRecord, folder);
         } catch (e) {
             logError(e, 'addFileExportedRecord failed');
@@ -375,20 +396,16 @@ class ExportService {
                 folder = getData(LS_KEYS.EXPORT)?.folder;
             }
             if (!folder) {
-                throw Error(CustomError.NO_EXPORT_FOLDER_SELECTED);
+                return null;
             }
             const exportFolderExists = this.exists(folder);
             if (!exportFolderExists) {
-                throw Error(CustomError.EXPORT_FOLDER_DOES_NOT_EXIST);
+                return null;
             }
             const recordFile = await this.electronAPIs.getExportRecord(
                 `${folder}/${EXPORT_RECORD_FILE_NAME}`
             );
-            if (recordFile) {
+            return JSON.parse(recordFile);
-                return JSON.parse(recordFile);
-            } else {
-                return {} as ExportRecord;
-            }
         } catch (e) {
             logError(e, 'export Record JSON parsing failed ');
             throw e;
@@ -675,6 +692,9 @@ class ExportService {
         if (exportRecord?.progress) {
             exportRecord.progress = undefined;
         }
+        if (exportRecord?.failedFiles) {
+            exportRecord.failedFiles = undefined;
+        }
         await this.updateExportRecord(exportRecord);
     }
 }

src/types/export/index.ts

@@ -9,9 +9,9 @@ export interface ExportProgress {
 export interface ExportedCollectionPaths {
     [collectionID: number]: string;
 }
-export interface ExportStats {
+export interface FileExportStats {
-    failed: number;
+    totalCount: number;
-    success: number;
+    pendingCount: number;
 }
 
 export interface ExportRecordV1 {
@@ -30,7 +30,6 @@ export interface ExportRecord {
     stage: ExportStage;
     lastAttemptTimestamp: number;
     exportedFiles: string[];
-    failedFiles: string[];
     exportedCollectionPaths: ExportedCollectionPaths;
 }
 

src/utils/export/index.ts

@@ -93,20 +93,6 @@ export const getExportedFiles = (
     return exportedFiles;
 };
 
-export const getExportFailedFiles = (
-    allFiles: EnteFile[],
-    exportRecord: ExportRecord
-) => {
-    const failedFiles = new Set(exportRecord?.failedFiles);
-    const filesToExport = allFiles.filter((file) => {
-        if (failedFiles.has(getExportRecordFileUID(file))) {
-            return true;
-        }
-        return false;
-    });
-    return filesToExport;
-};
-
 export const dedupe = (files: string[]) => {
     const fileSet = new Set(files);
     return Array.from(fileSet);

src/utils/file/index.ts

@@ -580,19 +580,11 @@ export function getLatestVersionFiles(files: EnteFile[]) {
     );
 }
 
-export function getUserPersonalFiles(files: EnteFile[]) {
+export async function getPersonalFiles() {
+    const files = await getLocalFiles();
     const user: User = getData(LS_KEYS.USER);
     if (!user?.id) {
         throw Error('user missing');
     }
     return files.filter((file) => file.ownerID === user.id);
 }
-
-export const getTotalFileCount = async () => {
-    try {
-        const userPersonalFiles = getUserPersonalFiles(await getLocalFiles());
-        return userPersonalFiles.length;
-    } catch (e) {
-        logError(e, 'updateTotalFileCount failed');
-    }
-};