beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Simplify export Flows (#988)

Browse source
This commit is contained in:
Abhinav Kumar 2023-03-23 14:20:34 +05:30 committed by GitHub
parent efb9af9b38 169ba33aeb
commit cf4b1ca6d9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 233 additions and 484 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
public/locales/en/translation.json
View file

@ -300,7 +300,7 @@
"EXPORT_DATA": "Export data",
"SELECT_FOLDER": "Select folder",
"DESTINATION": "Destination",
"EXPORT_SIZE": "Export size",
"TOTAL_FILE_COUNT": "Total file count",
"START": "Start",
"EXPORT_IN_PROGRESS": "Export in progress...",
"PAUSE": "Pause",
@ -568,5 +568,13 @@
"WEEK": "after a week",
"MONTH": "after a month",
"YEAR": "after a year"
},
"STOP_EXPORT": "Stop",
"EXPORT_PROGRESS": "<a>{{progress.current}} / {{progress.total}}</a> files exported",
"EXPORT_NOTIFICATION": {
"START": "Export started",
"IN_PROGRESS": "Export already in progress",
"FINISH": "Export finished",
"UP_TO_DATE": "No new files to export"
}
}

1
public/locales/fr/translation.json
View file

@ -298,7 +298,6 @@
"EXPORT_DATA": "Exporter les données",
"SELECT_FOLDER": "Sélectionner un dossier",
"DESTINATION": "Destination",
"EXPORT_SIZE": "Taille d'export",
"START": "Démarrer",
"EXPORT_IN_PROGRESS": "Export en cours...",
"PAUSE": "Pause",

12
src/components/Container.ts
View file

@ -27,10 +27,6 @@ export const Row = styled('div')`
flex: 1;
`;
export const Label = styled('div')<{ width?: string }>`
width: ${(props) => props.width ?? '70%'};
color: ${(props) => props.theme.palette.text.secondary};
`;
export const Value = styled('div')<{ width?: string }>`
display: flex;
justify-content: flex-start;
@ -77,3 +73,11 @@ export const IconButtonWithBG = styled(IconButton)(({ theme }) => ({
export const HorizontalFlex = styled(Box)({
display: 'flex',
});
export const VerticalFlex = styled(HorizontalFlex)({
flexDirection: 'column',
});
export const VerticallyCenteredFlex = styled(HorizontalFlex)({
alignItems: 'center',
});

72
src/components/ExportFinished.tsx
View file

@ -1,74 +1,62 @@
import { Button, DialogActions, DialogContent, Stack } from '@mui/material';
import {
Button,
DialogActions,
DialogContent,
Stack,
Typography,
} from '@mui/material';
import React from 'react';
import { t } from 'i18next';
import { ExportStats } from 'types/export';
import { formatDateTime } from 'utils/time/format';
import { FlexWrapper, Label, Value } from './Container';
import { ComfySpan } from './ExportInProgress';
import { SpaceBetweenFlex } from './Container';
interface Props {
onHide: () => void;
lastExportTime: number;
exportStats: ExportStats;
exportFiles: () => void;
retryFailed: () => void;
startExport: () => void;
}
export default function ExportFinished(props: Props) {
const totalFiles = props.exportStats.failed + props.exportStats.success;
return (
<>
<DialogContent>
<Stack spacing={2.5}>
<FlexWrapper>
<Label width="40%">{t('LAST_EXPORT_TIME')}</Label>
<Value width="60%">
<Stack spacing={2.5} pr={2}>
<SpaceBetweenFlex>
<Typography color="text.secondary">
{t('LAST_EXPORT_TIME')}
</Typography>
<Typography>
{formatDateTime(props.lastExportTime)}
</Value>
</FlexWrapper>
<FlexWrapper>
<Label width="40%">
</Typography>
</SpaceBetweenFlex>
<SpaceBetweenFlex>
<Typography color="text.secondary">
{t('SUCCESSFULLY_EXPORTED_FILES')}
</Label>
<Value width="60%">
<ComfySpan>
{props.exportStats.success} / {totalFiles}
</ComfySpan>
</Value>
</FlexWrapper>
</Typography>
<Typography>{props.exportStats.success}</Typography>
</SpaceBetweenFlex>
{props.exportStats.failed > 0 && (
<FlexWrapper>
<Label width="40%">
<SpaceBetweenFlex>
<Typography color="text.secondary">
{t('FAILED_EXPORTED_FILES')}
</Label>
<Value width="60%">
<ComfySpan>
{props.exportStats.failed} / {totalFiles}
</ComfySpan>
</Value>
</FlexWrapper>
</Typography>
<Typography>{props.exportStats.failed}</Typography>
</SpaceBetweenFlex>
)}
</Stack>
</DialogContent>
<DialogActions>
{props.exportStats.failed !== 0 ? (
<Button
size="large"
color="accent"
onClick={props.retryFailed}>
{t('RETRY_EXPORT')}
<Button color="secondary" size="large" onClick={props.onHide}>
{t('CLOSE')}
</Button>
) : (
<Button
size="large"
color="primary"
onClick={props.exportFiles}>
onClick={props.startExport}>
{t('EXPORT_AGAIN')}
</Button>
)}
<Button color="secondary" size="large" onClick={props.onHide}>
{t('CLOSE')}
</Button>
</DialogActions>
</>
);

53
src/components/ExportInProgress.tsx
View file

@ -11,8 +11,10 @@ import { ExportStage } from 'constants/export';
import VerticallyCentered, { FlexWrapper } from './Container';
import { ProgressBar } from 'react-bootstrap';
import { t } from 'i18next';
import { Trans } from 'react-i18next';
export const ComfySpan = styled('span')`
padding: 0 0.5rem;
word-spacing: 1rem;
color: #ddd;
`;
@ -20,9 +22,8 @@ export const ComfySpan = styled('span')`
interface Props {
exportStage: ExportStage;
exportProgress: ExportProgress;
resumeExport: () => void;
cancelExport: () => void;
pauseExport: () => void;
stopExport: () => void;
closeExportDialog: () => void;
}
export default function ExportInProgress(props: Props) {
@ -31,17 +32,15 @@ export default function ExportInProgress(props: Props) {
<DialogContent>
<VerticallyCentered>
<Box mb={1.5}>
<ComfySpan>
{' '}
{props.exportProgress.current} /{' '}
{props.exportProgress.total}{' '}
</ComfySpan>{' '}
<span>
{' '}
files exported{' '}
{props.exportStage === ExportStage.PAUSED &&
`(paused)`}
</span>
<Trans
i18nKey={'EXPORT_PROGRESS'}
components={{
a: <ComfySpan />,
}}
values={{
progress: props.exportProgress,
}}
/>
</Box>
<FlexWrapper px={1}>
<ProgressBar
@ -50,35 +49,21 @@ export default function ExportInProgress(props: Props) {
(props.exportProgress.current * 100) /
props.exportProgress.total
)}
animated={
!(props.exportStage === ExportStage.PAUSED)
}
animated
variant="upload-progress-bar"
/>
</FlexWrapper>
</VerticallyCentered>
</DialogContent>
<DialogActions>
{props.exportStage === ExportStage.PAUSED ? (
<Button
color="secondary"
size="large"
onClick={props.resumeExport}
color="accent">
{t('RESUME')}
onClick={props.closeExportDialog}>
{t('CLOSE')}
</Button>
) : (
<Button
size="large"
onClick={props.pauseExport}
color="primary">
{t('PAUSE')}
</Button>
)}
<Button
size="large"
onClick={props.cancelExport}
color="secondary">
{t('CANCEL')}
<Button size="large" color="danger" onClick={props.stopExport}>
{t('STOP_EXPORT')}
</Button>
</DialogActions>
</>

312
src/components/ExportModal.tsx
View file

@ -1,40 +1,34 @@
import isElectron from 'is-electron';
import React, { useEffect, useMemo, useState, useContext } from 'react';
import React, { useEffect, useState, useContext } from 'react';
import exportService from 'services/exportService';
import { ExportProgress, ExportStats } from 'types/export';
import { getLocalFiles } from 'services/fileService';
import { User } from 'types/user';
import {
Box,
Button,
Dialog,
DialogContent,
Divider,
Stack,
styled,
Tooltip,
Typography,
} from '@mui/material';
import { sleep } from 'utils/common';
import { getExportRecordFileUID } from 'utils/export';
import { logError } from 'utils/sentry';
import { getData, LS_KEYS, setData } from 'utils/storage/localStorage';
import { FlexWrapper, Label, Value } from './Container';
import { SpaceBetweenFlex, VerticallyCenteredFlex } from './Container';
import ExportFinished from './ExportFinished';
import ExportInit from './ExportInit';
import ExportInProgress from './ExportInProgress';
import FolderIcon from '@mui/icons-material/Folder';
import { ExportStage, ExportType } from 'constants/export';
import EnteSpinner from './EnteSpinner';
import { ExportStage } from 'constants/export';
import DialogTitleWithCloseButton from './DialogBox/TitleWithCloseButton';
import MoreHoriz from '@mui/icons-material/MoreHoriz';
import OverflowMenu from './OverflowMenu/menu';
import { OverflowMenuOption } from './OverflowMenu/option';
import { convertBytesToHumanReadable } from 'utils/file/size';
import { CustomError } from 'utils/error';
import { getLocalUserDetails } from 'utils/user';
import { AppContext } from 'pages/_app';
import { getExportDirectoryDoesNotExistMessage } from 'utils/ui';
import { addLogLine } from 'utils/logging';
import { t } from 'i18next';
import { getTotalFileCount } from 'utils/file';
import { eventBus, Events } from 'services/events';
const ExportFolderPathContainer = styled('span')`
white-space: nowrap;
@ -53,10 +47,9 @@ interface Props {
}
export default function ExportModal(props: Props) {
const appContext = useContext(AppContext);
const userDetails = useMemo(() => getLocalUserDetails(), []);
const [exportStage, setExportStage] = useState(ExportStage.INIT);
const [exportFolder, setExportFolder] = useState('');
const [exportSize, setExportSize] = useState('');
const [totalFileCount, setTotalFileCount] = useState(0);
const [exportProgress, setExportProgress] = useState<ExportProgress>({
current: 0,
total: 0,
@ -76,19 +69,11 @@ export default function ExportModal(props: Props) {
}
try {
setExportFolder(getData(LS_KEYS.EXPORT)?.folder);
exportService.electronAPIs.registerStopExportListener(
stopExportHandler
);
exportService.electronAPIs.registerPauseExportListener(
pauseExportHandler
);
exportService.electronAPIs.registerResumeExportListener(
resumeExportHandler
);
exportService.electronAPIs.registerRetryFailedExportListener(
retryFailedExportHandler
);
const localFileUpdateHandler = async () => {
setTotalFileCount(await getTotalFileCount());
};
localFileUpdateHandler();
eventBus.on(Events.LOCAL_FILES_UPDATED, localFileUpdateHandler);
} catch (e) {
logError(e, 'error in exportModal');
}
@ -103,15 +88,12 @@ export default function ExportModal(props: Props) {
const exportInfo = await exportService.getExportRecord();
setExportStage(exportInfo?.stage ?? ExportStage.INIT);
setLastExportTime(exportInfo?.lastAttemptTimestamp);
setExportProgress(
exportInfo?.progress ?? { current: 0, total: 0 }
);
setExportStats({
success: exportInfo?.exportedFiles?.length ?? 0,
failed: exportInfo?.failedFiles?.length ?? 0,
});
if (exportInfo?.stage === ExportStage.INPROGRESS) {
await resumeExport();
await startExport();
}
} catch (e) {
logError(e, 'error handling exportFolder change');
@ -120,56 +102,6 @@ export default function ExportModal(props: Props) {
void main();
}, [exportFolder]);
useEffect(() => {
if (!props.show) {
return;
}
const main = async () => {
const user: User = getData(LS_KEYS.USER);
if (exportStage === ExportStage.FINISHED) {
try {
const localFiles = await getLocalFiles();
const userPersonalFiles = localFiles.filter(
(file) => file.ownerID === user?.id
);
const exportRecord = await exportService.getExportRecord();
const exportedFileCnt = exportRecord.exportedFiles?.length;
const failedFilesCnt = exportRecord.failedFiles?.length;
const syncedFilesCnt = userPersonalFiles.length;
if (syncedFilesCnt > exportedFileCnt + failedFilesCnt) {
await updateExportProgress({
current: exportedFileCnt + failedFilesCnt,
total: syncedFilesCnt,
});
const exportFileUIDs = new Set([
...exportRecord.exportedFiles,
...exportRecord.failedFiles,
]);
const unExportedFiles = userPersonalFiles.filter(
(file) =>
!exportFileUIDs.has(
getExportRecordFileUID(file)
)
);
await exportService.addFilesQueuedRecord(
exportFolder,
unExportedFiles
);
await updateExportStage(ExportStage.PAUSED);
}
} catch (e) {
setExportStage(ExportStage.INIT);
logError(e, 'error while updating exportModal on reopen');
}
}
};
void main();
}, [props.show]);
useEffect(() => {
setExportSize(convertBytesToHumanReadable(userDetails?.usage));
}, [userDetails]);
// =============
// STATE UPDATERS
// ==============
@ -190,20 +122,12 @@ export default function ExportModal(props: Props) {
});
};
const updateExportProgress = async (newProgress: ExportProgress) => {
setExportProgress(newProgress);
await exportService.updateExportRecord({ progress: newProgress });
};
// ======================
// HELPER FUNCTIONS
// =========================
const preExportRun = async () => {
const exportFolder = getData(LS_KEYS.EXPORT)?.folder;
if (!exportFolder) {
await selectExportDirectory();
}
const exportFolderExists = exportService.exists(exportFolder);
if (!exportFolderExists) {
appContext.setDialogMessage(
@ -212,24 +136,12 @@ export default function ExportModal(props: Props) {
return;
}
await updateExportStage(ExportStage.INPROGRESS);
await sleep(100);
};
const postExportRun = async (exportResult?: { paused?: boolean }) => {
if (!exportResult?.paused) {
await updateExportStage(ExportStage.FINISHED);
await sleep(100);
await updateExportTime(Date.now());
await syncExportStatsWithRecord();
}
};
const selectExportDirectory = async () => {
const newFolder = await exportService.selectExportDirectory();
if (newFolder) {
updateExportFolder(newFolder);
} else {
throw Error(CustomError.REQUEST_CANCELLED);
}
const postExportRun = async () => {
await updateExportStage(ExportStage.FINISHED);
await updateExportTime(Date.now());
await syncExportStatsWithRecord();
};
const syncExportStatsWithRecord = async () => {
@ -243,19 +155,38 @@ export default function ExportModal(props: Props) {
// UI functions
// =============
const changeExportDirectory = async () => {
try {
const newFolder = await exportService.selectExportDirectory();
if (newFolder) {
updateExportFolder(newFolder);
}
} catch (e) {
logError(e, 'selectExportDirectory failed');
}
};
const startExport = async () => {
try {
await preExportRun();
await updateExportProgress({ current: 0, total: 0 });
const exportResult = await exportService.exportFiles(
updateExportProgress,
ExportType.NEW
);
await postExportRun(exportResult);
const exportRecord = await exportService.getExportRecord();
const totalFileCount = await getTotalFileCount();
const exportedFileCount = exportRecord.exportedFiles?.length ?? 0;
setExportProgress({
current: exportedFileCount,
total: totalFileCount,
});
const updateExportStatsWithOffset = (current: number) =>
setExportProgress({
current: exportedFileCount + current,
total: totalFileCount,
});
await exportService.exportFiles(updateExportStatsWithOffset);
await postExportRun();
} catch (e) {
if (e.message !== CustomError.REQUEST_CANCELLED) {
logError(e, 'startExport failed');
}
logError(e, 'resumeExport failed');
}
};
@ -264,105 +195,22 @@ export default function ExportModal(props: Props) {
exportService.stopRunningExport();
await postExportRun();
} catch (e) {
if (e.message !== CustomError.REQUEST_CANCELLED) {
logError(e, 'stopExport failed');
}
}
};
const pauseExport = async () => {
try {
await updateExportStage(ExportStage.PAUSED);
exportService.pauseRunningExport();
await postExportRun({ paused: true });
} catch (e) {
if (e.message !== CustomError.REQUEST_CANCELLED) {
logError(e, 'pauseExport failed');
}
}
};
const resumeExport = async () => {
try {
const exportRecord = await exportService.getExportRecord();
await preExportRun();
const pausedStageProgress = exportRecord.progress;
setExportProgress(pausedStageProgress);
addLogLine(
`resuming export, pausedStageProgress: ${JSON.stringify(
pausedStageProgress
)}`
);
const updateExportStatsWithOffset = (progress: ExportProgress) =>
updateExportProgress({
current: pausedStageProgress.current + progress.current,
total: pausedStageProgress.current + progress.total,
});
const exportResult = await exportService.exportFiles(
updateExportStatsWithOffset,
ExportType.PENDING
);
await postExportRun(exportResult);
} catch (e) {
if (e.message !== CustomError.REQUEST_CANCELLED) {
logError(e, 'resumeExport failed');
}
}
};
const retryFailedExport = async () => {
try {
await preExportRun();
await updateExportProgress({
current: 0,
total: exportStats.failed,
});
const exportResult = await exportService.exportFiles(
updateExportProgress,
ExportType.RETRY_FAILED
);
await postExportRun(exportResult);
} catch (e) {
if (e.message !== CustomError.REQUEST_CANCELLED) {
logError(e, 'retryFailedExport failed');
}
}
};
const startExportHandler = () => {
void startExport();
};
const stopExportHandler = () => {
void stopExport();
};
const pauseExportHandler = () => {
void pauseExport();
};
const resumeExportHandler = () => {
void resumeExport();
};
const retryFailedExportHandler = () => {
void retryFailedExport();
};
const ExportDynamicContent = () => {
switch (exportStage) {
case ExportStage.INIT:
return <ExportInit startExport={startExportHandler} />;
return <ExportInit startExport={startExport} />;
case ExportStage.INPROGRESS:
case ExportStage.PAUSED:
return (
<ExportInProgress
exportStage={exportStage}
exportProgress={exportProgress}
resumeExport={resumeExportHandler}
cancelExport={stopExportHandler}
pauseExport={pauseExportHandler}
stopExport={stopExport}
closeExportDialog={props.onHide}
/>
);
case ExportStage.FINISHED:
@ -371,8 +219,7 @@ export default function ExportModal(props: Props) {
onHide={props.onHide}
lastExportTime={lastExportTime}
exportStats={exportStats}
exportFiles={startExportHandler}
retryFailed={retryFailedExportHandler}
startExport={startExport}
/>
);
@ -387,14 +234,12 @@ export default function ExportModal(props: Props) {
{t('EXPORT_DATA')}
</DialogTitleWithCloseButton>
<DialogContent>
<Stack spacing={2}>
<ExportDirectory
exportFolder={exportFolder}
selectExportDirectory={selectExportDirectory}
changeExportDirectory={changeExportDirectory}
exportStage={exportStage}
/>
<ExportSize exportSize={exportSize} />
</Stack>
<TotalFileCount totalFileCount={totalFileCount} />
</DialogContent>
<Divider />
<ExportDynamicContent />
@ -402,56 +247,49 @@ export default function ExportModal(props: Props) {
);
}
function ExportDirectory({ exportFolder, selectExportDirectory, exportStage }) {
function ExportDirectory({ exportFolder, changeExportDirectory, exportStage }) {
return (
<FlexWrapper>
<Label width="30%">{t('DESTINATION')}</Label>
<Value width="70%">
<SpaceBetweenFlex minHeight={'48px'}>
<Typography color="text.secondary">{t('DESTINATION')}</Typography>
<>
{!exportFolder ? (
<Button color={'accent'} onClick={selectExportDirectory}>
<Button color={'accent'} onClick={changeExportDirectory}>
{t('SELECT_FOLDER')}
</Button>
) : (
<>
<VerticallyCenteredFlex>
<Tooltip title={exportFolder}>
<ExportFolderPathContainer>
{exportFolder}
</ExportFolderPathContainer>
</Tooltip>
{(exportStage === ExportStage.FINISHED ||
exportStage === ExportStage.INIT) && (
{exportStage === ExportStage.FINISHED ||
exportStage === ExportStage.INIT ? (
<ExportDirectoryOption
selectExportDirectory={selectExportDirectory}
changeExportDirectory={changeExportDirectory}
/>
) : (
<Box sx={{ width: '16px' }} />
)}
</VerticallyCenteredFlex>
)}
</>
)}
</Value>
</FlexWrapper>
</SpaceBetweenFlex>
);
}
function ExportSize({ exportSize }) {
function TotalFileCount({ totalFileCount }) {
return (
<FlexWrapper>
<Label width="30%">{t('EXPORT_SIZE')} </Label>
<Value width="70%">
{exportSize ? `${exportSize}` : <EnteSpinner />}
</Value>
</FlexWrapper>
<SpaceBetweenFlex minHeight={'40px'} pr={2}>
<Typography color={'text.secondary'}>
{t('TOTAL_FILE_COUNT')}{' '}
</Typography>
<Typography>{totalFileCount}</Typography>
</SpaceBetweenFlex>
);
}
function ExportDirectoryOption({ selectExportDirectory }) {
const handleClick = () => {
try {
selectExportDirectory();
} catch (e) {
if (e.message !== CustomError.REQUEST_CANCELLED) {
logError(e, 'startExport failed');
}
}
};
function ExportDirectoryOption({ changeExportDirectory }) {
return (
<OverflowMenu
triggerButtonProps={{
@ -462,7 +300,7 @@ function ExportDirectoryOption({ selectExportDirectory }) {
ariaControls={'export-option'}
triggerButtonIcon={<MoreHoriz />}>
<OverflowMenuOption
onClick={handleClick}
onClick={changeExportDirectory}
startIcon={<FolderIcon />}>
{t('CHANGE_FOLDER')}
</OverflowMenuOption>

9
src/components/PhotoViewer/FileInfo/RenderInfoItem.tsx
View file

@ -1,9 +0,0 @@
import React from 'react';
import { Label, Row, Value } from 'components/Container';
export const RenderInfoItem = (label: string, value: string | JSX.Element) => (
<Row>
<Label width="30%">{label}</Label>
<Value width="70%">{value}</Value>
</Row>
);

2
src/components/UserNameInputDialog.tsx
View file

@ -33,7 +33,7 @@ export default function UserNameInputDialog({
initialValue={uploaderName}
callback={handleSubmit}
placeholder={t('NAME_PLACEHOLDER')}
buttonText={t('add_photos', { count: toUploadFilesCount })}
buttonText={t('add_photos', { count: toUploadFilesCount ?? 0 })}
fieldType="text"
blockButton
secondaryButtonAction={onClose}

23
src/constants/export/index.ts
View file

@ -1,28 +1,11 @@
export const ENTE_METADATA_FOLDER = 'metadata';
export enum ExportNotification {
START = 'Export started',
IN_PROGRESS = 'Export already in progress',
FINISH = 'Export finished',
FAILED = 'Export failed',
ABORT = 'Export aborted',
PAUSE = 'Export paused',
UP_TO_DATE = `No new files to export`,
}
export enum RecordType {
SUCCESS = 'success',
FAILED = 'failed',
}
export enum ExportStage {
INIT,
INPROGRESS,
PAUSED,
FINISHED,
}
export enum ExportType {
NEW,
PENDING,
RETRY_FAILED,
INIT = 0,
INPROGRESS = 1,
FINISHED = 3,
}

157
src/services/exportService.ts
View file

@ -1,8 +1,6 @@
import { runningInBrowser } from 'utils/common';
import {
getExportQueuedFiles,
getExportFailedFiles,
getFilesUploadedAfterLastExport,
getUnExportedFiles,
dedupe,
getGoogleLikeMetadataFile,
getExportRecordFileUID,
@ -41,27 +39,25 @@ import { updateFileCreationDateInEXIF } from './upload/exifService';
import QueueProcessor from './queueProcessor';
import { Collection } from 'types/collection';
import {
ExportProgress,
CollectionIDPathMap,
ExportRecord,
ExportRecordV1,
} from 'types/export';
import { User } from 'types/user';
import { FILE_TYPE, TYPE_JPEG, TYPE_JPG } from 'constants/file';
import { ExportType, ExportNotification, RecordType } from 'constants/export';
import { RecordType } from 'constants/export';
import { ElectronAPIs } from 'types/electron';
import { CustomError } from 'utils/error';
import { addLogLine } from 'utils/logging';
import { t } from 'i18next';
const LATEST_EXPORT_VERSION = 1;
const EXPORT_RECORD_FILE_NAME = 'export_status.json';
class ExportService {
electronAPIs: ElectronAPIs;
private exportInProgress: Promise<{ paused: boolean }> = null;
private electronAPIs: ElectronAPIs;
private exportInProgress: Promise<void> = null;
private exportRecordUpdater = new QueueProcessor<void>(1);
private stopExport: boolean = false;
private pauseExport: boolean = false;
private allElectronAPIsExist: boolean = false;
private fileReader: FileReader = null;
@ -81,22 +77,15 @@ class ExportService {
stopRunningExport() {
this.stopExport = true;
}
pauseRunningExport() {
this.pauseExport = true;
}
async exportFiles(
updateProgress: (progress: ExportProgress) => Promise<void>,
exportType: ExportType
) {
async exportFiles(updateProgress: (current: number) => void) {
try {
// eslint-disable-next-line @typescript-eslint/no-misused-promises
if (this.exportInProgress) {
this.electronAPIs.sendNotification(
ExportNotification.IN_PROGRESS
t('EXPORT_NOTIFICATION.IN_PROGRESS')
);
return await this.exportInProgress;
}
this.electronAPIs.showOnTray('starting export');
const exportDir = getData(LS_KEYS.EXPORT)?.folder;
if (!exportDir) {
// no-export folder set
@ -104,7 +93,6 @@ class ExportService {
}
const user: User = getData(LS_KEYS.USER);
let filesToExport: EnteFile[];
const localFiles = await getLocalFiles();
const userPersonalFiles = localFiles
.filter((file) => file.ownerID === user?.id)
@ -130,32 +118,13 @@ class ExportService {
}
const exportRecord = await this.getExportRecord(exportDir);
const filesToExport = getUnExportedFiles(
userPersonalFiles,
exportRecord
);
addLogLine(
`export stats -> progress: ${JSON.stringify(
exportRecord.progress
)} stage:${exportRecord.stage} queuedFilesCount: ${
exportRecord?.queuedFiles?.length
} exportedFiles: ${exportRecord?.exportedFiles?.length}
failedFiles: ${exportRecord?.failedFiles?.length}`
);
if (exportType === ExportType.NEW) {
filesToExport = getFilesUploadedAfterLastExport(
userPersonalFiles,
exportRecord
);
} else if (exportType === ExportType.RETRY_FAILED) {
filesToExport = getExportFailedFiles(
userPersonalFiles,
exportRecord
);
} else {
filesToExport = getExportQueuedFiles(
userPersonalFiles,
exportRecord
);
}
addLogLine(
`starting export, type: ${exportType}, filesToExportCount: ${filesToExport?.length}, userPersonalFileCount: ${userPersonalFiles?.length}`
`starting export, filesToExportCount: ${filesToExport?.length}, userPersonalFileCount: ${userPersonalFiles?.length}`
);
const collectionIDPathMap: CollectionIDPathMap =
@ -192,9 +161,9 @@ class ExportService {
newCollections: Collection[],
renamedCollections: Collection[],
collectionIDPathMap: CollectionIDPathMap,
updateProgress: (progress: ExportProgress) => Promise<void>,
updateProgress: (current: number) => void,
exportDir: string
): Promise<{ paused: boolean }> {
): Promise<void> {
try {
if (newCollections?.length) {
await this.createNewCollectionFolders(
@ -215,32 +184,15 @@ class ExportService {
}
if (!files?.length) {
this.electronAPIs.sendNotification(
ExportNotification.UP_TO_DATE
t('EXPORT_NOTIFICATION.UP_TO_DATE')
);
return { paused: false };
return;
}
this.stopExport = false;
this.pauseExport = false;
await this.addFilesQueuedRecord(exportDir, files);
const failedFileCount = 0;
this.electronAPIs.showOnTray({
export_progress: `0 / ${files.length} files exported`,
});
await updateProgress({
current: 0,
total: files.length,
});
this.electronAPIs.sendNotification(ExportNotification.START);
this.electronAPIs.sendNotification(t('EXPORT_NOTIFICATION.START'));
for (const [index, file] of files.entries()) {
if (this.stopExport || this.pauseExport) {
if (this.pauseExport) {
this.electronAPIs.showOnTray({
export_progress: `${index} / ${files.length} files exported (paused)`,
paused: true,
});
}
if (this.stopExport) {
break;
}
const collectionPath = collectionIDPathMap.get(
@ -267,42 +219,18 @@ class ExportService {
RecordType.FAILED
);
}
this.electronAPIs.showOnTray({
export_progress: `${index + 1} / ${
files.length
} files exported`,
});
await updateProgress({
current: index + 1,
total: files.length,
});
updateProgress(index + 1);
}
if (this.stopExport) {
this.electronAPIs.sendNotification(ExportNotification.ABORT);
this.electronAPIs.showOnTray();
} else if (this.pauseExport) {
this.electronAPIs.sendNotification(ExportNotification.PAUSE);
return { paused: true };
} else if (failedFileCount > 0) {
this.electronAPIs.sendNotification(ExportNotification.FAILED);
this.electronAPIs.showOnTray({
retry_export: `Retry failed exports`,
});
} else {
this.electronAPIs.sendNotification(ExportNotification.FINISH);
this.electronAPIs.showOnTray();
if (!this.stopExport) {
this.electronAPIs.sendNotification(
t('EXPORT_NOTIFICATION.FINISH')
);
}
return { paused: false };
} catch (e) {
logError(e, 'fileExporter failed');
throw e;
}
}
async addFilesQueuedRecord(folder: string, files: EnteFile[]) {
const exportRecord = await this.getExportRecord(folder);
exportRecord.queuedFiles = files.map(getExportRecordFileUID);
await this.updateExportRecord(exportRecord, folder);
}
async addFileExportedRecord(
folder: string,
@ -312,9 +240,6 @@ class ExportService {
try {
const fileUID = getExportRecordFileUID(file);
const exportRecord = await this.getExportRecord(folder);
exportRecord.queuedFiles = exportRecord.queuedFiles.filter(
(queuedFilesUID) => queuedFilesUID !== fileUID
);
if (type === RecordType.SUCCESS) {
if (!exportRecord.exportedFiles) {
exportRecord.exportedFiles = [];
@ -333,7 +258,6 @@ class ExportService {
}
}
exportRecord.exportedFiles = dedupe(exportRecord.exportedFiles);
exportRecord.queuedFiles = dedupe(exportRecord.queuedFiles);
exportRecord.failedFiles = dedupe(exportRecord.failedFiles);
await this.updateExportRecord(exportRecord, folder);
} catch (e) {
@ -359,14 +283,17 @@ class ExportService {
await this.updateExportRecord(exportRecord, folder);
}
async updateExportRecord(newData: ExportRecord, folder?: string) {
async updateExportRecord(newData: Partial<ExportRecord>, folder?: string) {
const response = this.exportRecordUpdater.queueUpRequest(() =>
this.updateExportRecordHelper(folder, newData)
);
await response.promise;
}
async updateExportRecordHelper(folder: string, newData: ExportRecord) {
async updateExportRecordHelper(
folder: string,
newData: Partial<ExportRecord>
) {
try {
if (!folder) {
folder = getData(LS_KEYS.EXPORT)?.folder;
@ -574,7 +501,7 @@ class ExportService {
allFiles: EnteFile[]
) {
const exportRecord = await this.getExportRecord(exportDir);
const currentVersion = exportRecord?.version ?? 0;
let currentVersion = exportRecord?.version ?? 0;
if (currentVersion === 0) {
const collectionIDPathMap = new Map<number, string>();
@ -587,9 +514,16 @@ class ExportService {
getExportedFiles(allFiles, exportRecord),
collectionIDPathMap
);
currentVersion++;
await this.updateExportRecord({
version: LATEST_EXPORT_VERSION,
version: currentVersion,
});
}
if (currentVersion === 1) {
await this.removeDeprecatedExportRecordProperties();
currentVersion++;
await this.updateExportRecord({
version: currentVersion,
});
}
}
@ -670,5 +604,16 @@ class ExportService {
);
}
}
private async removeDeprecatedExportRecordProperties() {
const exportRecord = (await this.getExportRecord()) as ExportRecordV1;
if (exportRecord?.queuedFiles) {
exportRecord.queuedFiles = undefined;
}
if (exportRecord?.progress) {
exportRecord.progress = undefined;
}
await this.updateExportRecord(exportRecord);
}
}
export default new ExportService();

5
src/types/electron/index.ts
View file

@ -21,11 +21,6 @@ export interface ElectronAPIs {
saveFileToDisk: (path: string, file: any) => Promise<void>;
selectRootDirectory: () => Promise<string>;
sendNotification: (content: string) => void;
showOnTray: (content?: any) => void;
registerResumeExportListener: (resumeExport: () => void) => void;
registerStopExportListener: (abortExport: () => void) => void;
registerPauseExportListener: (pauseExport: () => void) => void;
registerRetryFailedExportListener: (retryFailedExport: () => void) => void;
getExportRecord: (filePath: string) => Promise<string>;
setExportRecord: (filePath: string, data: string) => Promise<void>;
showUploadFilesDialog: () => Promise<ElectronFile[]>;

11
src/types/export/index.ts
View file

@ -13,7 +13,7 @@ export interface ExportStats {
success: number;
}
export interface ExportRecord {
export interface ExportRecordV1 {
version?: number;
stage?: ExportStage;
lastAttemptTimestamp?: number;
@ -23,3 +23,12 @@ export interface ExportRecord {
failedFiles?: string[];
exportedCollectionPaths?: ExportedCollectionPaths;
}
export interface ExportRecord {
version: number;
stage: ExportStage;
lastAttemptTimestamp: number;
exportedFiles: string[];
failedFiles: string[];
exportedCollectionPaths: ExportedCollectionPaths;
}

16
src/utils/export/index.ts
View file

@ -13,20 +13,6 @@ import { formatDateTimeShort } from 'utils/time/format';
export const getExportRecordFileUID = (file: EnteFile) =>
`${file.id}_${file.collectionID}_${file.updationTime}`;
export const getExportQueuedFiles = (
allFiles: EnteFile[],
exportRecord: ExportRecord
) => {
const queuedFiles = new Set(exportRecord?.queuedFiles);
const unExportedFiles = allFiles.filter((file) => {
if (queuedFiles.has(getExportRecordFileUID(file))) {
return true;
}
return false;
});
return unExportedFiles;
};
export const getCollectionsCreatedAfterLastExport = (
collections: Collection[],
exportRecord: ExportRecord
@ -79,7 +65,7 @@ export const getCollectionsRenamedAfterLastExport = (
return renamedCollections;
};
export const getFilesUploadedAfterLastExport = (
export const getUnExportedFiles = (
allFiles: EnteFile[],
exportRecord: ExportRecord
) => {

18
src/utils/file/index.ts
View file

@ -31,6 +31,7 @@ import { addLogLine } from 'utils/logging';
import { CustomError } from 'utils/error';
import { convertBytesToHumanReadable } from './size';
import ComlinkCryptoWorker from 'utils/comlink/ComlinkCryptoWorker';
import { getLocalFiles } from 'services/fileService';
const WAIT_TIME_IMAGE_CONVERSION = 30 * 1000;
@ -578,3 +579,20 @@ export function getLatestVersionFiles(files: EnteFile[]) {
(file) => !file.isDeleted
);
}
export function getUserPersonalFiles(files: EnteFile[]) {
const user: User = getData(LS_KEYS.USER);
if (!user?.id) {
throw Error('user missing');
}
return files.filter((file) => file.ownerID === user.id);
}
export const getTotalFileCount = async () => {
try {
const userPersonalFiles = getUserPersonalFiles(await getLocalFiles());
return userPersonalFiles.length;
} catch (e) {
logError(e, 'updateTotalFileCount failed');
}
};