ente/src/services/exportService.ts

import { runningInBrowser } from 'utils/common';
import {
    getExportPendingFiles,
    getExportFailedFiles,
    getFilesUploadedAfterLastExport,
    dedupe,
    getGoogleLikeMetadataFile,
    getExportRecordFileUID,
} from 'utils/export';
import { retryAsyncFunction } from 'utils/network';
import { logError } from 'utils/sentry';
import { getData, LS_KEYS } from 'utils/storage/localStorage';
import {
    Collection,
    getLocalCollections,
    getNonEmptyCollections,
} from './collectionService';
import downloadManager from './downloadManager';
import { File, FILE_TYPE, getLocalFiles } from './fileService';
import { decodeMotionPhoto } from './motionPhotoService';
import {
    fileNameWithoutExtension,
    generateStreamFromArrayBuffer,
} from 'utils/file';

export interface ExportProgress {
    current: number;
    total: number;
}
export interface ExportStats {
    failed: number;
    success: number;
}

export interface ExportRecord {
    stage: ExportStage;
    lastAttemptTimestamp: number;
    progress: ExportProgress;
    queuedFiles: string[];
    exportedFiles: string[];
    failedFiles: string[];
}
export enum ExportStage {
    INIT,
    INPROGRESS,
    PAUSED,
    FINISHED,
}

enum ExportNotification {
    START = 'export started',
    IN_PROGRESS = 'export already in progress',
    FINISH = 'export finished',
    FAILED = 'export failed',
    ABORT = 'export aborted',
    PAUSE = 'export paused',
    UP_TO_DATE = `no new files to export`,
}

enum RecordType {
    SUCCESS = 'success',
    FAILED = 'failed',
}
export enum ExportType {
    NEW,
    PENDING,
    RETRY_FAILED,
}

const EXPORT_RECORD_FILE_NAME = 'export_status.json';
export const METADATA_FOLDER_NAME = 'metadata';

class ExportService {
    ElectronAPIs: any;

    private exportInProgress: Promise<{ paused: boolean }> = null;
    private recordUpdateInProgress = Promise.resolve();
    private stopExport: boolean = false;
    private pauseExport: boolean = false;

    constructor() {
        this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs'];
    }
    async selectExportDirectory() {
        return await this.ElectronAPIs.selectRootDirectory();
    }
    stopRunningExport() {
        this.stopExport = true;
    }
    pauseRunningExport() {
        this.pauseExport = true;
    }
    async exportFiles(
        updateProgress: (progress: ExportProgress) => void,
        exportType: ExportType
    ) {
        if (this.exportInProgress) {
            this.ElectronAPIs.sendNotification(ExportNotification.IN_PROGRESS);
            return this.exportInProgress;
        }
        this.ElectronAPIs.showOnTray('starting export');
        const exportDir = getData(LS_KEYS.EXPORT)?.folder;
        if (!exportDir) {
            // no-export folder set
            return;
        }
        let filesToExport: File[];
        const allFiles = await getLocalFiles();
        const collections = await getLocalCollections();
        const nonEmptyCollections = getNonEmptyCollections(
            collections,
            allFiles
        );
        const exportRecord = await this.getExportRecord(exportDir);

        if (exportType === ExportType.NEW) {
            filesToExport = await getFilesUploadedAfterLastExport(
                allFiles,
                exportRecord
            );
        } else if (exportType === ExportType.RETRY_FAILED) {
            filesToExport = await getExportFailedFiles(allFiles, exportRecord);
        } else {
            filesToExport = await getExportPendingFiles(allFiles, exportRecord);
        }
        this.exportInProgress = this.fileExporter(
            filesToExport,
            nonEmptyCollections,
            updateProgress,
            exportDir
        );
        const resp = await this.exportInProgress;
        this.exportInProgress = null;
        return resp;
    }

    async fileExporter(
        files: File[],
        collections: Collection[],
        updateProgress: (progress: ExportProgress) => void,
        dir: string
    ): Promise<{ paused: boolean }> {
        try {
            if (!files?.length) {
                this.ElectronAPIs.sendNotification(
                    ExportNotification.UP_TO_DATE
                );
                return { paused: false };
            }
            this.stopExport = false;
            this.pauseExport = false;
            this.addFilesQueuedRecord(dir, files);
            const failedFileCount = 0;

            this.ElectronAPIs.showOnTray({
                export_progress: `0 / ${files.length} files exported`,
            });
            updateProgress({
                current: 0,
                total: files.length,
            });
            this.ElectronAPIs.sendNotification(ExportNotification.START);

            const collectionIDMap = new Map<number, string>();
            for (const collection of collections) {
                const collectionFolderPath = `${dir}/${
                    collection.id
                }_${this.sanitizeName(collection.name)}`;
                await this.ElectronAPIs.checkExistsAndCreateCollectionDir(
                    collectionFolderPath
                );
                await this.ElectronAPIs.checkExistsAndCreateCollectionDir(
                    `${collectionFolderPath}/${METADATA_FOLDER_NAME}`
                );
                collectionIDMap.set(collection.id, collectionFolderPath);
            }
            for (const [index, file] of files.entries()) {
                if (this.stopExport || this.pauseExport) {
                    if (this.pauseExport) {
                        this.ElectronAPIs.showOnTray({
                            export_progress: `${index} / ${files.length} files exported (paused)`,
                            paused: true,
                        });
                    }
                    break;
                }
                const collectionPath = collectionIDMap.get(file.collectionID);
                try {
                    await this.downloadAndSave(file, collectionPath);
                    await this.addFileExportRecord(
                        dir,
                        file,
                        RecordType.SUCCESS
                    );
                } catch (e) {
                    await this.addFileExportRecord(
                        dir,
                        file,
                        RecordType.FAILED
                    );
                    logError(
                        e,
                        'download and save failed for file during export'
                    );
                }
                this.ElectronAPIs.showOnTray({
                    export_progress: `${index + 1} / ${
                        files.length
                    } files exported`,
                });
                updateProgress({ current: index + 1, total: files.length });
            }
            if (this.stopExport) {
                this.ElectronAPIs.sendNotification(ExportNotification.ABORT);
                this.ElectronAPIs.showOnTray();
            } else if (this.pauseExport) {
                this.ElectronAPIs.sendNotification(ExportNotification.PAUSE);
                return { paused: true };
            } else if (failedFileCount > 0) {
                this.ElectronAPIs.sendNotification(ExportNotification.FAILED);
                this.ElectronAPIs.showOnTray({
                    retry_export: `export failed - retry export`,
                });
            } else {
                this.ElectronAPIs.sendNotification(ExportNotification.FINISH);
                this.ElectronAPIs.showOnTray();
            }
            return { paused: false };
        } catch (e) {
            logError(e, 'export failed ');
        }
    }
    async addFilesQueuedRecord(folder: string, files: File[]) {
        const exportRecord = await this.getExportRecord(folder);
        exportRecord.queuedFiles = files.map(getExportRecordFileUID);
        await this.updateExportRecord(exportRecord, folder);
    }

    async addFileExportRecord(folder: string, file: File, type: RecordType) {
        const fileUID = getExportRecordFileUID(file);
        const exportRecord = await this.getExportRecord(folder);
        exportRecord.queuedFiles = exportRecord.queuedFiles.filter(
            (queuedFilesUID) => queuedFilesUID !== fileUID
        );
        if (type === RecordType.SUCCESS) {
            if (!exportRecord.exportedFiles) {
                exportRecord.exportedFiles = [];
            }
            exportRecord.exportedFiles.push(fileUID);
            exportRecord.failedFiles &&
                (exportRecord.failedFiles = exportRecord.failedFiles.filter(
                    (FailedFileUID) => FailedFileUID !== fileUID
                ));
        } else {
            if (!exportRecord.failedFiles) {
                exportRecord.failedFiles = [];
            }
            if (!exportRecord.failedFiles.find((x) => x === fileUID)) {
                exportRecord.failedFiles.push(fileUID);
            }
        }
        exportRecord.exportedFiles = dedupe(exportRecord.exportedFiles);
        exportRecord.queuedFiles = dedupe(exportRecord.queuedFiles);
        exportRecord.failedFiles = dedupe(exportRecord.failedFiles);
        await this.updateExportRecord(exportRecord, folder);
    }

    async updateExportRecord(newData: Record<string, any>, folder?: string) {
        await this.recordUpdateInProgress;
        this.recordUpdateInProgress = (async () => {
            try {
                if (!folder) {
                    folder = getData(LS_KEYS.EXPORT)?.folder;
                }
                const exportRecord = await this.getExportRecord(folder);
                const newRecord = { ...exportRecord, ...newData };
                await this.ElectronAPIs.setExportRecord(
                    `${folder}/${EXPORT_RECORD_FILE_NAME}`,
                    JSON.stringify(newRecord, null, 2)
                );
            } catch (e) {
                logError(e, 'error updating Export Record');
            }
        })();
    }

    async getExportRecord(folder?: string): Promise<ExportRecord> {
        try {
            await this.recordUpdateInProgress;
            if (!folder) {
                folder = getData(LS_KEYS.EXPORT)?.folder;
            }
            const recordFile = await this.ElectronAPIs.getExportRecord(
                `${folder}/${EXPORT_RECORD_FILE_NAME}`
            );
            if (recordFile) {
                return JSON.parse(recordFile);
            } else {
                return {} as ExportRecord;
            }
        } catch (e) {
            logError(e, 'export Record JSON parsing failed ');
        }
    }

    async downloadAndSave(file: File, collectionPath: string) {
        const uid = `${file.id}_${this.sanitizeName(file.metadata.title)}`;
        const fileStream = await retryAsyncFunction(() =>
            downloadManager.downloadFile(file)
        );
        if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
            this.exportMotionPhoto(fileStream, file, collectionPath);
        } else {
            this.saveMediaFile(collectionPath, uid, fileStream);
            this.saveMetadataFile(collectionPath, uid, file.metadata);
        }
    }

    private async exportMotionPhoto(
        fileStream: ReadableStream<any>,
        file: File,
        collectionPath: string
    ) {
        const fileBlob = await new Response(fileStream).blob();
        const originalName = fileNameWithoutExtension(file.metadata.title);
        const motionPhoto = await decodeMotionPhoto(fileBlob, originalName);

        const imageStream = generateStreamFromArrayBuffer(motionPhoto.image);
        const imageUID = `${file.id}_${motionPhoto.imageNameTitle}`;
        this.saveMediaFile(collectionPath, imageUID, imageStream);
        this.saveMetadataFile(collectionPath, imageUID, file.metadata);

        const videoStream = generateStreamFromArrayBuffer(motionPhoto.video);
        const videoUID = `${file.id}_${motionPhoto.videoNameTitle}`;
        this.saveMediaFile(collectionPath, videoUID, videoStream);
        this.saveMetadataFile(collectionPath, videoUID, file.metadata);
    }

    private saveMediaFile(collectionPath, uid, fileStream) {
        this.ElectronAPIs.saveStreamToDisk(
            `${collectionPath}/${uid}`,
            fileStream
        );
    }
    private saveMetadataFile(collectionPath, uid, metadata) {
        this.ElectronAPIs.saveFileToDisk(
            `${collectionPath}/${METADATA_FOLDER_NAME}/${uid}.json`,
            getGoogleLikeMetadataFile(uid, metadata)
        );
    }

    private sanitizeName(name) {
        return name.replaceAll('/', '_').replaceAll(' ', '_');
    }

    isExportInProgress = () => {
        return this.exportInProgress !== null;
    };
}
export default new ExportService();