import { runningInBrowser } from 'utils/common'; import { getExportPendingFiles, getExportFailedFiles, getFilesUploadedAfterLastExport, getFileUID, dedupe, getGoogleLikeMetadataFile, } from 'utils/export'; import { retryAsyncFunction } from 'utils/network'; import { logError } from 'utils/sentry'; import { getData, LS_KEYS } from 'utils/storage/localStorage'; import { Collection, getLocalCollections } from './collectionService'; import downloadManager from './downloadManager'; import { File, getLocalFiles } from './fileService'; export interface ExportProgress { current: number; total: number; } export interface ExportStats { failed: number; success: number; } export interface ExportRecord { stage: ExportStage; lastAttemptTimestamp: number; progress: ExportProgress; queuedFiles: string[]; exportedFiles: string[]; failedFiles: string[]; } export enum ExportStage { INIT, INPROGRESS, PAUSED, FINISHED, } enum ExportNotification { START = 'export started', IN_PROGRESS = 'export already in progress', FINISH = 'export finished', FAILED = 'export failed', ABORT = 'export aborted', PAUSE = 'export paused', UP_TO_DATE = `no new files to export`, } enum RecordType { SUCCESS = 'success', FAILED = 'failed', } export enum ExportType { NEW, PENDING, RETRY_FAILED, } const ExportRecordFileName = 'export_status.json'; const MetadataFolderName = 'metadata'; class ExportService { ElectronAPIs: any; private exportInProgress: Promise<{ paused: boolean }> = null; private recordUpdateInProgress = Promise.resolve(); private stopExport: boolean = false; private pauseExport: boolean = false; constructor() { this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs']; } async selectExportDirectory() { return await this.ElectronAPIs.selectRootDirectory(); } stopRunningExport() { this.stopExport = true; } pauseRunningExport() { this.pauseExport = true; } async exportFiles( updateProgress: (progress: ExportProgress) => void, exportType: ExportType, ) { if (this.exportInProgress) { this.ElectronAPIs.sendNotification(ExportNotification.IN_PROGRESS); return this.exportInProgress; } this.ElectronAPIs.showOnTray('starting export'); const exportDir = getData(LS_KEYS.EXPORT)?.folder; if (!exportDir) { // no-export folder set return; } let filesToExport: File[]; const allFiles = await getLocalFiles(); const collections = await getLocalCollections(); const exportRecord = await this.getExportRecord(exportDir); if (exportType === ExportType.NEW) { filesToExport = await getFilesUploadedAfterLastExport( allFiles, exportRecord, ); } else if (exportType === ExportType.RETRY_FAILED) { filesToExport = await getExportFailedFiles(allFiles, exportRecord); } else { filesToExport = await getExportPendingFiles(allFiles, exportRecord); } this.exportInProgress = this.fileExporter( filesToExport, collections, updateProgress, exportDir, ); const resp = await this.exportInProgress; this.exportInProgress = null; return resp; } async fileExporter( files: File[], collections: Collection[], updateProgress: (progress: ExportProgress) => void, dir: string, ): Promise<{ paused: boolean }> { try { if (!files?.length) { this.ElectronAPIs.sendNotification( ExportNotification.UP_TO_DATE, ); return { paused: false }; } this.stopExport = false; this.pauseExport = false; this.addFilesQueuedRecord(dir, files); const failedFileCount = 0; this.ElectronAPIs.showOnTray({ export_progress: `0 / ${files.length} files exported`, }); updateProgress({ current: 0, total: files.length, }); this.ElectronAPIs.sendNotification(ExportNotification.START); const collectionIDMap = new Map(); for (const collection of collections) { const collectionFolderPath = `${dir}/${ collection.id }_${this.sanitizeName(collection.name)}`; await this.ElectronAPIs.checkExistsAndCreateCollectionDir( collectionFolderPath, ); await this.ElectronAPIs.checkExistsAndCreateCollectionDir( `${collectionFolderPath}/${MetadataFolderName}`, ); collectionIDMap.set(collection.id, collectionFolderPath); } for (const [index, file] of files.entries()) { if (this.stopExport || this.pauseExport) { if (this.pauseExport) { this.ElectronAPIs.showOnTray({ export_progress: `${index} / ${files.length} files exported (paused)`, paused: true, }); } break; } const collectionPath = collectionIDMap.get(file.collectionID); try { await this.downloadAndSave(file, collectionPath); await this.addFileExportRecord( dir, file, RecordType.SUCCESS, ); } catch (e) { await this.addFileExportRecord( dir, file, RecordType.FAILED, ); logError( e, 'download and save failed for file during export', ); } this.ElectronAPIs.showOnTray({ export_progress: `${index + 1} / ${ files.length } files exported`, }); updateProgress({ current: index + 1, total: files.length }); } if (this.stopExport) { this.ElectronAPIs.sendNotification(ExportNotification.ABORT); this.ElectronAPIs.showOnTray(); } else if (this.pauseExport) { this.ElectronAPIs.sendNotification(ExportNotification.PAUSE); return { paused: true }; } else if (failedFileCount > 0) { this.ElectronAPIs.sendNotification(ExportNotification.FAILED); this.ElectronAPIs.showOnTray({ retry_export: `export failed - retry export`, }); } else { this.ElectronAPIs.sendNotification(ExportNotification.FINISH); this.ElectronAPIs.showOnTray(); } return { paused: false }; } catch (e) { logError(e); } } async addFilesQueuedRecord(folder: string, files: File[]) { const exportRecord = await this.getExportRecord(folder); exportRecord.queuedFiles = files.map(getFileUID); await this.updateExportRecord(exportRecord, folder); } async addFileExportRecord(folder: string, file: File, type: RecordType) { const fileUID = getFileUID(file); const exportRecord = await this.getExportRecord(folder); exportRecord.queuedFiles = exportRecord.queuedFiles.filter( (queuedFilesUID) => queuedFilesUID !== fileUID, ); if (type === RecordType.SUCCESS) { if (!exportRecord.exportedFiles) { exportRecord.exportedFiles = []; } exportRecord.exportedFiles.push(fileUID); exportRecord.failedFiles && (exportRecord.failedFiles = exportRecord.failedFiles.filter( (FailedFileUID) => FailedFileUID !== fileUID, )); } else { if (!exportRecord.failedFiles) { exportRecord.failedFiles = []; } if (!exportRecord.failedFiles.find((x) => x === fileUID)) { exportRecord.failedFiles.push(fileUID); } } exportRecord.exportedFiles = dedupe(exportRecord.exportedFiles); exportRecord.queuedFiles = dedupe(exportRecord.queuedFiles); exportRecord.failedFiles = dedupe(exportRecord.failedFiles); await this.updateExportRecord(exportRecord, folder); } async updateExportRecord(newData: Record, folder?: string) { await this.recordUpdateInProgress; this.recordUpdateInProgress = (async () => { try { if (!folder) { folder = getData(LS_KEYS.EXPORT)?.folder; } const exportRecord = await this.getExportRecord(folder); const newRecord = { ...exportRecord, ...newData }; await this.ElectronAPIs.setExportRecord( `${folder}/${ExportRecordFileName}`, JSON.stringify(newRecord, null, 2), ); } catch (e) { logError(e, 'error updating Export Record'); } })(); } async getExportRecord(folder?: string): Promise { try { await this.recordUpdateInProgress; if (!folder) { folder = getData(LS_KEYS.EXPORT)?.folder; } const recordFile = await this.ElectronAPIs.getExportRecord( `${folder}/${ExportRecordFileName}`, ); if (recordFile) { return JSON.parse(recordFile); } else { return {} as ExportRecord; } } catch (e) { logError(e, 'export Record JSON parsing failed '); } } async downloadAndSave(file: File, collectionPath: string) { const uid = `${file.id}_${this.sanitizeName(file.metadata.title)}`; const fileStream = await retryAsyncFunction(() => downloadManager.downloadFile(file), ); this.ElectronAPIs.saveStreamToDisk( `${collectionPath}/${uid}`, fileStream, ); this.ElectronAPIs.saveFileToDisk( `${collectionPath}/${MetadataFolderName}/${uid}.json`, getGoogleLikeMetadataFile(uid, file.metadata), ); } private sanitizeName(name) { return name.replaceAll('/', '_').replaceAll(' ', '_'); } isExportInProgress = () => { return this.exportInProgress !== null; }; } export default new ExportService();