migrate export if previous export version less than current

src/services/exportService.ts

@@ -8,6 +8,12 @@ import {
     getExportRecordFileUID,
     getUniqueCollectionFolderPath,
     getUniqueFileSaveName,
+    getOldFileSavePath,
+    getOldCollectionFolderPath,
+    getFileMetadataSavePath,
+    getFileSavePath,
+    getOldFileMetadataSavePath,
+    getExportedFiles,
 } from 'utils/export';
 import { retryAsyncFunction } from 'utils/network';
 import { logError } from 'utils/sentry';
@@ -30,6 +36,7 @@ import {
 } from 'utils/file';
 import { User } from './userService';
 import { updateFileModifyDateInEXIF } from './upload/exifService';
+import { MetadataObject } from './upload/uploadService';
 
 export interface ExportProgress {
     current: number;
@@ -40,13 +47,16 @@ export interface ExportStats {
     success: number;
 }
 
+const LATEST_EXPORT_VERSION = 1;
+
 export interface ExportRecord {
-    stage: ExportStage;
+    version?: number;
-    lastAttemptTimestamp: number;
+    stage?: ExportStage;
-    progress: ExportProgress;
+    lastAttemptTimestamp?: number;
-    queuedFiles: string[];
+    progress?: ExportProgress;
-    exportedFiles: string[];
+    queuedFiles?: string[];
-    failedFiles: string[];
+    exportedFiles?: string[];
+    failedFiles?: string[];
 }
 export enum ExportStage {
     INIT,
@@ -117,33 +127,44 @@ class ExportService {
                 return;
             }
             let filesToExport: File[];
-            const allFiles = await getLocalFiles();
+            const allFiles = (await getLocalFiles()).sort(
+                (fileA, fileB) => fileA.id - fileB.id
+            );
             const collections = await getLocalCollections();
             const nonEmptyCollections = getNonEmptyCollections(
                 collections,
                 allFiles
             );
             const user: User = getData(LS_KEYS.USER);
-            const userCollections = nonEmptyCollections.filter(
+            const userCollections = nonEmptyCollections
-                (collection) => collection.owner.id === user?.id
+                .filter((collection) => collection.owner.id === user?.id)
+                .sort(
+                    (collectionA, collectionB) =>
+                        collectionA.id - collectionB.id
                 );
             const exportRecord = await this.getExportRecord(exportDir);
+            if (
+                !exportRecord.version ||
+                exportRecord.version < LATEST_EXPORT_VERSION
+            ) {
+                const exportedFiles = getExportedFiles(allFiles, exportRecord);
+                await this.migrateExport(
+                    exportDir,
+                    exportRecord.version ?? 0,
+                    collections,
+                    exportedFiles
+                );
+            }
 
             if (exportType === ExportType.NEW) {
-                filesToExport = await getFilesUploadedAfterLastExport(
+                filesToExport = getFilesUploadedAfterLastExport(
                     allFiles,
                     exportRecord
                 );
             } else if (exportType === ExportType.RETRY_FAILED) {
-                filesToExport = await getExportFailedFiles(
+                filesToExport = getExportFailedFiles(allFiles, exportRecord);
-                    allFiles,
-                    exportRecord
-                );
             } else {
-                filesToExport = await getExportPendingFiles(
+                filesToExport = getExportPendingFiles(allFiles, exportRecord);
-                    allFiles,
-                    exportRecord
-                );
             }
             this.exportInProgress = this.fileExporter(
                 filesToExport,
@@ -186,9 +207,6 @@ class ExportService {
                 total: files.length,
             });
             this.ElectronAPIs.sendNotification(ExportNotification.START);
-            collections.sort(
-                (collectionA, collectionB) => collectionA.id - collectionB.id
-            );
             const collectionIDPathMap = new Map<number, string>();
             const usedCollectionPaths = new Set<string>();
             for (const collection of collections) {
@@ -206,7 +224,6 @@ class ExportService {
                     `${collectionFolderPath}/${METADATA_FOLDER_NAME}`
                 );
             }
-            files.sort((fileA, fileB) => fileA.id - fileB.id);
             for (const [index, file] of files.entries()) {
                 if (this.stopExport || this.pauseExport) {
                     if (this.pauseExport) {
@@ -301,7 +318,7 @@ class ExportService {
         await this.updateExportRecord(exportRecord, folder);
     }
 
-    async updateExportRecord(newData: Record<string, any>, folder?: string) {
+    async updateExportRecord(newData: ExportRecord, folder?: string) {
         await this.recordUpdateInProgress;
         this.recordUpdateInProgress = (async () => {
             try {
@@ -399,21 +416,122 @@ class ExportService {
         this.saveMetadataFile(collectionPath, videoSaveName, file.metadata);
     }
 
-    private saveMediaFile(collectionPath, uid, fileStream) {
+    private saveMediaFile(
+        collectionFolderPath: string,
+        fileSaveName: string,
+        fileStream: ReadableStream<any>
+    ) {
         this.ElectronAPIs.saveStreamToDisk(
-            `${collectionPath}/${uid}`,
+            getFileSavePath(collectionFolderPath, fileSaveName),
             fileStream
         );
     }
-    private saveMetadataFile(collectionPath, uid, metadata) {
+    private saveMetadataFile(
+        collectionFolderPath: string,
+        fileSaveName: string,
+        metadata: MetadataObject
+    ) {
         this.ElectronAPIs.saveFileToDisk(
-            `${collectionPath}/${METADATA_FOLDER_NAME}/${uid}.json`,
+            getFileMetadataSavePath(collectionFolderPath, fileSaveName),
-            getGoogleLikeMetadataFile(uid, metadata)
+            getGoogleLikeMetadataFile(fileSaveName, metadata)
         );
     }
 
     isExportInProgress = () => {
         return this.exportInProgress !== null;
     };
+
+    private async migrateExport(
+        exportDir: string,
+        currentVersion: number,
+        collections: Collection[],
+        files: File[]
+    ) {
+        if (currentVersion === 0) {
+            const collectionIDPathMap = new Map<number, string>();
+
+            await this.migrateCollectionFolders(
+                collections,
+                exportDir,
+                collectionIDPathMap
+            );
+            await this.migrateFiles(files, collectionIDPathMap);
+
+            await this.updateExportRecord({
+                version: LATEST_EXPORT_VERSION,
+            });
+        }
+    }
+
+    private async migrateCollectionFolders(
+        collections: Collection[],
+        dir: string,
+        collectionIDPathMap: Map<number, string>
+    ) {
+        const tempUsedCollectionPaths = new Set<string>();
+        for (const collection of collections) {
+            const oldCollectionFolderPath = getOldCollectionFolderPath(
+                dir,
+                collection
+            );
+            const newCollectionFolderPath = getUniqueCollectionFolderPath(
+                dir,
+                collection.name,
+                tempUsedCollectionPaths
+            );
+            tempUsedCollectionPaths.add(newCollectionFolderPath);
+            collectionIDPathMap.set(collection.id, newCollectionFolderPath);
+            await this.ElectronAPIs.checkExistsAndRename(
+                oldCollectionFolderPath,
+                newCollectionFolderPath
+            );
+            await this.ElectronAPIs.checkExistsAndRename(
+                `${oldCollectionFolderPath}/${METADATA_FOLDER_NAME}/`,
+                `${newCollectionFolderPath}/${METADATA_FOLDER_NAME}`
+            );
+        }
+    }
+
+    private async migrateFiles(
+        files: File[],
+        collectionIDPathMap: Map<number, string>
+    ) {
+        const tempUsedFileSaveNames = new Set<string>();
+        for (let file of files) {
+            const oldFileSavePath = getOldFileSavePath(
+                collectionIDPathMap.get(file.collectionID),
+                file
+            );
+            const oldFileMetadataSavePath = getOldFileMetadataSavePath(
+                collectionIDPathMap.get(file.collectionID),
+                file
+            );
+            file = mergeMetadata([file])[0];
+            const newFileSaveName = getUniqueFileSaveName(
+                file.metadata.title,
+                tempUsedFileSaveNames
+            );
+            tempUsedFileSaveNames.add(newFileSaveName);
+
+            const newFileSavePath = getFileSavePath(
+                collectionIDPathMap.get(file.collectionID),
+                newFileSaveName
+            );
+
+            const newFileMetadataSavePath = getFileMetadataSavePath(
+                collectionIDPathMap.get(file.collectionID),
+                newFileSaveName
+            );
+            await this.ElectronAPIs.checkExistsAndRename(
+                oldFileSavePath,
+                newFileSavePath
+            );
+            console.log(oldFileMetadataSavePath, newFileMetadataSavePath);
+            await this.ElectronAPIs.checkExistsAndRename(
+                oldFileMetadataSavePath,
+                newFileMetadataSavePath
+            );
+        }
+    }
 }
 export default new ExportService();

src/utils/export/index.ts

@@ -1,4 +1,5 @@
-import { ExportRecord } from 'services/exportService';
+import { Collection } from 'services/collectionService';
+import { ExportRecord, METADATA_FOLDER_NAME } from 'services/exportService';
 import { File } from 'services/fileService';
 import { MetadataObject } from 'services/upload/uploadService';
 import { formatDate } from 'utils/file';
@@ -6,7 +7,7 @@ import { formatDate } from 'utils/file';
 export const getExportRecordFileUID = (file: File) =>
     `${file.id}_${file.collectionID}_${file.updationTime}`;
 
-export const getExportPendingFiles = async (
+export const getExportPendingFiles = (
     allFiles: File[],
     exportRecord: ExportRecord
 ) => {
@@ -19,7 +20,7 @@ export const getExportPendingFiles = async (
     return unExportedFiles;
 };
 
-export const getFilesUploadedAfterLastExport = async (
+export const getFilesUploadedAfterLastExport = (
     allFiles: File[],
     exportRecord: ExportRecord
 ) => {
@@ -32,7 +33,20 @@ export const getFilesUploadedAfterLastExport = async (
     return unExportedFiles;
 };
 
-export const getExportFailedFiles = async (
+export const getExportedFiles = (
+    allFiles: File[],
+    exportRecord: ExportRecord
+) => {
+    const exportedFileIds = new Set(exportRecord?.exportedFiles);
+    const exportedFiles = allFiles.filter((file) => {
+        if (exportedFileIds.has(getExportRecordFileUID(file))) {
+            return file;
+        }
+    });
+    return exportedFiles;
+};
+
+export const getExportFailedFiles = (
     allFiles: File[],
     exportRecord: ExportRecord
 ) => {
@@ -52,14 +66,14 @@ export const dedupe = (files: any[]) => {
 };
 
 export const getGoogleLikeMetadataFile = (
-    uid: string,
+    fileSaveName: string,
     metadata: MetadataObject
 ) => {
     const creationTime = Math.floor(metadata.creationTime / 1000000);
     const modificationTime = Math.floor(metadata.modificationTime / 1000000);
     return JSON.stringify(
         {
-            title: uid,
+            title: fileSaveName,
             creationTime: {
                 timestamp: creationTime,
                 formatted: formatDate(creationTime * 1000),
@@ -101,17 +115,42 @@ export const getUniqueFileSaveName = (
     filename: string,
     usedFileNamesInCollection: Set<string>
 ) => {
-    let fileSaveName = filename;
+    let fileSaveName = sanitizeName(filename);
     const count = 1;
     while (
         usedFileNamesInCollection &&
         usedFileNamesInCollection.has(fileSaveName)
     ) {
-        fileSaveName = filename + `(${count})`;
+        fileSaveName = sanitizeName(filename) + `(${count})`;
     }
     return fileSaveName;
 };
 
-export const sanitizeName = (name: string) => {
+export const getFileMetadataSavePath = (
-    return name.replaceAll('/', '_').replaceAll(' ', '_');
+    collectionFolderPath: string,
-};
+    fileSaveName: string
+) => `${collectionFolderPath}/${METADATA_FOLDER_NAME}/${fileSaveName}.json`;
+
+export const getFileSavePath = (
+    collectionFolderPath: string,
+    fileSaveName: string
+) => `${collectionFolderPath}/${fileSaveName}`;
+
+export const sanitizeName = (name: string) =>
+    name.replaceAll('/', '_').replaceAll(' ', '_');
+
+export const getOldCollectionFolderPath = (
+    dir: string,
+    collection: Collection
+) => `${dir}/${collection.id}_${sanitizeName(collection.name)}`;
+
+export const getOldFileSavePath = (collectionFolderPath: string, file: File) =>
+    `${collectionFolderPath}/${file.id}_${sanitizeName(file.metadata.title)}`;
+
+export const getOldFileMetadataSavePath = (
+    collectionFolderPath: string,
+    file: File
+) =>
+    `${collectionFolderPath}/${METADATA_FOLDER_NAME}/${file.id}_${sanitizeName(
+        file.metadata.title
+    )}.json`;