fix migration issues

2023-04-28 13:09:12 +05:30 · 2023-04-28 13:09:12 +05:30 · e5d28c24c7
parent 43043df2c8
commit e5d28c24c7
4 changed files with 106 additions and 51 deletions
--- a/src/components/ExportModal.tsx
+++ b/src/components/ExportModal.tsx
@ -76,7 +76,7 @@ export default function ExportModal(props: Props) {
        }
        const main = async () => {
            try {
-                exportService.setUIUpdaters({
+                await exportService.init({
                    updateExportStage: updateExportStage,
                    updateExportProgress: setExportProgress,
                    updateFileExportStats: setFileExportStats,
--- a/src/services/export/index.ts
+++ b/src/services/export/index.ts
@ -17,6 +17,9 @@ import {
    getCollectionExportedFiles,
    getCollectionExportPath,
    getMetadataFolderExportPath,
+    getLivePhotoExportName,
+    isLivePhotoExportName,
+    parseLivePhotoExportName,
 } from 'utils/export';
 import { retryAsyncFunction } from 'utils/network';
 import { logError } from 'utils/sentry';
@ -53,7 +56,7 @@ import {
    getCollectionNameMap,
    getNonEmptyPersonalCollections,
 } from 'utils/collection';
-import { migrateExport } from './migration';
+import { migrateExportJSONHelper } from './migration';

 const EXPORT_RECORD_FILE_NAME = 'export_status.json';

@ -74,29 +77,26 @@ class ExportService {
        success: 0,
        failed: 0,
    };
+    private exportJSONMigrator = new QueueProcessor<void>(1);

    constructor() {
        if (runningInBrowser()) {
            this.electronAPIs = window['ElectronAPIs'];
            this.allElectronAPIsExist = !!this.electronAPIs?.exists;
            this.fileReader = new FileReader();
-            void this.migrateExportJSON();
        }
    }

    async migrateExportJSON() {
-        try {
-            const exportDir = getData(LS_KEYS.EXPORT)?.folder;
-            if (!exportDir) {
-                return;
-            }
-            const exportRecord = await this.getExportRecord(exportDir);
-            if (this.checkAllElectronAPIsExists()) {
-                await migrateExport(exportDir, exportRecord);
-            }
-        } catch (e) {
-            logError(e, 'migrateExportJSON failed');
-        }
+        const response = this.exportJSONMigrator.queueUpRequest(() =>
+            migrateExportJSONHelper()
+        );
+        return response.promise;
+    }
+
+    async init(uiUpdater: ExportUIUpdaters) {
+        this.setUIUpdaters(uiUpdater);
+        await this.migrateExportJSON();
    }

    async setUIUpdaters(uiUpdater: ExportUIUpdaters) {
@ -245,6 +245,8 @@ class ExportService {
            }
            try {
                await this.preExport();
+                // checking if migration is needed
+                await this.migrateExportJSON();
                addLogLine('export started');
                await this.runExport();
                addLogLine('export completed');
@ -618,15 +620,11 @@ class ExportService {
                    );
                    // check if filepath is for live photo
                    // livePhoto has the path in format: `JSON.stringify({image,video})`
-                    const isLivePhoto =
-                        fileExportName.startsWith('{') &&
-                        fileExportName.endsWith('}');
-
-                    if (isLivePhoto) {
+                    if (isLivePhotoExportName(fileExportName)) {
                        const {
                            image: imageExportName,
                            video: videoExportName,
-                        } = JSON.parse(fileExportName);
+                        } = parseLivePhotoExportName(fileExportName);
                        const imageExportPath = getFileExportPath(
                            collectionExportPath,
                            imageExportName
@ -672,7 +670,6 @@ class ExportService {
                                videoMetadataFileExportPath
                            )
                        );
-                        await this.removeFileExportedRecord(exportDir, fileUID);
                    } else {
                        const fileExportPath = getFileExportPath(
                            collectionExportPath,
@ -691,8 +688,8 @@ class ExportService {
                                metadataFileExportPath
                            )
                        );
-                        await this.removeFileExportedRecord(exportDir, fileUID);
                    }
+                    await this.removeFileExportedRecord(exportDir, fileUID);
                    incrementSuccess();
                } catch (e) {
                    incrementFailed();
@ -991,10 +988,7 @@ class ExportService {
            videoExportName
        );

-        return JSON.stringify({
-            image: imageExportPath,
-            video: videoExportPath,
-        });
+        return getLivePhotoExportName(imageExportPath, videoExportPath);
    }

    private async saveMediaFile(
--- a/src/services/export/migration.ts
+++ b/src/services/export/migration.ts
@ -9,7 +9,7 @@ import {
 import { EnteFile } from 'types/file';
 import { User } from 'types/user';
 import { getNonEmptyPersonalCollections } from 'utils/collection';
-import { getExportRecordFileUID, getFileExportPath } from 'utils/export';
+import { getExportRecordFileUID, getLivePhotoExportName } from 'utils/export';
 import {
    getIDBasedSortedFiles,
    getPersonalFiles,
@ -32,6 +32,25 @@ import {
    getFileMetadataSavePath,
    getFileSavePath,
 } from 'utils/export/migration';
+import { FILE_TYPE } from 'constants/file';
+import { decodeLivePhoto } from 'services/livePhotoService';
+import downloadManager from 'services/downloadManager';
+import { retryAsyncFunction } from 'utils/network';
+
+export async function migrateExportJSONHelper() {
+    try {
+        const exportDir = getData(LS_KEYS.EXPORT)?.folder;
+        if (!exportDir) {
+            return;
+        }
+        const exportRecord = await exportService.getExportRecord(exportDir);
+        if (exportService.checkAllElectronAPIsExists()) {
+            await migrateExport(exportDir, exportRecord);
+        }
+    } catch (e) {
+        logError(e, 'migrateExportJSON failed');
+    }
+}

 /*
    this function migrates the exportRecord file to apply any schema changes.
@ -40,13 +59,14 @@ import {
    later this will be converted to a loop which applies the migration one by one 
    till the files reaches the latest version 
    */
-export async function migrateExport(
+async function migrateExport(
    exportDir: string,
    exportRecord: ExportRecordV1 | ExportRecordV2 | ExportRecord
 ) {
    try {
-        if (!exportRecord) {
+        if (!exportRecord?.version) {
            exportRecord = {
+                ...exportRecord,
                version: 0,
            };
        }
@ -75,6 +95,7 @@ export async function migrateExport(
            });
            addLogLine('migration to version 3 complete');
        }
+        addLogLine(`Record at latest version`);
    } catch (e) {
        logError(e, 'export record migration failed');
    }
@ -120,7 +141,6 @@ async function migrationV2ToV3(
    const personalFiles = getIDBasedSortedFiles(
        getPersonalFiles(localFiles, user)
    );
-    addLogLine(`personal files count: ${personalFiles.length}`);
    // earlier the file were sorted by id,
    // which we can use to determine which file got which number suffix
    // this can be used to determine the filepaths of the those already exported files
@ -128,7 +148,6 @@ async function migrationV2ToV3(
    // This is based on the assumption new files have higher ids than the older ones
    await updateExportedFilesToExportedFilePathsProperty(
        exportRecord,
-
        getExportedFiles(personalFiles, exportRecord)
    );
    await extractExportDirPathPrefix(exportDir, exportRecord);
@ -155,16 +174,10 @@ export async function migrateCollectionFolders(
            collection.name
        );
        collectionIDPathMap.set(collection.id, newCollectionExportPath);
-        if (exportService.exists(oldCollectionExportPath)) {
-            await exportService.checkExistsAndRename(
-                oldCollectionExportPath,
-                newCollectionExportPath
-            );
-        } else {
-            await exportService.checkExistsAndCreateDir(
-                newCollectionExportPath
-            );
-        }
+        await exportService.checkExistsAndRename(
+            oldCollectionExportPath,
+            newCollectionExportPath
+        );
        await exportService.addCollectionExportedRecord(
            exportDir,
            collection.id,
@ -254,7 +267,9 @@ export async function updateExportedFilesToExportedFilePathsProperty(
    exportRecord: ExportRecordV2,
    exportedFiles: EnteFile[]
 ) {
-    addLocalLog(() => `${JSON.stringify(exportRecord)}}`);
+    if (!exportedFiles.length) {
+        return;
+    }
    addLogLine(
        'updating exported files to exported file paths property',
        `got ${exportedFiles.length} files`
@ -270,16 +285,37 @@ export async function updateExportedFilesToExportedFilePathsProperty(
            () =>
                `collection path for ${file.collectionID} is ${collectionPath}`
        );
-        const fileExportName = getUniqueFileExportNameForMigration(
-            collectionPath,
-            file.metadata.title,
-            usedFilePaths
-        );
+        let fileExportName: string;
+        if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
+            const fileStream = await retryAsyncFunction(() =>
+                downloadManager.downloadFile(file)
+            );
+            const fileBlob = await new Response(fileStream).blob();
+            const livePhoto = await decodeLivePhoto(file, fileBlob);
+            const imageExportName = getUniqueFileExportNameForMigration(
+                collectionPath,
+                livePhoto.imageNameTitle,
+                usedFilePaths
+            );
+            const videoExportName = getUniqueFileExportNameForMigration(
+                collectionPath,
+                livePhoto.videoNameTitle,
+                usedFilePaths
+            );
+            fileExportName = getLivePhotoExportName(
+                imageExportName,
+                videoExportName
+            );
+        } else {
+            fileExportName = getUniqueFileExportNameForMigration(
+                collectionPath,
+                file.metadata.title,
+                usedFilePaths
+            );
+        }
        addLocalLog(
            () =>
-                `file export path for ${
-                    file.metadata.title
-                } is ${getFileExportPath(collectionPath, fileExportName)}`
+                `file export name for ${file.metadata.title} is ${fileExportName}`
        );
        exportedFileNames = {
            ...exportedFileNames,
--- a/src/utils/export/index.ts
+++ b/src/utils/export/index.ts
@ -246,3 +246,28 @@ export const getMetadataFileExportPath = (filePath: string) => {
    const collectionExportPath = filePath.replace(`/${filename}`, '');
    return `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${filename}.json`;
 };
+
+export const getLivePhotoExportName = (
+    imageExportName: string,
+    videoExportName: string
+) =>
+    JSON.stringify({
+        image: imageExportName,
+        video: videoExportName,
+    });
+
+export const isLivePhotoExportName = (exportName: string) => {
+    try {
+        JSON.parse(exportName);
+        return true;
+    } catch (e) {
+        return false;
+    }
+};
+
+export const parseLivePhotoExportName = (
+    livePhotoExportName: string
+): { image: string; video: string } => {
+    const { image, video } = JSON.parse(livePhotoExportName);
+    return { image, video };
+};