fix migration issues
This commit is contained in:
parent
43043df2c8
commit
e5d28c24c7
|
@ -76,7 +76,7 @@ export default function ExportModal(props: Props) {
|
|||
}
|
||||
const main = async () => {
|
||||
try {
|
||||
exportService.setUIUpdaters({
|
||||
await exportService.init({
|
||||
updateExportStage: updateExportStage,
|
||||
updateExportProgress: setExportProgress,
|
||||
updateFileExportStats: setFileExportStats,
|
||||
|
|
|
@ -17,6 +17,9 @@ import {
|
|||
getCollectionExportedFiles,
|
||||
getCollectionExportPath,
|
||||
getMetadataFolderExportPath,
|
||||
getLivePhotoExportName,
|
||||
isLivePhotoExportName,
|
||||
parseLivePhotoExportName,
|
||||
} from 'utils/export';
|
||||
import { retryAsyncFunction } from 'utils/network';
|
||||
import { logError } from 'utils/sentry';
|
||||
|
@ -53,7 +56,7 @@ import {
|
|||
getCollectionNameMap,
|
||||
getNonEmptyPersonalCollections,
|
||||
} from 'utils/collection';
|
||||
import { migrateExport } from './migration';
|
||||
import { migrateExportJSONHelper } from './migration';
|
||||
|
||||
const EXPORT_RECORD_FILE_NAME = 'export_status.json';
|
||||
|
||||
|
@ -74,29 +77,26 @@ class ExportService {
|
|||
success: 0,
|
||||
failed: 0,
|
||||
};
|
||||
private exportJSONMigrator = new QueueProcessor<void>(1);
|
||||
|
||||
constructor() {
|
||||
if (runningInBrowser()) {
|
||||
this.electronAPIs = window['ElectronAPIs'];
|
||||
this.allElectronAPIsExist = !!this.electronAPIs?.exists;
|
||||
this.fileReader = new FileReader();
|
||||
void this.migrateExportJSON();
|
||||
}
|
||||
}
|
||||
|
||||
async migrateExportJSON() {
|
||||
try {
|
||||
const exportDir = getData(LS_KEYS.EXPORT)?.folder;
|
||||
if (!exportDir) {
|
||||
return;
|
||||
}
|
||||
const exportRecord = await this.getExportRecord(exportDir);
|
||||
if (this.checkAllElectronAPIsExists()) {
|
||||
await migrateExport(exportDir, exportRecord);
|
||||
}
|
||||
} catch (e) {
|
||||
logError(e, 'migrateExportJSON failed');
|
||||
const response = this.exportJSONMigrator.queueUpRequest(() =>
|
||||
migrateExportJSONHelper()
|
||||
);
|
||||
return response.promise;
|
||||
}
|
||||
|
||||
async init(uiUpdater: ExportUIUpdaters) {
|
||||
this.setUIUpdaters(uiUpdater);
|
||||
await this.migrateExportJSON();
|
||||
}
|
||||
|
||||
async setUIUpdaters(uiUpdater: ExportUIUpdaters) {
|
||||
|
@ -245,6 +245,8 @@ class ExportService {
|
|||
}
|
||||
try {
|
||||
await this.preExport();
|
||||
// checking if migration is needed
|
||||
await this.migrateExportJSON();
|
||||
addLogLine('export started');
|
||||
await this.runExport();
|
||||
addLogLine('export completed');
|
||||
|
@ -618,15 +620,11 @@ class ExportService {
|
|||
);
|
||||
// check if filepath is for live photo
|
||||
// livePhoto has the path in format: `JSON.stringify({image,video})`
|
||||
const isLivePhoto =
|
||||
fileExportName.startsWith('{') &&
|
||||
fileExportName.endsWith('}');
|
||||
|
||||
if (isLivePhoto) {
|
||||
if (isLivePhotoExportName(fileExportName)) {
|
||||
const {
|
||||
image: imageExportName,
|
||||
video: videoExportName,
|
||||
} = JSON.parse(fileExportName);
|
||||
} = parseLivePhotoExportName(fileExportName);
|
||||
const imageExportPath = getFileExportPath(
|
||||
collectionExportPath,
|
||||
imageExportName
|
||||
|
@ -672,7 +670,6 @@ class ExportService {
|
|||
videoMetadataFileExportPath
|
||||
)
|
||||
);
|
||||
await this.removeFileExportedRecord(exportDir, fileUID);
|
||||
} else {
|
||||
const fileExportPath = getFileExportPath(
|
||||
collectionExportPath,
|
||||
|
@ -691,8 +688,8 @@ class ExportService {
|
|||
metadataFileExportPath
|
||||
)
|
||||
);
|
||||
await this.removeFileExportedRecord(exportDir, fileUID);
|
||||
}
|
||||
await this.removeFileExportedRecord(exportDir, fileUID);
|
||||
incrementSuccess();
|
||||
} catch (e) {
|
||||
incrementFailed();
|
||||
|
@ -991,10 +988,7 @@ class ExportService {
|
|||
videoExportName
|
||||
);
|
||||
|
||||
return JSON.stringify({
|
||||
image: imageExportPath,
|
||||
video: videoExportPath,
|
||||
});
|
||||
return getLivePhotoExportName(imageExportPath, videoExportPath);
|
||||
}
|
||||
|
||||
private async saveMediaFile(
|
||||
|
|
|
@ -9,7 +9,7 @@ import {
|
|||
import { EnteFile } from 'types/file';
|
||||
import { User } from 'types/user';
|
||||
import { getNonEmptyPersonalCollections } from 'utils/collection';
|
||||
import { getExportRecordFileUID, getFileExportPath } from 'utils/export';
|
||||
import { getExportRecordFileUID, getLivePhotoExportName } from 'utils/export';
|
||||
import {
|
||||
getIDBasedSortedFiles,
|
||||
getPersonalFiles,
|
||||
|
@ -32,6 +32,25 @@ import {
|
|||
getFileMetadataSavePath,
|
||||
getFileSavePath,
|
||||
} from 'utils/export/migration';
|
||||
import { FILE_TYPE } from 'constants/file';
|
||||
import { decodeLivePhoto } from 'services/livePhotoService';
|
||||
import downloadManager from 'services/downloadManager';
|
||||
import { retryAsyncFunction } from 'utils/network';
|
||||
|
||||
export async function migrateExportJSONHelper() {
|
||||
try {
|
||||
const exportDir = getData(LS_KEYS.EXPORT)?.folder;
|
||||
if (!exportDir) {
|
||||
return;
|
||||
}
|
||||
const exportRecord = await exportService.getExportRecord(exportDir);
|
||||
if (exportService.checkAllElectronAPIsExists()) {
|
||||
await migrateExport(exportDir, exportRecord);
|
||||
}
|
||||
} catch (e) {
|
||||
logError(e, 'migrateExportJSON failed');
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
this function migrates the exportRecord file to apply any schema changes.
|
||||
|
@ -40,13 +59,14 @@ import {
|
|||
later this will be converted to a loop which applies the migration one by one
|
||||
till the files reaches the latest version
|
||||
*/
|
||||
export async function migrateExport(
|
||||
async function migrateExport(
|
||||
exportDir: string,
|
||||
exportRecord: ExportRecordV1 | ExportRecordV2 | ExportRecord
|
||||
) {
|
||||
try {
|
||||
if (!exportRecord) {
|
||||
if (!exportRecord?.version) {
|
||||
exportRecord = {
|
||||
...exportRecord,
|
||||
version: 0,
|
||||
};
|
||||
}
|
||||
|
@ -75,6 +95,7 @@ export async function migrateExport(
|
|||
});
|
||||
addLogLine('migration to version 3 complete');
|
||||
}
|
||||
addLogLine(`Record at latest version`);
|
||||
} catch (e) {
|
||||
logError(e, 'export record migration failed');
|
||||
}
|
||||
|
@ -120,7 +141,6 @@ async function migrationV2ToV3(
|
|||
const personalFiles = getIDBasedSortedFiles(
|
||||
getPersonalFiles(localFiles, user)
|
||||
);
|
||||
addLogLine(`personal files count: ${personalFiles.length}`);
|
||||
// earlier the file were sorted by id,
|
||||
// which we can use to determine which file got which number suffix
|
||||
// this can be used to determine the filepaths of the those already exported files
|
||||
|
@ -128,7 +148,6 @@ async function migrationV2ToV3(
|
|||
// This is based on the assumption new files have higher ids than the older ones
|
||||
await updateExportedFilesToExportedFilePathsProperty(
|
||||
exportRecord,
|
||||
|
||||
getExportedFiles(personalFiles, exportRecord)
|
||||
);
|
||||
await extractExportDirPathPrefix(exportDir, exportRecord);
|
||||
|
@ -155,16 +174,10 @@ export async function migrateCollectionFolders(
|
|||
collection.name
|
||||
);
|
||||
collectionIDPathMap.set(collection.id, newCollectionExportPath);
|
||||
if (exportService.exists(oldCollectionExportPath)) {
|
||||
await exportService.checkExistsAndRename(
|
||||
oldCollectionExportPath,
|
||||
newCollectionExportPath
|
||||
);
|
||||
} else {
|
||||
await exportService.checkExistsAndCreateDir(
|
||||
newCollectionExportPath
|
||||
);
|
||||
}
|
||||
await exportService.addCollectionExportedRecord(
|
||||
exportDir,
|
||||
collection.id,
|
||||
|
@ -254,7 +267,9 @@ export async function updateExportedFilesToExportedFilePathsProperty(
|
|||
exportRecord: ExportRecordV2,
|
||||
exportedFiles: EnteFile[]
|
||||
) {
|
||||
addLocalLog(() => `${JSON.stringify(exportRecord)}}`);
|
||||
if (!exportedFiles.length) {
|
||||
return;
|
||||
}
|
||||
addLogLine(
|
||||
'updating exported files to exported file paths property',
|
||||
`got ${exportedFiles.length} files`
|
||||
|
@ -270,16 +285,37 @@ export async function updateExportedFilesToExportedFilePathsProperty(
|
|||
() =>
|
||||
`collection path for ${file.collectionID} is ${collectionPath}`
|
||||
);
|
||||
const fileExportName = getUniqueFileExportNameForMigration(
|
||||
let fileExportName: string;
|
||||
if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
|
||||
const fileStream = await retryAsyncFunction(() =>
|
||||
downloadManager.downloadFile(file)
|
||||
);
|
||||
const fileBlob = await new Response(fileStream).blob();
|
||||
const livePhoto = await decodeLivePhoto(file, fileBlob);
|
||||
const imageExportName = getUniqueFileExportNameForMigration(
|
||||
collectionPath,
|
||||
livePhoto.imageNameTitle,
|
||||
usedFilePaths
|
||||
);
|
||||
const videoExportName = getUniqueFileExportNameForMigration(
|
||||
collectionPath,
|
||||
livePhoto.videoNameTitle,
|
||||
usedFilePaths
|
||||
);
|
||||
fileExportName = getLivePhotoExportName(
|
||||
imageExportName,
|
||||
videoExportName
|
||||
);
|
||||
} else {
|
||||
fileExportName = getUniqueFileExportNameForMigration(
|
||||
collectionPath,
|
||||
file.metadata.title,
|
||||
usedFilePaths
|
||||
);
|
||||
}
|
||||
addLocalLog(
|
||||
() =>
|
||||
`file export path for ${
|
||||
file.metadata.title
|
||||
} is ${getFileExportPath(collectionPath, fileExportName)}`
|
||||
`file export name for ${file.metadata.title} is ${fileExportName}`
|
||||
);
|
||||
exportedFileNames = {
|
||||
...exportedFileNames,
|
||||
|
|
|
@ -246,3 +246,28 @@ export const getMetadataFileExportPath = (filePath: string) => {
|
|||
const collectionExportPath = filePath.replace(`/${filename}`, '');
|
||||
return `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${filename}.json`;
|
||||
};
|
||||
|
||||
export const getLivePhotoExportName = (
|
||||
imageExportName: string,
|
||||
videoExportName: string
|
||||
) =>
|
||||
JSON.stringify({
|
||||
image: imageExportName,
|
||||
video: videoExportName,
|
||||
});
|
||||
|
||||
export const isLivePhotoExportName = (exportName: string) => {
|
||||
try {
|
||||
JSON.parse(exportName);
|
||||
return true;
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
export const parseLivePhotoExportName = (
|
||||
livePhotoExportName: string
|
||||
): { image: string; video: string } => {
|
||||
const { image, video } = JSON.parse(livePhotoExportName);
|
||||
return { image, video };
|
||||
};
|
||||
|
|
Loading…
Reference in a new issue