beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refactor stuff to migration

Browse source
This commit is contained in:
Abhinav 2023-04-27 16:39:38 +05:30
parent 7749de84fc
commit 2264be4370
4 changed files with 265 additions and 209 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

132
src/services/export/index.ts
View file

@ -7,7 +7,6 @@ import {
getUniqueFileExportName,
getFileMetadataExportPath,
getFileExportPath,
getMetadataFolderPath,
getRenamedCollections,
getDeletedExportedFiles,
convertCollectionIDExportNameObjectToMap,
@ -17,9 +16,7 @@ import {
getMetadataFileExportPath,
getCollectionExportedFiles,
getCollectionExportPath,
getOldCollectionExportPath,
getOldFileExportPath,
getOldFileMetadataExportPath,
getMetadataFolderExportPath,
} from 'utils/export';
import { retryAsyncFunction } from 'utils/network';
import { logError } from 'utils/sentry';
@ -34,7 +31,6 @@ import {
generateStreamFromArrayBuffer,
getFileExtension,
getPersonalFiles,
mergeMetadata,
} from 'utils/file';
import { updateFileCreationDateInEXIF } from '../upload/exifService';
@ -421,7 +417,6 @@ class ExportService {
const newCollectionExportName =
getUniqueCollectionExportName(
exportFolder,
collection.id,
collection.name
);
const newCollectionExportPath = getCollectionExportPath(
@ -854,7 +849,6 @@ class ExportService {
const collectionName = collectionIDNameMap.get(collectionID);
const collectionExportName = getUniqueCollectionExportName(
exportFolder,
collectionID,
collectionName
);
const collectionExportPath = getCollectionExportPath(
@ -863,37 +857,36 @@ class ExportService {
);
await this.electronAPIs.checkExistsAndCreateDir(collectionExportPath);
await this.electronAPIs.checkExistsAndCreateDir(
getMetadataFolderPath(collectionExportPath)
getMetadataFolderExportPath(collectionExportPath)
);
return collectionExportName;
}
async renamecollectionExports(
async renameCollectionExports(
renamedCollections: Collection[],
exportFolder: string,
collectionIDPathMap: Map<number, string>
) {
for (const collection of renamedCollections) {
const oldcollectionExportPath = collectionIDPathMap.get(
const oldCollectionExportPath = collectionIDPathMap.get(
collection.id
);
const newcollectionExportPath = getUniqueCollectionExportName(
const newCollectionExportPath = getUniqueCollectionExportName(
exportFolder,
collection.id,
collection.name
);
await this.electronAPIs.checkExistsAndRename(
oldcollectionExportPath,
newcollectionExportPath
oldCollectionExportPath,
newCollectionExportPath
);
await this.addCollectionExportedRecord(
exportFolder,
collection.id,
newcollectionExportPath
newCollectionExportPath
);
collectionIDPathMap.set(collection.id, newcollectionExportPath);
collectionIDPathMap.set(collection.id, newCollectionExportPath);
}
}
@ -904,8 +897,7 @@ class ExportService {
try {
const fileExportName = getUniqueFileExportName(
collectionExportPath,
file.metadata.title,
file.id
file.metadata.title
);
let fileStream = await retryAsyncFunction(() =>
downloadManager.downloadFile(file)
@ -965,8 +957,7 @@ class ExportService {
const imageStream = generateStreamFromArrayBuffer(motionPhoto.image);
const imageExportName = getUniqueFileExportName(
collectionExportPath,
motionPhoto.imageNameTitle,
file.id
motionPhoto.imageNameTitle
);
await this.saveMediaFile(
collectionExportPath,
@ -982,8 +973,7 @@ class ExportService {
const videoStream = generateStreamFromArrayBuffer(motionPhoto.video);
const videoExportName = getUniqueFileExportName(
collectionExportPath,
motionPhoto.videoNameTitle,
file.id
motionPhoto.videoNameTitle
);
await this.saveMediaFile(
collectionExportPath,
@ -1032,97 +1022,6 @@ class ExportService {
);
}
/*
This updates the folder name of already exported folders from the earlier format of
`collectionID_collectionName` to newer `collectionName(numbered)` format
*/
async migrateCollectionExports(
collections: Collection[],
exportDir: string,
collectionIDPathMap: Map<number, string>
) {
for (const collection of collections) {
const oldCollectionExportPath = getOldCollectionExportPath(
exportDir,
collection.id,
collection.name
);
const newCollectionExportPath = getCollectionExportPath(
exportDir,
getUniqueCollectionExportName(
exportDir,
collection.id,
collection.name
)
);
collectionIDPathMap.set(collection.id, newCollectionExportPath);
if (this.exists(oldCollectionExportPath)) {
await this.electronAPIs.checkExistsAndRename(
oldCollectionExportPath,
newCollectionExportPath
);
} else {
await this.electronAPIs.checkExistsAndCreateDir(
newCollectionExportPath
);
}
await this.addCollectionExportedRecord(
exportDir,
collection.id,
newCollectionExportPath
);
}
}
/*
This updates the file name of already exported files from the earlier format of
`fileID_fileName` to newer `fileName(numbered)` format
*/
async migrateFiles(
exportDir: string,
files: EnteFile[],
collectionIDExportNameMap: Map<number, string>
) {
for (let file of files) {
const collectionExportPath = getCollectionExportPath(
exportDir,
collectionIDExportNameMap.get(file.collectionID)
);
const oldFileExportPath = getOldFileExportPath(
collectionExportPath,
file
);
const oldFileMetadataExportPath = getOldFileMetadataExportPath(
collectionExportPath,
file
);
file = mergeMetadata([file])[0];
const newFileExportName = getUniqueFileExportName(
collectionExportPath,
file.metadata.title,
file.id
);
const newFileExportPath = getFileExportPath(
collectionExportPath,
newFileExportName
);
const newFileMetadataExportPath = getFileMetadataExportPath(
collectionExportPath,
newFileExportName
);
await this.electronAPIs.checkExistsAndRename(
oldFileExportPath,
newFileExportPath
);
await this.electronAPIs.checkExistsAndRename(
oldFileMetadataExportPath,
newFileMetadataExportPath
);
}
}
isExportInProgress = () => {
return this.exportInProgress;
};
@ -1130,6 +1029,13 @@ class ExportService {
exists = (path: string) => {
return this.electronAPIs.exists(path);
};
checkExistsAndRename = (oldPath: string, newPath: string) => {
return this.electronAPIs.checkExistsAndRename(oldPath, newPath);
};
checkExistsAndCreateDir = (path: string) => {
return this.electronAPIs.checkExistsAndCreateDir(path);
};
checkAllElectronAPIsExists = () => this.allElectronAPIsExist;
}

106
src/services/export/migration.ts
View file

@ -14,13 +14,25 @@ import {
getUniqueFileExportNameForMigration,
getExportRecordFileUID,
getFileExportPath,
getCollectionExportPath,
getFileMetadataExportPath,
getOldFileExportPath,
getOldFileMetadataExportPath,
getOldCollectionFolderPath,
convertCollectionIDFolderPathObjectToMap,
getUniqueCollectionFolderPath,
getUniqueFileSaveName,
} from 'utils/export';
import { getIDBasedSortedFiles, getPersonalFiles } from 'utils/file';
import {
getIDBasedSortedFiles,
getPersonalFiles,
mergeMetadata,
} from 'utils/file';
import { addLocalLog, addLogLine } from 'utils/logging';
import { logError } from 'utils/sentry';
import { getData, LS_KEYS } from 'utils/storage/localStorage';
import exportService from './index';
import { Collection } from 'types/collection';
type UpdatedExportRecord = (
exportRecord: Partial<ExportRecord>
@ -90,12 +102,12 @@ async function migrationV0ToV1(
personalFiles,
user
);
await exportService.migrateCollectionExports(
await migrateCollectionExports(
nonEmptyPersonalCollections,
exportDir,
collectionIDPathMap
);
await exportService.migrateFiles(
await migrateFiles(
exportDir,
getExportedFiles(personalFiles, exportRecord),
collectionIDPathMap
@ -221,3 +233,91 @@ export async function updateExportedFilesToExportedFilePathsProperty(
await updateExportRecord(updatedExportRecord);
}
/*
This updates the folder name of already exported folders from the earlier format of
`collectionID_collectionName` to newer `collectionName(numbered)` format
*/
export async function migrateCollectionExports(
collections: Collection[],
exportDir: string,
collectionIDPathMap: Map<number, string>
) {
for (const collection of collections) {
const oldCollectionExportPath = getOldCollectionFolderPath(
exportDir,
collection.id,
collection.name
);
const newCollectionExportPath = getUniqueCollectionFolderPath(
exportDir,
collection.id,
collection.name
);
collectionIDPathMap.set(collection.id, newCollectionExportPath);
if (exportService.exists(oldCollectionExportPath)) {
await exportService.checkExistsAndRename(
oldCollectionExportPath,
newCollectionExportPath
);
} else {
await exportService.checkExistsAndCreateDir(
newCollectionExportPath
);
}
await exportService.addCollectionExportedRecord(
exportDir,
collection.id,
newCollectionExportPath
);
}
}
/*
This updates the file name of already exported files from the earlier format of
`fileID_fileName` to newer `fileName(numbered)` format
*/
export async function migrateFiles(
exportDir: string,
files: EnteFile[],
collectionIDExportNameMap: Map<number, string>
) {
for (let file of files) {
const collectionExportPath = getCollectionExportPath(
exportDir,
collectionIDExportNameMap.get(file.collectionID)
);
const oldFileExportPath = getOldFileExportPath(
collectionExportPath,
file
);
const oldFileMetadataExportPath = getOldFileMetadataExportPath(
collectionExportPath,
file
);
file = mergeMetadata([file])[0];
const newFileExportName = getUniqueFileSaveName(
collectionExportPath,
file.metadata.title,
file.id
);
const newFileExportPath = getFileExportPath(
collectionExportPath,
newFileExportName
);
const newFileMetadataExportPath = getFileMetadataExportPath(
collectionExportPath,
newFileExportName
);
await exportService.checkExistsAndRename(
oldFileExportPath,
newFileExportPath
);
await exportService.checkExistsAndRename(
oldFileMetadataExportPath,
newFileMetadataExportPath
);
}
}

114
src/utils/export/index.ts
View file

@ -2,11 +2,9 @@ import { Collection } from 'types/collection';
import exportService from 'services/export';
import {
ExportRecord,
ExportRecordV1,
ExportRecordV2,
CollectionExportNames,
FileExportNames,
ExportedCollectionPaths,
} from 'types/export';
import { EnteFile } from 'types/file';
@ -20,31 +18,21 @@ import { formatDateTimeShort } from 'utils/time/format';
export const getExportRecordFileUID = (file: EnteFile) =>
`${file.id}_${file.collectionID}_${file.updationTime}`;
export const convertCollectionIDFolderPathObjectToMap = (
exportedCollectionPaths: ExportedCollectionPaths
): Map<number, string> => {
return new Map<number, string>(
Object.entries(exportedCollectionPaths ?? {}).map((e) => {
return [Number(e[0]), String(e[1])];
})
);
};
export const convertCollectionIDExportNameObjectToMap = (
exportedCollectionNames: CollectionExportNames
collectionExportNames: CollectionExportNames
): Map<number, string> => {
return new Map<number, string>(
Object.entries(exportedCollectionNames ?? {}).map((e) => {
Object.entries(collectionExportNames ?? {}).map((e) => {
return [Number(e[0]), String(e[1])];
})
);
};
export const convertFileIDExportNameObjectToMap = (
exportedFileNames: FileExportNames
fileExportNames: FileExportNames
): Map<string, string> => {
return new Map<string, string>(
Object.entries(exportedFileNames ?? {}).map((e) => {
Object.entries(fileExportNames ?? {}).map((e) => {
return [String(e[0]), String(e[1])];
})
);
@ -57,14 +45,16 @@ export const getRenamedCollections = (
if (!exportRecord?.collectionExportNames) {
return [];
}
const collectionIDFolderMap = convertCollectionIDExportNameObjectToMap(
const collectionIDExportNameMap = convertCollectionIDExportNameObjectToMap(
exportRecord.collectionExportNames
);
const renamedCollections = collections.filter((collection) => {
if (collectionIDFolderMap.has(collection.id)) {
const currentFolderName = collectionIDFolderMap.get(collection.id);
if (collectionIDExportNameMap.has(collection.id)) {
const currentExportName = collectionIDExportNameMap.get(
collection.id
);
if (currentFolderName !== sanitizeName(collection.name)) {
if (!currentExportName.startsWith(sanitizeName(collection.name))) {
return true;
}
}
@ -113,42 +103,6 @@ export const getUnExportedFiles = (
return unExportedFiles;
};
export const getExportedFiles = (
allFiles: EnteFile[],
exportRecord: ExportRecordV1 | ExportRecordV2
) => {
if (!exportRecord?.exportedFiles) {
return [];
}
const exportedFileIds = new Set(exportRecord?.exportedFiles);
const exportedFiles = allFiles.filter((file) => {
if (exportedFileIds.has(getExportRecordFileUID(file))) {
return true;
} else {
return false;
}
});
return exportedFiles;
};
export const getExportedFilePaths = (
allFiles: EnteFile[],
exportRecord: ExportRecord
) => {
if (!exportRecord?.fileExportNames) {
return [];
}
const exportedFileIds = new Set(Object.keys(exportRecord?.fileExportNames));
const exportedFiles = allFiles.filter((file) => {
if (exportedFileIds.has(getExportRecordFileUID(file))) {
return true;
}
return false;
});
return exportedFiles;
};
export const getDeletedExportedFiles = (
allFiles: EnteFile[],
exportRecord: ExportRecord
@ -222,20 +176,23 @@ export const getGoogleLikeMetadataFile = (
);
};
export const oldSanitizeName = (name: string) =>
name.replaceAll('/', '_').replaceAll(' ', '_');
export const getCollectionIDPathMapFromExportRecord = (
exportRecord: ExportRecordV2
): Map<number, string> => {
return new Map<number, string>(
Object.entries(exportRecord.exportedCollectionPaths ?? {}).map((e) => {
return [Number(e[0]), String(e[1])];
})
);
};
export const sanitizeName = (name: string) =>
sanitize(name, { replacement: '_' });
export const getUniqueCollectionExportName = (
dir: string,
collectionID: number,
collectionName: string
): string => {
if (!exportService.checkAllElectronAPIsExists()) {
return getOldCollectionExportPath(dir, collectionID, collectionName);
}
let collectionExportName = sanitizeName(collectionName);
let count = 1;
while (
@ -247,17 +204,13 @@ export const getUniqueCollectionExportName = (
return collectionExportName;
};
export const getMetadataFolderPath = (collectionExportPath: string) =>
export const getMetadataFolderExportPath = (collectionExportPath: string) =>
`${collectionExportPath}/${ENTE_METADATA_FOLDER}`;
export const getUniqueFileExportName = (
collectionExportPath: string,
filename: string,
fileID: number
filename: string
) => {
if (!exportService.checkAllElectronAPIsExists()) {
return getOldFileExportName(filename, fileID);
}
let fileExportName = sanitizeName(filename);
let count = 1;
while (
@ -276,9 +229,6 @@ export const getUniqueFileExportName = (
return fileExportName;
};
export const getOldFileExportName = (filename: string, fileID: number) =>
`${fileID}_${oldSanitizeName(filename)}`;
export const getFileMetadataExportPath = (
collectionExportPath: string,
fileExportName: string
@ -294,28 +244,6 @@ export const getFileExportPath = (
fileExportName: string
) => `${collectionExportPath}/${fileExportName}`;
export const getOldCollectionExportPath = (
dir: string,
collectionID: number,
collectionName: string
) => `${dir}/${collectionID}_${oldSanitizeName(collectionName)}`;
export const getOldFileExportPath = (
collectionExportPath: string,
file: EnteFile
) =>
`${collectionExportPath}/${file.id}_${oldSanitizeName(
file.metadata.title
)}`;
export const getOldFileMetadataExportPath = (
collectionExportPath: string,
file: EnteFile
) =>
`${collectionExportPath}/${ENTE_METADATA_FOLDER}/${
file.id
}_${oldSanitizeName(file.metadata.title)}.json`;
export const getUniqueFileExportNameForMigration = (
collectionPath: string,
filename: string,

122
src/utils/export/migration.ts Normal file
View file

@ -0,0 +1,122 @@
import { ENTE_METADATA_FOLDER } from 'constants/export';
import {
ExportedCollectionPaths,
ExportRecordV1,
ExportRecordV2,
} from 'types/export';
import { EnteFile } from 'types/file';
import { splitFilenameAndExtension } from 'utils/ffmpeg';
import { getExportRecordFileUID, sanitizeName } from '.';
import exportService from 'services/export';
export const convertCollectionIDFolderPathObjectToMap = (
exportedCollectionPaths: ExportedCollectionPaths
): Map<number, string> => {
return new Map<number, string>(
Object.entries(exportedCollectionPaths ?? {}).map((e) => {
return [Number(e[0]), String(e[1])];
})
);
};
export const getExportedFiles = (
allFiles: EnteFile[],
exportRecord: ExportRecordV1 | ExportRecordV2
) => {
if (!exportRecord?.exportedFiles) {
return [];
}
const exportedFileIds = new Set(exportRecord?.exportedFiles);
const exportedFiles = allFiles.filter((file) => {
if (exportedFileIds.has(getExportRecordFileUID(file))) {
return true;
} else {
return false;
}
});
return exportedFiles;
};
export const oldSanitizeName = (name: string) =>
name.replaceAll('/', '_').replaceAll(' ', '_');
export const getUniqueCollectionFolderPath = (
dir: string,
collectionID: number,
collectionName: string
): string => {
if (!exportService.checkAllElectronAPIsExists()) {
return getOldCollectionFolderPath(dir, collectionID, collectionName);
}
let collectionFolderPath = `${dir}/${sanitizeName(collectionName)}`;
let count = 1;
while (exportService.exists(collectionFolderPath)) {
collectionFolderPath = `${dir}/${sanitizeName(
collectionName
)}(${count})`;
count++;
}
return collectionFolderPath;
};
export const getMetadataFolderPath = (collectionFolderPath: string) =>
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}`;
export const getUniqueFileSaveName = (
collectionPath: string,
filename: string,
fileID: number
) => {
if (!exportService.checkAllElectronAPIsExists()) {
return getOldFileSaveName(filename, fileID);
}
let fileSaveName = sanitizeName(filename);
let count = 1;
while (
exportService.exists(getFileSavePath(collectionPath, fileSaveName))
) {
const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
if (filenameParts[1]) {
fileSaveName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
} else {
fileSaveName = `${filenameParts[0]}(${count})`;
}
count++;
}
return fileSaveName;
};
export const getOldFileSaveName = (filename: string, fileID: number) =>
`${fileID}_${oldSanitizeName(filename)}`;
export const getFileMetadataSavePath = (
collectionFolderPath: string,
fileSaveName: string
) => `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${fileSaveName}.json`;
export const getFileSavePath = (
collectionFolderPath: string,
fileSaveName: string
) => `${collectionFolderPath}/${fileSaveName}`;
export const getOldCollectionFolderPath = (
dir: string,
collectionID: number,
collectionName: string
) => `${dir}/${collectionID}_${oldSanitizeName(collectionName)}`;
export const getOldFileSavePath = (
collectionFolderPath: string,
file: EnteFile
) =>
`${collectionFolderPath}/${file.id}_${oldSanitizeName(
file.metadata.title
)}`;
export const getOldFileMetadataSavePath = (
collectionFolderPath: string,
file: EnteFile
) =>
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${
file.id
}_${oldSanitizeName(file.metadata.title)}.json`;