beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Inline

Browse source
This commit is contained in:
Manav Rathi 2024-04-13 16:44:55 +05:30
parent e72003c199
commit f7e961e86b
No known key found for this signature in database
4 changed files with 136 additions and 168 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
web/apps/photos/src/constants/export.ts
View file

@ -1,6 +1,5 @@
export const ENTE_METADATA_FOLDER = "metadata";
export const ENTE_TRASH_FOLDER = "Trash";
export enum ExportStage {
INIT = 0,

143
web/apps/photos/src/services/export/migration.ts
View file

@ -2,6 +2,7 @@ import log from "@/next/log";
import { LS_KEYS, getData } from "@ente/shared/storage/localStorage";
import { User } from "@ente/shared/user/types";
import { sleep } from "@ente/shared/utils";
import { ENTE_METADATA_FOLDER } from "constants/export";
import { FILE_TYPE } from "constants/file";
import { getLocalCollections } from "services/collectionService";
import downloadManager from "services/download";
@ -15,6 +16,7 @@ import {
ExportRecordV0,
ExportRecordV1,
ExportRecordV2,
ExportedCollectionPaths,
FileExportNames,
} from "types/export";
import { EnteFile } from "types/file";
@ -25,19 +27,9 @@ import {
getExportRecordFileUID,
getLivePhotoExportName,
getMetadataFolderExportPath,
sanitizeName,
} from "utils/export";
import {
convertCollectionIDFolderPathObjectToMap,
getExportedFiles,
getFileMetadataSavePath,
getFileSavePath,
getOldCollectionFolderPath,
getOldFileMetadataSavePath,
getOldFileSavePath,
getUniqueCollectionFolderPath,
getUniqueFileExportNameForMigration,
getUniqueFileSaveName,
} from "utils/export/migration";
import { splitFilenameAndExtension } from "utils/ffmpeg";
import {
getIDBasedSortedFiles,
getPersonalFiles,
@ -475,3 +467,130 @@ async function removeCollectionExportMissingMetadataFolder(
};
await exportService.updateExportRecord(exportDir, updatedExportRecord);
}
const convertCollectionIDFolderPathObjectToMap = (
exportedCollectionPaths: ExportedCollectionPaths,
): Map<number, string> => {
return new Map<number, string>(
Object.entries(exportedCollectionPaths ?? {}).map((e) => {
return [Number(e[0]), String(e[1])];
}),
);
};
const getExportedFiles = (
allFiles: EnteFile[],
exportRecord: ExportRecordV0 | ExportRecordV1 | ExportRecordV2,
) => {
if (!exportRecord?.exportedFiles) {
return [];
}
const exportedFileIds = new Set(exportRecord?.exportedFiles);
const exportedFiles = allFiles.filter((file) => {
if (exportedFileIds.has(getExportRecordFileUID(file))) {
return true;
} else {
return false;
}
});
return exportedFiles;
};
const oldSanitizeName = (name: string) =>
name.replaceAll("/", "_").replaceAll(" ", "_");
const getUniqueCollectionFolderPath = async (
dir: string,
collectionName: string,
): Promise<string> => {
let collectionFolderPath = `${dir}/${sanitizeName(collectionName)}`;
let count = 1;
while (await exportService.exists(collectionFolderPath)) {
collectionFolderPath = `${dir}/${sanitizeName(
collectionName,
)}(${count})`;
count++;
}
return collectionFolderPath;
};
export const getMetadataFolderPath = (collectionFolderPath: string) =>
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}`;
const getUniqueFileSaveName = async (
collectionPath: string,
filename: string,
) => {
let fileSaveName = sanitizeName(filename);
let count = 1;
while (
await exportService.exists(
getFileSavePath(collectionPath, fileSaveName),
)
) {
const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
if (filenameParts[1]) {
fileSaveName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
} else {
fileSaveName = `${filenameParts[0]}(${count})`;
}
count++;
}
return fileSaveName;
};
const getFileMetadataSavePath = (
collectionFolderPath: string,
fileSaveName: string,
) => `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${fileSaveName}.json`;
const getFileSavePath = (collectionFolderPath: string, fileSaveName: string) =>
`${collectionFolderPath}/${fileSaveName}`;
const getOldCollectionFolderPath = (
dir: string,
collectionID: number,
collectionName: string,
) => `${dir}/${collectionID}_${oldSanitizeName(collectionName)}`;
const getOldFileSavePath = (collectionFolderPath: string, file: EnteFile) =>
`${collectionFolderPath}/${file.id}_${oldSanitizeName(
file.metadata.title,
)}`;
const getOldFileMetadataSavePath = (
collectionFolderPath: string,
file: EnteFile,
) =>
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${
file.id
}_${oldSanitizeName(file.metadata.title)}.json`;
const getUniqueFileExportNameForMigration = (
collectionPath: string,
filename: string,
usedFilePaths: Map<string, Set<string>>,
) => {
let fileExportName = sanitizeName(filename);
let count = 1;
while (
usedFilePaths
.get(collectionPath)
?.has(getFileSavePath(collectionPath, fileExportName))
) {
const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
if (filenameParts[1]) {
fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
} else {
fileExportName = `${filenameParts[0]}(${count})`;
}
count++;
}
if (!usedFilePaths.has(collectionPath)) {
usedFilePaths.set(collectionPath, new Set());
}
usedFilePaths
.get(collectionPath)
.add(getFileSavePath(collectionPath, fileExportName));
return fileExportName;
};

14
web/apps/photos/src/utils/export/index.ts
View file

@ -1,3 +1,6 @@
import { formatDateTimeShort } from "@ente/shared/time/format";
import { ENTE_METADATA_FOLDER, ExportStage } from "constants/export";
import sanitize from "sanitize-filename";
import exportService from "services/export";
import { Collection } from "types/collection";
import {
@ -5,20 +8,13 @@ import {
ExportRecord,
FileExportNames,
} from "types/export";
import { EnteFile } from "types/file";
import { formatDateTimeShort } from "@ente/shared/time/format";
import {
ENTE_METADATA_FOLDER,
ENTE_TRASH_FOLDER,
ExportStage,
} from "constants/export";
import sanitize from "sanitize-filename";
import { Metadata } from "types/upload";
import { getCollectionUserFacingName } from "utils/collection";
import { splitFilenameAndExtension } from "utils/file";
export const ENTE_TRASH_FOLDER = "Trash";
export const getExportRecordFileUID = (file: EnteFile) =>
`${file.id}_${file.collectionID}_${file.updationTime}`;

146
web/apps/photos/src/utils/export/migration.ts
View file

@ -1,146 +0,0 @@
import { ENTE_METADATA_FOLDER } from "constants/export";
import exportService from "services/export";
import {
ExportedCollectionPaths,
ExportRecordV0,
ExportRecordV1,
ExportRecordV2,
} from "types/export";
import { EnteFile } from "types/file";
import { splitFilenameAndExtension } from "utils/ffmpeg";
import { getExportRecordFileUID, sanitizeName } from ".";
export const convertCollectionIDFolderPathObjectToMap = (
exportedCollectionPaths: ExportedCollectionPaths,
): Map<number, string> => {
return new Map<number, string>(
Object.entries(exportedCollectionPaths ?? {}).map((e) => {
return [Number(e[0]), String(e[1])];
}),
);
};
export const getExportedFiles = (
allFiles: EnteFile[],
exportRecord: ExportRecordV0 | ExportRecordV1 | ExportRecordV2,
) => {
if (!exportRecord?.exportedFiles) {
return [];
}
const exportedFileIds = new Set(exportRecord?.exportedFiles);
const exportedFiles = allFiles.filter((file) => {
if (exportedFileIds.has(getExportRecordFileUID(file))) {
return true;
} else {
return false;
}
});
return exportedFiles;
};
export const oldSanitizeName = (name: string) =>
name.replaceAll("/", "_").replaceAll(" ", "_");
export const getUniqueCollectionFolderPath = async (
dir: string,
collectionName: string,
): Promise<string> => {
let collectionFolderPath = `${dir}/${sanitizeName(collectionName)}`;
let count = 1;
while (await exportService.exists(collectionFolderPath)) {
collectionFolderPath = `${dir}/${sanitizeName(
collectionName,
)}(${count})`;
count++;
}
return collectionFolderPath;
};
export const getMetadataFolderPath = (collectionFolderPath: string) =>
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}`;
export const getUniqueFileSaveName = async (
collectionPath: string,
filename: string,
) => {
let fileSaveName = sanitizeName(filename);
let count = 1;
while (
await exportService.exists(
getFileSavePath(collectionPath, fileSaveName),
)
) {
const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
if (filenameParts[1]) {
fileSaveName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
} else {
fileSaveName = `${filenameParts[0]}(${count})`;
}
count++;
}
return fileSaveName;
};
export const getOldFileSaveName = (filename: string, fileID: number) =>
`${fileID}_${oldSanitizeName(filename)}`;
export const getFileMetadataSavePath = (
collectionFolderPath: string,
fileSaveName: string,
) => `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${fileSaveName}.json`;
export const getFileSavePath = (
collectionFolderPath: string,
fileSaveName: string,
) => `${collectionFolderPath}/${fileSaveName}`;
export const getOldCollectionFolderPath = (
dir: string,
collectionID: number,
collectionName: string,
) => `${dir}/${collectionID}_${oldSanitizeName(collectionName)}`;
export const getOldFileSavePath = (
collectionFolderPath: string,
file: EnteFile,
) =>
`${collectionFolderPath}/${file.id}_${oldSanitizeName(
file.metadata.title,
)}`;
export const getOldFileMetadataSavePath = (
collectionFolderPath: string,
file: EnteFile,
) =>
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${
file.id
}_${oldSanitizeName(file.metadata.title)}.json`;
export const getUniqueFileExportNameForMigration = (
collectionPath: string,
filename: string,
usedFilePaths: Map<string, Set<string>>,
) => {
let fileExportName = sanitizeName(filename);
let count = 1;
while (
usedFilePaths
.get(collectionPath)
?.has(getFileSavePath(collectionPath, fileExportName))
) {
const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
if (filenameParts[1]) {
fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
} else {
fileExportName = `${filenameParts[0]}(${count})`;
}
count++;
}
if (!usedFilePaths.has(collectionPath)) {
usedFilePaths.set(collectionPath, new Set());
}
usedFilePaths
.get(collectionPath)
.add(getFileSavePath(collectionPath, fileExportName));
return fileExportName;
};