beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

[desktop] Fix export related IPC - Part 1/x (#1432)

Browse source
This commit is contained in:
Manav Rathi 2024-04-13 18:31:36 +05:30 committed by GitHub
parent e72003c199 4f764dc77c
commit 27fb43837b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 495 additions and 612 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
web/apps/photos/src/components/ExportInProgress.tsx
View file

@ -10,9 +10,9 @@ import {
LinearProgress,
styled,
} from "@mui/material";
import { ExportStage } from "constants/export";
import { t } from "i18next";
import { Trans } from "react-i18next";
import { ExportStage } from "services/export";
import { ExportProgress } from "types/export";
export const ComfySpan = styled("span")`

3
web/apps/photos/src/components/ExportModal.tsx
View file

@ -14,12 +14,11 @@ import {
Switch,
Typography,
} from "@mui/material";
import { ExportStage } from "constants/export";
import { t } from "i18next";
import isElectron from "is-electron";
import { AppContext } from "pages/_app";
import { useContext, useEffect, useState } from "react";
import exportService from "services/export";
import exportService, { ExportStage } from "services/export";
import { ExportProgress, ExportSettings } from "types/export";
import { EnteFile } from "types/file";
import { getExportDirectoryDoesNotExistMessage } from "utils/ui";

14
web/apps/photos/src/constants/export.ts
View file

@ -1,14 +0,0 @@
export const ENTE_METADATA_FOLDER = "metadata";
export const ENTE_TRASH_FOLDER = "Trash";
export enum ExportStage {
INIT = 0,
MIGRATION = 1,
STARTING = 2,
EXPORTING_FILES = 3,
TRASHING_DELETED_FILES = 4,
RENAMING_COLLECTION_FOLDERS = 5,
TRASHING_DELETED_COLLECTIONS = 6,
FINISHED = 7,
}

38
web/apps/photos/src/pages/_app.tsx
View file

@ -52,7 +52,7 @@ import "photoswipe/dist/photoswipe.css";
import { createContext, useEffect, useRef, useState } from "react";
import LoadingBar from "react-top-loading-bar";
import DownloadManager from "services/download";
import exportService from "services/export";
import exportService, { resumeExportsIfNeeded } from "services/export";
import mlWorkManager from "services/machineLearning/mlWorkManager";
import {
getFamilyPortalRedirectURL,
@ -64,7 +64,6 @@ import {
NotificationAttributes,
SetNotificationAttributes,
} from "types/Notification";
import { isExportInProgress } from "utils/export";
import {
getMLSearchConfig,
updateMLSearchConfig,
@ -214,37 +213,10 @@ export default function App({ Component, pageProps }: AppProps) {
return;
}
const initExport = async () => {
try {
log.info("init export");
const token = getToken();
if (!token) {
log.info(
"User not logged in, not starting export continuous sync job",
);
return;
}
await DownloadManager.init(APPS.PHOTOS, { token });
const exportSettings = exportService.getExportSettings();
if (
!(await exportService.exportFolderExists(
exportSettings?.folder,
))
) {
return;
}
const exportRecord = await exportService.getExportRecord(
exportSettings.folder,
);
if (exportSettings.continuousExport) {
exportService.enableContinuousExport();
}
if (isExportInProgress(exportRecord.stage)) {
log.info("export was in progress, resuming");
exportService.scheduleExport();
}
} catch (e) {
log.error("init export failed", e);
}
const token = getToken();
if (!token) return;
await DownloadManager.init(APPS.PHOTOS, { token });
await resumeExportsIfNeeded();
};
initExport();
try {

364
web/apps/photos/src/services/export/index.ts
View file

@ -3,56 +3,42 @@ import log from "@/next/log";
import { CustomError } from "@ente/shared/error";
import { Events, eventBus } from "@ente/shared/events";
import { LS_KEYS, getData, setData } from "@ente/shared/storage/localStorage";
import { formatDateTimeShort } from "@ente/shared/time/format";
import { User } from "@ente/shared/user/types";
import { sleep } from "@ente/shared/utils";
import QueueProcessor, {
CancellationStatus,
RequestCanceller,
} from "@ente/shared/utils/queueProcessor";
import { ExportStage } from "constants/export";
import { FILE_TYPE } from "constants/file";
import { Collection } from "types/collection";
import {
CollectionExportNames,
ExportProgress,
ExportRecord,
ExportSettings,
ExportUIUpdaters,
FileExportNames,
} from "types/export";
import { EnteFile } from "types/file";
import { Metadata } from "types/upload";
import {
constructCollectionNameMap,
getCollectionUserFacingName,
getNonEmptyPersonalCollections,
} from "utils/collection";
import {
convertCollectionIDExportNameObjectToMap,
convertFileIDExportNameObjectToMap,
getCollectionExportPath,
getCollectionExportedFiles,
getCollectionIDFromFileUID,
getDeletedExportedCollections,
getDeletedExportedFiles,
getExportRecordFileUID,
getFileExportPath,
getFileMetadataExportPath,
getGoogleLikeMetadataFile,
getLivePhotoExportName,
getMetadataFileExportPath,
getMetadataFolderExportPath,
getRenamedExportedCollections,
getTrashedFileExportPath,
getUnExportedFiles,
getUniqueCollectionExportName,
getUniqueFileExportName,
isLivePhotoExportName,
parseLivePhotoExportName,
} from "utils/export";
import {
generateStreamFromArrayBuffer,
getPersonalFiles,
getUpdatedEXIFFileForDownload,
mergeMetadata,
splitFilenameAndExtension,
} from "utils/file";
import {
ENTE_TRASH_FOLDER,
getUniqueCollectionExportName,
getUniqueFileExportName,
} from "utils/native-fs";
import { getAllLocalCollections } from "../collectionService";
import downloadManager from "../download";
import { getAllLocalFiles } from "../fileService";
@ -63,6 +49,19 @@ const EXPORT_RECORD_FILE_NAME = "export_status.json";
export const ENTE_EXPORT_DIRECTORY = "ente Photos";
export const ENTE_METADATA_FOLDER = "metadata";
export enum ExportStage {
INIT = 0,
MIGRATION = 1,
STARTING = 2,
EXPORTING_FILES = 3,
TRASHING_DELETED_FILES = 4,
RENAMING_COLLECTION_FOLDERS = 5,
TRASHING_DELETED_COLLECTIONS = 6,
FINISHED = 7,
}
export const NULL_EXPORT_RECORD: ExportRecord = {
version: 3,
lastAttemptTimestamp: null,
@ -484,11 +483,7 @@ class ExportService {
await this.verifyExportFolderExists(exportFolder);
const oldCollectionExportName =
collectionIDExportNameMap.get(collection.id);
const oldCollectionExportPath = getCollectionExportPath(
exportFolder,
oldCollectionExportName,
);
const oldCollectionExportPath = `${exportFolder}/${oldCollectionExportName}`;
const newCollectionExportName =
await getUniqueCollectionExportName(
exportFolder,
@ -497,11 +492,7 @@ class ExportService {
log.info(
`renaming collection with id ${collection.id} from ${oldCollectionExportName} to ${newCollectionExportName}`,
);
const newCollectionExportPath = getCollectionExportPath(
exportFolder,
newCollectionExportName,
);
const newCollectionExportPath = `${exportFolder}/${newCollectionExportName}`;
await this.addCollectionExportedRecord(
exportFolder,
collection.id,
@ -587,10 +578,7 @@ class ExportService {
"collection is not empty, can't remove",
);
}
const collectionExportPath = getCollectionExportPath(
exportFolder,
collectionExportName,
);
const collectionExportPath = `${exportFolder}/${collectionExportName}`;
await this.removeCollectionExportedRecord(
exportFolder,
collectionID,
@ -682,10 +670,7 @@ class ExportService {
collectionExportName,
);
}
const collectionExportPath = getCollectionExportPath(
exportDir,
collectionExportName,
);
const collectionExportPath = `${exportDir}/${collectionExportName}`;
await ensureElectron().checkExistsAndCreateDir(
collectionExportPath,
);
@ -750,10 +735,10 @@ class ExportService {
try {
const fileExportName = fileIDExportNameMap.get(fileUID);
const collectionID = getCollectionIDFromFileUID(fileUID);
const collectionExportPath = getCollectionExportPath(
exportDir,
collectionIDExportNameMap.get(collectionID),
);
const collectionExportName =
collectionIDExportNameMap.get(collectionID);
const collectionExportPath = `${exportDir}/${collectionExportName}`;
await this.removeFileExportedRecord(exportDir, fileUID);
try {
if (isLivePhotoExportName(fileExportName)) {
@ -761,10 +746,7 @@ class ExportService {
image: imageExportName,
video: videoExportName,
} = parseLivePhotoExportName(fileExportName);
const imageExportPath = getFileExportPath(
collectionExportPath,
imageExportName,
);
const imageExportPath = `${collectionExportPath}/${imageExportName}`;
log.info(
`moving image file ${imageExportPath} to trash folder`,
);
@ -793,10 +775,7 @@ class ExportService {
);
}
const videoExportPath = getFileExportPath(
collectionExportPath,
videoExportName,
);
const videoExportPath = `${collectionExportPath}/${videoExportName}`;
log.info(
`moving video file ${videoExportPath} to trash folder`,
);
@ -823,10 +802,7 @@ class ExportService {
);
}
} else {
const fileExportPath = getFileExportPath(
collectionExportPath,
fileExportName,
);
const fileExportPath = `${collectionExportPath}/${fileExportName}`;
const trashedFilePath =
await getTrashedFileExportPath(
exportDir,
@ -1037,10 +1013,7 @@ class ExportService {
exportFolder,
collectionName,
);
const collectionExportPath = getCollectionExportPath(
exportFolder,
collectionExportName,
);
const collectionExportPath = `${exportFolder}/${collectionExportName}`;
await ensureElectron().checkExistsAndCreateDir(collectionExportPath);
await ensureElectron().checkExistsAndCreateDir(
getMetadataFolderExportPath(collectionExportPath),
@ -1090,7 +1063,7 @@ class ExportService {
file,
);
await ensureElectron().saveStreamToDisk(
getFileExportPath(collectionExportPath, fileExportName),
`${collectionExportPath}/${fileExportName}`,
updatedFileStream,
);
} catch (e) {
@ -1138,7 +1111,7 @@ class ExportService {
file,
);
await ensureElectron().saveStreamToDisk(
getFileExportPath(collectionExportPath, imageExportName),
`${collectionExportPath}/${imageExportName}`,
imageStream,
);
@ -1150,12 +1123,12 @@ class ExportService {
);
try {
await ensureElectron().saveStreamToDisk(
getFileExportPath(collectionExportPath, videoExportName),
`${collectionExportPath}/${videoExportName}`,
videoStream,
);
} catch (e) {
await ensureElectron().deleteFile(
getFileExportPath(collectionExportPath, imageExportName),
`${collectionExportPath}/${imageExportName}`,
);
throw e;
}
@ -1218,4 +1191,261 @@ class ExportService {
return exportRecord;
};
}
export default new ExportService();
const exportService = new ExportService();
export default exportService;
/**
* If there are any in-progress exports, or if continuous exports are enabled,
* resume them.
*/
export const resumeExportsIfNeeded = async () => {
const exportSettings = exportService.getExportSettings();
if (!(await exportService.exportFolderExists(exportSettings?.folder))) {
return;
}
const exportRecord = await exportService.getExportRecord(
exportSettings.folder,
);
if (exportSettings.continuousExport) {
exportService.enableContinuousExport();
}
if (isExportInProgress(exportRecord.stage)) {
log.debug(() => "Resuming in-progress export");
exportService.scheduleExport();
}
};
export const getExportRecordFileUID = (file: EnteFile) =>
`${file.id}_${file.collectionID}_${file.updationTime}`;
export const getCollectionIDFromFileUID = (fileUID: string) =>
Number(fileUID.split("_")[1]);
const convertCollectionIDExportNameObjectToMap = (
collectionExportNames: CollectionExportNames,
): Map<number, string> => {
return new Map<number, string>(
Object.entries(collectionExportNames ?? {}).map((e) => {
return [Number(e[0]), String(e[1])];
}),
);
};
const convertFileIDExportNameObjectToMap = (
fileExportNames: FileExportNames,
): Map<string, string> => {
return new Map<string, string>(
Object.entries(fileExportNames ?? {}).map((e) => {
return [String(e[0]), String(e[1])];
}),
);
};
const getRenamedExportedCollections = (
collections: Collection[],
exportRecord: ExportRecord,
) => {
if (!exportRecord?.collectionExportNames) {
return [];
}
const collectionIDExportNameMap = convertCollectionIDExportNameObjectToMap(
exportRecord.collectionExportNames,
);
const renamedCollections = collections.filter((collection) => {
if (collectionIDExportNameMap.has(collection.id)) {
const currentExportName = collectionIDExportNameMap.get(
collection.id,
);
const collectionExportName =
getCollectionUserFacingName(collection);
if (currentExportName === collectionExportName) {
return false;
}
const hasNumberedSuffix = currentExportName.match(/\(\d+\)$/);
const currentExportNameWithoutNumberedSuffix = hasNumberedSuffix
? currentExportName.replace(/\(\d+\)$/, "")
: currentExportName;
return (
collectionExportName !== currentExportNameWithoutNumberedSuffix
);
}
return false;
});
return renamedCollections;
};
const getDeletedExportedCollections = (
collections: Collection[],
exportRecord: ExportRecord,
) => {
if (!exportRecord?.collectionExportNames) {
return [];
}
const presentCollections = new Set(
collections.map((collection) => collection.id),
);
const deletedExportedCollections = Object.keys(
exportRecord?.collectionExportNames,
)
.map(Number)
.filter((collectionID) => {
if (!presentCollections.has(collectionID)) {
return true;
}
return false;
});
return deletedExportedCollections;
};
const getUnExportedFiles = (
allFiles: EnteFile[],
exportRecord: ExportRecord,
) => {
if (!exportRecord?.fileExportNames) {
return allFiles;
}
const exportedFiles = new Set(Object.keys(exportRecord?.fileExportNames));
const unExportedFiles = allFiles.filter((file) => {
if (!exportedFiles.has(getExportRecordFileUID(file))) {
return true;
}
return false;
});
return unExportedFiles;
};
const getDeletedExportedFiles = (
allFiles: EnteFile[],
exportRecord: ExportRecord,
): string[] => {
if (!exportRecord?.fileExportNames) {
return [];
}
const presentFileUIDs = new Set(
allFiles?.map((file) => getExportRecordFileUID(file)),
);
const deletedExportedFiles = Object.keys(
exportRecord?.fileExportNames,
).filter((fileUID) => {
if (!presentFileUIDs.has(fileUID)) {
return true;
}
return false;
});
return deletedExportedFiles;
};
const getCollectionExportedFiles = (
exportRecord: ExportRecord,
collectionID: number,
): string[] => {
if (!exportRecord?.fileExportNames) {
return [];
}
const collectionExportedFiles = Object.keys(
exportRecord?.fileExportNames,
).filter((fileUID) => {
const fileCollectionID = Number(fileUID.split("_")[1]);
if (fileCollectionID === collectionID) {
return true;
} else {
return false;
}
});
return collectionExportedFiles;
};
const getGoogleLikeMetadataFile = (fileExportName: string, file: EnteFile) => {
const metadata: Metadata = file.metadata;
const creationTime = Math.floor(metadata.creationTime / 1000000);
const modificationTime = Math.floor(
(metadata.modificationTime ?? metadata.creationTime) / 1000000,
);
const captionValue: string = file?.pubMagicMetadata?.data?.caption;
return JSON.stringify(
{
title: fileExportName,
caption: captionValue,
creationTime: {
timestamp: creationTime,
formatted: formatDateTimeShort(creationTime * 1000),
},
modificationTime: {
timestamp: modificationTime,
formatted: formatDateTimeShort(modificationTime * 1000),
},
geoData: {
latitude: metadata.latitude,
longitude: metadata.longitude,
},
},
null,
2,
);
};
export const getMetadataFolderExportPath = (collectionExportPath: string) =>
`${collectionExportPath}/${ENTE_METADATA_FOLDER}`;
const getFileMetadataExportPath = (
collectionExportPath: string,
fileExportName: string,
) => `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${fileExportName}.json`;
const getTrashedFileExportPath = async (exportDir: string, path: string) => {
const fileRelativePath = path.replace(`${exportDir}/`, "");
let trashedFilePath = `${exportDir}/${ENTE_TRASH_FOLDER}/${fileRelativePath}`;
let count = 1;
while (await exportService.exists(trashedFilePath)) {
const trashedFilePathParts = splitFilenameAndExtension(trashedFilePath);
if (trashedFilePathParts[1]) {
trashedFilePath = `${trashedFilePathParts[0]}(${count}).${trashedFilePathParts[1]}`;
} else {
trashedFilePath = `${trashedFilePathParts[0]}(${count})`;
}
count++;
}
return trashedFilePath;
};
// if filepath is /home/user/Ente/Export/Collection1/1.jpg
// then metadata path is /home/user/Ente/Export/Collection1/ENTE_METADATA_FOLDER/1.jpg.json
const getMetadataFileExportPath = (filePath: string) => {
// extract filename and collection folder path
const filename = filePath.split("/").pop();
const collectionExportPath = filePath.replace(`/${filename}`, "");
return `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${filename}.json`;
};
export const getLivePhotoExportName = (
imageExportName: string,
videoExportName: string,
) =>
JSON.stringify({
image: imageExportName,
video: videoExportName,
});
export const isLivePhotoExportName = (exportName: string) => {
try {
JSON.parse(exportName);
return true;
} catch (e) {
return false;
}
};
const parseLivePhotoExportName = (
livePhotoExportName: string,
): { image: string; video: string } => {
const { image, video } = JSON.parse(livePhotoExportName);
return { image, video };
};
const isExportInProgress = (exportStage: ExportStage) =>
exportStage > ExportStage.INIT && exportStage < ExportStage.FINISHED;

158
web/apps/photos/src/services/export/migration.ts
View file

@ -15,34 +15,25 @@ import {
ExportRecordV0,
ExportRecordV1,
ExportRecordV2,
ExportedCollectionPaths,
FileExportNames,
} from "types/export";
import { EnteFile } from "types/file";
import { getNonEmptyPersonalCollections } from "utils/collection";
import {
getCollectionExportPath,
getCollectionIDFromFileUID,
getExportRecordFileUID,
getLivePhotoExportName,
getMetadataFolderExportPath,
} from "utils/export";
import {
convertCollectionIDFolderPathObjectToMap,
getExportedFiles,
getFileMetadataSavePath,
getFileSavePath,
getOldCollectionFolderPath,
getOldFileMetadataSavePath,
getOldFileSavePath,
getUniqueCollectionFolderPath,
getUniqueFileExportNameForMigration,
getUniqueFileSaveName,
} from "utils/export/migration";
import { splitFilenameAndExtension } from "utils/ffmpeg";
import {
getIDBasedSortedFiles,
getPersonalFiles,
mergeMetadata,
} from "utils/file";
import { sanitizeName } from "utils/native-fs";
import {
ENTE_METADATA_FOLDER,
getCollectionIDFromFileUID,
getExportRecordFileUID,
getLivePhotoExportName,
getMetadataFolderExportPath,
} from ".";
import exportService from "./index";
export async function migrateExport(
@ -441,7 +432,7 @@ async function removeCollectionExportMissingMetadataFolder(
if (
await exportService.exists(
getMetadataFolderExportPath(
getCollectionExportPath(exportDir, collectionExportName),
`${exportDir}/${collectionExportName}`,
),
)
) {
@ -475,3 +466,130 @@ async function removeCollectionExportMissingMetadataFolder(
};
await exportService.updateExportRecord(exportDir, updatedExportRecord);
}
const convertCollectionIDFolderPathObjectToMap = (
exportedCollectionPaths: ExportedCollectionPaths,
): Map<number, string> => {
return new Map<number, string>(
Object.entries(exportedCollectionPaths ?? {}).map((e) => {
return [Number(e[0]), String(e[1])];
}),
);
};
const getExportedFiles = (
allFiles: EnteFile[],
exportRecord: ExportRecordV0 | ExportRecordV1 | ExportRecordV2,
) => {
if (!exportRecord?.exportedFiles) {
return [];
}
const exportedFileIds = new Set(exportRecord?.exportedFiles);
const exportedFiles = allFiles.filter((file) => {
if (exportedFileIds.has(getExportRecordFileUID(file))) {
return true;
} else {
return false;
}
});
return exportedFiles;
};
const oldSanitizeName = (name: string) =>
name.replaceAll("/", "_").replaceAll(" ", "_");
const getUniqueCollectionFolderPath = async (
dir: string,
collectionName: string,
): Promise<string> => {
let collectionFolderPath = `${dir}/${sanitizeName(collectionName)}`;
let count = 1;
while (await exportService.exists(collectionFolderPath)) {
collectionFolderPath = `${dir}/${sanitizeName(
collectionName,
)}(${count})`;
count++;
}
return collectionFolderPath;
};
export const getMetadataFolderPath = (collectionFolderPath: string) =>
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}`;
const getUniqueFileSaveName = async (
collectionPath: string,
filename: string,
) => {
let fileSaveName = sanitizeName(filename);
let count = 1;
while (
await exportService.exists(
getFileSavePath(collectionPath, fileSaveName),
)
) {
const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
if (filenameParts[1]) {
fileSaveName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
} else {
fileSaveName = `${filenameParts[0]}(${count})`;
}
count++;
}
return fileSaveName;
};
const getFileMetadataSavePath = (
collectionFolderPath: string,
fileSaveName: string,
) => `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${fileSaveName}.json`;
const getFileSavePath = (collectionFolderPath: string, fileSaveName: string) =>
`${collectionFolderPath}/${fileSaveName}`;
const getOldCollectionFolderPath = (
dir: string,
collectionID: number,
collectionName: string,
) => `${dir}/${collectionID}_${oldSanitizeName(collectionName)}`;
const getOldFileSavePath = (collectionFolderPath: string, file: EnteFile) =>
`${collectionFolderPath}/${file.id}_${oldSanitizeName(
file.metadata.title,
)}`;
const getOldFileMetadataSavePath = (
collectionFolderPath: string,
file: EnteFile,
) =>
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${
file.id
}_${oldSanitizeName(file.metadata.title)}.json`;
const getUniqueFileExportNameForMigration = (
collectionPath: string,
filename: string,
usedFilePaths: Map<string, Set<string>>,
) => {
let fileExportName = sanitizeName(filename);
let count = 1;
while (
usedFilePaths
.get(collectionPath)
?.has(getFileSavePath(collectionPath, fileExportName))
) {
const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
if (filenameParts[1]) {
fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
} else {
fileExportName = `${filenameParts[0]}(${count})`;
}
count++;
}
if (!usedFilePaths.has(collectionPath)) {
usedFilePaths.set(collectionPath, new Set());
}
usedFilePaths
.get(collectionPath)
.add(getFileSavePath(collectionPath, fileExportName));
return fileExportName;
};

2
web/apps/photos/src/types/export/index.ts
View file

@ -1,4 +1,4 @@
import { ExportStage } from "constants/export";
import type { ExportStage } from "services/export";
import { EnteFile } from "types/file";
export interface ExportProgress {

10
web/apps/photos/src/utils/collection/index.ts
View file

@ -42,12 +42,9 @@ import {
import { EnteFile } from "types/file";
import { SetFilesDownloadProgressAttributes } from "types/gallery";
import { SUB_TYPE, VISIBILITY_STATE } from "types/magicMetadata";
import {
getCollectionExportPath,
getUniqueCollectionExportName,
} from "utils/export";
import { downloadFilesWithProgress } from "utils/file";
import { isArchivedCollection, updateMagicMetadata } from "utils/magicMetadata";
import { getUniqueCollectionExportName } from "utils/native-fs";
export enum COLLECTION_OPS_TYPE {
ADD,
@ -176,10 +173,7 @@ async function createCollectionDownloadFolder(
downloadDirPath,
collectionName,
);
const collectionDownloadPath = getCollectionExportPath(
downloadDirPath,
collectionDownloadName,
);
const collectionDownloadPath = `${downloadDirPath}/${collectionDownloadName}`;
await exportService.checkExistsAndCreateDir(collectionDownloadPath);
return collectionDownloadPath;
}

312
web/apps/photos/src/utils/export/index.ts
View file

@ -1,312 +0,0 @@
import exportService from "services/export";
import { Collection } from "types/collection";
import {
CollectionExportNames,
ExportRecord,
FileExportNames,
} from "types/export";
import { EnteFile } from "types/file";
import { formatDateTimeShort } from "@ente/shared/time/format";
import {
ENTE_METADATA_FOLDER,
ENTE_TRASH_FOLDER,
ExportStage,
} from "constants/export";
import sanitize from "sanitize-filename";
import { Metadata } from "types/upload";
import { getCollectionUserFacingName } from "utils/collection";
import { splitFilenameAndExtension } from "utils/file";
export const getExportRecordFileUID = (file: EnteFile) =>
`${file.id}_${file.collectionID}_${file.updationTime}`;
export const getCollectionIDFromFileUID = (fileUID: string) =>
Number(fileUID.split("_")[1]);
export const convertCollectionIDExportNameObjectToMap = (
collectionExportNames: CollectionExportNames,
): Map<number, string> => {
return new Map<number, string>(
Object.entries(collectionExportNames ?? {}).map((e) => {
return [Number(e[0]), String(e[1])];
}),
);
};
export const convertFileIDExportNameObjectToMap = (
fileExportNames: FileExportNames,
): Map<string, string> => {
return new Map<string, string>(
Object.entries(fileExportNames ?? {}).map((e) => {
return [String(e[0]), String(e[1])];
}),
);
};
export const getRenamedExportedCollections = (
collections: Collection[],
exportRecord: ExportRecord,
) => {
if (!exportRecord?.collectionExportNames) {
return [];
}
const collectionIDExportNameMap = convertCollectionIDExportNameObjectToMap(
exportRecord.collectionExportNames,
);
const renamedCollections = collections.filter((collection) => {
if (collectionIDExportNameMap.has(collection.id)) {
const currentExportName = collectionIDExportNameMap.get(
collection.id,
);
const collectionExportName =
getCollectionUserFacingName(collection);
if (currentExportName === collectionExportName) {
return false;
}
const hasNumberedSuffix = currentExportName.match(/\(\d+\)$/);
const currentExportNameWithoutNumberedSuffix = hasNumberedSuffix
? currentExportName.replace(/\(\d+\)$/, "")
: currentExportName;
return (
collectionExportName !== currentExportNameWithoutNumberedSuffix
);
}
return false;
});
return renamedCollections;
};
export const getDeletedExportedCollections = (
collections: Collection[],
exportRecord: ExportRecord,
) => {
if (!exportRecord?.collectionExportNames) {
return [];
}
const presentCollections = new Set(
collections.map((collection) => collection.id),
);
const deletedExportedCollections = Object.keys(
exportRecord?.collectionExportNames,
)
.map(Number)
.filter((collectionID) => {
if (!presentCollections.has(collectionID)) {
return true;
}
return false;
});
return deletedExportedCollections;
};
export const getUnExportedFiles = (
allFiles: EnteFile[],
exportRecord: ExportRecord,
) => {
if (!exportRecord?.fileExportNames) {
return allFiles;
}
const exportedFiles = new Set(Object.keys(exportRecord?.fileExportNames));
const unExportedFiles = allFiles.filter((file) => {
if (!exportedFiles.has(getExportRecordFileUID(file))) {
return true;
}
return false;
});
return unExportedFiles;
};
export const getDeletedExportedFiles = (
allFiles: EnteFile[],
exportRecord: ExportRecord,
): string[] => {
if (!exportRecord?.fileExportNames) {
return [];
}
const presentFileUIDs = new Set(
allFiles?.map((file) => getExportRecordFileUID(file)),
);
const deletedExportedFiles = Object.keys(
exportRecord?.fileExportNames,
).filter((fileUID) => {
if (!presentFileUIDs.has(fileUID)) {
return true;
}
return false;
});
return deletedExportedFiles;
};
export const getCollectionExportedFiles = (
exportRecord: ExportRecord,
collectionID: number,
): string[] => {
if (!exportRecord?.fileExportNames) {
return [];
}
const collectionExportedFiles = Object.keys(
exportRecord?.fileExportNames,
).filter((fileUID) => {
const fileCollectionID = Number(fileUID.split("_")[1]);
if (fileCollectionID === collectionID) {
return true;
} else {
return false;
}
});
return collectionExportedFiles;
};
export const getGoogleLikeMetadataFile = (
fileExportName: string,
file: EnteFile,
) => {
const metadata: Metadata = file.metadata;
const creationTime = Math.floor(metadata.creationTime / 1000000);
const modificationTime = Math.floor(
(metadata.modificationTime ?? metadata.creationTime) / 1000000,
);
const captionValue: string = file?.pubMagicMetadata?.data?.caption;
return JSON.stringify(
{
title: fileExportName,
caption: captionValue,
creationTime: {
timestamp: creationTime,
formatted: formatDateTimeShort(creationTime * 1000),
},
modificationTime: {
timestamp: modificationTime,
formatted: formatDateTimeShort(modificationTime * 1000),
},
geoData: {
latitude: metadata.latitude,
longitude: metadata.longitude,
},
},
null,
2,
);
};
export const sanitizeName = (name: string) =>
sanitize(name, { replacement: "_" });
export const getUniqueCollectionExportName = async (
dir: string,
collectionName: string,
): Promise<string> => {
let collectionExportName = sanitizeName(collectionName);
let count = 1;
while (
(await exportService.exists(
getCollectionExportPath(dir, collectionExportName),
)) ||
collectionExportName === ENTE_TRASH_FOLDER
) {
collectionExportName = `${sanitizeName(collectionName)}(${count})`;
count++;
}
return collectionExportName;
};
export const getMetadataFolderExportPath = (collectionExportPath: string) =>
`${collectionExportPath}/${ENTE_METADATA_FOLDER}`;
export const getUniqueFileExportName = async (
collectionExportPath: string,
filename: string,
) => {
let fileExportName = sanitizeName(filename);
let count = 1;
while (
await exportService.exists(
getFileExportPath(collectionExportPath, fileExportName),
)
) {
const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
if (filenameParts[1]) {
fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
} else {
fileExportName = `${filenameParts[0]}(${count})`;
}
count++;
}
return fileExportName;
};
export const getFileMetadataExportPath = (
collectionExportPath: string,
fileExportName: string,
) => `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${fileExportName}.json`;
export const getCollectionExportPath = (
exportFolder: string,
collectionExportName: string,
) => `${exportFolder}/${collectionExportName}`;
export const getFileExportPath = (
collectionExportPath: string,
fileExportName: string,
) => `${collectionExportPath}/${fileExportName}`;
export const getTrashedFileExportPath = async (
exportDir: string,
path: string,
) => {
const fileRelativePath = path.replace(`${exportDir}/`, "");
let trashedFilePath = `${exportDir}/${ENTE_TRASH_FOLDER}/${fileRelativePath}`;
let count = 1;
while (await exportService.exists(trashedFilePath)) {
const trashedFilePathParts = splitFilenameAndExtension(trashedFilePath);
if (trashedFilePathParts[1]) {
trashedFilePath = `${trashedFilePathParts[0]}(${count}).${trashedFilePathParts[1]}`;
} else {
trashedFilePath = `${trashedFilePathParts[0]}(${count})`;
}
count++;
}
return trashedFilePath;
};
// if filepath is /home/user/Ente/Export/Collection1/1.jpg
// then metadata path is /home/user/Ente/Export/Collection1/ENTE_METADATA_FOLDER/1.jpg.json
export const getMetadataFileExportPath = (filePath: string) => {
// extract filename and collection folder path
const filename = filePath.split("/").pop();
const collectionExportPath = filePath.replace(`/${filename}`, "");
return `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${filename}.json`;
};
export const getLivePhotoExportName = (
imageExportName: string,
videoExportName: string,
) =>
JSON.stringify({
image: imageExportName,
video: videoExportName,
});
export const isLivePhotoExportName = (exportName: string) => {
try {
JSON.parse(exportName);
return true;
} catch (e) {
return false;
}
};
export const parseLivePhotoExportName = (
livePhotoExportName: string,
): { image: string; video: string } => {
const { image, video } = JSON.parse(livePhotoExportName);
return { image, video };
};
export const isExportInProgress = (exportStage: ExportStage) =>
exportStage > ExportStage.INIT && exportStage < ExportStage.FINISHED;

146
web/apps/photos/src/utils/export/migration.ts
View file

@ -1,146 +0,0 @@
import { ENTE_METADATA_FOLDER } from "constants/export";
import exportService from "services/export";
import {
ExportedCollectionPaths,
ExportRecordV0,
ExportRecordV1,
ExportRecordV2,
} from "types/export";
import { EnteFile } from "types/file";
import { splitFilenameAndExtension } from "utils/ffmpeg";
import { getExportRecordFileUID, sanitizeName } from ".";
export const convertCollectionIDFolderPathObjectToMap = (
exportedCollectionPaths: ExportedCollectionPaths,
): Map<number, string> => {
return new Map<number, string>(
Object.entries(exportedCollectionPaths ?? {}).map((e) => {
return [Number(e[0]), String(e[1])];
}),
);
};
export const getExportedFiles = (
allFiles: EnteFile[],
exportRecord: ExportRecordV0 | ExportRecordV1 | ExportRecordV2,
) => {
if (!exportRecord?.exportedFiles) {
return [];
}
const exportedFileIds = new Set(exportRecord?.exportedFiles);
const exportedFiles = allFiles.filter((file) => {
if (exportedFileIds.has(getExportRecordFileUID(file))) {
return true;
} else {
return false;
}
});
return exportedFiles;
};
export const oldSanitizeName = (name: string) =>
name.replaceAll("/", "_").replaceAll(" ", "_");
export const getUniqueCollectionFolderPath = async (
dir: string,
collectionName: string,
): Promise<string> => {
let collectionFolderPath = `${dir}/${sanitizeName(collectionName)}`;
let count = 1;
while (await exportService.exists(collectionFolderPath)) {
collectionFolderPath = `${dir}/${sanitizeName(
collectionName,
)}(${count})`;
count++;
}
return collectionFolderPath;
};
export const getMetadataFolderPath = (collectionFolderPath: string) =>
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}`;
export const getUniqueFileSaveName = async (
collectionPath: string,
filename: string,
) => {
let fileSaveName = sanitizeName(filename);
let count = 1;
while (
await exportService.exists(
getFileSavePath(collectionPath, fileSaveName),
)
) {
const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
if (filenameParts[1]) {
fileSaveName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
} else {
fileSaveName = `${filenameParts[0]}(${count})`;
}
count++;
}
return fileSaveName;
};
export const getOldFileSaveName = (filename: string, fileID: number) =>
`${fileID}_${oldSanitizeName(filename)}`;
export const getFileMetadataSavePath = (
collectionFolderPath: string,
fileSaveName: string,
) => `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${fileSaveName}.json`;
export const getFileSavePath = (
collectionFolderPath: string,
fileSaveName: string,
) => `${collectionFolderPath}/${fileSaveName}`;
export const getOldCollectionFolderPath = (
dir: string,
collectionID: number,
collectionName: string,
) => `${dir}/${collectionID}_${oldSanitizeName(collectionName)}`;
export const getOldFileSavePath = (
collectionFolderPath: string,
file: EnteFile,
) =>
`${collectionFolderPath}/${file.id}_${oldSanitizeName(
file.metadata.title,
)}`;
export const getOldFileMetadataSavePath = (
collectionFolderPath: string,
file: EnteFile,
) =>
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${
file.id
}_${oldSanitizeName(file.metadata.title)}.json`;
export const getUniqueFileExportNameForMigration = (
collectionPath: string,
filename: string,
usedFilePaths: Map<string, Set<string>>,
) => {
let fileExportName = sanitizeName(filename);
let count = 1;
while (
usedFilePaths
.get(collectionPath)
?.has(getFileSavePath(collectionPath, fileExportName))
) {
const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
if (filenameParts[1]) {
fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
} else {
fileExportName = `${filenameParts[0]}(${count})`;
}
count++;
}
if (!usedFilePaths.has(collectionPath)) {
usedFilePaths.set(collectionPath, new Set());
}
usedFilePaths
.get(collectionPath)
.add(getFileSavePath(collectionPath, fileExportName));
return fileExportName;
};

12
web/apps/photos/src/utils/file/index.ts
View file

@ -51,8 +51,8 @@ import {
} from "types/gallery";
import { VISIBILITY_STATE } from "types/magicMetadata";
import { FileTypeInfo } from "types/upload";
import { getFileExportPath, getUniqueFileExportName } from "utils/export";
import { isArchivedFile, updateMagicMetadata } from "utils/magicMetadata";
import { getUniqueFileExportName } from "utils/native-fs";
const WAIT_TIME_IMAGE_CONVERSION = 30 * 1000;
@ -818,7 +818,7 @@ async function downloadFileDesktop(
);
const imageStream = generateStreamFromArrayBuffer(livePhoto.image);
await electron.saveStreamToDisk(
getFileExportPath(downloadPath, imageExportName),
`${downloadPath}/${imageExportName}`,
imageStream,
);
try {
@ -828,13 +828,11 @@ async function downloadFileDesktop(
);
const videoStream = generateStreamFromArrayBuffer(livePhoto.video);
await electron.saveStreamToDisk(
getFileExportPath(downloadPath, videoExportName),
`${downloadPath}/${videoExportName}`,
videoStream,
);
} catch (e) {
await electron.deleteFile(
getFileExportPath(downloadPath, imageExportName),
);
await electron.deleteFile(`${downloadPath}/${imageExportName}`);
throw e;
}
} else {
@ -843,7 +841,7 @@ async function downloadFileDesktop(
file.metadata.title,
);
await electron.saveStreamToDisk(
getFileExportPath(downloadPath, fileExportName),
`${downloadPath}/${fileExportName}`,
updatedFileStream,
);
}

44
web/apps/photos/src/utils/native-fs.ts Normal file
View file

@ -0,0 +1,44 @@
import sanitize from "sanitize-filename";
import exportService from "services/export";
import { splitFilenameAndExtension } from "utils/file";
export const ENTE_TRASH_FOLDER = "Trash";
export const sanitizeName = (name: string) =>
sanitize(name, { replacement: "_" });
export const getUniqueCollectionExportName = async (
dir: string,
collectionName: string,
): Promise<string> => {
let collectionExportName = sanitizeName(collectionName);
let count = 1;
while (
(await exportService.exists(`${dir}/${collectionExportName}`)) ||
collectionExportName === ENTE_TRASH_FOLDER
) {
collectionExportName = `${sanitizeName(collectionName)}(${count})`;
count++;
}
return collectionExportName;
};
export const getUniqueFileExportName = async (
collectionExportPath: string,
filename: string,
) => {
let fileExportName = sanitizeName(filename);
let count = 1;
while (
await exportService.exists(`${collectionExportPath}/${fileExportName}`)
) {
const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
if (filenameParts[1]) {
fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
} else {
fileExportName = `${filenameParts[0]}(${count})`;
}
count++;
}
return fileExportName;
};

2
web/apps/photos/src/utils/upload/index.ts
View file

@ -1,4 +1,3 @@
import { ENTE_METADATA_FOLDER } from "constants/export";
import { FILE_TYPE } from "constants/file";
import {
A_SEC_IN_MICROSECONDS,
@ -6,6 +5,7 @@ import {
PICKED_UPLOAD_TYPE,
} from "constants/upload";
import isElectron from "is-electron";
import { ENTE_METADATA_FOLDER } from "services/export";
import { EnteFile } from "types/file";
import {
ElectronFile,