beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

[web] Upload refactoring - Part x/x (#1536)

Browse source
This commit is contained in:
Manav Rathi 2024-04-25 21:33:03 +05:30 committed by GitHub
parent 6d864a71a7 8067d9b9b9
commit c1103b656c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
41 changed files with 1347 additions and 1191 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
desktop/src/main/fs.ts
View file

@ -27,3 +27,5 @@ export const fsIsDir = async (dirPath: string) => {
const stat = await fs.stat(dirPath);
return stat.isDirectory();
};
export const fsSize = (path: string) => fs.stat(path).then((s) => s.size);

3
desktop/src/main/ipc.ts
View file

@ -29,6 +29,7 @@ import {
fsRename,
fsRm,
fsRmdir,
fsSize,
fsWriteFile,
} from "./fs";
import { logToDisk } from "./log";
@ -139,6 +140,8 @@ export const attachIPCHandlers = () => {
ipcMain.handle("fsIsDir", (_, dirPath: string) => fsIsDir(dirPath));
ipcMain.handle("fsSize", (_, path: string) => fsSize(path));
// - Conversion
ipcMain.handle("convertToJPEG", (_, imageData: Uint8Array) =>

2
desktop/src/main/services/ffmpeg.ts
View file

@ -77,7 +77,7 @@ export const ffmpegExec = async (
if (isInputFileTemporary) await deleteTempFile(inputFilePath);
await deleteTempFile(outputFilePath);
} catch (e) {
log.error("Ignoring error when cleaning up temp files", e);
log.error("Could not clean up temp files", e);
}
}
};

4
desktop/src/main/services/image.ts
View file

@ -23,7 +23,7 @@ export const convertToJPEG = async (imageData: Uint8Array) => {
await deleteTempFile(inputFilePath);
await deleteTempFile(outputFilePath);
} catch (e) {
log.error("Ignoring error when cleaning up temp files", e);
log.error("Could not clean up temp files", e);
}
}
};
@ -110,7 +110,7 @@ export const generateImageThumbnail = async (
if (isInputFileTemporary) await deleteTempFile(inputFilePath);
await deleteTempFile(outputFilePath);
} catch (e) {
log.error("Ignoring error when cleaning up temp files", e);
log.error("Could not clean up temp files", e);
}
}
};

8
desktop/src/main/stream.ts
View file

@ -62,9 +62,15 @@ const handleRead = async (path: string) => {
// this is binary data.
res.headers.set("Content-Type", "application/octet-stream");
const stat = await fs.stat(path);
// Add the file's size as the Content-Length header.
const fileSize = (await fs.stat(path)).size;
const fileSize = stat.size;
res.headers.set("Content-Length", `${fileSize}`);
// Add the file's last modified time (as epoch milliseconds).
const mtimeMs = stat.mtimeMs;
res.headers.set("X-Last-Modified-Ms", `${mtimeMs}`);
}
return res;
} catch (e) {

4
desktop/src/preload.ts
View file

@ -122,6 +122,9 @@ const fsWriteFile = (path: string, contents: string): Promise<void> =>
const fsIsDir = (dirPath: string): Promise<boolean> =>
ipcRenderer.invoke("fsIsDir", dirPath);
const fsSize = (path: string): Promise<number> =>
ipcRenderer.invoke("fsSize", path);
// - Conversion
const convertToJPEG = (imageData: Uint8Array): Promise<Uint8Array> =>
@ -331,6 +334,7 @@ contextBridge.exposeInMainWorld("electron", {
readTextFile: fsReadTextFile,
writeFile: fsWriteFile,
isDir: fsIsDir,
size: fsSize,
},
// - Conversion

3
web/apps/cast/package.json
View file

@ -7,7 +7,6 @@
"@/next": "*",
"@ente/accounts": "*",
"@ente/eslint-config": "*",
"@ente/shared": "*",
"mime-types": "^2.1.35"
"@ente/shared": "*"
}
}

7
web/apps/cast/src/services/detect-type.ts
View file

@ -1,5 +1,5 @@
import { KnownFileTypeInfos } from "@/media/file-type";
import { nameAndExtension } from "@/next/file";
import { lowercaseExtension } from "@/next/file";
import FileType from "file-type";
/**
@ -22,8 +22,7 @@ export const detectMediaMIMEType = async (file: File): Promise<string> => {
else throw new Error(`Detected MIME type ${mime} is not a media file`);
}
let [, ext] = nameAndExtension(file.name);
const ext = lowercaseExtension(file.name);
if (!ext) return undefined;
ext = ext.toLowerCase();
return KnownFileTypeInfos.find((f) => f.exactType == ext)?.mimeType;
return KnownFileTypeInfos.find((f) => f.extension == ext)?.mimeType;
};

2
web/apps/cast/src/types/file/index.ts
View file

@ -1,9 +1,9 @@
import type { Metadata } from "@/media/types/file";
import {
EncryptedMagicMetadata,
MagicMetadataCore,
VISIBILITY_STATE,
} from "types/magicMetadata";
import { Metadata } from "types/upload";
export interface MetadataFileAttributes {
encryptedData: string;

25
web/apps/cast/src/types/upload.ts
View file

@ -1,25 +0,0 @@
import { FILE_TYPE } from "@/media/file-type";
export interface Metadata {
title: string;
creationTime: number;
modificationTime: number;
latitude: number;
longitude: number;
fileType: FILE_TYPE;
hasStaticThumbnail?: boolean;
hash?: string;
imageHash?: string;
videoHash?: string;
localID?: number;
version?: number;
deviceFolder?: string;
}
export interface FileTypeInfo {
fileType: FILE_TYPE;
exactType: string;
mimeType?: string;
imageType?: string;
videoType?: string;
}

1
web/apps/photos/package.json
View file

@ -29,6 +29,7 @@
"leaflet-defaulticon-compatibility": "^0.1.1",
"localforage": "^1.9.0",
"memoize-one": "^6.0.0",
"mime-types": "^2.1.35",
"ml-matrix": "^6.10.4",
"otpauth": "^9.0.2",
"p-debounce": "^4.0.0",

4
web/apps/photos/src/components/PhotoViewer/ImageEditorOverlay/index.tsx
View file

@ -42,8 +42,8 @@ import { t } from "i18next";
import mime from "mime-types";
import { AppContext } from "pages/_app";
import { getLocalCollections } from "services/collectionService";
import { detectFileTypeInfo } from "services/detect-type";
import downloadManager from "services/download";
import { getFileType } from "services/typeDetectionService";
import uploadManager from "services/upload/uploadManager";
import { EnteFile } from "types/file";
import { FileWithCollection } from "types/upload";
@ -486,7 +486,7 @@ const ImageEditorOverlay = (props: IProps) => {
if (!canvasRef.current) return;
const editedFile = await getEditedFile();
const fileType = await getFileType(editedFile);
const fileType = await detectFileTypeInfo(editedFile);
const tempImgURL = URL.createObjectURL(
new Blob([editedFile], { type: fileType.mimeType }),
);

11
web/apps/photos/src/components/PhotoViewer/index.tsx
View file

@ -10,13 +10,13 @@ import { EnteFile } from "types/file";
import {
copyFileToClipboard,
downloadSingleFile,
getFileExtension,
getFileFromURL,
isRawFile,
isSupportedRawFormat,
} from "utils/file";
import { FILE_TYPE } from "@/media/file-type";
import { lowercaseExtension } from "@/next/file";
import { FlexWrapper } from "@ente/shared/components/Container";
import EnteSpinner from "@ente/shared/components/EnteSpinner";
import AlbumOutlined from "@mui/icons-material/AlbumOutlined";
@ -43,10 +43,10 @@ import { t } from "i18next";
import isElectron from "is-electron";
import { AppContext } from "pages/_app";
import { GalleryContext } from "pages/gallery";
import { detectFileTypeInfo } from "services/detect-type";
import downloadManager, { LoadedLivePhotoSourceURL } from "services/download";
import { getParsedExifData } from "services/exif";
import { trashFiles } from "services/fileService";
import { getFileType } from "services/typeDetectionService";
import { SetFilesDownloadProgressAttributesCreator } from "types/gallery";
import { isClipboardItemPresent } from "utils/common";
import { pauseVideo, playVideo } from "utils/photoFrame";
@ -348,7 +348,7 @@ function PhotoViewer(props: Iprops) {
}
function updateShowEditButton(file: EnteFile) {
const extension = getFileExtension(file.metadata.title);
const extension = lowercaseExtension(file.metadata.title);
const isSupported =
!isRawFile(extension) || isSupportedRawFormat(extension);
setShowEditButton(
@ -594,7 +594,7 @@ function PhotoViewer(props: Iprops) {
.image;
fileObject = await getFileFromURL(url, file.metadata.title);
}
const fileTypeInfo = await getFileType(fileObject);
const fileTypeInfo = await detectFileTypeInfo(fileObject);
const exifData = await getParsedExifData(
fileObject,
fileTypeInfo,
@ -611,9 +611,8 @@ function PhotoViewer(props: Iprops) {
}
} catch (e) {
setExif({ key: file.src, value: null });
const fileExtension = getFileExtension(file.metadata.title);
log.error(
`checkExifAvailable failed for extension ${fileExtension}`,
`checkExifAvailable failed for file ${file.metadata.title}`,
e,
);
}

2
web/apps/photos/src/services/deduplicationService.ts
View file

@ -1,10 +1,10 @@
import { FILE_TYPE } from "@/media/file-type";
import type { Metadata } from "@/media/types/file";
import log from "@/next/log";
import HTTPService from "@ente/shared/network/HTTPService";
import { getEndpoint } from "@ente/shared/network/api";
import { getToken } from "@ente/shared/storage/localStorage/helpers";
import { EnteFile } from "types/file";
import { Metadata } from "types/upload";
import { hasFileHash } from "utils/upload";
const ENDPOINT = getEndpoint();

101
web/apps/photos/src/services/detect-type.ts Normal file
View file

@ -0,0 +1,101 @@
import {
FILE_TYPE,
KnownFileTypeInfos,
KnownNonMediaFileExtensions,
type FileTypeInfo,
} from "@/media/file-type";
import { lowercaseExtension } from "@/next/file";
import { CustomError } from "@ente/shared/error";
import FileType from "file-type";
import { getUint8ArrayView } from "./readerService";
/**
* Read the file's initial contents or use the file's name to detect its type.
*
* This function first reads an initial chunk of the file and tries to detect
* the file's {@link FileTypeInfo} from it. If that doesn't work, it then falls
* back to using the file's name to detect it.
*
* If neither of these two approaches work, it throws an exception.
*
* If we were able to detect the file type, but it is explicitly not a media
* (image or video) format that we support, this function throws an error with
* the message `CustomError.UNSUPPORTED_FILE_FORMAT`.
*
* @param file A {@link File} object
*
* @returns The detected {@link FileTypeInfo}.
*/
export const detectFileTypeInfo = async (file: File): Promise<FileTypeInfo> =>
detectFileTypeInfoFromChunk(() => readInitialChunkOfFile(file), file.name);
/**
* The lower layer implementation of the type detector.
*
* Usually, when the code already has a {@link File} object at hand, it is
* easier to use the higher level {@link detectFileTypeInfo} function.
*
* However, this lower level function is also exposed for use in cases like
* during upload where we might not have a File object and would like to provide
* the initial chunk of the file's contents in a different way.
*
* @param readInitialChunk A function to call to read the initial chunk of the
* file's data. There is no strict requirement for the size of the chunk this
* function should return, generally the first few KBs should be good.
*
* @param fileNameOrPath The full path or just the file name of the file whose
* type we're trying to determine. This is used by the fallback layer that tries
* to detect the type info from the file's extension.
*/
export const detectFileTypeInfoFromChunk = async (
readInitialChunk: () => Promise<Uint8Array>,
fileNameOrPath: string,
): Promise<FileTypeInfo> => {
try {
const typeResult = await detectFileTypeFromBuffer(
await readInitialChunk(),
);
const mimeType = typeResult.mime;
let fileType: FILE_TYPE;
if (mimeType.startsWith("image/")) {
fileType = FILE_TYPE.IMAGE;
} else if (mimeType.startsWith("video/")) {
fileType = FILE_TYPE.VIDEO;
} else {
throw new Error(CustomError.UNSUPPORTED_FILE_FORMAT);
}
return {
fileType,
// See https://github.com/sindresorhus/file-type/blob/main/core.d.ts
// for the full list of ext values.
extension: typeResult.ext,
mimeType,
};
} catch (e) {
const extension = lowercaseExtension(fileNameOrPath);
const known = KnownFileTypeInfos.find((f) => f.extension == extension);
if (known) return known;
if (KnownNonMediaFileExtensions.includes(extension))
throw Error(CustomError.UNSUPPORTED_FILE_FORMAT);
throw e;
}
};
const readInitialChunkOfFile = async (file: File) => {
const chunkSizeForTypeDetection = 4100;
const chunk = file.slice(0, chunkSizeForTypeDetection);
return await getUint8ArrayView(chunk);
};
const detectFileTypeFromBuffer = async (buffer: Uint8Array) => {
const result = await FileType.fromBuffer(buffer);
if (!result?.ext || !result?.mime) {
throw Error(`Could not deduce file type from buffer`);
}
return result;
};

55
web/apps/photos/src/services/exif.ts
View file

@ -4,7 +4,7 @@ import { validateAndGetCreationUnixTimeInMicroSeconds } from "@ente/shared/time"
import { NULL_LOCATION } from "constants/upload";
import exifr from "exifr";
import piexif from "piexifjs";
import { Location } from "types/upload";
import { Location, type ParsedExtractedMetadata } from "types/upload";
type ParsedEXIFData = Record<string, any> &
Partial<{
@ -34,18 +34,59 @@ type RawEXIFData = Record<string, any> &
ImageHeight: number;
}>;
const exifTagsNeededForParsingImageMetadata = [
"DateTimeOriginal",
"CreateDate",
"ModifyDate",
"GPSLatitude",
"GPSLongitude",
"GPSLatitudeRef",
"GPSLongitudeRef",
"DateCreated",
"ExifImageWidth",
"ExifImageHeight",
"ImageWidth",
"ImageHeight",
"PixelXDimension",
"PixelYDimension",
"MetadataDate",
];
/**
* Read EXIF data from an image {@link file} and use that to construct and
* return an {@link ParsedExtractedMetadata}.
*
* This function is tailored for use when we upload files.
*/
export const parseImageMetadata = async (
file: File,
fileTypeInfo: FileTypeInfo,
): Promise<ParsedExtractedMetadata> => {
const exifData = await getParsedExifData(
file,
fileTypeInfo,
exifTagsNeededForParsingImageMetadata,
);
return {
location: getEXIFLocation(exifData),
creationTime: getEXIFTime(exifData),
width: exifData?.imageWidth ?? null,
height: exifData?.imageHeight ?? null,
};
};
export async function getParsedExifData(
receivedFile: File,
{ exactType }: FileTypeInfo,
{ extension }: FileTypeInfo,
tags?: string[],
): Promise<ParsedEXIFData> {
const exifLessFormats = ["gif", "bmp"];
const exifrUnsupportedFileFormatMessage = "Unknown file format";
try {
if (exifLessFormats.includes(exactType)) {
return null;
}
if (exifLessFormats.includes(extension)) return null;
const exifData: RawEXIFData = await exifr.parse(receivedFile, {
reviveValues: false,
tiff: true,
@ -68,10 +109,10 @@ export async function getParsedExifData(
return parseExifData(filteredExifData);
} catch (e) {
if (e.message == exifrUnsupportedFileFormatMessage) {
log.error(`EXIFR does not support format ${exactType}`, e);
log.error(`EXIFR does not support ${extension} files`, e);
return undefined;
} else {
log.error(`Failed to parse EXIF data of ${exactType} file`, e);
log.error(`Failed to parse EXIF data for a ${extension} file`, e);
throw e;
}
}

2
web/apps/photos/src/services/export/index.ts
View file

@ -1,5 +1,6 @@
import { FILE_TYPE } from "@/media/file-type";
import { decodeLivePhoto } from "@/media/live-photo";
import type { Metadata } from "@/media/types/file";
import { ensureElectron } from "@/next/electron";
import log from "@/next/log";
import { CustomError } from "@ente/shared/error";
@ -22,7 +23,6 @@ import {
FileExportNames,
} from "types/export";
import { EnteFile } from "types/file";
import { Metadata } from "types/upload";
import {
constructCollectionNameMap,
getCollectionUserFacingName,

6
web/apps/photos/src/services/export/migration.ts
View file

@ -1,6 +1,7 @@
import { FILE_TYPE } from "@/media/file-type";
import { decodeLivePhoto } from "@/media/live-photo";
import { ensureElectron } from "@/next/electron";
import { nameAndExtension } from "@/next/file";
import log from "@/next/log";
import { LS_KEYS, getData } from "@ente/shared/storage/localStorage";
import { User } from "@ente/shared/user/types";
@ -25,7 +26,6 @@ import {
getIDBasedSortedFiles,
getPersonalFiles,
mergeMetadata,
splitFilenameAndExtension,
} from "utils/file";
import {
safeDirectoryName,
@ -501,9 +501,7 @@ const getUniqueFileExportNameForMigration = (
.get(collectionPath)
?.has(getFileSavePath(collectionPath, fileExportName))
) {
const filenameParts = splitFilenameAndExtension(
sanitizeFilename(filename),
);
const filenameParts = nameAndExtension(sanitizeFilename(filename));
if (filenameParts[1]) {
fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
} else {

90
web/apps/photos/src/services/ffmpeg.ts
View file

@ -26,11 +26,11 @@ import { type DedicatedFFmpegWorker } from "worker/ffmpeg.worker";
* See also {@link generateVideoThumbnailNative}.
*/
export const generateVideoThumbnailWeb = async (blob: Blob) =>
generateVideoThumbnail((seekTime: number) =>
ffmpegExecWeb(genThumbnailCommand(seekTime), blob, "jpeg", 0),
_generateVideoThumbnail((seekTime: number) =>
ffmpegExecWeb(makeGenThumbnailCommand(seekTime), blob, "jpeg", 0),
);
const generateVideoThumbnail = async (
const _generateVideoThumbnail = async (
thumbnailAtTime: (seekTime: number) => Promise<Uint8Array>,
) => {
try {
@ -51,7 +51,7 @@ const generateVideoThumbnail = async (
* for the new files that the user is adding.
*
* @param dataOrPath The input video's data or the path to the video on the
* user's local filesystem. See: [Note: The fileOrPath parameter to upload].
* user's local filesystem. See: [Note: Reading a fileOrPath].
*
* @returns JPEG data of the generated thumbnail.
*
@ -61,16 +61,16 @@ export const generateVideoThumbnailNative = async (
electron: Electron,
dataOrPath: Uint8Array | string,
) =>
generateVideoThumbnail((seekTime: number) =>
_generateVideoThumbnail((seekTime: number) =>
electron.ffmpegExec(
genThumbnailCommand(seekTime),
makeGenThumbnailCommand(seekTime),
dataOrPath,
"jpeg",
0,
),
);
const genThumbnailCommand = (seekTime: number) => [
const makeGenThumbnailCommand = (seekTime: number) => [
ffmpegPathPlaceholder,
"-i",
inputPathPlaceholder,
@ -83,30 +83,58 @@ const genThumbnailCommand = (seekTime: number) => [
outputPathPlaceholder,
];
/** Called during upload */
export async function extractVideoMetadata(file: File | ElectronFile) {
// https://stackoverflow.com/questions/9464617/retrieving-and-saving-media-metadata-using-ffmpeg
// -c [short for codex] copy[(stream_specifier)[ffmpeg.org/ffmpeg.html#Stream-specifiers]] => copies all the stream without re-encoding
// -map_metadata [http://ffmpeg.org/ffmpeg.html#Advanced-options search for map_metadata] => copies all stream metadata to the out
// -f ffmetadata [https://ffmpeg.org/ffmpeg-formats.html#Metadata-1] => dump metadata from media files into a simple UTF-8-encoded INI-like text file
const metadata = await ffmpegExec2(
[
ffmpegPathPlaceholder,
"-i",
inputPathPlaceholder,
"-c",
"copy",
"-map_metadata",
"0",
"-f",
"ffmetadata",
outputPathPlaceholder,
],
file,
"txt",
);
return parseFFmpegExtractedMetadata(metadata);
}
/**
* Extract metadata from the given video
*
* When we're running in the context of our desktop app _and_ we're passed a
* file path , this uses the native FFmpeg bundled with our desktop app.
* Otherwise it uses a wasm FFmpeg running in a web worker.
*
* This function is called during upload, when we need to extract the metadata
* of videos that the user is uploading.
*
* @param fileOrPath A {@link File}, or the absolute path to a file on the
* user's local filesytem. A path can only be provided when we're running in the
* context of our desktop app.
*/
export const extractVideoMetadata = async (
fileOrPath: File | string,
): Promise<ParsedExtractedMetadata> => {
const command = extractVideoMetadataCommand;
const outputData =
fileOrPath instanceof File
? await ffmpegExecWeb(command, fileOrPath, "txt", 0)
: await electron.ffmpegExec(command, fileOrPath, "txt", 0);
return parseFFmpegExtractedMetadata(outputData);
};
// Options:
//
// - `-c [short for codex] copy`
// - copy is the [stream_specifier](ffmpeg.org/ffmpeg.html#Stream-specifiers)
// - copies all the stream without re-encoding
//
// - `-map_metadata`
// - http://ffmpeg.org/ffmpeg.html#Advanced-options (search for map_metadata)
// - copies all stream metadata to the output
//
// - `-f ffmetadata`
// - https://ffmpeg.org/ffmpeg-formats.html#Metadata-1
// - dump metadata from media files into a simple INI-like utf-8 text file
//
const extractVideoMetadataCommand = [
ffmpegPathPlaceholder,
"-i",
inputPathPlaceholder,
"-c",
"copy",
"-map_metadata",
"0",
"-f",
"ffmetadata",
outputPathPlaceholder,
];
enum MetadataTags {
CREATION_TIME = "creation_time",

4
web/apps/photos/src/services/fix-exif.ts
View file

@ -2,7 +2,7 @@ import { FILE_TYPE } from "@/media/file-type";
import log from "@/next/log";
import { validateAndGetCreationUnixTimeInMicroSeconds } from "@ente/shared/time";
import type { FixOption } from "components/FixCreationTime";
import { getFileType } from "services/typeDetectionService";
import { detectFileTypeInfo } from "services/detect-type";
import { EnteFile } from "types/file";
import {
changeFileCreationTime,
@ -53,7 +53,7 @@ export async function updateCreationTimeWithExif(
[fileBlob],
file.metadata.title,
);
const fileTypeInfo = await getFileType(fileObject);
const fileTypeInfo = await detectFileTypeInfo(fileObject);
const exifData = await getParsedExifData(
fileObject,
fileTypeInfo,

97
web/apps/photos/src/services/typeDetectionService.ts
View file

@ -1,97 +0,0 @@
import {
FILE_TYPE,
KnownFileTypeInfos,
KnownNonMediaFileExtensions,
type FileTypeInfo,
} from "@/media/file-type";
import log from "@/next/log";
import { ElectronFile } from "@/next/types/file";
import { CustomError } from "@ente/shared/error";
import FileType, { type FileTypeResult } from "file-type";
import { getFileExtension } from "utils/file";
import { getUint8ArrayView } from "./readerService";
const TYPE_VIDEO = "video";
const TYPE_IMAGE = "image";
const CHUNK_SIZE_FOR_TYPE_DETECTION = 4100;
export async function getFileType(
receivedFile: File | ElectronFile,
): Promise<FileTypeInfo> {
try {
let fileType: FILE_TYPE;
let typeResult: FileTypeResult;
if (receivedFile instanceof File) {
typeResult = await extractFileType(receivedFile);
} else {
typeResult = await extractElectronFileType(receivedFile);
}
const mimTypeParts: string[] = typeResult.mime?.split("/");
if (mimTypeParts?.length !== 2) {
throw Error(CustomError.INVALID_MIME_TYPE(typeResult.mime));
}
switch (mimTypeParts[0]) {
case TYPE_IMAGE:
fileType = FILE_TYPE.IMAGE;
break;
case TYPE_VIDEO:
fileType = FILE_TYPE.VIDEO;
break;
default:
throw Error(CustomError.NON_MEDIA_FILE);
}
return {
fileType,
exactType: typeResult.ext,
mimeType: typeResult.mime,
};
} catch (e) {
const fileFormat = getFileExtension(receivedFile.name);
const whiteListedFormat = KnownFileTypeInfos.find(
(a) => a.exactType === fileFormat,
);
if (whiteListedFormat) {
return whiteListedFormat;
}
if (KnownNonMediaFileExtensions.includes(fileFormat)) {
throw Error(CustomError.UNSUPPORTED_FILE_FORMAT);
}
if (e.message === CustomError.NON_MEDIA_FILE) {
log.error(`unsupported file format ${fileFormat}`, e);
throw Error(CustomError.UNSUPPORTED_FILE_FORMAT);
}
log.error(`type detection failed for format ${fileFormat}`, e);
throw Error(CustomError.TYPE_DETECTION_FAILED(fileFormat));
}
}
async function extractFileType(file: File) {
const fileBlobChunk = file.slice(0, CHUNK_SIZE_FOR_TYPE_DETECTION);
const fileDataChunk = await getUint8ArrayView(fileBlobChunk);
return getFileTypeFromBuffer(fileDataChunk);
}
async function extractElectronFileType(file: ElectronFile) {
const stream = await file.stream();
const reader = stream.getReader();
const { value: fileDataChunk } = await reader.read();
await reader.cancel();
return getFileTypeFromBuffer(fileDataChunk);
}
async function getFileTypeFromBuffer(buffer: Uint8Array) {
const result = await FileType.fromBuffer(buffer);
if (!result?.mime) {
let logableInfo = "";
try {
logableInfo = `result: ${JSON.stringify(result)}`;
} catch (e) {
logableInfo = "failed to stringify result";
}
throw Error(`mimetype missing from file type result - ${logableInfo}`);
}
return result;
}

166
web/apps/photos/src/services/upload/date.ts Normal file
View file

@ -0,0 +1,166 @@
import log from "@/next/log";
import { validateAndGetCreationUnixTimeInMicroSeconds } from "@ente/shared/time";
/**
* Try to extract a date (as epoch microseconds) from a file name by matching it
* against certain known patterns for media files.
*
* If it doesn't match a known pattern, or if there is some error during the
* parsing, return `undefined`.
*/
export const tryParseEpochMicrosecondsFromFileName = (
fileName: string,
): number | undefined => {
try {
fileName = fileName.trim();
let parsedDate: Date;
if (fileName.startsWith("IMG-") || fileName.startsWith("VID-")) {
// WhatsApp media files
// Sample name: IMG-20171218-WA0028.jpg
parsedDate = parseDateFromFusedDateString(fileName.split("-")[1]);
} else if (fileName.startsWith("Screenshot_")) {
// Screenshots on Android
// Sample name: Screenshot_20181227-152914.jpg
parsedDate = parseDateFromFusedDateString(
fileName.replaceAll("Screenshot_", ""),
);
} else if (fileName.startsWith("signal-")) {
// Signal images
// Sample name: signal-2018-08-21-100217.jpg
const p = fileName.split("-");
const dateString = `${p[1]}${p[2]}${p[3]}-${p[4]}`;
parsedDate = parseDateFromFusedDateString(dateString);
}
if (!parsedDate) {
parsedDate = tryToParseDateTime(fileName);
}
return validateAndGetCreationUnixTimeInMicroSeconds(parsedDate);
} catch (e) {
log.error(`Could not extract date from file name ${fileName}`, e);
return undefined;
}
};
interface DateComponent<T = number> {
year: T;
month: T;
day: T;
hour: T;
minute: T;
second: T;
}
const currentYear = new Date().getFullYear();
/*
generates data component for date in format YYYYMMDD-HHMMSS
*/
function parseDateFromFusedDateString(dateTime: string) {
const dateComponent: DateComponent<number> = convertDateComponentToNumber({
year: dateTime.slice(0, 4),
month: dateTime.slice(4, 6),
day: dateTime.slice(6, 8),
hour: dateTime.slice(9, 11),
minute: dateTime.slice(11, 13),
second: dateTime.slice(13, 15),
});
return validateAndGetDateFromComponents(dateComponent);
}
/* sample date format = 2018-08-19 12:34:45
the date has six symbol separated number values
which we would extract and use to form the date
*/
export function tryToParseDateTime(dateTime: string): Date {
const dateComponent = getDateComponentsFromSymbolJoinedString(dateTime);
if (dateComponent.year?.length === 8 && dateComponent.month?.length === 6) {
// the filename has size 8 consecutive and then 6 consecutive digits
// high possibility that the it is a date in format YYYYMMDD-HHMMSS
const possibleDateTime = dateComponent.year + "-" + dateComponent.month;
return parseDateFromFusedDateString(possibleDateTime);
}
return validateAndGetDateFromComponents(
convertDateComponentToNumber(dateComponent),
);
}
function getDateComponentsFromSymbolJoinedString(
dateTime: string,
): DateComponent<string> {
const [year, month, day, hour, minute, second] =
dateTime.match(/\d+/g) ?? [];
return { year, month, day, hour, minute, second };
}
function validateAndGetDateFromComponents(
dateComponent: DateComponent<number>,
options = { minYear: 1990, maxYear: currentYear + 1 },
) {
let date = getDateFromComponents(dateComponent);
if (hasTimeValues(dateComponent) && !isTimePartValid(date, dateComponent)) {
// if the date has time values but they are not valid
// then we remove the time values and try to validate the date
date = getDateFromComponents(removeTimeValues(dateComponent));
}
if (!isDatePartValid(date, dateComponent)) {
return null;
}
if (
date.getFullYear() < options.minYear ||
date.getFullYear() > options.maxYear
) {
return null;
}
return date;
}
function isTimePartValid(date: Date, dateComponent: DateComponent<number>) {
return (
date.getHours() === dateComponent.hour &&
date.getMinutes() === dateComponent.minute &&
date.getSeconds() === dateComponent.second
);
}
function isDatePartValid(date: Date, dateComponent: DateComponent<number>) {
return (
date.getFullYear() === dateComponent.year &&
date.getMonth() === dateComponent.month &&
date.getDate() === dateComponent.day
);
}
function convertDateComponentToNumber(
dateComponent: DateComponent<string>,
): DateComponent<number> {
return {
year: Number(dateComponent.year),
// https://stackoverflow.com/questions/2552483/why-does-the-month-argument-range-from-0-to-11-in-javascripts-date-constructor
month: Number(dateComponent.month) - 1,
day: Number(dateComponent.day),
hour: Number(dateComponent.hour),
minute: Number(dateComponent.minute),
second: Number(dateComponent.second),
};
}
function getDateFromComponents(dateComponent: DateComponent<number>) {
const { year, month, day, hour, minute, second } = dateComponent;
if (hasTimeValues(dateComponent)) {
return new Date(year, month, day, hour, minute, second);
} else {
return new Date(year, month, day);
}
}
function hasTimeValues(dateComponent: DateComponent<number>) {
const { hour, minute, second } = dateComponent;
return !isNaN(hour) && !isNaN(minute) && !isNaN(second);
}
function removeTimeValues(
dateComponent: DateComponent<number>,
): DateComponent<number> {
return { ...dateComponent, hour: 0, minute: 0, second: 0 };
}

317
web/apps/photos/src/services/upload/metadata.ts
View file

@ -1,317 +0,0 @@
import { FILE_TYPE, type FileTypeInfo } from "@/media/file-type";
import { getFileNameSize } from "@/next/file";
import log from "@/next/log";
import { ElectronFile } from "@/next/types/file";
import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker";
import { CustomError } from "@ente/shared/error";
import {
parseDateFromFusedDateString,
tryToParseDateTime,
validateAndGetCreationUnixTimeInMicroSeconds,
} from "@ente/shared/time";
import type { DataStream } from "@ente/shared/utils/data-stream";
import { Remote } from "comlink";
import { FILE_READER_CHUNK_SIZE, NULL_LOCATION } from "constants/upload";
import { getEXIFLocation, getEXIFTime, getParsedExifData } from "services/exif";
import * as ffmpegService from "services/ffmpeg";
import { getElectronFileStream, getFileStream } from "services/readerService";
import { FilePublicMagicMetadataProps } from "types/file";
import {
Metadata,
ParsedExtractedMetadata,
type LivePhotoAssets2,
type UploadAsset2,
} from "types/upload";
import {
MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT,
getClippedMetadataJSONMapKeyForFile,
getMetadataJSONMapKeyForFile,
type ParsedMetadataJSON,
} from "./takeout";
import { getFileName } from "./uploadService";
const EXIF_TAGS_NEEDED = [
"DateTimeOriginal",
"CreateDate",
"ModifyDate",
"GPSLatitude",
"GPSLongitude",
"GPSLatitudeRef",
"GPSLongitudeRef",
"DateCreated",
"ExifImageWidth",
"ExifImageHeight",
"ImageWidth",
"ImageHeight",
"PixelXDimension",
"PixelYDimension",
"MetadataDate",
];
const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = {
location: NULL_LOCATION,
creationTime: null,
width: null,
height: null,
};
interface ExtractMetadataResult {
metadata: Metadata;
publicMagicMetadata: FilePublicMagicMetadataProps;
}
export const extractAssetMetadata = async (
worker: Remote<DedicatedCryptoWorker>,
parsedMetadataJSONMap: Map<string, ParsedMetadataJSON>,
{ isLivePhoto, file, livePhotoAssets }: UploadAsset2,
collectionID: number,
fileTypeInfo: FileTypeInfo,
): Promise<ExtractMetadataResult> => {
return isLivePhoto
? await extractLivePhotoMetadata(
worker,
parsedMetadataJSONMap,
collectionID,
fileTypeInfo,
livePhotoAssets,
)
: await extractFileMetadata(
worker,
parsedMetadataJSONMap,
collectionID,
fileTypeInfo,
file,
);
};
async function extractFileMetadata(
worker: Remote<DedicatedCryptoWorker>,
parsedMetadataJSONMap: Map<string, ParsedMetadataJSON>,
collectionID: number,
fileTypeInfo: FileTypeInfo,
rawFile: File | ElectronFile | string,
): Promise<ExtractMetadataResult> {
const rawFileName = getFileName(rawFile);
let key = getMetadataJSONMapKeyForFile(collectionID, rawFileName);
let googleMetadata: ParsedMetadataJSON = parsedMetadataJSONMap.get(key);
if (!googleMetadata && key.length > MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT) {
key = getClippedMetadataJSONMapKeyForFile(collectionID, rawFileName);
googleMetadata = parsedMetadataJSONMap.get(key);
}
const { metadata, publicMagicMetadata } = await extractMetadata(
worker,
/* TODO(MR): ElectronFile changes */
rawFile as File | ElectronFile,
fileTypeInfo,
);
for (const [key, value] of Object.entries(googleMetadata ?? {})) {
if (!value) {
continue;
}
metadata[key] = value;
}
return { metadata, publicMagicMetadata };
}
async function extractMetadata(
worker: Remote<DedicatedCryptoWorker>,
receivedFile: File | ElectronFile,
fileTypeInfo: FileTypeInfo,
): Promise<ExtractMetadataResult> {
let extractedMetadata: ParsedExtractedMetadata = NULL_EXTRACTED_METADATA;
if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) {
extractedMetadata = await getImageMetadata(receivedFile, fileTypeInfo);
} else if (fileTypeInfo.fileType === FILE_TYPE.VIDEO) {
extractedMetadata = await getVideoMetadata(receivedFile);
}
const hash = await getFileHash(worker, receivedFile);
const metadata: Metadata = {
title: receivedFile.name,
creationTime:
extractedMetadata.creationTime ??
extractDateFromFileName(receivedFile.name) ??
receivedFile.lastModified * 1000,
modificationTime: receivedFile.lastModified * 1000,
latitude: extractedMetadata.location.latitude,
longitude: extractedMetadata.location.longitude,
fileType: fileTypeInfo.fileType,
hash,
};
const publicMagicMetadata: FilePublicMagicMetadataProps = {
w: extractedMetadata.width,
h: extractedMetadata.height,
};
return { metadata, publicMagicMetadata };
}
async function getImageMetadata(
receivedFile: File | ElectronFile,
fileTypeInfo: FileTypeInfo,
): Promise<ParsedExtractedMetadata> {
let imageMetadata = NULL_EXTRACTED_METADATA;
try {
if (!(receivedFile instanceof File)) {
receivedFile = new File(
[await receivedFile.blob()],
receivedFile.name,
{
lastModified: receivedFile.lastModified,
},
);
}
const exifData = await getParsedExifData(
receivedFile,
fileTypeInfo,
EXIF_TAGS_NEEDED,
);
imageMetadata = {
location: getEXIFLocation(exifData),
creationTime: getEXIFTime(exifData),
width: exifData?.imageWidth ?? null,
height: exifData?.imageHeight ?? null,
};
} catch (e) {
log.error("getExifData failed", e);
}
return imageMetadata;
}
// tries to extract date from file name if available else returns null
function extractDateFromFileName(filename: string): number {
try {
filename = filename.trim();
let parsedDate: Date;
if (filename.startsWith("IMG-") || filename.startsWith("VID-")) {
// Whatsapp media files
// sample name IMG-20171218-WA0028.jpg
parsedDate = parseDateFromFusedDateString(filename.split("-")[1]);
} else if (filename.startsWith("Screenshot_")) {
// Screenshots on droid
// sample name Screenshot_20181227-152914.jpg
parsedDate = parseDateFromFusedDateString(
filename.replaceAll("Screenshot_", ""),
);
} else if (filename.startsWith("signal-")) {
// signal images
// sample name :signal-2018-08-21-100217.jpg
const dateString = convertSignalNameToFusedDateString(filename);
parsedDate = parseDateFromFusedDateString(dateString);
}
if (!parsedDate) {
parsedDate = tryToParseDateTime(filename);
}
return validateAndGetCreationUnixTimeInMicroSeconds(parsedDate);
} catch (e) {
log.error("failed to extract date From FileName ", e);
return null;
}
}
function convertSignalNameToFusedDateString(filename: string) {
const dateStringParts = filename.split("-");
return `${dateStringParts[1]}${dateStringParts[2]}${dateStringParts[3]}-${dateStringParts[4]}`;
}
async function getVideoMetadata(file: File | ElectronFile) {
let videoMetadata = NULL_EXTRACTED_METADATA;
try {
log.info(`getVideoMetadata called for ${getFileNameSize(file)}`);
videoMetadata = await ffmpegService.extractVideoMetadata(file);
log.info(
`videoMetadata successfully extracted ${getFileNameSize(file)}`,
);
} catch (e) {
log.error("failed to get video metadata", e);
log.info(
`videoMetadata extracted failed ${getFileNameSize(file)} ,${
e.message
} `,
);
}
return videoMetadata;
}
async function extractLivePhotoMetadata(
worker: Remote<DedicatedCryptoWorker>,
parsedMetadataJSONMap: Map<string, ParsedMetadataJSON>,
collectionID: number,
fileTypeInfo: FileTypeInfo,
livePhotoAssets: LivePhotoAssets2,
): Promise<ExtractMetadataResult> {
const imageFileTypeInfo: FileTypeInfo = {
fileType: FILE_TYPE.IMAGE,
exactType: fileTypeInfo.imageType,
};
const {
metadata: imageMetadata,
publicMagicMetadata: imagePublicMagicMetadata,
} = await extractFileMetadata(
worker,
parsedMetadataJSONMap,
collectionID,
imageFileTypeInfo,
livePhotoAssets.image,
);
const videoHash = await getFileHash(
worker,
/* TODO(MR): ElectronFile changes */
livePhotoAssets.video as File | ElectronFile,
);
return {
metadata: {
...imageMetadata,
title: getFileName(livePhotoAssets.image),
fileType: FILE_TYPE.LIVE_PHOTO,
imageHash: imageMetadata.hash,
videoHash: videoHash,
hash: undefined,
},
publicMagicMetadata: imagePublicMagicMetadata,
};
}
async function getFileHash(
worker: Remote<DedicatedCryptoWorker>,
file: File | ElectronFile,
) {
try {
log.info(`getFileHash called for ${getFileNameSize(file)}`);
let filedata: DataStream;
if (file instanceof File) {
filedata = getFileStream(file, FILE_READER_CHUNK_SIZE);
} else {
filedata = await getElectronFileStream(
file,
FILE_READER_CHUNK_SIZE,
);
}
const hashState = await worker.initChunkHashing();
const streamReader = filedata.stream.getReader();
for (let i = 0; i < filedata.chunkCount; i++) {
const { done, value: chunk } = await streamReader.read();
if (done) {
throw Error(CustomError.CHUNK_LESS_THAN_EXPECTED);
}
await worker.hashFileChunk(hashState, Uint8Array.from(chunk));
}
const { done } = await streamReader.read();
if (!done) {
throw Error(CustomError.CHUNK_MORE_THAN_EXPECTED);
}
const hash = await worker.completeChunkHashing(hashState);
log.info(
`file hashing completed successfully ${getFileNameSize(file)}`,
);
return hash;
} catch (e) {
log.error("getFileHash failed", e);
log.info(`file hashing failed ${getFileNameSize(file)} ,${e.message} `);
}
}

22
web/apps/photos/src/services/upload/takeout.ts
View file

@ -111,7 +111,7 @@ const parseMetadataJSONText = (text: string) => {
return undefined;
}
const parsedMetadataJSON: ParsedMetadataJSON = NULL_PARSED_METADATA_JSON;
const parsedMetadataJSON = { ...NULL_PARSED_METADATA_JSON };
if (
metadataJSON["photoTakenTime"] &&
@ -153,3 +153,23 @@ const parseMetadataJSONText = (text: string) => {
}
return parsedMetadataJSON;
};
/**
* Return the matching entry (if any) from {@link parsedMetadataJSONMap} for the
* {@link fileName} and {@link collectionID} combination.
*/
export const matchTakeoutMetadata = (
fileName: string,
collectionID: number,
parsedMetadataJSONMap: Map<string, ParsedMetadataJSON>,
) => {
let key = getMetadataJSONMapKeyForFile(collectionID, fileName);
let takeoutMetadata = parsedMetadataJSONMap.get(key);
if (!takeoutMetadata && key.length > MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT) {
key = getClippedMetadataJSONMapKeyForFile(collectionID, fileName);
takeoutMetadata = parsedMetadataJSONMap.get(key);
}
return takeoutMetadata;
};

5
web/apps/photos/src/services/upload/thumbnail.ts
View file

@ -5,7 +5,6 @@ import { withTimeout } from "@ente/shared/utils";
import { BLACK_THUMBNAIL_BASE64 } from "constants/upload";
import * as ffmpeg from "services/ffmpeg";
import { heicToJPEG } from "services/heic-convert";
import { isFileHEIC } from "utils/file";
/** Maximum width or height of the generated thumbnail */
const maxThumbnailDimension = 720;
@ -36,9 +35,9 @@ export const generateThumbnailWeb = async (
const generateImageThumbnailUsingCanvas = async (
blob: Blob,
fileTypeInfo: FileTypeInfo,
{ extension }: FileTypeInfo,
) => {
if (isFileHEIC(fileTypeInfo.exactType)) {
if (extension == "heic" || extension == "heif") {
log.debug(() => `Pre-converting HEIC to JPEG for thumbnail generation`);
blob = await heicToJPEG(blob);
}

342
web/apps/photos/src/services/upload/uploadManager.ts
View file

@ -1,10 +1,11 @@
import { FILE_TYPE } from "@/media/file-type";
import { potentialFileTypeFromExtension } from "@/media/live-photo";
import { ensureElectron } from "@/next/electron";
import { nameAndExtension } from "@/next/file";
import { lowercaseExtension, nameAndExtension } from "@/next/file";
import log from "@/next/log";
import { ElectronFile } from "@/next/types/file";
import { ComlinkWorker } from "@/next/worker/comlink-worker";
import { ensure } from "@/utils/ensure";
import { getDedicatedCryptoWorker } from "@ente/shared/crypto";
import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker";
import { CustomError } from "@ente/shared/error";
@ -41,7 +42,6 @@ import {
SegregatedFinishedUploads,
} from "types/upload/ui";
import { decryptFile, getUserOwnedFiles, sortFiles } from "utils/file";
import { segregateMetadataAndMediaFiles } from "utils/upload";
import { getLocalFiles } from "../fileService";
import {
getMetadataJSONMapKeyForJSON,
@ -50,7 +50,7 @@ import {
} from "./takeout";
import UploadService, {
assetName,
getAssetName,
fopSize,
getFileName,
uploader,
} from "./uploadService";
@ -302,6 +302,7 @@ class UploadManager {
constructor() {
this.uiService = new UIService();
}
public async init(
progressUpdater: ProgressUpdater,
setFiles: SetFiles,
@ -333,7 +334,7 @@ class UploadManager {
this.uploaderName = null;
}
prepareForNewUpload() {
public prepareForNewUpload() {
this.resetState();
this.uiService.reset();
uploadCancelService.reset();
@ -344,78 +345,58 @@ class UploadManager {
this.uiService.setUploadProgressView(true);
}
async updateExistingFilesAndCollections(collections: Collection[]) {
if (this.publicUploadProps.accessedThroughSharedURL) {
this.existingFiles = await getLocalPublicFiles(
getPublicCollectionUID(this.publicUploadProps.token),
);
} else {
this.existingFiles = getUserOwnedFiles(await getLocalFiles());
}
this.collections = new Map(
collections.map((collection) => [collection.id, collection]),
);
}
public async queueFilesForUpload(
filesWithCollectionToUploadIn: FileWithCollection[],
collections: Collection[],
uploaderName?: string,
) {
try {
if (this.uploadInProgress) {
throw Error("can't run multiple uploads at once");
}
if (this.uploadInProgress)
throw new Error("Cannot run multiple uploads at once");
log.info(`Uploading ${filesWithCollectionToUploadIn.length} files`);
this.uploadInProgress = true;
await this.updateExistingFilesAndCollections(collections);
this.uploaderName = uploaderName;
log.info(
`received ${filesWithCollectionToUploadIn.length} files to upload`,
);
await this.updateExistingFilesAndCollections(collections);
const namedFiles: FileWithCollectionIDAndName[] =
filesWithCollectionToUploadIn.map(
makeFileWithCollectionIDAndName,
);
this.uiService.setFilenames(
new Map<number, string>(
filesWithCollectionToUploadIn.map((mediaFile) => [
mediaFile.localID,
getAssetName(mediaFile),
]),
namedFiles.map((f) => [f.localID, f.fileName]),
),
);
const { metadataJSONFiles, mediaFiles } =
segregateMetadataAndMediaFiles(filesWithCollectionToUploadIn);
log.info(`has ${metadataJSONFiles.length} metadata json files`);
log.info(`has ${mediaFiles.length} media files`);
if (metadataJSONFiles.length) {
const [metadataFiles, mediaFiles] =
splitMetadataAndMediaFiles(namedFiles);
if (metadataFiles.length) {
this.uiService.setUploadStage(
UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES,
);
/* TODO(MR): ElectronFile changes */
await this.parseMetadataJSONFiles(
metadataJSONFiles as FileWithCollection2[],
);
await this.parseMetadataJSONFiles(metadataFiles);
}
if (mediaFiles.length) {
/* TODO(MR): ElectronFile changes */
const clusteredMediaFiles = clusterLivePhotos(
mediaFiles as ClusterableFile[],
);
const clusteredMediaFiles = await clusterLivePhotos(mediaFiles);
if (uploadCancelService.isUploadCancelationRequested()) {
throw Error(CustomError.UPLOAD_CANCELLED);
}
this.abortIfCancelled();
this.uiService.setFilenames(
new Map<number, string>(
clusteredMediaFiles.map((mediaFile) => [
mediaFile.localID,
/* TODO(MR): ElectronFile changes */
assetName(mediaFile as FileWithCollection2),
clusteredMediaFiles.map((file) => [
file.localID,
file.fileName,
]),
),
);
this.uiService.setHasLivePhoto(
mediaFiles.length !== clusteredMediaFiles.length,
mediaFiles.length != clusteredMediaFiles.length,
);
/* TODO(MR): ElectronFile changes */
@ -458,50 +439,38 @@ class UploadManager {
}
};
private async parseMetadataJSONFiles(metadataFiles: FileWithCollection2[]) {
try {
log.info(`parseMetadataJSONFiles function executed `);
private async updateExistingFilesAndCollections(collections: Collection[]) {
if (this.publicUploadProps.accessedThroughSharedURL) {
this.existingFiles = await getLocalPublicFiles(
getPublicCollectionUID(this.publicUploadProps.token),
);
} else {
this.existingFiles = getUserOwnedFiles(await getLocalFiles());
}
this.collections = new Map(
collections.map((collection) => [collection.id, collection]),
);
}
this.uiService.reset(metadataFiles.length);
private async parseMetadataJSONFiles(files: FileWithCollectionIDAndName[]) {
this.uiService.reset(files.length);
for (const { file, collectionID } of metadataFiles) {
this.abortIfCancelled();
const name = getFileName(file);
try {
log.info(`parsing metadata json file ${name}`);
for (const { file, fileName, collectionID } of files) {
this.abortIfCancelled();
const metadataJSON =
await tryParseTakeoutMetadataJSON(file);
if (metadataJSON) {
this.parsedMetadataJSONMap.set(
getMetadataJSONMapKeyForJSON(collectionID, name),
metadataJSON && { ...metadataJSON },
);
this.uiService.increaseFileUploaded();
}
log.info(`successfully parsed metadata json file ${name}`);
} catch (e) {
if (e.message === CustomError.UPLOAD_CANCELLED) {
throw e;
} else {
// and don't break for subsequent files just log and move on
log.error("parsing failed for a file", e);
log.info(
`failed to parse metadata json file ${name} error: ${e.message}`,
);
}
}
log.info(`Parsing metadata JSON ${fileName}`);
const metadataJSON = await tryParseTakeoutMetadataJSON(file);
if (metadataJSON) {
this.parsedMetadataJSONMap.set(
getMetadataJSONMapKeyForJSON(collectionID, fileName),
metadataJSON,
);
this.uiService.increaseFileUploaded();
}
} catch (e) {
if (e.message !== CustomError.UPLOAD_CANCELLED) {
log.error("error seeding MetadataMap", e);
}
throw e;
}
}
private async uploadMediaFiles(mediaFiles: FileWithCollection2[]) {
log.info(`uploadMediaFiles called`);
this.filesToBeUploaded = [...this.filesToBeUploaded, ...mediaFiles];
if (isElectron()) {
@ -531,9 +500,8 @@ class UploadManager {
const uiService = this.uiService;
while (this.filesToBeUploaded.length > 0) {
if (uploadCancelService.isUploadCancelationRequested()) {
throw Error(CustomError.UPLOAD_CANCELLED);
}
this.abortIfCancelled();
let fileWithCollection = this.filesToBeUploaded.pop();
const { collectionID } = fileWithCollection;
const collection = this.collections.get(collectionID);
@ -543,11 +511,11 @@ class UploadManager {
await wait(0);
const { fileUploadResult, uploadedFile } = await uploader(
worker,
this.existingFiles,
fileWithCollection,
this.parsedMetadataJSONMap,
this.uploaderName,
this.existingFiles,
this.parsedMetadataJSONMap,
worker,
this.isCFUploadProxyDisabled,
() => {
this.abortIfCancelled();
@ -579,16 +547,14 @@ class UploadManager {
}
}
async postUploadTask(
private async postUploadTask(
fileUploadResult: UPLOAD_RESULT,
uploadedFile: EncryptedEnteFile | EnteFile | null,
fileWithCollection: FileWithCollection2,
) {
try {
let decryptedFile: EnteFile;
log.info(
`post upload action -> fileUploadResult: ${fileUploadResult} uploadedFile present ${!!uploadedFile}`,
);
log.info(`Upload completed with result: ${fileUploadResult}`);
await this.removeFromPendingUploads(fileWithCollection);
switch (fileUploadResult) {
case UPLOAD_RESULT.FAILED:
@ -614,7 +580,9 @@ class UploadManager {
// no-op
break;
default:
throw Error("Invalid Upload Result" + fileUploadResult);
throw new Error(
`Invalid Upload Result ${fileUploadResult}`,
);
}
if (
[
@ -631,7 +599,7 @@ class UploadManager {
fileWithCollection.livePhotoAssets.image,
});
} catch (e) {
log.error("Error in fileUploaded handlers", e);
log.warn("Ignoring error in fileUploaded handlers", e);
}
this.updateExistingFiles(decryptedFile);
}
@ -664,19 +632,19 @@ class UploadManager {
}
public cancelRunningUpload() {
log.info("user cancelled running upload");
log.info("User cancelled running upload");
this.uiService.setUploadStage(UPLOAD_STAGES.CANCELLING);
uploadCancelService.requestUploadCancelation();
}
getFailedFilesWithCollections() {
public getFailedFilesWithCollections() {
return {
files: this.failedFiles,
collections: [...this.collections.values()],
};
}
getUploaderName() {
public getUploaderName() {
return this.uploaderName;
}
@ -708,6 +676,101 @@ class UploadManager {
export default new UploadManager();
/**
* The data operated on by the intermediate stages of the upload.
*
* [Note: Intermediate file types during upload]
*
* As files progress through stages, they get more and more bits tacked on to
* them. These types document the journey.
*
* - The input is {@link FileWithCollection}. This can either be a new
* {@link FileWithCollection}, in which case it'll only have a
* {@link localID}, {@link collectionID} and a {@link file}. Or it could be a
* retry, in which case it'll not have a {@link file} but instead will have
* data from a previous stage, like a snake eating its tail.
*
* - Immediately we convert it to {@link FileWithCollectionIDAndName}. This is
* to mostly systematize what we have, and also attach a {@link fileName}.
*
* - These then get converted to "assets", whereby both parts of a live photo
* are combined. This is a {@link ClusteredFile}.
*
* - On to the {@link ClusteredFile} we attach the corresponding
* {@link collection}, giving us {@link UploadableFile}. This is what gets
* queued and then passed to the {@link uploader}.
*/
type FileWithCollectionIDAndName = {
/** A unique ID for the duration of the upload */
localID: number;
/** The ID of the collection to which this file should be uploaded. */
collectionID: number;
/**
* The name of the file.
*
* In case of live photos, this'll be the name of the image part.
*/
fileName: string;
/** `true` if this is a live photo. */
isLivePhoto?: boolean;
/* Valid for non-live photos */
fileOrPath?: File | string;
/** Alias */
file?: File | string;
/* Valid for live photos */
livePhotoAssets?: LivePhotoAssets2;
};
const makeFileWithCollectionIDAndName = (
f: FileWithCollection,
): FileWithCollectionIDAndName => {
/* TODO(MR): ElectronFile */
const fileOrPath = (f.fileOrPath ?? f.file) as File | string;
if (!(fileOrPath instanceof File || typeof fileOrPath == "string"))
throw new Error(`Unexpected file ${f}`);
return {
localID: ensure(f.localID),
collectionID: ensure(f.collectionID),
fileName: ensure(
f.isLivePhoto
? getFileName(f.livePhotoAssets.image)
: getFileName(fileOrPath),
),
isLivePhoto: f.isLivePhoto,
/* TODO(MR): ElectronFile */
file: fileOrPath,
fileOrPath: fileOrPath,
/* TODO(MR): ElectronFile */
livePhotoAssets: f.livePhotoAssets as LivePhotoAssets2,
};
};
type ClusteredFile = {
localID: number;
collectionID: number;
fileName: string;
isLivePhoto: boolean;
file?: File | string;
livePhotoAssets?: LivePhotoAssets2;
};
const splitMetadataAndMediaFiles = (
files: FileWithCollectionIDAndName[],
): [
metadata: FileWithCollectionIDAndName[],
media: FileWithCollectionIDAndName[],
] =>
files.reduce(
([metadata, media], file) => {
if (lowercaseExtension(file.fileName) == "json")
metadata.push(file);
else media.push(file);
return [metadata, media];
},
[[], []],
);
export const setToUploadCollection = async (collections: Collection[]) => {
let collectionName: string = null;
/* collection being one suggest one of two things
@ -751,73 +814,50 @@ const cancelRemainingUploads = async () => {
await electron.setPendingUploadFiles("files", []);
};
/**
* The data needed by {@link clusterLivePhotos} to do its thing.
*
* As files progress through stages, they get more and more bits tacked on to
* them. These types document the journey.
*/
type ClusterableFile = {
localID: number;
collectionID: number;
// fileOrPath: File | ElectronFile | string;
file?: File | ElectronFile | string;
};
type ClusteredFile = ClusterableFile & {
isLivePhoto: boolean;
livePhotoAssets?: LivePhotoAssets2;
};
/**
* Go through the given files, combining any sibling image + video assets into a
* single live photo when appropriate.
*/
const clusterLivePhotos = (mediaFiles: ClusterableFile[]) => {
const clusterLivePhotos = async (files: FileWithCollectionIDAndName[]) => {
const result: ClusteredFile[] = [];
mediaFiles
files
.sort((f, g) =>
nameAndExtension(getFileName(f.file))[0].localeCompare(
nameAndExtension(getFileName(g.file))[0],
nameAndExtension(f.fileName)[0].localeCompare(
nameAndExtension(g.fileName)[0],
),
)
.sort((f, g) => f.collectionID - g.collectionID);
let index = 0;
while (index < mediaFiles.length - 1) {
const f = mediaFiles[index];
const g = mediaFiles[index + 1];
const fFileName = getFileName(f.file);
const gFileName = getFileName(g.file);
const fFileType = potentialFileTypeFromExtension(fFileName);
const gFileType = potentialFileTypeFromExtension(gFileName);
while (index < files.length - 1) {
const f = files[index];
const g = files[index + 1];
const fFileType = potentialFileTypeFromExtension(f.fileName);
const gFileType = potentialFileTypeFromExtension(g.fileName);
const fa: PotentialLivePhotoAsset = {
fileName: fFileName,
fileName: f.fileName,
fileType: fFileType,
collectionID: f.collectionID,
/* TODO(MR): ElectronFile changes */
size: (f as FileWithCollection).file.size,
fileOrPath: f.file,
};
const ga: PotentialLivePhotoAsset = {
fileName: gFileName,
fileName: g.fileName,
fileType: gFileType,
collectionID: g.collectionID,
/* TODO(MR): ElectronFile changes */
size: (g as FileWithCollection).file.size,
fileOrPath: g.file,
};
if (areLivePhotoAssets(fa, ga)) {
result.push({
if (await areLivePhotoAssets(fa, ga)) {
const livePhoto = {
localID: f.localID,
collectionID: f.collectionID,
isLivePhoto: true,
livePhotoAssets: {
/* TODO(MR): ElectronFile changes */
image: (fFileType == FILE_TYPE.IMAGE ? f.file : g.file) as
| string
| File,
video: (fFileType == FILE_TYPE.IMAGE ? g.file : f.file) as
| string
| File,
image: fFileType == FILE_TYPE.IMAGE ? f.file : g.file,
video: fFileType == FILE_TYPE.IMAGE ? g.file : f.file,
},
};
result.push({
...livePhoto,
fileName: assetName(livePhoto),
});
index += 2;
} else {
@ -828,9 +868,9 @@ const clusterLivePhotos = (mediaFiles: ClusterableFile[]) => {
index += 1;
}
}
if (index === mediaFiles.length - 1) {
if (index === files.length - 1) {
result.push({
...mediaFiles[index],
...files[index],
isLivePhoto: false,
});
}
@ -841,10 +881,10 @@ interface PotentialLivePhotoAsset {
fileName: string;
fileType: FILE_TYPE;
collectionID: number;
size: number;
fileOrPath: File | string;
}
const areLivePhotoAssets = (
const areLivePhotoAssets = async (
f: PotentialLivePhotoAsset,
g: PotentialLivePhotoAsset,
) => {
@ -884,9 +924,11 @@ const areLivePhotoAssets = (
// we use doesn't support stream as a input.
const maxAssetSize = 20 * 1024 * 1024; /* 20MB */
if (f.size > maxAssetSize || g.size > maxAssetSize) {
const fSize = await fopSize(f.fileOrPath);
const gSize = await fopSize(g.fileOrPath);
if (fSize > maxAssetSize || gSize > maxAssetSize) {
log.info(
`Not classifying assets with too large sizes ${[f.size, g.size]} as a live photo`,
`Not classifying assets with too large sizes ${[fSize, gSize]} as a live photo`,
);
return false;
}

805
web/apps/photos/src/services/upload/uploadService.ts
View file

@ -1,10 +1,12 @@
import { FILE_TYPE, type FileTypeInfo } from "@/media/file-type";
import { encodeLivePhoto } from "@/media/live-photo";
import type { Metadata } from "@/media/types/file";
import { ensureElectron } from "@/next/electron";
import { basename } from "@/next/file";
import log from "@/next/log";
import { ElectronFile } from "@/next/types/file";
import { CustomErrorMessage } from "@/next/types/ipc";
import { ensure } from "@/utils/ensure";
import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker";
import { EncryptionResult } from "@ente/shared/crypto/types";
import { CustomError, handleUploadError } from "@ente/shared/error";
@ -14,10 +16,13 @@ import {
FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART,
FILE_READER_CHUNK_SIZE,
MULTIPART_PART_SIZE,
NULL_LOCATION,
RANDOM_PERCENTAGE_PROGRESS_FOR_PUT,
UPLOAD_RESULT,
} from "constants/upload";
import { addToCollection } from "services/collectionService";
import { parseImageMetadata } from "services/exif";
import * as ffmpeg from "services/ffmpeg";
import {
EnteFile,
type FilePublicMagicMetadata,
@ -29,15 +34,14 @@ import {
EncryptedFile,
FileInMemory,
FileWithMetadata,
ParsedExtractedMetadata,
ProcessedFile,
PublicUploadProps,
UploadAsset,
UploadFile,
UploadURL,
type FileWithCollection2,
type LivePhotoAssets,
type LivePhotoAssets2,
type Metadata,
type UploadAsset2,
} from "types/upload";
import {
@ -47,11 +51,12 @@ import {
import { readStream } from "utils/native-stream";
import { hasFileHash } from "utils/upload";
import * as convert from "xml-js";
import { detectFileTypeInfoFromChunk } from "../detect-type";
import { getFileStream } from "../readerService";
import { getFileType } from "../typeDetectionService";
import { extractAssetMetadata } from "./metadata";
import { tryParseEpochMicrosecondsFromFileName } from "./date";
import publicUploadHttpClient from "./publicUploadHttpClient";
import type { ParsedMetadataJSON } from "./takeout";
import { matchTakeoutMetadata } from "./takeout";
import {
fallbackThumbnail,
generateThumbnailNative,
@ -160,38 +165,52 @@ interface UploadResponse {
}
export const uploader = async (
worker: Remote<DedicatedCryptoWorker>,
existingFiles: EnteFile[],
fileWithCollection: FileWithCollection2,
parsedMetadataJSONMap: Map<string, ParsedMetadataJSON>,
uploaderName: string,
existingFiles: EnteFile[],
parsedMetadataJSONMap: Map<string, ParsedMetadataJSON>,
worker: Remote<DedicatedCryptoWorker>,
isCFUploadProxyDisabled: boolean,
abortIfCancelled: () => void,
makeProgessTracker: MakeProgressTracker,
): Promise<UploadResponse> => {
const name = assetName(fileWithCollection);
const { collection, localID, ...uploadAsset } = fileWithCollection;
const name = assetName(uploadAsset);
log.info(`Uploading ${name}`);
const { collection, localID, ...uploadAsset2 } = fileWithCollection;
/* TODO(MR): ElectronFile changes */
const uploadAsset = uploadAsset2 as UploadAsset;
let fileTypeInfo: FileTypeInfo;
let fileSize: number;
try {
const maxFileSize = 4 * 1024 * 1024 * 1024; // 4 GB
/*
* We read the file four times:
* 1. To determine its MIME type (only needs first few KBs).
* 2. To extract its metadata.
* 3. To calculate its hash.
* 4. To encrypt it.
*
* When we already have a File object the multiple reads are fine.
*
* When we're in the context of our desktop app and have a path, it
* might be possible to optimize further by using `ReadableStream.tee`
* to perform these steps simultaneously. However, that'll require
* restructuring the code so that these steps run in a parallel manner
* (tee will not work for strictly sequential reads of large streams).
*/
fileSize = getAssetSize(uploadAsset);
if (fileSize >= maxFileSize) {
const { fileTypeInfo, fileSize, lastModifiedMs } =
await readAssetDetails(uploadAsset);
const maxFileSize = 4 * 1024 * 1024 * 1024; /* 4 GB */
if (fileSize >= maxFileSize)
return { fileUploadResult: UPLOAD_RESULT.TOO_LARGE };
}
fileTypeInfo = await getAssetFileType(uploadAsset);
abortIfCancelled();
const { metadata, publicMagicMetadata } = await extractAssetMetadata(
worker,
parsedMetadataJSONMap,
uploadAsset2,
collection.id,
uploadAsset,
fileTypeInfo,
lastModifiedMs,
collection.id,
parsedMetadataJSONMap,
worker,
);
const matches = existingFiles.filter((file) =>
@ -223,9 +242,12 @@ export const uploader = async (
abortIfCancelled();
const file = await readAsset(fileTypeInfo, uploadAsset2);
const { filedata, thumbnail, hasStaticThumbnail } = await readAsset(
fileTypeInfo,
uploadAsset,
);
if (file.hasStaticThumbnail) metadata.hasStaticThumbnail = true;
if (hasStaticThumbnail) metadata.hasStaticThumbnail = true;
const pubMagicMetadata = await constructPublicMagicMetadata({
...publicMagicMetadata,
@ -236,16 +258,16 @@ export const uploader = async (
const fileWithMetadata: FileWithMetadata = {
localID,
filedata: file.filedata,
thumbnail: file.thumbnail,
filedata,
thumbnail,
metadata,
pubMagicMetadata,
};
const encryptedFile = await encryptFile(
worker,
fileWithMetadata,
collection.key,
worker,
);
abortIfCancelled();
@ -296,12 +318,20 @@ export const uploader = async (
}
};
/**
* Return the size of the given file
*
* @param fileOrPath The {@link File}, or the path to it. Note that it is only
* valid to specify a path if we are running in the context of our desktop app.
*/
export const fopSize = async (fileOrPath: File | string): Promise<number> =>
fileOrPath instanceof File
? fileOrPath.size
: await ensureElectron().fs.size(fileOrPath);
export const getFileName = (file: File | ElectronFile | string) =>
typeof file == "string" ? basename(file) : file.name;
function getFileSize(file: File | ElectronFile) {
return file.size;
}
export const getAssetName = ({
isLivePhoto,
file,
@ -316,67 +346,10 @@ export const assetName = ({
}: UploadAsset2) =>
isLivePhoto ? getFileName(livePhotoAssets.image) : getFileName(file);
const getAssetSize = ({ isLivePhoto, file, livePhotoAssets }: UploadAsset) => {
return isLivePhoto ? getLivePhotoSize(livePhotoAssets) : getFileSize(file);
};
const getLivePhotoSize = (livePhotoAssets: LivePhotoAssets) => {
return livePhotoAssets.image.size + livePhotoAssets.video.size;
};
const getAssetFileType = ({
isLivePhoto,
file,
livePhotoAssets,
}: UploadAsset) => {
return isLivePhoto
? getLivePhotoFileType(livePhotoAssets)
: getFileType(file);
};
const getLivePhotoFileType = async (
livePhotoAssets: LivePhotoAssets,
): Promise<FileTypeInfo> => {
const imageFileTypeInfo = await getFileType(livePhotoAssets.image);
const videoFileTypeInfo = await getFileType(livePhotoAssets.video);
return {
fileType: FILE_TYPE.LIVE_PHOTO,
exactType: `${imageFileTypeInfo.exactType}+${videoFileTypeInfo.exactType}`,
imageType: imageFileTypeInfo.exactType,
videoType: videoFileTypeInfo.exactType,
};
};
const readAsset = async (
fileTypeInfo: FileTypeInfo,
{ isLivePhoto, file, livePhotoAssets }: UploadAsset2,
) => {
return isLivePhoto
? await readLivePhoto(livePhotoAssets, fileTypeInfo)
: await readImageOrVideo(file, fileTypeInfo);
};
// TODO(MR): Merge with the uploader
class ModuleState {
/**
* This will be set to true if we get an error from the Node.js side of our
* desktop app telling us that native image thumbnail generation is not
* available for the current OS/arch combination.
*
* That way, we can stop pestering it again and again (saving an IPC
* round-trip).
*
* Note the double negative when it is used.
*/
isNativeImageThumbnailGenerationNotAvailable = false;
}
const moduleState = new ModuleState();
/**
* Read the given file or path into an in-memory representation.
*
* [Note: The fileOrPath parameter to upload]
* See: [Note: Reading a fileOrPath]
*
* The file can be either a web
* [File](https://developer.mozilla.org/en-US/docs/Web/API/File) or the absolute
@ -423,21 +396,32 @@ const moduleState = new ModuleState();
*/
const readFileOrPath = async (
fileOrPath: File | string,
): Promise<{ dataOrStream: Uint8Array | DataStream; fileSize: number }> => {
): Promise<{
dataOrStream: Uint8Array | DataStream;
fileSize: number;
lastModifiedMs: number;
}> => {
let dataOrStream: Uint8Array | DataStream;
let fileSize: number;
let lastModifiedMs: number;
if (fileOrPath instanceof File) {
const file = fileOrPath;
fileSize = file.size;
lastModifiedMs = file.lastModified;
dataOrStream =
fileSize > MULTIPART_PART_SIZE
? getFileStream(file, FILE_READER_CHUNK_SIZE)
: new Uint8Array(await file.arrayBuffer());
} else {
const path = fileOrPath;
const { response, size } = await readStream(ensureElectron(), path);
const {
response,
size,
lastModifiedMs: lm,
} = await readStream(ensureElectron(), path);
fileSize = size;
lastModifiedMs = lm;
if (size > MULTIPART_PART_SIZE) {
const chunkCount = Math.ceil(size / FILE_READER_CHUNK_SIZE);
dataOrStream = { stream: response.body, chunkCount };
@ -446,9 +430,424 @@ const readFileOrPath = async (
}
}
return { dataOrStream, fileSize };
return { dataOrStream, fileSize, lastModifiedMs };
};
/** A variant of {@readFileOrPath} that always returns an {@link DataStream}. */
const readFileOrPathStream = async (
fileOrPath: File | string,
): Promise<DataStream> => {
if (fileOrPath instanceof File) {
return getFileStream(fileOrPath, FILE_READER_CHUNK_SIZE);
} else {
const { response, size } = await readStream(
ensureElectron(),
fileOrPath,
);
const chunkCount = Math.ceil(size / FILE_READER_CHUNK_SIZE);
return { stream: response.body, chunkCount };
}
};
interface ReadAssetDetailsResult {
fileTypeInfo: FileTypeInfo;
fileSize: number;
lastModifiedMs: number;
}
/**
* Read the file(s) to determine the type, size and last modified time of the
* given {@link asset}.
*/
const readAssetDetails = async ({
isLivePhoto,
livePhotoAssets,
file,
}: UploadAsset2): Promise<ReadAssetDetailsResult> =>
isLivePhoto
? readLivePhotoDetails(livePhotoAssets)
: readImageOrVideoDetails(file);
const readLivePhotoDetails = async ({ image, video }: LivePhotoAssets2) => {
const img = await readImageOrVideoDetails(image);
const vid = await readImageOrVideoDetails(video);
return {
fileTypeInfo: {
fileType: FILE_TYPE.LIVE_PHOTO,
extension: `${img.fileTypeInfo.extension}+${vid.fileTypeInfo.extension}`,
imageType: img.fileTypeInfo.extension,
videoType: vid.fileTypeInfo.extension,
},
fileSize: img.fileSize + vid.fileSize,
lastModifiedMs: img.lastModifiedMs,
};
};
/**
* Read the beginning of the given file (or its path), or use its filename as a
* fallback, to determine its MIME type. From that, construct and return a
* {@link FileTypeInfo}.
*
* While we're at it, also return the size of the file, and its last modified
* time (expressed as epoch milliseconds).
*
* @param fileOrPath See: [Note: Reading a fileOrPath]
*/
const readImageOrVideoDetails = async (fileOrPath: File | string) => {
const { dataOrStream, fileSize, lastModifiedMs } =
await readFileOrPath(fileOrPath);
const fileTypeInfo = await detectFileTypeInfoFromChunk(async () => {
if (dataOrStream instanceof Uint8Array) {
return dataOrStream;
} else {
const reader = dataOrStream.stream.getReader();
const chunk = ensure((await reader.read()).value);
await reader.cancel();
return chunk;
}
}, getFileName(fileOrPath));
return { fileTypeInfo, fileSize, lastModifiedMs };
};
/**
* Read the entirety of a readable stream.
*
* It is not recommended to use this for large (say, multi-hundred MB) files. It
* is provided as a syntactic shortcut for cases where we already know that the
* size of the stream will be reasonable enough to be read in its entirety
* without us running out of memory.
*/
const readEntireStream = async (stream: ReadableStream) =>
new Uint8Array(await new Response(stream).arrayBuffer());
interface ExtractAssetMetadataResult {
metadata: Metadata;
publicMagicMetadata: FilePublicMagicMetadataProps;
}
/**
* Compute the hash, extract EXIF or other metadata, and merge in data from the
* {@link parsedMetadataJSONMap} for the assets. Return the resultant metadatum.
*/
const extractAssetMetadata = async (
{ isLivePhoto, file, livePhotoAssets }: UploadAsset2,
fileTypeInfo: FileTypeInfo,
lastModifiedMs: number,
collectionID: number,
parsedMetadataJSONMap: Map<string, ParsedMetadataJSON>,
worker: Remote<DedicatedCryptoWorker>,
): Promise<ExtractAssetMetadataResult> =>
isLivePhoto
? await extractLivePhotoMetadata(
livePhotoAssets,
fileTypeInfo,
lastModifiedMs,
collectionID,
parsedMetadataJSONMap,
worker,
)
: await extractImageOrVideoMetadata(
file,
fileTypeInfo,
lastModifiedMs,
collectionID,
parsedMetadataJSONMap,
worker,
);
const extractLivePhotoMetadata = async (
livePhotoAssets: LivePhotoAssets2,
fileTypeInfo: FileTypeInfo,
lastModifiedMs: number,
collectionID: number,
parsedMetadataJSONMap: Map<string, ParsedMetadataJSON>,
worker: Remote<DedicatedCryptoWorker>,
) => {
const imageFileTypeInfo: FileTypeInfo = {
fileType: FILE_TYPE.IMAGE,
extension: fileTypeInfo.imageType,
};
const { metadata: imageMetadata, publicMagicMetadata } =
await extractImageOrVideoMetadata(
livePhotoAssets.image,
imageFileTypeInfo,
lastModifiedMs,
collectionID,
parsedMetadataJSONMap,
worker,
);
const videoHash = await computeHash(livePhotoAssets.video, worker);
return {
metadata: {
...imageMetadata,
title: getFileName(livePhotoAssets.image),
fileType: FILE_TYPE.LIVE_PHOTO,
imageHash: imageMetadata.hash,
videoHash: videoHash,
hash: undefined,
},
publicMagicMetadata,
};
};
const extractImageOrVideoMetadata = async (
fileOrPath: File | string,
fileTypeInfo: FileTypeInfo,
lastModifiedMs: number,
collectionID: number,
parsedMetadataJSONMap: Map<string, ParsedMetadataJSON>,
worker: Remote<DedicatedCryptoWorker>,
) => {
const fileName = getFileName(fileOrPath);
const { fileType } = fileTypeInfo;
let extractedMetadata: ParsedExtractedMetadata;
if (fileType === FILE_TYPE.IMAGE) {
extractedMetadata =
(await tryExtractImageMetadata(
fileOrPath,
fileTypeInfo,
lastModifiedMs,
)) ?? NULL_EXTRACTED_METADATA;
} else if (fileType === FILE_TYPE.VIDEO) {
extractedMetadata =
(await tryExtractVideoMetadata(fileOrPath)) ??
NULL_EXTRACTED_METADATA;
} else {
throw new Error(`Unexpected file type ${fileType} for ${fileOrPath}`);
}
const hash = await computeHash(fileOrPath, worker);
const modificationTime = lastModifiedMs * 1000;
const creationTime =
extractedMetadata.creationTime ??
tryParseEpochMicrosecondsFromFileName(fileName) ??
modificationTime;
const metadata: Metadata = {
title: fileName,
creationTime,
modificationTime,
latitude: extractedMetadata.location.latitude,
longitude: extractedMetadata.location.longitude,
fileType,
hash,
};
const publicMagicMetadata: FilePublicMagicMetadataProps = {
w: extractedMetadata.width,
h: extractedMetadata.height,
};
const takeoutMetadata = matchTakeoutMetadata(
fileName,
collectionID,
parsedMetadataJSONMap,
);
if (takeoutMetadata)
for (const [key, value] of Object.entries(takeoutMetadata))
if (value) metadata[key] = value;
return { metadata, publicMagicMetadata };
};
const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = {
location: NULL_LOCATION,
creationTime: null,
width: null,
height: null,
};
async function tryExtractImageMetadata(
fileOrPath: File | string,
fileTypeInfo: FileTypeInfo,
lastModifiedMs: number,
): Promise<ParsedExtractedMetadata> {
let file: File;
if (fileOrPath instanceof File) {
file = fileOrPath;
} else {
const path = fileOrPath;
// The library we use for extracting EXIF from images, exifr, doesn't
// support streams. But unlike videos, for images it is reasonable to
// read the entire stream into memory here.
const { response } = await readStream(ensureElectron(), path);
file = new File([await response.arrayBuffer()], basename(path), {
lastModified: lastModifiedMs,
});
}
try {
return await parseImageMetadata(file, fileTypeInfo);
} catch (e) {
log.error(`Failed to extract image metadata for ${fileOrPath}`, e);
return undefined;
}
}
const tryExtractVideoMetadata = async (fileOrPath: File | string) => {
try {
return await ffmpeg.extractVideoMetadata(fileOrPath);
} catch (e) {
log.error(`Failed to extract video metadata for ${fileOrPath}`, e);
return undefined;
}
};
const computeHash = async (
fileOrPath: File | string,
worker: Remote<DedicatedCryptoWorker>,
) => {
const { stream, chunkCount } = await readFileOrPathStream(fileOrPath);
const hashState = await worker.initChunkHashing();
const streamReader = stream.getReader();
for (let i = 0; i < chunkCount; i++) {
const { done, value: chunk } = await streamReader.read();
if (done) throw new Error("Less chunks than expected");
await worker.hashFileChunk(hashState, Uint8Array.from(chunk));
}
const { done } = await streamReader.read();
if (!done) throw new Error("More chunks than expected");
return await worker.completeChunkHashing(hashState);
};
/**
* Return true if the two files, as represented by their metadata, are same.
*
* Note that the metadata includes the hash of the file's contents (when
* available), so this also in effect compares the contents of the files, not
* just the "meta" information about them.
*/
const areFilesSame = (f: Metadata, g: Metadata) =>
hasFileHash(f) && hasFileHash(g)
? areFilesSameHash(f, g)
: areFilesSameNoHash(f, g);
const areFilesSameHash = (f: Metadata, g: Metadata) => {
if (f.fileType !== g.fileType || f.title !== g.title) {
return false;
}
if (f.fileType === FILE_TYPE.LIVE_PHOTO) {
return f.imageHash === g.imageHash && f.videoHash === g.videoHash;
} else {
return f.hash === g.hash;
}
};
/**
* Older files that were uploaded before we introduced hashing will not have
* hashes, so retain and use the logic we used back then for such files.
*
* Deprecation notice April 2024: Note that hashing was introduced very early
* (years ago), so the chance of us finding files without hashes is rare. And
* even in these cases, the worst that'll happen is that a duplicate file would
* get uploaded which can later be deduped. So we can get rid of this case at
* some point (e.g. the mobile app doesn't do this extra check, just uploads).
*/
const areFilesSameNoHash = (f: Metadata, g: Metadata) => {
/*
* The maximum difference in the creation/modification times of two similar
* files is set to 1 second. This is because while uploading files in the
* web - browsers and users could have set reduced precision of file times
* to prevent timing attacks and fingerprinting.
*
* See:
* https://developer.mozilla.org/en-US/docs/Web/API/File/lastModified#reduced_time_precision
*/
const oneSecond = 1e6;
return (
f.fileType == g.fileType &&
f.title == g.title &&
Math.abs(f.creationTime - g.creationTime) < oneSecond &&
Math.abs(f.modificationTime - g.modificationTime) < oneSecond
);
};
const readAsset = async (
fileTypeInfo: FileTypeInfo,
{ isLivePhoto, file, livePhotoAssets }: UploadAsset2,
) =>
isLivePhoto
? await readLivePhoto(livePhotoAssets, fileTypeInfo)
: await readImageOrVideo(file, fileTypeInfo);
const readLivePhoto = async (
livePhotoAssets: LivePhotoAssets2,
fileTypeInfo: FileTypeInfo,
) => {
const readImage = await readFileOrPath(livePhotoAssets.image);
const {
filedata: imageDataOrStream,
thumbnail,
hasStaticThumbnail,
} = await withThumbnail(
livePhotoAssets.image,
{
extension: fileTypeInfo.imageType,
fileType: FILE_TYPE.IMAGE,
},
readImage.dataOrStream,
readImage.fileSize,
);
const readVideo = await readFileOrPath(livePhotoAssets.video);
// We can revisit this later, but the existing code always read the entire
// file into memory here, and to avoid changing the rest of the scaffolding
// retain the same behaviour.
//
// This is a reasonable assumption too, since the videos corresponding to
// live photos are only a couple of seconds long.
const toData = async (dataOrStream: Uint8Array | DataStream) =>
dataOrStream instanceof Uint8Array
? dataOrStream
: await readEntireStream(dataOrStream.stream);
return {
filedata: await encodeLivePhoto({
imageFileName: getFileName(livePhotoAssets.image),
imageData: await toData(imageDataOrStream),
videoFileName: getFileName(livePhotoAssets.video),
videoData: await toData(readVideo.dataOrStream),
}),
thumbnail,
hasStaticThumbnail,
};
};
const readImageOrVideo = async (
fileOrPath: File | string,
fileTypeInfo: FileTypeInfo,
) => {
const { dataOrStream, fileSize } = await readFileOrPath(fileOrPath);
return withThumbnail(fileOrPath, fileTypeInfo, dataOrStream, fileSize);
};
// TODO(MR): Merge with the uploader
class ModuleState {
/**
* This will be set to true if we get an error from the Node.js side of our
* desktop app telling us that native image thumbnail generation is not
* available for the current OS/arch combination.
*
* That way, we can stop pestering it again and again (saving an IPC
* round-trip).
*
* Note the double negative when it is used.
*/
isNativeImageThumbnailGenerationNotAvailable = false;
}
const moduleState = new ModuleState();
/**
* Augment the given {@link dataOrStream} with thumbnail information.
*
@ -557,68 +956,6 @@ const withThumbnail = async (
};
};
/**
* Read the entirety of a readable stream.
*
* It is not recommended to use this for large (say, multi-hundred MB) files. It
* is provided as a syntactic shortcut for cases where we already know that the
* size of the stream will be reasonable enough to be read in its entirety
* without us running out of memory.
*/
const readEntireStream = async (stream: ReadableStream) =>
new Uint8Array(await new Response(stream).arrayBuffer());
const readImageOrVideo = async (
fileOrPath: File | string,
fileTypeInfo: FileTypeInfo,
) => {
const { dataOrStream, fileSize } = await readFileOrPath(fileOrPath);
return withThumbnail(fileOrPath, fileTypeInfo, dataOrStream, fileSize);
};
const readLivePhoto = async (
livePhotoAssets: LivePhotoAssets2,
fileTypeInfo: FileTypeInfo,
) => {
const readImage = await readFileOrPath(livePhotoAssets.image);
const {
filedata: imageDataOrStream,
thumbnail,
hasStaticThumbnail,
} = await withThumbnail(
livePhotoAssets.image,
{
exactType: fileTypeInfo.imageType,
fileType: FILE_TYPE.IMAGE,
},
readImage.dataOrStream,
readImage.fileSize,
);
const readVideo = await readFileOrPath(livePhotoAssets.video);
// We can revisit this later, but the existing code always read the
// full files into memory here, and to avoid changing the rest of
// the scaffolding retain the same behaviour.
//
// This is a reasonable assumption too, since the videos
// corresponding to live photos are only a couple of seconds long.
const toData = async (dataOrStream: Uint8Array | DataStream) =>
dataOrStream instanceof Uint8Array
? dataOrStream
: await readEntireStream(dataOrStream.stream);
return {
filedata: await encodeLivePhoto({
imageFileName: getFileName(livePhotoAssets.image),
imageData: await toData(imageDataOrStream),
videoFileName: getFileName(livePhotoAssets.video),
videoData: await toData(readVideo.dataOrStream),
}),
thumbnail,
hasStaticThumbnail,
};
};
const constructPublicMagicMetadata = async (
publicMagicMetadataProps: FilePublicMagicMetadataProps,
): Promise<FilePublicMagicMetadata> => {
@ -632,73 +969,65 @@ const constructPublicMagicMetadata = async (
return await updateMagicMetadata(publicMagicMetadataProps);
};
async function encryptFile(
worker: Remote<DedicatedCryptoWorker>,
const encryptFile = async (
file: FileWithMetadata,
encryptionKey: string,
): Promise<EncryptedFile> {
try {
const { key: fileKey, file: encryptedFiledata } = await encryptFiledata(
worker,
file.filedata,
);
worker: Remote<DedicatedCryptoWorker>,
): Promise<EncryptedFile> => {
const { key: fileKey, file: encryptedFiledata } = await encryptFiledata(
file.filedata,
worker,
);
const { file: encryptedThumbnail } = await worker.encryptThumbnail(
file.thumbnail,
fileKey,
);
const { file: encryptedMetadata } = await worker.encryptMetadata(
file.metadata,
fileKey,
);
const { file: encryptedThumbnail } = await worker.encryptThumbnail(
file.thumbnail,
fileKey,
);
let encryptedPubMagicMetadata: EncryptedMagicMetadata;
if (file.pubMagicMetadata) {
const { file: encryptedPubMagicMetadataData } =
await worker.encryptMetadata(
file.pubMagicMetadata.data,
fileKey,
);
encryptedPubMagicMetadata = {
version: file.pubMagicMetadata.version,
count: file.pubMagicMetadata.count,
data: encryptedPubMagicMetadataData.encryptedData,
header: encryptedPubMagicMetadataData.decryptionHeader,
};
}
const { file: encryptedMetadata } = await worker.encryptMetadata(
file.metadata,
fileKey,
);
const encryptedKey = await worker.encryptToB64(fileKey, encryptionKey);
const result: EncryptedFile = {
file: {
file: encryptedFiledata,
thumbnail: encryptedThumbnail,
metadata: encryptedMetadata,
pubMagicMetadata: encryptedPubMagicMetadata,
localID: file.localID,
},
fileKey: encryptedKey,
let encryptedPubMagicMetadata: EncryptedMagicMetadata;
if (file.pubMagicMetadata) {
const { file: encryptedPubMagicMetadataData } =
await worker.encryptMetadata(file.pubMagicMetadata.data, fileKey);
encryptedPubMagicMetadata = {
version: file.pubMagicMetadata.version,
count: file.pubMagicMetadata.count,
data: encryptedPubMagicMetadataData.encryptedData,
header: encryptedPubMagicMetadataData.decryptionHeader,
};
return result;
} catch (e) {
log.error("Error encrypting files", e);
throw e;
}
}
async function encryptFiledata(
worker: Remote<DedicatedCryptoWorker>,
const encryptedKey = await worker.encryptToB64(fileKey, encryptionKey);
const result: EncryptedFile = {
file: {
file: encryptedFiledata,
thumbnail: encryptedThumbnail,
metadata: encryptedMetadata,
pubMagicMetadata: encryptedPubMagicMetadata,
localID: file.localID,
},
fileKey: encryptedKey,
};
return result;
};
const encryptFiledata = async (
filedata: Uint8Array | DataStream,
): Promise<EncryptionResult<Uint8Array | DataStream>> {
return isDataStream(filedata)
? await encryptFileStream(worker, filedata)
: await worker.encryptFile(filedata);
}
async function encryptFileStream(
worker: Remote<DedicatedCryptoWorker>,
): Promise<EncryptionResult<Uint8Array | DataStream>> =>
isDataStream(filedata)
? await encryptFileStream(filedata, worker)
: await worker.encryptFile(filedata);
const encryptFileStream = async (
fileData: DataStream,
) {
worker: Remote<DedicatedCryptoWorker>,
) => {
const { stream, chunkCount } = fileData;
const fileStreamReader = stream.getReader();
const { key, decryptionHeader, pushState } =
@ -726,58 +1055,6 @@ async function encryptFileStream(
encryptedData: { stream: encryptedFileStream, chunkCount },
},
};
}
/**
* Return true if the two files, as represented by their metadata, are same.
*
* Note that the metadata includes the hash of the file's contents (when
* available), so this also in effect compares the contents of the files, not
* just the "meta" information about them.
*/
const areFilesSame = (f: Metadata, g: Metadata) =>
hasFileHash(f) && hasFileHash(g)
? areFilesSameHash(f, g)
: areFilesSameNoHash(f, g);
const areFilesSameHash = (f: Metadata, g: Metadata) => {
if (f.fileType !== g.fileType || f.title !== g.title) {
return false;
}
if (f.fileType === FILE_TYPE.LIVE_PHOTO) {
return f.imageHash === g.imageHash && f.videoHash === g.videoHash;
} else {
return f.hash === g.hash;
}
};
/**
* Older files that were uploaded before we introduced hashing will not have
* hashes, so retain and use the logic we used back then for such files.
*
* Deprecation notice April 2024: Note that hashing was introduced very early
* (years ago), so the chance of us finding files without hashes is rare. And
* even in these cases, the worst that'll happen is that a duplicate file would
* get uploaded which can later be deduped. So we can get rid of this case at
* some point (e.g. the mobile app doesn't do this extra check, just uploads).
*/
const areFilesSameNoHash = (f: Metadata, g: Metadata) => {
/*
* The maximum difference in the creation/modification times of two similar
* files is set to 1 second. This is because while uploading files in the
* web - browsers and users could have set reduced precision of file times
* to prevent timing attacks and fingerprinting.
*
* See:
* https://developer.mozilla.org/en-US/docs/Web/API/File/lastModified#reduced_time_precision
*/
const oneSecond = 1e6;
return (
f.fileType == g.fileType &&
f.title == g.title &&
Math.abs(f.creationTime - g.creationTime) < oneSecond &&
Math.abs(f.modificationTime - g.modificationTime) < oneSecond
);
};
const uploadToBucket = async (
@ -845,7 +1122,7 @@ const uploadToBucket = async (
return backupedFile;
} catch (e) {
if (e.message !== CustomError.UPLOAD_CANCELLED) {
log.error("error uploading to bucket", e);
log.error("Error when uploading to bucket", e);
}
throw e;
}
@ -904,9 +1181,7 @@ async function uploadStreamUsingMultipart(
partEtags.push({ PartNumber: index + 1, ETag: eTag });
}
const { done } = await streamReader.read();
if (!done) {
throw Error(CustomError.CHUNK_MORE_THAN_EXPECTED);
}
if (!done) throw new Error("More chunks than expected");
const completeURL = multipartUploadURLs.completeURL;
const cBody = convert.js2xml(

2
web/apps/photos/src/types/file/index.ts
View file

@ -1,10 +1,10 @@
import type { Metadata } from "@/media/types/file";
import { SourceURLs } from "services/download";
import {
EncryptedMagicMetadata,
MagicMetadataCore,
VISIBILITY_STATE,
} from "types/magicMetadata";
import { Metadata } from "types/upload";
export interface MetadataFileAttributes {
encryptedData: string;

28
web/apps/photos/src/types/upload/index.ts
View file

@ -1,4 +1,4 @@
import { FILE_TYPE } from "@/media/file-type";
import type { Metadata } from "@/media/types/file";
import type { ElectronFile } from "@/next/types/file";
import {
B64EncryptionResult,
@ -13,28 +13,6 @@ import {
} from "types/file";
import { EncryptedMagicMetadata } from "types/magicMetadata";
/** Information about the file that never changes post upload. */
export interface Metadata {
/**
* The file name.
*
* See: [Note: File name for local EnteFile objects]
*/
title: string;
creationTime: number;
modificationTime: number;
latitude: number;
longitude: number;
fileType: FILE_TYPE;
hasStaticThumbnail?: boolean;
hash?: string;
imageHash?: string;
videoHash?: string;
localID?: number;
version?: number;
deviceFolder?: string;
}
export interface Location {
latitude: number;
longitude: number;
@ -49,6 +27,7 @@ export interface MultipartUploadURLs {
export interface UploadAsset {
isLivePhoto?: boolean;
file?: File | ElectronFile;
fileOrPath?: File | ElectronFile;
livePhotoAssets?: LivePhotoAssets;
}
@ -66,6 +45,7 @@ export interface FileWithCollection extends UploadAsset {
export interface UploadAsset2 {
isLivePhoto?: boolean;
file?: File | string;
fileOrPath?: File | string;
livePhotoAssets?: LivePhotoAssets2;
}
@ -77,7 +57,7 @@ export interface LivePhotoAssets2 {
export interface FileWithCollection2 extends UploadAsset2 {
localID: number;
collection?: Collection;
collectionID?: number;
collectionID: number;
}
export interface UploadURL {

67
web/apps/photos/src/utils/file/index.ts
View file

@ -1,5 +1,6 @@
import { FILE_TYPE, type FileTypeInfo } from "@/media/file-type";
import { FILE_TYPE } from "@/media/file-type";
import { decodeLivePhoto } from "@/media/live-photo";
import { lowercaseExtension } from "@/next/file";
import log from "@/next/log";
import { CustomErrorMessage, type Electron } from "@/next/types/ipc";
import { workerBridge } from "@/next/worker/worker-bridge";
@ -10,6 +11,7 @@ import { downloadUsingAnchor, withTimeout } from "@ente/shared/utils";
import { t } from "i18next";
import isElectron from "is-electron";
import { moveToHiddenCollection } from "services/collectionService";
import { detectFileTypeInfo } from "services/detect-type";
import DownloadManager from "services/download";
import { updateFileCreationDateInEXIF } from "services/exif";
import {
@ -19,7 +21,6 @@ import {
updateFilePublicMagicMetadata,
} from "services/fileService";
import { heicToJPEG } from "services/heic-convert";
import { getFileType } from "services/typeDetectionService";
import {
EncryptedEnteFile,
EnteFile,
@ -39,11 +40,6 @@ import { isArchivedFile, updateMagicMetadata } from "utils/magicMetadata";
import { safeFileName } from "utils/native-fs";
import { writeStream } from "utils/native-stream";
const TYPE_HEIC = "heic";
const TYPE_HEIF = "heif";
const TYPE_JPEG = "jpeg";
const TYPE_JPG = "jpg";
const RAW_FORMATS = [
"heic",
"rw2",
@ -102,11 +98,11 @@ export async function getUpdatedEXIFFileForDownload(
file: EnteFile,
fileStream: ReadableStream<Uint8Array>,
): Promise<ReadableStream<Uint8Array>> {
const extension = getFileExtension(file.metadata.title);
const extension = lowercaseExtension(file.metadata.title);
if (
file.metadata.fileType === FILE_TYPE.IMAGE &&
file.pubMagicMetadata?.data.editedTime &&
(extension === TYPE_JPEG || extension === TYPE_JPG)
(extension == "jpeg" || extension == "jpg")
) {
const fileBlob = await new Response(fileStream).blob();
const updatedFileBlob = await updateFileCreationDateInEXIF(
@ -130,19 +126,19 @@ export async function downloadFile(file: EnteFile) {
const { imageFileName, imageData, videoFileName, videoData } =
await decodeLivePhoto(file.metadata.title, fileBlob);
const image = new File([imageData], imageFileName);
const imageType = await getFileType(image);
const imageType = await detectFileTypeInfo(image);
const tempImageURL = URL.createObjectURL(
new Blob([imageData], { type: imageType.mimeType }),
);
const video = new File([videoData], videoFileName);
const videoType = await getFileType(video);
const videoType = await detectFileTypeInfo(video);
const tempVideoURL = URL.createObjectURL(
new Blob([videoData], { type: videoType.mimeType }),
);
downloadUsingAnchor(tempImageURL, imageFileName);
downloadUsingAnchor(tempVideoURL, videoFileName);
} else {
const fileType = await getFileType(
const fileType = await detectFileTypeInfo(
new File([fileBlob], file.metadata.title),
);
fileBlob = await new Response(
@ -278,20 +274,6 @@ export async function decryptFile(
}
}
export function splitFilenameAndExtension(filename: string): [string, string] {
const lastDotPosition = filename.lastIndexOf(".");
if (lastDotPosition === -1) return [filename, null];
else
return [
filename.slice(0, lastDotPosition),
filename.slice(lastDotPosition + 1),
];
}
export function getFileExtension(filename: string) {
return splitFilenameAndExtension(filename)[1]?.toLocaleLowerCase();
}
export function generateStreamFromArrayBuffer(data: Uint8Array) {
return new ReadableStream({
async start(controller: ReadableStreamDefaultController) {
@ -302,23 +284,22 @@ export function generateStreamFromArrayBuffer(data: Uint8Array) {
}
export const getRenderableImage = async (fileName: string, imageBlob: Blob) => {
let fileTypeInfo: FileTypeInfo;
try {
const tempFile = new File([imageBlob], fileName);
fileTypeInfo = await getFileType(tempFile);
const fileTypeInfo = await detectFileTypeInfo(tempFile);
log.debug(
() =>
`Obtaining renderable image for ${JSON.stringify(fileTypeInfo)}`,
() => `Need renderable image for ${JSON.stringify(fileTypeInfo)}`,
);
const { exactType } = fileTypeInfo;
const { extension } = fileTypeInfo;
if (!isRawFile(exactType)) {
// Not something we know how to handle yet, give back the original.
if (!isRawFile(extension)) {
// Either it is not something we know how to handle yet, or
// something that the browser already knows how to render.
return imageBlob;
}
const available = !moduleState.isNativeJPEGConversionNotAvailable;
if (isElectron() && available && isSupportedRawFormat(exactType)) {
if (isElectron() && available && isSupportedRawFormat(extension)) {
// If we're running in our desktop app, see if our Node.js layer can
// convert this into a JPEG using native tools for us.
try {
@ -332,17 +313,14 @@ export const getRenderableImage = async (fileName: string, imageBlob: Blob) => {
}
}
if (isFileHEIC(exactType)) {
// If it is an HEIC file, use our web HEIC converter.
if (extension == "heic" || extension == "heif") {
// For HEIC/HEIF files we can use our web HEIC converter.
return await heicToJPEG(imageBlob);
}
return undefined;
} catch (e) {
log.error(
`Failed to get renderable image for ${JSON.stringify(fileTypeInfo ?? fileName)}`,
e,
);
log.error(`Failed to get renderable image for ${fileName}`, e);
return undefined;
}
};
@ -361,13 +339,6 @@ const nativeConvertToJPEG = async (imageBlob: Blob) => {
return new Blob([jpegData]);
};
export function isFileHEIC(exactType: string) {
return (
exactType.toLowerCase().endsWith(TYPE_HEIC) ||
exactType.toLowerCase().endsWith(TYPE_HEIF)
);
}
export function isRawFile(exactType: string) {
return RAW_FORMATS.includes(exactType.toLowerCase());
}
@ -724,7 +695,7 @@ export const getArchivedFiles = (files: EnteFile[]) => {
};
export const createTypedObjectURL = async (blob: Blob, fileName: string) => {
const type = await getFileType(new File([blob], fileName));
const type = await detectFileTypeInfo(new File([blob], fileName));
return URL.createObjectURL(new Blob([blob], { type: type.mimeType }));
};

25
web/apps/photos/src/utils/native-stream.ts
View file

@ -19,18 +19,21 @@ import type { Electron } from "@/next/types/ipc";
* @param path The path on the file on the user's local filesystem whose
* contents we want to stream.
*
* @return A ({@link Response}, size) tuple.
* @return A ({@link Response}, size, lastModifiedMs) triple.
*
* * The response contains the contents of the file. In particular, the `body`
* {@link ReadableStream} property of this response can be used to read the
* files contents in a streaming manner.
*
* * The size is the size of the file that we'll be reading from disk.
*
* * The lastModifiedMs value is the last modified time of the file that we're
* reading, expressed as epoch milliseconds.
*/
export const readStream = async (
_: Electron,
path: string,
): Promise<{ response: Response; size: number }> => {
): Promise<{ response: Response; size: number; lastModifiedMs: number }> => {
const req = new Request(`stream://read${path}`, {
method: "GET",
});
@ -41,13 +44,19 @@ export const readStream = async (
`Failed to read stream from ${path}: HTTP ${res.status}`,
);
const size = +res.headers["Content-Length"];
if (isNaN(size))
throw new Error(
`Got a numeric Content-Length when reading a stream. The response was ${res}`,
);
const size = readNumericHeader(res, "Content-Length");
const lastModifiedMs = readNumericHeader(res, "X-Last-Modified-Ms");
return { response: res, size };
return { response: res, size, lastModifiedMs };
};
const readNumericHeader = (res: Response, key: string) => {
const value = +res.headers[key];
if (isNaN(value))
throw new Error(
`Expected a numeric ${key} when reading a stream response: ${res}`,
);
return value;
};
/**

31
web/apps/photos/src/utils/upload/index.ts
View file

@ -1,42 +1,13 @@
import type { Metadata } from "@/media/types/file";
import { basename, dirname } from "@/next/file";
import { ElectronFile } from "@/next/types/file";
import { PICKED_UPLOAD_TYPE } from "constants/upload";
import isElectron from "is-electron";
import { exportMetadataDirectoryName } from "services/export";
import {
FileWithCollection,
Metadata,
type FileWithCollection2,
} from "types/upload";
const TYPE_JSON = "json";
export const hasFileHash = (file: Metadata) =>
file.hash || (file.imageHash && file.videoHash);
export function segregateMetadataAndMediaFiles(
filesWithCollectionToUpload: FileWithCollection[],
) {
const metadataJSONFiles: FileWithCollection[] = [];
const mediaFiles: FileWithCollection[] = [];
filesWithCollectionToUpload.forEach((fileWithCollection) => {
const file = fileWithCollection.file;
if (file.name.toLowerCase().endsWith(TYPE_JSON)) {
metadataJSONFiles.push(fileWithCollection);
} else {
mediaFiles.push(fileWithCollection);
}
});
return { mediaFiles, metadataJSONFiles };
}
export function areFileWithCollectionsSame(
firstFile: FileWithCollection2,
secondFile: FileWithCollection2,
): boolean {
return firstFile.localID === secondFile.localID;
}
/**
* Return true if all the paths in the given list are items that belong to the
* same (arbitrary) directory.

2
web/apps/photos/tests/upload.test.ts
View file

@ -1,7 +1,7 @@
import { FILE_TYPE } from "@/media/file-type";
import { tryToParseDateTime } from "@ente/shared/time";
import { getLocalCollections } from "services/collectionService";
import { getLocalFiles } from "services/fileService";
import { tryToParseDateTime } from "services/upload/date";
import {
MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT,
getClippedMetadataJSONMapKeyForFile,

33
web/packages/media/file-type.ts
View file

@ -7,7 +7,12 @@ export enum FILE_TYPE {
export interface FileTypeInfo {
fileType: FILE_TYPE;
exactType: string;
/**
* A lowercased, standardized extension for files of the current type.
*
* TODO(MR): This in not valid for LIVE_PHOTO.
*/
extension: string;
mimeType?: string;
imageType?: string;
videoType?: string;
@ -15,42 +20,42 @@ export interface FileTypeInfo {
// list of format that were missed by type-detection for some files.
export const KnownFileTypeInfos: FileTypeInfo[] = [
{ fileType: FILE_TYPE.IMAGE, exactType: "jpeg", mimeType: "image/jpeg" },
{ fileType: FILE_TYPE.IMAGE, exactType: "jpg", mimeType: "image/jpeg" },
{ fileType: FILE_TYPE.VIDEO, exactType: "webm", mimeType: "video/webm" },
{ fileType: FILE_TYPE.VIDEO, exactType: "mod", mimeType: "video/mpeg" },
{ fileType: FILE_TYPE.VIDEO, exactType: "mp4", mimeType: "video/mp4" },
{ fileType: FILE_TYPE.IMAGE, exactType: "gif", mimeType: "image/gif" },
{ fileType: FILE_TYPE.VIDEO, exactType: "dv", mimeType: "video/x-dv" },
{ fileType: FILE_TYPE.IMAGE, extension: "jpeg", mimeType: "image/jpeg" },
{ fileType: FILE_TYPE.IMAGE, extension: "jpg", mimeType: "image/jpeg" },
{ fileType: FILE_TYPE.VIDEO, extension: "webm", mimeType: "video/webm" },
{ fileType: FILE_TYPE.VIDEO, extension: "mod", mimeType: "video/mpeg" },
{ fileType: FILE_TYPE.VIDEO, extension: "mp4", mimeType: "video/mp4" },
{ fileType: FILE_TYPE.IMAGE, extension: "gif", mimeType: "image/gif" },
{ fileType: FILE_TYPE.VIDEO, extension: "dv", mimeType: "video/x-dv" },
{
fileType: FILE_TYPE.VIDEO,
exactType: "wmv",
extension: "wmv",
mimeType: "video/x-ms-asf",
},
{
fileType: FILE_TYPE.VIDEO,
exactType: "hevc",
extension: "hevc",
mimeType: "video/hevc",
},
{
fileType: FILE_TYPE.IMAGE,
exactType: "raf",
extension: "raf",
mimeType: "image/x-fuji-raf",
},
{
fileType: FILE_TYPE.IMAGE,
exactType: "orf",
extension: "orf",
mimeType: "image/x-olympus-orf",
},
{
fileType: FILE_TYPE.IMAGE,
exactType: "crw",
extension: "crw",
mimeType: "image/x-canon-crw",
},
{
fileType: FILE_TYPE.VIDEO,
exactType: "mov",
extension: "mov",
mimeType: "video/quicktime",
},
];

10
web/packages/media/live-photo.ts
View file

@ -1,4 +1,8 @@
import { fileNameFromComponents, nameAndExtension } from "@/next/file";
import {
fileNameFromComponents,
lowercaseExtension,
nameAndExtension,
} from "@/next/file";
import JSZip from "jszip";
import { FILE_TYPE } from "./file-type";
@ -38,11 +42,9 @@ const potentialVideoExtensions = [
export const potentialFileTypeFromExtension = (
fileName: string,
): FILE_TYPE | undefined => {
let [, ext] = nameAndExtension(fileName);
const ext = lowercaseExtension(fileName);
if (!ext) return undefined;
ext = ext.toLowerCase();
if (potentialImageExtensions.includes(ext)) return FILE_TYPE.IMAGE;
else if (potentialVideoExtensions.includes(ext)) return FILE_TYPE.VIDEO;
else return undefined;

73
web/packages/media/types/file.ts Normal file
View file

@ -0,0 +1,73 @@
import type { FILE_TYPE } from "../file-type";
/**
* Information about the file that never changes post upload.
*
* [Note: Metadatum]
*
* There are three different sources of metadata relating to a file.
*
* 1. Metadata
* 2. Magic Metadata
* 3. Public Magic Metadata
*
* The names of API entities are such for historical reasons, but we can think
* of them as:
*
* 1. Metadata
* 2. Private Mutable Metadata
* 3. Shared Mutable Metadata
*
* Metadata is the original metadata that we attached to the file when it was
* uploaded. It is immutable, and it never changes.
*
* Later on, the user might make changes to the file's metadata. Since the
* metadata is immutable, we need a place to keep these mutations.
*
* Some mutations are "private" to the user who owns the file. For example, the
* user might archive the file. Such modifications get written to (2), Private
* Mutable Metadata.
*
* Other mutations are "public" across all the users with whom the file is
* shared. For example, if the user (owner) edits the name of the file, all
* people with whom this file is shared can see the new edited name. Such
* modifications get written to (3), Shared Mutable Metadata.
*
* When the client needs to show a file, it needs to "merge" in 2 or 3 of these
* sources.
*
* - When showing a shared file, (1) and (3) are merged, with changes from (3)
* taking precedence, to obtain the full metadata pertinent to the file.
* - When showing a normal (un-shared) file, (1), (2) and (3) are merged, with
* changes from (2) and (3) taking precedence, to obtain the full metadata.
* (2) and (3) have no intersection of keys, so they can be merged in any
* order.
*
* While these sources can be conceptually merged, it is important for the
* client to also retain the original sources unchanged. This is because the
* metadatas (any of the three) might have keys that the current client does not
* yet understand, so when updating some key, say filename in (3), it should
* only edit the key it knows about but retain the rest of the source JSON
* unchanged.
*/
export interface Metadata {
/**
* The file name.
*
* See: [Note: File name for local EnteFile objects]
*/
title: string;
creationTime: number;
modificationTime: number;
latitude: number;
longitude: number;
/** The "Ente" file type. */
fileType: FILE_TYPE;
hasStaticThumbnail?: boolean;
hash?: string;
imageHash?: string;
videoHash?: string;
localID?: number;
version?: number;
deviceFolder?: string;
}

17
web/packages/next/file.ts
View file

@ -25,6 +25,23 @@ export const nameAndExtension = (fileName: string): FileNameComponents => {
return [fileName.slice(0, i), fileName.slice(i + 1)];
};
/**
* If the file name or path has an extension, return a lowercased version of it.
*
* This is handy when comparing the extension to a known set without worrying
* about case sensitivity.
*
* See {@link nameAndExtension} for its more generic sibling.
*/
export const lowercaseExtension = (
fileNameOrPath: string,
): string | undefined => {
// We rely on the implementation of nameAndExtension using lastIndexOf to
// allow us to also work on paths.
const [, ext] = nameAndExtension(fileNameOrPath);
return ext?.toLowerCase();
};
/**
* Construct a file name from its components (name and extension).
*

26
web/packages/next/log.ts
View file

@ -27,27 +27,30 @@ const workerLogToDisk = (message: string) => {
});
};
const logError = (message: string, e?: unknown) => {
if (!e) {
logError_(message);
return;
}
const messageWithError = (message: string, e?: unknown) => {
if (!e) return message;
let es: string;
if (e instanceof Error) {
// In practice, we expect ourselves to be called with Error objects, so
// this is the happy path so to say.
es = `${e.name}: ${e.message}\n${e.stack}`;
return `${e.name}: ${e.message}\n${e.stack}`;
} else {
// For the rest rare cases, use the default string serialization of e.
es = String(e);
}
logError_(`${message}: ${es}`);
return `${message}: ${es}`;
};
const logError_ = (message: string) => {
const m = `[error] ${message}`;
const logError = (message: string, e?: unknown) => {
const m = `[error] ${messageWithError(message, e)}`;
if (isDevBuild) console.error(m);
logToDisk(m);
};
const logWarn = (message: string, e?: unknown) => {
const m = `[warn] ${messageWithError(message, e)}`;
if (isDevBuild) console.error(m);
logToDisk(m);
};
@ -90,6 +93,11 @@ export default {
* printed to the browser console.
*/
error: logError,
/**
* Sibling of {@link error}, with the same parameters and behaviour, except
* it gets prefixed with a warning instead of an error tag.
*/
warn: logWarn,
/**
* Log a message.
*

5
web/packages/next/types/ipc.ts
View file

@ -189,6 +189,11 @@ export interface Electron {
* directory.
*/
isDir: (dirPath: string) => Promise<boolean>;
/**
* Return the size in bytes of the file at {@link path}.
*/
size: (path: string) => Promise<number>;
};
// - Conversion

5
web/packages/shared/error/index.ts
View file

@ -26,8 +26,6 @@ export const CustomError = {
ETAG_MISSING: "no header/etag present in response body",
KEY_MISSING: "encrypted key missing from localStorage",
FAILED_TO_LOAD_WEB_WORKER: "failed to load web worker",
CHUNK_MORE_THAN_EXPECTED: "chunks more than expected",
CHUNK_LESS_THAN_EXPECTED: "chunks less than expected",
UNSUPPORTED_FILE_FORMAT: "unsupported file format",
FILE_TOO_LARGE: "file too large",
SUBSCRIPTION_EXPIRED: "subscription expired",
@ -56,8 +54,6 @@ export const CustomError = {
HIDDEN_COLLECTION_SYNC_FILE_ATTEMPTED:
"hidden collection sync file attempted",
UNKNOWN_ERROR: "Something went wrong, please try again",
TYPE_DETECTION_FAILED: (fileFormat: string) =>
`type detection failed ${fileFormat}`,
WINDOWS_NATIVE_IMAGE_PROCESSING_NOT_SUPPORTED:
"Windows native image processing is not supported",
NETWORK_ERROR: "Network Error",
@ -69,7 +65,6 @@ export const CustomError = {
AUTH_KEY_NOT_FOUND: "auth key not found",
EXIF_DATA_NOT_FOUND: "exif data not found",
SELECT_FOLDER_ABORTED: "select folder aborted",
NON_MEDIA_FILE: "non media file",
PROCESSING_FAILED: "processing failed",
EXPORT_RECORD_JSON_PARSING_FAILED: "export record json parsing failed",
TWO_FACTOR_ENABLED: "two factor enabled",

124
web/packages/shared/time/index.ts
View file

@ -5,17 +5,6 @@ export interface TimeDelta {
years?: number;
}
interface DateComponent<T = number> {
year: T;
month: T;
day: T;
hour: T;
minute: T;
second: T;
}
const currentYear = new Date().getFullYear();
export function getUnixTimeInMicroSecondsWithDelta(delta: TimeDelta): number {
let currentDate = new Date();
if (delta?.hours) {
@ -71,116 +60,3 @@ function _addYears(date: Date, years: number) {
result.setFullYear(date.getFullYear() + years);
return result;
}
/*
generates data component for date in format YYYYMMDD-HHMMSS
*/
export function parseDateFromFusedDateString(dateTime: string) {
const dateComponent: DateComponent<number> = convertDateComponentToNumber({
year: dateTime.slice(0, 4),
month: dateTime.slice(4, 6),
day: dateTime.slice(6, 8),
hour: dateTime.slice(9, 11),
minute: dateTime.slice(11, 13),
second: dateTime.slice(13, 15),
});
return validateAndGetDateFromComponents(dateComponent);
}
/* sample date format = 2018-08-19 12:34:45
the date has six symbol separated number values
which we would extract and use to form the date
*/
export function tryToParseDateTime(dateTime: string): Date {
const dateComponent = getDateComponentsFromSymbolJoinedString(dateTime);
if (dateComponent.year?.length === 8 && dateComponent.month?.length === 6) {
// the filename has size 8 consecutive and then 6 consecutive digits
// high possibility that the it is a date in format YYYYMMDD-HHMMSS
const possibleDateTime = dateComponent.year + "-" + dateComponent.month;
return parseDateFromFusedDateString(possibleDateTime);
}
return validateAndGetDateFromComponents(
convertDateComponentToNumber(dateComponent),
);
}
function getDateComponentsFromSymbolJoinedString(
dateTime: string,
): DateComponent<string> {
const [year, month, day, hour, minute, second] =
dateTime.match(/\d+/g) ?? [];
return { year, month, day, hour, minute, second };
}
function validateAndGetDateFromComponents(
dateComponent: DateComponent<number>,
options = { minYear: 1990, maxYear: currentYear + 1 },
) {
let date = getDateFromComponents(dateComponent);
if (hasTimeValues(dateComponent) && !isTimePartValid(date, dateComponent)) {
// if the date has time values but they are not valid
// then we remove the time values and try to validate the date
date = getDateFromComponents(removeTimeValues(dateComponent));
}
if (!isDatePartValid(date, dateComponent)) {
return null;
}
if (
date.getFullYear() < options.minYear ||
date.getFullYear() > options.maxYear
) {
return null;
}
return date;
}
function isTimePartValid(date: Date, dateComponent: DateComponent<number>) {
return (
date.getHours() === dateComponent.hour &&
date.getMinutes() === dateComponent.minute &&
date.getSeconds() === dateComponent.second
);
}
function isDatePartValid(date: Date, dateComponent: DateComponent<number>) {
return (
date.getFullYear() === dateComponent.year &&
date.getMonth() === dateComponent.month &&
date.getDate() === dateComponent.day
);
}
function convertDateComponentToNumber(
dateComponent: DateComponent<string>,
): DateComponent<number> {
return {
year: Number(dateComponent.year),
// https://stackoverflow.com/questions/2552483/why-does-the-month-argument-range-from-0-to-11-in-javascripts-date-constructor
month: Number(dateComponent.month) - 1,
day: Number(dateComponent.day),
hour: Number(dateComponent.hour),
minute: Number(dateComponent.minute),
second: Number(dateComponent.second),
};
}
function getDateFromComponents(dateComponent: DateComponent<number>) {
const { year, month, day, hour, minute, second } = dateComponent;
if (hasTimeValues(dateComponent)) {
return new Date(year, month, day, hour, minute, second);
} else {
return new Date(year, month, day);
}
}
function hasTimeValues(dateComponent: DateComponent<number>) {
const { hour, minute, second } = dateComponent;
return !isNaN(hour) && !isNaN(minute) && !isNaN(second);
}
function removeTimeValues(
dateComponent: DateComponent<number>,
): DateComponent<number> {
return { ...dateComponent, hour: 0, minute: 0, second: 0 };
}