beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Impl 1

Browse source
This commit is contained in:
Manav Rathi 2024-04-23 11:07:10 +05:30
parent cd22400136
commit 4a12774a3c
No known key found for this signature in database
4 changed files with 158 additions and 112 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

95
web/apps/photos/src/services/ffmpeg.ts
View file

@ -1,4 +1,5 @@
import { ElectronFile } from "@/next/types/file";
import type { Electron } from "@/next/types/ipc";
import { ComlinkWorker } from "@/next/worker/comlink-worker";
import { validateAndGetCreationUnixTimeInMicroSeconds } from "@ente/shared/time";
import { Remote } from "comlink";
@ -12,31 +13,21 @@ import { ParsedExtractedMetadata } from "types/upload";
import { type DedicatedFFmpegWorker } from "worker/ffmpeg.worker";
/**
* Generate a thumbnail of the given video using FFmpeg.
* Generate a thumbnail for the given video using a wasm FFmpeg running in a web
* worker.
*
* This function is called during upload, when we need to generate thumbnails
* for the new files that the user is adding.
*
* @param blob The input video blob.
*
* @returns JPEG data of the generated thumbnail.
*
* See also {@link generateVideoThumbnailNative}.
*/
export const generateVideoThumbnail = async (blob: Blob) => {
export const generateVideoThumbnailWeb = async (blob: Blob) => {
const thumbnailAtTime = (seekTime: number) =>
ffmpegExec(
[
ffmpegPathPlaceholder,
"-i",
inputPathPlaceholder,
"-ss",
`00:00:0${seekTime}`,
"-vframes",
"1",
"-vf",
"scale=-1:720",
outputPathPlaceholder,
],
blob,
);
ffmpegExecWeb(commandForThumbnailAtTime(seekTime), blob, 0);
try {
// Try generating thumbnail at seekTime 1 second.
@ -48,6 +39,50 @@ export const generateVideoThumbnail = async (blob: Blob) => {
}
};
/**
* Generate a thumbnail for the given video using a native FFmpeg binary bundled
* with our desktop app.
*
* This function is called during upload, when we need to generate thumbnails
* for the new files that the user is adding.
*
* @param dataOrPath The input video's data or the path to the video on the
* user's local filesystem. See: [Note: The fileOrPath parameter to upload].
*
* @returns JPEG data of the generated thumbnail.
*
* See also {@link generateVideoThumbnailNative}.
*/
export const generateVideoThumbnailNative = async (
electron: Electron,
dataOrPath: Uint8Array | string,
) => {
const thumbnailAtTime = (seekTime: number) =>
electron.ffmpegExec(commandForThumbnailAtTime(seekTime), dataOrPath, 0);
try {
// Try generating thumbnail at seekTime 1 second.
return await thumbnailAtTime(1);
} catch (e) {
// If that fails, try again at the beginning. If even this throws, let
// it fail.
return await thumbnailAtTime(0);
}
};
const commandForThumbnailAtTime = (seekTime: number) => [
ffmpegPathPlaceholder,
"-i",
inputPathPlaceholder,
"-ss",
`00:00:0${seekTime}`,
"-vframes",
"1",
"-vf",
"scale=-1:720",
outputPathPlaceholder,
];
/** Called during upload */
export async function extractVideoMetadata(file: File | ElectronFile) {
// https://stackoverflow.com/questions/9464617/retrieving-and-saving-media-metadata-using-ffmpeg
@ -157,16 +192,28 @@ export async function convertToMP4(file: File) {
}
/**
* Run the given FFmpeg command.
*
* If we're running in the context of our desktop app, use the FFmpeg binary we
* bundle with our desktop app to run the command. Otherwise fallback to using a
* wasm FFmpeg in a web worker.
* Run the given FFmpeg command using a wasm FFmpeg running in a web worker.
*
* As a rough ballpark, currently the native FFmpeg integration in the desktop
* app is 10-20x faster than the wasm one. See: [Note: FFmpeg in Electron].
*/
const ffmpegExec = async (
const ffmpegExecWeb = async (
command: string[],
blob: Blob,
timeoutMs: number,
) => {
const worker = await workerFactory.lazy();
return await worker.exec(command, blob, timeoutMs);
};
/**
* Run the given FFmpeg command using a native FFmpeg binary bundled with our
* desktop app.
*
* See also: {@link ffmpegExecWeb}.
*/
const ffmpegExecNative = async (
electron: Electron,
command: string[],
blob: Blob,
timeoutMs: number = 0,
@ -176,7 +223,7 @@ const ffmpegExec = async (
const data = new Uint8Array(await blob.arrayBuffer());
return await electron.ffmpegExec(command, data, timeoutMs);
} else {
const worker = await workerFactory.lazy()
const worker = await workerFactory.lazy();
return await worker.exec(command, blob, timeoutMs);
}
};

132
web/apps/photos/src/services/upload/thumbnail.ts
View file

@ -30,79 +30,16 @@ export const generateThumbnail = async (
blob: Blob,
fileTypeInfo: FileTypeInfo,
): Promise<Uint8Array> =>
fileTypeInfo.fileType === FILE_TYPE.IMAGE
? await generateImageThumbnailUsingCanvas(blob, fileTypeInfo)
: await generateVideoThumbnail(blob);
};
/**
* A fallback, black, thumbnail for use in cases where thumbnail generation
* fails.
*/
export const fallbackThumbnail = () =>
Uint8Array.from(atob(BLACK_THUMBNAIL_BASE64), (c) => c.charCodeAt(0));
/**
* Generate a JPEG thumbnail for the given file using native tools.
*
* This function only works when we're running in the context of our desktop
* app, and this dependency is enforced by the need to pass the {@link electron}
* object which we use to perform IPC with the Node.js side of our desktop app.
*
* @param fileOrPath Either the image or video File, or the path to the image or
* video file on the user's local filesystem, whose thumbnail we want to
* generate.
*
* @param fileTypeInfo The type information for the file.
*
* @return The JPEG data of the generated thumbnail.
*
* @see {@link generateThumbnail}.
*/
export const generateThumbnailNative = async (
electron: Electron,
fileOrPath: File | string,
fileTypeInfo: FileTypeInfo,
): Promise<GeneratedThumbnail> => {
try {
const thumbnail =
fileTypeInfo.fileType === FILE_TYPE.IMAGE
? await generateImageThumbnailNative(electron, fileOrPath)
: await generateVideoThumbnail(blob);
if (thumbnail.length == 0) throw new Error("Empty thumbnail");
return { thumbnail, hasStaticThumbnail: false };
} catch (e) {
log.error(`Failed to generate ${fileTypeInfo.exactType} thumbnail`, e);
return { thumbnail: fallbackThumbnail(), hasStaticThumbnail: true };
}
};
const generateImageThumbnailNative = async (
electron: Electron,
fileOrPath: File | string,
): Promise<Uint8Array> => {
const startTime = Date.now();
const jpegData = await electron.generateImageThumbnail(
fileOrPath instanceof File
? new Uint8Array(await fileOrPath.arrayBuffer())
: fileOrPath,
maxThumbnailDimension,
maxThumbnailSize,
);
log.debug(
() => `Native thumbnail generation took ${Date.now() - startTime} ms`,
);
return jpegData;
};
fileTypeInfo.fileType === FILE_TYPE.IMAGE
? await generateImageThumbnailUsingCanvas(blob, fileTypeInfo)
: await generateVideoThumbnail(blob);
const generateImageThumbnailUsingCanvas = async (
blob: Blob,
fileTypeInfo: FileTypeInfo,
) => {
if (isFileHEIC(fileTypeInfo.exactType)) {
log.debug(() => `Pre-converting ${fileTypeInfo.exactType} to JPEG`);
log.debug(() => `Pre-converting HEIC to JPEG for thumbnail generation`);
blob = await heicToJPEG(blob);
}
@ -234,3 +171,64 @@ const percentageSizeDiff = (
newThumbnailSize: number,
oldThumbnailSize: number,
) => ((oldThumbnailSize - newThumbnailSize) * 100) / oldThumbnailSize;
/**
* A fallback, black, thumbnail for use in cases where thumbnail generation
* fails.
*/
export const fallbackThumbnail = () =>
Uint8Array.from(atob(BLACK_THUMBNAIL_BASE64), (c) => c.charCodeAt(0));
/**
* Generate a JPEG thumbnail for the given file using native tools.
*
* This function only works when we're running in the context of our desktop
* app, and this dependency is enforced by the need to pass the {@link electron}
* object which we use to perform IPC with the Node.js side of our desktop app.
*
* @param fileOrPath Either the image or video File, or the path to the image or
* video file on the user's local filesystem, whose thumbnail we want to
* generate.
*
* @param fileTypeInfo The type information for the file.
*
* @return The JPEG data of the generated thumbnail.
*
* @see {@link generateThumbnail}.
*/
export const generateThumbnailNative = async (
electron: Electron,
fileOrPath: File | string,
fileTypeInfo: FileTypeInfo,
): Promise<GeneratedThumbnail> => {
try {
const thumbnail =
fileTypeInfo.fileType === FILE_TYPE.IMAGE
? await generateImageThumbnailNative(electron, fileOrPath)
: await generateVideoThumbnail(blob);
if (thumbnail.length == 0) throw new Error("Empty thumbnail");
return { thumbnail, hasStaticThumbnail: false };
} catch (e) {
log.error(`Failed to generate ${fileTypeInfo.exactType} thumbnail`, e);
return { thumbnail: fallbackThumbnail(), hasStaticThumbnail: true };
}
};
const generateImageThumbnailNative = async (
electron: Electron,
fileOrPath: File | string,
): Promise<Uint8Array> => {
const startTime = Date.now();
const jpegData = await electron.generateImageThumbnail(
fileOrPath instanceof File
? new Uint8Array(await fileOrPath.arrayBuffer())
: fileOrPath,
maxThumbnailDimension,
maxThumbnailSize,
);
log.debug(
() => `Native thumbnail generation took ${Date.now() - startTime} ms`,
);
return jpegData;
};

37
web/apps/photos/src/services/upload/uploadService.ts
View file

@ -2,6 +2,7 @@ import { encodeLivePhoto } from "@/media/live-photo";
import {
basename,
convertBytesToHumanReadable,
fopLabel,
getFileNameSize,
} from "@/next/file";
import log from "@/next/log";
@ -424,26 +425,10 @@ const moduleState = new ModuleState();
* the read during upload using a streaming IPC mechanism.
*/
async function readFile(
fileOrPath
fileOrPath: File | string,
fileTypeInfo: FileTypeInfo,
rawFile: File | ElectronFile,
): Promise<FileInMemory> {
log.info(`reading file data ${getFileNameSize(rawFile)} `);
let filedata: Uint8Array | DataStream;
if (!(rawFile instanceof File)) {
if (rawFile.size > MULTIPART_PART_SIZE) {
filedata = await getElectronFileStream(
rawFile,
FILE_READER_CHUNK_SIZE,
);
} else {
filedata = await getUint8ArrayView(rawFile);
}
} else if (rawFile.size > MULTIPART_PART_SIZE) {
filedata = getFileStream(rawFile, FILE_READER_CHUNK_SIZE);
} else {
filedata = await getUint8ArrayView(rawFile);
}
log.info(`Reading file ${fopLabel(fileOrPath)} `);
let thumbnail: Uint8Array
@ -461,6 +446,22 @@ async function readFile(
}
}
let filedata: Uint8Array | DataStream;
if (!(rawFile instanceof File)) {
if (rawFile.size > MULTIPART_PART_SIZE) {
filedata = await getElectronFileStream(
rawFile,
FILE_READER_CHUNK_SIZE,
);
} else {
filedata = await getUint8ArrayView(rawFile);
}
} else if (rawFile.size > MULTIPART_PART_SIZE) {
filedata = getFileStream(rawFile, FILE_READER_CHUNK_SIZE);
} else {
filedata = await getUint8ArrayView(rawFile);
}
try {
const thumbnail =

6
web/packages/next/file.ts
View file

@ -1,4 +1,4 @@
import type { DesktopFilePath, ElectronFile } from "./types/file";
import type { ElectronFile } from "./types/file";
/**
* The two parts of a file name - the name itself, and an (optional) extension.
@ -70,8 +70,8 @@ export const dirname = (path: string) => {
* Return a short description of the given {@link fileOrPath} suitable for
* helping identify it in log messages.
*/
export const fopLabel = (fileOrPath: File | DesktopFilePath) =>
fileOrPath instanceof File ? `File(${fileOrPath.name})` : fileOrPath.path;
export const fopLabel = (fileOrPath: File | string) =>
fileOrPath instanceof File ? `File(${fileOrPath.name})` : fileOrPath;
export function getFileNameSize(file: File | ElectronFile) {
return `${file.name}_${convertBytesToHumanReadable(file.size)}`;