diff --git a/web/apps/photos/src/services/ffmpeg.ts b/web/apps/photos/src/services/ffmpeg.ts index 3c3f58a5f..2778efb26 100644 --- a/web/apps/photos/src/services/ffmpeg.ts +++ b/web/apps/photos/src/services/ffmpeg.ts @@ -1,4 +1,5 @@ import { ElectronFile } from "@/next/types/file"; +import type { Electron } from "@/next/types/ipc"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { validateAndGetCreationUnixTimeInMicroSeconds } from "@ente/shared/time"; import { Remote } from "comlink"; @@ -12,31 +13,21 @@ import { ParsedExtractedMetadata } from "types/upload"; import { type DedicatedFFmpegWorker } from "worker/ffmpeg.worker"; /** - * Generate a thumbnail of the given video using FFmpeg. + * Generate a thumbnail for the given video using a wasm FFmpeg running in a web + * worker. * * This function is called during upload, when we need to generate thumbnails * for the new files that the user is adding. * * @param blob The input video blob. + * * @returns JPEG data of the generated thumbnail. + * + * See also {@link generateVideoThumbnailNative}. */ -export const generateVideoThumbnail = async (blob: Blob) => { +export const generateVideoThumbnailWeb = async (blob: Blob) => { const thumbnailAtTime = (seekTime: number) => - ffmpegExec( - [ - ffmpegPathPlaceholder, - "-i", - inputPathPlaceholder, - "-ss", - `00:00:0${seekTime}`, - "-vframes", - "1", - "-vf", - "scale=-1:720", - outputPathPlaceholder, - ], - blob, - ); + ffmpegExecWeb(commandForThumbnailAtTime(seekTime), blob, 0); try { // Try generating thumbnail at seekTime 1 second. @@ -48,6 +39,50 @@ export const generateVideoThumbnail = async (blob: Blob) => { } }; +/** + * Generate a thumbnail for the given video using a native FFmpeg binary bundled + * with our desktop app. + * + * This function is called during upload, when we need to generate thumbnails + * for the new files that the user is adding. + * + * @param dataOrPath The input video's data or the path to the video on the + * user's local filesystem. See: [Note: The fileOrPath parameter to upload]. + * + * @returns JPEG data of the generated thumbnail. + * + * See also {@link generateVideoThumbnailNative}. + */ +export const generateVideoThumbnailNative = async ( + electron: Electron, + dataOrPath: Uint8Array | string, +) => { + const thumbnailAtTime = (seekTime: number) => + electron.ffmpegExec(commandForThumbnailAtTime(seekTime), dataOrPath, 0); + + try { + // Try generating thumbnail at seekTime 1 second. + return await thumbnailAtTime(1); + } catch (e) { + // If that fails, try again at the beginning. If even this throws, let + // it fail. + return await thumbnailAtTime(0); + } +}; + +const commandForThumbnailAtTime = (seekTime: number) => [ + ffmpegPathPlaceholder, + "-i", + inputPathPlaceholder, + "-ss", + `00:00:0${seekTime}`, + "-vframes", + "1", + "-vf", + "scale=-1:720", + outputPathPlaceholder, +]; + /** Called during upload */ export async function extractVideoMetadata(file: File | ElectronFile) { // https://stackoverflow.com/questions/9464617/retrieving-and-saving-media-metadata-using-ffmpeg @@ -157,16 +192,28 @@ export async function convertToMP4(file: File) { } /** - * Run the given FFmpeg command. - * - * If we're running in the context of our desktop app, use the FFmpeg binary we - * bundle with our desktop app to run the command. Otherwise fallback to using a - * wasm FFmpeg in a web worker. + * Run the given FFmpeg command using a wasm FFmpeg running in a web worker. * * As a rough ballpark, currently the native FFmpeg integration in the desktop * app is 10-20x faster than the wasm one. See: [Note: FFmpeg in Electron]. */ -const ffmpegExec = async ( +const ffmpegExecWeb = async ( + command: string[], + blob: Blob, + timeoutMs: number, +) => { + const worker = await workerFactory.lazy(); + return await worker.exec(command, blob, timeoutMs); +}; + +/** + * Run the given FFmpeg command using a native FFmpeg binary bundled with our + * desktop app. + * + * See also: {@link ffmpegExecWeb}. + */ +const ffmpegExecNative = async ( + electron: Electron, command: string[], blob: Blob, timeoutMs: number = 0, @@ -176,7 +223,7 @@ const ffmpegExec = async ( const data = new Uint8Array(await blob.arrayBuffer()); return await electron.ffmpegExec(command, data, timeoutMs); } else { - const worker = await workerFactory.lazy() + const worker = await workerFactory.lazy(); return await worker.exec(command, blob, timeoutMs); } }; diff --git a/web/apps/photos/src/services/upload/thumbnail.ts b/web/apps/photos/src/services/upload/thumbnail.ts index b9a648221..01bb6c4f0 100644 --- a/web/apps/photos/src/services/upload/thumbnail.ts +++ b/web/apps/photos/src/services/upload/thumbnail.ts @@ -30,79 +30,16 @@ export const generateThumbnail = async ( blob: Blob, fileTypeInfo: FileTypeInfo, ): Promise => - fileTypeInfo.fileType === FILE_TYPE.IMAGE - ? await generateImageThumbnailUsingCanvas(blob, fileTypeInfo) - : await generateVideoThumbnail(blob); - -}; - -/** - * A fallback, black, thumbnail for use in cases where thumbnail generation - * fails. - */ -export const fallbackThumbnail = () => - Uint8Array.from(atob(BLACK_THUMBNAIL_BASE64), (c) => c.charCodeAt(0)); - -/** - * Generate a JPEG thumbnail for the given file using native tools. - * - * This function only works when we're running in the context of our desktop - * app, and this dependency is enforced by the need to pass the {@link electron} - * object which we use to perform IPC with the Node.js side of our desktop app. - * - * @param fileOrPath Either the image or video File, or the path to the image or - * video file on the user's local filesystem, whose thumbnail we want to - * generate. - * - * @param fileTypeInfo The type information for the file. - * - * @return The JPEG data of the generated thumbnail. - * - * @see {@link generateThumbnail}. - */ -export const generateThumbnailNative = async ( - electron: Electron, - fileOrPath: File | string, - fileTypeInfo: FileTypeInfo, -): Promise => { - try { - const thumbnail = - fileTypeInfo.fileType === FILE_TYPE.IMAGE - ? await generateImageThumbnailNative(electron, fileOrPath) - : await generateVideoThumbnail(blob); - - if (thumbnail.length == 0) throw new Error("Empty thumbnail"); - return { thumbnail, hasStaticThumbnail: false }; - } catch (e) { - log.error(`Failed to generate ${fileTypeInfo.exactType} thumbnail`, e); - return { thumbnail: fallbackThumbnail(), hasStaticThumbnail: true }; - } -}; - -const generateImageThumbnailNative = async ( - electron: Electron, - fileOrPath: File | string, -): Promise => { - const startTime = Date.now(); - const jpegData = await electron.generateImageThumbnail( - fileOrPath instanceof File - ? new Uint8Array(await fileOrPath.arrayBuffer()) - : fileOrPath, - maxThumbnailDimension, - maxThumbnailSize, - ); - log.debug( - () => `Native thumbnail generation took ${Date.now() - startTime} ms`, - ); - return jpegData; -}; + fileTypeInfo.fileType === FILE_TYPE.IMAGE + ? await generateImageThumbnailUsingCanvas(blob, fileTypeInfo) + : await generateVideoThumbnail(blob); const generateImageThumbnailUsingCanvas = async ( blob: Blob, fileTypeInfo: FileTypeInfo, ) => { if (isFileHEIC(fileTypeInfo.exactType)) { - log.debug(() => `Pre-converting ${fileTypeInfo.exactType} to JPEG`); + log.debug(() => `Pre-converting HEIC to JPEG for thumbnail generation`); blob = await heicToJPEG(blob); } @@ -234,3 +171,64 @@ const percentageSizeDiff = ( newThumbnailSize: number, oldThumbnailSize: number, ) => ((oldThumbnailSize - newThumbnailSize) * 100) / oldThumbnailSize; + +/** + * A fallback, black, thumbnail for use in cases where thumbnail generation + * fails. + */ +export const fallbackThumbnail = () => + Uint8Array.from(atob(BLACK_THUMBNAIL_BASE64), (c) => c.charCodeAt(0)); + +/** + * Generate a JPEG thumbnail for the given file using native tools. + * + * This function only works when we're running in the context of our desktop + * app, and this dependency is enforced by the need to pass the {@link electron} + * object which we use to perform IPC with the Node.js side of our desktop app. + * + * @param fileOrPath Either the image or video File, or the path to the image or + * video file on the user's local filesystem, whose thumbnail we want to + * generate. + * + * @param fileTypeInfo The type information for the file. + * + * @return The JPEG data of the generated thumbnail. + * + * @see {@link generateThumbnail}. + */ +export const generateThumbnailNative = async ( + electron: Electron, + fileOrPath: File | string, + fileTypeInfo: FileTypeInfo, +): Promise => { + try { + const thumbnail = + fileTypeInfo.fileType === FILE_TYPE.IMAGE + ? await generateImageThumbnailNative(electron, fileOrPath) + : await generateVideoThumbnail(blob); + + if (thumbnail.length == 0) throw new Error("Empty thumbnail"); + return { thumbnail, hasStaticThumbnail: false }; + } catch (e) { + log.error(`Failed to generate ${fileTypeInfo.exactType} thumbnail`, e); + return { thumbnail: fallbackThumbnail(), hasStaticThumbnail: true }; + } +}; + +const generateImageThumbnailNative = async ( + electron: Electron, + fileOrPath: File | string, +): Promise => { + const startTime = Date.now(); + const jpegData = await electron.generateImageThumbnail( + fileOrPath instanceof File + ? new Uint8Array(await fileOrPath.arrayBuffer()) + : fileOrPath, + maxThumbnailDimension, + maxThumbnailSize, + ); + log.debug( + () => `Native thumbnail generation took ${Date.now() - startTime} ms`, + ); + return jpegData; +}; diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index b2ca9c063..f49a2a1fb 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -2,6 +2,7 @@ import { encodeLivePhoto } from "@/media/live-photo"; import { basename, convertBytesToHumanReadable, + fopLabel, getFileNameSize, } from "@/next/file"; import log from "@/next/log"; @@ -424,26 +425,10 @@ const moduleState = new ModuleState(); * the read during upload using a streaming IPC mechanism. */ async function readFile( - fileOrPath + fileOrPath: File | string, fileTypeInfo: FileTypeInfo, - rawFile: File | ElectronFile, ): Promise { - log.info(`reading file data ${getFileNameSize(rawFile)} `); - let filedata: Uint8Array | DataStream; - if (!(rawFile instanceof File)) { - if (rawFile.size > MULTIPART_PART_SIZE) { - filedata = await getElectronFileStream( - rawFile, - FILE_READER_CHUNK_SIZE, - ); - } else { - filedata = await getUint8ArrayView(rawFile); - } - } else if (rawFile.size > MULTIPART_PART_SIZE) { - filedata = getFileStream(rawFile, FILE_READER_CHUNK_SIZE); - } else { - filedata = await getUint8ArrayView(rawFile); - } + log.info(`Reading file ${fopLabel(fileOrPath)} `); let thumbnail: Uint8Array @@ -461,6 +446,22 @@ async function readFile( } } + let filedata: Uint8Array | DataStream; + if (!(rawFile instanceof File)) { + if (rawFile.size > MULTIPART_PART_SIZE) { + filedata = await getElectronFileStream( + rawFile, + FILE_READER_CHUNK_SIZE, + ); + } else { + filedata = await getUint8ArrayView(rawFile); + } + } else if (rawFile.size > MULTIPART_PART_SIZE) { + filedata = getFileStream(rawFile, FILE_READER_CHUNK_SIZE); + } else { + filedata = await getUint8ArrayView(rawFile); + } + try { const thumbnail = diff --git a/web/packages/next/file.ts b/web/packages/next/file.ts index 4d05225c8..02f936a18 100644 --- a/web/packages/next/file.ts +++ b/web/packages/next/file.ts @@ -1,4 +1,4 @@ -import type { DesktopFilePath, ElectronFile } from "./types/file"; +import type { ElectronFile } from "./types/file"; /** * The two parts of a file name - the name itself, and an (optional) extension. @@ -70,8 +70,8 @@ export const dirname = (path: string) => { * Return a short description of the given {@link fileOrPath} suitable for * helping identify it in log messages. */ -export const fopLabel = (fileOrPath: File | DesktopFilePath) => - fileOrPath instanceof File ? `File(${fileOrPath.name})` : fileOrPath.path; +export const fopLabel = (fileOrPath: File | string) => + fileOrPath instanceof File ? `File(${fileOrPath.name})` : fileOrPath; export function getFileNameSize(file: File | ElectronFile) { return `${file.name}_${convertBytesToHumanReadable(file.size)}`;