From cfced851c66edb2e91ed93c2099bc41f2f502df8 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sun, 21 Apr 2024 10:26:17 +0530 Subject: [PATCH] Expectation --- web/apps/photos/src/constants/ffmpeg.ts | 6 +-- web/apps/photos/src/services/ffmpeg.ts | 37 ++++++-------- web/apps/photos/src/worker/ffmpeg.worker.ts | 54 +++++++++------------ web/packages/next/types/ipc.ts | 37 ++++++++++++-- 4 files changed, 76 insertions(+), 58 deletions(-) diff --git a/web/apps/photos/src/constants/ffmpeg.ts b/web/apps/photos/src/constants/ffmpeg.ts index 9ecc41eb5..fb0d762e5 100644 --- a/web/apps/photos/src/constants/ffmpeg.ts +++ b/web/apps/photos/src/constants/ffmpeg.ts @@ -1,3 +1,3 @@ -export const INPUT_PATH_PLACEHOLDER = "INPUT"; -export const FFMPEG_PLACEHOLDER = "FFMPEG"; -export const OUTPUT_PATH_PLACEHOLDER = "OUTPUT"; +export const ffmpegPathPlaceholder = "FFMPEG"; +export const inputPathPlaceholder = "INPUT"; +export const outputPathPlaceholder = "OUTPUT"; diff --git a/web/apps/photos/src/services/ffmpeg.ts b/web/apps/photos/src/services/ffmpeg.ts index ea1e6b4b2..17833f426 100644 --- a/web/apps/photos/src/services/ffmpeg.ts +++ b/web/apps/photos/src/services/ffmpeg.ts @@ -2,9 +2,9 @@ import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { validateAndGetCreationUnixTimeInMicroSeconds } from "@ente/shared/time"; import { Remote } from "comlink"; import { - FFMPEG_PLACEHOLDER, - INPUT_PATH_PLACEHOLDER, - OUTPUT_PATH_PLACEHOLDER, + ffmpegPathPlaceholder, + inputPathPlaceholder, + outputPathPlaceholder, } from "constants/ffmpeg"; import { NULL_LOCATION } from "constants/upload"; import { ElectronFile, ParsedExtractedMetadata } from "types/upload"; @@ -19,16 +19,16 @@ export async function generateVideoThumbnail( try { return await ffmpegExec( [ - FFMPEG_PLACEHOLDER, + ffmpegPathPlaceholder, "-i", - INPUT_PATH_PLACEHOLDER, + inputPathPlaceholder, "-ss", `00:00:0${seekTime}`, "-vframes", "1", "-vf", "scale=-1:720", - OUTPUT_PATH_PLACEHOLDER, + outputPathPlaceholder, ], file, "thumb.jpeg", @@ -50,16 +50,16 @@ export async function extractVideoMetadata(file: File | ElectronFile) { // -f ffmetadata [https://ffmpeg.org/ffmpeg-formats.html#Metadata-1] => dump metadata from media files into a simple UTF-8-encoded INI-like text file const metadata = await ffmpegExec( [ - FFMPEG_PLACEHOLDER, + ffmpegPathPlaceholder, "-i", - INPUT_PATH_PLACEHOLDER, + inputPathPlaceholder, "-c", "copy", "-map_metadata", "0", "-f", "ffmetadata", - OUTPUT_PATH_PLACEHOLDER, + outputPathPlaceholder, ], file, `metadata.txt`, @@ -137,16 +137,16 @@ function parseCreationTime(creationTime: string) { export async function convertToMP4(file: File) { return await ffmpegExec( [ - FFMPEG_PLACEHOLDER, + ffmpegPathPlaceholder, "-i", - INPUT_PATH_PLACEHOLDER, + inputPathPlaceholder, "-preset", "ultrafast", - OUTPUT_PATH_PLACEHOLDER, + outputPathPlaceholder, ], file, "output.mp4", - true, + 30 * 1000, ); } @@ -164,21 +164,16 @@ const ffmpegExec = async ( cmd: string[], inputFile: File | ElectronFile, outputFilename: string, - dontTimeout?: boolean, + timeoutMS: number = 0, ): Promise => { const electron = globalThis.electron; if (electron) { - return electron.runFFmpegCmd( - cmd, - inputFile, - outputFilename, - dontTimeout, - ); + return electron.runFFmpegCmd(cmd, inputFile, outputFilename, timeoutMS); } else { return workerFactory .instance() .then((worker) => - worker.run(cmd, inputFile, outputFilename, dontTimeout), + worker.run(cmd, inputFile, outputFilename, timeoutMS), ); } }; diff --git a/web/apps/photos/src/worker/ffmpeg.worker.ts b/web/apps/photos/src/worker/ffmpeg.worker.ts index 7d75db58e..2e6045008 100644 --- a/web/apps/photos/src/worker/ffmpeg.worker.ts +++ b/web/apps/photos/src/worker/ffmpeg.worker.ts @@ -3,29 +3,34 @@ import log from "@/next/log"; import { withTimeout } from "@ente/shared/utils"; import QueueProcessor from "@ente/shared/utils/queueProcessor"; import { generateTempName } from "@ente/shared/utils/temp"; -import * as Comlink from "comlink"; +import { expose } from "comlink"; import { - FFMPEG_PLACEHOLDER, - INPUT_PATH_PLACEHOLDER, - OUTPUT_PATH_PLACEHOLDER, + ffmpegPathPlaceholder, + inputPathPlaceholder, + outputPathPlaceholder, } from "constants/ffmpeg"; import { FFmpeg, createFFmpeg } from "ffmpeg-wasm"; import { getUint8ArrayView } from "services/readerService"; export class DedicatedFFmpegWorker { - wasmFFmpeg: WasmFFmpeg; + private wasmFFmpeg: WasmFFmpeg; + constructor() { this.wasmFFmpeg = new WasmFFmpeg(); } - run(cmd, inputFile, outputFileName, dontTimeout) { - return this.wasmFFmpeg.run(cmd, inputFile, outputFileName, dontTimeout); + /** + * Execute a FFMPEG {@link command}. + * + * This is a sibling of {@link ffmpegExec} in ipc.ts exposed by the desktop + * app. See [Note: FFMPEG in Electron]. + */ + run(cmd, inputFile, outputFileName, timeoutMS) { + return this.wasmFFmpeg.run(cmd, inputFile, outputFileName, timeoutMS); } } -Comlink.expose(DedicatedFFmpegWorker, self); - -const FFMPEG_EXECUTION_WAIT_TIME = 30 * 1000; +expose(DedicatedFFmpegWorker, self); export class WasmFFmpeg { private ffmpeg: FFmpeg; @@ -51,24 +56,13 @@ export class WasmFFmpeg { cmd: string[], inputFile: File, outputFileName: string, - dontTimeout = false, + timeoutMS, ) { - const response = this.ffmpegTaskQueue.queueUpRequest(() => { - if (dontTimeout) { - return this.execute(cmd, inputFile, outputFileName); - } else { - return withTimeout( - this.execute(cmd, inputFile, outputFileName), - FFMPEG_EXECUTION_WAIT_TIME, - ); - } - }); - try { - return await response.promise; - } catch (e) { - log.error("ffmpeg run failed", e); - throw e; - } + const exec = () => this.execute(cmd, inputFile, outputFileName); + const request = this.ffmpegTaskQueue.queueUpRequest(() => + timeoutMS ? withTimeout(exec(), timeoutMS) : exec(), + ); + return await request.promise; } private async execute( @@ -91,11 +85,11 @@ export class WasmFFmpeg { tempOutputFilePath = `${generateTempName(10, outputFileName)}`; cmd = cmd.map((cmdPart) => { - if (cmdPart === FFMPEG_PLACEHOLDER) { + if (cmdPart === ffmpegPathPlaceholder) { return ""; - } else if (cmdPart === INPUT_PATH_PLACEHOLDER) { + } else if (cmdPart === inputPathPlaceholder) { return tempInputFilePath; - } else if (cmdPart === OUTPUT_PATH_PLACEHOLDER) { + } else if (cmdPart === outputPathPlaceholder) { return tempOutputFilePath; } else { return cmdPart; diff --git a/web/packages/next/types/ipc.ts b/web/packages/next/types/ipc.ts index 10aeeae4f..400067153 100644 --- a/web/packages/next/types/ipc.ts +++ b/web/packages/next/types/ipc.ts @@ -236,11 +236,40 @@ export interface Electron { maxSize: number, ) => Promise; - runFFmpegCmd: ( - cmd: string[], - inputFile: File | ElectronFile, + /** + * Execute a FFMPEG {@link command}. + * + * This executes the command using the FFMPEG executable we bundle with our + * desktop app. There is also a FFMPEG WASM implementation that we use when + * running on the web, it also has a sibling function with the same + * parameters. See [Note: FFMPEG in Electron]. + * + * @param command An array of strings, each representing one positional + * parameter in the command to execute. Placeholders for the input, output + * and ffmpeg's own path are replaced before executing the command + * (respectively {@link inputPathPlaceholder}, + * {@link outputPathPlaceholder}, {@link ffmpegPathPlaceholder}). + * + * @param inputDataOrPath The bytes of the input file, or the path to the + * input file on the user's local disk. In both cases, the data gets + * serialized to a temporary file, and then that path gets substituted in + * the FFMPEG {@link command} by {@link inputPathPlaceholder}. + * + * @param outputFileName The name of the file we instruct FFMPEG to produce + * when giving it the given {@link command}. The contents of this file get + * returned as the result. + * + * @param timeoutMS If non-zero, then throw a timeout error if the FFMPEG + * command takes more than the given number of milliseconds. + * + * @returns The contents of the output file produced by the ffmpeg command + * at {@link outputFileName}. + */ + ffmpegExec: ( + command: string[], + inputDataOrPath: Uint8Array | string, outputFileName: string, - dontTimeout?: boolean, + timeoutMS: number, ) => Promise; // - ML