Merge pull request #122 from ente-io/ffmpeg

Ffmpeg

.gitignore

@@ -45,3 +45,5 @@ public/sw.js
 public/sw.js.map
 public/worker-*.js
 public/worker-*.js.map
+
+public/js/ffmpeg

next.config.js

@@ -11,7 +11,21 @@ const gitSha = cp.execSync('git rev-parse --short HEAD', {
     encoding: 'utf8',
 });
 
-module.exports = withSentryConfig(withWorkbox(withBundleAnalyzer({
+// eslint-disable-next-line camelcase
+const COOP_COEP_Headers = [
+    {
+        key: 'Cross-Origin-Opener-Policy',
+        value: 'same-origin',
+    },
+    {
+        key: 'Cross-Origin-Embedder-Policy',
+        value: 'require-corp',
+    },
+];
+
+module.exports = withSentryConfig(
+    withWorkbox(
+        withBundleAnalyzer({
             future: {
                 webpack5: true,
             },
@@ -27,4 +41,17 @@ module.exports = withSentryConfig(withWorkbox(withBundleAnalyzer({
                     'static/webpack/[fullhash].[runtime].hot-update.json';
                 return config;
             },
-})), { release: gitSha });
+
+            async headers() {
+                return [
+                    {
+                        // Apply these headers to all routes in your application.
+                        source: '/(.*)',
+                        headers: COOP_COEP_Headers,
+                    },
+                ];
+            },
+        })
+    ),
+    { release: gitSha }
+);

package.json

@@ -13,6 +13,8 @@
   },
   "dependencies": {
     "@ente-io/next-with-workbox": "^1.0.3",
+    "@ffmpeg/core": "^0.10.0",
+    "@ffmpeg/ffmpeg": "^0.10.1",
     "@sentry/nextjs": "^6.7.1",
     "@stripe/stripe-js": "^1.13.2",
     "@typescript-eslint/eslint-plugin": "^4.25.0",

src/services/ffmpegService.ts

@@ -0,0 +1,76 @@
+import { createFFmpeg, FFmpeg } from '@ffmpeg/ffmpeg';
+import { CustomError } from 'utils/common/errorUtil';
+import { logError } from 'utils/sentry';
+import QueueProcessor from './upload/queueProcessor';
+import { getUint8ArrayView } from './upload/readFileService';
+
+class FFmpegService {
+    private ffmpeg: FFmpeg = null;
+    private isLoading = null;
+
+    private generateThumbnailProcessor = new QueueProcessor<Uint8Array>(1);
+    async init() {
+        try {
+            this.ffmpeg = createFFmpeg({
+                ...(!process.env.NEXT_PUBLIC_SENTRY_ENV && {
+                    corePath: '/js/ffmpeg/ffmpeg-core.js',
+                }),
+            });
+            this.isLoading = this.ffmpeg.load();
+            await this.isLoading;
+            this.isLoading = null;
+        } catch (e) {
+            logError(e, 'ffmpeg load failed');
+            throw e;
+        }
+    }
+
+    async generateThumbnail(file: File) {
+        if (!this.ffmpeg) {
+            await this.init();
+        }
+        if (this.isLoading) {
+            await this.isLoading;
+        }
+        const response = this.generateThumbnailProcessor.queueUpRequest(
+            generateThumbnailHelper.bind(null, this.ffmpeg, file)
+        );
+
+        const thumbnail = await response.promise;
+        if (!thumbnail) {
+            throw Error(CustomError.THUMBNAIL_GENERATION_FAILED);
+        }
+        return thumbnail;
+    }
+}
+
+async function generateThumbnailHelper(ffmpeg: FFmpeg, file: File) {
+    try {
+        const inputFileName = `${Date.now().toString}-${file.name}`;
+        const thumbFileName = `${Date.now().toString}-thumb.png`;
+        ffmpeg.FS(
+            'writeFile',
+            inputFileName,
+            await getUint8ArrayView(new FileReader(), file)
+        );
+
+        await ffmpeg.run(
+            '-i',
+            inputFileName,
+            '-ss',
+            '00:00:01.000',
+            '-vframes',
+            '1',
+            thumbFileName
+        );
+        const thumb = ffmpeg.FS('readFile', thumbFileName);
+        ffmpeg.FS('unlink', 'thumb.png');
+        ffmpeg.FS('unlink', file.name);
+        return thumb;
+    } catch (e) {
+        logError(e, 'ffmpeg thumbnail generation failed');
+        throw e;
+    }
+}
+
+export default new FFmpegService();

src/services/upload/thumbnailService.ts

@@ -3,6 +3,7 @@ import { CustomError } from 'utils/common/errorUtil';
 import { convertHEIC2JPEG } from 'utils/file';
 import { logError } from 'utils/sentry';
 import { BLACK_THUMBNAIL_BASE64 } from '../../../public/images/black-thumbnail-b64';
+import FFmpegService from 'services/ffmpegService';
 
 const THUMBNAIL_HEIGHT = 720;
 const MAX_ATTEMPTS = 3;
@@ -24,15 +25,20 @@ export async function generateThumbnail(
             if (fileType === FILE_TYPE.IMAGE) {
                 canvas = await generateImageThumbnail(file, isHEIC);
             } else {
+                try {
+                    const thumb = await FFmpegService.generateThumbnail(file);
+                    return { thumbnail: thumb, hasStaticThumbnail: false };
+                } catch (e) {
                     canvas = await generateVideoThumbnail(file);
                 }
+            }
             const thumbnailBlob = await thumbnailCanvasToBlob(canvas);
             thumbnail = await worker.getUint8ArrayView(thumbnailBlob);
             if (thumbnail.length === 0) {
                 throw Error('EMPTY THUMBNAIL');
             }
         } catch (e) {
-            logError(e);
+            logError(e, 'uploading static thumbnail');
             thumbnail = Uint8Array.from(atob(BLACK_THUMBNAIL_BASE64), (c) =>
                 c.charCodeAt(0)
             );
@@ -157,7 +163,7 @@ export async function generateVideoThumbnail(file: globalThis.File) {
     return canvas;
 }
 
-export async function thumbnailCanvasToBlob(canvas: HTMLCanvasElement) {
+async function thumbnailCanvasToBlob(canvas: HTMLCanvasElement) {
     let thumbnailBlob = null;
     let attempts = 0;
     let quality = 1;

yarn.lock

@@ -1005,6 +1005,21 @@
     minimatch "^3.0.4"
     strip-json-comments "^3.1.1"
 
+"@ffmpeg/core@^0.10.0":
+  version "0.10.0"
+  resolved "https://registry.yarnpkg.com/@ffmpeg/core/-/core-0.10.0.tgz#f6a58361b22d7c23c6f7071b9fff6d572bc3f499"
+  integrity sha512-qunWJl5PezpXEm31tb8Qu5z37B5KVA1VYZCpXchMhuAb3X9T7PuE3SlhOwphEoRhzaOa3lpofDfzihAUMFaVPQ==
+
+"@ffmpeg/ffmpeg@^0.10.1":
+  version "0.10.1"
+  resolved "https://registry.yarnpkg.com/@ffmpeg/ffmpeg/-/ffmpeg-0.10.1.tgz#3dacf3985de9c83a95fbf79fe709920cc009b00a"
+  integrity sha512-ChQkH7Rh57hmVo1LhfQFibWX/xqneolJKSwItwZdKPcLZuKigtYAYDIvB55pDfP17VtR1R77SxgkB2/UApB+Og==
+  dependencies:
+    is-url "^1.2.4"
+    node-fetch "^2.6.1"
+    regenerator-runtime "^0.13.7"
+    resolve-url "^0.2.1"
+
 "@hapi/accept@5.0.2":
   version "5.0.2"
   resolved "https://registry.yarnpkg.com/@hapi/accept/-/accept-5.0.2.tgz#ab7043b037e68b722f93f376afb05e85c0699523"
@@ -3985,6 +4000,11 @@ is-unicode-supported@^0.1.0:
   resolved "https://registry.yarnpkg.com/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz#3f26c76a809593b52bfa2ecb5710ed2779b522a7"
   integrity sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==
 
+is-url@^1.2.4:
+  version "1.2.4"
+  resolved "https://registry.yarnpkg.com/is-url/-/is-url-1.2.4.tgz#04a4df46d28c4cff3d73d01ff06abeb318a1aa52"
+  integrity sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==
+
 isarray@^1.0.0, isarray@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
@@ -4534,7 +4554,7 @@ next@^10.2.3:
     vm-browserify "1.1.2"
     watchpack "2.1.1"
 
-node-fetch@2.6.1, node-fetch@^2.6.0:
+node-fetch@2.6.1, node-fetch@^2.6.0, node-fetch@^2.6.1:
   version "2.6.1"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052"
   integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==
@@ -5377,7 +5397,7 @@ regenerator-runtime@^0.11.0:
   resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz#be05ad7f9bf7d22e056f9726cee5017fbf19e2e9"
   integrity sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==
 
-regenerator-runtime@^0.13.4:
+regenerator-runtime@^0.13.4, regenerator-runtime@^0.13.7:
   version "0.13.9"
   resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz#8925742a98ffd90814988d7566ad30ca3b263b52"
   integrity sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==
@@ -5441,6 +5461,11 @@ resolve-from@^4.0.0:
   resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6"
   integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==
 
+resolve-url@^0.2.1:
+  version "0.2.1"
+  resolved "https://registry.yarnpkg.com/resolve-url/-/resolve-url-0.2.1.tgz#2c637fe77c893afd2a663fe21aa9080068e2052a"
+  integrity sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=
+
 resolve@1.1.7:
   version "1.1.7"
   resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.1.7.tgz#203114d82ad2c5ed9e8e0411b3932875e889e97b"