From f6d550d689ccb7928ca1a98d21d214ab5ab88032 Mon Sep 17 00:00:00 2001
From: Rushikesh Tote <rushi.tote@gmail.com>
Date: Mon, 30 May 2022 20:21:12 +0530
Subject: [PATCH] move scene minScore to config

---
 src/constants/machineLearning/config.ts       |  1 +
 src/pages/scene-debug/index.tsx               |  8 +++-
 .../machineLearning/imageSceneService.ts      | 47 +++----------------
 src/services/machineLearning/objectService.ts |  3 +-
 src/types/machineLearning/index.ts            |  6 ++-
 5 files changed, 21 insertions(+), 44 deletions(-)

diff --git a/src/constants/machineLearning/config.ts b/src/constants/machineLearning/config.ts
index 4355c9d88..2bcebb7d0 100644
--- a/src/constants/machineLearning/config.ts
+++ b/src/constants/machineLearning/config.ts
@@ -54,6 +54,7 @@ export const DEFAULT_ML_SYNC_CONFIG: MLSyncConfig = {
     },
     sceneDetection: {
         method: 'Image-Scene',
+        minScore: 0.1,
     },
     // tsne: {
     //     samples: 200,
diff --git a/src/pages/scene-debug/index.tsx b/src/pages/scene-debug/index.tsx
index 2a7153346..966121194 100644
--- a/src/pages/scene-debug/index.tsx
+++ b/src/pages/scene-debug/index.tsx
@@ -10,7 +10,13 @@ function SceneDebug() {
 
     const handleSubmission = async () => {
         for (const file of selectedFiles) {
-            await sceneDetectionService.detectByFile(file);
+            console.log(
+                `scene detection for file ${file.name}`,
+                await sceneDetectionService.detectScenes(
+                    await createImageBitmap(file),
+                    0.1
+                )
+            );
         }
         console.log('done with scene detection');
     };
diff --git a/src/services/machineLearning/imageSceneService.ts b/src/services/machineLearning/imageSceneService.ts
index b092a9f91..6639f2f5d 100644
--- a/src/services/machineLearning/imageSceneService.ts
+++ b/src/services/machineLearning/imageSceneService.ts
@@ -7,8 +7,6 @@ import {
 } from 'types/machineLearning';
 import sceneMap from 'utils/machineLearning/sceneMap';
 
-const MIN_SCENE_DETECTION_SCORE = 0.1;
-
 class ImageScene implements SceneDetectionService {
     method: Versioned<SceneDetectionMethod>;
     model: tf.GraphModel;
@@ -30,42 +28,7 @@ class ImageScene implements SceneDetectionService {
         this.model = model;
     }
 
-    async detectByFile(file: File) {
-        const bmp = await createImageBitmap(file);
-
-        await tf.ready();
-
-        if (!this.model) {
-            await this.init();
-        }
-
-        const currTime = new Date().getTime();
-        const output = tf.tidy(() => {
-            let tensor = tf.browser.fromPixels(bmp);
-
-            tensor = tf.image.resizeBilinear(tensor, [224, 224]);
-            tensor = tf.expandDims(tensor);
-            tensor = tf.cast(tensor, 'float32');
-
-            const output = this.model.predict(tensor, {
-                verbose: true,
-            });
-
-            return output;
-        });
-
-        console.log('done in', new Date().getTime() - currTime, 'ms');
-
-        const data = await (output as tf.Tensor).data();
-        const scenes = this.getScenes(
-            data as Float32Array,
-            bmp.width,
-            bmp.height
-        );
-        console.log(`scenes for ${file.name}`, scenes);
-    }
-
-    async detectScenes(image: ImageBitmap) {
+    async detectScenes(image: ImageBitmap, minScore: number) {
         await tf.ready();
 
         if (!this.model) {
@@ -88,7 +51,8 @@ class ImageScene implements SceneDetectionService {
         const scenes = this.getScenes(
             data as Float32Array,
             image.width,
-            image.height
+            image.height,
+            minScore
         );
 
         return scenes;
@@ -97,11 +61,12 @@ class ImageScene implements SceneDetectionService {
     private getScenes(
         outputData: Float32Array,
         width: number,
-        height: number
+        height: number,
+        minScore: number
     ): ObjectDetection[] {
         const scenes = [];
         for (let i = 0; i < outputData.length; i++) {
-            if (outputData[i] >= MIN_SCENE_DETECTION_SCORE) {
+            if (outputData[i] >= minScore) {
                 scenes.push({
                     class: sceneMap.get(i),
                     score: outputData[i],
diff --git a/src/services/machineLearning/objectService.ts b/src/services/machineLearning/objectService.ts
index 30753da0c..7000ad355 100644
--- a/src/services/machineLearning/objectService.ts
+++ b/src/services/machineLearning/objectService.ts
@@ -56,7 +56,8 @@ class ObjectService {
             );
         objectDetections.push(
             ...(await syncContext.sceneDetectionService.detectScenes(
-                imageBitmap
+                imageBitmap,
+                syncContext.config.sceneDetection.minScore
             ))
         );
         // console.log('3 TF Memory stats: ', tf.memory());
diff --git a/src/types/machineLearning/index.ts b/src/types/machineLearning/index.ts
index c23429ad0..b4c57fb67 100644
--- a/src/types/machineLearning/index.ts
+++ b/src/types/machineLearning/index.ts
@@ -255,6 +255,7 @@ export interface ObjectDetectionConfig {
 
 export interface SceneDetectionConfig {
     method: SceneDetectionMethod;
+    minScore: number;
 }
 
 export interface TextDetectionConfig {
@@ -393,7 +394,10 @@ export interface ObjectDetectionService {
 export interface SceneDetectionService {
     method: Versioned<SceneDetectionMethod>;
     // init(): Promise<void>;
-    detectScenes(image: ImageBitmap): Promise<ObjectDetection[]>;
+    detectScenes(
+        image: ImageBitmap,
+        minScore: number
+    ): Promise<ObjectDetection[]>;
 }
 
 export interface TextDetectionService {