update text detection result format

src/components/PhotoFrame.tsx

@@ -199,7 +199,7 @@ const PhotoFrame = ({
                     return false;
                 }
 
-                if (search.text && search.text.indexOf(item.id) === -1) {
+                if (search.text && search.text.files.indexOf(item.id) === -1) {
                     return false;
                 }
 

src/components/SearchBar.tsx

@@ -34,7 +34,7 @@ import { FILE_TYPE } from 'constants/file';
 import { GalleryContext } from 'pages/gallery';
 import { AppContext } from 'pages/_app';
 import { Col } from 'react-bootstrap';
-import { Person, ThingClass } from 'types/machineLearning';
+import { Person, ThingClass, WordGroup } from 'types/machineLearning';
 import { IndexStatus } from 'types/machineLearning/ui';
 import { PeopleList } from './MachineLearning/PeopleList';
 
@@ -217,14 +217,18 @@ export default function SearchBar(props: Props) {
             )
         );
 
-        const textResult = await searchText(searchPhrase);
+        const textResults = await searchText(searchPhrase);
-
-        options.push({
-            type: SuggestionType.TEXT,
-            value: textResult.files,
-            label: textResult.text,
-        } as Suggestion);
 
+        options.push(
+            ...textResults.map(
+                (searchResult) =>
+                    ({
+                        type: SuggestionType.TEXT,
+                        value: searchResult,
+                        label: searchResult.word,
+                    } as Suggestion)
+            )
+        );
         return options;
     };
 
@@ -266,7 +270,7 @@ export default function SearchBar(props: Props) {
                 props.setOpen(true);
                 break;
             case SuggestionType.TEXT:
-                props.setSearch({ text: selectedOption.value as number[] });
+                props.setSearch({ text: selectedOption.value as WordGroup });
                 props.setOpen(true);
                 break;
         }

src/services/machineLearning/textService.ts

@@ -2,6 +2,7 @@ import {
     MLSyncContext,
     MLSyncFileContext,
     DetectedText,
+    WordGroup,
 } from 'types/machineLearning';
 import { imageBitmapToBlob } from 'utils/image';
 import { isDifferentOrOld, getAllTextFromMap } from 'utils/machineLearning';
@@ -35,19 +36,24 @@ class TextService {
         );
         const textDetections =
             await syncContext.textDetectionService.detectText(
-                await imageBitmapToBlob(imageBitmap)
+                new File(
+                    [await imageBitmapToBlob(imageBitmap)],
+                    fileContext.enteFile.id.toString()
+                )
             );
-        // console.log('3 TF Memory stats: ', tf.memory());
+
-        // TODO: reenable faces filtering based on width
+        const detectedText: DetectedText[] = textDetections.data.words.map(
-        const detectedText: DetectedText = {
+            ({ bbox, confidence, text }) => ({
-            fileID: fileContext.enteFile.id,
+                fileID: fileContext.enteFile.id,
-            detection: textDetections,
+                detection: { bbox, confidence, word: text },
-        };
+            })
+        );
         newMlFile.text = detectedText;
-        // ?.filter((f) =>
+        console.log(
-        //     f.box.width > syncContext.config.faceDetection.minFaceSize
+            '[MLService] Detected text: ',
-        // );
+            fileContext.enteFile.metadata.title,
-        console.log('[MLService] Detected text: ', newMlFile.text);
+            newMlFile.text
+        );
     }
 
     async getAllSyncedTextMap(syncContext: MLSyncContext) {
@@ -59,32 +65,23 @@ class TextService {
         return syncContext.allSyncedTextMap;
     }
 
-    public async getAllText() {
+    public async clusterWords(): Promise<WordGroup[]> {
         const allTextMap = await mlIDbStorage.getAllTextMap();
         const allText = getAllTextFromMap(allTextMap);
-        return allText;
+        const textCluster = new Map<string, number[]>();
+        allText.map((text) => {
+            if (!textCluster.has(text.detection.word)) {
+                textCluster.set(text.detection.word, []);
+            }
+            const objectsInCluster = textCluster.get(text.detection.word);
+            objectsInCluster.push(text.fileID);
+        });
+        return [...textCluster.entries()].map(([word, files]) => ({
+            word,
+            files,
+        }));
     }
 
-    // public async clusterThingClasses(
-    //     syncContext: MLSyncContext
-    // ): Promise<ThingClass[]> {
-    //     const allTextMap = await this.getAllSyncedTextMap(syncContext);
-    //     const allText = getAllTextFromMap(allTextMap);
-    //     const textCluster = new Map<string, number[]>();
-    //     allObjects.map((object) => {
-    //         if (!objectClusters.has(object.detection.class)) {
-    //             objectClusters.set(object.detection.class, []);
-    //         }
-    //         const objectsInCluster = objectClusters.get(object.detection.class);
-    //         objectsInCluster.push(object.fileID);
-    //     });
-    //     return [...objectClusters.entries()].map(([className, files], id) => ({
-    //         id,
-    //         className,
-    //         files,
-    //     }));
-    // }
-
     // async syncThingClassesIndex(syncContext: MLSyncContext) {
     //     const filesVersion = await mlIDbStorage.getIndexVersion('files');
     //     console.log(

src/services/searchService.ts

@@ -184,11 +184,12 @@ export async function searchThing(searchPhrase: string) {
 }
 
 export async function searchText(searchPhrase: string) {
-    const texts = await textService.getAllText();
+    const texts = await textService.clusterWords();
-    const files = texts
+    return texts
-        .filter((text) =>
+        .filter((text) => text.word.toLocaleLowerCase().includes(searchPhrase))
-            text.detection.data.text.toLocaleLowerCase().includes(searchPhrase)
+        .map(({ word, files }) => ({
-        )
+            word,
-        .map(({ fileID }) => fileID);
+            files,
-    return { text: searchPhrase, files };
+        }))
+        .slice(0, 4);
 }

src/types/gallery/index.ts

@@ -1,7 +1,7 @@
 import { SetDialogMessage } from 'components/MessageDialog';
 import { Collection } from 'types/collection';
 import { EnteFile } from 'types/file';
-import { Person, ThingClass } from 'types/machineLearning';
+import { Person, ThingClass, WordGroup } from 'types/machineLearning';
 import { DateValue, Bbox } from 'types/search';
 
 export type SelectedState = {
@@ -20,7 +20,7 @@ export type Search = {
     fileIndex?: number;
     person?: Person;
     thing?: ThingClass;
-    text?: number[];
+    text?: WordGroup;
 };
 export interface SearchStats {
     resultCount: number;

src/types/machineLearning/index.ts

@@ -205,7 +205,16 @@ export interface ThingClass {
     files: Array<number>;
 }
 
-export declare type TextDetection = Tesseract.RecognizeResult;
+export interface WordGroup {
+    word: string;
+    files: Array<number>;
+}
+
+export interface TextDetection {
+    bbox: Tesseract.Bbox;
+    word: string;
+    confidence: number;
+}
 
 export interface DetectedText {
     fileID: number;
@@ -216,7 +225,7 @@ export interface MlFileData {
     fileId: number;
     faces?: Face[];
     things?: Thing[];
-    text?: DetectedText;
+    text?: DetectedText[];
     imageSource?: ImageType;
     imageDimensions?: Dimensions;
     faceDetectionMethod?: Versioned<FaceDetectionMethod>;
@@ -315,7 +324,7 @@ export interface MLSyncContext {
     nSyncedFaces: number;
     allSyncedFacesMap?: Map<number, Array<Face>>;
     allSyncedThingsMap?: Map<number, Array<Thing>>;
-    allSyncedTextMap?: Map<number, DetectedText>;
+    allSyncedTextMap?: Map<number, Array<DetectedText>>;
     tsne?: any;
 
     error?: Error;
@@ -376,7 +385,7 @@ export interface ObjectDetectionService {
 export interface TextDetectionService {
     method: Versioned<TextDetectionMethod>;
     // init(): Promise<void>;
-    detectText(image: Blob): Promise<TextDetection>;
+    detectText(image: File): Promise<Tesseract.RecognizeResult>;
     dispose(): Promise<void>;
 }
 

src/utils/machineLearning/index.ts

@@ -205,8 +205,8 @@ export function getAllThingsFromMap(allObjectsMap: Map<number, Array<Thing>>) {
     return [...allObjectsMap.values()].flat();
 }
 
-export function getAllTextFromMap(allTextMap: Map<number, DetectedText>) {
+export function getAllTextFromMap(allTextMap: Map<number, DetectedText[]>) {
-    return [...allTextMap.values()];
+    return [...allTextMap.values()].flat();
 }
 export async function getLocalFile(fileId: number) {
     const localFiles = await getLocalFiles();

src/utils/storage/mlIDbStorage.ts

@@ -315,7 +315,7 @@ class MLIDbStorage {
         console.time('getAllTextMap');
         const db = await this.db;
         const allFiles = await db.getAll('files');
-        const allTextMap = new Map<number, DetectedText>();
+        const allTextMap = new Map<number, DetectedText[]>();
         allFiles.forEach(
             (mlFileData) =>
                 mlFileData.text &&