update text detection result format
This commit is contained in:
parent
a6e731b09e
commit
4d6d441798
|
@ -199,7 +199,7 @@ const PhotoFrame = ({
|
|||
return false;
|
||||
}
|
||||
|
||||
if (search.text && search.text.indexOf(item.id) === -1) {
|
||||
if (search.text && search.text.files.indexOf(item.id) === -1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
|
@ -34,7 +34,7 @@ import { FILE_TYPE } from 'constants/file';
|
|||
import { GalleryContext } from 'pages/gallery';
|
||||
import { AppContext } from 'pages/_app';
|
||||
import { Col } from 'react-bootstrap';
|
||||
import { Person, ThingClass } from 'types/machineLearning';
|
||||
import { Person, ThingClass, WordGroup } from 'types/machineLearning';
|
||||
import { IndexStatus } from 'types/machineLearning/ui';
|
||||
import { PeopleList } from './MachineLearning/PeopleList';
|
||||
|
||||
|
@ -217,14 +217,18 @@ export default function SearchBar(props: Props) {
|
|||
)
|
||||
);
|
||||
|
||||
const textResult = await searchText(searchPhrase);
|
||||
const textResults = await searchText(searchPhrase);
|
||||
|
||||
options.push({
|
||||
options.push(
|
||||
...textResults.map(
|
||||
(searchResult) =>
|
||||
({
|
||||
type: SuggestionType.TEXT,
|
||||
value: textResult.files,
|
||||
label: textResult.text,
|
||||
} as Suggestion);
|
||||
|
||||
value: searchResult,
|
||||
label: searchResult.word,
|
||||
} as Suggestion)
|
||||
)
|
||||
);
|
||||
return options;
|
||||
};
|
||||
|
||||
|
@ -266,7 +270,7 @@ export default function SearchBar(props: Props) {
|
|||
props.setOpen(true);
|
||||
break;
|
||||
case SuggestionType.TEXT:
|
||||
props.setSearch({ text: selectedOption.value as number[] });
|
||||
props.setSearch({ text: selectedOption.value as WordGroup });
|
||||
props.setOpen(true);
|
||||
break;
|
||||
}
|
||||
|
|
|
@ -2,6 +2,7 @@ import {
|
|||
MLSyncContext,
|
||||
MLSyncFileContext,
|
||||
DetectedText,
|
||||
WordGroup,
|
||||
} from 'types/machineLearning';
|
||||
import { imageBitmapToBlob } from 'utils/image';
|
||||
import { isDifferentOrOld, getAllTextFromMap } from 'utils/machineLearning';
|
||||
|
@ -35,19 +36,24 @@ class TextService {
|
|||
);
|
||||
const textDetections =
|
||||
await syncContext.textDetectionService.detectText(
|
||||
await imageBitmapToBlob(imageBitmap)
|
||||
new File(
|
||||
[await imageBitmapToBlob(imageBitmap)],
|
||||
fileContext.enteFile.id.toString()
|
||||
)
|
||||
);
|
||||
// console.log('3 TF Memory stats: ', tf.memory());
|
||||
// TODO: reenable faces filtering based on width
|
||||
const detectedText: DetectedText = {
|
||||
|
||||
const detectedText: DetectedText[] = textDetections.data.words.map(
|
||||
({ bbox, confidence, text }) => ({
|
||||
fileID: fileContext.enteFile.id,
|
||||
detection: textDetections,
|
||||
};
|
||||
detection: { bbox, confidence, word: text },
|
||||
})
|
||||
);
|
||||
newMlFile.text = detectedText;
|
||||
// ?.filter((f) =>
|
||||
// f.box.width > syncContext.config.faceDetection.minFaceSize
|
||||
// );
|
||||
console.log('[MLService] Detected text: ', newMlFile.text);
|
||||
console.log(
|
||||
'[MLService] Detected text: ',
|
||||
fileContext.enteFile.metadata.title,
|
||||
newMlFile.text
|
||||
);
|
||||
}
|
||||
|
||||
async getAllSyncedTextMap(syncContext: MLSyncContext) {
|
||||
|
@ -59,31 +65,22 @@ class TextService {
|
|||
return syncContext.allSyncedTextMap;
|
||||
}
|
||||
|
||||
public async getAllText() {
|
||||
public async clusterWords(): Promise<WordGroup[]> {
|
||||
const allTextMap = await mlIDbStorage.getAllTextMap();
|
||||
const allText = getAllTextFromMap(allTextMap);
|
||||
return allText;
|
||||
const textCluster = new Map<string, number[]>();
|
||||
allText.map((text) => {
|
||||
if (!textCluster.has(text.detection.word)) {
|
||||
textCluster.set(text.detection.word, []);
|
||||
}
|
||||
const objectsInCluster = textCluster.get(text.detection.word);
|
||||
objectsInCluster.push(text.fileID);
|
||||
});
|
||||
return [...textCluster.entries()].map(([word, files]) => ({
|
||||
word,
|
||||
files,
|
||||
}));
|
||||
}
|
||||
|
||||
// public async clusterThingClasses(
|
||||
// syncContext: MLSyncContext
|
||||
// ): Promise<ThingClass[]> {
|
||||
// const allTextMap = await this.getAllSyncedTextMap(syncContext);
|
||||
// const allText = getAllTextFromMap(allTextMap);
|
||||
// const textCluster = new Map<string, number[]>();
|
||||
// allObjects.map((object) => {
|
||||
// if (!objectClusters.has(object.detection.class)) {
|
||||
// objectClusters.set(object.detection.class, []);
|
||||
// }
|
||||
// const objectsInCluster = objectClusters.get(object.detection.class);
|
||||
// objectsInCluster.push(object.fileID);
|
||||
// });
|
||||
// return [...objectClusters.entries()].map(([className, files], id) => ({
|
||||
// id,
|
||||
// className,
|
||||
// files,
|
||||
// }));
|
||||
// }
|
||||
|
||||
// async syncThingClassesIndex(syncContext: MLSyncContext) {
|
||||
// const filesVersion = await mlIDbStorage.getIndexVersion('files');
|
||||
|
|
|
@ -184,11 +184,12 @@ export async function searchThing(searchPhrase: string) {
|
|||
}
|
||||
|
||||
export async function searchText(searchPhrase: string) {
|
||||
const texts = await textService.getAllText();
|
||||
const files = texts
|
||||
.filter((text) =>
|
||||
text.detection.data.text.toLocaleLowerCase().includes(searchPhrase)
|
||||
)
|
||||
.map(({ fileID }) => fileID);
|
||||
return { text: searchPhrase, files };
|
||||
const texts = await textService.clusterWords();
|
||||
return texts
|
||||
.filter((text) => text.word.toLocaleLowerCase().includes(searchPhrase))
|
||||
.map(({ word, files }) => ({
|
||||
word,
|
||||
files,
|
||||
}))
|
||||
.slice(0, 4);
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
import { SetDialogMessage } from 'components/MessageDialog';
|
||||
import { Collection } from 'types/collection';
|
||||
import { EnteFile } from 'types/file';
|
||||
import { Person, ThingClass } from 'types/machineLearning';
|
||||
import { Person, ThingClass, WordGroup } from 'types/machineLearning';
|
||||
import { DateValue, Bbox } from 'types/search';
|
||||
|
||||
export type SelectedState = {
|
||||
|
@ -20,7 +20,7 @@ export type Search = {
|
|||
fileIndex?: number;
|
||||
person?: Person;
|
||||
thing?: ThingClass;
|
||||
text?: number[];
|
||||
text?: WordGroup;
|
||||
};
|
||||
export interface SearchStats {
|
||||
resultCount: number;
|
||||
|
|
|
@ -205,7 +205,16 @@ export interface ThingClass {
|
|||
files: Array<number>;
|
||||
}
|
||||
|
||||
export declare type TextDetection = Tesseract.RecognizeResult;
|
||||
export interface WordGroup {
|
||||
word: string;
|
||||
files: Array<number>;
|
||||
}
|
||||
|
||||
export interface TextDetection {
|
||||
bbox: Tesseract.Bbox;
|
||||
word: string;
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
export interface DetectedText {
|
||||
fileID: number;
|
||||
|
@ -216,7 +225,7 @@ export interface MlFileData {
|
|||
fileId: number;
|
||||
faces?: Face[];
|
||||
things?: Thing[];
|
||||
text?: DetectedText;
|
||||
text?: DetectedText[];
|
||||
imageSource?: ImageType;
|
||||
imageDimensions?: Dimensions;
|
||||
faceDetectionMethod?: Versioned<FaceDetectionMethod>;
|
||||
|
@ -315,7 +324,7 @@ export interface MLSyncContext {
|
|||
nSyncedFaces: number;
|
||||
allSyncedFacesMap?: Map<number, Array<Face>>;
|
||||
allSyncedThingsMap?: Map<number, Array<Thing>>;
|
||||
allSyncedTextMap?: Map<number, DetectedText>;
|
||||
allSyncedTextMap?: Map<number, Array<DetectedText>>;
|
||||
tsne?: any;
|
||||
|
||||
error?: Error;
|
||||
|
@ -376,7 +385,7 @@ export interface ObjectDetectionService {
|
|||
export interface TextDetectionService {
|
||||
method: Versioned<TextDetectionMethod>;
|
||||
// init(): Promise<void>;
|
||||
detectText(image: Blob): Promise<TextDetection>;
|
||||
detectText(image: File): Promise<Tesseract.RecognizeResult>;
|
||||
dispose(): Promise<void>;
|
||||
}
|
||||
|
||||
|
|
|
@ -205,8 +205,8 @@ export function getAllThingsFromMap(allObjectsMap: Map<number, Array<Thing>>) {
|
|||
return [...allObjectsMap.values()].flat();
|
||||
}
|
||||
|
||||
export function getAllTextFromMap(allTextMap: Map<number, DetectedText>) {
|
||||
return [...allTextMap.values()];
|
||||
export function getAllTextFromMap(allTextMap: Map<number, DetectedText[]>) {
|
||||
return [...allTextMap.values()].flat();
|
||||
}
|
||||
export async function getLocalFile(fileId: number) {
|
||||
const localFiles = await getLocalFiles();
|
||||
|
|
|
@ -315,7 +315,7 @@ class MLIDbStorage {
|
|||
console.time('getAllTextMap');
|
||||
const db = await this.db;
|
||||
const allFiles = await db.getAll('files');
|
||||
const allTextMap = new Map<number, DetectedText>();
|
||||
const allTextMap = new Map<number, DetectedText[]>();
|
||||
allFiles.forEach(
|
||||
(mlFileData) =>
|
||||
mlFileData.text &&
|
||||
|
|
Loading…
Reference in a new issue