Distribute noise to nearest initial clusters if distance is within limit

src/components/MlDebug.tsx

@@ -17,7 +17,7 @@ export default function MLDebug() {
     const [batchSize, setBatchSize] = useState<number>(50);
     const [mlResult, setMlResult] = useState<MLSyncResult>({
         allFaces: [],
-        clusterResults: {
+        clustersWithNoise: {
             clusters: [],
             noise: [],
         },
@@ -111,12 +111,12 @@ export default function MLDebug() {
             <p></p>
             <button onClick={onSync}>Run ML Sync</button>
 
-            <p>{JSON.stringify(mlResult.clusterResults)}</p>
+            <p>{JSON.stringify(mlResult.clustersWithNoise)}</p>
             <div>
                 <p>Clusters: </p>
-                {mlResult.clusterResults.clusters.map((cluster, index) => (
+                {mlResult.clustersWithNoise.clusters.map((cluster, index) => (
                     <div key={index} style={{ display: 'flex' }}>
-                        {cluster.map((faceIndex, ind) => (
+                        {cluster.faces.map((faceIndex, ind) => (
                             <div key={ind}>
                                 <TFJSImage
                                     faceImage={
@@ -129,14 +129,16 @@ export default function MLDebug() {
 
                 <p style={{ marginTop: '1em' }}>Noise: </p>
                 <div style={{ display: 'flex' }}>
-                    {mlResult.clusterResults.noise.map((faceIndex, index) => (
+                    {mlResult.clustersWithNoise.noise.map(
-                        <div key={index}>
+                        (faceIndex, index) => (
-                            <TFJSImage
+                            <div key={index}>
-                                faceImage={
+                                <TFJSImage
-                                    mlResult.allFaces[faceIndex].faceImage
+                                    faceImage={
-                                }></TFJSImage>
+                                        mlResult.allFaces[faceIndex].faceImage
-                        </div>
+                                    }></TFJSImage>
-                    ))}
+                            </div>
+                        )
+                    )}
                 </div>
             </div>
         </div>

src/services/machineLearning/clusteringService.ts

@@ -1,4 +1,5 @@
 import { DBSCAN, OPTICS, KMEANS } from 'density-clustering';
+import { ClusteringResults } from 'utils/machineLearning/types';
 
 class ClusteringService {
     private dbscan: DBSCAN;
@@ -15,7 +16,7 @@ class ClusteringService {
         dataset: Array<Array<number>>,
         epsilon: number = 1.0,
         minPts: number = 2
-    ) {
+    ): ClusteringResults {
         // console.log("distanceFunction", DBSCAN._);
         const clusters = this.dbscan.run(dataset, epsilon, minPts);
         const noise = this.dbscan.noise;

src/services/machineLearning/machineLearningService.ts

@@ -7,10 +7,16 @@ import * as tf from '@tensorflow/tfjs-core';
 // import TFJSFaceDetectionService from './tfjsFaceDetectionService';
 // import TFJSFaceEmbeddingService from './tfjsFaceEmbeddingService';
 import {
+    Cluster,
+    ClusterFaces,
+    ClusteringResults,
+    ClustersWithNoise,
     FaceApiResult,
+    FaceDescriptor,
     FaceImage,
     FaceWithEmbedding,
     MLSyncResult,
+    NearestCluster,
 } from 'utils/machineLearning/types';
 
 import * as jpeg from 'jpeg-js';
@@ -18,7 +24,7 @@ import ClusteringService from './clusteringService';
 
 import './faceEnvPatch';
 import * as faceapi from 'face-api.js';
-import { SsdMobilenetv1Options } from 'face-api.js';
+import { euclideanDistance, SsdMobilenetv1Options } from 'face-api.js';
 
 class MachineLearningService {
     // private faceDetectionService: TFJSFaceDetectionService;
@@ -26,11 +32,14 @@ class MachineLearningService {
     private clusteringService: ClusteringService;
 
     private clusterFaceDistance = 0.4;
+    private maxFaceDistance = 0.6;
     private minClusterSize = 4;
     private minFaceSize = 24;
     private batchSize = 50;
 
-    public allFaces: FaceWithEmbedding[];
+    private allFaces: FaceWithEmbedding[];
+    private clusteringResults: ClusteringResults;
+    private clustersWithNoise: ClustersWithNoise;
     private allFaceImages: FaceImage[];
 
     public constructor() {
@@ -40,6 +49,14 @@ class MachineLearningService {
 
         this.allFaces = [];
         this.allFaceImages = [];
+        this.clusteringResults = {
+            clusters: [],
+            noise: [],
+        };
+        this.clustersWithNoise = {
+            clusters: [],
+            noise: [],
+        };
     }
 
     public async init(
@@ -67,6 +84,78 @@ class MachineLearningService {
         console.log('04 TF Memory stats: ', tf.memory());
     }
 
+    private getClusterSummary(cluster: ClusterFaces): FaceDescriptor {
+        const faceScore = (f) => f.detection.score; // f.alignedRect.box.width *
+
+        return cluster
+            .map((f) => this.allFaces[f].face)
+            .sort((f1, f2) => faceScore(f2) - faceScore(f1))[0].descriptor;
+    }
+
+    private updateClusterSummaries() {
+        if (
+            !this.clusteringResults ||
+            !this.clusteringResults.clusters ||
+            this.clusteringResults.clusters.length < 1
+        ) {
+            return;
+        }
+
+        const resultClusters = this.clusteringResults.clusters;
+
+        resultClusters.forEach((resultCluster) => {
+            this.clustersWithNoise.clusters.push({
+                faces: resultCluster,
+                summary: this.getClusterSummary(resultCluster),
+            });
+        });
+    }
+
+    private getNearestCluster(noise: FaceWithEmbedding): NearestCluster {
+        let nearest: Cluster = null;
+        let nearestDist = 100000;
+        this.clustersWithNoise.clusters.forEach((c) => {
+            const dist = euclideanDistance(noise.face.descriptor, c.summary);
+            if (dist < nearestDist) {
+                nearestDist = dist;
+                nearest = c;
+            }
+        });
+
+        console.log('nearestDist: ', nearestDist);
+        return { cluster: nearest, distance: nearestDist };
+    }
+
+    private assignNoiseWithinLimit() {
+        if (
+            !this.clusteringResults ||
+            !this.clusteringResults.noise ||
+            this.clusteringResults.noise.length < 1
+        ) {
+            return;
+        }
+
+        const noise = this.clusteringResults.noise;
+
+        noise.forEach((n) => {
+            const noiseFace = this.allFaces[n];
+            const nearest = this.getNearestCluster(noiseFace);
+
+            if (nearest.cluster && nearest.distance < this.maxFaceDistance) {
+                console.log('Adding noise to cluser: ', n, nearest.distance);
+                nearest.cluster.faces.push(n);
+            } else {
+                console.log(
+                    'No cluster for noise: ',
+                    n,
+                    'within distance: ',
+                    this.maxFaceDistance
+                );
+                this.clustersWithNoise.noise.push(n);
+            }
+        });
+    }
+
     private getUniqueFiles(files: File[], limit: number) {
         const uniqueFiles: Map<number, File> = new Map<number, File>();
         for (let i = 0; uniqueFiles.size < limit && i < files.length; i++) {
@@ -125,7 +214,7 @@ class MachineLearningService {
         // this.allFaces[0].alignedRect.box,
         // this.allFaces[0].alignedRect.imageDims
 
-        const clusterResults = this.clusteringService.clusterUsingDBSCAN(
+        this.clusteringResults = this.clusteringService.clusterUsingDBSCAN(
             this.allFaces.map((f) => Array.from(f.face.descriptor)),
             this.clusterFaceDistance,
             this.minClusterSize
@@ -135,11 +224,17 @@ class MachineLearningService {
         //     this.allFaces.map((f) => f.embedding),
         //     10);
 
-        console.log('[MLService] Got cluster results: ', clusterResults);
+        console.log(
+            '[MLService] Got cluster results: ',
+            this.clusteringResults
+        );
+
+        this.updateClusterSummaries();
+        this.assignNoiseWithinLimit();
 
         return {
             allFaces: this.allFaces,
-            clusterResults,
+            clustersWithNoise: this.clustersWithNoise,
         };
     }
 

src/utils/machineLearning/types.ts

@@ -8,7 +8,7 @@ import {
 
 export interface MLSyncResult {
     allFaces: FaceWithEmbedding[];
-    clusterResults: ClusteringResults;
+    clustersWithNoise: ClustersWithNoise;
 }
 
 export interface AlignedFace extends NormalizedFace {
@@ -28,6 +28,30 @@ export declare type FaceApiResult = WithFaceDescriptor<
     >
 >;
 
+export declare type FaceDescriptor = Float32Array;
+
+export declare type ClusterFaces = Array<number>;
+
+export interface Cluster {
+    faces: ClusterFaces;
+    summary: FaceDescriptor;
+}
+
+export interface ClustersWithNoise {
+    clusters: Array<Cluster>;
+    noise: ClusterFaces;
+}
+
+export interface ClusteringResults {
+    clusters: Array<ClusterFaces>;
+    noise: ClusterFaces;
+}
+
+export interface NearestCluster {
+    cluster: Cluster;
+    distance: number;
+}
+
 export interface FaceWithEmbedding {
     fileId: string;
     face: FaceApiResult;
@@ -35,10 +59,3 @@ export interface FaceWithEmbedding {
     // embedding: FaceEmbedding;
     faceImage: FaceImage;
 }
-
-export declare type Cluster = Array<number>;
-
-export interface ClusteringResults {
-    clusters: Cluster[];
-    noise: Cluster;
-}