diff --git a/src/components/MlDebug.tsx b/src/components/MlDebug.tsx index e587e94a4..2c3736645 100644 --- a/src/components/MlDebug.tsx +++ b/src/components/MlDebug.tsx @@ -17,7 +17,7 @@ export default function MLDebug() { const [batchSize, setBatchSize] = useState(50); const [mlResult, setMlResult] = useState({ allFaces: [], - clusterResults: { + clustersWithNoise: { clusters: [], noise: [], }, @@ -111,12 +111,12 @@ export default function MLDebug() {

-

{JSON.stringify(mlResult.clusterResults)}

+

{JSON.stringify(mlResult.clustersWithNoise)}

Clusters:

- {mlResult.clusterResults.clusters.map((cluster, index) => ( + {mlResult.clustersWithNoise.clusters.map((cluster, index) => (
- {cluster.map((faceIndex, ind) => ( + {cluster.faces.map((faceIndex, ind) => (
Noise:

- {mlResult.clusterResults.noise.map((faceIndex, index) => ( -
- -
- ))} + {mlResult.clustersWithNoise.noise.map( + (faceIndex, index) => ( +
+ +
+ ) + )}
diff --git a/src/services/machineLearning/clusteringService.ts b/src/services/machineLearning/clusteringService.ts index 7caccbd47..45dffabbf 100644 --- a/src/services/machineLearning/clusteringService.ts +++ b/src/services/machineLearning/clusteringService.ts @@ -1,4 +1,5 @@ import { DBSCAN, OPTICS, KMEANS } from 'density-clustering'; +import { ClusteringResults } from 'utils/machineLearning/types'; class ClusteringService { private dbscan: DBSCAN; @@ -15,7 +16,7 @@ class ClusteringService { dataset: Array>, epsilon: number = 1.0, minPts: number = 2 - ) { + ): ClusteringResults { // console.log("distanceFunction", DBSCAN._); const clusters = this.dbscan.run(dataset, epsilon, minPts); const noise = this.dbscan.noise; diff --git a/src/services/machineLearning/machineLearningService.ts b/src/services/machineLearning/machineLearningService.ts index c080341f7..b385f21da 100644 --- a/src/services/machineLearning/machineLearningService.ts +++ b/src/services/machineLearning/machineLearningService.ts @@ -7,10 +7,16 @@ import * as tf from '@tensorflow/tfjs-core'; // import TFJSFaceDetectionService from './tfjsFaceDetectionService'; // import TFJSFaceEmbeddingService from './tfjsFaceEmbeddingService'; import { + Cluster, + ClusterFaces, + ClusteringResults, + ClustersWithNoise, FaceApiResult, + FaceDescriptor, FaceImage, FaceWithEmbedding, MLSyncResult, + NearestCluster, } from 'utils/machineLearning/types'; import * as jpeg from 'jpeg-js'; @@ -18,7 +24,7 @@ import ClusteringService from './clusteringService'; import './faceEnvPatch'; import * as faceapi from 'face-api.js'; -import { SsdMobilenetv1Options } from 'face-api.js'; +import { euclideanDistance, SsdMobilenetv1Options } from 'face-api.js'; class MachineLearningService { // private faceDetectionService: TFJSFaceDetectionService; @@ -26,11 +32,14 @@ class MachineLearningService { private clusteringService: ClusteringService; private clusterFaceDistance = 0.4; + private maxFaceDistance = 0.6; private minClusterSize = 4; private minFaceSize = 24; private batchSize = 50; - public allFaces: FaceWithEmbedding[]; + private allFaces: FaceWithEmbedding[]; + private clusteringResults: ClusteringResults; + private clustersWithNoise: ClustersWithNoise; private allFaceImages: FaceImage[]; public constructor() { @@ -40,6 +49,14 @@ class MachineLearningService { this.allFaces = []; this.allFaceImages = []; + this.clusteringResults = { + clusters: [], + noise: [], + }; + this.clustersWithNoise = { + clusters: [], + noise: [], + }; } public async init( @@ -67,6 +84,78 @@ class MachineLearningService { console.log('04 TF Memory stats: ', tf.memory()); } + private getClusterSummary(cluster: ClusterFaces): FaceDescriptor { + const faceScore = (f) => f.detection.score; // f.alignedRect.box.width * + + return cluster + .map((f) => this.allFaces[f].face) + .sort((f1, f2) => faceScore(f2) - faceScore(f1))[0].descriptor; + } + + private updateClusterSummaries() { + if ( + !this.clusteringResults || + !this.clusteringResults.clusters || + this.clusteringResults.clusters.length < 1 + ) { + return; + } + + const resultClusters = this.clusteringResults.clusters; + + resultClusters.forEach((resultCluster) => { + this.clustersWithNoise.clusters.push({ + faces: resultCluster, + summary: this.getClusterSummary(resultCluster), + }); + }); + } + + private getNearestCluster(noise: FaceWithEmbedding): NearestCluster { + let nearest: Cluster = null; + let nearestDist = 100000; + this.clustersWithNoise.clusters.forEach((c) => { + const dist = euclideanDistance(noise.face.descriptor, c.summary); + if (dist < nearestDist) { + nearestDist = dist; + nearest = c; + } + }); + + console.log('nearestDist: ', nearestDist); + return { cluster: nearest, distance: nearestDist }; + } + + private assignNoiseWithinLimit() { + if ( + !this.clusteringResults || + !this.clusteringResults.noise || + this.clusteringResults.noise.length < 1 + ) { + return; + } + + const noise = this.clusteringResults.noise; + + noise.forEach((n) => { + const noiseFace = this.allFaces[n]; + const nearest = this.getNearestCluster(noiseFace); + + if (nearest.cluster && nearest.distance < this.maxFaceDistance) { + console.log('Adding noise to cluser: ', n, nearest.distance); + nearest.cluster.faces.push(n); + } else { + console.log( + 'No cluster for noise: ', + n, + 'within distance: ', + this.maxFaceDistance + ); + this.clustersWithNoise.noise.push(n); + } + }); + } + private getUniqueFiles(files: File[], limit: number) { const uniqueFiles: Map = new Map(); for (let i = 0; uniqueFiles.size < limit && i < files.length; i++) { @@ -125,7 +214,7 @@ class MachineLearningService { // this.allFaces[0].alignedRect.box, // this.allFaces[0].alignedRect.imageDims - const clusterResults = this.clusteringService.clusterUsingDBSCAN( + this.clusteringResults = this.clusteringService.clusterUsingDBSCAN( this.allFaces.map((f) => Array.from(f.face.descriptor)), this.clusterFaceDistance, this.minClusterSize @@ -135,11 +224,17 @@ class MachineLearningService { // this.allFaces.map((f) => f.embedding), // 10); - console.log('[MLService] Got cluster results: ', clusterResults); + console.log( + '[MLService] Got cluster results: ', + this.clusteringResults + ); + + this.updateClusterSummaries(); + this.assignNoiseWithinLimit(); return { allFaces: this.allFaces, - clusterResults, + clustersWithNoise: this.clustersWithNoise, }; } diff --git a/src/utils/machineLearning/types.ts b/src/utils/machineLearning/types.ts index fad442150..1fb5cba91 100644 --- a/src/utils/machineLearning/types.ts +++ b/src/utils/machineLearning/types.ts @@ -8,7 +8,7 @@ import { export interface MLSyncResult { allFaces: FaceWithEmbedding[]; - clusterResults: ClusteringResults; + clustersWithNoise: ClustersWithNoise; } export interface AlignedFace extends NormalizedFace { @@ -28,6 +28,30 @@ export declare type FaceApiResult = WithFaceDescriptor< > >; +export declare type FaceDescriptor = Float32Array; + +export declare type ClusterFaces = Array; + +export interface Cluster { + faces: ClusterFaces; + summary: FaceDescriptor; +} + +export interface ClustersWithNoise { + clusters: Array; + noise: ClusterFaces; +} + +export interface ClusteringResults { + clusters: Array; + noise: ClusterFaces; +} + +export interface NearestCluster { + cluster: Cluster; + distance: number; +} + export interface FaceWithEmbedding { fileId: string; face: FaceApiResult; @@ -35,10 +59,3 @@ export interface FaceWithEmbedding { // embedding: FaceEmbedding; faceImage: FaceImage; } - -export declare type Cluster = Array; - -export interface ClusteringResults { - clusters: Cluster[]; - noise: Cluster; -}