beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Inline

Browse source
This commit is contained in:
Manav Rathi 2024-05-16 14:29:09 +05:30
parent fb53ae7163
commit 865ddc0fa9
No known key found for this signature in database
2 changed files with 42 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
web/apps/photos/src/services/face/f-index.ts
View file

@ -7,13 +7,11 @@ import { faceEmbeddings, mobileFaceNetFaceSize } from "services/face/embed";
import {
DetectedFace,
Face,
MLSyncContext,
MLSyncFileContext,
type FaceAlignment,
} from "services/face/types";
import { imageBitmapToBlob, warpAffineFloat32List } from "utils/image";
import { detectBlur } from "./blur";
import { clusterFaces } from "./cluster";
import { getFaceCrop } from "./crop";
import {
fetchImageBitmap,
@ -161,51 +159,6 @@ export const saveFaceCrop = async (imageBitmap: ImageBitmap, face: Face) => {
return blob;
};
export const getAllSyncedFacesMap = async (syncContext: MLSyncContext) => {
if (syncContext.allSyncedFacesMap) {
return syncContext.allSyncedFacesMap;
}
syncContext.allSyncedFacesMap = await mlIDbStorage.getAllFacesMap();
return syncContext.allSyncedFacesMap;
};
export const runFaceClustering = async (
syncContext: MLSyncContext,
allFaces: Array<Face>,
) => {
// await this.init();
if (!allFaces || allFaces.length < 50) {
log.info(
`Skipping clustering since number of faces (${allFaces.length}) is less than the clustering threshold (50)`,
);
return;
}
log.info("Running clustering allFaces: ", allFaces.length);
syncContext.mlLibraryData.faceClusteringResults = await clusterFaces(
allFaces.map((f) => Array.from(f.embedding)),
);
syncContext.mlLibraryData.faceClusteringMethod = {
value: "Hdbscan",
version: 1,
};
log.info(
"[MLService] Got face clustering results: ",
JSON.stringify(syncContext.mlLibraryData.faceClusteringResults),
);
// syncContext.faceClustersWithNoise = {
// clusters: syncContext.faceClusteringResults.clusters.map(
// (faces) => ({
// faces,
// })
// ),
// noise: syncContext.faceClusteringResults.noise,
// };
};
export const regenerateFaceCrop = async (faceID: string) => {
const fileID = Number(faceID.split("-")[0]);
const personFace = await mlIDbStorage.getFace(fileID, faceID);

48
web/apps/photos/src/services/face/people.ts
View file

@ -1,10 +1,8 @@
import log from "@/next/log";
import mlIDbStorage from "services/face/db";
import { Face, MLSyncContext, Person } from "services/face/types";
import {
getAllSyncedFacesMap,
runFaceClustering,
saveFaceCrop,
} from "./f-index";
import { clusterFaces } from "./cluster";
import { saveFaceCrop } from "./f-index";
import { fetchImageBitmap, getLocalFile } from "./image";
export const syncPeopleIndex = async (syncContext: MLSyncContext) => {
@ -16,7 +14,9 @@ export const syncPeopleIndex = async (syncContext: MLSyncContext) => {
// TODO: have faces addresable through fileId + faceId
// to avoid index based addressing, which is prone to wrong results
// one way could be to match nearest face within threshold in the file
const allFacesMap = await getAllSyncedFacesMap(syncContext);
const allFacesMap =
syncContext.allSyncedFacesMap ??
(syncContext.allSyncedFacesMap = await mlIDbStorage.getAllFacesMap());
const allFaces = [...allFacesMap.values()].flat();
await runFaceClustering(syncContext, allFaces);
@ -25,6 +25,42 @@ export const syncPeopleIndex = async (syncContext: MLSyncContext) => {
await mlIDbStorage.setIndexVersion("people", filesVersion);
};
const runFaceClustering = async (
syncContext: MLSyncContext,
allFaces: Array<Face>,
) => {
// await this.init();
if (!allFaces || allFaces.length < 50) {
log.info(
`Skipping clustering since number of faces (${allFaces.length}) is less than the clustering threshold (50)`,
);
return;
}
log.info("Running clustering allFaces: ", allFaces.length);
syncContext.mlLibraryData.faceClusteringResults = await clusterFaces(
allFaces.map((f) => Array.from(f.embedding)),
);
syncContext.mlLibraryData.faceClusteringMethod = {
value: "Hdbscan",
version: 1,
};
log.info(
"[MLService] Got face clustering results: ",
JSON.stringify(syncContext.mlLibraryData.faceClusteringResults),
);
// syncContext.faceClustersWithNoise = {
// clusters: syncContext.faceClusteringResults.clusters.map(
// (faces) => ({
// faces,
// })
// ),
// noise: syncContext.faceClusteringResults.noise,
// };
};
const syncPeopleFromClusters = async (
syncContext: MLSyncContext,
allFacesMap: Map<number, Array<Face>>,