diff --git a/desktop/src/main/services/ml-clip.ts b/desktop/src/main/services/ml-clip.ts index 3fe6da2eb..63fa75148 100644 --- a/desktop/src/main/services/ml-clip.ts +++ b/desktop/src/main/services/ml-clip.ts @@ -134,7 +134,7 @@ const clipImageEmbedding_ = async (jpegFilePath: string) => { const results = await imageSession.run(feeds); log.debug( () => - `CLIP image embedding took ${Date.now() - t1} ms (prep: ${t2 - t1} ms, inference: ${Date.now() - t2} ms)`, + `onnx/clip image embedding took ${Date.now() - t1} ms (prep: ${t2 - t1} ms, inference: ${Date.now() - t2} ms)`, ); const imageEmbedding = results["output"].data; // Float32Array return normalizeEmbedding(imageEmbedding); @@ -241,7 +241,7 @@ export const clipTextEmbedding = async (text: string) => { const results = await imageSession.run(feeds); log.debug( () => - `CLIP text embedding took ${Date.now() - t1} ms (prep: ${t2 - t1} ms, inference: ${Date.now() - t2} ms)`, + `onnx/clip text embedding took ${Date.now() - t1} ms (prep: ${t2 - t1} ms, inference: ${Date.now() - t2} ms)`, ); const textEmbedding = results["output"].data; return normalizeEmbedding(textEmbedding); diff --git a/desktop/src/main/services/ml-face.ts b/desktop/src/main/services/ml-face.ts index 78eb82bd1..c79ae591f 100644 --- a/desktop/src/main/services/ml-face.ts +++ b/desktop/src/main/services/ml-face.ts @@ -2,7 +2,7 @@ * @file Various face recognition related tasks. * * - Face detection with the YOLO model. - * - Face embedding with the mobilefacenet model. + * - Face embedding with the MobileFaceNet model. * * The runtime used is ONNX. */ @@ -99,51 +99,10 @@ export const detectFaces = async (input: Float32Array) => { input: new ort.Tensor("float32", input, [1, 3, 640, 640]), }; const results = await session.run(feeds); - log.debug(() => `onnx/yolo inference took ${Date.now() - t} ms`); + log.debug(() => `onnx/yolo face detection took ${Date.now() - t} ms`); return results["output"].data; }; export const faceEmbedding = async (input: Float32Array) => { throw new Error("test"); }; - -/* - -private async initOnnx() { -} - -private async getOnnxInferenceSession() { - if (!this.onnxInferenceSession) { - await this.initOnnx(); - } - return this.onnxInferenceSession; -} -*/ - -// export const clipImageEmbedding = async (jpegImageData: Uint8Array) => { -// const tempFilePath = await generateTempFilePath(""); -// const imageStream = new Response(jpegImageData.buffer).body; -// await writeStream(tempFilePath, imageStream); -// try { -// return await clipImageEmbedding_(tempFilePath); -// } finally { -// await deleteTempFile(tempFilePath); -// } -// }; - -// const clipImageEmbedding_ = async (jpegFilePath: string) => { -// const imageSession = await onnxImageSession(); -// const t1 = Date.now(); -// const rgbData = await getRGBData(jpegFilePath); -// const feeds = { -// input: new ort.Tensor("float32", rgbData, [1, 3, 224, 224]), -// }; -// const t2 = Date.now(); -// const results = await imageSession.run(feeds); -// log.debug( -// () => -// `CLIP image embedding took ${Date.now() - t1} ms (prep: ${t2 - t1} ms, inference: ${Date.now() - t2} ms)`, -// ); -// const imageEmbedding = results["output"].data; // Float32Array -// return normalizeEmbedding(imageEmbedding); -// }; diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index 449ae0b96..0f26950f8 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -55,7 +55,7 @@ class FaceService { await syncContext.faceDetectionService.detectFaces(imageBitmap); console.timeEnd(timerId); console.log("faceDetections: ", faceDetections?.length); - // log.info('3 TF Memory stats: ',JSON.stringify(tf.memory())); + // TODO: reenable faces filtering based on width const detectedFaces = faceDetections?.map((detection) => { return { @@ -150,7 +150,7 @@ class FaceService { imageBitmap.close(); log.info("[MLService] alignedFaces: ", newMlFile.faces?.length); - // log.info('4 TF Memory stats: ',JSON.stringify(tf.memory())); + return faceImages; } @@ -187,7 +187,6 @@ class FaceService { newMlFile.faces.forEach((f, i) => (f.embedding = embeddings[i])); log.info("[MLService] facesWithEmbeddings: ", newMlFile.faces.length); - // log.info('5 TF Memory stats: ',JSON.stringify(tf.memory())); } async syncFileFaceMakeRelativeDetections(