From 76848c826e48640181d42712c0be5afd314361c3 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Mon, 20 May 2024 14:18:34 +0530 Subject: [PATCH] [mob][photos] Change embeddings server structure --- mobile/lib/face/model/face.dart | 20 ++++++--- .../face_ml/face_ml_service.dart | 42 +++++++++++-------- .../machine_learning/file_ml/file_ml.dart | 16 +++---- .../file_ml/remote_fileml_service.dart | 3 +- 4 files changed, 44 insertions(+), 37 deletions(-) diff --git a/mobile/lib/face/model/face.dart b/mobile/lib/face/model/face.dart index 3176ef90d..c21538949 100644 --- a/mobile/lib/face/model/face.dart +++ b/mobile/lib/face/model/face.dart @@ -1,5 +1,6 @@ import "package:photos/face/model/detection.dart"; import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; +import "package:photos/services/machine_learning/face_ml/face_ml_result.dart"; // FileInfo contains the image width and height of the image the face was detected in. class FileInfo { @@ -12,13 +13,17 @@ class FileInfo { } class Face { - final int fileID; final String faceID; final List embedding; Detection detection; final double score; final double blur; + + ///#region Local DB fields + // This is not stored on the server, using it for local DB row FileInfo? fileInfo; + final int fileID; + ///#endregion bool get isBlurry => blur < kLaplacianHardThreshold; @@ -55,10 +60,12 @@ class Face { } factory Face.fromJson(Map json) { + final String faceID = json['faceID'] as String; + final int fileID = getFileIdFromFaceId(faceID); return Face( - json['faceID'] as String, - json['fileID'] as int, - List.from(json['embeddings'] as List), + faceID, + fileID, + List.from((json['embedding'] ?? json['embeddings']) as List), json['score'] as double, Detection.fromJson(json['detection'] as Map), // high value means t @@ -66,10 +73,11 @@ class Face { ); } + // Note: Keep the information in toJson minimum. Keep in sync with desktop. + // Derive fields like fileID from other values whenever possible Map toJson() => { 'faceID': faceID, - 'fileID': fileID, - 'embeddings': embedding, + 'embedding': embedding, 'detection': detection.toJson(), 'score': score, 'blur': blur, diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 66668fccf..974d36c10 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -12,6 +12,7 @@ import "package:flutter/foundation.dart" show debugPrint, kDebugMode; import "package:flutter_image_compress/flutter_image_compress.dart"; import "package:logging/logging.dart"; import "package:onnxruntime/onnxruntime.dart"; +import "package:package_info_plus/package_info_plus.dart"; import "package:photos/core/configuration.dart"; import "package:photos/core/event_bus.dart"; import "package:photos/db/files_db.dart"; @@ -88,6 +89,7 @@ class FaceMlService { final _computer = Computer.shared(); bool isInitialized = false; + late String client; bool canRunMLController = false; bool isImageIndexRunning = false; @@ -125,6 +127,11 @@ class FaceMlService { _logger.severe("Could not initialize mobilefacenet", e, s); } + // Get client name + final packageInfo = await PackageInfo.fromPlatform(); + client = "${packageInfo.packageName}/${packageInfo.version}"; + _logger.info("client: $client"); + isInitialized = true; canRunMLController = !Platform.isAndroid || kDebugMode; @@ -621,7 +628,6 @@ class FaceMlService { faces.add( Face.empty( fileMl.fileID, - error: (fileMl.faceEmbedding.error ?? false), ), ); } else { @@ -715,11 +721,6 @@ class FaceMlService { "because version is ${fileMl.faceEmbedding.version} and we need $faceMlVersion"); return true; } - if (fileMl.faceEmbedding.error ?? false) { - debugPrint("Discarding remote embedding for fileID ${fileMl.fileID} " - "because error is true"); - return true; - } // are all landmarks equal? bool allLandmarksEqual = true; if (fileMl.faceEmbedding.faces.isEmpty) { @@ -823,19 +824,24 @@ class FaceMlService { } } _logger.info("inserting ${faces.length} faces for ${result.fileId}"); - await RemoteFileMLService.instance.putFileEmbedding( - enteFile, - FileMl( - enteFile.uploadedFileID!, - FaceEmbeddings( - faces, - result.mlVersion, - error: result.errorOccured ? true : null, + if (!result.errorOccured) { + await RemoteFileMLService.instance.putFileEmbedding( + enteFile, + FileMl( + enteFile.uploadedFileID!, + FaceEmbeddings( + faces, + result.mlVersion, + client: client, + ), + height: result.decodedImageSize.height, + width: result.decodedImageSize.width, ), - height: result.decodedImageSize.height, - width: result.decodedImageSize.width, - ), - ); + ); + } else { + _logger.warning( + 'Skipped putting embedding because of error ${result.toJsonString()}',); + } await FaceMLDataDB.instance.bulkInsertFaces(faces); return true; } catch (e, s) { diff --git a/mobile/lib/services/machine_learning/file_ml/file_ml.dart b/mobile/lib/services/machine_learning/file_ml/file_ml.dart index cbd706dd7..990990276 100644 --- a/mobile/lib/services/machine_learning/file_ml/file_ml.dart +++ b/mobile/lib/services/machine_learning/file_ml/file_ml.dart @@ -4,11 +4,8 @@ class FileMl { final int fileID; final int? height; final int? width; - // json: face final FaceEmbeddings faceEmbedding; final ClipEmbedding? clipEmbedding; - // int updationTime that is not serialized - int? updationTime; FileMl( this.fileID, @@ -45,15 +42,13 @@ class FileMl { class FaceEmbeddings { final List faces; final int version; - // Platform: appVersion - final String? client; - final bool? error; + // pkgname/version + final String client; FaceEmbeddings( this.faces, this.version, { - this.client, - this.error, + required this.client, }); // toJson @@ -61,7 +56,6 @@ class FaceEmbeddings { 'faces': faces.map((x) => x.toJson()).toList(), 'version': version, 'client': client, - 'error': error, }; // fromJson factory FaceEmbeddings.fromJson(Map json) { @@ -70,8 +64,8 @@ class FaceEmbeddings { json['faces'].map((x) => Face.fromJson(x as Map)), ), json['version'] as int, - client: json['client'] as String?, - error: json['error'] as bool?, + client: json['client'] ?? + 'unknown', ); } } diff --git a/mobile/lib/services/machine_learning/file_ml/remote_fileml_service.dart b/mobile/lib/services/machine_learning/file_ml/remote_fileml_service.dart index 3a8e383be..ec4e632b3 100644 --- a/mobile/lib/services/machine_learning/file_ml/remote_fileml_service.dart +++ b/mobile/lib/services/machine_learning/file_ml/remote_fileml_service.dart @@ -1,6 +1,5 @@ import "dart:async"; import "dart:convert"; -import "dart:typed_data"; import "package:computer/computer.dart"; import "package:logging/logging.dart"; @@ -37,7 +36,7 @@ class RemoteFileMLService { final encryptionKey = getFileKey(file); final embeddingJSON = jsonEncode(fileML.toJson()); final encryptedEmbedding = await CryptoUtil.encryptChaCha( - utf8.encode(embeddingJSON) as Uint8List, + utf8.encode(embeddingJSON), encryptionKey, ); final encryptedData =