From 39f16ff517af3bdc08655148dab5bb06144a59b4 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 20 Mar 2024 14:34:12 +0530 Subject: [PATCH] Only show high quality faces in file info --- mobile/lib/face/db.dart | 9 ++++++--- mobile/lib/face/db_fields.dart | 2 +- mobile/lib/face/model/face.dart | 6 +++++- .../face_ml/blur_detection/blur_constants.dart | 2 -- .../blur_detection_service.dart | 2 +- .../face_filtering/face_filtering_constants.dart | 8 ++++++++ mobile/lib/services/face_ml/face_ml_result.dart | 2 +- mobile/lib/services/face_ml/face_ml_service.dart | 3 ++- .../lib/ui/viewer/file_details/face_widget.dart | 4 ++++ .../viewer/file_details/faces_item_widget.dart | 16 ++++++++++++++-- mobile/lib/utils/image_ml_util.dart | 2 +- 11 files changed, 43 insertions(+), 13 deletions(-) delete mode 100644 mobile/lib/services/face_ml/blur_detection/blur_constants.dart rename mobile/lib/services/face_ml/{blur_detection => face_filtering}/blur_detection_service.dart (97%) create mode 100644 mobile/lib/services/face_ml/face_filtering/face_filtering_constants.dart diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index 0b85e303c..c8b4f75f7 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -11,7 +11,7 @@ import "package:photos/face/db_model_mappers.dart"; import "package:photos/face/model/face.dart"; import "package:photos/face/model/person.dart"; import "package:photos/models/file/file.dart"; -import "package:photos/services/face_ml/blur_detection/blur_constants.dart"; +import 'package:photos/services/face_ml/face_filtering/face_filtering_constants.dart'; import 'package:sqflite/sqflite.dart'; /// Stores all data for the ML-related features. The database can be accessed by `MlDataDB.instance.database`. @@ -226,7 +226,7 @@ class FaceMLDataDB { return null; } - Future> getFacesForGivenFileID(int fileUploadID) async { + Future?> getFacesForGivenFileID(int fileUploadID) async { final db = await instance.database; final List> maps = await db.query( facesTable, @@ -246,6 +246,9 @@ class FaceMLDataDB { where: '$fileIDColumn = ?', whereArgs: [fileUploadID], ); + if (maps.isEmpty) { + return null; + } return maps.map((e) => mapRowToFace(e)).toList(); } @@ -338,7 +341,7 @@ class FaceMLDataDB { /// /// Only selects faces with score greater than [minScore] and blur score greater than [minClarity] Future> getFaceEmbeddingMap({ - double minScore = 0.78, + double minScore = kMinFaceScore, int minClarity = kLaplacianThreshold, int maxRows = 20000, }) async { diff --git a/mobile/lib/face/db_fields.dart b/mobile/lib/face/db_fields.dart index a1f185e7e..8b1719575 100644 --- a/mobile/lib/face/db_fields.dart +++ b/mobile/lib/face/db_fields.dart @@ -1,5 +1,5 @@ // Faces Table Fields & Schema Queries -import "package:photos/services/face_ml/blur_detection/blur_constants.dart"; +import 'package:photos/services/face_ml/face_filtering/face_filtering_constants.dart'; const facesTable = 'faces'; const fileIDColumn = 'file_id'; diff --git a/mobile/lib/face/model/face.dart b/mobile/lib/face/model/face.dart index aa352cc19..c29c03ae1 100644 --- a/mobile/lib/face/model/face.dart +++ b/mobile/lib/face/model/face.dart @@ -1,5 +1,5 @@ import "package:photos/face/model/detection.dart"; -import "package:photos/services/face_ml/blur_detection/blur_constants.dart"; +import 'package:photos/services/face_ml/face_filtering/face_filtering_constants.dart'; class Face { final int fileID; @@ -11,6 +11,10 @@ class Face { bool get isBlurry => blur < kLaplacianThreshold; + bool get hasHighScore => score > kMinFaceScore; + + bool get isHighQuality => (!isBlurry) && hasHighScore; + Face( this.faceID, this.fileID, diff --git a/mobile/lib/services/face_ml/blur_detection/blur_constants.dart b/mobile/lib/services/face_ml/blur_detection/blur_constants.dart deleted file mode 100644 index 614096ed4..000000000 --- a/mobile/lib/services/face_ml/blur_detection/blur_constants.dart +++ /dev/null @@ -1,2 +0,0 @@ -const kLaplacianThreshold = 15; -const kLapacianDefault = 10000.0; diff --git a/mobile/lib/services/face_ml/blur_detection/blur_detection_service.dart b/mobile/lib/services/face_ml/face_filtering/blur_detection_service.dart similarity index 97% rename from mobile/lib/services/face_ml/blur_detection/blur_detection_service.dart rename to mobile/lib/services/face_ml/face_filtering/blur_detection_service.dart index ff5830468..9c1552364 100644 --- a/mobile/lib/services/face_ml/blur_detection/blur_detection_service.dart +++ b/mobile/lib/services/face_ml/face_filtering/blur_detection_service.dart @@ -1,5 +1,5 @@ import 'package:logging/logging.dart'; -import "package:photos/services/face_ml/blur_detection/blur_constants.dart"; +import 'package:photos/services/face_ml/face_filtering/face_filtering_constants.dart'; class BlurDetectionService { final _logger = Logger('BlurDetectionService'); diff --git a/mobile/lib/services/face_ml/face_filtering/face_filtering_constants.dart b/mobile/lib/services/face_ml/face_filtering/face_filtering_constants.dart new file mode 100644 index 000000000..478161f22 --- /dev/null +++ b/mobile/lib/services/face_ml/face_filtering/face_filtering_constants.dart @@ -0,0 +1,8 @@ +/// Blur detection threshold +const kLaplacianThreshold = 15; + +/// Default blur value +const kLapacianDefault = 10000.0; + +/// The minimum score for a face to be considered a face +const kMinFaceScore = 0.78; \ No newline at end of file diff --git a/mobile/lib/services/face_ml/face_ml_result.dart b/mobile/lib/services/face_ml/face_ml_result.dart index c770efde7..3a981b377 100644 --- a/mobile/lib/services/face_ml/face_ml_result.dart +++ b/mobile/lib/services/face_ml/face_ml_result.dart @@ -6,11 +6,11 @@ import "package:photos/db/ml_data_db.dart"; import "package:photos/models/file/file.dart"; import 'package:photos/models/ml/ml_typedefs.dart'; import "package:photos/models/ml/ml_versions.dart"; -import "package:photos/services/face_ml/blur_detection/blur_constants.dart"; import "package:photos/services/face_ml/face_alignment/alignment_result.dart"; import "package:photos/services/face_ml/face_clustering/cosine_distance.dart"; import "package:photos/services/face_ml/face_detection/detection.dart"; import "package:photos/services/face_ml/face_feedback.dart/cluster_feedback.dart"; +import 'package:photos/services/face_ml/face_filtering/face_filtering_constants.dart'; import "package:photos/services/face_ml/face_ml_methods.dart"; final _logger = Logger('ClusterResult_FaceMlResult'); diff --git a/mobile/lib/services/face_ml/face_ml_service.dart b/mobile/lib/services/face_ml/face_ml_service.dart index 789430386..db3931ed7 100644 --- a/mobile/lib/services/face_ml/face_ml_service.dart +++ b/mobile/lib/services/face_ml/face_ml_service.dart @@ -31,6 +31,7 @@ import 'package:photos/services/face_ml/face_detection/yolov5face/onnx_face_dete import "package:photos/services/face_ml/face_detection/yolov5face/yolo_face_detection_exceptions.dart"; import "package:photos/services/face_ml/face_embedding/face_embedding_exceptions.dart"; import 'package:photos/services/face_ml/face_embedding/onnx_face_embedding.dart'; +import "package:photos/services/face_ml/face_filtering/face_filtering_constants.dart"; import "package:photos/services/face_ml/face_ml_exceptions.dart"; import "package:photos/services/face_ml/face_ml_result.dart"; import 'package:photos/services/machine_learning/file_ml/file_ml.dart'; @@ -361,7 +362,7 @@ class FaceMlService { await clusterAllImages(); } - Future clusterAllImages({double minFaceScore = 0.75}) async { + Future clusterAllImages({double minFaceScore = kMinFaceScore}) async { _logger.info("`clusterAllImages()` called"); try { diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index a7045f92c..6dd0b4aeb 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -88,6 +88,10 @@ class FaceWidget extends StatelessWidget { ), ), const SizedBox(height: 8), + Text( + (face.score).toStringAsFixed(2), + style: Theme.of(context).textTheme.bodySmall, + ), if (person != null) Text( person!.attr.name.trim(), diff --git a/mobile/lib/ui/viewer/file_details/faces_item_widget.dart b/mobile/lib/ui/viewer/file_details/faces_item_widget.dart index fb2d8cbd5..68de105c7 100644 --- a/mobile/lib/ui/viewer/file_details/faces_item_widget.dart +++ b/mobile/lib/ui/viewer/file_details/faces_item_widget.dart @@ -36,9 +36,18 @@ class FacesItemWidget extends StatelessWidget { ]; } - final List faces = await FaceMLDataDB.instance + final List? faces = await FaceMLDataDB.instance .getFacesForGivenFileID(file.uploadedFileID!); - if (faces.isEmpty || faces.every((face) => face.score < 0.5)) { + if (faces == null) { + return [ + const ChipButtonWidget( + "Image not analyzed", + noChips: true, + ), + ]; + } + if (faces.isEmpty || + faces.every((face) => face.score < 0.75 || face.isBlurry)) { return [ const ChipButtonWidget( "No faces found", @@ -50,6 +59,9 @@ class FacesItemWidget extends StatelessWidget { // Sort the faces by score in descending order, so that the highest scoring face is first. faces.sort((Face a, Face b) => b.score.compareTo(a.score)); + // Remove faces with low scores and blurry faces + faces.removeWhere((face) => face.isHighQuality == false); + // TODO: add deduplication of faces of same person final faceIdsToClusterIds = await FaceMLDataDB.instance .getFaceIdsToClusterIds(faces.map((face) => face.faceID)); diff --git a/mobile/lib/utils/image_ml_util.dart b/mobile/lib/utils/image_ml_util.dart index fd04cc1c1..9b11efa8a 100644 --- a/mobile/lib/utils/image_ml_util.dart +++ b/mobile/lib/utils/image_ml_util.dart @@ -18,10 +18,10 @@ import 'package:flutter/painting.dart' as paint show decodeImageFromList; import 'package:ml_linalg/linalg.dart'; import "package:photos/face/model/box.dart"; import 'package:photos/models/ml/ml_typedefs.dart'; -import "package:photos/services/face_ml/blur_detection/blur_detection_service.dart"; import "package:photos/services/face_ml/face_alignment/alignment_result.dart"; import "package:photos/services/face_ml/face_alignment/similarity_transform.dart"; import "package:photos/services/face_ml/face_detection/detection.dart"; +import 'package:photos/services/face_ml/face_filtering/blur_detection_service.dart'; /// All of the functions in this file are helper functions for the [ImageMlIsolate] isolate. /// Don't use them outside of the isolate, unless you are okay with UI jank!!!!