diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index 6c4c27563..add547376 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import "dart:convert" show json; import "dart:io" show Directory; import "dart:math"; @@ -10,6 +11,7 @@ import 'package:path_provider/path_provider.dart'; import "package:photos/extensions/stop_watch.dart"; import 'package:photos/face/db_fields.dart'; import "package:photos/face/db_model_mappers.dart"; +import "package:photos/face/model/detection.dart"; import "package:photos/face/model/face.dart"; import "package:photos/models/file/file.dart"; import "package:photos/services/machine_learning/face_ml/face_clustering/face_info_for_clustering.dart"; @@ -455,6 +457,31 @@ class FaceMLDataDB { ); } + // TODO: remove this method and replace by correct logic during indexing + Future> getFaceIdsToIsSidewaysFaceTEMP() async { + final db = await instance.sqliteAsyncDB; + final List> maps = await db.getAll( + 'SELECT $faceIDColumn, $faceDetectionColumn FROM $facesTable', + ); + + final time = DateTime.now(); + final Map result = {}; + for (final map in maps) { + final faceID = map[faceIDColumn] as String; + final detection = + Detection.fromJson(json.decode(map[faceDetectionColumn] as String)); + result[faceID] = detection.faceIsSideways(); + } + _logger.info( + 'decoding detections and calculating face sideways bools took ${DateTime.now().difference(time).inMilliseconds} ms', + ); + final double sidewaysRatio = + result.values.where((e) => e).length / result.length; + _logger.info('sideways face ratio: $sidewaysRatio'); + + return result; + } + Future> getFaceInfoForClustering({ double minScore = kMinHighQualityFaceScore, int minClarity = kLaplacianHardThreshold, @@ -468,6 +495,9 @@ class FaceMLDataDB { ); final db = await instance.sqliteAsyncDB; + final Map faceIdsToIsSideways = + await getFaceIdsToIsSidewaysFaceTEMP(); + final Set result = {}; while (true) { // Query a batch of rows @@ -494,6 +524,7 @@ class FaceMLDataDB { embeddingBytes: map[faceEmbeddingBlob] as Uint8List, faceScore: map[faceScore] as double, blurValue: map[faceBlur] as double, + isSideways: faceIdsToIsSideways[faceID] ?? false, ); result.add(faceInfo); } diff --git a/mobile/lib/face/model/detection.dart b/mobile/lib/face/model/detection.dart index af6324ac8..49e8c3652 100644 --- a/mobile/lib/face/model/detection.dart +++ b/mobile/lib/face/model/detection.dart @@ -22,7 +22,7 @@ class Detection { bool get isEmpty => box.width == 0 && box.height == 0 && landmarks.isEmpty; - // emoty box + // empty box Detection.empty() : box = FaceBox( xMin: 0, @@ -94,6 +94,9 @@ class Detection { } FaceDirection getFaceDirection() { + if (isEmpty) { + return FaceDirection.straight; + } final leftEye = [landmarks[0].x, landmarks[0].y]; final rightEye = [landmarks[1].x, landmarks[1].y]; final nose = [landmarks[2].x, landmarks[2].y]; @@ -131,6 +134,9 @@ class Detection { } bool faceIsSideways() { + if (isEmpty) { + return false; + } final leftEye = [landmarks[0].x, landmarks[0].y]; final rightEye = [landmarks[1].x, landmarks[1].y]; final nose = [landmarks[2].x, landmarks[2].y]; diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart index 7d822f9e7..792ec3b4c 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart @@ -452,7 +452,8 @@ class FaceClusteringService { faceScore: face.faceScore, blurValue: face.blurValue, badFace: face.faceScore < kMinHighQualityFaceScore || - face.blurValue < kLaplacianSoftThreshold, + face.blurValue < kLaplacianSoftThreshold || + face.isSideways, vEmbedding: Vector.fromList( EVector.fromBuffer(face.embeddingBytes).values, dtype: DType.float32, @@ -606,7 +607,7 @@ class FaceClusteringService { ); if (useDynamicThreshold) { log( - "[ClusterIsolate] ${DateTime.now()} Dynamic thresholding: $dynamicThresholdCount faces had a low face score or high blur value", + "[ClusterIsolate] ${DateTime.now()} Dynamic thresholding: $dynamicThresholdCount faces had a low face score or low blur clarity", ); } diff --git a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_info_for_clustering.dart b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_info_for_clustering.dart index 8dd78ff1f..273d85da5 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_clustering/face_info_for_clustering.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_clustering/face_info_for_clustering.dart @@ -1,4 +1,3 @@ - import "dart:typed_data" show Uint8List; class FaceInfoForClustering { @@ -7,6 +6,7 @@ class FaceInfoForClustering { final Uint8List embeddingBytes; final double faceScore; final double blurValue; + final bool isSideways; FaceInfoForClustering({ required this.faceID, @@ -14,5 +14,6 @@ class FaceInfoForClustering { required this.embeddingBytes, required this.faceScore, required this.blurValue, + this.isSideways = false, }); -} \ No newline at end of file +}