Merge branch 'mobile_face' of https://github.com/ente-io/auth into mobile_face

mobile/lib/face/db.dart

@@ -322,6 +322,17 @@ class FaceMLDataDB {
     return mapRowToFace(result.first);
   }
 
+  Future<Iterable<String>> getFaceIDsForCluster(int clusterID) async {
+    final db = await instance.database;
+    final List<Map<String, dynamic>> maps = await db.query(
+      faceClustersTable,
+      columns: [fcFaceId],
+      where: '$fcClusterID = ?',
+      whereArgs: [clusterID],
+    );
+    return maps.map((e) => e[fcFaceId] as String).toSet();
+  }
+
   Future<Map<String, int?>> getFaceIdsToClusterIds(
     Iterable<String> faceIds,
   ) async {

mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart

@@ -47,7 +47,7 @@ class FaceLinearClustering {
   bool isSpawned = false;
   bool isRunning = false;
 
-  static const recommendedDistanceThreshold = 0.3;
+  static const kRecommendedDistanceThreshold = 0.3;
 
   // singleton pattern
   FaceLinearClustering._privateConstructor();
@@ -102,9 +102,11 @@ class FaceLinearClustering {
             final input = args['input'] as Map<String, (int?, Uint8List)>;
             final fileIDToCreationTime =
                 args['fileIDToCreationTime'] as Map<int, int>?;
+            final distanceThreshold = args['distanceThreshold'] as double;
             final result = FaceLinearClustering._runLinearClustering(
               input,
               fileIDToCreationTime: fileIDToCreationTime,
+              distanceThreshold: distanceThreshold,
             );
             sendPort.send(result);
             break;
@@ -183,6 +185,7 @@ class FaceLinearClustering {
   Future<Map<String, int>?> predict(
     Map<String, (int?, Uint8List)> input, {
     Map<int, int>? fileIDToCreationTime,
+    double distanceThreshold = kRecommendedDistanceThreshold,
   }) async {
     if (input.isEmpty) {
       _logger.warning(
@@ -207,7 +210,7 @@ class FaceLinearClustering {
     final Map<String, int> faceIdToCluster = await _runInIsolate(
       (
         ClusterOperation.linearIncrementalClustering,
-        {'input': input, 'fileIDToCreationTime': fileIDToCreationTime}
+        {'input': input, 'fileIDToCreationTime': fileIDToCreationTime, 'distanceThreshold': distanceThreshold}
       ),
     );
     // return _runLinearClusteringInComputer(input);
@@ -223,6 +226,7 @@ class FaceLinearClustering {
   static Map<String, int> _runLinearClustering(
     Map<String, (int?, Uint8List)> x, {
     Map<int, int>? fileIDToCreationTime,
+    double distanceThreshold = kRecommendedDistanceThreshold,
   }) {
     log(
       "[ClusterIsolate] ${DateTime.now()} Copied to isolate ${x.length} faces",
@@ -332,7 +336,7 @@ class FaceLinearClustering {
         }
       }
 
-      if (closestDistance < recommendedDistanceThreshold) {
+      if (closestDistance < distanceThreshold) {
         if (sortedFaceInfos[closestIdx].clusterId == null) {
           // Ideally this should never happen, but just in case log it
           log(

mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart

@@ -11,9 +11,24 @@ import "package:photos/face/model/person.dart";
 import "package:photos/generated/protos/ente/common/vector.pb.dart";
 import "package:photos/models/file/file.dart";
 import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart';
+import "package:photos/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart";
 import "package:photos/services/machine_learning/face_ml/face_ml_result.dart";
 import "package:photos/services/search_service.dart";
 
+class ClusterSuggestion {
+  final int clusterIDToMerge;
+  final double distancePersonToCluster;
+  final bool usedOnlyMeanForSuggestion;
+  final List<EnteFile> filesInCluster;
+
+  ClusterSuggestion(
+    this.clusterIDToMerge,
+    this.distancePersonToCluster,
+    this.usedOnlyMeanForSuggestion,
+    this.filesInCluster,
+  );
+}
+
 class ClusterFeedbackService {
   final Logger _logger = Logger("ClusterFeedbackService");
   ClusterFeedbackService._privateConstructor();
@@ -241,7 +256,7 @@ class ClusterFeedbackService {
   /// 2. distance: the distance between the person's cluster and the suggestion
   /// 3. bool: whether the suggestion was found using the mean (true) or the median (false)
   /// 4. List<EnteFile>: the files in the cluster
-  Future<List<(int, double, bool, List<EnteFile>)>> getClusterFilesForPersonID(
+  Future<List<ClusterSuggestion>> getSuggestionForPerson(
     Person person, {
     bool extremeFilesFirst = true,
   }) async {
@@ -275,15 +290,15 @@ class ClusterFeedbackService {
         }
       }
 
-      final List<(int, double, bool, List<EnteFile>)> clusterIdAndFiles = [];
+      final List<ClusterSuggestion> clusterIdAndFiles = [];
       for (final clusterSuggestion in suggestClusterIds) {
         if (clusterIDToFiles.containsKey(clusterSuggestion.$1)) {
           clusterIdAndFiles.add(
-            (
+            ClusterSuggestion(
               clusterSuggestion.$1,
               clusterSuggestion.$2,
               clusterSuggestion.$3,
-              clusterIDToFiles[clusterSuggestion.$1]!
+              clusterIDToFiles[clusterSuggestion.$1]!,
             ),
           );
         }
@@ -365,6 +380,44 @@ class ClusterFeedbackService {
     return true;
   }
 
+  // TODO: iterate over this method and actually use it
+  Future<Map<int, List<String>>> breakUpCluster(int clusterID) async {
+    final faceMlDb = FaceMLDataDB.instance;
+
+    final faceIDs = await faceMlDb.getFaceIDsForCluster(clusterID);
+    final fileIDs = faceIDs.map((e) => getFileIdFromFaceId(e)).toList();
+
+    final embeddings = await faceMlDb.getFaceEmbeddingMapForFile(fileIDs);
+    embeddings.removeWhere((key, value) => !faceIDs.contains(key));
+    final clusteringInput = embeddings.map((key, value) {
+      return MapEntry(key, (null, value));
+    });
+
+    final faceIdToCluster = await FaceLinearClustering.instance
+        .predict(clusteringInput, distanceThreshold: 0.15);
+
+    if (faceIdToCluster == null) {
+      return {};
+    }
+
+    final clusterIdToFaceIds = <int, List<String>>{};
+    for (final entry in faceIdToCluster.entries) {
+      final clusterID = entry.value;
+      if (clusterIdToFaceIds.containsKey(clusterID)) {
+        clusterIdToFaceIds[clusterID]!.add(entry.key);
+      } else {
+        clusterIdToFaceIds[clusterID] = [entry.key];
+      }
+    }
+
+    final clusterIdToCount = clusterIdToFaceIds.map((key, value) {
+      return MapEntry(key, value.length);
+    });
+    final amountOfNewClusters = clusterIdToCount.length;
+
+    return clusterIdToFaceIds;
+  }
+
   Future<Map<int, List<double>>> _getUpdateClusterAvg(
     Map<int, int> allClusterIdsToCountMap,
     Set<int> ignoredClusters, {
@@ -558,7 +611,7 @@ class ClusterFeedbackService {
 
   Future<void> _sortSuggestionsOnDistanceToPerson(
     Person person,
-    List<(int, double, bool, List<EnteFile>)> suggestions,
+    List<ClusterSuggestion> suggestions,
   ) async {
     if (suggestions.isEmpty) {
       debugPrint('No suggestions to sort');
@@ -590,9 +643,9 @@ class ClusterFeedbackService {
 
     // Sort the suggestions based on the distance to the person
     for (final suggestion in suggestions) {
-      final clusterID = suggestion.$1;
+      final clusterID = suggestion.clusterIDToMerge;
       final faceIdToEmbeddingMap = await faceMlDb.getFaceEmbeddingMapForFile(
-        suggestion.$4.map((e) => e.uploadedFileID!).toList(),
+        suggestion.filesInCluster.map((e) => e.uploadedFileID!).toList(),
       );
       final fileIdToDistanceMap = {};
       for (final entry in faceIdToEmbeddingMap.entries) {
@@ -602,7 +655,7 @@ class ClusterFeedbackService {
           EVector.fromBuffer(entry.value).values,
         );
       }
-      suggestion.$4.sort((b, a) {
+      suggestion.filesInCluster.sort((b, a) {
         //todo: review with @laurens, added this to avoid null safety issue
         final double distanceA = fileIdToDistanceMap[a.uploadedFileID!] ?? -1;
         final double distanceB = fileIdToDistanceMap[b.uploadedFileID!] ?? -1;
@@ -610,7 +663,7 @@ class ClusterFeedbackService {
       });
 
       debugPrint(
-        "[${_logger.name}] Sorted suggestions for cluster $clusterID based on distance to person: ${suggestion.$4.map((e) => fileIdToDistanceMap[e.uploadedFileID]).toList()}",
+        "[${_logger.name}] Sorted suggestions for cluster $clusterID based on distance to person: ${suggestion.filesInCluster.map((e) => fileIdToDistanceMap[e.uploadedFileID]).toList()}",
       );
     }
 

mobile/lib/ui/viewer/people/person_cluster_suggestion.dart

@@ -31,7 +31,7 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
   Key futureBuilderKey = UniqueKey();
 
   // Declare a variable for the future
-  late Future<List<(int, double, bool, List<EnteFile>)>>
+  late Future<List<ClusterSuggestion>>
       futureClusterSuggestions;
 
   @override
@@ -47,7 +47,7 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
       appBar: AppBar(
         title: const Text('Review suggestions'),
       ),
-      body: FutureBuilder<List<(int, double, bool, List<EnteFile>)>>(
+      body: FutureBuilder<List<ClusterSuggestion>>(
         key: futureBuilderKey,
         future: futureClusterSuggestions,
         builder: (context, snapshot) {
@@ -63,10 +63,10 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
             }
             final numberOfDifferentSuggestions = snapshot.data!.length;
             final currentSuggestion = snapshot.data![currentSuggestionIndex];
-            final int clusterID = currentSuggestion.$1;
-            final double distance = currentSuggestion.$2;
-            final bool usingMean = currentSuggestion.$3;
-            final List<EnteFile> files = currentSuggestion.$4;
+            final int clusterID = currentSuggestion.clusterIDToMerge;
+            final double distance = currentSuggestion.distancePersonToCluster;
+            final bool usingMean = currentSuggestion.usedOnlyMeanForSuggestion;
+            final List<EnteFile> files = currentSuggestion.filesInCluster;
             return InkWell(
               onTap: () {
                 Navigator.of(context).push(
@@ -140,8 +140,8 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
 
   // Method to fetch cluster suggestions
   void _fetchClusterSuggestions() {
-    futureClusterSuggestions = ClusterFeedbackService.instance
-        .getClusterFilesForPersonID(widget.person);
+    futureClusterSuggestions =
+        ClusterFeedbackService.instance.getSuggestionForPerson(widget.person);
   }
 
   Widget _buildSuggestionView(