beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'mobile_face' of https://github.com/ente-io/auth into mobile_face

Browse source
This commit is contained in:
Neeraj Gupta 2024-04-01 15:37:41 +05:30
parent f663bbfc53 1af3d2d2db
commit 5b339fc30e
11 changed files with 6 additions and 1921 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

714
mobile/lib/db/ml_data_db.dart
View file

@ -1,714 +0,0 @@
import 'dart:async';
import 'package:logging/logging.dart';
import 'package:path/path.dart' show join;
import 'package:path_provider/path_provider.dart';
import 'package:photos/models/ml/ml_typedefs.dart';
import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart';
import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart';
import 'package:photos/services/machine_learning/face_ml/face_ml_result.dart';
import 'package:sqflite/sqflite.dart';
/// Stores all data for the ML-related features. The database can be accessed by `MlDataDB.instance.database`.
///
/// This includes:
/// [facesTable] - Stores all the detected faces and its embeddings in the images.
/// [peopleTable] - Stores all the clusters of faces which are considered to be the same person.
class MlDataDB {
static final Logger _logger = Logger("MlDataDB");
// TODO: [BOB] put the db in files
static const _databaseName = "ente.ml_data.db";
static const _databaseVersion = 1;
static const facesTable = 'faces';
static const fileIDColumn = 'file_id';
static const faceMlResultColumn = 'face_ml_result';
static const mlVersionColumn = 'ml_version';
static const peopleTable = 'people';
static const personIDColumn = 'person_id';
static const clusterResultColumn = 'cluster_result';
static const centroidColumn = 'cluster_centroid';
static const centroidDistanceThresholdColumn = 'centroid_distance_threshold';
static const feedbackTable = 'feedback';
static const feedbackIDColumn = 'feedback_id';
static const feedbackTypeColumn = 'feedback_type';
static const feedbackDataColumn = 'feedback_data';
static const feedbackTimestampColumn = 'feedback_timestamp';
static const feedbackFaceMlVersionColumn = 'feedback_face_ml_version';
static const feedbackClusterMlVersionColumn = 'feedback_cluster_ml_version';
static const createFacesTable = '''CREATE TABLE IF NOT EXISTS $facesTable (
$fileIDColumn INTEGER NOT NULL UNIQUE,
$faceMlResultColumn TEXT NOT NULL,
$mlVersionColumn INTEGER NOT NULL,
PRIMARY KEY($fileIDColumn)
);
''';
static const createPeopleTable = '''CREATE TABLE IF NOT EXISTS $peopleTable (
$personIDColumn INTEGER NOT NULL UNIQUE,
$clusterResultColumn TEXT NOT NULL,
$centroidColumn TEXT NOT NULL,
$centroidDistanceThresholdColumn REAL NOT NULL,
PRIMARY KEY($personIDColumn)
);
''';
static const createFeedbackTable =
'''CREATE TABLE IF NOT EXISTS $feedbackTable (
$feedbackIDColumn TEXT NOT NULL UNIQUE,
$feedbackTypeColumn TEXT NOT NULL,
$feedbackDataColumn TEXT NOT NULL,
$feedbackTimestampColumn TEXT NOT NULL,
$feedbackFaceMlVersionColumn INTEGER NOT NULL,
$feedbackClusterMlVersionColumn INTEGER NOT NULL,
PRIMARY KEY($feedbackIDColumn)
);
''';
static const _deleteFacesTable = 'DROP TABLE IF EXISTS $facesTable';
static const _deletePeopleTable = 'DROP TABLE IF EXISTS $peopleTable';
static const _deleteFeedbackTable = 'DROP TABLE IF EXISTS $feedbackTable';
MlDataDB._privateConstructor();
static final MlDataDB instance = MlDataDB._privateConstructor();
static Future<Database>? _dbFuture;
Future<Database> get database async {
_dbFuture ??= _initDatabase();
return _dbFuture!;
}
Future<Database> _initDatabase() async {
final documentsDirectory = await getApplicationDocumentsDirectory();
final String databaseDirectory =
join(documentsDirectory.path, _databaseName);
return await openDatabase(
databaseDirectory,
version: _databaseVersion,
onCreate: _onCreate,
);
}
Future _onCreate(Database db, int version) async {
await db.execute(createFacesTable);
await db.execute(createPeopleTable);
await db.execute(createFeedbackTable);
}
/// WARNING: This will delete ALL data in the database! Only use this for debug/testing purposes!
Future<void> cleanTables({
bool cleanFaces = false,
bool cleanPeople = false,
bool cleanFeedback = false,
}) async {
_logger.fine('`cleanTables()` called');
final db = await instance.database;
if (cleanFaces) {
_logger.fine('`cleanTables()`: Cleaning faces table');
await db.execute(_deleteFacesTable);
}
if (cleanPeople) {
_logger.fine('`cleanTables()`: Cleaning people table');
await db.execute(_deletePeopleTable);
}
if (cleanFeedback) {
_logger.fine('`cleanTables()`: Cleaning feedback table');
await db.execute(_deleteFeedbackTable);
}
if (!cleanFaces && !cleanPeople && !cleanFeedback) {
_logger.fine(
'`cleanTables()`: No tables cleaned, since no table was specified. Please be careful with this function!',
);
}
await db.execute(createFacesTable);
await db.execute(createPeopleTable);
await db.execute(createFeedbackTable);
}
Future<void> createFaceMlResult(FaceMlResult faceMlResult) async {
_logger.fine('createFaceMlResult called');
final existingResult = await getFaceMlResult(faceMlResult.fileId);
if (existingResult != null) {
if (faceMlResult.mlVersion <= existingResult.mlVersion) {
_logger.fine(
'FaceMlResult with file ID ${faceMlResult.fileId} already exists with equal or higher version. Skipping insert.',
);
return;
}
}
final db = await instance.database;
await db.insert(
facesTable,
{
fileIDColumn: faceMlResult.fileId,
faceMlResultColumn: faceMlResult.toJsonString(),
mlVersionColumn: faceMlResult.mlVersion,
},
conflictAlgorithm: ConflictAlgorithm.replace,
);
}
Future<bool> doesFaceMlResultExist(int fileId, {int? mlVersion}) async {
_logger.fine('doesFaceMlResultExist called');
final db = await instance.database;
String whereString = '$fileIDColumn = ?';
final List<dynamic> whereArgs = [fileId];
if (mlVersion != null) {
whereString += ' AND $mlVersionColumn = ?';
whereArgs.add(mlVersion);
}
final result = await db.query(
facesTable,
where: whereString,
whereArgs: whereArgs,
limit: 1,
);
return result.isNotEmpty;
}
Future<FaceMlResult?> getFaceMlResult(int fileId, {int? mlVersion}) async {
_logger.fine('getFaceMlResult called');
final db = await instance.database;
String whereString = '$fileIDColumn = ?';
final List<dynamic> whereArgs = [fileId];
if (mlVersion != null) {
whereString += ' AND $mlVersionColumn = ?';
whereArgs.add(mlVersion);
}
final result = await db.query(
facesTable,
where: whereString,
whereArgs: whereArgs,
limit: 1,
);
if (result.isNotEmpty) {
return FaceMlResult.fromJsonString(
result.first[faceMlResultColumn] as String,
);
}
_logger.fine(
'No faceMlResult found for fileID $fileId and mlVersion $mlVersion (null if not specified)',
);
return null;
}
/// Returns the faceMlResults for the given [fileIds].
Future<List<FaceMlResult>> getSelectedFaceMlResults(
List<int> fileIds,
) async {
_logger.fine('getSelectedFaceMlResults called');
final db = await instance.database;
if (fileIds.isEmpty) {
_logger.warning('getSelectedFaceMlResults called with empty fileIds');
return <FaceMlResult>[];
}
final List<Map<String, Object?>> results = await db.query(
facesTable,
columns: [faceMlResultColumn],
where: '$fileIDColumn IN (${fileIds.join(',')})',
orderBy: fileIDColumn,
);
return results
.map(
(result) =>
FaceMlResult.fromJsonString(result[faceMlResultColumn] as String),
)
.toList();
}
Future<List<FaceMlResult>> getAllFaceMlResults({int? mlVersion}) async {
_logger.fine('getAllFaceMlResults called');
final db = await instance.database;
String? whereString;
List<dynamic>? whereArgs;
if (mlVersion != null) {
whereString = '$mlVersionColumn = ?';
whereArgs = [mlVersion];
}
final results = await db.query(
facesTable,
where: whereString,
whereArgs: whereArgs,
orderBy: fileIDColumn,
);
return results
.map(
(result) =>
FaceMlResult.fromJsonString(result[faceMlResultColumn] as String),
)
.toList();
}
/// getAllFileIDs returns a set of all fileIDs from the facesTable, meaning all the fileIDs for which a FaceMlResult exists, optionally filtered by mlVersion.
Future<Set<int>> getAllFaceMlResultFileIDs({int? mlVersion}) async {
_logger.fine('getAllFaceMlResultFileIDs called');
final db = await instance.database;
String? whereString;
List<dynamic>? whereArgs;
if (mlVersion != null) {
whereString = '$mlVersionColumn = ?';
whereArgs = [mlVersion];
}
final List<Map<String, Object?>> results = await db.query(
facesTable,
where: whereString,
whereArgs: whereArgs,
orderBy: fileIDColumn,
);
return results.map((result) => result[fileIDColumn] as int).toSet();
}
Future<Set<int>> getAllFaceMlResultFileIDsProcessedWithThumbnailOnly({
int? mlVersion,
}) async {
_logger.fine('getAllFaceMlResultFileIDsProcessedWithThumbnailOnly called');
final db = await instance.database;
String? whereString;
List<dynamic>? whereArgs;
if (mlVersion != null) {
whereString = '$mlVersionColumn = ?';
whereArgs = [mlVersion];
}
final List<Map<String, Object?>> results = await db.query(
facesTable,
where: whereString,
whereArgs: whereArgs,
orderBy: fileIDColumn,
);
return results
.map(
(result) =>
FaceMlResult.fromJsonString(result[faceMlResultColumn] as String),
)
.where((element) => element.onlyThumbnailUsed)
.map((result) => result.fileId)
.toSet();
}
/// Updates the faceMlResult for the given [faceMlResult.fileId]. Update is done regardless of the [faceMlResult.mlVersion].
/// However, if [updateHigherVersionOnly] is set to true, the update is only done if the [faceMlResult.mlVersion] is higher than the existing one.
Future<int> updateFaceMlResult(
FaceMlResult faceMlResult, {
bool updateHigherVersionOnly = false,
}) async {
_logger.fine('updateFaceMlResult called');
if (updateHigherVersionOnly) {
final existingResult = await getFaceMlResult(faceMlResult.fileId);
if (existingResult != null) {
if (faceMlResult.mlVersion <= existingResult.mlVersion) {
_logger.fine(
'FaceMlResult with file ID ${faceMlResult.fileId} already exists with equal or higher version. Skipping update.',
);
return 0;
}
}
}
final db = await instance.database;
return await db.update(
facesTable,
{
fileIDColumn: faceMlResult.fileId,
faceMlResultColumn: faceMlResult.toJsonString(),
mlVersionColumn: faceMlResult.mlVersion,
},
where: '$fileIDColumn = ?',
whereArgs: [faceMlResult.fileId],
);
}
Future<int> deleteFaceMlResult(int fileId) async {
_logger.fine('deleteFaceMlResult called');
final db = await instance.database;
final deleteCount = await db.delete(
facesTable,
where: '$fileIDColumn = ?',
whereArgs: [fileId],
);
_logger.fine('Deleted $deleteCount faceMlResults');
return deleteCount;
}
Future<void> createAllClusterResults(
List<ClusterResult> clusterResults, {
bool cleanExistingClusters = true,
}) async {
_logger.fine('createClusterResults called');
final db = await instance.database;
if (clusterResults.isEmpty) {
_logger.fine('No clusterResults given, skipping insert.');
return;
}
// Completely clean the table and start fresh
if (cleanExistingClusters) {
await deleteAllClusterResults();
}
// Insert all the cluster results
for (final clusterResult in clusterResults) {
await db.insert(
peopleTable,
{
personIDColumn: clusterResult.personId,
clusterResultColumn: clusterResult.toJsonString(),
centroidColumn: clusterResult.medoid.toString(),
centroidDistanceThresholdColumn:
clusterResult.medoidDistanceThreshold,
},
conflictAlgorithm: ConflictAlgorithm.replace,
);
}
}
Future<ClusterResult?> getClusterResult(int personId) async {
_logger.fine('getClusterResult called');
final db = await instance.database;
final result = await db.query(
peopleTable,
where: '$personIDColumn = ?',
whereArgs: [personId],
limit: 1,
);
if (result.isNotEmpty) {
return ClusterResult.fromJsonString(
result.first[clusterResultColumn] as String,
);
}
_logger.fine('No clusterResult found for personID $personId');
return null;
}
/// Returns the ClusterResult objects for the given [personIDs].
Future<List<ClusterResult>> getSelectedClusterResults(
List<int> personIDs,
) async {
_logger.fine('getSelectedClusterResults called');
final db = await instance.database;
if (personIDs.isEmpty) {
_logger.warning('getSelectedClusterResults called with empty personIDs');
return <ClusterResult>[];
}
final results = await db.query(
peopleTable,
where: '$personIDColumn IN (${personIDs.join(',')})',
orderBy: personIDColumn,
);
return results
.map(
(result) => ClusterResult.fromJsonString(
result[clusterResultColumn] as String,
),
)
.toList();
}
Future<List<ClusterResult>> getAllClusterResults() async {
_logger.fine('getAllClusterResults called');
final db = await instance.database;
final results = await db.query(
peopleTable,
);
return results
.map(
(result) => ClusterResult.fromJsonString(
result[clusterResultColumn] as String,
),
)
.toList();
}
/// Returns the personIDs of all clustered people in the database.
Future<List<int>> getAllClusterIds() async {
_logger.fine('getAllClusterIds called');
final db = await instance.database;
final results = await db.query(
peopleTable,
columns: [personIDColumn],
);
return results.map((result) => result[personIDColumn] as int).toList();
}
/// Returns the fileIDs of all files associated with a given [personId].
Future<List<int>> getClusterFileIds(int personId) async {
_logger.fine('getClusterFileIds called');
final ClusterResult? clusterResult = await getClusterResult(personId);
if (clusterResult == null) {
return <int>[];
}
return clusterResult.uniqueFileIds;
}
Future<List<String>> getClusterFaceIds(int personId) async {
_logger.fine('getClusterFaceIds called');
final ClusterResult? clusterResult = await getClusterResult(personId);
if (clusterResult == null) {
return <String>[];
}
return clusterResult.faceIDs;
}
Future<List<Embedding>> getClusterEmbeddings(
int personId,
) async {
_logger.fine('getClusterEmbeddings called');
final ClusterResult? clusterResult = await getClusterResult(personId);
if (clusterResult == null) return <Embedding>[];
final fileIds = clusterResult.uniqueFileIds;
final faceIds = clusterResult.faceIDs;
if (fileIds.length != faceIds.length) {
_logger.severe(
'fileIds and faceIds have different lengths: ${fileIds.length} vs ${faceIds.length}. This should not happen!',
);
return <Embedding>[];
}
final faceMlResults = await getSelectedFaceMlResults(fileIds);
if (faceMlResults.isEmpty) return <Embedding>[];
final embeddings = <Embedding>[];
for (var i = 0; i < faceMlResults.length; i++) {
final faceMlResult = faceMlResults[i];
final int faceIndex = faceMlResult.allFaceIds.indexOf(faceIds[i]);
if (faceIndex == -1) {
_logger.severe(
'Could not find faceIndex for faceId ${faceIds[i]} in faceMlResult ${faceMlResult.fileId}',
);
return <Embedding>[];
}
embeddings.add(faceMlResult.faces[faceIndex].embedding);
}
return embeddings;
}
Future<void> updateClusterResult(ClusterResult clusterResult) async {
_logger.fine('updateClusterResult called');
final db = await instance.database;
await db.update(
peopleTable,
{
personIDColumn: clusterResult.personId,
clusterResultColumn: clusterResult.toJsonString(),
centroidColumn: clusterResult.medoid.toString(),
centroidDistanceThresholdColumn: clusterResult.medoidDistanceThreshold,
},
where: '$personIDColumn = ?',
whereArgs: [clusterResult.personId],
);
}
Future<int> deleteClusterResult(int personId) async {
_logger.fine('deleteClusterResult called');
final db = await instance.database;
final deleteCount = await db.delete(
peopleTable,
where: '$personIDColumn = ?',
whereArgs: [personId],
);
_logger.fine('Deleted $deleteCount clusterResults');
return deleteCount;
}
Future<void> deleteAllClusterResults() async {
_logger.fine('deleteAllClusterResults called');
final db = await instance.database;
await db.execute(_deletePeopleTable);
await db.execute(createPeopleTable);
}
// TODO: current function implementation will skip inserting for a similar feedback, which means I can't remove two photos from the same person in a row
Future<void> createClusterFeedback<T extends ClusterFeedback>(
T feedback, {
bool skipIfSimilarFeedbackExists = false,
}) async {
_logger.fine('createClusterFeedback called');
// TODO: this skipping might cause issues for adding photos to the same person in a row!!
if (skipIfSimilarFeedbackExists &&
await doesSimilarClusterFeedbackExist(feedback)) {
_logger.fine(
'ClusterFeedback with ID ${feedback.feedbackID} already has a similar feedback installed. Skipping insert.',
);
return;
}
final db = await instance.database;
await db.insert(
feedbackTable,
{
feedbackIDColumn: feedback.feedbackID,
feedbackTypeColumn: feedback.typeString,
feedbackDataColumn: feedback.toJsonString(),
feedbackTimestampColumn: feedback.timestampString,
feedbackFaceMlVersionColumn: feedback.madeOnFaceMlVersion,
feedbackClusterMlVersionColumn: feedback.madeOnClusterMlVersion,
},
conflictAlgorithm: ConflictAlgorithm.replace,
);
return;
}
Future<bool> doesSimilarClusterFeedbackExist<T extends ClusterFeedback>(
T feedback,
) async {
_logger.fine('doesClusterFeedbackExist called');
final List<T> existingFeedback =
await getAllClusterFeedback<T>(type: feedback.type);
if (existingFeedback.isNotEmpty) {
for (final existingFeedbackItem in existingFeedback) {
assert(
existingFeedbackItem.type == feedback.type,
'Feedback types should be the same!',
);
if (feedback.looselyMatchesMedoid(existingFeedbackItem)) {
_logger.fine(
'ClusterFeedback of type ${feedback.typeString} with ID ${feedback.feedbackID} already has a similar feedback installed!',
);
return true;
}
}
}
return false;
}
/// Returns all the clusterFeedbacks of type [T] which match the given [feedback], sorted by timestamp (latest first).
Future<List<T>> getAllMatchingClusterFeedback<T extends ClusterFeedback>(
T feedback, {
bool sortNewestFirst = true,
}) async {
_logger.fine('getAllMatchingClusterFeedback called');
final List<T> existingFeedback =
await getAllClusterFeedback<T>(type: feedback.type);
final List<T> matchingFeedback = <T>[];
if (existingFeedback.isNotEmpty) {
for (final existingFeedbackItem in existingFeedback) {
assert(
existingFeedbackItem.type == feedback.type,
'Feedback types should be the same!',
);
if (feedback.looselyMatchesMedoid(existingFeedbackItem)) {
_logger.fine(
'ClusterFeedback of type ${feedback.typeString} with ID ${feedback.feedbackID} already has a similar feedback installed!',
);
matchingFeedback.add(existingFeedbackItem);
}
}
}
if (sortNewestFirst) {
matchingFeedback.sort((a, b) => b.timestamp.compareTo(a.timestamp));
}
return matchingFeedback;
}
Future<List<T>> getAllClusterFeedback<T extends ClusterFeedback>({
required FeedbackType type,
int? mlVersion,
int? clusterMlVersion,
}) async {
_logger.fine('getAllClusterFeedback called');
final db = await instance.database;
// TODO: implement the versions for FeedbackType.imageFeedback and FeedbackType.faceFeedback and rename this function to getAllFeedback?
String whereString = '$feedbackTypeColumn = ?';
final List<dynamic> whereArgs = [type.toValueString()];
if (mlVersion != null) {
whereString += ' AND $feedbackFaceMlVersionColumn = ?';
whereArgs.add(mlVersion);
}
if (clusterMlVersion != null) {
whereString += ' AND $feedbackClusterMlVersionColumn = ?';
whereArgs.add(clusterMlVersion);
}
final results = await db.query(
feedbackTable,
where: whereString,
whereArgs: whereArgs,
);
if (results.isNotEmpty) {
if (ClusterFeedback.fromJsonStringRegistry.containsKey(type)) {
final Function(String) fromJsonString =
ClusterFeedback.fromJsonStringRegistry[type]!;
return results
.map((e) => fromJsonString(e[feedbackDataColumn] as String) as T)
.toList();
} else {
_logger.severe(
'No fromJsonString function found for type ${type.name}. This should not happen!',
);
}
}
_logger.fine(
'No clusterFeedback results found of type $type' +
(mlVersion != null ? ' and mlVersion $mlVersion' : '') +
(clusterMlVersion != null
? ' and clusterMlVersion $clusterMlVersion'
: ''),
);
return <T>[];
}
Future<int> deleteClusterFeedback<T extends ClusterFeedback>(
T feedback,
) async {
_logger.fine('deleteClusterFeedback called');
final db = await instance.database;
final deleteCount = await db.delete(
feedbackTable,
where: '$feedbackIDColumn = ?',
whereArgs: [feedback.feedbackID],
);
_logger.fine('Deleted $deleteCount clusterFeedbacks');
return deleteCount;
}
}

379
mobile/lib/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart
View file

@ -1,379 +0,0 @@
import "dart:convert";
import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart';
import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/feedback.dart';
import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart';
abstract class ClusterFeedback extends Feedback {
static final Map<FeedbackType, Function(String)> fromJsonStringRegistry = {
FeedbackType.deleteClusterFeedback: DeleteClusterFeedback.fromJsonString,
FeedbackType.mergeClusterFeedback: MergeClusterFeedback.fromJsonString,
FeedbackType.renameOrCustomThumbnailClusterFeedback:
RenameOrCustomThumbnailClusterFeedback.fromJsonString,
FeedbackType.removePhotosClusterFeedback:
RemovePhotosClusterFeedback.fromJsonString,
FeedbackType.addPhotosClusterFeedback:
AddPhotosClusterFeedback.fromJsonString,
};
final List<double> medoid;
final double medoidDistanceThreshold;
// TODO: work out the optimal distance threshold so there's never an overlap between clusters
ClusterFeedback(
FeedbackType type,
this.medoid,
this.medoidDistanceThreshold, {
String? feedbackID,
DateTime? timestamp,
int? madeOnFaceMlVersion,
int? madeOnClusterMlVersion,
}) : super(
type,
feedbackID: feedbackID,
timestamp: timestamp,
madeOnFaceMlVersion: madeOnFaceMlVersion,
madeOnClusterMlVersion: madeOnClusterMlVersion,
);
/// Compares this feedback with another [ClusterFeedback] to see if they are similar enough that only one should be kept.
///
/// It checks this by comparing the distance between the two medoids with the medoidDistanceThreshold of each feedback.
///
/// Returns true if they are similar enough, false otherwise.
/// // TODO: Should it maybe return a merged feedback instead, when you are similar enough?
bool looselyMatchesMedoid(ClusterFeedback other) {
// Using the cosineDistance function you mentioned
final double distance = cosineDistance(medoid, other.medoid);
// Check if the distance is less than either of the threshold values
return distance < medoidDistanceThreshold ||
distance < other.medoidDistanceThreshold;
}
bool exactlyMatchesMedoid(ClusterFeedback other) {
if (medoid.length != other.medoid.length) {
return false;
}
for (int i = 0; i < medoid.length; i++) {
if (medoid[i] != other.medoid[i]) {
return false;
}
}
return true;
}
}
class DeleteClusterFeedback extends ClusterFeedback {
DeleteClusterFeedback({
required List<double> medoid,
required double medoidDistanceThreshold,
String? feedbackID,
DateTime? timestamp,
int? madeOnFaceMlVersion,
int? madeOnClusterMlVersion,
}) : super(
FeedbackType.deleteClusterFeedback,
medoid,
medoidDistanceThreshold,
feedbackID: feedbackID,
timestamp: timestamp,
madeOnFaceMlVersion: madeOnFaceMlVersion,
madeOnClusterMlVersion: madeOnClusterMlVersion,
);
@override
Map<String, dynamic> toJson() {
return {
'type': type.toValueString(),
'medoid': medoid,
'medoidDistanceThreshold': medoidDistanceThreshold,
'feedbackID': feedbackID,
'timestamp': timestamp.toIso8601String(),
'madeOnFaceMlVersion': madeOnFaceMlVersion,
'madeOnClusterMlVersion': madeOnClusterMlVersion,
};
}
@override
String toJsonString() => jsonEncode(toJson());
static DeleteClusterFeedback fromJson(Map<String, dynamic> json) {
assert(json['type'] == FeedbackType.deleteClusterFeedback.toValueString());
return DeleteClusterFeedback(
medoid:
(json['medoid'] as List?)?.map((item) => item as double).toList() ??
[],
medoidDistanceThreshold: json['medoidDistanceThreshold'],
feedbackID: json['feedbackID'],
timestamp: DateTime.parse(json['timestamp']),
madeOnFaceMlVersion: json['madeOnFaceMlVersion'],
madeOnClusterMlVersion: json['madeOnClusterMlVersion'],
);
}
static fromJsonString(String jsonString) {
return fromJson(jsonDecode(jsonString));
}
}
class MergeClusterFeedback extends ClusterFeedback {
final List<double> medoidToMoveTo;
MergeClusterFeedback({
required List<double> medoid,
required double medoidDistanceThreshold,
required this.medoidToMoveTo,
String? feedbackID,
DateTime? timestamp,
int? madeOnFaceMlVersion,
int? madeOnClusterMlVersion,
}) : super(
FeedbackType.mergeClusterFeedback,
medoid,
medoidDistanceThreshold,
feedbackID: feedbackID,
timestamp: timestamp,
madeOnFaceMlVersion: madeOnFaceMlVersion,
madeOnClusterMlVersion: madeOnClusterMlVersion,
);
@override
Map<String, dynamic> toJson() {
return {
'type': type.toValueString(),
'medoid': medoid,
'medoidDistanceThreshold': medoidDistanceThreshold,
'medoidToMoveTo': medoidToMoveTo,
'feedbackID': feedbackID,
'timestamp': timestamp.toIso8601String(),
'madeOnFaceMlVersion': madeOnFaceMlVersion,
'madeOnClusterMlVersion': madeOnClusterMlVersion,
};
}
@override
String toJsonString() => jsonEncode(toJson());
static MergeClusterFeedback fromJson(Map<String, dynamic> json) {
assert(json['type'] == FeedbackType.mergeClusterFeedback.toValueString());
return MergeClusterFeedback(
medoid:
(json['medoid'] as List?)?.map((item) => item as double).toList() ??
[],
medoidDistanceThreshold: json['medoidDistanceThreshold'],
medoidToMoveTo: (json['medoidToMoveTo'] as List?)
?.map((item) => item as double)
.toList() ??
[],
feedbackID: json['feedbackID'],
timestamp: DateTime.parse(json['timestamp']),
madeOnFaceMlVersion: json['madeOnFaceMlVersion'],
madeOnClusterMlVersion: json['madeOnClusterMlVersion'],
);
}
static MergeClusterFeedback fromJsonString(String jsonString) {
return fromJson(jsonDecode(jsonString));
}
}
class RenameOrCustomThumbnailClusterFeedback extends ClusterFeedback {
String? customName;
String? customThumbnailFaceId;
RenameOrCustomThumbnailClusterFeedback({
required List<double> medoid,
required double medoidDistanceThreshold,
this.customName,
this.customThumbnailFaceId,
String? feedbackID,
DateTime? timestamp,
int? madeOnFaceMlVersion,
int? madeOnClusterMlVersion,
}) : assert(
customName != null || customThumbnailFaceId != null,
"Either customName or customThumbnailFaceId must be non-null!",
),
super(
FeedbackType.renameOrCustomThumbnailClusterFeedback,
medoid,
medoidDistanceThreshold,
feedbackID: feedbackID,
timestamp: timestamp,
madeOnFaceMlVersion: madeOnFaceMlVersion,
madeOnClusterMlVersion: madeOnClusterMlVersion,
);
@override
Map<String, dynamic> toJson() {
return {
'type': type.toValueString(),
'medoid': medoid,
'medoidDistanceThreshold': medoidDistanceThreshold,
if (customName != null) 'customName': customName,
if (customThumbnailFaceId != null)
'customThumbnailFaceId': customThumbnailFaceId,
'feedbackID': feedbackID,
'timestamp': timestamp.toIso8601String(),
'madeOnFaceMlVersion': madeOnFaceMlVersion,
'madeOnClusterMlVersion': madeOnClusterMlVersion,
};
}
@override
String toJsonString() => jsonEncode(toJson());
static RenameOrCustomThumbnailClusterFeedback fromJson(
Map<String, dynamic> json,
) {
assert(
json['type'] ==
FeedbackType.renameOrCustomThumbnailClusterFeedback.toValueString(),
);
return RenameOrCustomThumbnailClusterFeedback(
medoid:
(json['medoid'] as List?)?.map((item) => item as double).toList() ??
[],
medoidDistanceThreshold: json['medoidDistanceThreshold'],
customName: json['customName'],
customThumbnailFaceId: json['customThumbnailFaceId'],
feedbackID: json['feedbackID'],
timestamp: DateTime.parse(json['timestamp']),
madeOnFaceMlVersion: json['madeOnFaceMlVersion'],
madeOnClusterMlVersion: json['madeOnClusterMlVersion'],
);
}
static RenameOrCustomThumbnailClusterFeedback fromJsonString(
String jsonString,
) {
return fromJson(jsonDecode(jsonString));
}
}
class RemovePhotosClusterFeedback extends ClusterFeedback {
final List<int> removedPhotosFileID;
RemovePhotosClusterFeedback({
required List<double> medoid,
required double medoidDistanceThreshold,
required this.removedPhotosFileID,
String? feedbackID,
DateTime? timestamp,
int? madeOnFaceMlVersion,
int? madeOnClusterMlVersion,
}) : super(
FeedbackType.removePhotosClusterFeedback,
medoid,
medoidDistanceThreshold,
feedbackID: feedbackID,
timestamp: timestamp,
madeOnFaceMlVersion: madeOnFaceMlVersion,
madeOnClusterMlVersion: madeOnClusterMlVersion,
);
@override
Map<String, dynamic> toJson() {
return {
'type': type.toValueString(),
'medoid': medoid,
'medoidDistanceThreshold': medoidDistanceThreshold,
'removedPhotosFileID': removedPhotosFileID,
'feedbackID': feedbackID,
'timestamp': timestamp.toIso8601String(),
'madeOnFaceMlVersion': madeOnFaceMlVersion,
'madeOnClusterMlVersion': madeOnClusterMlVersion,
};
}
@override
String toJsonString() => jsonEncode(toJson());
static RemovePhotosClusterFeedback fromJson(Map<String, dynamic> json) {
assert(
json['type'] == FeedbackType.removePhotosClusterFeedback.toValueString(),
);
return RemovePhotosClusterFeedback(
medoid:
(json['medoid'] as List?)?.map((item) => item as double).toList() ??
[],
medoidDistanceThreshold: json['medoidDistanceThreshold'],
removedPhotosFileID: (json['removedPhotosFileID'] as List?)
?.map((item) => item as int)
.toList() ??
[],
feedbackID: json['feedbackID'],
timestamp: DateTime.parse(json['timestamp']),
madeOnFaceMlVersion: json['madeOnFaceMlVersion'],
madeOnClusterMlVersion: json['madeOnClusterMlVersion'],
);
}
static RemovePhotosClusterFeedback fromJsonString(String jsonString) {
return fromJson(jsonDecode(jsonString));
}
}
class AddPhotosClusterFeedback extends ClusterFeedback {
final List<int> addedPhotoFileIDs;
AddPhotosClusterFeedback({
required List<double> medoid,
required double medoidDistanceThreshold,
required this.addedPhotoFileIDs,
String? feedbackID,
DateTime? timestamp,
int? madeOnFaceMlVersion,
int? madeOnClusterMlVersion,
}) : super(
FeedbackType.addPhotosClusterFeedback,
medoid,
medoidDistanceThreshold,
feedbackID: feedbackID,
timestamp: timestamp,
madeOnFaceMlVersion: madeOnFaceMlVersion,
madeOnClusterMlVersion: madeOnClusterMlVersion,
);
@override
Map<String, dynamic> toJson() {
return {
'type': type.toValueString(),
'medoid': medoid,
'medoidDistanceThreshold': medoidDistanceThreshold,
'addedPhotoFileIDs': addedPhotoFileIDs,
'feedbackID': feedbackID,
'timestamp': timestamp.toIso8601String(),
'madeOnFaceMlVersion': madeOnFaceMlVersion,
'madeOnClusterMlVersion': madeOnClusterMlVersion,
};
}
@override
String toJsonString() => jsonEncode(toJson());
static AddPhotosClusterFeedback fromJson(Map<String, dynamic> json) {
assert(
json['type'] == FeedbackType.addPhotosClusterFeedback.toValueString(),
);
return AddPhotosClusterFeedback(
medoid:
(json['medoid'] as List?)?.map((item) => item as double).toList() ??
[],
medoidDistanceThreshold: json['medoidDistanceThreshold'],
addedPhotoFileIDs: (json['addedPhotoFileIDs'] as List?)
?.map((item) => item as int)
.toList() ??
[],
feedbackID: json['feedbackID'],
timestamp: DateTime.parse(json['timestamp']),
madeOnFaceMlVersion: json['madeOnFaceMlVersion'],
madeOnClusterMlVersion: json['madeOnClusterMlVersion'],
);
}
static AddPhotosClusterFeedback fromJsonString(String jsonString) {
return fromJson(jsonDecode(jsonString));
}
}

416
mobile/lib/services/machine_learning/face_ml/face_feedback.dart/face_feedback_service.dart
View file

@ -1,416 +0,0 @@
import "package:logging/logging.dart";
import "package:photos/db/ml_data_db.dart";
import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart';
import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart';
import 'package:photos/services/machine_learning/face_ml/face_ml_result.dart';
class FaceFeedbackService {
final _logger = Logger("FaceFeedbackService");
final _mlDatabase = MlDataDB.instance;
int executedFeedbackCount = 0;
final int _reclusterFeedbackThreshold = 10;
// singleton pattern
FaceFeedbackService._privateConstructor();
static final instance = FaceFeedbackService._privateConstructor();
factory FaceFeedbackService() => instance;
/// Returns the updated cluster after removing the given file from the given person's cluster.
///
/// If the file is not in the cluster, returns null.
///
/// The updated cluster is also updated in [MlDataDB].
Future<ClusterResult> removePhotosFromCluster(
List<int> fileIDs,
int personID,
) async {
// TODO: check if photo was originally added to cluster by user. If so, we should remove that addition instead of changing the embedding, because there is no embedding...
_logger.info(
'removePhotoFromCluster called with fileIDs $fileIDs and personID $personID',
);
if (fileIDs.isEmpty) {
_logger.severe(
"No fileIDs given, unable to add photos to cluster!",
);
throw ArgumentError(
"No fileIDs given, unable to add photos to cluster!",
);
}
// Get the relevant cluster
final ClusterResult? cluster = await _mlDatabase.getClusterResult(personID);
if (cluster == null) {
_logger.severe(
"No cluster found for personID $personID, unable to remove photo from non-existent cluster!",
);
throw ArgumentError(
"No cluster found for personID $personID, unable to remove photo from non-existent cluster!",
);
}
// Get the relevant faceMlResults
final List<FaceMlResult> faceMlResults =
await _mlDatabase.getSelectedFaceMlResults(fileIDs);
if (faceMlResults.length != fileIDs.length) {
final List<int> foundFileIDs =
faceMlResults.map((faceMlResult) => faceMlResult.fileId).toList();
_logger.severe(
"Couldn't find all facemlresults for fileIDs $fileIDs, only found for $foundFileIDs. Unable to remove unindexed photos from cluster!",
);
throw ArgumentError(
"Couldn't find all facemlresults for fileIDs $fileIDs, only found for $foundFileIDs. Unable to remove unindexed photos from cluster!",
);
}
// Check if at least one of the files is in the cluster. If all files are already not in the cluster, return the cluster.
final List<int> fileIDsInCluster = fileIDs
.where((fileID) => cluster.uniqueFileIds.contains(fileID))
.toList();
if (fileIDsInCluster.isEmpty) {
_logger.warning(
"All fileIDs are already not in the cluster, unable to remove photos from cluster!",
);
return cluster;
}
final List<FaceMlResult> faceMlResultsInCluster = faceMlResults
.where((faceMlResult) => fileIDsInCluster.contains(faceMlResult.fileId))
.toList();
assert(faceMlResultsInCluster.length == fileIDsInCluster.length);
for (var i = 0; i < fileIDsInCluster.length; i++) {
// Find the faces/embeddings associated with both the fileID and personID
final List<String> faceIDs = faceMlResultsInCluster[i].allFaceIds;
final List<String> faceIDsInCluster = cluster.faceIDs;
final List<String> relevantFaceIDs =
faceIDsInCluster.where((faceID) => faceIDs.contains(faceID)).toList();
if (relevantFaceIDs.isEmpty) {
_logger.severe(
"No faces found in both cluster and file, unable to remove photo from cluster!",
);
throw ArgumentError(
"No faces found in both cluster and file, unable to remove photo from cluster!",
);
}
// Set the embeddings to [10, 10,..., 10] and save the updated faceMlResult
faceMlResultsInCluster[i].setEmbeddingsToTen(relevantFaceIDs);
await _mlDatabase.updateFaceMlResult(faceMlResultsInCluster[i]);
// Make sure there is a manual override for [10, 10,..., 10] embeddings (not actually here, but in building the clusters, see _checkIfClusterIsDeleted function)
// Manually remove the fileID from the cluster
cluster.removeFileId(fileIDsInCluster[i]);
}
// TODO: see below
// Re-cluster and check if this leads to more deletions. If so, save them and ask the user if they want to delete them too.
executedFeedbackCount++;
if (executedFeedbackCount % _reclusterFeedbackThreshold == 0) {
// await recluster();
}
// Update the cluster in the database
await _mlDatabase.updateClusterResult(cluster);
// TODO: see below
// Safe the given feedback to the database
final removePhotoFeedback = RemovePhotosClusterFeedback(
medoid: cluster.medoid,
medoidDistanceThreshold: cluster.medoidDistanceThreshold,
removedPhotosFileID: fileIDsInCluster,
);
await _mlDatabase.createClusterFeedback(
removePhotoFeedback,
skipIfSimilarFeedbackExists: false,
);
// Return the updated cluster
return cluster;
}
Future<ClusterResult> addPhotosToCluster(List<int> fileIDs, personID) async {
_logger.info(
'addPhotosToCluster called with fileIDs $fileIDs and personID $personID',
);
if (fileIDs.isEmpty) {
_logger.severe(
"No fileIDs given, unable to add photos to cluster!",
);
throw ArgumentError(
"No fileIDs given, unable to add photos to cluster!",
);
}
// Get the relevant cluster
final ClusterResult? cluster = await _mlDatabase.getClusterResult(personID);
if (cluster == null) {
_logger.severe(
"No cluster found for personID $personID, unable to add photos to non-existent cluster!",
);
throw ArgumentError(
"No cluster found for personID $personID, unable to add photos to non-existent cluster!",
);
}
// Check if at least one of the files is not in the cluster. If all files are already in the cluster, return the cluster.
final List<int> fileIDsNotInCluster = fileIDs
.where((fileID) => !cluster.uniqueFileIds.contains(fileID))
.toList();
if (fileIDsNotInCluster.isEmpty) {
_logger.warning(
"All fileIDs are already in the cluster, unable to add new photos to cluster!",
);
return cluster;
}
final List<String> faceIDsNotInCluster = fileIDsNotInCluster
.map((fileID) => FaceDetectionRelative.toFaceIDEmpty(fileID: fileID))
.toList();
// Add the new files to the cluster
cluster.addFileIDsAndFaceIDs(fileIDsNotInCluster, faceIDsNotInCluster);
// Update the cluster in the database
await _mlDatabase.updateClusterResult(cluster);
// Build the addPhotoFeedback
final AddPhotosClusterFeedback addPhotosFeedback = AddPhotosClusterFeedback(
medoid: cluster.medoid,
medoidDistanceThreshold: cluster.medoidDistanceThreshold,
addedPhotoFileIDs: fileIDsNotInCluster,
);
// TODO: check for exact match and update feedback if necessary
// Save the addPhotoFeedback to the database
await _mlDatabase.createClusterFeedback(
addPhotosFeedback,
skipIfSimilarFeedbackExists: false,
);
// Return the updated cluster
return cluster;
}
/// Deletes the given cluster completely.
Future<void> deleteCluster(int personID) async {
_logger.info(
'deleteCluster called with personID $personID',
);
// Get the relevant cluster
final cluster = await _mlDatabase.getClusterResult(personID);
if (cluster == null) {
_logger.severe(
"No cluster found for personID $personID, unable to delete non-existent cluster!",
);
throw ArgumentError(
"No cluster found for personID $personID, unable to delete non-existent cluster!",
);
}
// Delete the cluster from the database
await _mlDatabase.deleteClusterResult(cluster.personId);
// TODO: look into the right threshold distance.
// Build the deleteClusterFeedback
final DeleteClusterFeedback deleteClusterFeedback = DeleteClusterFeedback(
medoid: cluster.medoid,
medoidDistanceThreshold: cluster.medoidDistanceThreshold,
);
// TODO: maybe I should merge the two feedbacks if they are similar enough? Or alternatively, I keep them both?
// Check if feedback doesn't already exist
if (await _mlDatabase
.doesSimilarClusterFeedbackExist(deleteClusterFeedback)) {
_logger.warning(
"Feedback already exists for deleting cluster $personID, unable to delete cluster!",
);
return;
}
// Save the deleteClusterFeedback to the database
await _mlDatabase.createClusterFeedback(deleteClusterFeedback);
}
/// Renames the given cluster and/or sets the thumbnail of the given cluster.
///
/// Requires either a [customName] or a [customFaceID]. If both are given, both are used. If neither are given, an error is thrown.
Future<ClusterResult> renameOrSetThumbnailCluster(
int personID, {
String? customName,
String? customFaceID,
}) async {
_logger.info(
'renameOrSetThumbnailCluster called with personID $personID, customName $customName, and customFaceID $customFaceID',
);
if (customFaceID != null &&
FaceDetectionRelative.isFaceIDEmpty(customFaceID)) {
_logger.severe(
"customFaceID $customFaceID is belongs to empty detection, unable to set as thumbnail of cluster!",
);
customFaceID = null;
}
if (customName == null && customFaceID == null) {
_logger.severe(
"No name or faceID given, unable to rename or set thumbnail of cluster!",
);
throw ArgumentError(
"No name or faceID given, unable to rename or set thumbnail of cluster!",
);
}
// Get the relevant cluster
final cluster = await _mlDatabase.getClusterResult(personID);
if (cluster == null) {
_logger.severe(
"No cluster found for personID $personID, unable to delete non-existent cluster!",
);
throw ArgumentError(
"No cluster found for personID $personID, unable to delete non-existent cluster!",
);
}
// Update the cluster
if (customName != null) cluster.setUserDefinedName = customName;
if (customFaceID != null) cluster.setThumbnailFaceId = customFaceID;
// Update the cluster in the database
await _mlDatabase.updateClusterResult(cluster);
// Build the RenameOrCustomThumbnailClusterFeedback
final RenameOrCustomThumbnailClusterFeedback renameClusterFeedback =
RenameOrCustomThumbnailClusterFeedback(
medoid: cluster.medoid,
medoidDistanceThreshold: cluster.medoidDistanceThreshold,
customName: customName,
customThumbnailFaceId: customFaceID,
);
// TODO: maybe I should merge the two feedbacks if they are similar enough?
// Check if feedback doesn't already exist
final matchingFeedbacks =
await _mlDatabase.getAllMatchingClusterFeedback(renameClusterFeedback);
for (final matchingFeedback in matchingFeedbacks) {
// Update the current feedback wherever possible
renameClusterFeedback.customName ??= matchingFeedback.customName;
renameClusterFeedback.customThumbnailFaceId ??=
matchingFeedback.customThumbnailFaceId;
// Delete the old feedback (since we want the user to be able to overwrite their earlier feedback)
await _mlDatabase.deleteClusterFeedback(matchingFeedback);
}
// Save the RenameOrCustomThumbnailClusterFeedback to the database
await _mlDatabase.createClusterFeedback(renameClusterFeedback);
// Return the updated cluster
return cluster;
}
/// Merges the given clusters. The largest cluster is kept and the other clusters are deleted.
///
/// Requires either a [clusters] or [personIDs]. If both are given, the [clusters] are used.
Future<ClusterResult> mergeClusters(List<int> personIDs) async {
_logger.info(
'mergeClusters called with personIDs $personIDs',
);
// Get the relevant clusters
final List<ClusterResult> clusters =
await _mlDatabase.getSelectedClusterResults(personIDs);
if (clusters.length <= 1) {
_logger.severe(
"${clusters.length} clusters found for personIDs $personIDs, unable to merge non-existent clusters!",
);
throw ArgumentError(
"${clusters.length} clusters found for personIDs $personIDs, unable to merge non-existent clusters!",
);
}
// Find the largest cluster
clusters.sort((a, b) => b.clusterSize.compareTo(a.clusterSize));
final ClusterResult largestCluster = clusters.first;
// Now iterate through the clusters to be merged and deleted
for (var i = 1; i < clusters.length; i++) {
final ClusterResult clusterToBeMerged = clusters[i];
// Add the files and faces of the cluster to be merged to the largest cluster
largestCluster.addFileIDsAndFaceIDs(
clusterToBeMerged.fileIDsIncludingPotentialDuplicates,
clusterToBeMerged.faceIDs,
);
// TODO: maybe I should wrap the logic below in a separate function, since it's also used in renameOrSetThumbnailCluster
// Merge any names and thumbnails if the largest cluster doesn't have them
bool shouldCreateNamingFeedback = false;
String? nameToBeMerged;
String? thumbnailToBeMerged;
if (!largestCluster.hasUserDefinedName &&
clusterToBeMerged.hasUserDefinedName) {
largestCluster.setUserDefinedName = clusterToBeMerged.userDefinedName!;
nameToBeMerged = clusterToBeMerged.userDefinedName!;
shouldCreateNamingFeedback = true;
}
if (!largestCluster.thumbnailFaceIdIsUserDefined &&
clusterToBeMerged.thumbnailFaceIdIsUserDefined) {
largestCluster.setThumbnailFaceId = clusterToBeMerged.thumbnailFaceId;
thumbnailToBeMerged = clusterToBeMerged.thumbnailFaceId;
shouldCreateNamingFeedback = true;
}
if (shouldCreateNamingFeedback) {
final RenameOrCustomThumbnailClusterFeedback renameClusterFeedback =
RenameOrCustomThumbnailClusterFeedback(
medoid: largestCluster.medoid,
medoidDistanceThreshold: largestCluster.medoidDistanceThreshold,
customName: nameToBeMerged,
customThumbnailFaceId: thumbnailToBeMerged,
);
// Check if feedback doesn't already exist
final matchingFeedbacks = await _mlDatabase
.getAllMatchingClusterFeedback(renameClusterFeedback);
for (final matchingFeedback in matchingFeedbacks) {
// Update the current feedback wherever possible
renameClusterFeedback.customName ??= matchingFeedback.customName;
renameClusterFeedback.customThumbnailFaceId ??=
matchingFeedback.customThumbnailFaceId;
// Delete the old feedback (since we want the user to be able to overwrite their earlier feedback)
await _mlDatabase.deleteClusterFeedback(matchingFeedback);
}
// Save the RenameOrCustomThumbnailClusterFeedback to the database
await _mlDatabase.createClusterFeedback(renameClusterFeedback);
}
// Build the mergeClusterFeedback
final MergeClusterFeedback mergeClusterFeedback = MergeClusterFeedback(
medoid: clusterToBeMerged.medoid,
medoidDistanceThreshold: clusterToBeMerged.medoidDistanceThreshold,
medoidToMoveTo: largestCluster.medoid,
);
// Save the mergeClusterFeedback to the database and delete any old matching feedbacks
final matchingFeedbacks =
await _mlDatabase.getAllMatchingClusterFeedback(mergeClusterFeedback);
for (final matchingFeedback in matchingFeedbacks) {
await _mlDatabase.deleteClusterFeedback(matchingFeedback);
}
await _mlDatabase.createClusterFeedback(mergeClusterFeedback);
// Delete the cluster from the database
await _mlDatabase.deleteClusterResult(clusterToBeMerged.personId);
}
// TODO: should I update the medoid of this new cluster? My intuition says no, but I'm not sure.
// Update the largest cluster in the database
await _mlDatabase.updateClusterResult(largestCluster);
// Return the merged cluster
return largestCluster;
}
}

34
mobile/lib/services/machine_learning/face_ml/face_feedback.dart/feedback.dart
View file

@ -1,34 +0,0 @@
import "package:photos/models/ml/ml_versions.dart";
import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart';
import "package:uuid/uuid.dart";
abstract class Feedback {
final FeedbackType type;
final String feedbackID;
final DateTime timestamp;
final int madeOnFaceMlVersion;
final int madeOnClusterMlVersion;
get typeString => type.toValueString();
get timestampString => timestamp.toIso8601String();
Feedback(
this.type, {
String? feedbackID,
DateTime? timestamp,
int? madeOnFaceMlVersion,
int? madeOnClusterMlVersion,
}) : feedbackID = feedbackID ?? const Uuid().v4(),
timestamp = timestamp ?? DateTime.now(),
madeOnFaceMlVersion = madeOnFaceMlVersion ?? faceMlVersion,
madeOnClusterMlVersion = madeOnClusterMlVersion ?? clusterMlVersion;
Map<String, dynamic> toJson();
String toJsonString();
// Feedback fromJson(Map<String, dynamic> json);
// Feedback fromJsonString(String jsonString);
}

26
mobile/lib/services/machine_learning/face_ml/face_feedback.dart/feedback_types.dart
View file

@ -1,26 +0,0 @@
enum FeedbackType {
removePhotosClusterFeedback,
addPhotosClusterFeedback,
deleteClusterFeedback,
mergeClusterFeedback,
renameOrCustomThumbnailClusterFeedback; // I have merged renameClusterFeedback and customThumbnailClusterFeedback, since I suspect they will be used together often
factory FeedbackType.fromValueString(String value) {
switch (value) {
case 'deleteClusterFeedback':
return FeedbackType.deleteClusterFeedback;
case 'mergeClusterFeedback':
return FeedbackType.mergeClusterFeedback;
case 'renameOrCustomThumbnailClusterFeedback':
return FeedbackType.renameOrCustomThumbnailClusterFeedback;
case 'removePhotoClusterFeedback':
return FeedbackType.removePhotosClusterFeedback;
case 'addPhotoClusterFeedback':
return FeedbackType.addPhotosClusterFeedback;
default:
throw Exception('Invalid FeedbackType: $value');
}
}
String toValueString() => name;
}

168
mobile/lib/services/machine_learning/face_ml/face_ml_result.dart
View file

@ -2,20 +2,19 @@ import "dart:convert" show jsonEncode, jsonDecode;
import "package:flutter/material.dart" show Size, debugPrint, immutable;
import "package:logging/logging.dart";
import "package:photos/db/ml_data_db.dart";
import "package:photos/models/file/file.dart";
import 'package:photos/models/ml/ml_typedefs.dart';
import "package:photos/models/ml/ml_versions.dart";
import 'package:photos/services/machine_learning/face_ml/face_alignment/alignment_result.dart';
import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart';
import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart';
import 'package:photos/services/machine_learning/face_ml/face_feedback.dart/cluster_feedback.dart';
import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart';
import 'package:photos/services/machine_learning/face_ml/face_ml_methods.dart';
final _logger = Logger('ClusterResult_FaceMlResult');
// TODO: should I add [faceMlVersion] and [clusterMlVersion] to the [ClusterResult] class?
@Deprecated('We are now just storing the cluster results directly in DB')
class ClusterResult {
final int personId;
String? userDefinedName;
@ -263,64 +262,6 @@ class ClusterResultBuilder {
return (medoid!, kthDistance);
}
Future<bool> _checkIfClusterIsDeleted() async {
assert(medoidAndThresholdCalculated);
// Check if the medoid is the default medoid for deleted faces
if (cosineDistance(medoid, List.filled(medoid.length, 10.0)) < 0.001) {
return true;
}
final tempFeedback = DeleteClusterFeedback(
medoid: medoid,
medoidDistanceThreshold: medoidDistanceThreshold,
);
return await MlDataDB.instance
.doesSimilarClusterFeedbackExist(tempFeedback);
}
Future<void> _checkAndAddPhotos() async {
assert(medoidAndThresholdCalculated);
final tempFeedback = AddPhotosClusterFeedback(
medoid: medoid,
medoidDistanceThreshold: medoidDistanceThreshold,
addedPhotoFileIDs: [],
);
final allAddPhotosFeedbacks =
await MlDataDB.instance.getAllMatchingClusterFeedback(tempFeedback);
for (final addPhotosFeedback in allAddPhotosFeedbacks) {
final fileIDsToAdd = addPhotosFeedback.addedPhotoFileIDs;
final faceIDsToAdd = fileIDsToAdd
.map((fileID) => FaceDetectionRelative.toFaceIDEmpty(fileID: fileID))
.toList();
addFileIDsAndFaceIDs(fileIDsToAdd, faceIDsToAdd);
}
}
Future<void> _checkAndAddCustomName() async {
assert(medoidAndThresholdCalculated);
final tempFeedback = RenameOrCustomThumbnailClusterFeedback(
medoid: medoid,
medoidDistanceThreshold: medoidDistanceThreshold,
customName: 'test',
);
final allRenameFeedbacks =
await MlDataDB.instance.getAllMatchingClusterFeedback(tempFeedback);
for (final nameFeedback in allRenameFeedbacks) {
userDefinedName ??= nameFeedback.customName;
if (!thumbnailFaceIdIsUserDefined) {
thumbnailFaceId = nameFeedback.customThumbnailFaceId ?? thumbnailFaceId;
thumbnailFaceIdIsUserDefined =
nameFeedback.customThumbnailFaceId != null;
}
}
return;
}
void changeThumbnailFaceId(String faceId) {
if (!faceIds.contains(faceId)) {
throw Exception(
@ -335,113 +276,6 @@ class ClusterResultBuilder {
fileIds.addAll(addedFileIDs);
faceIds.addAll(addedFaceIDs);
}
static Future<List<ClusterResult>> buildClusters(
List<ClusterResultBuilder> clusterBuilders,
) async {
final List<int> deletedClusterIndices = [];
for (var i = 0; i < clusterBuilders.length; i++) {
final clusterBuilder = clusterBuilders[i];
clusterBuilder.calculateAndSetMedoidAndThreshold();
// Check if the cluster has been deleted
if (await clusterBuilder._checkIfClusterIsDeleted()) {
deletedClusterIndices.add(i);
}
await clusterBuilder._checkAndAddPhotos();
}
// Check if a cluster should be merged with another cluster
for (var i = 0; i < clusterBuilders.length; i++) {
// Don't check for clusters that have been deleted
if (deletedClusterIndices.contains(i)) {
continue;
}
final clusterBuilder = clusterBuilders[i];
final List<MergeClusterFeedback> allMatchingMergeFeedback =
await MlDataDB.instance.getAllMatchingClusterFeedback(
MergeClusterFeedback(
medoid: clusterBuilder.medoid,
medoidDistanceThreshold: clusterBuilder.medoidDistanceThreshold,
medoidToMoveTo: clusterBuilder.medoid,
),
);
if (allMatchingMergeFeedback.isEmpty) {
continue;
}
// Merge the cluster with the first merge feedback
final mainFeedback = allMatchingMergeFeedback.first;
if (allMatchingMergeFeedback.length > 1) {
// This is the BUG!!!!
_logger.warning(
"There are ${allMatchingMergeFeedback.length} merge feedbacks for cluster ${clusterBuilder.personId}. Using the first one.",
);
}
for (var j = 0; j < clusterBuilders.length; j++) {
if (i == j) continue;
final clusterBuilderToMergeTo = clusterBuilders[j];
final distance = cosineDistance(
// BUG: it hasn't calculated the medoid for every clusterBuilder yet!!!
mainFeedback.medoidToMoveTo,
clusterBuilderToMergeTo.medoid,
);
if (distance < mainFeedback.medoidDistanceThreshold ||
distance < clusterBuilderToMergeTo.medoidDistanceThreshold) {
clusterBuilderToMergeTo.addFileIDsAndFaceIDs(
clusterBuilder.fileIds,
clusterBuilder.faceIds,
);
deletedClusterIndices.add(i);
}
}
}
final clusterResults = <ClusterResult>[];
for (var i = 0; i < clusterBuilders.length; i++) {
// Don't build the cluster if it has been deleted or merged
if (deletedClusterIndices.contains(i)) {
continue;
}
final clusterBuilder = clusterBuilders[i];
// Check if the cluster has a custom name or thumbnail
await clusterBuilder._checkAndAddCustomName();
// Build the clusterResult
clusterResults.add(
ClusterResult(
personId: clusterBuilder.personId,
thumbnailFaceId: clusterBuilder.thumbnailFaceId,
fileIds: clusterBuilder.fileIds,
faceIds: clusterBuilder.faceIds,
medoid: clusterBuilder.medoid,
medoidDistanceThreshold: clusterBuilder.medoidDistanceThreshold,
userDefinedName: clusterBuilder.userDefinedName,
thumbnailFaceIdIsUserDefined:
clusterBuilder.thumbnailFaceIdIsUserDefined,
),
);
}
return clusterResults;
}
// TODO: This function should include the feedback from the user. Should also be nullable, since user might want to delete the cluster.
Future<ClusterResult?> _buildSingleCluster() async {
calculateAndSetMedoidAndThreshold();
if (await _checkIfClusterIsDeleted()) {
return null;
}
await _checkAndAddCustomName();
return ClusterResult(
personId: personId,
thumbnailFaceId: thumbnailFaceId,
fileIds: fileIds,
faceIds: faceIds,
medoid: medoid,
medoidDistanceThreshold: medoidDistanceThreshold,
);
}
}
@immutable

61
mobile/lib/services/machine_learning/face_ml/face_ml_service.dart
View file

@ -14,7 +14,6 @@ import "package:onnxruntime/onnxruntime.dart";
import "package:photos/core/configuration.dart";
import "package:photos/core/event_bus.dart";
import "package:photos/db/files_db.dart";
import "package:photos/db/ml_data_db.dart";
import "package:photos/events/diff_sync_complete_event.dart";
import "package:photos/extensions/list.dart";
import "package:photos/extensions/stop_watch.dart";
@ -699,48 +698,6 @@ class FaceMlService {
isImageIndexRunning = false;
}
/// Analyzes the given image data by running the full pipeline using [analyzeImageInComputerAndImageIsolate] and stores the result in the database [MlDataDB].
/// This function first checks if the image has already been analyzed (with latest ml version) and stored in the database. If so, it returns the stored result.
///
/// 'enteFile': The ente file to analyze.
///
/// Returns an immutable [FaceMlResult] instance containing the results of the analysis. The result is also stored in the database.
Future<FaceMlResult> indexImage(EnteFile enteFile) async {
_logger.info(
"`indexImage` called on image with uploadedFileID ${enteFile.uploadedFileID}",
);
_checkEnteFileForID(enteFile);
// Check if the image has already been analyzed and stored in the database with the latest ml version
final existingResult = await _checkForExistingUpToDateResult(enteFile);
if (existingResult != null) {
return existingResult;
}
// If the image has not been analyzed and stored in the database, analyze it and store the result in the database
_logger.info(
"Image with uploadedFileID ${enteFile.uploadedFileID} has not been analyzed and stored in the database. Analyzing it now.",
);
FaceMlResult result;
try {
result = await analyzeImageInComputerAndImageIsolate(enteFile);
} catch (e, s) {
_logger.severe(
"`indexImage` failed on image with uploadedFileID ${enteFile.uploadedFileID}",
e,
s,
);
throw GeneralFaceMlException(
"`indexImage` failed on image with uploadedFileID ${enteFile.uploadedFileID}",
);
}
// Store the result in the database
await MlDataDB.instance.createFaceMlResult(result);
return result;
}
/// Analyzes the given image data by running the full pipeline (face detection, face alignment, face embedding).
///
/// [enteFile] The ente file to analyze.
@ -1266,22 +1223,4 @@ class FaceMlService {
indexedFileIds[id]! >= faceMlVersion;
}
Future<FaceMlResult?> _checkForExistingUpToDateResult(
EnteFile enteFile,
) async {
// Check if the image has already been analyzed and stored in the database
final existingResult =
await MlDataDB.instance.getFaceMlResult(enteFile.uploadedFileID!);
// If the image has already been analyzed and stored in the database, return the stored result
if (existingResult != null) {
if (existingResult.mlVersion >= faceMlVersion) {
_logger.info(
"Image with uploadedFileID ${enteFile.uploadedFileID} has already been analyzed and stored in the database with the latest ml version. Returning the stored result.",
);
return existingResult;
}
}
return null;
}
}

123
mobile/lib/services/machine_learning/face_ml/face_search_service.dart
View file

@ -1,123 +0,0 @@
// import "dart:io" show File;
// import "dart:typed_data" show Uint8List;
import "package:logging/logging.dart";
import "package:photos/db/files_db.dart";
import "package:photos/db/ml_data_db.dart";
import "package:photos/models/file/file.dart";
// import 'package:photos/utils/image_ml_isolate.dart';
// import "package:photos/utils/thumbnail_util.dart";
class FaceSearchService {
final _logger = Logger("FaceSearchService");
final _mlDatabase = MlDataDB.instance;
final _filesDatabase = FilesDB.instance;
// singleton pattern
FaceSearchService._privateConstructor();
static final instance = FaceSearchService._privateConstructor();
factory FaceSearchService() => instance;
/// Returns the personIDs of all clustered people in the database.
Future<List<int>> getAllPeople() async {
final peopleIds = await _mlDatabase.getAllClusterIds();
return peopleIds;
}
// /// Returns the thumbnail associated with a given personId.
// Future<Uint8List?> getPersonThumbnail(int personID) async {
// // get the cluster associated with the personID
// final cluster = await _mlDatabase.getClusterResult(personID);
// if (cluster == null) {
// _logger.warning(
// "No cluster found for personID $personID, unable to get thumbnail.",
// );
// return null;
// }
// // get the faceID and fileID you want to use to generate the thumbnail
// final String thumbnailFaceID = cluster.thumbnailFaceId;
// final int thumbnailFileID = cluster.thumbnailFileId;
// // get the full file thumbnail
// final EnteFile enteFile = await _filesDatabase
// .getFilesFromIDs([thumbnailFileID]).then((value) => value.values.first);
// final File? fileThumbnail = await getThumbnailForUploadedFile(enteFile);
// if (fileThumbnail == null) {
// _logger.warning(
// "No full file thumbnail found for thumbnail faceID $thumbnailFaceID, unable to get thumbnail.",
// );
// return null;
// }
// // get the face detection for the thumbnail
// final thumbnailMlResult =
// await _mlDatabase.getFaceMlResult(thumbnailFileID);
// if (thumbnailMlResult == null) {
// _logger.warning(
// "No face ml result found for thumbnail faceID $thumbnailFaceID, unable to get thumbnail.",
// );
// return null;
// }
// final detection = thumbnailMlResult.getDetectionForFaceId(thumbnailFaceID);
// // create the thumbnail from the full file thumbnail and the face detection
// Uint8List faceThumbnail;
// try {
// faceThumbnail = await ImageMlIsolate.instance
// .generateFaceThumbnailsForImage(
// fileThumbnail.path,
// detection,
// )
// .then((value) => value[0]);
// } catch (e, s) {
// _logger.warning(
// "Unable to generate face thumbnail for thumbnail faceID $thumbnailFaceID, unable to get thumbnail.",
// e,
// s,
// );
// return null;
// }
// return faceThumbnail;
// }
/// Returns all files associated with a given personId.
Future<List<EnteFile>> getFilesForPerson(int personID) async {
final fileIDs = await _mlDatabase.getClusterFileIds(personID);
final Map<int, EnteFile> files =
await _filesDatabase.getFilesFromIDs(fileIDs);
return files.values.toList();
}
Future<List<EnteFile>> getFilesForIntersectOfPeople(
List<int> personIDs,
) async {
if (personIDs.length <= 1) {
_logger
.warning('Cannot get intersection of files for less than 2 people');
return <EnteFile>[];
}
final Set<int> fileIDsFirstCluster = await _mlDatabase
.getClusterFileIds(personIDs.first)
.then((value) => value.toSet());
for (final personID in personIDs.sublist(1)) {
final fileIDsSingleCluster =
await _mlDatabase.getClusterFileIds(personID);
fileIDsFirstCluster.retainAll(fileIDsSingleCluster);
// Early termination if intersection is empty
if (fileIDsFirstCluster.isEmpty) {
return <EnteFile>[];
}
}
final Map<int, EnteFile> files =
await _filesDatabase.getFilesFromIDs(fileIDsFirstCluster.toList());
return files.values.toList();
}
}

2
mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart
View file

@ -402,7 +402,7 @@ class _FileSelectionActionsWidgetState
);
// if (widget.type == GalleryType.cluster && widget.clusterID != null) {
if (widget.type == GalleryType.cluster) {
if (widget.type == GalleryType.cluster && widget.clusterID != null) {
items.add(
SelectionActionButton(
labelText: 'Remove',

3
mobile/lib/ui/viewer/actions/file_selection_overlay_bar.dart
View file

@ -12,6 +12,7 @@ class FileSelectionOverlayBar extends StatefulWidget {
final Collection? collection;
final Color? backgroundColor;
final Person? person;
final int? clusterID;
const FileSelectionOverlayBar(
this.galleryType,
@ -19,6 +20,7 @@ class FileSelectionOverlayBar extends StatefulWidget {
this.collection,
this.backgroundColor,
this.person,
this.clusterID,
Key? key,
}) : super(key: key);
@ -69,6 +71,7 @@ class _FileSelectionOverlayBarState extends State<FileSelectionOverlayBar> {
galleryType: widget.galleryType,
collection: widget.collection,
person: widget.person,
clusterID: widget.clusterID,
onCancel: () {
if (widget.selectedFiles.files.isNotEmpty) {
widget.selectedFiles.clearAll();

1
mobile/lib/ui/viewer/people/cluster_page.dart
View file

@ -159,6 +159,7 @@ class _ClusterPageState extends State<ClusterPage> {
FileSelectionOverlayBar(
ClusterPage.overlayType,
_selectedFiles,
clusterID: widget.cluserID,
),
],
),