beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Face (#1885)

Browse source 
## Description

- Several fixes for Faces
This commit is contained in:
Laurens Priem 2024-05-27 17:46:05 +05:30 committed by GitHub
parent c7be2270ff f25f119ca1
commit 232acfa211
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 127 additions and 87 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

76
mobile/lib/face/db.dart
View file

@ -35,6 +35,15 @@ class FaceMLDataDB {
static final FaceMLDataDB instance = FaceMLDataDB._privateConstructor();
static final _migrationScripts = [
createFacesTable,
createFaceClustersTable,
createClusterPersonTable,
createClusterSummaryTable,
createNotPersonFeedbackTable,
fcClusterIDIndex,
];
// only have a single app-wide reference to the database
static Future<SqliteDatabase>? _sqliteAsyncDBFuture;
@ -50,17 +59,42 @@ class FaceMLDataDB {
_logger.info("Opening sqlite_async access: DB path " + databaseDirectory);
final asyncDBConnection =
SqliteDatabase(path: databaseDirectory, maxReaders: 2);
await _onCreate(asyncDBConnection);
final stopwatch = Stopwatch()..start();
_logger.info("FaceMLDataDB: Starting migration");
await _migrate(asyncDBConnection);
_logger.info(
"FaceMLDataDB Migration took ${stopwatch.elapsedMilliseconds} ms",
);
stopwatch.stop();
return asyncDBConnection;
}
Future<void> _onCreate(SqliteDatabase asyncDBConnection) async {
await asyncDBConnection.execute(createFacesTable);
await asyncDBConnection.execute(createFaceClustersTable);
await asyncDBConnection.execute(createClusterPersonTable);
await asyncDBConnection.execute(createClusterSummaryTable);
await asyncDBConnection.execute(createNotPersonFeedbackTable);
await asyncDBConnection.execute(fcClusterIDIndex);
Future<void> _migrate(
SqliteDatabase database,
) async {
final result = await database.execute('PRAGMA user_version');
final currentVersion = result[0]['user_version'] as int;
final toVersion = _migrationScripts.length;
if (currentVersion < toVersion) {
_logger.info("Migrating database from $currentVersion to $toVersion");
await database.writeTransaction((tx) async {
for (int i = currentVersion + 1; i <= toVersion; i++) {
try {
await tx.execute(_migrationScripts[i - 1]);
} catch (e) {
_logger.severe("Error running migration script index ${i - 1}", e);
rethrow;
}
}
await tx.execute('PRAGMA user_version = $toVersion');
});
} else if (currentVersion > toVersion) {
throw AssertionError(
"currentVersion($currentVersion) cannot be greater than toVersion($toVersion)",
);
}
}
// bulkInsertFaces inserts the faces in the database in batches of 1000.
@ -195,10 +229,10 @@ class FaceMLDataDB {
final db = await instance.asyncDB;
await db.execute(deleteFacesTable);
await db.execute(dropClusterPersonTable);
await db.execute(dropClusterSummaryTable);
await db.execute(deletePersonTable);
await db.execute(dropNotPersonFeedbackTable);
await db.execute(deleteFaceClustersTable);
await db.execute(deleteClusterPersonTable);
await db.execute(deleteClusterSummaryTable);
await db.execute(deleteNotPersonFeedbackTable);
}
Future<Iterable<Uint8List>> getFaceEmbeddingsForCluster(
@ -734,7 +768,7 @@ class FaceMLDataDB {
try {
final db = await instance.asyncDB;
await db.execute(dropFaceClustersTable);
await db.execute(deleteFaceClustersTable);
await db.execute(createFaceClustersTable);
await db.execute(fcClusterIDIndex);
} catch (e, s) {
@ -945,16 +979,15 @@ class FaceMLDataDB {
if (faces) {
await db.execute(deleteFacesTable);
await db.execute(createFacesTable);
await db.execute(dropFaceClustersTable);
await db.execute(deleteFaceClustersTable);
await db.execute(createFaceClustersTable);
await db.execute(fcClusterIDIndex);
}
await db.execute(deletePersonTable);
await db.execute(dropClusterPersonTable);
await db.execute(dropNotPersonFeedbackTable);
await db.execute(dropClusterSummaryTable);
await db.execute(dropFaceClustersTable);
await db.execute(deleteClusterPersonTable);
await db.execute(deleteNotPersonFeedbackTable);
await db.execute(deleteClusterSummaryTable);
await db.execute(deleteFaceClustersTable);
await db.execute(createClusterPersonTable);
await db.execute(createNotPersonFeedbackTable);
@ -972,9 +1005,8 @@ class FaceMLDataDB {
final db = await instance.asyncDB;
// Drop the tables
await db.execute(deletePersonTable);
await db.execute(dropClusterPersonTable);
await db.execute(dropNotPersonFeedbackTable);
await db.execute(deleteClusterPersonTable);
await db.execute(deleteNotPersonFeedbackTable);
// Recreate the tables
await db.execute(createClusterPersonTable);

16
mobile/lib/face/db_fields.dart
View file

@ -29,7 +29,7 @@ const createFacesTable = '''CREATE TABLE IF NOT EXISTS $facesTable (
);
''';
const deleteFacesTable = 'DROP TABLE IF EXISTS $facesTable';
const deleteFacesTable = 'DELETE FROM $facesTable';
// End of Faces Table Fields & Schema Queries
//##region Face Clusters Table Fields & Schema Queries
@ -48,15 +48,9 @@ CREATE TABLE IF NOT EXISTS $faceClustersTable (
// -- Creating a non-unique index on clusterID for query optimization
const fcClusterIDIndex =
'''CREATE INDEX IF NOT EXISTS idx_fcClusterID ON $faceClustersTable($fcClusterID);''';
const dropFaceClustersTable = 'DROP TABLE IF EXISTS $faceClustersTable';
const deleteFaceClustersTable = 'DELETE FROM $faceClustersTable';
//##endregion
// People Table Fields & Schema Queries
const personTable = 'person';
const deletePersonTable = 'DROP TABLE IF EXISTS $personTable';
//End People Table Fields & Schema Queries
// Clusters Table Fields & Schema Queries
const clusterPersonTable = 'cluster_person';
const personIdColumn = 'person_id';
@ -69,7 +63,7 @@ CREATE TABLE IF NOT EXISTS $clusterPersonTable (
PRIMARY KEY($personIdColumn, $clusterIDColumn)
);
''';
const dropClusterPersonTable = 'DROP TABLE IF EXISTS $clusterPersonTable';
const deleteClusterPersonTable = 'DELETE FROM $clusterPersonTable';
// End Clusters Table Fields & Schema Queries
/// Cluster Summary Table Fields & Schema Queries
@ -85,7 +79,7 @@ CREATE TABLE IF NOT EXISTS $clusterSummaryTable (
);
''';
const dropClusterSummaryTable = 'DROP TABLE IF EXISTS $clusterSummaryTable';
const deleteClusterSummaryTable = 'DELETE FROM $clusterSummaryTable';
/// End Cluster Summary Table Fields & Schema Queries
@ -99,5 +93,5 @@ CREATE TABLE IF NOT EXISTS $notPersonFeedback (
PRIMARY KEY($personIdColumn, $clusterIDColumn)
);
''';
const dropNotPersonFeedbackTable = 'DROP TABLE IF EXISTS $notPersonFeedback';
const deleteNotPersonFeedbackTable = 'DELETE FROM $notPersonFeedback';
// End Clusters Table Fields & Schema Queries

2
mobile/lib/generated/intl/messages_en.dart generated
View file

@ -814,7 +814,7 @@ class MessageLookup extends MessageLookupByLibrary {
MessageLookupByLibrary.simpleMessage("Incorrect recovery key"),
"indexedItems": MessageLookupByLibrary.simpleMessage("Indexed items"),
"indexingIsPaused": MessageLookupByLibrary.simpleMessage(
"Indexing is paused, will automatically resume when device is ready"),
"Indexing is paused. It will automatically resume when device is ready."),
"insecureDevice":
MessageLookupByLibrary.simpleMessage("Insecure device"),
"installManually":

4
mobile/lib/generated/l10n.dart generated
View file

@ -8794,10 +8794,10 @@ class S {
);
}
/// `Indexing is paused, will automatically resume when device is ready`
/// `Indexing is paused. It will automatically resume when device is ready.`
String get indexingIsPaused {
return Intl.message(
'Indexing is paused, will automatically resume when device is ready',
'Indexing is paused. It will automatically resume when device is ready.',
name: 'indexingIsPaused',
desc: '',
args: [],

4
mobile/lib/l10n/intl_en.arb
View file

@ -1236,5 +1236,5 @@
"faceRecognitionIndexingDescription": "Please note that this will result in a higher bandwidth and battery usage until all items are indexed.",
"foundFaces": "Found faces",
"clusteringProgress": "Clustering progress",
"indexingIsPaused": "Indexing is paused, will automatically resume when device is ready"
}
"indexingIsPaused": "Indexing is paused. It will automatically resume when device is ready."
}

16
mobile/lib/main.dart
View file

@ -246,17 +246,11 @@ Future<void> _init(bool isBackground, {String via = ''}) async {
unawaited(SemanticSearchService.instance.init());
MachineLearningController.instance.init();
// Can not including existing tf/ml binaries as they are not being built
// from source.
// See https://gitlab.com/fdroid/fdroiddata/-/merge_requests/12671#note_1294346819
if (!UpdateService.instance.isFdroidFlavor()) {
// unawaited(ObjectDetectionService.instance.init());
if (flagService.faceSearchEnabled) {
unawaited(FaceMlService.instance.init());
} else {
if (LocalSettings.instance.isFaceIndexingEnabled) {
unawaited(LocalSettings.instance.toggleFaceIndexing());
}
if (flagService.faceSearchEnabled) {
unawaited(FaceMlService.instance.init());
} else {
if (LocalSettings.instance.isFaceIndexingEnabled) {
unawaited(LocalSettings.instance.toggleFaceIndexing());
}
}
PersonService.init(

18
mobile/lib/services/machine_learning/face_ml/face_ml_service.dart
View file

@ -43,6 +43,7 @@ import 'package:photos/services/machine_learning/face_ml/face_ml_result.dart';
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import 'package:photos/services/machine_learning/file_ml/file_ml.dart';
import 'package:photos/services/machine_learning/file_ml/remote_fileml_service.dart';
import "package:photos/services/machine_learning/machine_learning_controller.dart";
import "package:photos/services/search_service.dart";
import "package:photos/utils/file_util.dart";
import 'package:photos/utils/image_ml_isolate.dart';
@ -99,7 +100,7 @@ class FaceMlService {
final int _fileDownloadLimit = 5;
final int _embeddingFetchLimit = 200;
final int _kForceClusteringFaceCount = 4000;
final int _kForceClusteringFaceCount = 8000;
Future<void> init({bool initializeImageMlIsolate = false}) async {
if (LocalSettings.instance.isFaceIndexingEnabled == false) {
@ -163,9 +164,16 @@ class FaceMlService {
pauseIndexingAndClustering();
}
});
if (Platform.isIOS &&
MachineLearningController.instance.isDeviceHealthy) {
_logger.info("Starting face indexing and clustering on iOS from init");
unawaited(indexAndClusterAll());
}
_listenIndexOnDiffSync();
_listenOnPeopleChangedSync();
_logger.info('init done');
});
}
@ -1016,9 +1024,13 @@ class FaceMlService {
File? file;
if (enteFile.fileType == FileType.video) {
try {
file = await getThumbnailForUploadedFile(enteFile);
file = await getThumbnailForUploadedFile(enteFile);
} on PlatformException catch (e, s) {
_logger.severe("Could not get thumbnail for $enteFile due to PlatformException", e, s);
_logger.severe(
"Could not get thumbnail for $enteFile due to PlatformException",
e,
s,
);
throw ThumbnailRetrievalException(e.toString(), s);
}
} else {

6
mobile/lib/services/machine_learning/machine_learning_controller.dart
View file

@ -4,7 +4,6 @@ import "dart:io";
import "package:battery_info/battery_info_plugin.dart";
import "package:battery_info/model/android_battery_info.dart";
import "package:battery_info/model/iso_battery_info.dart";
import "package:flutter/foundation.dart" show kDebugMode;
import "package:logging/logging.dart";
import "package:photos/core/event_bus.dart";
import "package:photos/events/machine_learning_control_event.dart";
@ -19,8 +18,7 @@ class MachineLearningController {
static const kMaximumTemperature = 42; // 42 degree celsius
static const kMinimumBatteryLevel = 20; // 20%
static const kDefaultInteractionTimeout =
kDebugMode ? Duration(seconds: 3) : Duration(seconds: 5);
static const kDefaultInteractionTimeout = Duration(seconds: 10);
static const kUnhealthyStates = ["over_heat", "over_voltage", "dead"];
bool _isDeviceHealthy = true;
@ -31,6 +29,7 @@ class MachineLearningController {
bool get isDeviceHealthy => _isDeviceHealthy;
void init() {
_logger.info('init called');
if (Platform.isAndroid) {
_startInteractionTimer();
BatteryInfoPlugin()
@ -47,6 +46,7 @@ class MachineLearningController {
});
}
_fireControlEvent();
_logger.info('init done');
}
void onUserInteraction() {

4
mobile/lib/ui/settings/machine_learning_settings_page.dart
View file

@ -89,8 +89,8 @@ class _MachineLearningSettingsPageState
iconButtonType: IconButtonType.secondary,
onTap: () {
Navigator.pop(context);
Navigator.pop(context);
Navigator.pop(context);
if (Navigator.canPop(context)) Navigator.pop(context);
if (Navigator.canPop(context)) Navigator.pop(context);
},
),
],

2
mobile/lib/ui/viewer/people/cluster_app_bar.dart
View file

@ -97,7 +97,7 @@ class _AppBarWidgetState extends State<ClusterAppBar> {
maxLines: 2,
overflow: TextOverflow.ellipsis,
),
actions: kDebugMode ? _getDefaultActions(context) : null,
actions: _getDefaultActions(context),
);
}

64
mobile/lib/ui/viewer/people/person_clusters_page.dart
View file

@ -38,12 +38,17 @@ class _PersonClustersPageState extends State<PersonClustersPage> {
.getClusterFilesForPersonID(widget.person.remoteID),
builder: (context, snapshot) {
if (snapshot.hasData) {
final List<int> keys = snapshot.data!.keys.toList();
final clusters = snapshot.data!;
final List<int> keys = clusters.keys.toList();
// Sort the clusters by the number of files in each cluster, largest first
keys.sort(
(b, a) => clusters[a]!.length.compareTo(clusters[b]!.length),
);
return ListView.builder(
itemCount: keys.length,
itemBuilder: (context, index) {
final int clusterID = keys[index];
final List<EnteFile> files = snapshot.data![keys[index]]!;
final List<EnteFile> files = clusters[clusterID]!;
return InkWell(
onTap: () {
Navigator.of(context).push(
@ -93,34 +98,37 @@ class _PersonClustersPageState extends State<PersonClustersPage> {
mainAxisAlignment: MainAxisAlignment.spaceBetween,
children: <Widget>[
Text(
"${snapshot.data![keys[index]]!.length} photos",
"${files.length} photos",
style: getEnteTextTheme(context).body,
),
GestureDetector(
onTap: () async {
try {
await PersonService.instance
.removeClusterToPerson(
personID: widget.person.remoteID,
clusterID: clusterID,
);
_logger.info(
"Removed cluster $clusterID from person ${widget.person.remoteID}",
);
Bus.instance.fire(PeopleChangedEvent());
setState(() {});
} catch (e) {
_logger.severe(
"removing cluster from person,",
e,
);
}
},
child: const Icon(
CupertinoIcons.minus_circled,
color: Colors.red,
),
),
(index != 0)
? GestureDetector(
onTap: () async {
try {
await PersonService.instance
.removeClusterToPerson(
personID: widget.person.remoteID,
clusterID: clusterID,
);
_logger.info(
"Removed cluster $clusterID from person ${widget.person.remoteID}",
);
Bus.instance
.fire(PeopleChangedEvent());
setState(() {});
} catch (e) {
_logger.severe(
"removing cluster from person,",
e,
);
}
},
child: const Icon(
CupertinoIcons.minus_circled,
color: Colors.red,
),
)
: const SizedBox.shrink(),
],
),
),

2
mobile/pubspec.yaml
View file

@ -12,7 +12,7 @@ description: ente photos application
# Read more about iOS versioning at
# https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html
version: 0.8.110+634
version: 0.8.112+636
publish_to: none
environment: