beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

[mob] Return list instead of set

Browse source
This commit is contained in:
Neeraj Gupta 2024-05-10 13:07:24 +05:30
parent 33a0641c52
commit 7623e69de3
2 changed files with 52 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

85
mobile/lib/face/db.dart
View file

@ -511,56 +511,61 @@ class FaceMLDataDB {
});
}
Future<Set<FaceInfoForClustering>> getFaceInfoForClustering({
Future<List<FaceInfoForClustering>> getFaceInfoForClustering({
double minScore = kMinimumQualityFaceScore,
int minClarity = kLaplacianHardThreshold,
int maxFaces = 20000,
int offset = 0,
int batchSize = 10000,
}) async {
final EnteWatch w = EnteWatch("getFaceEmbeddingMap")..start();
w.logAndReset(
'reading as float offset: $offset, maxFaces: $maxFaces, batchSize: $batchSize',
);
final db = await instance.asyncDB;
final Set<FaceInfoForClustering> result = {};
while (true) {
// Query a batch of rows
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $faceIDColumn, $faceEmbeddingBlob, $faceScore, $faceBlur, $isSideways FROM $facesTable'
' WHERE $faceScore > $minScore AND $faceBlur > $minClarity'
' ORDER BY $faceIDColumn'
' DESC LIMIT $batchSize OFFSET $offset',
try {
final EnteWatch w = EnteWatch("getFaceEmbeddingMap")..start();
w.logAndReset(
'reading as float offset: $offset, maxFaces: $maxFaces, batchSize: $batchSize',
);
// Break the loop if no more rows
if (maps.isEmpty) {
break;
}
final List<String> faceIds = [];
for (final map in maps) {
faceIds.add(map[faceIDColumn] as String);
}
final faceIdToClusterId = await getFaceIdsToClusterIds(faceIds);
for (final map in maps) {
final faceID = map[faceIDColumn] as String;
final faceInfo = FaceInfoForClustering(
faceID: faceID,
clusterId: faceIdToClusterId[faceID],
embeddingBytes: map[faceEmbeddingBlob] as Uint8List,
faceScore: map[faceScore] as double,
blurValue: map[faceBlur] as double,
isSideways: (map[isSideways] as int) == 1,
final db = await instance.asyncDB;
final List<FaceInfoForClustering> result = {};
while (true) {
// Query a batch of rows
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $faceIDColumn, $faceEmbeddingBlob, $faceScore, $faceBlur, $isSideways FROM $facesTable'
' WHERE $faceScore > $minScore AND $faceBlur > $minClarity'
' ORDER BY $faceIDColumn'
' DESC LIMIT $batchSize OFFSET $offset',
);
result.add(faceInfo);
// Break the loop if no more rows
if (maps.isEmpty) {
break;
}
final List<String> faceIds = [];
for (final map in maps) {
faceIds.add(map[faceIDColumn] as String);
}
final faceIdToClusterId = await getFaceIdsToClusterIds(faceIds);
for (final map in maps) {
final faceID = map[faceIDColumn] as String;
final faceInfo = FaceInfoForClustering(
faceID: faceID,
clusterId: faceIdToClusterId[faceID],
embeddingBytes: map[faceEmbeddingBlob] as Uint8List,
faceScore: map[faceScore] as double,
blurValue: map[faceBlur] as double,
isSideways: (map[isSideways] as int) == 1,
);
result.add(faceInfo);
}
if (result.length >= maxFaces) {
break;
}
offset += batchSize;
}
if (result.length >= maxFaces) {
break;
}
offset += batchSize;
w.stopWithLog('done reading face embeddings ${result.length}');
return result;
} catch (e) {
_logger.severe('Error in getFaceInfoForClustering', e);
rethrow;
}
w.stopWithLog('done reading face embeddings ${result.length}');
return result;
}
/// Returns a map of faceID to record of clusterId and faceEmbeddingBlob

16
mobile/lib/services/machine_learning/face_ml/face_ml_service.dart
View file

@ -71,7 +71,9 @@ class FaceMlService {
// singleton pattern
FaceMlService._privateConstructor();
static final instance = FaceMlService._privateConstructor();
factory FaceMlService() => instance;
final _initLock = Lock();
@ -302,19 +304,15 @@ class FaceMlService {
// Get a sense of the total number of faces in the database
final int totalFaces = await FaceMLDataDB.instance
.getTotalFaceCount(minFaceScore: minFaceScore);
// read the creation times from Files DB, in a map from fileID to creation time
final fileIDToCreationTime =
await FilesDB.instance.getFileIDToCreationTime();
final startEmbeddingFetch = DateTime.now();
// read all embeddings
final allFaceInfoForClustering = await FaceMLDataDB.instance
.getFaceInfoForClustering(
minScore: minFaceScore,
maxFaces: totalFaces,
)
.then((set) => set.toList());
final allFaceInfoForClustering =
await FaceMLDataDB.instance.getFaceInfoForClustering(
minScore: minFaceScore,
maxFaces: totalFaces,
);
// sort the embeddings based on file creation time, oldest first
allFaceInfoForClustering.sort((a, b) {
final aFileId = getFileIdFromFaceId(a.faceID);