beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Change Face to use relative coordinates

Browse source
This commit is contained in:
laurenspriem 2024-03-15 17:07:17 +05:30
parent ca16c6f0d6
commit 9285baace2
6 changed files with 58 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
mobile/lib/face/db_model_mappers.dart
View file

@ -4,7 +4,6 @@ import 'package:photos/face/db_fields.dart';
import "package:photos/face/model/detection.dart";
import "package:photos/face/model/face.dart";
import "package:photos/face/model/person.dart";
import 'package:photos/face/model/person_face.dart';
import "package:photos/generated/protos/ente/common/vector.pb.dart";
int boolToSQLInt(bool? value, {bool defaultValue = false}) {

29
mobile/lib/face/model/box.dart
View file

@ -1,29 +1,30 @@
/// Bounding box of a face.
///
/// [`x`] and [y] are the coordinates of the top left corner of the box, so the minimim values
/// [xMin] and [yMin] are the coordinates of the top left corner of the box, and
/// [width] and [height] are the width and height of the box.
/// All values are in absolute pixels relative to the original image size.
///
/// WARNING: All values are relative to the original image size, so in the range [0, 1].
class FaceBox {
final double x;
final double y;
final double xMin;
final double yMin;
final double width;
final double height;
FaceBox({
required this.x,
required this.y,
required this.xMin,
required this.yMin,
required this.width,
required this.height,
});
factory FaceBox.fromJson(Map<String, dynamic> json) {
return FaceBox(
x: (json['x'] is int
? (json['x'] as int).toDouble()
: json['x'] as double),
y: (json['y'] is int
? (json['y'] as int).toDouble()
: json['y'] as double),
xMin: (json['xMin'] is int
? (json['xMin'] as int).toDouble()
: json['xMin'] as double),
yMin: (json['yMin'] is int
? (json['yMin'] as int).toDouble()
: json['yMin'] as double),
width: (json['width'] is int
? (json['width'] as int).toDouble()
: json['width'] as double),
@ -34,8 +35,8 @@ class FaceBox {
}
Map<String, dynamic> toJson() => {
'x': x,
'y': y,
'xMin': xMin,
'yMin': yMin,
'width': width,
'height': height,
};

9
mobile/lib/face/model/detection.dart
View file

@ -1,6 +1,9 @@
import "package:photos/face/model/box.dart";
import "package:photos/face/model/landmark.dart";
/// Stores the face detection data, notably the bounding box and landmarks.
///
/// WARNING: All coordinates are relative to the image size, so in the range [0, 1]!
class Detection {
FaceBox box;
List<Landmark> landmarks;
@ -10,11 +13,13 @@ class Detection {
required this.landmarks,
});
bool get isEmpty => box.width == 0 && box.height == 0 && landmarks.isEmpty;
// emoty box
Detection.empty()
: box = FaceBox(
x: 0,
y: 0,
xMin: 0,
yMin: 0,
width: 0,
height: 0,
),

4
mobile/lib/face/model/landmark.dart
View file

@ -1,4 +1,6 @@
// Class for the 'landmark' sub-object
/// Landmark coordinate data.
///
/// WARNING: All coordinates are relative to the image size, so in the range [0, 1]!
class Landmark {
double x;
double y;

26
mobile/lib/services/face_ml/face_ml_service.dart
View file

@ -496,35 +496,21 @@ class FaceMlService {
_logger.severe(
"faceDetectionImageSize or faceDetectionImageSize is null for image with "
"ID: ${enteFile.uploadedFileID}");
}
final bool useAlign = result.faceAlignmentImageSize != null &&
result.faceAlignmentImageSize!.width > 0 &&
result.faceAlignmentImageSize!.height > 0 &&
result.onlyThumbnailUsed == false;
if (useAlign) {
_logger.info(
"Using aligned image size for image with ID: ${enteFile.uploadedFileID}. This size is ${result.faceAlignmentImageSize!.width}x${result.faceAlignmentImageSize!.height} compared to size of ${enteFile.width}x${enteFile.height} in the metadata",
);
}
for (int i = 0; i < result.faces.length; ++i) {
final FaceResult faceRes = result.faces[i];
final FaceDetectionAbsolute absoluteDetection =
faceRes.detection.toAbsolute(
imageWidth: useAlign
? result.faceAlignmentImageSize!.width.toInt()
: enteFile.width,
imageHeight: useAlign
? result.faceAlignmentImageSize!.height.toInt()
: enteFile.height,
);
final FaceDetectionRelative relativeDetection = faceRes.detection;
final detection = face_detection.Detection(
box: FaceBox(
x: absoluteDetection.xMinBox,
y: absoluteDetection.yMinBox,
width: absoluteDetection.width,
height: absoluteDetection.height,
xMin: relativeDetection.xMinBox,
yMin: relativeDetection.yMinBox,
width: relativeDetection.width,
height: relativeDetection.height,
),
landmarks: absoluteDetection.allKeypoints
landmarks: relativeDetection.allKeypoints
.map(
(keypoint) => Landmark(
x: keypoint[0],

42
mobile/lib/utils/image_ml_util.dart
View file

@ -1232,23 +1232,25 @@ Future<List<Uint8List>> generateFaceThumbnails(
) async {
final stopwatch = Stopwatch()..start();
final Image image = await decodeImageFromData(imageData);
final ByteData imgByteData = await getByteDataFromImage(image);
final Image img = await decodeImageFromData(imageData);
final ByteData imgByteData = await getByteDataFromImage(img);
try {
final List<Uint8List> faceThumbnails = [];
for (final faceBox in faceBoxes) {
final int xCrop =
(faceBox.x - faceBox.width / 2).round().clamp(0, image.width);
final int yCrop =
(faceBox.y - faceBox.height / 2).round().clamp(0, image.height);
final int widthCrop =
min((faceBox.width * 2).round(), image.width - xCrop);
final int heightCrop =
min((faceBox.height * 2).round(), image.height - yCrop);
// Note that the faceBox values are relative to the image size, so we need to convert them to absolute values first
final double xMinAbs = faceBox.xMin * img.width;
final double yMinAbs = faceBox.yMin * img.height;
final double widthAbs = faceBox.width * img.width;
final double heightAbs = faceBox.height * img.height;
final int xCrop = (xMinAbs - widthAbs / 2).round().clamp(0, img.width);
final int yCrop = (yMinAbs - heightAbs / 2).round().clamp(0, img.height);
final int widthCrop = min((widthAbs * 2).round(), img.width - xCrop);
final int heightCrop = min((heightAbs * 2).round(), img.height - yCrop);
final Image faceThumbnail = await cropImage(
image,
img,
imgByteData,
x: xCrop,
y: yCrop,
@ -1280,19 +1282,25 @@ Future<List<Uint8List>> generateFaceThumbnailsFromDataAndDetections(
Uint8List imageData,
List<FaceBox> faceBoxes,
) async {
final Image image = await decodeImageFromData(imageData);
final Image img = await decodeImageFromData(imageData);
int i = 0;
try {
final List<Uint8List> faceThumbnails = [];
for (final faceBox in faceBoxes) {
// Note that the faceBox values are relative to the image size, so we need to convert them to absolute values first
final double xMinAbs = faceBox.xMin * img.width;
final double yMinAbs = faceBox.yMin * img.height;
final double widthAbs = faceBox.width * img.width;
final double heightAbs = faceBox.height * img.height;
final Image faceThumbnail = await cropImageWithCanvas(
image,
x: faceBox.x - faceBox.width / 2,
y: faceBox.y - faceBox.height / 2,
width: faceBox.width * 2,
height: faceBox.height * 2,
img,
x: xMinAbs - widthAbs / 2,
y: yMinAbs - heightAbs / 2,
width: widthAbs * 2,
height: heightAbs * 2,
);
final Uint8List faceThumbnailPng = await encodeImageToUint8List(
faceThumbnail,