[mob][photos] Generate face crops faster (#1542)
## Description Have written two new methods, `generateImgFaceThumbnails()` and `generateJpgFaceThumbnails()`. Using `generateJpgFaceThumbnails()` now since it returns `Future<List<Uint8List>>` and is easier to integrate within the code base because the return type remains the same with the older `generateFaceThumbnailsForImage()` There is performance improvement with `generateImgFaceThumbnails()`, but it's not very significant and it requires changes in codebase to work with it's return type `Future<List<Image>>` (`Image` from the `Image` package). Can consider using it if it feels necessary in future. If multiple faces are being generated from the same image, the image can be decoded once and passed to `generateImgFaceThumbnails()` or `generateJpgFaceThumbnails()` to avoid repeated decoding of the same image. `generateImgFaceThumbnails()` and `generateJpgFaceThumbnails()` uses the isolates available from the pool of 4 spawned by `Computer` and processes multiple faces in parallel unlike `generateImgFaceThumbnails()`, which processes only one at a time.
This commit is contained in:
commit
9eeab36392
|
@ -41,3 +41,23 @@ class FaceBox {
|
|||
'height': height,
|
||||
};
|
||||
}
|
||||
|
||||
/// Bounding box of a face.
|
||||
///
|
||||
/// [xMin] and [yMin] are the coordinates of the top left corner of the box, and
|
||||
/// [width] and [height] are the width and height of the box.
|
||||
///
|
||||
/// One unit is equal to one pixel in the original image.
|
||||
class FaceBoxImage {
|
||||
final int xMin;
|
||||
final int yMin;
|
||||
final int width;
|
||||
final int height;
|
||||
|
||||
FaceBoxImage({
|
||||
required this.xMin,
|
||||
required this.yMin,
|
||||
required this.width,
|
||||
required this.height,
|
||||
});
|
||||
}
|
||||
|
|
|
@ -21,7 +21,7 @@ import "package:photos/utils/face/face_box_crop.dart";
|
|||
import "package:photos/utils/thumbnail_util.dart";
|
||||
// import "package:photos/utils/toast_util.dart";
|
||||
|
||||
const useGeneratedFaceCrops = false;
|
||||
const useGeneratedFaceCrops = true;
|
||||
|
||||
class FaceWidget extends StatefulWidget {
|
||||
final EnteFile file;
|
||||
|
|
|
@ -90,7 +90,7 @@ class PersonFaceWidget extends StatelessWidget {
|
|||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
if (!useGeneratedFaceCrops) {
|
||||
if (useGeneratedFaceCrops) {
|
||||
return FutureBuilder<Uint8List?>(
|
||||
future: getFaceCrop(),
|
||||
builder: (context, snapshot) {
|
||||
|
|
|
@ -5,8 +5,8 @@ import "package:photos/core/cache/lru_map.dart";
|
|||
import "package:photos/face/model/box.dart";
|
||||
import "package:photos/models/file/file.dart";
|
||||
import "package:photos/models/file/file_type.dart";
|
||||
import "package:photos/utils/face/face_util.dart";
|
||||
import "package:photos/utils/file_util.dart";
|
||||
import "package:photos/utils/image_ml_isolate.dart";
|
||||
import "package:photos/utils/thumbnail_util.dart";
|
||||
import "package:pool/pool.dart";
|
||||
|
||||
|
@ -37,7 +37,8 @@ Future<Map<String, Uint8List>?> getFaceCrops(
|
|||
faceBoxes.add(e.value);
|
||||
}
|
||||
final List<Uint8List> faceCrop =
|
||||
await ImageMlIsolate.instance.generateFaceThumbnailsForImage(
|
||||
// await ImageMlIsolate.instance.generateFaceThumbnailsForImage(
|
||||
await generateJpgFaceThumbnails(
|
||||
imagePath,
|
||||
faceBoxes,
|
||||
);
|
||||
|
|
161
mobile/lib/utils/face/face_util.dart
Normal file
161
mobile/lib/utils/face/face_util.dart
Normal file
|
@ -0,0 +1,161 @@
|
|||
import "dart:math";
|
||||
import "dart:typed_data";
|
||||
|
||||
import "package:computer/computer.dart";
|
||||
import "package:flutter_image_compress/flutter_image_compress.dart";
|
||||
import "package:image/image.dart" as img;
|
||||
import "package:logging/logging.dart";
|
||||
import "package:photos/face/model/box.dart";
|
||||
|
||||
final _logger = Logger("FaceUtil");
|
||||
final _computer = Computer.shared();
|
||||
const _faceImageBufferFactor = 0.2;
|
||||
|
||||
///Convert img.Image to ui.Image and use RawImage to display.
|
||||
Future<List<img.Image>> generateImgFaceThumbnails(
|
||||
String imagePath,
|
||||
List<FaceBox> faceBoxes, {
|
||||
///Pass decodedImage decoded by [decodeToImgImage] to avoid decoding image
|
||||
///multiple times if all faces are from the same image (eg: File info).
|
||||
img.Image? decodedImage,
|
||||
}) async {
|
||||
final faceThumbnails = <img.Image>[];
|
||||
|
||||
final image = decodedImage ?? await decodeToImgImage(imagePath);
|
||||
|
||||
for (FaceBox faceBox in faceBoxes) {
|
||||
final croppedImage = cropFaceBoxFromImage(image, faceBox);
|
||||
faceThumbnails.add(croppedImage);
|
||||
}
|
||||
|
||||
return faceThumbnails;
|
||||
}
|
||||
|
||||
Future<List<Uint8List>> generateJpgFaceThumbnails(
|
||||
String imagePath,
|
||||
List<FaceBox> faceBoxes, {
|
||||
///Pass decodedImage decoded by [decodeToImgImage] to avoid decoding image
|
||||
///multiple times if all faces are from the same image (eg: File info).
|
||||
img.Image? decodedImage,
|
||||
}) async {
|
||||
final image = decodedImage ?? await decodeToImgImage(imagePath);
|
||||
final croppedImages = <img.Image>[];
|
||||
for (FaceBox faceBox in faceBoxes) {
|
||||
final croppedImage = cropFaceBoxFromImage(image, faceBox);
|
||||
croppedImages.add(croppedImage);
|
||||
}
|
||||
|
||||
return await _computer
|
||||
.compute(_encodeImagesToJpg, param: {"images": croppedImages});
|
||||
}
|
||||
|
||||
Future<img.Image> decodeToImgImage(String imagePath) async {
|
||||
img.Image? image =
|
||||
await _computer.compute(_decodeImageFile, param: {"filePath": imagePath});
|
||||
|
||||
if (image == null) {
|
||||
_logger.info(
|
||||
"Failed to decode image. Compressing to jpg and decoding",
|
||||
);
|
||||
final compressedJPGImage =
|
||||
await FlutterImageCompress.compressWithFile(imagePath);
|
||||
image = await _computer.compute(
|
||||
_decodeJpg,
|
||||
param: {"image": compressedJPGImage},
|
||||
);
|
||||
|
||||
if (image == null) {
|
||||
throw Exception("Failed to decode image");
|
||||
} else {
|
||||
return image;
|
||||
}
|
||||
} else {
|
||||
return image;
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns an Image from 'package:image/image.dart'
|
||||
img.Image cropFaceBoxFromImage(img.Image image, FaceBox faceBox) {
|
||||
final squareFaceBox = _getSquareFaceBoxImage(image, faceBox);
|
||||
final squareFaceBoxWithBuffer =
|
||||
_addBufferAroundFaceBox(squareFaceBox, _faceImageBufferFactor);
|
||||
return img.copyCrop(
|
||||
image,
|
||||
x: squareFaceBoxWithBuffer.xMin,
|
||||
y: squareFaceBoxWithBuffer.yMin,
|
||||
width: squareFaceBoxWithBuffer.width,
|
||||
height: squareFaceBoxWithBuffer.height,
|
||||
antialias: false,
|
||||
);
|
||||
}
|
||||
|
||||
/// Returns a square face box image from the original image with
|
||||
/// side length equal to the maximum of the width and height of the face box in
|
||||
/// the OG image.
|
||||
FaceBoxImage _getSquareFaceBoxImage(img.Image image, FaceBox faceBox) {
|
||||
final width = (image.width * faceBox.width).round();
|
||||
final height = (image.height * faceBox.height).round();
|
||||
final side = max(width, height);
|
||||
final xImage = (image.width * faceBox.xMin).round();
|
||||
final yImage = (image.height * faceBox.yMin).round();
|
||||
|
||||
if (height >= width) {
|
||||
final xImageAdj = (xImage - (height - width) / 2).round();
|
||||
return FaceBoxImage(
|
||||
xMin: xImageAdj,
|
||||
yMin: yImage,
|
||||
width: side,
|
||||
height: side,
|
||||
);
|
||||
} else {
|
||||
final yImageAdj = (yImage - (width - height) / 2).round();
|
||||
return FaceBoxImage(
|
||||
xMin: xImage,
|
||||
yMin: yImageAdj,
|
||||
width: side,
|
||||
height: side,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
///To add some buffer around the face box so that the face isn't cropped
|
||||
///too close to the face.
|
||||
FaceBoxImage _addBufferAroundFaceBox(
|
||||
FaceBoxImage faceBoxImage,
|
||||
double bufferFactor,
|
||||
) {
|
||||
final heightBuffer = faceBoxImage.height * bufferFactor;
|
||||
final widthBuffer = faceBoxImage.width * bufferFactor;
|
||||
final xMinWithBuffer = faceBoxImage.xMin - widthBuffer;
|
||||
final yMinWithBuffer = faceBoxImage.yMin - heightBuffer;
|
||||
final widthWithBuffer = faceBoxImage.width + 2 * widthBuffer;
|
||||
final heightWithBuffer = faceBoxImage.height + 2 * heightBuffer;
|
||||
//Do not add buffer if the top left edge of the image is out of bounds
|
||||
//after adding the buffer.
|
||||
if (xMinWithBuffer < 0 || yMinWithBuffer < 0) {
|
||||
return faceBoxImage;
|
||||
}
|
||||
//Another similar case that can be handled is when the bottom right edge
|
||||
//of the image is out of bounds after adding the buffer. But the
|
||||
//the visual difference is not as significant as when the top left edge
|
||||
//is out of bounds, so we are not handling that case.
|
||||
return FaceBoxImage(
|
||||
xMin: xMinWithBuffer.round(),
|
||||
yMin: yMinWithBuffer.round(),
|
||||
width: widthWithBuffer.round(),
|
||||
height: heightWithBuffer.round(),
|
||||
);
|
||||
}
|
||||
|
||||
List<Uint8List> _encodeImagesToJpg(Map args) {
|
||||
final images = args["images"] as List<img.Image>;
|
||||
return images.map((img.Image image) => img.encodeJpg(image)).toList();
|
||||
}
|
||||
|
||||
Future<img.Image?> _decodeImageFile(Map args) async {
|
||||
return await img.decodeImageFile(args["filePath"]);
|
||||
}
|
||||
|
||||
img.Image? _decodeJpg(Map args) {
|
||||
return img.decodeJpg(args["image"])!;
|
||||
}
|
|
@ -1,6 +1,8 @@
|
|||
import 'dart:async';
|
||||
import 'dart:ui' as ui;
|
||||
|
||||
import 'package:flutter/widgets.dart';
|
||||
import 'package:image/image.dart' as img;
|
||||
|
||||
Future<ImageInfo> getImageInfo(ImageProvider imageProvider) {
|
||||
final completer = Completer<ImageInfo>();
|
||||
|
@ -14,3 +16,35 @@ Future<ImageInfo> getImageInfo(ImageProvider imageProvider) {
|
|||
completer.future.whenComplete(() => imageStream.removeListener(listener));
|
||||
return completer.future;
|
||||
}
|
||||
|
||||
Future<ui.Image> convertImageToFlutterUi(img.Image image) async {
|
||||
if (image.format != img.Format.uint8 || image.numChannels != 4) {
|
||||
final cmd = img.Command()
|
||||
..image(image)
|
||||
..convert(format: img.Format.uint8, numChannels: 4);
|
||||
final rgba8 = await cmd.getImageThread();
|
||||
if (rgba8 != null) {
|
||||
image = rgba8;
|
||||
}
|
||||
}
|
||||
|
||||
final ui.ImmutableBuffer buffer =
|
||||
await ui.ImmutableBuffer.fromUint8List(image.toUint8List());
|
||||
|
||||
final ui.ImageDescriptor id = ui.ImageDescriptor.raw(
|
||||
buffer,
|
||||
height: image.height,
|
||||
width: image.width,
|
||||
pixelFormat: ui.PixelFormat.rgba8888,
|
||||
);
|
||||
|
||||
final ui.Codec codec = await id.instantiateCodec(
|
||||
targetHeight: image.height,
|
||||
targetWidth: image.width,
|
||||
);
|
||||
|
||||
final ui.FrameInfo fi = await codec.getNextFrame();
|
||||
final ui.Image uiImage = fi.image;
|
||||
|
||||
return uiImage;
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue