[mob][photos] Generate face crops faster (#1542)

## Description

Have written two new methods, `generateImgFaceThumbnails()` and
`generateJpgFaceThumbnails()`.
Using `generateJpgFaceThumbnails()` now since it returns
`Future<List<Uint8List>>` and is easier to integrate within the code
base because the return type remains the same with the older
`generateFaceThumbnailsForImage()`

There is performance improvement with `generateImgFaceThumbnails()`, but
it's not very significant and it requires changes in codebase to work
with it's return type `Future<List<Image>>` (`Image` from the `Image`
package). Can consider using it if it feels necessary in future.

If multiple faces are being generated from the same image, the image can
be decoded once and passed to `generateImgFaceThumbnails()` or
`generateJpgFaceThumbnails()` to avoid repeated decoding of the same
image.

`generateImgFaceThumbnails()` and `generateJpgFaceThumbnails()` uses the
isolates available from the pool of 4 spawned by `Computer` and
processes multiple faces in parallel unlike
`generateImgFaceThumbnails()`, which processes only one at a time.

mobile/lib/face/model/box.dart

@@ -41,3 +41,23 @@ class FaceBox {
         'height': height,
       };
 }
+
+/// Bounding box of a face.
+///
+/// [xMin] and [yMin] are the coordinates of the top left corner of the box, and
+/// [width] and [height] are the width and height of the box.
+///
+/// One unit is equal to one pixel in the original image.
+class FaceBoxImage {
+  final int xMin;
+  final int yMin;
+  final int width;
+  final int height;
+
+  FaceBoxImage({
+    required this.xMin,
+    required this.yMin,
+    required this.width,
+    required this.height,
+  });
+}

mobile/lib/ui/viewer/file_details/face_widget.dart

@@ -21,7 +21,7 @@ import "package:photos/utils/face/face_box_crop.dart";
 import "package:photos/utils/thumbnail_util.dart";
 // import "package:photos/utils/toast_util.dart";
 
-const useGeneratedFaceCrops = false;
+const useGeneratedFaceCrops = true;
 
 class FaceWidget extends StatefulWidget {
   final EnteFile file;

mobile/lib/ui/viewer/search/result/person_face_widget.dart

@@ -90,7 +90,7 @@ class PersonFaceWidget extends StatelessWidget {
 
   @override
   Widget build(BuildContext context) {
-    if (!useGeneratedFaceCrops) {
+    if (useGeneratedFaceCrops) {
       return FutureBuilder<Uint8List?>(
         future: getFaceCrop(),
         builder: (context, snapshot) {

mobile/lib/utils/face/face_box_crop.dart

@@ -5,8 +5,8 @@ import "package:photos/core/cache/lru_map.dart";
 import "package:photos/face/model/box.dart";
 import "package:photos/models/file/file.dart";
 import "package:photos/models/file/file_type.dart";
+import "package:photos/utils/face/face_util.dart";
 import "package:photos/utils/file_util.dart";
-import "package:photos/utils/image_ml_isolate.dart";
 import "package:photos/utils/thumbnail_util.dart";
 import "package:pool/pool.dart";
 
@@ -37,7 +37,8 @@ Future<Map<String, Uint8List>?> getFaceCrops(
     faceBoxes.add(e.value);
   }
   final List<Uint8List> faceCrop =
-      await ImageMlIsolate.instance.generateFaceThumbnailsForImage(
+      // await ImageMlIsolate.instance.generateFaceThumbnailsForImage(
+      await generateJpgFaceThumbnails(
     imagePath,
     faceBoxes,
   );

mobile/lib/utils/face/face_util.dart

@@ -0,0 +1,161 @@
+import "dart:math";
+import "dart:typed_data";
+
+import "package:computer/computer.dart";
+import "package:flutter_image_compress/flutter_image_compress.dart";
+import "package:image/image.dart" as img;
+import "package:logging/logging.dart";
+import "package:photos/face/model/box.dart";
+
+final _logger = Logger("FaceUtil");
+final _computer = Computer.shared();
+const _faceImageBufferFactor = 0.2;
+
+///Convert img.Image to ui.Image and use RawImage to display.
+Future<List<img.Image>> generateImgFaceThumbnails(
+  String imagePath,
+  List<FaceBox> faceBoxes, {
+  ///Pass decodedImage decoded by [decodeToImgImage] to avoid decoding image
+  ///multiple times if all faces are from the same image (eg: File info).
+  img.Image? decodedImage,
+}) async {
+  final faceThumbnails = <img.Image>[];
+
+  final image = decodedImage ?? await decodeToImgImage(imagePath);
+
+  for (FaceBox faceBox in faceBoxes) {
+    final croppedImage = cropFaceBoxFromImage(image, faceBox);
+    faceThumbnails.add(croppedImage);
+  }
+
+  return faceThumbnails;
+}
+
+Future<List<Uint8List>> generateJpgFaceThumbnails(
+  String imagePath,
+  List<FaceBox> faceBoxes, {
+  ///Pass decodedImage decoded by [decodeToImgImage] to avoid decoding image
+  ///multiple times if all faces are from the same image (eg: File info).
+  img.Image? decodedImage,
+}) async {
+  final image = decodedImage ?? await decodeToImgImage(imagePath);
+  final croppedImages = <img.Image>[];
+  for (FaceBox faceBox in faceBoxes) {
+    final croppedImage = cropFaceBoxFromImage(image, faceBox);
+    croppedImages.add(croppedImage);
+  }
+
+  return await _computer
+      .compute(_encodeImagesToJpg, param: {"images": croppedImages});
+}
+
+Future<img.Image> decodeToImgImage(String imagePath) async {
+  img.Image? image =
+      await _computer.compute(_decodeImageFile, param: {"filePath": imagePath});
+
+  if (image == null) {
+    _logger.info(
+      "Failed to decode image. Compressing to jpg and decoding",
+    );
+    final compressedJPGImage =
+        await FlutterImageCompress.compressWithFile(imagePath);
+    image = await _computer.compute(
+      _decodeJpg,
+      param: {"image": compressedJPGImage},
+    );
+
+    if (image == null) {
+      throw Exception("Failed to decode image");
+    } else {
+      return image;
+    }
+  } else {
+    return image;
+  }
+}
+
+/// Returns an Image from 'package:image/image.dart'
+img.Image cropFaceBoxFromImage(img.Image image, FaceBox faceBox) {
+  final squareFaceBox = _getSquareFaceBoxImage(image, faceBox);
+  final squareFaceBoxWithBuffer =
+      _addBufferAroundFaceBox(squareFaceBox, _faceImageBufferFactor);
+  return img.copyCrop(
+    image,
+    x: squareFaceBoxWithBuffer.xMin,
+    y: squareFaceBoxWithBuffer.yMin,
+    width: squareFaceBoxWithBuffer.width,
+    height: squareFaceBoxWithBuffer.height,
+    antialias: false,
+  );
+}
+
+/// Returns a square face box image from the original image with
+/// side length equal to the maximum of the width and height of the face box in
+/// the OG image.
+FaceBoxImage _getSquareFaceBoxImage(img.Image image, FaceBox faceBox) {
+  final width = (image.width * faceBox.width).round();
+  final height = (image.height * faceBox.height).round();
+  final side = max(width, height);
+  final xImage = (image.width * faceBox.xMin).round();
+  final yImage = (image.height * faceBox.yMin).round();
+
+  if (height >= width) {
+    final xImageAdj = (xImage - (height - width) / 2).round();
+    return FaceBoxImage(
+      xMin: xImageAdj,
+      yMin: yImage,
+      width: side,
+      height: side,
+    );
+  } else {
+    final yImageAdj = (yImage - (width - height) / 2).round();
+    return FaceBoxImage(
+      xMin: xImage,
+      yMin: yImageAdj,
+      width: side,
+      height: side,
+    );
+  }
+}
+
+///To add some buffer around the face box so that the face isn't cropped
+///too close to the face.
+FaceBoxImage _addBufferAroundFaceBox(
+  FaceBoxImage faceBoxImage,
+  double bufferFactor,
+) {
+  final heightBuffer = faceBoxImage.height * bufferFactor;
+  final widthBuffer = faceBoxImage.width * bufferFactor;
+  final xMinWithBuffer = faceBoxImage.xMin - widthBuffer;
+  final yMinWithBuffer = faceBoxImage.yMin - heightBuffer;
+  final widthWithBuffer = faceBoxImage.width + 2 * widthBuffer;
+  final heightWithBuffer = faceBoxImage.height + 2 * heightBuffer;
+  //Do not add buffer if the top left edge of the image is out of bounds
+  //after adding the buffer.
+  if (xMinWithBuffer < 0 || yMinWithBuffer < 0) {
+    return faceBoxImage;
+  }
+  //Another similar case that can be handled is when the bottom right edge
+  //of the image is out of bounds after adding the buffer. But the
+  //the visual difference is not as significant as when the top left edge
+  //is out of bounds, so we are not handling that case.
+  return FaceBoxImage(
+    xMin: xMinWithBuffer.round(),
+    yMin: yMinWithBuffer.round(),
+    width: widthWithBuffer.round(),
+    height: heightWithBuffer.round(),
+  );
+}
+
+List<Uint8List> _encodeImagesToJpg(Map args) {
+  final images = args["images"] as List<img.Image>;
+  return images.map((img.Image image) => img.encodeJpg(image)).toList();
+}
+
+Future<img.Image?> _decodeImageFile(Map args) async {
+  return await img.decodeImageFile(args["filePath"]);
+}
+
+img.Image? _decodeJpg(Map args) {
+  return img.decodeJpg(args["image"])!;
+}

mobile/lib/utils/image_util.dart

@@ -1,6 +1,8 @@
 import 'dart:async';
+import 'dart:ui' as ui;
 
 import 'package:flutter/widgets.dart';
+import 'package:image/image.dart' as img;
 
 Future<ImageInfo> getImageInfo(ImageProvider imageProvider) {
   final completer = Completer<ImageInfo>();
@@ -14,3 +16,35 @@ Future<ImageInfo> getImageInfo(ImageProvider imageProvider) {
   completer.future.whenComplete(() => imageStream.removeListener(listener));
   return completer.future;
 }
+
+Future<ui.Image> convertImageToFlutterUi(img.Image image) async {
+  if (image.format != img.Format.uint8 || image.numChannels != 4) {
+    final cmd = img.Command()
+      ..image(image)
+      ..convert(format: img.Format.uint8, numChannels: 4);
+    final rgba8 = await cmd.getImageThread();
+    if (rgba8 != null) {
+      image = rgba8;
+    }
+  }
+
+  final ui.ImmutableBuffer buffer =
+      await ui.ImmutableBuffer.fromUint8List(image.toUint8List());
+
+  final ui.ImageDescriptor id = ui.ImageDescriptor.raw(
+    buffer,
+    height: image.height,
+    width: image.width,
+    pixelFormat: ui.PixelFormat.rgba8888,
+  );
+
+  final ui.Codec codec = await id.instantiateCodec(
+    targetHeight: image.height,
+    targetWidth: image.width,
+  );
+
+  final ui.FrameInfo fi = await codec.getNextFrame();
+  final ui.Image uiImage = fi.image;
+
+  return uiImage;
+}