From b022ef6d1e6420205ae6a6baa28feffe70679d17 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 25 Apr 2024 09:35:55 +0530 Subject: [PATCH 01/12] [mob] Crop image instead of using scale and translate transforms on OG image in CroppedFaceImageView widget --- .../people/cropped_face_image_view.dart | 144 +++++++++--------- mobile/lib/utils/image_util.dart | 35 +++++ 2 files changed, 103 insertions(+), 76 deletions(-) diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart index 4ef369259..cb7baff42 100644 --- a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -1,11 +1,14 @@ -import 'dart:developer' show log; import "dart:io" show File; +import 'dart:ui' as ui; import 'package:flutter/material.dart'; +import "package:image/image.dart" as img; +import "package:logging/logging.dart"; import "package:photos/face/model/face.dart"; import "package:photos/models/file/file.dart"; import "package:photos/ui/viewer/file/thumbnail_widget.dart"; import "package:photos/utils/file_util.dart"; +import "package:photos/utils/image_util.dart"; class CroppedFaceInfo { final Image image; @@ -21,7 +24,7 @@ class CroppedFaceInfo { }); } -class CroppedFaceImageView extends StatelessWidget { +class CroppedFaceImageView extends StatefulWidget { final EnteFile enteFile; final Face face; @@ -32,85 +35,74 @@ class CroppedFaceImageView extends StatelessWidget { }) : super(key: key); @override - Widget build(BuildContext context) { - return FutureBuilder( - future: getImage(), - builder: (context, snapshot) { - if (snapshot.hasData) { - return LayoutBuilder( - builder: ((context, constraints) { - final double imageAspectRatio = enteFile.width / enteFile.height; - final Image image = snapshot.data!; + CroppedFaceImageViewState createState() => CroppedFaceImageViewState(); +} - final double viewWidth = constraints.maxWidth; - final double viewHeight = constraints.maxHeight; +class CroppedFaceImageViewState extends State { + ui.Image? _image; + final _logger = Logger("CroppedFaceImageView"); - final faceBox = face.detection.box; - - final double relativeFaceCenterX = - faceBox.xMin + faceBox.width / 2; - final double relativeFaceCenterY = - faceBox.yMin + faceBox.height / 2; - - const double desiredFaceHeightRelativeToWidget = 8 / 10; - final double scale = - (1 / faceBox.height) * desiredFaceHeightRelativeToWidget; - - final double widgetCenterX = viewWidth / 2; - final double widgetCenterY = viewHeight / 2; - - final double widgetAspectRatio = viewWidth / viewHeight; - final double imageToWidgetRatio = - imageAspectRatio / widgetAspectRatio; - - double offsetX = - (widgetCenterX - relativeFaceCenterX * viewWidth) * scale; - double offsetY = - (widgetCenterY - relativeFaceCenterY * viewHeight) * scale; - - if (imageAspectRatio < widgetAspectRatio) { - // Landscape Image: Adjust offsetX more conservatively - offsetX = offsetX * imageToWidgetRatio; - } else { - // Portrait Image: Adjust offsetY more conservatively - offsetY = offsetY / imageToWidgetRatio; - } - return ClipRRect( - borderRadius: const BorderRadius.all(Radius.elliptical(16, 12)), - child: Transform.translate( - offset: Offset( - offsetX, - offsetY, - ), - child: Transform.scale( - scale: scale, - child: image, - ), - ), - ); - }), - ); - } else { - if (snapshot.hasError) { - log('Error getting cover face for person: ${snapshot.error}'); - } - return ThumbnailWidget( - enteFile, - ); - } - }, - ); + @override + void initState() { + super.initState(); + _loadImage(); } - Future getImage() async { - final File? ioFile = await getFile(enteFile); - if (ioFile == null) { + @override + void dispose() { + super.dispose(); + _image?.dispose(); + } + + Future _loadImage() async { + final image = await getImage(); + if (mounted) { + setState(() { + _image = image; + }); + } + } + + @override + Widget build(BuildContext context) { + return _image != null + ? LayoutBuilder( + builder: (context, constraints) { + return RawImage( + image: _image!, + ); + }, + ) + : ThumbnailWidget(widget.enteFile); + } + + Future getImage() async { + try { + final faceBox = widget.face.detection.box; + final File? ioFile = await getFile(widget.enteFile); + if (ioFile == null) { + return null; + } + + final image = await img.decodeImageFile(ioFile.path); + + if (image == null) { + throw Exception("Failed decoding image file ${widget.enteFile.title}}"); + } + + final croppedImage = img.copyCrop( + image, + x: (image.width * faceBox.xMin).round(), + y: (image.height * faceBox.yMin).round(), + width: (image.width * faceBox.width).round(), + height: (image.height * faceBox.height).round(), + antialias: false, + ); + + return convertImageToFlutterUi(croppedImage); + } catch (e, s) { + _logger.severe("Error getting image", e, s); return null; } - - final imageData = await ioFile.readAsBytes(); - final image = Image.memory(imageData, fit: BoxFit.contain); - - return image; } } diff --git a/mobile/lib/utils/image_util.dart b/mobile/lib/utils/image_util.dart index a5bcb03a7..7eb1e39fc 100644 --- a/mobile/lib/utils/image_util.dart +++ b/mobile/lib/utils/image_util.dart @@ -1,6 +1,8 @@ import 'dart:async'; +import 'dart:ui' as ui; import 'package:flutter/widgets.dart'; +import 'package:image/image.dart' as img; Future getImageInfo(ImageProvider imageProvider) { final completer = Completer(); @@ -14,3 +16,36 @@ Future getImageInfo(ImageProvider imageProvider) { completer.future.whenComplete(() => imageStream.removeListener(listener)); return completer.future; } + +///https://github.com/brendan-duncan/image/blob/main/doc/flutter.md +Future convertImageToFlutterUi(img.Image image) async { + if (image.format != img.Format.uint8 || image.numChannels != 4) { + final cmd = img.Command() + ..image(image) + ..convert(format: img.Format.uint8, numChannels: 4); + final rgba8 = await cmd.getImageThread(); + if (rgba8 != null) { + image = rgba8; + } + } + + final ui.ImmutableBuffer buffer = + await ui.ImmutableBuffer.fromUint8List(image.toUint8List()); + + final ui.ImageDescriptor id = ui.ImageDescriptor.raw( + buffer, + height: image.height, + width: image.width, + pixelFormat: ui.PixelFormat.rgba8888, + ); + + final ui.Codec codec = await id.instantiateCodec( + targetHeight: image.height, + targetWidth: image.width, + ); + + final ui.FrameInfo fi = await codec.getNextFrame(); + final ui.Image uiImage = fi.image; + + return uiImage; +} From 2f7e0cd1ef6e5676e1d294b850835f1487e6c71c Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 25 Apr 2024 12:29:29 +0530 Subject: [PATCH 02/12] [mob] perf: Decode images from which face is to be cropped, in an isolate to avoid jank --- .../people/cropped_face_image_view.dart | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart index cb7baff42..0da4129eb 100644 --- a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -1,7 +1,10 @@ import "dart:io" show File; import 'dart:ui' as ui; +import "package:computer/computer.dart"; import 'package:flutter/material.dart'; +import "package:flutter/widgets.dart"; +import "package:flutter_image_compress/flutter_image_compress.dart"; import "package:image/image.dart" as img; import "package:logging/logging.dart"; import "package:photos/face/model/face.dart"; @@ -40,6 +43,7 @@ class CroppedFaceImageView extends StatefulWidget { class CroppedFaceImageViewState extends State { ui.Image? _image; + final _computer = Computer.shared(); final _logger = Logger("CroppedFaceImageView"); @override @@ -79,17 +83,32 @@ class CroppedFaceImageViewState extends State { Future getImage() async { try { final faceBox = widget.face.detection.box; + final File? ioFile = await getFile(widget.enteFile); if (ioFile == null) { return null; } - final image = await img.decodeImageFile(ioFile.path); + img.Image? image = await _computer + .compute(decodeImage, param: {"filePath": ioFile.path}); if (image == null) { - throw Exception("Failed decoding image file ${widget.enteFile.title}}"); + _logger.info( + "Failed to decode image ${widget.enteFile.title}. Compressing to jpg and decoding", + ); + final compressedJPGImage = + await FlutterImageCompress.compressWithFile(ioFile.path); + image = await _computer.compute( + decodeJPGImage, + param: {"image": compressedJPGImage}, + ); + + if (image == null) { + throw Exception("Failed to decode image"); + } } + final stopwatch = Stopwatch()..start(); final croppedImage = img.copyCrop( image, x: (image.width * faceBox.xMin).round(), @@ -98,7 +117,10 @@ class CroppedFaceImageViewState extends State { height: (image.height * faceBox.height).round(), antialias: false, ); - + _logger.info( + "Image crop took ${stopwatch.elapsedMilliseconds}ms ----------------", + ); + stopwatch.stop(); return convertImageToFlutterUi(croppedImage); } catch (e, s) { _logger.severe("Error getting image", e, s); @@ -106,3 +128,11 @@ class CroppedFaceImageViewState extends State { } } } + +Future decodeImage(Map args) async { + return await img.decodeImageFile(args["filePath"]); +} + +img.Image? decodeJPGImage(Map args) { + return img.decodeJpg(args["image"])!; +} From f173bc4038d1325dbcc3290098a1f53187f228de Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 25 Apr 2024 16:56:43 +0530 Subject: [PATCH 03/12] [mob] Wrote util methods to generate face thumbnails from an image path Need to decide on which util method to use of the two after performance testing --- mobile/lib/utils/face/face_util.dart | 101 +++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 mobile/lib/utils/face/face_util.dart diff --git a/mobile/lib/utils/face/face_util.dart b/mobile/lib/utils/face/face_util.dart new file mode 100644 index 000000000..99ec04512 --- /dev/null +++ b/mobile/lib/utils/face/face_util.dart @@ -0,0 +1,101 @@ +import "dart:typed_data"; + +import "package:computer/computer.dart"; +import "package:flutter_image_compress/flutter_image_compress.dart"; +import "package:image/image.dart" as img; +import "package:logging/logging.dart"; +import "package:photos/face/model/box.dart"; + +final _logger = Logger("FaceUtil"); +final _computer = Computer.shared(); + +///Convert img.Image to ui.Image and use RawImage to display. +Future> generateImgFaceThumbnails( + String imagePath, + List faceBoxes, +) async { + final faceThumbnails = []; + + img.Image? image = + await _computer.compute(_decodeImageFile, param: {"filePath": imagePath}); + + if (image == null) { + _logger.info( + "Failed to decode image. Compressing to jpg and decoding", + ); + final compressedJPGImage = + await FlutterImageCompress.compressWithFile(imagePath); + image = await _computer.compute( + _decodeJpg, + param: {"image": compressedJPGImage}, + ); + + if (image == null) { + throw Exception("Failed to decode image"); + } + } + + for (FaceBox faceBox in faceBoxes) { + final croppedImage = cropFaceBoxFromImage(image, faceBox); + faceThumbnails.add(croppedImage); + } + + return faceThumbnails; +} + +Future> generateJpgFaceThumbnails( + String imagePath, + List faceBoxes, +) async { + img.Image? image = + await _computer.compute(_decodeImageFile, param: {"filePath": imagePath}); + + if (image == null) { + _logger.info( + "Failed to decode image. Compressing to jpg and decoding", + ); + final compressedJPGImage = + await FlutterImageCompress.compressWithFile(imagePath); + image = await _computer.compute( + _decodeJpg, + param: {"image": compressedJPGImage}, + ); + + if (image == null) { + throw Exception("Failed to decode image"); + } + } + final croppedImages = []; + for (FaceBox faceBox in faceBoxes) { + final croppedImage = cropFaceBoxFromImage(image, faceBox); + croppedImages.add(croppedImage); + } + + return await _computer + .compute(_encodeImagesToJpg, param: {"images": croppedImages}); +} + +/// Returns an Image from 'package:image/image.dart' +img.Image cropFaceBoxFromImage(img.Image image, FaceBox faceBox) { + return img.copyCrop( + image, + x: (image.width * faceBox.xMin).round(), + y: (image.height * faceBox.yMin).round(), + width: (image.width * faceBox.width).round(), + height: (image.height * faceBox.height).round(), + antialias: false, + ); +} + +List _encodeImagesToJpg(Map args) { + final images = args["images"] as List; + return images.map((img.Image image) => img.encodeJpg(image)).toList(); +} + +Future _decodeImageFile(Map args) async { + return await img.decodeImageFile(args["filePath"]); +} + +img.Image? _decodeJpg(Map args) { + return img.decodeJpg(args["image"])!; +} From 7617817798d0f6888a9c0f9a620ba29069e4e7dd Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 25 Apr 2024 16:58:58 +0530 Subject: [PATCH 04/12] [mob] Two varients of CroppedFaceImageView for testing out which is more performant --- .../ui/viewer/file_details/face_widget.dart | 2 +- .../people/cropped_face_image_view.dart | 118 +++++++++++------- .../search/result/person_face_widget.dart | 2 +- 3 files changed, 76 insertions(+), 46 deletions(-) diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index da592a150..9d1fa8dec 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -267,7 +267,7 @@ class _FaceWidgetState extends State { SizedBox( width: 60, height: 60, - child: CroppedFaceImageView( + child: CroppedFaceImgImageView( enteFile: widget.file, face: widget.face, ), diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart index 0da4129eb..c7e840e71 100644 --- a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -1,15 +1,14 @@ import "dart:io" show File; +import "dart:typed_data"; import 'dart:ui' as ui; -import "package:computer/computer.dart"; import 'package:flutter/material.dart'; import "package:flutter/widgets.dart"; -import "package:flutter_image_compress/flutter_image_compress.dart"; -import "package:image/image.dart" as img; import "package:logging/logging.dart"; import "package:photos/face/model/face.dart"; import "package:photos/models/file/file.dart"; import "package:photos/ui/viewer/file/thumbnail_widget.dart"; +import "package:photos/utils/face/face_util.dart"; import "package:photos/utils/file_util.dart"; import "package:photos/utils/image_util.dart"; @@ -27,23 +26,22 @@ class CroppedFaceInfo { }); } -class CroppedFaceImageView extends StatefulWidget { +class CroppedFaceImgImageView extends StatefulWidget { final EnteFile enteFile; final Face face; - const CroppedFaceImageView({ + const CroppedFaceImgImageView({ Key? key, required this.enteFile, required this.face, }) : super(key: key); @override - CroppedFaceImageViewState createState() => CroppedFaceImageViewState(); + CroppedFaceImgImageViewState createState() => CroppedFaceImgImageViewState(); } -class CroppedFaceImageViewState extends State { +class CroppedFaceImgImageViewState extends State { ui.Image? _image; - final _computer = Computer.shared(); final _logger = Logger("CroppedFaceImageView"); @override @@ -89,39 +87,9 @@ class CroppedFaceImageViewState extends State { return null; } - img.Image? image = await _computer - .compute(decodeImage, param: {"filePath": ioFile.path}); + final image = await generateImgFaceThumbnails(ioFile.path, [faceBox]); - if (image == null) { - _logger.info( - "Failed to decode image ${widget.enteFile.title}. Compressing to jpg and decoding", - ); - final compressedJPGImage = - await FlutterImageCompress.compressWithFile(ioFile.path); - image = await _computer.compute( - decodeJPGImage, - param: {"image": compressedJPGImage}, - ); - - if (image == null) { - throw Exception("Failed to decode image"); - } - } - - final stopwatch = Stopwatch()..start(); - final croppedImage = img.copyCrop( - image, - x: (image.width * faceBox.xMin).round(), - y: (image.height * faceBox.yMin).round(), - width: (image.width * faceBox.width).round(), - height: (image.height * faceBox.height).round(), - antialias: false, - ); - _logger.info( - "Image crop took ${stopwatch.elapsedMilliseconds}ms ----------------", - ); - stopwatch.stop(); - return convertImageToFlutterUi(croppedImage); + return convertImageToFlutterUi(image.first); } catch (e, s) { _logger.severe("Error getting image", e, s); return null; @@ -129,10 +97,72 @@ class CroppedFaceImageViewState extends State { } } -Future decodeImage(Map args) async { - return await img.decodeImageFile(args["filePath"]); +class CroppedFaceJpgImageView extends StatefulWidget { + final EnteFile enteFile; + final Face face; + + const CroppedFaceJpgImageView({ + Key? key, + required this.enteFile, + required this.face, + }) : super(key: key); + + @override + CroppedFaceJpgImageViewState createState() => CroppedFaceJpgImageViewState(); } -img.Image? decodeJPGImage(Map args) { - return img.decodeJpg(args["image"])!; +class CroppedFaceJpgImageViewState extends State { + Uint8List? _image; + final _logger = Logger("CroppedFaceImageView"); + + @override + void initState() { + super.initState(); + _loadImage(); + } + + @override + void dispose() { + super.dispose(); + } + + Future _loadImage() async { + final image = await getImage(); + if (mounted) { + setState(() { + _image = image; + }); + } + } + + @override + Widget build(BuildContext context) { + return _image != null + ? LayoutBuilder( + builder: (context, constraints) { + return Image.memory( + _image!, + ); + }, + ) + : ThumbnailWidget(widget.enteFile); + } + + Future getImage() async { + try { + final faceBox = widget.face.detection.box; + + final File? ioFile = await getFile(widget.enteFile); + if (ioFile == null) { + return null; + } + + final image = await generateJpgFaceThumbnails(ioFile.path, [faceBox]); + + return image.first; + } catch (e, s) { + _logger.severe("Error getting image", e, s); + return null; + } + } } diff --git a/mobile/lib/ui/viewer/search/result/person_face_widget.dart b/mobile/lib/ui/viewer/search/result/person_face_widget.dart index f2c96fc7a..0f872e9f3 100644 --- a/mobile/lib/ui/viewer/search/result/person_face_widget.dart +++ b/mobile/lib/ui/viewer/search/result/person_face_widget.dart @@ -69,7 +69,7 @@ class PersonFaceWidget extends StatelessWidget { return Stack( fit: StackFit.expand, children: [ - CroppedFaceImageView(enteFile: file, face: face), + CroppedFaceImgImageView(enteFile: file, face: face), ], ); } else { From 3eebfdd037ae491ff283b718bb8a449d660a4376 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 26 Apr 2024 12:54:29 +0530 Subject: [PATCH 05/12] Revert "[mob] Two varients of CroppedFaceImageView for testing out which is more performant" This reverts commit 7617817798d0f6888a9c0f9a620ba29069e4e7dd. --- .../ui/viewer/file_details/face_widget.dart | 2 +- .../people/cropped_face_image_view.dart | 120 +++++++----------- .../search/result/person_face_widget.dart | 2 +- 3 files changed, 47 insertions(+), 77 deletions(-) diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index fa455eba4..4caf1305f 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -288,7 +288,7 @@ class _FaceWidgetState extends State { child: SizedBox( width: 60, height: 60, - child: CroppedFaceImgImageView( + child: CroppedFaceImageView( enteFile: widget.file, face: widget.face, ), diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart index c7e840e71..0da4129eb 100644 --- a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -1,14 +1,15 @@ import "dart:io" show File; -import "dart:typed_data"; import 'dart:ui' as ui; +import "package:computer/computer.dart"; import 'package:flutter/material.dart'; import "package:flutter/widgets.dart"; +import "package:flutter_image_compress/flutter_image_compress.dart"; +import "package:image/image.dart" as img; import "package:logging/logging.dart"; import "package:photos/face/model/face.dart"; import "package:photos/models/file/file.dart"; import "package:photos/ui/viewer/file/thumbnail_widget.dart"; -import "package:photos/utils/face/face_util.dart"; import "package:photos/utils/file_util.dart"; import "package:photos/utils/image_util.dart"; @@ -26,22 +27,23 @@ class CroppedFaceInfo { }); } -class CroppedFaceImgImageView extends StatefulWidget { +class CroppedFaceImageView extends StatefulWidget { final EnteFile enteFile; final Face face; - const CroppedFaceImgImageView({ + const CroppedFaceImageView({ Key? key, required this.enteFile, required this.face, }) : super(key: key); @override - CroppedFaceImgImageViewState createState() => CroppedFaceImgImageViewState(); + CroppedFaceImageViewState createState() => CroppedFaceImageViewState(); } -class CroppedFaceImgImageViewState extends State { +class CroppedFaceImageViewState extends State { ui.Image? _image; + final _computer = Computer.shared(); final _logger = Logger("CroppedFaceImageView"); @override @@ -87,82 +89,50 @@ class CroppedFaceImgImageViewState extends State { return null; } - final image = await generateImgFaceThumbnails(ioFile.path, [faceBox]); + img.Image? image = await _computer + .compute(decodeImage, param: {"filePath": ioFile.path}); - return convertImageToFlutterUi(image.first); - } catch (e, s) { - _logger.severe("Error getting image", e, s); - return null; - } - } -} + if (image == null) { + _logger.info( + "Failed to decode image ${widget.enteFile.title}. Compressing to jpg and decoding", + ); + final compressedJPGImage = + await FlutterImageCompress.compressWithFile(ioFile.path); + image = await _computer.compute( + decodeJPGImage, + param: {"image": compressedJPGImage}, + ); -class CroppedFaceJpgImageView extends StatefulWidget { - final EnteFile enteFile; - final Face face; - - const CroppedFaceJpgImageView({ - Key? key, - required this.enteFile, - required this.face, - }) : super(key: key); - - @override - CroppedFaceJpgImageViewState createState() => CroppedFaceJpgImageViewState(); -} - -class CroppedFaceJpgImageViewState extends State { - Uint8List? _image; - final _logger = Logger("CroppedFaceImageView"); - - @override - void initState() { - super.initState(); - _loadImage(); - } - - @override - void dispose() { - super.dispose(); - } - - Future _loadImage() async { - final image = await getImage(); - if (mounted) { - setState(() { - _image = image; - }); - } - } - - @override - Widget build(BuildContext context) { - return _image != null - ? LayoutBuilder( - builder: (context, constraints) { - return Image.memory( - _image!, - ); - }, - ) - : ThumbnailWidget(widget.enteFile); - } - - Future getImage() async { - try { - final faceBox = widget.face.detection.box; - - final File? ioFile = await getFile(widget.enteFile); - if (ioFile == null) { - return null; + if (image == null) { + throw Exception("Failed to decode image"); + } } - final image = await generateJpgFaceThumbnails(ioFile.path, [faceBox]); - - return image.first; + final stopwatch = Stopwatch()..start(); + final croppedImage = img.copyCrop( + image, + x: (image.width * faceBox.xMin).round(), + y: (image.height * faceBox.yMin).round(), + width: (image.width * faceBox.width).round(), + height: (image.height * faceBox.height).round(), + antialias: false, + ); + _logger.info( + "Image crop took ${stopwatch.elapsedMilliseconds}ms ----------------", + ); + stopwatch.stop(); + return convertImageToFlutterUi(croppedImage); } catch (e, s) { _logger.severe("Error getting image", e, s); return null; } } } + +Future decodeImage(Map args) async { + return await img.decodeImageFile(args["filePath"]); +} + +img.Image? decodeJPGImage(Map args) { + return img.decodeJpg(args["image"])!; +} diff --git a/mobile/lib/ui/viewer/search/result/person_face_widget.dart b/mobile/lib/ui/viewer/search/result/person_face_widget.dart index f30bb6f9f..bdbfaab94 100644 --- a/mobile/lib/ui/viewer/search/result/person_face_widget.dart +++ b/mobile/lib/ui/viewer/search/result/person_face_widget.dart @@ -124,7 +124,7 @@ class PersonFaceWidget extends StatelessWidget { return Stack( fit: StackFit.expand, children: [ - CroppedFaceImgImageView(enteFile: file, face: face), + CroppedFaceImageView(enteFile: file, face: face), ], ); } else { From b256bb2757f07464fc4226fecda8faf489d6191b Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 26 Apr 2024 12:55:29 +0530 Subject: [PATCH 06/12] Revert "[mob] perf: Decode images from which face is to be cropped, in an isolate to avoid jank" This reverts commit 2f7e0cd1ef6e5676e1d294b850835f1487e6c71c. --- .../people/cropped_face_image_view.dart | 36 ++----------------- 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart index 0da4129eb..cb7baff42 100644 --- a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -1,10 +1,7 @@ import "dart:io" show File; import 'dart:ui' as ui; -import "package:computer/computer.dart"; import 'package:flutter/material.dart'; -import "package:flutter/widgets.dart"; -import "package:flutter_image_compress/flutter_image_compress.dart"; import "package:image/image.dart" as img; import "package:logging/logging.dart"; import "package:photos/face/model/face.dart"; @@ -43,7 +40,6 @@ class CroppedFaceImageView extends StatefulWidget { class CroppedFaceImageViewState extends State { ui.Image? _image; - final _computer = Computer.shared(); final _logger = Logger("CroppedFaceImageView"); @override @@ -83,32 +79,17 @@ class CroppedFaceImageViewState extends State { Future getImage() async { try { final faceBox = widget.face.detection.box; - final File? ioFile = await getFile(widget.enteFile); if (ioFile == null) { return null; } - img.Image? image = await _computer - .compute(decodeImage, param: {"filePath": ioFile.path}); + final image = await img.decodeImageFile(ioFile.path); if (image == null) { - _logger.info( - "Failed to decode image ${widget.enteFile.title}. Compressing to jpg and decoding", - ); - final compressedJPGImage = - await FlutterImageCompress.compressWithFile(ioFile.path); - image = await _computer.compute( - decodeJPGImage, - param: {"image": compressedJPGImage}, - ); - - if (image == null) { - throw Exception("Failed to decode image"); - } + throw Exception("Failed decoding image file ${widget.enteFile.title}}"); } - final stopwatch = Stopwatch()..start(); final croppedImage = img.copyCrop( image, x: (image.width * faceBox.xMin).round(), @@ -117,10 +98,7 @@ class CroppedFaceImageViewState extends State { height: (image.height * faceBox.height).round(), antialias: false, ); - _logger.info( - "Image crop took ${stopwatch.elapsedMilliseconds}ms ----------------", - ); - stopwatch.stop(); + return convertImageToFlutterUi(croppedImage); } catch (e, s) { _logger.severe("Error getting image", e, s); @@ -128,11 +106,3 @@ class CroppedFaceImageViewState extends State { } } } - -Future decodeImage(Map args) async { - return await img.decodeImageFile(args["filePath"]); -} - -img.Image? decodeJPGImage(Map args) { - return img.decodeJpg(args["image"])!; -} From a0e9913f43693886352ea7715b932308f09d0b95 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 26 Apr 2024 12:56:12 +0530 Subject: [PATCH 07/12] Revert "[mob] Crop image instead of using scale and translate transforms on OG image in CroppedFaceImageView widget" This reverts commit b022ef6d1e6420205ae6a6baa28feffe70679d17. --- .../people/cropped_face_image_view.dart | 144 +++++++++--------- mobile/lib/utils/image_util.dart | 35 ----- 2 files changed, 76 insertions(+), 103 deletions(-) diff --git a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart index cb7baff42..4ef369259 100644 --- a/mobile/lib/ui/viewer/people/cropped_face_image_view.dart +++ b/mobile/lib/ui/viewer/people/cropped_face_image_view.dart @@ -1,14 +1,11 @@ +import 'dart:developer' show log; import "dart:io" show File; -import 'dart:ui' as ui; import 'package:flutter/material.dart'; -import "package:image/image.dart" as img; -import "package:logging/logging.dart"; import "package:photos/face/model/face.dart"; import "package:photos/models/file/file.dart"; import "package:photos/ui/viewer/file/thumbnail_widget.dart"; import "package:photos/utils/file_util.dart"; -import "package:photos/utils/image_util.dart"; class CroppedFaceInfo { final Image image; @@ -24,7 +21,7 @@ class CroppedFaceInfo { }); } -class CroppedFaceImageView extends StatefulWidget { +class CroppedFaceImageView extends StatelessWidget { final EnteFile enteFile; final Face face; @@ -34,75 +31,86 @@ class CroppedFaceImageView extends StatefulWidget { required this.face, }) : super(key: key); - @override - CroppedFaceImageViewState createState() => CroppedFaceImageViewState(); -} - -class CroppedFaceImageViewState extends State { - ui.Image? _image; - final _logger = Logger("CroppedFaceImageView"); - - @override - void initState() { - super.initState(); - _loadImage(); - } - - @override - void dispose() { - super.dispose(); - _image?.dispose(); - } - - Future _loadImage() async { - final image = await getImage(); - if (mounted) { - setState(() { - _image = image; - }); - } - } - @override Widget build(BuildContext context) { - return _image != null - ? LayoutBuilder( - builder: (context, constraints) { - return RawImage( - image: _image!, + return FutureBuilder( + future: getImage(), + builder: (context, snapshot) { + if (snapshot.hasData) { + return LayoutBuilder( + builder: ((context, constraints) { + final double imageAspectRatio = enteFile.width / enteFile.height; + final Image image = snapshot.data!; + + final double viewWidth = constraints.maxWidth; + final double viewHeight = constraints.maxHeight; + + final faceBox = face.detection.box; + + final double relativeFaceCenterX = + faceBox.xMin + faceBox.width / 2; + final double relativeFaceCenterY = + faceBox.yMin + faceBox.height / 2; + + const double desiredFaceHeightRelativeToWidget = 8 / 10; + final double scale = + (1 / faceBox.height) * desiredFaceHeightRelativeToWidget; + + final double widgetCenterX = viewWidth / 2; + final double widgetCenterY = viewHeight / 2; + + final double widgetAspectRatio = viewWidth / viewHeight; + final double imageToWidgetRatio = + imageAspectRatio / widgetAspectRatio; + + double offsetX = + (widgetCenterX - relativeFaceCenterX * viewWidth) * scale; + double offsetY = + (widgetCenterY - relativeFaceCenterY * viewHeight) * scale; + + if (imageAspectRatio < widgetAspectRatio) { + // Landscape Image: Adjust offsetX more conservatively + offsetX = offsetX * imageToWidgetRatio; + } else { + // Portrait Image: Adjust offsetY more conservatively + offsetY = offsetY / imageToWidgetRatio; + } + return ClipRRect( + borderRadius: const BorderRadius.all(Radius.elliptical(16, 12)), + child: Transform.translate( + offset: Offset( + offsetX, + offsetY, + ), + child: Transform.scale( + scale: scale, + child: image, + ), + ), ); - }, - ) - : ThumbnailWidget(widget.enteFile); + }), + ); + } else { + if (snapshot.hasError) { + log('Error getting cover face for person: ${snapshot.error}'); + } + return ThumbnailWidget( + enteFile, + ); + } + }, + ); } - Future getImage() async { - try { - final faceBox = widget.face.detection.box; - final File? ioFile = await getFile(widget.enteFile); - if (ioFile == null) { - return null; - } - - final image = await img.decodeImageFile(ioFile.path); - - if (image == null) { - throw Exception("Failed decoding image file ${widget.enteFile.title}}"); - } - - final croppedImage = img.copyCrop( - image, - x: (image.width * faceBox.xMin).round(), - y: (image.height * faceBox.yMin).round(), - width: (image.width * faceBox.width).round(), - height: (image.height * faceBox.height).round(), - antialias: false, - ); - - return convertImageToFlutterUi(croppedImage); - } catch (e, s) { - _logger.severe("Error getting image", e, s); + Future getImage() async { + final File? ioFile = await getFile(enteFile); + if (ioFile == null) { return null; } + + final imageData = await ioFile.readAsBytes(); + final image = Image.memory(imageData, fit: BoxFit.contain); + + return image; } } diff --git a/mobile/lib/utils/image_util.dart b/mobile/lib/utils/image_util.dart index 7eb1e39fc..a5bcb03a7 100644 --- a/mobile/lib/utils/image_util.dart +++ b/mobile/lib/utils/image_util.dart @@ -1,8 +1,6 @@ import 'dart:async'; -import 'dart:ui' as ui; import 'package:flutter/widgets.dart'; -import 'package:image/image.dart' as img; Future getImageInfo(ImageProvider imageProvider) { final completer = Completer(); @@ -16,36 +14,3 @@ Future getImageInfo(ImageProvider imageProvider) { completer.future.whenComplete(() => imageStream.removeListener(listener)); return completer.future; } - -///https://github.com/brendan-duncan/image/blob/main/doc/flutter.md -Future convertImageToFlutterUi(img.Image image) async { - if (image.format != img.Format.uint8 || image.numChannels != 4) { - final cmd = img.Command() - ..image(image) - ..convert(format: img.Format.uint8, numChannels: 4); - final rgba8 = await cmd.getImageThread(); - if (rgba8 != null) { - image = rgba8; - } - } - - final ui.ImmutableBuffer buffer = - await ui.ImmutableBuffer.fromUint8List(image.toUint8List()); - - final ui.ImageDescriptor id = ui.ImageDescriptor.raw( - buffer, - height: image.height, - width: image.width, - pixelFormat: ui.PixelFormat.rgba8888, - ); - - final ui.Codec codec = await id.instantiateCodec( - targetHeight: image.height, - targetWidth: image.width, - ); - - final ui.FrameInfo fi = await codec.getNextFrame(); - final ui.Image uiImage = fi.image; - - return uiImage; -} From caa72ba83022de5b48355ab9870497b2f598a42a Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 26 Apr 2024 15:44:07 +0530 Subject: [PATCH 08/12] [mob][photos] add option to pass decoded image to face thumbnail generation methods to avoid unnecessary decoding when possible --- mobile/lib/utils/face/face_util.dart | 57 +++++++++++++--------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/mobile/lib/utils/face/face_util.dart b/mobile/lib/utils/face/face_util.dart index 99ec04512..3d7853f08 100644 --- a/mobile/lib/utils/face/face_util.dart +++ b/mobile/lib/utils/face/face_util.dart @@ -12,28 +12,14 @@ final _computer = Computer.shared(); ///Convert img.Image to ui.Image and use RawImage to display. Future> generateImgFaceThumbnails( String imagePath, - List faceBoxes, -) async { + List faceBoxes, { + ///Pass decodedImage decoded by [decodeToImgImage] to avoid decoding image + ///multiple times if all faces are from the same image (eg: File info). + img.Image? decodedImage, +}) async { final faceThumbnails = []; - img.Image? image = - await _computer.compute(_decodeImageFile, param: {"filePath": imagePath}); - - if (image == null) { - _logger.info( - "Failed to decode image. Compressing to jpg and decoding", - ); - final compressedJPGImage = - await FlutterImageCompress.compressWithFile(imagePath); - image = await _computer.compute( - _decodeJpg, - param: {"image": compressedJPGImage}, - ); - - if (image == null) { - throw Exception("Failed to decode image"); - } - } + final image = decodedImage ?? await decodeToImgImage(imagePath); for (FaceBox faceBox in faceBoxes) { final croppedImage = cropFaceBoxFromImage(image, faceBox); @@ -45,8 +31,23 @@ Future> generateImgFaceThumbnails( Future> generateJpgFaceThumbnails( String imagePath, - List faceBoxes, -) async { + List faceBoxes, { + ///Pass decodedImage decoded by [decodeToImgImage] to avoid decoding image + ///multiple times if all faces are from the same image (eg: File info). + img.Image? decodedImage, +}) async { + final image = decodedImage ?? await decodeToImgImage(imagePath); + final croppedImages = []; + for (FaceBox faceBox in faceBoxes) { + final croppedImage = cropFaceBoxFromImage(image, faceBox); + croppedImages.add(croppedImage); + } + + return await _computer + .compute(_encodeImagesToJpg, param: {"images": croppedImages}); +} + +Future decodeToImgImage(String imagePath) async { img.Image? image = await _computer.compute(_decodeImageFile, param: {"filePath": imagePath}); @@ -63,16 +64,12 @@ Future> generateJpgFaceThumbnails( if (image == null) { throw Exception("Failed to decode image"); + } else { + return image; } + } else { + return image; } - final croppedImages = []; - for (FaceBox faceBox in faceBoxes) { - final croppedImage = cropFaceBoxFromImage(image, faceBox); - croppedImages.add(croppedImage); - } - - return await _computer - .compute(_encodeImagesToJpg, param: {"images": croppedImages}); } /// Returns an Image from 'package:image/image.dart' From 8b236cde0946f014eabc86da9b81757d7d4a54d2 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Sat, 27 Apr 2024 09:59:16 +0530 Subject: [PATCH 09/12] [mob][photos] When cropping a face from an image, make the image a square and add some buffer around it --- mobile/lib/face/model/box.dart | 20 ++++++++ mobile/lib/utils/face/face_util.dart | 71 ++++++++++++++++++++++++++-- 2 files changed, 87 insertions(+), 4 deletions(-) diff --git a/mobile/lib/face/model/box.dart b/mobile/lib/face/model/box.dart index 73d7dea38..dafe3d838 100644 --- a/mobile/lib/face/model/box.dart +++ b/mobile/lib/face/model/box.dart @@ -41,3 +41,23 @@ class FaceBox { 'height': height, }; } + +/// Bounding box of a face. +/// +/// [xMin] and [yMin] are the coordinates of the top left corner of the box, and +/// [width] and [height] are the width and height of the box. +/// +/// One unit is equal to one pixel in the original image. +class FaceBoxImage { + final int xMin; + final int yMin; + final int width; + final int height; + + FaceBoxImage({ + required this.xMin, + required this.yMin, + required this.width, + required this.height, + }); +} diff --git a/mobile/lib/utils/face/face_util.dart b/mobile/lib/utils/face/face_util.dart index 3d7853f08..b41765791 100644 --- a/mobile/lib/utils/face/face_util.dart +++ b/mobile/lib/utils/face/face_util.dart @@ -1,3 +1,4 @@ +import "dart:math"; import "dart:typed_data"; import "package:computer/computer.dart"; @@ -8,6 +9,7 @@ import "package:photos/face/model/box.dart"; final _logger = Logger("FaceUtil"); final _computer = Computer.shared(); +const _faceImageBufferFactor = 0.2; ///Convert img.Image to ui.Image and use RawImage to display. Future> generateImgFaceThumbnails( @@ -74,16 +76,77 @@ Future decodeToImgImage(String imagePath) async { /// Returns an Image from 'package:image/image.dart' img.Image cropFaceBoxFromImage(img.Image image, FaceBox faceBox) { + final squareFaceBox = _getSquareFaceBoxImage(image, faceBox); + final squareFaceBoxWithBuffer = + _addBufferAroundFaceBox(squareFaceBox, _faceImageBufferFactor); return img.copyCrop( image, - x: (image.width * faceBox.xMin).round(), - y: (image.height * faceBox.yMin).round(), - width: (image.width * faceBox.width).round(), - height: (image.height * faceBox.height).round(), + x: squareFaceBoxWithBuffer.xMin, + y: squareFaceBoxWithBuffer.yMin, + width: squareFaceBoxWithBuffer.width, + height: squareFaceBoxWithBuffer.height, antialias: false, ); } +/// Returns a square face box image from the original image with +/// side length equal to the maximum of the width and height of the face box in +/// the OG image. +FaceBoxImage _getSquareFaceBoxImage(img.Image image, FaceBox faceBox) { + final width = (image.width * faceBox.width).round(); + final height = (image.height * faceBox.height).round(); + final side = max(width, height); + final xImage = (image.width * faceBox.xMin).round(); + final yImage = (image.height * faceBox.yMin).round(); + + if (height >= width) { + final xImageAdj = (xImage - (height - width) / 2).round(); + return FaceBoxImage( + xMin: xImageAdj, + yMin: yImage, + width: side, + height: side, + ); + } else { + final yImageAdj = (yImage - (width - height) / 2).round(); + return FaceBoxImage( + xMin: xImage, + yMin: yImageAdj, + width: side, + height: side, + ); + } +} + +///To add some buffer around the face box so that the face isn't cropped +///too close to the face. +FaceBoxImage _addBufferAroundFaceBox( + FaceBoxImage faceBoxImage, + double bufferFactor, +) { + final heightBuffer = faceBoxImage.height * bufferFactor; + final widthBuffer = faceBoxImage.width * bufferFactor; + final xMinWithBuffer = faceBoxImage.xMin - widthBuffer; + final yMinWithBuffer = faceBoxImage.yMin - heightBuffer; + final widthWithBuffer = faceBoxImage.width + 2 * widthBuffer; + final heightWithBuffer = faceBoxImage.height + 2 * heightBuffer; + //Do not add buffer if the top left edge of the image is out of bounds + //after adding the buffer. + if (xMinWithBuffer < 0 || yMinWithBuffer < 0) { + return faceBoxImage; + } + //Another similar case that can be handled is when the bottom right edge + //of the image is out of bounds after adding the buffer. But the + //the visual difference is not as significant as when the top left edge + //is out of bounds, so we are not handling that case. + return FaceBoxImage( + xMin: xMinWithBuffer.round(), + yMin: yMinWithBuffer.round(), + width: widthWithBuffer.round(), + height: heightWithBuffer.round(), + ); +} + List _encodeImagesToJpg(Map args) { final images = args["images"] as List; return images.map((img.Image image) => img.encodeJpg(image)).toList(); From 707916f677c4b961e0cb80dd35ca38c6ee9ae72b Mon Sep 17 00:00:00 2001 From: ashilkn Date: Sat, 27 Apr 2024 10:57:49 +0530 Subject: [PATCH 10/12] [mob][photos] Add method to convert Image from Image package to UI image --- mobile/lib/utils/image_util.dart | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/mobile/lib/utils/image_util.dart b/mobile/lib/utils/image_util.dart index a5bcb03a7..e5b0d72fa 100644 --- a/mobile/lib/utils/image_util.dart +++ b/mobile/lib/utils/image_util.dart @@ -1,6 +1,8 @@ import 'dart:async'; +import 'dart:ui' as ui; import 'package:flutter/widgets.dart'; +import 'package:image/image.dart' as img; Future getImageInfo(ImageProvider imageProvider) { final completer = Completer(); @@ -14,3 +16,35 @@ Future getImageInfo(ImageProvider imageProvider) { completer.future.whenComplete(() => imageStream.removeListener(listener)); return completer.future; } + +Future convertImageToFlutterUi(img.Image image) async { + if (image.format != img.Format.uint8 || image.numChannels != 4) { + final cmd = img.Command() + ..image(image) + ..convert(format: img.Format.uint8, numChannels: 4); + final rgba8 = await cmd.getImageThread(); + if (rgba8 != null) { + image = rgba8; + } + } + + final ui.ImmutableBuffer buffer = + await ui.ImmutableBuffer.fromUint8List(image.toUint8List()); + + final ui.ImageDescriptor id = ui.ImageDescriptor.raw( + buffer, + height: image.height, + width: image.width, + pixelFormat: ui.PixelFormat.rgba8888, + ); + + final ui.Codec codec = await id.instantiateCodec( + targetHeight: image.height, + targetWidth: image.width, + ); + + final ui.FrameInfo fi = await codec.getNextFrame(); + final ui.Image uiImage = fi.image; + + return uiImage; +} From 19f2c5f00a026478f7e22104586658c0e4ad0117 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Sat, 27 Apr 2024 11:02:38 +0530 Subject: [PATCH 11/12] [mob][photos] remove negation --- mobile/lib/ui/viewer/search/result/person_face_widget.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/ui/viewer/search/result/person_face_widget.dart b/mobile/lib/ui/viewer/search/result/person_face_widget.dart index bdbfaab94..de0293861 100644 --- a/mobile/lib/ui/viewer/search/result/person_face_widget.dart +++ b/mobile/lib/ui/viewer/search/result/person_face_widget.dart @@ -90,7 +90,7 @@ class PersonFaceWidget extends StatelessWidget { @override Widget build(BuildContext context) { - if (!useGeneratedFaceCrops) { + if (useGeneratedFaceCrops) { return FutureBuilder( future: getFaceCrop(), builder: (context, snapshot) { From ab5985a08b58f5f98e272ed8214532e34289c406 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Sat, 27 Apr 2024 11:06:54 +0530 Subject: [PATCH 12/12] [mob][photos] Use generated face crops and crop it using the new method --- mobile/lib/ui/viewer/file_details/face_widget.dart | 2 +- mobile/lib/utils/face/face_box_crop.dart | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index 4caf1305f..52f173377 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -20,7 +20,7 @@ import "package:photos/utils/face/face_box_crop.dart"; import "package:photos/utils/thumbnail_util.dart"; // import "package:photos/utils/toast_util.dart"; -const useGeneratedFaceCrops = false; +const useGeneratedFaceCrops = true; class FaceWidget extends StatefulWidget { final EnteFile file; diff --git a/mobile/lib/utils/face/face_box_crop.dart b/mobile/lib/utils/face/face_box_crop.dart index 9bf36fbdd..7d032998a 100644 --- a/mobile/lib/utils/face/face_box_crop.dart +++ b/mobile/lib/utils/face/face_box_crop.dart @@ -5,8 +5,8 @@ import "package:photos/core/cache/lru_map.dart"; import "package:photos/face/model/box.dart"; import "package:photos/models/file/file.dart"; import "package:photos/models/file/file_type.dart"; +import "package:photos/utils/face/face_util.dart"; import "package:photos/utils/file_util.dart"; -import "package:photos/utils/image_ml_isolate.dart"; import "package:photos/utils/thumbnail_util.dart"; import "package:pool/pool.dart"; @@ -37,7 +37,8 @@ Future?> getFaceCrops( faceBoxes.add(e.value); } final List faceCrop = - await ImageMlIsolate.instance.generateFaceThumbnailsForImage( + // await ImageMlIsolate.instance.generateFaceThumbnailsForImage( + await generateJpgFaceThumbnails( imagePath, faceBoxes, );