Merge remote-tracking branch 'origin/main' into mobile_face

auth/android/app/src/main/AndroidManifest.xml

@@ -1,7 +1,7 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
           xmlns:tools="http://schemas.android.com/tools">
     <application android:name="${applicationName}"
-                 android:label="auth"
+                 android:label="Auth"
                  android:icon="@mipmap/launcher_icon"
                  android:usesCleartextTraffic="true"
                  android:requestLegacyExternalStorage="true"

auth/ios/Runner/Info.plist

@@ -20,7 +20,7 @@
 			<string>es</string>
 		</array>
 		<key>CFBundleName</key>
-		<string>auth</string>
+		<string>Auth</string>
 		<key>CFBundlePackageType</key>
 		<string>APPL</string>
 		<key>CFBundleShortVersionString</key>

auth/lib/ui/code_widget.dart

@@ -279,7 +279,7 @@ class _CodeWidgetState extends State<CodeWidget> {
           crossAxisAlignment: CrossAxisAlignment.start,
           mainAxisAlignment: MainAxisAlignment.spaceBetween,
           children: [
-            if (widget.code.type == Type.totp)
+            if (widget.code.type.isTOTPCompatible)
               CodeTimerProgress(
                 period: widget.code.period,
               ),

auth/lib/utils/totp_util.dart

@@ -1,8 +1,12 @@
 import 'package:ente_auth/models/code.dart';
 import 'package:flutter/foundation.dart';
 import 'package:otp/otp.dart' as otp;
+import 'package:steam_totp/steam_totp.dart';
 
 String getOTP(Code code) {
+  if (code.issuer.toLowerCase() == 'steam') {
+    return _getSteamCode(code);
+  }
   if (code.type == Type.hotp) {
     return _getHOTPCode(code);
   }
@@ -26,7 +30,18 @@ String _getHOTPCode(Code code) {
   );
 }
 
+String _getSteamCode(Code code, [bool isNext = false]) {
+  final SteamTOTP steamtotp = SteamTOTP(secret: code.secret);
+
+  return steamtotp.generate(
+    DateTime.now().millisecondsSinceEpoch ~/ 1000 + (isNext ? code.period : 0),
+  );
+}
+
 String getNextTotp(Code code) {
+  if (code.issuer.toLowerCase() == 'steam') {
+    return _getSteamCode(code, true);
+  }
   return otp.OTP.generateTOTPCodeString(
     getSanitizedSecret(code.secret),
     DateTime.now().millisecondsSinceEpoch + code.period * 1000,

auth/pubspec.lock

@@ -745,6 +745,22 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "2.1.0"
+  hashlib:
+    dependency: transitive
+    description:
+      name: hashlib
+      sha256: "67e640e19cc33070113acab3125cd48ebe480a0300e15554dec089b8878a729f"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.16.0"
+  hashlib_codecs:
+    dependency: transitive
+    description:
+      name: hashlib_codecs
+      sha256: "49e2a471f74b15f1854263e58c2ac11f2b631b5b12c836f9708a35397d36d626"
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.2.0"
   hex:
     dependency: transitive
     description:
@@ -1439,6 +1455,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "1.11.1"
+  steam_totp:
+    dependency: "direct main"
+    description:
+      name: steam_totp
+      sha256: "3c09143c983f6bb05bb53e9232f9d40bbcc01c596ba0273c3e6bb246729abfa1"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.0.1"
   step_progress_indicator:
     dependency: "direct main"
     description:

auth/pubspec.yaml

@@ -1,6 +1,6 @@
 name: ente_auth
 description: ente two-factor authenticator
-version: 3.0.1+301
+version: 3.0.3+303
 publish_to: none
 
 environment:
@@ -94,6 +94,7 @@ dependencies:
   sqflite_common_ffi: ^2.3.0+4
   sqlite3: ^2.1.0
   sqlite3_flutter_libs: ^0.5.19+1
+  steam_totp: ^0.0.1
   step_progress_indicator: ^1.0.2
   styled_text: ^8.1.0
   tray_manager: ^0.2.1

desktop/src/main/services/dir.ts

@@ -51,14 +51,6 @@ export const openLogDirectory = () => openDirectory(logDirectoryPath());
  * "userData" directory. This is the **primary** place applications are meant to
  * store user's data, e.g. various configuration files and saved state.
  *
- * During development, our app name is "Electron", so this'd be, for example,
- * `~/Library/Application Support/Electron` if we run using `yarn dev`. For the
- * packaged production app, our app name is "ente", so this would be:
- *
- * - Windows: `%APPDATA%\ente`, e.g. `C:\Users\<username>\AppData\Local\ente`
- * - Linux: `~/.config/ente`
- * - macOS: `~/Library/Application Support/ente`
- *
  * Note that Chromium also stores the browser state, e.g. localStorage or disk
  * caches, in userData.
  *
@@ -71,7 +63,6 @@ export const openLogDirectory = () => openDirectory(logDirectoryPath());
  * "ente.log", it can be found at:
  *
  * - macOS: ~/Library/Logs/ente/ente.log (production)
- * - macOS: ~/Library/Logs/Electron/ente.log    (dev)
  * - Linux: ~/.config/ente/logs/ente.log
  * - Windows: %USERPROFILE%\AppData\Roaming\ente\logs\ente.log
  */

desktop/src/main/services/store.ts

@@ -18,10 +18,7 @@ export const clearStores = () => {
  * [Note: Safe storage keys]
  *
  * On macOS, `safeStorage` stores our data under a Keychain entry named
- * "<app-name> Safe Storage". Which resolves to:
- *
- * - Electron Safe Storage (dev)
- * - ente Safe Storage    (prod)
+ * "<app-name> Safe Storage". In our case, "ente Safe Storage".
  */
 export const saveEncryptionKey = (encryptionKey: string) => {
     const encryptedKey = safeStorage.encryptString(encryptionKey);

desktop/src/preload.ts

@@ -65,7 +65,7 @@ const selectDirectory = () => ipcRenderer.invoke("selectDirectory");
 
 const logout = () => {
     watchRemoveListeners();
-    ipcRenderer.send("logout");
+    return ipcRenderer.invoke("logout");
 };
 
 const encryptionKey = () => ipcRenderer.invoke("encryptionKey");

desktop/thirdparty/next-electron-server

@@ -1 +0,0 @@
-Subproject commit a88030295c89dd8f43d9e3a45025678d95c78a45

mobile/lib/services/machine_learning/file_ml/remote_fileml_service.dart

@@ -1,7 +1,6 @@
 import "dart:async";
 import "dart:convert";
 
-import "package:computer/computer.dart";
 import "package:logging/logging.dart";
 import "package:photos/core/network/network.dart";
 import "package:photos/db/files_db.dart";
@@ -22,15 +21,8 @@ class RemoteFileMLService {
 
   final _logger = Logger("RemoteFileMLService");
   final _dio = NetworkClient.instance.enteDio;
-  final _computer = Computer.shared();
 
-  late SharedPreferences _preferences;
-
-  Completer<void>? _syncStatus;
-
-  void init(SharedPreferences prefs) {
-    _preferences = prefs;
-  }
+  void init(SharedPreferences prefs) {}
 
   Future<void> putFileEmbedding(EnteFile file, FileMl fileML) async {
     final encryptionKey = getFileKey(file);

mobile/lib/ui/viewer/file_details/faces_item_widget.dart

@@ -142,7 +142,7 @@ class _FacesItemWidgetState extends State<FacesItemWidget> {
       final faceWidgets = <FaceWidget>[];
 
       // await generation of the face crops here, so that the file info shows one central loading spinner
-      final test = await getRelevantFaceCrops(faces);
+      final _ = await getRelevantFaceCrops(faces);
 
       final faceCrops = getRelevantFaceCrops(faces);
       for (final Face face in faces) {

mobile/lib/utils/local_settings.dart

@@ -86,7 +86,7 @@ class LocalSettings {
 
   //#region todo:(NG) remove this section, only needed for internal testing to see
   // if the OS stops the app during indexing
-  bool get remoteFetchEnabled => _prefs.getBool("remoteFetchEnabled") ?? false;
+  bool get remoteFetchEnabled => _prefs.getBool("remoteFetchEnabled") ?? true;
   Future<void> toggleRemoteFetch() async {
     await _prefs.setBool("remoteFetchEnabled", !remoteFetchEnabled);
   }

mobile/pubspec.yaml

@@ -12,7 +12,7 @@ description: ente photos application
 # Read more about iOS versioning at
 # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html
 
-version: 0.8.96+616
+version: 0.8.97+617
 publish_to: none
 
 environment:

mobile/thirdparty/flutter

@@ -1 +0,0 @@
-Subproject commit 367f9ea16bfae1ca451b9cc27c1366870b187ae2

mobile/thirdparty/isar

@@ -1 +0,0 @@
-Subproject commit 6643d064abf22606b6c6a741ea873e4781115ef4

web/apps/photos/package.json

@@ -27,7 +27,7 @@
         "leaflet-defaulticon-compatibility": "^0.1.1",
         "localforage": "^1.9.0",
         "memoize-one": "^6.0.0",
-        "ml-matrix": "^6.10.4",
+        "ml-matrix": "^6.11",
         "otpauth": "^9.0.2",
         "p-debounce": "^4.0.0",
         "p-queue": "^7.1.0",
@@ -42,7 +42,7 @@
         "react-window": "^1.8.6",
         "sanitize-filename": "^1.6.3",
         "similarity-transformation": "^0.0.1",
-        "transformation-matrix": "^2.15.0",
+        "transformation-matrix": "^2.16",
         "uuid": "^9.0.1",
         "vscode-uri": "^3.0.7",
         "xml-js": "^1.6.11",

web/apps/photos/src/pages/gallery/index.tsx

@@ -717,10 +717,10 @@ export default function Gallery() {
             await syncTrash(collections, setTrashedFiles);
             await syncEntities();
             await syncMapEnabled();
-            await syncCLIPEmbeddings();
             const electron = globalThis.electron;
-            if (isInternalUserForML() && electron) {
-                await syncFaceEmbeddings();
+            if (electron) {
+                await syncCLIPEmbeddings();
+                if (isInternalUserForML()) await syncFaceEmbeddings();
             }
             if (clipService.isPlatformSupported()) {
                 void clipService.scheduleImageEmbeddingExtraction();

web/apps/photos/src/services/face/f-index.ts

@@ -2,7 +2,6 @@ import { FILE_TYPE } from "@/media/file-type";
 import { blobCache } from "@/next/blob-cache";
 import log from "@/next/log";
 import { workerBridge } from "@/next/worker/worker-bridge";
-import { euclidean } from "hdbscan";
 import { Matrix } from "ml-matrix";
 import {
     Box,
@@ -19,6 +18,13 @@ import type {
 } from "services/face/types";
 import { defaultMLVersion } from "services/machineLearning/machineLearningService";
 import { getSimilarityTransformation } from "similarity-transformation";
+import {
+    Matrix as TransformationMatrix,
+    applyToPoint,
+    compose,
+    scale,
+    translate,
+} from "transformation-matrix";
 import type { EnteFile } from "types/file";
 import { fetchImageBitmap, getLocalFileImageBitmap } from "./file";
 import {
@@ -27,7 +33,6 @@ import {
     pixelRGBBilinear,
     warpAffineFloat32List,
 } from "./image";
-import { transformFaceDetections } from "./transform-box";
 
 /**
  * Index faces in the given file.
@@ -138,7 +143,7 @@ const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => {
 /**
  * Detect faces in the given {@link imageBitmap}.
  *
- * The model used is YOLO, running in an ONNX runtime.
+ * The model used is YOLOv5Face, running in an ONNX runtime.
  */
 const detectFaces = async (
     imageBitmap: ImageBitmap,
@@ -149,16 +154,14 @@ const detectFaces = async (
     const { yoloInput, yoloSize } =
         convertToYOLOInputFloat32ChannelsFirst(imageBitmap);
     const yoloOutput = await workerBridge.detectFaces(yoloInput);
-    const faces = faceDetectionsFromYOLOOutput(yoloOutput);
+    const faces = filterExtractDetectionsFromYOLOOutput(yoloOutput);
     const faceDetections = transformFaceDetections(
         faces,
         rect(yoloSize),
         rect(imageBitmap),
     );
 
-    const maxFaceDistancePercent = Math.sqrt(2) / 100;
-    const maxFaceDistance = imageBitmap.width * maxFaceDistancePercent;
-    return removeDuplicateDetections(faceDetections, maxFaceDistance);
+    return naiveNonMaxSuppression(faceDetections, 0.4);
 };
 
 /**
@@ -214,14 +217,24 @@ const convertToYOLOInputFloat32ChannelsFirst = (imageBitmap: ImageBitmap) => {
 };
 
 /**
- * Extract detected faces from the YOLO's output.
+ * Extract detected faces from the YOLOv5Face's output.
  *
  * Only detections that exceed a minimum score are returned.
  *
- * @param rows A Float32Array of shape [25200, 16], where each row
- * represents a bounding box.
+ * @param rows A Float32Array of shape [25200, 16], where each row represents a
+ * face detection.
+ *
+ * YOLO detects a fixed number of faces, 25200, always from the input it is
+ * given. Each detection is a "row" of 16 bytes, containing the bounding box,
+ * score, and landmarks of the detection.
+ *
+ * We prune out detections with a score lower than our threshold. However, we
+ * will still be left with some overlapping detections of the same face: these
+ * we will deduplicate in {@link removeDuplicateDetections}.
  */
-const faceDetectionsFromYOLOOutput = (rows: Float32Array): FaceDetection[] => {
+const filterExtractDetectionsFromYOLOOutput = (
+    rows: Float32Array,
+): FaceDetection[] => {
     const faces: FaceDetection[] = [];
     // Iterate over each row.
     for (let i = 0; i < rows.length; i += 16) {
@@ -266,61 +279,121 @@ const faceDetectionsFromYOLOOutput = (rows: Float32Array): FaceDetection[] => {
 };
 
 /**
- * Removes duplicate face detections from an array of detections.
+ * Transform the given {@link faceDetections} from their coordinate system in
+ * which they were detected ({@link inBox}) back to the coordinate system of the
+ * original image ({@link toBox}).
+ */
+const transformFaceDetections = (
+    faceDetections: FaceDetection[],
+    inBox: Box,
+    toBox: Box,
+): FaceDetection[] => {
+    const transform = boxTransformationMatrix(inBox, toBox);
+    return faceDetections.map((f) => ({
+        box: transformBox(f.box, transform),
+        landmarks: f.landmarks.map((p) => transformPoint(p, transform)),
+        probability: f.probability,
+    }));
+};
+
+const boxTransformationMatrix = (
+    inBox: Box,
+    toBox: Box,
+): TransformationMatrix =>
+    compose(
+        translate(toBox.x, toBox.y),
+        scale(toBox.width / inBox.width, toBox.height / inBox.height),
+    );
+
+const transformPoint = (point: Point, transform: TransformationMatrix) => {
+    const txdPoint = applyToPoint(transform, point);
+    return new Point(txdPoint.x, txdPoint.y);
+};
+
+const transformBox = (box: Box, transform: TransformationMatrix) => {
+    const topLeft = transformPoint(new Point(box.x, box.y), transform);
+    const bottomRight = transformPoint(
+        new Point(box.x + box.width, box.y + box.height),
+        transform,
+    );
+
+    return new Box({
+        x: topLeft.x,
+        y: topLeft.y,
+        width: bottomRight.x - topLeft.x,
+        height: bottomRight.y - topLeft.y,
+    });
+};
+
+/**
+ * Remove overlapping faces from an array of face detections through non-maximum
+ * suppression algorithm.
  *
  * This function sorts the detections by their probability in descending order,
  * then iterates over them.
  *
- * For each detection, it calculates the Euclidean distance to all other
- * detections.
+ * For each detection, it calculates the Intersection over Union (IoU) with all
+ * other detections.
  *
- * If the distance is less than or equal to the specified threshold
- * (`withinDistance`), the other detection is considered a duplicate and is
+ * If the IoU is greater than or equal to the specified threshold
+ * (`iouThreshold`), the other detection is considered overlapping and is
  * removed.
  *
- * @param detections - An array of face detections to remove duplicates from.
+ * @param detections - An array of face detections to remove overlapping faces
+ * from.
  *
- * @param withinDistance - The maximum Euclidean distance between two detections
- * for them to be considered duplicates.
+ * @param iouThreshold - The minimum IoU between two detections for them to be
+ * considered overlapping.
  *
- * @returns An array of face detections with duplicates removed.
+ * @returns An array of face detections with overlapping faces removed
  */
-const removeDuplicateDetections = (
+const naiveNonMaxSuppression = (
     detections: FaceDetection[],
-    withinDistance: number,
-) => {
+    iouThreshold: number,
+): FaceDetection[] => {
+    // Sort the detections by score, the highest first.
     detections.sort((a, b) => b.probability - a.probability);
 
-    const dupIndices = new Set<number>();
-    for (let i = 0; i < detections.length; i++) {
-        if (dupIndices.has(i)) continue;
-
+    // Loop through the detections and calculate the IOU.
+    for (let i = 0; i < detections.length - 1; i++) {
         for (let j = i + 1; j < detections.length; j++) {
-            if (dupIndices.has(j)) continue;
-
-            const centeri = faceDetectionCenter(detections[i]);
-            const centerj = faceDetectionCenter(detections[j]);
-            const dist = euclidean(
-                [centeri.x, centeri.y],
-                [centerj.x, centerj.y],
-            );
-
-            if (dist <= withinDistance) dupIndices.add(j);
+            const iou = intersectionOverUnion(detections[i], detections[j]);
+            if (iou >= iouThreshold) {
+                detections.splice(j, 1);
+                j--;
+            }
         }
     }
 
-    return detections.filter((_, i) => !dupIndices.has(i));
+    return detections;
 };
 
-const faceDetectionCenter = (detection: FaceDetection) => {
-    const center = new Point(0, 0);
-    // TODO-ML(LAURENS): first 4 landmarks is applicable to blazeface only this
-    // needs to consider eyes, nose and mouth landmarks to take center
-    detection.landmarks?.slice(0, 4).forEach((p) => {
-        center.x += p.x;
-        center.y += p.y;
-    });
-    return new Point(center.x / 4, center.y / 4);
+const intersectionOverUnion = (a: FaceDetection, b: FaceDetection): number => {
+    const intersectionMinX = Math.max(a.box.x, b.box.x);
+    const intersectionMinY = Math.max(a.box.y, b.box.y);
+    const intersectionMaxX = Math.min(
+        a.box.x + a.box.width,
+        b.box.x + b.box.width,
+    );
+    const intersectionMaxY = Math.min(
+        a.box.y + a.box.height,
+        b.box.y + b.box.height,
+    );
+
+    const intersectionWidth = intersectionMaxX - intersectionMinX;
+    const intersectionHeight = intersectionMaxY - intersectionMinY;
+
+    if (intersectionWidth < 0 || intersectionHeight < 0) {
+        return 0.0; // If boxes do not overlap, IoU is 0
+    }
+
+    const areaA = a.box.width * a.box.height;
+    const areaB = b.box.width * b.box.height;
+
+    const intersectionArea = intersectionWidth * intersectionHeight;
+    const unionArea = areaA + areaB - intersectionArea;
+
+    return intersectionArea / unionArea;
 };
 
 const makeFaceID = (
@@ -398,12 +471,15 @@ const faceAlignmentUsingSimilarityTransform = (
     const meanTranslation = simTransform.toMean.sub(0.5).mul(size);
     const centerMat = simTransform.fromMean.sub(meanTranslation);
     const center = new Point(centerMat.get(0, 0), centerMat.get(1, 0));
-    const rotation = -Math.atan2(
-        simTransform.rotation.get(0, 1),
-        simTransform.rotation.get(0, 0),
-    );
 
-    return { affineMatrix, center, size, rotation };
+    const boundingBox = new Box({
+        x: center.x - size / 2,
+        y: center.y - size / 2,
+        width: size,
+        height: size,
+    });
+
+    return { affineMatrix, boundingBox };
 };
 
 const convertToMobileFaceNetInput = (
@@ -678,35 +754,22 @@ const extractFaceCrop = (
     imageBitmap: ImageBitmap,
     alignment: FaceAlignment,
 ): ImageBitmap => {
-    const alignmentBox = new Box({
-        x: alignment.center.x - alignment.size / 2,
-        y: alignment.center.y - alignment.size / 2,
-        width: alignment.size,
-        height: alignment.size,
-    });
+    // TODO-ML: This algorithm is different from what is used by the mobile app.
+    // Also, it needs to be something that can work fully using the embedding we
+    // receive from remote - the `alignment.boundingBox` will not be available
+    // to us in such cases.
+    const paddedBox = roundBox(enlargeBox(alignment.boundingBox, 1.5));
+    const outputSize = { width: paddedBox.width, height: paddedBox.height };
 
-    const padding = 0.25;
-    const scaleForPadding = 1 + padding * 2;
-    const paddedBox = roundBox(enlargeBox(alignmentBox, scaleForPadding));
+    const maxDimension = 256;
+    const scale = Math.min(
+        maxDimension / paddedBox.width,
+        maxDimension / paddedBox.height,
+    );
 
-    // TODO-ML(LAURENS): The rotation doesn't seem to be used? it's set to 0.
-    return cropWithRotation(imageBitmap, paddedBox, 0, 256);
-};
-
-const cropWithRotation = (
-    imageBitmap: ImageBitmap,
-    cropBox: Box,
-    rotation: number,
-    maxDimension: number,
-) => {
-    const box = roundBox(cropBox);
-
-    const outputSize = { width: box.width, height: box.height };
-
-    const scale = Math.min(maxDimension / box.width, maxDimension / box.height);
     if (scale < 1) {
-        outputSize.width = Math.round(scale * box.width);
-        outputSize.height = Math.round(scale * box.height);
+        outputSize.width = Math.round(scale * paddedBox.width);
+        outputSize.height = Math.round(scale * paddedBox.height);
     }
 
     const offscreen = new OffscreenCanvas(outputSize.width, outputSize.height);
@@ -714,7 +777,6 @@ const cropWithRotation = (
     offscreenCtx.imageSmoothingQuality = "high";
 
     offscreenCtx.translate(outputSize.width / 2, outputSize.height / 2);
-    rotation && offscreenCtx.rotate(rotation);
 
     const outputBox = new Box({
         x: -outputSize.width / 2,
@@ -723,7 +785,7 @@ const cropWithRotation = (
         height: outputSize.height,
     });
 
-    const enlargedBox = enlargeBox(box, 1.5);
+    const enlargedBox = enlargeBox(paddedBox, 1.5);
     const enlargedOutputBox = enlargeBox(outputBox, 1.5);
 
     offscreenCtx.drawImage(

web/apps/photos/src/services/face/remote.ts

@@ -20,16 +20,18 @@ export const putFaceEmbedding = async (
     const comlinkCryptoWorker = await ComlinkCryptoWorker.getInstance();
     const { file: encryptedEmbeddingData } =
         await comlinkCryptoWorker.encryptMetadata(serverMl, enteFile.key);
-    log.info(
-        `putEmbedding embedding to server for file: ${enteFile.metadata.title} fileID: ${enteFile.id}`,
-    );
-    const res = await putEmbedding({
+    // TODO-ML(MR): Do we need any of these fields
+    // log.info(
+    //     `putEmbedding embedding to server for file: ${enteFile.metadata.title} fileID: ${enteFile.id}`,
+    // );
+    /*const res =*/ await putEmbedding({
         fileID: enteFile.id,
         encryptedEmbedding: encryptedEmbeddingData.encryptedData,
         decryptionHeader: encryptedEmbeddingData.decryptionHeader,
         model: "file-ml-clip-face",
     });
-    log.info("putEmbedding response: ", res);
+    // TODO-ML(MR): Do we need any of these fields
+    // log.info("putEmbedding response: ", res);
 };
 
 export interface FileML extends ServerFileMl {

web/apps/photos/src/services/face/transform-box.ts

@@ -1,57 +0,0 @@
-import { Box, Point } from "services/face/geom";
-import type { FaceDetection } from "services/face/types";
-// TODO-ML(LAURENS): Do we need two separate Matrix libraries?
-//
-// Keeping this in a separate file so that we can audit this. If these can be
-// expressed using ml-matrix, then we can move this code to f-index.ts
-import {
-    Matrix,
-    applyToPoint,
-    compose,
-    scale,
-    translate,
-} from "transformation-matrix";
-
-/**
- * Transform the given {@link faceDetections} from their coordinate system in
- * which they were detected ({@link inBox}) back to the coordinate system of the
- * original image ({@link toBox}).
- */
-export const transformFaceDetections = (
-    faceDetections: FaceDetection[],
-    inBox: Box,
-    toBox: Box,
-): FaceDetection[] => {
-    const transform = boxTransformationMatrix(inBox, toBox);
-    return faceDetections.map((f) => ({
-        box: transformBox(f.box, transform),
-        landmarks: f.landmarks.map((p) => transformPoint(p, transform)),
-        probability: f.probability,
-    }));
-};
-
-const boxTransformationMatrix = (inBox: Box, toBox: Box): Matrix =>
-    compose(
-        translate(toBox.x, toBox.y),
-        scale(toBox.width / inBox.width, toBox.height / inBox.height),
-    );
-
-const transformPoint = (point: Point, transform: Matrix) => {
-    const txdPoint = applyToPoint(transform, point);
-    return new Point(txdPoint.x, txdPoint.y);
-};
-
-const transformBox = (box: Box, transform: Matrix) => {
-    const topLeft = transformPoint(new Point(box.x, box.y), transform);
-    const bottomRight = transformPoint(
-        new Point(box.x + box.width, box.y + box.height),
-        transform,
-    );
-
-    return new Box({
-        x: topLeft.x,
-        y: topLeft.y,
-        width: bottomRight.x - topLeft.x,
-        height: bottomRight.y - topLeft.y,
-    });
-};

web/apps/photos/src/services/face/types.ts

@@ -8,13 +8,20 @@ export interface FaceDetection {
 }
 
 export interface FaceAlignment {
-    // TODO-ML(MR): remove affine matrix as rotation, size and center
-    // are simple to store and use, affine matrix adds complexity while getting crop
+    /**
+     * An affine transformation matrix (rotation, translation, scaling) to align
+     * the face extracted from the image.
+     */
     affineMatrix: number[][];
-    rotation: number;
-    // size and center is relative to image dimentions stored at mlFileData
-    size: number;
-    center: Point;
+    /**
+     * The bounding box of the transformed box.
+     *
+     * The affine transformation shifts the original detection box a new,
+     * transformed, box (possibily rotated). This property is the bounding box
+     * of that transformed box. It is in the coordinate system of the original,
+     * full, image on which the detection occurred.
+     */
+    boundingBox: Box;
 }
 
 export interface Face {

web/docs/dependencies.md

@@ -177,12 +177,19 @@ some cases.
 
 ## Face search
 
--   [matrix](https://github.com/mljs/matrix) and
-    [similarity-transformation](https://github.com/shaileshpandit/similarity-transformation-js)
-    are used during face alignment.
-
 -   [transformation-matrix](https://github.com/chrvadala/transformation-matrix)
-    is used during face detection.
+    is used for performing 2D affine transformations using transformation
+    matrices. It is used during face detection.
+
+-   [matrix](https://github.com/mljs/matrix) is mathematical matrix abstraction.
+    It is used alongwith
+    [similarity-transformation](https://github.com/shaileshpandit/similarity-transformation-js)
+    during face alignment.
+
+    > Note that while both `transformation-matrix` and `matrix` are "matrix"
+    > libraries, they have different foci and purposes: `transformation-matrix`
+    > provides affine transforms, while `matrix` is for performing computations
+    > on matrices, say inverting them or performing their decomposition.
 
 -   [hdbscan](https://github.com/shaileshpandit/hdbscan-js) is used for face
     clustering.

web/packages/next/locales/pt-BR/translation.json

@@ -428,7 +428,7 @@
     "USED": "usado",
     "YOU": "Você",
     "FAMILY": "Família",
-    "FREE": "grátis",
+    "FREE": "livre",
     "OF": "de",
     "WATCHED_FOLDERS": "Pastas monitoradas",
     "NO_FOLDERS_ADDED": "Nenhuma pasta adicionada ainda!",

web/yarn.lock

@@ -3528,7 +3528,7 @@ ml-array-rescale@^1.3.7:
     ml-array-max "^1.2.4"
     ml-array-min "^1.2.3"
 
-ml-matrix@^6.10.4:
+ml-matrix@^6.11:
   version "6.11.0"
   resolved "https://registry.yarnpkg.com/ml-matrix/-/ml-matrix-6.11.0.tgz#3cf2260ef04cbb8e0e0425e71d200f5cbcf82772"
   integrity sha512-7jr9NmFRkaUxbKslfRu3aZOjJd2LkSitCGv+QH9PF0eJoEG7jIpjXra1Vw8/kgao8+kHCSsJONG6vfWmXQ+/Eg==
@@ -4628,7 +4628,7 @@ tr46@~0.0.3:
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
   integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==
 
-transformation-matrix@^2.15.0:
+transformation-matrix@^2.16:
   version "2.16.1"
   resolved "https://registry.yarnpkg.com/transformation-matrix/-/transformation-matrix-2.16.1.tgz#4a2de06331b94ae953193d1b9a5ba002ec5f658a"
   integrity sha512-tdtC3wxVEuzU7X/ydL131Q3JU5cPMEn37oqVLITjRDSDsnSHVFzW2JiCLfZLIQEgWzZHdSy3J6bZzvKEN24jGA==