Merge remote-tracking branch 'origin/main' into mobile_face
|
@ -1,7 +1,7 @@
|
|||
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
xmlns:tools="http://schemas.android.com/tools">
|
||||
<application android:name="${applicationName}"
|
||||
android:label="auth"
|
||||
android:label="Auth"
|
||||
android:icon="@mipmap/launcher_icon"
|
||||
android:usesCleartextTraffic="true"
|
||||
android:requestLegacyExternalStorage="true"
|
||||
|
|
Before Width: | Height: | Size: 3.1 KiB After Width: | Height: | Size: 133 KiB |
Before Width: | Height: | Size: 38 KiB After Width: | Height: | Size: 801 KiB |
|
@ -20,7 +20,7 @@
|
|||
<string>es</string>
|
||||
</array>
|
||||
<key>CFBundleName</key>
|
||||
<string>auth</string>
|
||||
<string>Auth</string>
|
||||
<key>CFBundlePackageType</key>
|
||||
<string>APPL</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
|
|
|
@ -279,7 +279,7 @@ class _CodeWidgetState extends State<CodeWidget> {
|
|||
crossAxisAlignment: CrossAxisAlignment.start,
|
||||
mainAxisAlignment: MainAxisAlignment.spaceBetween,
|
||||
children: [
|
||||
if (widget.code.type == Type.totp)
|
||||
if (widget.code.type.isTOTPCompatible)
|
||||
CodeTimerProgress(
|
||||
period: widget.code.period,
|
||||
),
|
||||
|
|
|
@ -1,8 +1,12 @@
|
|||
import 'package:ente_auth/models/code.dart';
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:otp/otp.dart' as otp;
|
||||
import 'package:steam_totp/steam_totp.dart';
|
||||
|
||||
String getOTP(Code code) {
|
||||
if (code.issuer.toLowerCase() == 'steam') {
|
||||
return _getSteamCode(code);
|
||||
}
|
||||
if (code.type == Type.hotp) {
|
||||
return _getHOTPCode(code);
|
||||
}
|
||||
|
@ -26,7 +30,18 @@ String _getHOTPCode(Code code) {
|
|||
);
|
||||
}
|
||||
|
||||
String _getSteamCode(Code code, [bool isNext = false]) {
|
||||
final SteamTOTP steamtotp = SteamTOTP(secret: code.secret);
|
||||
|
||||
return steamtotp.generate(
|
||||
DateTime.now().millisecondsSinceEpoch ~/ 1000 + (isNext ? code.period : 0),
|
||||
);
|
||||
}
|
||||
|
||||
String getNextTotp(Code code) {
|
||||
if (code.issuer.toLowerCase() == 'steam') {
|
||||
return _getSteamCode(code, true);
|
||||
}
|
||||
return otp.OTP.generateTOTPCodeString(
|
||||
getSanitizedSecret(code.secret),
|
||||
DateTime.now().millisecondsSinceEpoch + code.period * 1000,
|
||||
|
|
Before Width: | Height: | Size: 14 KiB After Width: | Height: | Size: 22 KiB |
Before Width: | Height: | Size: 2.1 KiB After Width: | Height: | Size: 2.9 KiB |
Before Width: | Height: | Size: 240 B After Width: | Height: | Size: 324 B |
Before Width: | Height: | Size: 3.9 KiB After Width: | Height: | Size: 5.6 KiB |
Before Width: | Height: | Size: 628 B After Width: | Height: | Size: 780 B |
Before Width: | Height: | Size: 7.6 KiB After Width: | Height: | Size: 11 KiB |
Before Width: | Height: | Size: 1.1 KiB After Width: | Height: | Size: 1.5 KiB |
|
@ -745,6 +745,22 @@ packages:
|
|||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "2.1.0"
|
||||
hashlib:
|
||||
dependency: transitive
|
||||
description:
|
||||
name: hashlib
|
||||
sha256: "67e640e19cc33070113acab3125cd48ebe480a0300e15554dec089b8878a729f"
|
||||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "1.16.0"
|
||||
hashlib_codecs:
|
||||
dependency: transitive
|
||||
description:
|
||||
name: hashlib_codecs
|
||||
sha256: "49e2a471f74b15f1854263e58c2ac11f2b631b5b12c836f9708a35397d36d626"
|
||||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "2.2.0"
|
||||
hex:
|
||||
dependency: transitive
|
||||
description:
|
||||
|
@ -1439,6 +1455,14 @@ packages:
|
|||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "1.11.1"
|
||||
steam_totp:
|
||||
dependency: "direct main"
|
||||
description:
|
||||
name: steam_totp
|
||||
sha256: "3c09143c983f6bb05bb53e9232f9d40bbcc01c596ba0273c3e6bb246729abfa1"
|
||||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "0.0.1"
|
||||
step_progress_indicator:
|
||||
dependency: "direct main"
|
||||
description:
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
name: ente_auth
|
||||
description: ente two-factor authenticator
|
||||
version: 3.0.1+301
|
||||
version: 3.0.3+303
|
||||
publish_to: none
|
||||
|
||||
environment:
|
||||
|
@ -94,6 +94,7 @@ dependencies:
|
|||
sqflite_common_ffi: ^2.3.0+4
|
||||
sqlite3: ^2.1.0
|
||||
sqlite3_flutter_libs: ^0.5.19+1
|
||||
steam_totp: ^0.0.1
|
||||
step_progress_indicator: ^1.0.2
|
||||
styled_text: ^8.1.0
|
||||
tray_manager: ^0.2.1
|
||||
|
|
|
@ -51,14 +51,6 @@ export const openLogDirectory = () => openDirectory(logDirectoryPath());
|
|||
* "userData" directory. This is the **primary** place applications are meant to
|
||||
* store user's data, e.g. various configuration files and saved state.
|
||||
*
|
||||
* During development, our app name is "Electron", so this'd be, for example,
|
||||
* `~/Library/Application Support/Electron` if we run using `yarn dev`. For the
|
||||
* packaged production app, our app name is "ente", so this would be:
|
||||
*
|
||||
* - Windows: `%APPDATA%\ente`, e.g. `C:\Users\<username>\AppData\Local\ente`
|
||||
* - Linux: `~/.config/ente`
|
||||
* - macOS: `~/Library/Application Support/ente`
|
||||
*
|
||||
* Note that Chromium also stores the browser state, e.g. localStorage or disk
|
||||
* caches, in userData.
|
||||
*
|
||||
|
@ -71,7 +63,6 @@ export const openLogDirectory = () => openDirectory(logDirectoryPath());
|
|||
* "ente.log", it can be found at:
|
||||
*
|
||||
* - macOS: ~/Library/Logs/ente/ente.log (production)
|
||||
* - macOS: ~/Library/Logs/Electron/ente.log (dev)
|
||||
* - Linux: ~/.config/ente/logs/ente.log
|
||||
* - Windows: %USERPROFILE%\AppData\Roaming\ente\logs\ente.log
|
||||
*/
|
||||
|
|
|
@ -18,10 +18,7 @@ export const clearStores = () => {
|
|||
* [Note: Safe storage keys]
|
||||
*
|
||||
* On macOS, `safeStorage` stores our data under a Keychain entry named
|
||||
* "<app-name> Safe Storage". Which resolves to:
|
||||
*
|
||||
* - Electron Safe Storage (dev)
|
||||
* - ente Safe Storage (prod)
|
||||
* "<app-name> Safe Storage". In our case, "ente Safe Storage".
|
||||
*/
|
||||
export const saveEncryptionKey = (encryptionKey: string) => {
|
||||
const encryptedKey = safeStorage.encryptString(encryptionKey);
|
||||
|
|
|
@ -65,7 +65,7 @@ const selectDirectory = () => ipcRenderer.invoke("selectDirectory");
|
|||
|
||||
const logout = () => {
|
||||
watchRemoveListeners();
|
||||
ipcRenderer.send("logout");
|
||||
return ipcRenderer.invoke("logout");
|
||||
};
|
||||
|
||||
const encryptionKey = () => ipcRenderer.invoke("encryptionKey");
|
||||
|
|
1
desktop/thirdparty/next-electron-server
vendored
|
@ -1 +0,0 @@
|
|||
Subproject commit a88030295c89dd8f43d9e3a45025678d95c78a45
|
|
@ -1,7 +1,6 @@
|
|||
import "dart:async";
|
||||
import "dart:convert";
|
||||
|
||||
import "package:computer/computer.dart";
|
||||
import "package:logging/logging.dart";
|
||||
import "package:photos/core/network/network.dart";
|
||||
import "package:photos/db/files_db.dart";
|
||||
|
@ -22,15 +21,8 @@ class RemoteFileMLService {
|
|||
|
||||
final _logger = Logger("RemoteFileMLService");
|
||||
final _dio = NetworkClient.instance.enteDio;
|
||||
final _computer = Computer.shared();
|
||||
|
||||
late SharedPreferences _preferences;
|
||||
|
||||
Completer<void>? _syncStatus;
|
||||
|
||||
void init(SharedPreferences prefs) {
|
||||
_preferences = prefs;
|
||||
}
|
||||
void init(SharedPreferences prefs) {}
|
||||
|
||||
Future<void> putFileEmbedding(EnteFile file, FileMl fileML) async {
|
||||
final encryptionKey = getFileKey(file);
|
||||
|
|
|
@ -142,7 +142,7 @@ class _FacesItemWidgetState extends State<FacesItemWidget> {
|
|||
final faceWidgets = <FaceWidget>[];
|
||||
|
||||
// await generation of the face crops here, so that the file info shows one central loading spinner
|
||||
final test = await getRelevantFaceCrops(faces);
|
||||
final _ = await getRelevantFaceCrops(faces);
|
||||
|
||||
final faceCrops = getRelevantFaceCrops(faces);
|
||||
for (final Face face in faces) {
|
||||
|
|
|
@ -86,7 +86,7 @@ class LocalSettings {
|
|||
|
||||
//#region todo:(NG) remove this section, only needed for internal testing to see
|
||||
// if the OS stops the app during indexing
|
||||
bool get remoteFetchEnabled => _prefs.getBool("remoteFetchEnabled") ?? false;
|
||||
bool get remoteFetchEnabled => _prefs.getBool("remoteFetchEnabled") ?? true;
|
||||
Future<void> toggleRemoteFetch() async {
|
||||
await _prefs.setBool("remoteFetchEnabled", !remoteFetchEnabled);
|
||||
}
|
||||
|
|
|
@ -12,7 +12,7 @@ description: ente photos application
|
|||
# Read more about iOS versioning at
|
||||
# https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html
|
||||
|
||||
version: 0.8.96+616
|
||||
version: 0.8.97+617
|
||||
publish_to: none
|
||||
|
||||
environment:
|
||||
|
|
1
mobile/thirdparty/flutter
vendored
|
@ -1 +0,0 @@
|
|||
Subproject commit 367f9ea16bfae1ca451b9cc27c1366870b187ae2
|
1
mobile/thirdparty/isar
vendored
|
@ -1 +0,0 @@
|
|||
Subproject commit 6643d064abf22606b6c6a741ea873e4781115ef4
|
|
@ -27,7 +27,7 @@
|
|||
"leaflet-defaulticon-compatibility": "^0.1.1",
|
||||
"localforage": "^1.9.0",
|
||||
"memoize-one": "^6.0.0",
|
||||
"ml-matrix": "^6.10.4",
|
||||
"ml-matrix": "^6.11",
|
||||
"otpauth": "^9.0.2",
|
||||
"p-debounce": "^4.0.0",
|
||||
"p-queue": "^7.1.0",
|
||||
|
@ -42,7 +42,7 @@
|
|||
"react-window": "^1.8.6",
|
||||
"sanitize-filename": "^1.6.3",
|
||||
"similarity-transformation": "^0.0.1",
|
||||
"transformation-matrix": "^2.15.0",
|
||||
"transformation-matrix": "^2.16",
|
||||
"uuid": "^9.0.1",
|
||||
"vscode-uri": "^3.0.7",
|
||||
"xml-js": "^1.6.11",
|
||||
|
|
|
@ -717,10 +717,10 @@ export default function Gallery() {
|
|||
await syncTrash(collections, setTrashedFiles);
|
||||
await syncEntities();
|
||||
await syncMapEnabled();
|
||||
await syncCLIPEmbeddings();
|
||||
const electron = globalThis.electron;
|
||||
if (isInternalUserForML() && electron) {
|
||||
await syncFaceEmbeddings();
|
||||
if (electron) {
|
||||
await syncCLIPEmbeddings();
|
||||
if (isInternalUserForML()) await syncFaceEmbeddings();
|
||||
}
|
||||
if (clipService.isPlatformSupported()) {
|
||||
void clipService.scheduleImageEmbeddingExtraction();
|
||||
|
|
|
@ -2,7 +2,6 @@ import { FILE_TYPE } from "@/media/file-type";
|
|||
import { blobCache } from "@/next/blob-cache";
|
||||
import log from "@/next/log";
|
||||
import { workerBridge } from "@/next/worker/worker-bridge";
|
||||
import { euclidean } from "hdbscan";
|
||||
import { Matrix } from "ml-matrix";
|
||||
import {
|
||||
Box,
|
||||
|
@ -19,6 +18,13 @@ import type {
|
|||
} from "services/face/types";
|
||||
import { defaultMLVersion } from "services/machineLearning/machineLearningService";
|
||||
import { getSimilarityTransformation } from "similarity-transformation";
|
||||
import {
|
||||
Matrix as TransformationMatrix,
|
||||
applyToPoint,
|
||||
compose,
|
||||
scale,
|
||||
translate,
|
||||
} from "transformation-matrix";
|
||||
import type { EnteFile } from "types/file";
|
||||
import { fetchImageBitmap, getLocalFileImageBitmap } from "./file";
|
||||
import {
|
||||
|
@ -27,7 +33,6 @@ import {
|
|||
pixelRGBBilinear,
|
||||
warpAffineFloat32List,
|
||||
} from "./image";
|
||||
import { transformFaceDetections } from "./transform-box";
|
||||
|
||||
/**
|
||||
* Index faces in the given file.
|
||||
|
@ -138,7 +143,7 @@ const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => {
|
|||
/**
|
||||
* Detect faces in the given {@link imageBitmap}.
|
||||
*
|
||||
* The model used is YOLO, running in an ONNX runtime.
|
||||
* The model used is YOLOv5Face, running in an ONNX runtime.
|
||||
*/
|
||||
const detectFaces = async (
|
||||
imageBitmap: ImageBitmap,
|
||||
|
@ -149,16 +154,14 @@ const detectFaces = async (
|
|||
const { yoloInput, yoloSize } =
|
||||
convertToYOLOInputFloat32ChannelsFirst(imageBitmap);
|
||||
const yoloOutput = await workerBridge.detectFaces(yoloInput);
|
||||
const faces = faceDetectionsFromYOLOOutput(yoloOutput);
|
||||
const faces = filterExtractDetectionsFromYOLOOutput(yoloOutput);
|
||||
const faceDetections = transformFaceDetections(
|
||||
faces,
|
||||
rect(yoloSize),
|
||||
rect(imageBitmap),
|
||||
);
|
||||
|
||||
const maxFaceDistancePercent = Math.sqrt(2) / 100;
|
||||
const maxFaceDistance = imageBitmap.width * maxFaceDistancePercent;
|
||||
return removeDuplicateDetections(faceDetections, maxFaceDistance);
|
||||
return naiveNonMaxSuppression(faceDetections, 0.4);
|
||||
};
|
||||
|
||||
/**
|
||||
|
@ -214,14 +217,24 @@ const convertToYOLOInputFloat32ChannelsFirst = (imageBitmap: ImageBitmap) => {
|
|||
};
|
||||
|
||||
/**
|
||||
* Extract detected faces from the YOLO's output.
|
||||
* Extract detected faces from the YOLOv5Face's output.
|
||||
*
|
||||
* Only detections that exceed a minimum score are returned.
|
||||
*
|
||||
* @param rows A Float32Array of shape [25200, 16], where each row
|
||||
* represents a bounding box.
|
||||
* @param rows A Float32Array of shape [25200, 16], where each row represents a
|
||||
* face detection.
|
||||
*
|
||||
* YOLO detects a fixed number of faces, 25200, always from the input it is
|
||||
* given. Each detection is a "row" of 16 bytes, containing the bounding box,
|
||||
* score, and landmarks of the detection.
|
||||
*
|
||||
* We prune out detections with a score lower than our threshold. However, we
|
||||
* will still be left with some overlapping detections of the same face: these
|
||||
* we will deduplicate in {@link removeDuplicateDetections}.
|
||||
*/
|
||||
const faceDetectionsFromYOLOOutput = (rows: Float32Array): FaceDetection[] => {
|
||||
const filterExtractDetectionsFromYOLOOutput = (
|
||||
rows: Float32Array,
|
||||
): FaceDetection[] => {
|
||||
const faces: FaceDetection[] = [];
|
||||
// Iterate over each row.
|
||||
for (let i = 0; i < rows.length; i += 16) {
|
||||
|
@ -266,61 +279,121 @@ const faceDetectionsFromYOLOOutput = (rows: Float32Array): FaceDetection[] => {
|
|||
};
|
||||
|
||||
/**
|
||||
* Removes duplicate face detections from an array of detections.
|
||||
* Transform the given {@link faceDetections} from their coordinate system in
|
||||
* which they were detected ({@link inBox}) back to the coordinate system of the
|
||||
* original image ({@link toBox}).
|
||||
*/
|
||||
const transformFaceDetections = (
|
||||
faceDetections: FaceDetection[],
|
||||
inBox: Box,
|
||||
toBox: Box,
|
||||
): FaceDetection[] => {
|
||||
const transform = boxTransformationMatrix(inBox, toBox);
|
||||
return faceDetections.map((f) => ({
|
||||
box: transformBox(f.box, transform),
|
||||
landmarks: f.landmarks.map((p) => transformPoint(p, transform)),
|
||||
probability: f.probability,
|
||||
}));
|
||||
};
|
||||
|
||||
const boxTransformationMatrix = (
|
||||
inBox: Box,
|
||||
toBox: Box,
|
||||
): TransformationMatrix =>
|
||||
compose(
|
||||
translate(toBox.x, toBox.y),
|
||||
scale(toBox.width / inBox.width, toBox.height / inBox.height),
|
||||
);
|
||||
|
||||
const transformPoint = (point: Point, transform: TransformationMatrix) => {
|
||||
const txdPoint = applyToPoint(transform, point);
|
||||
return new Point(txdPoint.x, txdPoint.y);
|
||||
};
|
||||
|
||||
const transformBox = (box: Box, transform: TransformationMatrix) => {
|
||||
const topLeft = transformPoint(new Point(box.x, box.y), transform);
|
||||
const bottomRight = transformPoint(
|
||||
new Point(box.x + box.width, box.y + box.height),
|
||||
transform,
|
||||
);
|
||||
|
||||
return new Box({
|
||||
x: topLeft.x,
|
||||
y: topLeft.y,
|
||||
width: bottomRight.x - topLeft.x,
|
||||
height: bottomRight.y - topLeft.y,
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Remove overlapping faces from an array of face detections through non-maximum
|
||||
* suppression algorithm.
|
||||
*
|
||||
* This function sorts the detections by their probability in descending order,
|
||||
* then iterates over them.
|
||||
*
|
||||
* For each detection, it calculates the Euclidean distance to all other
|
||||
* detections.
|
||||
* For each detection, it calculates the Intersection over Union (IoU) with all
|
||||
* other detections.
|
||||
*
|
||||
* If the distance is less than or equal to the specified threshold
|
||||
* (`withinDistance`), the other detection is considered a duplicate and is
|
||||
* If the IoU is greater than or equal to the specified threshold
|
||||
* (`iouThreshold`), the other detection is considered overlapping and is
|
||||
* removed.
|
||||
*
|
||||
* @param detections - An array of face detections to remove duplicates from.
|
||||
* @param detections - An array of face detections to remove overlapping faces
|
||||
* from.
|
||||
*
|
||||
* @param withinDistance - The maximum Euclidean distance between two detections
|
||||
* for them to be considered duplicates.
|
||||
* @param iouThreshold - The minimum IoU between two detections for them to be
|
||||
* considered overlapping.
|
||||
*
|
||||
* @returns An array of face detections with duplicates removed.
|
||||
* @returns An array of face detections with overlapping faces removed
|
||||
*/
|
||||
const removeDuplicateDetections = (
|
||||
const naiveNonMaxSuppression = (
|
||||
detections: FaceDetection[],
|
||||
withinDistance: number,
|
||||
) => {
|
||||
iouThreshold: number,
|
||||
): FaceDetection[] => {
|
||||
// Sort the detections by score, the highest first.
|
||||
detections.sort((a, b) => b.probability - a.probability);
|
||||
|
||||
const dupIndices = new Set<number>();
|
||||
for (let i = 0; i < detections.length; i++) {
|
||||
if (dupIndices.has(i)) continue;
|
||||
|
||||
// Loop through the detections and calculate the IOU.
|
||||
for (let i = 0; i < detections.length - 1; i++) {
|
||||
for (let j = i + 1; j < detections.length; j++) {
|
||||
if (dupIndices.has(j)) continue;
|
||||
|
||||
const centeri = faceDetectionCenter(detections[i]);
|
||||
const centerj = faceDetectionCenter(detections[j]);
|
||||
const dist = euclidean(
|
||||
[centeri.x, centeri.y],
|
||||
[centerj.x, centerj.y],
|
||||
);
|
||||
|
||||
if (dist <= withinDistance) dupIndices.add(j);
|
||||
const iou = intersectionOverUnion(detections[i], detections[j]);
|
||||
if (iou >= iouThreshold) {
|
||||
detections.splice(j, 1);
|
||||
j--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return detections.filter((_, i) => !dupIndices.has(i));
|
||||
return detections;
|
||||
};
|
||||
|
||||
const faceDetectionCenter = (detection: FaceDetection) => {
|
||||
const center = new Point(0, 0);
|
||||
// TODO-ML(LAURENS): first 4 landmarks is applicable to blazeface only this
|
||||
// needs to consider eyes, nose and mouth landmarks to take center
|
||||
detection.landmarks?.slice(0, 4).forEach((p) => {
|
||||
center.x += p.x;
|
||||
center.y += p.y;
|
||||
});
|
||||
return new Point(center.x / 4, center.y / 4);
|
||||
const intersectionOverUnion = (a: FaceDetection, b: FaceDetection): number => {
|
||||
const intersectionMinX = Math.max(a.box.x, b.box.x);
|
||||
const intersectionMinY = Math.max(a.box.y, b.box.y);
|
||||
const intersectionMaxX = Math.min(
|
||||
a.box.x + a.box.width,
|
||||
b.box.x + b.box.width,
|
||||
);
|
||||
const intersectionMaxY = Math.min(
|
||||
a.box.y + a.box.height,
|
||||
b.box.y + b.box.height,
|
||||
);
|
||||
|
||||
const intersectionWidth = intersectionMaxX - intersectionMinX;
|
||||
const intersectionHeight = intersectionMaxY - intersectionMinY;
|
||||
|
||||
if (intersectionWidth < 0 || intersectionHeight < 0) {
|
||||
return 0.0; // If boxes do not overlap, IoU is 0
|
||||
}
|
||||
|
||||
const areaA = a.box.width * a.box.height;
|
||||
const areaB = b.box.width * b.box.height;
|
||||
|
||||
const intersectionArea = intersectionWidth * intersectionHeight;
|
||||
const unionArea = areaA + areaB - intersectionArea;
|
||||
|
||||
return intersectionArea / unionArea;
|
||||
};
|
||||
|
||||
const makeFaceID = (
|
||||
|
@ -398,12 +471,15 @@ const faceAlignmentUsingSimilarityTransform = (
|
|||
const meanTranslation = simTransform.toMean.sub(0.5).mul(size);
|
||||
const centerMat = simTransform.fromMean.sub(meanTranslation);
|
||||
const center = new Point(centerMat.get(0, 0), centerMat.get(1, 0));
|
||||
const rotation = -Math.atan2(
|
||||
simTransform.rotation.get(0, 1),
|
||||
simTransform.rotation.get(0, 0),
|
||||
);
|
||||
|
||||
return { affineMatrix, center, size, rotation };
|
||||
const boundingBox = new Box({
|
||||
x: center.x - size / 2,
|
||||
y: center.y - size / 2,
|
||||
width: size,
|
||||
height: size,
|
||||
});
|
||||
|
||||
return { affineMatrix, boundingBox };
|
||||
};
|
||||
|
||||
const convertToMobileFaceNetInput = (
|
||||
|
@ -678,35 +754,22 @@ const extractFaceCrop = (
|
|||
imageBitmap: ImageBitmap,
|
||||
alignment: FaceAlignment,
|
||||
): ImageBitmap => {
|
||||
const alignmentBox = new Box({
|
||||
x: alignment.center.x - alignment.size / 2,
|
||||
y: alignment.center.y - alignment.size / 2,
|
||||
width: alignment.size,
|
||||
height: alignment.size,
|
||||
});
|
||||
// TODO-ML: This algorithm is different from what is used by the mobile app.
|
||||
// Also, it needs to be something that can work fully using the embedding we
|
||||
// receive from remote - the `alignment.boundingBox` will not be available
|
||||
// to us in such cases.
|
||||
const paddedBox = roundBox(enlargeBox(alignment.boundingBox, 1.5));
|
||||
const outputSize = { width: paddedBox.width, height: paddedBox.height };
|
||||
|
||||
const padding = 0.25;
|
||||
const scaleForPadding = 1 + padding * 2;
|
||||
const paddedBox = roundBox(enlargeBox(alignmentBox, scaleForPadding));
|
||||
const maxDimension = 256;
|
||||
const scale = Math.min(
|
||||
maxDimension / paddedBox.width,
|
||||
maxDimension / paddedBox.height,
|
||||
);
|
||||
|
||||
// TODO-ML(LAURENS): The rotation doesn't seem to be used? it's set to 0.
|
||||
return cropWithRotation(imageBitmap, paddedBox, 0, 256);
|
||||
};
|
||||
|
||||
const cropWithRotation = (
|
||||
imageBitmap: ImageBitmap,
|
||||
cropBox: Box,
|
||||
rotation: number,
|
||||
maxDimension: number,
|
||||
) => {
|
||||
const box = roundBox(cropBox);
|
||||
|
||||
const outputSize = { width: box.width, height: box.height };
|
||||
|
||||
const scale = Math.min(maxDimension / box.width, maxDimension / box.height);
|
||||
if (scale < 1) {
|
||||
outputSize.width = Math.round(scale * box.width);
|
||||
outputSize.height = Math.round(scale * box.height);
|
||||
outputSize.width = Math.round(scale * paddedBox.width);
|
||||
outputSize.height = Math.round(scale * paddedBox.height);
|
||||
}
|
||||
|
||||
const offscreen = new OffscreenCanvas(outputSize.width, outputSize.height);
|
||||
|
@ -714,7 +777,6 @@ const cropWithRotation = (
|
|||
offscreenCtx.imageSmoothingQuality = "high";
|
||||
|
||||
offscreenCtx.translate(outputSize.width / 2, outputSize.height / 2);
|
||||
rotation && offscreenCtx.rotate(rotation);
|
||||
|
||||
const outputBox = new Box({
|
||||
x: -outputSize.width / 2,
|
||||
|
@ -723,7 +785,7 @@ const cropWithRotation = (
|
|||
height: outputSize.height,
|
||||
});
|
||||
|
||||
const enlargedBox = enlargeBox(box, 1.5);
|
||||
const enlargedBox = enlargeBox(paddedBox, 1.5);
|
||||
const enlargedOutputBox = enlargeBox(outputBox, 1.5);
|
||||
|
||||
offscreenCtx.drawImage(
|
||||
|
|
|
@ -20,16 +20,18 @@ export const putFaceEmbedding = async (
|
|||
const comlinkCryptoWorker = await ComlinkCryptoWorker.getInstance();
|
||||
const { file: encryptedEmbeddingData } =
|
||||
await comlinkCryptoWorker.encryptMetadata(serverMl, enteFile.key);
|
||||
log.info(
|
||||
`putEmbedding embedding to server for file: ${enteFile.metadata.title} fileID: ${enteFile.id}`,
|
||||
);
|
||||
const res = await putEmbedding({
|
||||
// TODO-ML(MR): Do we need any of these fields
|
||||
// log.info(
|
||||
// `putEmbedding embedding to server for file: ${enteFile.metadata.title} fileID: ${enteFile.id}`,
|
||||
// );
|
||||
/*const res =*/ await putEmbedding({
|
||||
fileID: enteFile.id,
|
||||
encryptedEmbedding: encryptedEmbeddingData.encryptedData,
|
||||
decryptionHeader: encryptedEmbeddingData.decryptionHeader,
|
||||
model: "file-ml-clip-face",
|
||||
});
|
||||
log.info("putEmbedding response: ", res);
|
||||
// TODO-ML(MR): Do we need any of these fields
|
||||
// log.info("putEmbedding response: ", res);
|
||||
};
|
||||
|
||||
export interface FileML extends ServerFileMl {
|
||||
|
|
|
@ -1,57 +0,0 @@
|
|||
import { Box, Point } from "services/face/geom";
|
||||
import type { FaceDetection } from "services/face/types";
|
||||
// TODO-ML(LAURENS): Do we need two separate Matrix libraries?
|
||||
//
|
||||
// Keeping this in a separate file so that we can audit this. If these can be
|
||||
// expressed using ml-matrix, then we can move this code to f-index.ts
|
||||
import {
|
||||
Matrix,
|
||||
applyToPoint,
|
||||
compose,
|
||||
scale,
|
||||
translate,
|
||||
} from "transformation-matrix";
|
||||
|
||||
/**
|
||||
* Transform the given {@link faceDetections} from their coordinate system in
|
||||
* which they were detected ({@link inBox}) back to the coordinate system of the
|
||||
* original image ({@link toBox}).
|
||||
*/
|
||||
export const transformFaceDetections = (
|
||||
faceDetections: FaceDetection[],
|
||||
inBox: Box,
|
||||
toBox: Box,
|
||||
): FaceDetection[] => {
|
||||
const transform = boxTransformationMatrix(inBox, toBox);
|
||||
return faceDetections.map((f) => ({
|
||||
box: transformBox(f.box, transform),
|
||||
landmarks: f.landmarks.map((p) => transformPoint(p, transform)),
|
||||
probability: f.probability,
|
||||
}));
|
||||
};
|
||||
|
||||
const boxTransformationMatrix = (inBox: Box, toBox: Box): Matrix =>
|
||||
compose(
|
||||
translate(toBox.x, toBox.y),
|
||||
scale(toBox.width / inBox.width, toBox.height / inBox.height),
|
||||
);
|
||||
|
||||
const transformPoint = (point: Point, transform: Matrix) => {
|
||||
const txdPoint = applyToPoint(transform, point);
|
||||
return new Point(txdPoint.x, txdPoint.y);
|
||||
};
|
||||
|
||||
const transformBox = (box: Box, transform: Matrix) => {
|
||||
const topLeft = transformPoint(new Point(box.x, box.y), transform);
|
||||
const bottomRight = transformPoint(
|
||||
new Point(box.x + box.width, box.y + box.height),
|
||||
transform,
|
||||
);
|
||||
|
||||
return new Box({
|
||||
x: topLeft.x,
|
||||
y: topLeft.y,
|
||||
width: bottomRight.x - topLeft.x,
|
||||
height: bottomRight.y - topLeft.y,
|
||||
});
|
||||
};
|
|
@ -8,13 +8,20 @@ export interface FaceDetection {
|
|||
}
|
||||
|
||||
export interface FaceAlignment {
|
||||
// TODO-ML(MR): remove affine matrix as rotation, size and center
|
||||
// are simple to store and use, affine matrix adds complexity while getting crop
|
||||
/**
|
||||
* An affine transformation matrix (rotation, translation, scaling) to align
|
||||
* the face extracted from the image.
|
||||
*/
|
||||
affineMatrix: number[][];
|
||||
rotation: number;
|
||||
// size and center is relative to image dimentions stored at mlFileData
|
||||
size: number;
|
||||
center: Point;
|
||||
/**
|
||||
* The bounding box of the transformed box.
|
||||
*
|
||||
* The affine transformation shifts the original detection box a new,
|
||||
* transformed, box (possibily rotated). This property is the bounding box
|
||||
* of that transformed box. It is in the coordinate system of the original,
|
||||
* full, image on which the detection occurred.
|
||||
*/
|
||||
boundingBox: Box;
|
||||
}
|
||||
|
||||
export interface Face {
|
||||
|
|
|
@ -177,12 +177,19 @@ some cases.
|
|||
|
||||
## Face search
|
||||
|
||||
- [matrix](https://github.com/mljs/matrix) and
|
||||
[similarity-transformation](https://github.com/shaileshpandit/similarity-transformation-js)
|
||||
are used during face alignment.
|
||||
|
||||
- [transformation-matrix](https://github.com/chrvadala/transformation-matrix)
|
||||
is used during face detection.
|
||||
is used for performing 2D affine transformations using transformation
|
||||
matrices. It is used during face detection.
|
||||
|
||||
- [matrix](https://github.com/mljs/matrix) is mathematical matrix abstraction.
|
||||
It is used alongwith
|
||||
[similarity-transformation](https://github.com/shaileshpandit/similarity-transformation-js)
|
||||
during face alignment.
|
||||
|
||||
> Note that while both `transformation-matrix` and `matrix` are "matrix"
|
||||
> libraries, they have different foci and purposes: `transformation-matrix`
|
||||
> provides affine transforms, while `matrix` is for performing computations
|
||||
> on matrices, say inverting them or performing their decomposition.
|
||||
|
||||
- [hdbscan](https://github.com/shaileshpandit/hdbscan-js) is used for face
|
||||
clustering.
|
||||
|
|
|
@ -428,7 +428,7 @@
|
|||
"USED": "usado",
|
||||
"YOU": "Você",
|
||||
"FAMILY": "Família",
|
||||
"FREE": "grátis",
|
||||
"FREE": "livre",
|
||||
"OF": "de",
|
||||
"WATCHED_FOLDERS": "Pastas monitoradas",
|
||||
"NO_FOLDERS_ADDED": "Nenhuma pasta adicionada ainda!",
|
||||
|
|
|
@ -3528,7 +3528,7 @@ ml-array-rescale@^1.3.7:
|
|||
ml-array-max "^1.2.4"
|
||||
ml-array-min "^1.2.3"
|
||||
|
||||
ml-matrix@^6.10.4:
|
||||
ml-matrix@^6.11:
|
||||
version "6.11.0"
|
||||
resolved "https://registry.yarnpkg.com/ml-matrix/-/ml-matrix-6.11.0.tgz#3cf2260ef04cbb8e0e0425e71d200f5cbcf82772"
|
||||
integrity sha512-7jr9NmFRkaUxbKslfRu3aZOjJd2LkSitCGv+QH9PF0eJoEG7jIpjXra1Vw8/kgao8+kHCSsJONG6vfWmXQ+/Eg==
|
||||
|
@ -4628,7 +4628,7 @@ tr46@~0.0.3:
|
|||
resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
|
||||
integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==
|
||||
|
||||
transformation-matrix@^2.15.0:
|
||||
transformation-matrix@^2.16:
|
||||
version "2.16.1"
|
||||
resolved "https://registry.yarnpkg.com/transformation-matrix/-/transformation-matrix-2.16.1.tgz#4a2de06331b94ae953193d1b9a5ba002ec5f658a"
|
||||
integrity sha512-tdtC3wxVEuzU7X/ydL131Q3JU5cPMEn37oqVLITjRDSDsnSHVFzW2JiCLfZLIQEgWzZHdSy3J6bZzvKEN24jGA==
|
||||
|
|