Remove old file handling
This commit is contained in:
parent
8a071fd45b
commit
ad684c46c3
|
@ -19,7 +19,7 @@ export class DedicatedMLWorker implements MachineLearningWorker {
|
||||||
enteFile: EnteFile,
|
enteFile: EnteFile,
|
||||||
localFile: globalThis.File,
|
localFile: globalThis.File,
|
||||||
) {
|
) {
|
||||||
return mlService.syncLocalFile(token, userID, enteFile, localFile);
|
mlService.syncLocalFile(token, userID, enteFile, localFile);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async sync(token: string, userID: number) {
|
public async sync(token: string, userID: number) {
|
||||||
|
|
|
@ -197,7 +197,6 @@ export interface MLSearchConfig {
|
||||||
export interface MLSyncContext {
|
export interface MLSyncContext {
|
||||||
token: string;
|
token: string;
|
||||||
userID: number;
|
userID: number;
|
||||||
shouldUpdateMLVersion: boolean;
|
|
||||||
|
|
||||||
faceDetectionService: FaceDetectionService;
|
faceDetectionService: FaceDetectionService;
|
||||||
faceCropService: FaceCropService;
|
faceCropService: FaceCropService;
|
||||||
|
@ -281,7 +280,7 @@ export interface MachineLearningWorker {
|
||||||
userID: number,
|
userID: number,
|
||||||
enteFile: EnteFile,
|
enteFile: EnteFile,
|
||||||
localFile: globalThis.File,
|
localFile: globalThis.File,
|
||||||
): Promise<MlFileData | Error>;
|
);
|
||||||
|
|
||||||
sync(token: string, userID: number): Promise<MLSyncResult>;
|
sync(token: string, userID: number): Promise<MLSyncResult>;
|
||||||
|
|
||||||
|
|
|
@ -8,7 +8,6 @@ import {
|
||||||
MLSyncContext,
|
MLSyncContext,
|
||||||
MLSyncFileContext,
|
MLSyncFileContext,
|
||||||
type FaceAlignment,
|
type FaceAlignment,
|
||||||
type Versioned,
|
|
||||||
} from "services/face/types";
|
} from "services/face/types";
|
||||||
import { imageBitmapToBlob, warpAffineFloat32List } from "utils/image";
|
import { imageBitmapToBlob, warpAffineFloat32List } from "utils/image";
|
||||||
import { clusterFaces } from "../face/cluster";
|
import { clusterFaces } from "../face/cluster";
|
||||||
|
@ -24,36 +23,12 @@ class FaceService {
|
||||||
syncContext: MLSyncContext,
|
syncContext: MLSyncContext,
|
||||||
fileContext: MLSyncFileContext,
|
fileContext: MLSyncFileContext,
|
||||||
) {
|
) {
|
||||||
const { oldMlFile, newMlFile } = fileContext;
|
const { newMlFile } = fileContext;
|
||||||
if (
|
|
||||||
!isDifferentOrOld(
|
|
||||||
oldMlFile?.faceDetectionMethod,
|
|
||||||
syncContext.faceDetectionService.method,
|
|
||||||
) &&
|
|
||||||
oldMlFile?.imageSource === "Original"
|
|
||||||
) {
|
|
||||||
newMlFile.faces = oldMlFile?.faces?.map((existingFace) => ({
|
|
||||||
id: existingFace.id,
|
|
||||||
fileId: existingFace.fileId,
|
|
||||||
detection: existingFace.detection,
|
|
||||||
}));
|
|
||||||
|
|
||||||
newMlFile.imageSource = oldMlFile.imageSource;
|
|
||||||
newMlFile.imageDimensions = oldMlFile.imageDimensions;
|
|
||||||
newMlFile.faceDetectionMethod = oldMlFile.faceDetectionMethod;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
newMlFile.faceDetectionMethod = syncContext.faceDetectionService.method;
|
newMlFile.faceDetectionMethod = syncContext.faceDetectionService.method;
|
||||||
fileContext.newDetection = true;
|
fileContext.newDetection = true;
|
||||||
const imageBitmap = await fetchImageBitmapForContext(fileContext);
|
const imageBitmap = await fetchImageBitmapForContext(fileContext);
|
||||||
const timerId = `faceDetection-${fileContext.enteFile.id}`;
|
|
||||||
console.time(timerId);
|
|
||||||
const faceDetections =
|
const faceDetections =
|
||||||
await syncContext.faceDetectionService.detectFaces(imageBitmap);
|
await syncContext.faceDetectionService.detectFaces(imageBitmap);
|
||||||
console.timeEnd(timerId);
|
|
||||||
console.log("faceDetections: ", faceDetections?.length);
|
|
||||||
|
|
||||||
// TODO: reenable faces filtering based on width
|
// TODO: reenable faces filtering based on width
|
||||||
const detectedFaces = faceDetections?.map((detection) => {
|
const detectedFaces = faceDetections?.map((detection) => {
|
||||||
return {
|
return {
|
||||||
|
@ -75,23 +50,7 @@ class FaceService {
|
||||||
syncContext: MLSyncContext,
|
syncContext: MLSyncContext,
|
||||||
fileContext: MLSyncFileContext,
|
fileContext: MLSyncFileContext,
|
||||||
) {
|
) {
|
||||||
const { oldMlFile, newMlFile } = fileContext;
|
const { newMlFile } = fileContext;
|
||||||
if (
|
|
||||||
// !syncContext.config.faceCrop.enabled ||
|
|
||||||
!fileContext.newDetection &&
|
|
||||||
!isDifferentOrOld(
|
|
||||||
oldMlFile?.faceCropMethod,
|
|
||||||
syncContext.faceCropService.method,
|
|
||||||
) &&
|
|
||||||
areFaceIdsSame(newMlFile.faces, oldMlFile?.faces)
|
|
||||||
) {
|
|
||||||
for (const [index, face] of newMlFile.faces.entries()) {
|
|
||||||
face.crop = oldMlFile.faces[index].crop;
|
|
||||||
}
|
|
||||||
newMlFile.faceCropMethod = oldMlFile.faceCropMethod;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const imageBitmap = await fetchImageBitmapForContext(fileContext);
|
const imageBitmap = await fetchImageBitmapForContext(fileContext);
|
||||||
newMlFile.faceCropMethod = syncContext.faceCropService.method;
|
newMlFile.faceCropMethod = syncContext.faceCropService.method;
|
||||||
|
|
||||||
|
@ -104,24 +63,7 @@ class FaceService {
|
||||||
syncContext: MLSyncContext,
|
syncContext: MLSyncContext,
|
||||||
fileContext: MLSyncFileContext,
|
fileContext: MLSyncFileContext,
|
||||||
): Promise<Float32Array> {
|
): Promise<Float32Array> {
|
||||||
const { oldMlFile, newMlFile } = fileContext;
|
const { newMlFile } = fileContext;
|
||||||
// TODO-ML(MR):
|
|
||||||
const method = {
|
|
||||||
value: "ArcFace",
|
|
||||||
version: 1,
|
|
||||||
};
|
|
||||||
if (
|
|
||||||
!fileContext.newDetection &&
|
|
||||||
!isDifferentOrOld(oldMlFile?.faceAlignmentMethod, method) &&
|
|
||||||
areFaceIdsSame(newMlFile.faces, oldMlFile?.faces)
|
|
||||||
) {
|
|
||||||
for (const [index, face] of newMlFile.faces.entries()) {
|
|
||||||
face.alignment = oldMlFile.faces[index].alignment;
|
|
||||||
}
|
|
||||||
newMlFile.faceAlignmentMethod = oldMlFile.faceAlignmentMethod;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
newMlFile.faceAlignmentMethod = {
|
newMlFile.faceAlignmentMethod = {
|
||||||
value: "ArcFace",
|
value: "ArcFace",
|
||||||
version: 1,
|
version: 1,
|
||||||
|
@ -159,22 +101,7 @@ class FaceService {
|
||||||
fileContext: MLSyncFileContext,
|
fileContext: MLSyncFileContext,
|
||||||
alignedFacesInput: Float32Array,
|
alignedFacesInput: Float32Array,
|
||||||
) {
|
) {
|
||||||
const { oldMlFile, newMlFile } = fileContext;
|
const { newMlFile } = fileContext;
|
||||||
if (
|
|
||||||
!fileContext.newAlignment &&
|
|
||||||
!isDifferentOrOld(
|
|
||||||
oldMlFile?.faceEmbeddingMethod,
|
|
||||||
syncContext.faceEmbeddingService.method,
|
|
||||||
) &&
|
|
||||||
areFaceIdsSame(newMlFile.faces, oldMlFile?.faces)
|
|
||||||
) {
|
|
||||||
for (const [index, face] of newMlFile.faces.entries()) {
|
|
||||||
face.embedding = oldMlFile.faces[index].embedding;
|
|
||||||
}
|
|
||||||
newMlFile.faceEmbeddingMethod = oldMlFile.faceEmbeddingMethod;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
newMlFile.faceEmbeddingMethod = syncContext.faceEmbeddingService.method;
|
newMlFile.faceEmbeddingMethod = syncContext.faceEmbeddingService.method;
|
||||||
// TODO: when not storing face crops, image will be needed to extract faces
|
// TODO: when not storing face crops, image will be needed to extract faces
|
||||||
// fileContext.imageBitmap ||
|
// fileContext.imageBitmap ||
|
||||||
|
@ -193,17 +120,7 @@ class FaceService {
|
||||||
syncContext: MLSyncContext,
|
syncContext: MLSyncContext,
|
||||||
fileContext: MLSyncFileContext,
|
fileContext: MLSyncFileContext,
|
||||||
) {
|
) {
|
||||||
const { oldMlFile, newMlFile } = fileContext;
|
const { newMlFile } = fileContext;
|
||||||
if (
|
|
||||||
!fileContext.newAlignment &&
|
|
||||||
!isDifferentOrOld(
|
|
||||||
oldMlFile?.faceEmbeddingMethod,
|
|
||||||
syncContext.faceEmbeddingService.method,
|
|
||||||
) &&
|
|
||||||
areFaceIdsSame(newMlFile.faces, oldMlFile?.faces)
|
|
||||||
) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
for (let i = 0; i < newMlFile.faces.length; i++) {
|
for (let i = 0; i < newMlFile.faces.length; i++) {
|
||||||
const face = newMlFile.faces[i];
|
const face = newMlFile.faces[i];
|
||||||
if (face.detection.box.x + face.detection.box.width < 2) continue; // Skip if somehow already relative
|
if (face.detection.box.x + face.detection.box.width < 2) continue; // Skip if somehow already relative
|
||||||
|
@ -298,39 +215,6 @@ class FaceService {
|
||||||
|
|
||||||
export default new FaceService();
|
export default new FaceService();
|
||||||
|
|
||||||
export function areFaceIdsSame(ofFaces: Array<Face>, toFaces: Array<Face>) {
|
|
||||||
if (
|
|
||||||
(ofFaces === null || ofFaces === undefined) &&
|
|
||||||
(toFaces === null || toFaces === undefined)
|
|
||||||
) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return primitiveArrayEquals(
|
|
||||||
ofFaces?.map((f) => f.id),
|
|
||||||
toFaces?.map((f) => f.id),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function primitiveArrayEquals(a, b) {
|
|
||||||
return (
|
|
||||||
Array.isArray(a) &&
|
|
||||||
Array.isArray(b) &&
|
|
||||||
a.length === b.length &&
|
|
||||||
a.every((val, index) => val === b[index])
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function isDifferentOrOld(
|
|
||||||
method: Versioned<string>,
|
|
||||||
thanMethod: Versioned<string>,
|
|
||||||
) {
|
|
||||||
return (
|
|
||||||
!method ||
|
|
||||||
method.value !== thanMethod.value ||
|
|
||||||
method.version < thanMethod.version
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function extractFaceImagesToFloat32(
|
async function extractFaceImagesToFloat32(
|
||||||
faceAlignments: Array<FaceAlignment>,
|
faceAlignments: Array<FaceAlignment>,
|
||||||
faceSize: number,
|
faceSize: number,
|
||||||
|
|
|
@ -157,7 +157,6 @@ export class MLFactory {
|
||||||
export class LocalMLSyncContext implements MLSyncContext {
|
export class LocalMLSyncContext implements MLSyncContext {
|
||||||
public token: string;
|
public token: string;
|
||||||
public userID: number;
|
public userID: number;
|
||||||
public shouldUpdateMLVersion: boolean;
|
|
||||||
|
|
||||||
public faceDetectionService: FaceDetectionService;
|
public faceDetectionService: FaceDetectionService;
|
||||||
public faceCropService: FaceCropService;
|
public faceCropService: FaceCropService;
|
||||||
|
@ -184,15 +183,9 @@ export class LocalMLSyncContext implements MLSyncContext {
|
||||||
>;
|
>;
|
||||||
private enteWorkers: Array<any>;
|
private enteWorkers: Array<any>;
|
||||||
|
|
||||||
constructor(
|
constructor(token: string, userID: number, concurrency?: number) {
|
||||||
token: string,
|
|
||||||
userID: number,
|
|
||||||
shouldUpdateMLVersion: boolean = true,
|
|
||||||
concurrency?: number,
|
|
||||||
) {
|
|
||||||
this.token = token;
|
this.token = token;
|
||||||
this.userID = userID;
|
this.userID = userID;
|
||||||
this.shouldUpdateMLVersion = shouldUpdateMLVersion;
|
|
||||||
|
|
||||||
this.faceDetectionService =
|
this.faceDetectionService =
|
||||||
MLFactory.getFaceDetectionService("YoloFace");
|
MLFactory.getFaceDetectionService("YoloFace");
|
||||||
|
@ -424,7 +417,7 @@ class MachineLearningService {
|
||||||
|
|
||||||
// TODO-ML(MR): Keep as promise for now.
|
// TODO-ML(MR): Keep as promise for now.
|
||||||
this.syncContext = new Promise((resolve) => {
|
this.syncContext = new Promise((resolve) => {
|
||||||
resolve(new LocalMLSyncContext(token, userID, true));
|
resolve(new LocalMLSyncContext(token, userID));
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
log.info("reusing existing syncContext");
|
log.info("reusing existing syncContext");
|
||||||
|
@ -433,11 +426,12 @@ class MachineLearningService {
|
||||||
}
|
}
|
||||||
|
|
||||||
private async getLocalSyncContext(token: string, userID: number) {
|
private async getLocalSyncContext(token: string, userID: number) {
|
||||||
|
// TODO-ML(MR): This is updating the file ML version. verify.
|
||||||
if (!this.localSyncContext) {
|
if (!this.localSyncContext) {
|
||||||
log.info("Creating localSyncContext");
|
log.info("Creating localSyncContext");
|
||||||
// TODO-ML(MR):
|
// TODO-ML(MR):
|
||||||
this.localSyncContext = new Promise((resolve) => {
|
this.localSyncContext = new Promise((resolve) => {
|
||||||
resolve(new LocalMLSyncContext(token, userID, false));
|
resolve(new LocalMLSyncContext(token, userID));
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
log.info("reusing existing localSyncContext");
|
log.info("reusing existing localSyncContext");
|
||||||
|
@ -459,11 +453,11 @@ class MachineLearningService {
|
||||||
userID: number,
|
userID: number,
|
||||||
enteFile: EnteFile,
|
enteFile: EnteFile,
|
||||||
localFile?: globalThis.File,
|
localFile?: globalThis.File,
|
||||||
): Promise<MlFileData | Error> {
|
) {
|
||||||
const syncContext = await this.getLocalSyncContext(token, userID);
|
const syncContext = await this.getLocalSyncContext(token, userID);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const mlFileData = await this.syncFileWithErrorHandler(
|
await this.syncFileWithErrorHandler(
|
||||||
syncContext,
|
syncContext,
|
||||||
enteFile,
|
enteFile,
|
||||||
localFile,
|
localFile,
|
||||||
|
@ -473,10 +467,8 @@ class MachineLearningService {
|
||||||
await this.closeLocalSyncContext();
|
await this.closeLocalSyncContext();
|
||||||
}
|
}
|
||||||
// await syncContext.dispose();
|
// await syncContext.dispose();
|
||||||
return mlFileData;
|
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.error("Error while syncing local file: ", enteFile.id, e);
|
console.error("Error while syncing local file: ", enteFile.id, e);
|
||||||
return e;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -484,7 +476,7 @@ class MachineLearningService {
|
||||||
syncContext: MLSyncContext,
|
syncContext: MLSyncContext,
|
||||||
enteFile: EnteFile,
|
enteFile: EnteFile,
|
||||||
localFile?: globalThis.File,
|
localFile?: globalThis.File,
|
||||||
): Promise<MlFileData> {
|
) {
|
||||||
try {
|
try {
|
||||||
console.log(
|
console.log(
|
||||||
`Indexing ${enteFile.title ?? "<untitled>"} ${enteFile.id}`,
|
`Indexing ${enteFile.title ?? "<untitled>"} ${enteFile.id}`,
|
||||||
|
@ -533,22 +525,13 @@ class MachineLearningService {
|
||||||
) {
|
) {
|
||||||
console.log("Syncing for file" + enteFile.title);
|
console.log("Syncing for file" + enteFile.title);
|
||||||
const fileContext: MLSyncFileContext = { enteFile, localFile };
|
const fileContext: MLSyncFileContext = { enteFile, localFile };
|
||||||
const oldMlFile =
|
const oldMlFile = await this.getMLFileData(enteFile.id);
|
||||||
(fileContext.oldMlFile = await this.getMLFileData(enteFile.id)) ??
|
if (oldMlFile) {
|
||||||
this.newMlData(enteFile.id);
|
return oldMlFile;
|
||||||
if (
|
|
||||||
fileContext.oldMlFile?.mlVersion === defaultMLVersion
|
|
||||||
// TODO: reset mlversion of all files when user changes image source
|
|
||||||
) {
|
|
||||||
return fileContext.oldMlFile;
|
|
||||||
}
|
}
|
||||||
const newMlFile = (fileContext.newMlFile = this.newMlData(enteFile.id));
|
|
||||||
|
|
||||||
if (syncContext.shouldUpdateMLVersion) {
|
const newMlFile = (fileContext.newMlFile = this.newMlData(enteFile.id));
|
||||||
newMlFile.mlVersion = defaultMLVersion;
|
newMlFile.mlVersion = defaultMLVersion;
|
||||||
} else if (fileContext.oldMlFile?.mlVersion) {
|
|
||||||
newMlFile.mlVersion = fileContext.oldMlFile.mlVersion;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await fetchImageBitmapForContext(fileContext);
|
await fetchImageBitmapForContext(fileContext);
|
||||||
|
@ -628,6 +611,7 @@ class MachineLearningService {
|
||||||
public async syncIndex(syncContext: MLSyncContext) {
|
public async syncIndex(syncContext: MLSyncContext) {
|
||||||
await this.getMLLibraryData(syncContext);
|
await this.getMLLibraryData(syncContext);
|
||||||
|
|
||||||
|
// TODO-ML(MR): Ensure this doesn't run until fixed.
|
||||||
await syncPeopleIndex(syncContext);
|
await syncPeopleIndex(syncContext);
|
||||||
|
|
||||||
await this.persistMLLibraryData(syncContext);
|
await this.persistMLLibraryData(syncContext);
|
||||||
|
|
|
@ -5,8 +5,8 @@ import { eventBus, Events } from "@ente/shared/events";
|
||||||
import { getToken, getUserID } from "@ente/shared/storage/localStorage/helpers";
|
import { getToken, getUserID } from "@ente/shared/storage/localStorage/helpers";
|
||||||
import debounce from "debounce";
|
import debounce from "debounce";
|
||||||
import PQueue from "p-queue";
|
import PQueue from "p-queue";
|
||||||
import mlIDbStorage from "services/face/db";
|
|
||||||
import { createFaceComlinkWorker } from "services/face";
|
import { createFaceComlinkWorker } from "services/face";
|
||||||
|
import mlIDbStorage from "services/face/db";
|
||||||
import type { DedicatedMLWorker } from "services/face/face.worker";
|
import type { DedicatedMLWorker } from "services/face/face.worker";
|
||||||
import { MLSyncResult } from "services/face/types";
|
import { MLSyncResult } from "services/face/types";
|
||||||
import { EnteFile } from "types/file";
|
import { EnteFile } from "types/file";
|
||||||
|
@ -232,19 +232,13 @@ class MLWorkManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
public async syncLocalFile(enteFile: EnteFile, localFile: globalThis.File) {
|
public async syncLocalFile(enteFile: EnteFile, localFile: globalThis.File) {
|
||||||
const result = await this.liveSyncQueue.add(async () => {
|
await this.liveSyncQueue.add(async () => {
|
||||||
this.stopSyncJob();
|
this.stopSyncJob();
|
||||||
const token = getToken();
|
const token = getToken();
|
||||||
const userID = getUserID();
|
const userID = getUserID();
|
||||||
const mlWorker = await this.getLiveSyncWorker();
|
const mlWorker = await this.getLiveSyncWorker();
|
||||||
return mlWorker.syncLocalFile(token, userID, enteFile, localFile);
|
return mlWorker.syncLocalFile(token, userID, enteFile, localFile);
|
||||||
});
|
});
|
||||||
|
|
||||||
if (result instanceof Error) {
|
|
||||||
// TODO: redirect/refresh to gallery in case of session_expired
|
|
||||||
// may not be required as uploader should anyways take care of this
|
|
||||||
console.error("Error while syncing local file: ", result);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Sync Job
|
// Sync Job
|
||||||
|
|
Loading…
Reference in a new issue