Sync people index only after new files are synced

Maintain files and people latest versions

src/services/machineLearning/machineLearningService.ts

@@ -23,7 +23,13 @@ import {
 import ClusteringService from './clusteringService';
 
 import { toTSNE } from 'utils/machineLearning/visualization';
-import { mlFilesStore, mlPeopleStore } from 'utils/storage/localForage';
+import {
+    getIndexVersion,
+    incrementIndexVersion,
+    mlFilesStore,
+    mlPeopleStore,
+    setIndexVersion,
+} from 'utils/storage/mlStorage';
 import ArcfaceAlignmentService from './arcfaceAlignmentService';
 import {
     findFirstIfSorted,
@@ -154,6 +160,8 @@ class MachineLearningService {
             }
         }
         console.log('allFaces: ', syncContext.syncedFaces);
+
+        await incrementIndexVersion('files');
         // await this.disposeMLModels();
     }
 
@@ -246,11 +254,21 @@ class MachineLearningService {
     }
 
     private async syncPeopleIndex(syncContext: MLSyncContext) {
+        const filesVersion = await getIndexVersion('files');
+        if (filesVersion <= (await getIndexVersion('people'))) {
+            console.log(
+                '[MLService] Skipping people index as already synced to latest version'
+            );
+            return;
+        }
+
         const allFacesMap = await this.getAllSyncedFacesMap(syncContext);
         const allFaces = getAllFacesFromMap(allFacesMap);
 
         await this.runFaceClustering(syncContext, allFaces);
         await this.syncPeopleFromClusters(syncContext, allFacesMap, allFaces);
+
+        await setIndexVersion('people', filesVersion);
     }
 
     private async getAllSyncedFacesMap(syncContext: MLSyncContext) {
@@ -277,7 +295,7 @@ class MachineLearningService {
 
         if (!allFaces || allFaces.length < clusteringConfig.minInputSize) {
             console.log(
-                'Too few faces to cluster, not running clustering: ',
+                '[MLService] Too few faces to cluster, not running clustering: ',
                 allFaces.length
             );
             return;

src/utils/machineLearning/index.ts

@@ -6,7 +6,7 @@ import { File, getLocalFiles } from 'services/fileService';
 import { Box, Point } from '../../../thirdparty/face-api/classes';
 import { Face, MlFileData, MLSyncConfig, Person } from './types';
 import { extractFaceImage } from './faceAlign';
-import { mlFilesStore, mlPeopleStore } from 'utils/storage/localForage';
+import { mlFilesStore, mlPeopleStore } from 'utils/storage/mlStorage';
 
 export function f32Average(descriptors: Float32Array[]) {
     if (descriptors.length < 1) {
@@ -229,7 +229,7 @@ export function findFirstIfSorted<T>(
 }
 
 export const DEFAULT_ML_SYNC_CONFIG: MLSyncConfig = {
-    syncIntervalSec: 5, // 20
+    syncIntervalSec: 30,
     batchSize: 200,
     faceDetection: {
         method: {

src/utils/machineLearning/types.ts

@@ -206,6 +206,8 @@ export class MLSyncContext {
     }
 }
 
+export declare type MLIndex = 'files' | 'people';
+
 export interface FaceDetectionService {
     init(): Promise<void>;
     detectFaces(image: tf.Tensor3D): Promise<Array<DetectedFace>>;

src/utils/storage/localForage.ts

@@ -11,18 +11,4 @@ if (runningInBrowser()) {
     });
 }
 
-export const mlFilesStore = localForage.createInstance({
-    driver: localForage.INDEXEDDB,
-    name: 'ml-data',
-    version: 1.0,
-    storeName: 'files',
-});
-
-export const mlPeopleStore = localForage.createInstance({
-    driver: localForage.INDEXEDDB,
-    name: 'ml-data',
-    version: 1.0,
-    storeName: 'people',
-});
-
 export default localForage;

src/utils/storage/mlStorage.ts

@@ -0,0 +1,51 @@
+import { MLIndex } from 'utils/machineLearning/types';
+import localForage from './localForage';
+
+export const mlFilesStore = localForage.createInstance({
+    driver: localForage.INDEXEDDB,
+    name: 'ml-data',
+    version: 1.0,
+    storeName: 'files',
+});
+
+export const mlPeopleStore = localForage.createInstance({
+    driver: localForage.INDEXEDDB,
+    name: 'ml-data',
+    version: 1.0,
+    storeName: 'people',
+});
+
+export const mlVersionStore = localForage.createInstance({
+    driver: localForage.INDEXEDDB,
+    name: 'ml-data',
+    version: 1.0,
+    storeName: 'versions',
+});
+
+export async function getIndexVersion(index: MLIndex): Promise<number> {
+    return ((await mlVersionStore.getItem(`${index}`)) as number) || 0;
+}
+
+export async function setIndexVersion(
+    index: MLIndex,
+    version: number
+): Promise<number> {
+    await mlVersionStore.setItem(`${index}`, version);
+
+    return version;
+}
+
+export async function incrementIndexVersion(index: MLIndex): Promise<number> {
+    let currentVersion = await getIndexVersion(index);
+    currentVersion = currentVersion + 1;
+    await setIndexVersion(index, currentVersion);
+
+    return currentVersion;
+}
+
+export async function isVersionOutdated(index: MLIndex, thanIndex: MLIndex) {
+    const indexVersion = await getIndexVersion(index);
+    const thanIndexVersion = await getIndexVersion(thanIndex);
+
+    return indexVersion < thanIndexVersion;
+}