diff --git a/src/services/machineLearning/machineLearningService.ts b/src/services/machineLearning/machineLearningService.ts index a36a51366..78c6ed8e2 100644 --- a/src/services/machineLearning/machineLearningService.ts +++ b/src/services/machineLearning/machineLearningService.ts @@ -278,8 +278,7 @@ class MachineLearningService { syncContext.syncedFiles.push(outOfSyncfile); } ); - syncContext.syncQueue.on('error', (error) => { - console.error('syncQueue: onError: ', error); + syncContext.syncQueue.on('error', () => { syncContext.syncQueue.clear(); }); await syncContext.syncQueue.addAll(functions); diff --git a/src/services/machineLearning/mlWorkManager.ts b/src/services/machineLearning/mlWorkManager.ts index 7f1da3b66..3455155fd 100644 --- a/src/services/machineLearning/mlWorkManager.ts +++ b/src/services/machineLearning/mlWorkManager.ts @@ -4,6 +4,7 @@ import { eventBus, Events } from 'services/events'; import { File, FILE_TYPE } from 'services/fileService'; import { FACE_CROPS_CACHE_NAME, MLSyncConfig } from 'types/machineLearning'; import { getToken } from 'utils/common/key'; +import { logQueueStats } from 'utils/machineLearning'; import { getMLSyncJobConfig } from 'utils/machineLearning/config'; import { MLWorkerWithProxy } from 'utils/machineLearning/worker'; import { logError } from 'utils/sentry'; @@ -11,6 +12,7 @@ import mlIDbStorage from 'utils/storage/mlIDbStorage'; import { MLSyncJobResult, MLSyncJob } from './mlSyncJob'; const LIVE_SYNC_IDLE_DEBOUNCE_SEC = 30; +const LIVE_SYNC_QUEUE_TIMEOUT_SEC = 300; const LOCAL_FILES_UPDATED_DEBOUNCE_SEC = 30; class MLWorkManager { @@ -21,7 +23,13 @@ class MLWorkManager { private liveSyncWorker: MLWorkerWithProxy; constructor() { - this.liveSyncQueue = new PQueue({ concurrency: 1 }); + this.liveSyncQueue = new PQueue({ + concurrency: 1, + // TODO: temp, remove + timeout: LIVE_SYNC_QUEUE_TIMEOUT_SEC * 1000, + throwOnTimeout: true, + }); + logQueueStats(this.liveSyncQueue, 'livesync'); const debouncedLiveSyncIdle = debounce( () => this.onLiveSyncIdle(), @@ -82,9 +90,10 @@ class MLWorkManager { } try { await this.syncLocalFile(arg.enteFile, arg.localFile); - } catch (e) { - // console.error(e); - logError(e, 'Failed in ML fileUploaded Handler'); + } catch (error) { + console.error('Error in syncLocalFile', error); + this.liveSyncQueue.clear(); + // logError(e, 'Failed in ML fileUploaded Handler'); } } diff --git a/src/utils/machineLearning/index.ts b/src/utils/machineLearning/index.ts index 24330ba0f..e3c1439c0 100644 --- a/src/utils/machineLearning/index.ts +++ b/src/utils/machineLearning/index.ts @@ -514,9 +514,13 @@ export function getNearestPointIndex( } export function logQueueStats(queue: PQueue, name: string) { - queue.on('active', () => { + queue.on('active', () => console.log( - `queuestats: ${name}: Working on next item. Size: ${queue.size} Pending: ${queue.pending}` - ); - }); + `queuestats: ${name}: Active, Size: ${queue.size} Pending: ${queue.pending}` + ) + ); + queue.on('idle', () => console.log(`queuestats: ${name}: Idle`)); + queue.on('error', (error) => + console.error(`queuestats: ${name}: Error, `, error) + ); } diff --git a/src/utils/machineLearning/worker.ts b/src/utils/machineLearning/worker.ts index 5df9d41cc..f1b3f9fd7 100644 --- a/src/utils/machineLearning/worker.ts +++ b/src/utils/machineLearning/worker.ts @@ -15,6 +15,9 @@ export class MLWorkerWithProxy { new URL('worker/machineLearning.worker', import.meta.url), { name: 'ml-worker' } ); + this.worker.onerror = (errorEvent) => { + console.error('Got error event from worker', errorEvent); + }; console.log('Initiated ml-worker'); const comlink = wrap(this.worker); this.proxy = new comlink();