beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

reuse same syncJobWorker instead of terminating

Browse source
This commit is contained in:
Abhinav 2022-08-19 15:19:02 +05:30
parent e38c663e1a
commit 5c93a7ccea
2 changed files with 47 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

73
src/services/machineLearning/machineLearningService.ts
View file

@ -42,6 +42,7 @@ class MachineLearningService {
// private clusteringService: ClusteringService; // private clusteringService: ClusteringService;
private localSyncContext: Promise<MLSyncContext>; private localSyncContext: Promise<MLSyncContext>;
private syncContext: Promise<MLSyncContext>;
public constructor() { public constructor() {
// setWasmPaths('/js/tfjs/'); // setWasmPaths('/js/tfjs/');
@ -64,12 +65,7 @@ class MachineLearningService {
// needs to be cleaned using tf.dispose or tf.tidy // needs to be cleaned using tf.dispose or tf.tidy
// tf.engine().startScope(); // tf.engine().startScope();
const mlSyncConfig = await getMLSyncConfig(); const syncContext = await this.getSyncContext(token);
const syncContext = MLFactory.getMLSyncContext(
token,
mlSyncConfig,
true
);
await this.syncLocalFiles(syncContext); await this.syncLocalFiles(syncContext);
@ -81,13 +77,13 @@ class MachineLearningService {
// TODO: running index before all files are on latest ml version // TODO: running index before all files are on latest ml version
// may be need to just take synced files on latest ml version for indexing // may be need to just take synced files on latest ml version for indexing
if ( // if (
syncContext.outOfSyncFiles.length <= 0 || // syncContext.outOfSyncFiles.length <= 0 ||
(syncContext.nSyncedFiles === syncContext.config.batchSize && // (syncContext.nSyncedFiles === syncContext.config.batchSize &&
Math.random() < 0.2) // Math.random() < 0.2)
) { // ) {
await this.syncIndex(syncContext); // await this.syncIndex(syncContext);
} // }
// tf.engine().endScope(); // tf.engine().endScope();
@ -109,7 +105,7 @@ class MachineLearningService {
}; };
// console.log('[MLService] sync results: ', mlSyncResult); // console.log('[MLService] sync results: ', mlSyncResult);
await syncContext.dispose(); // await syncContext.dispose();
console.log('Final TF Memory stats: ', tf.memory()); console.log('Final TF Memory stats: ', tf.memory());
return mlSyncResult; return mlSyncResult;
@ -295,14 +291,28 @@ class MachineLearningService {
// await this.disposeMLModels(); // await this.disposeMLModels();
} }
private async getSyncContext(token: string) {
if (!this.syncContext) {
console.log('Creating syncContext');
this.syncContext = getMLSyncConfig().then((mlSyncConfig) =>
MLFactory.getMLSyncContext(token, mlSyncConfig, true)
);
} else {
console.log('reusing existing syncContext');
}
return this.syncContext;
}
private async getLocalSyncContext(token: string) { private async getLocalSyncContext(token: string) {
if (!this.localSyncContext) { if (!this.localSyncContext) {
console.log('Creating localSyncContext'); console.log('Creating localSyncContext');
this.localSyncContext = getMLSyncConfig().then((mlSyncConfig) => this.localSyncContext = getMLSyncConfig().then((mlSyncConfig) =>
MLFactory.getMLSyncContext(token, mlSyncConfig, false) MLFactory.getMLSyncContext(token, mlSyncConfig, false)
); );
} else {
console.log('reusing existing localSyncContext');
} }
return this.localSyncContext; return this.localSyncContext;
} }
@ -396,14 +406,17 @@ class MachineLearningService {
// eslint-disable-next-line @typescript-eslint/no-unused-vars // eslint-disable-next-line @typescript-eslint/no-unused-vars
textDetectionTimeoutIndex?: number textDetectionTimeoutIndex?: number
) { ) {
console.log('syncFile called');
const fileContext: MLSyncFileContext = { enteFile, localFile }; const fileContext: MLSyncFileContext = { enteFile, localFile };
const oldMlFile = const oldMlFile =
(fileContext.oldMlFile = await this.getMLFileData(enteFile.id)) ?? (fileContext.oldMlFile = await this.getMLFileData(enteFile.id)) ??
this.newMlData(enteFile.id); this.newMlData(enteFile.id);
console.log('======1===========');
if ( if (
fileContext.oldMlFile?.mlVersion === syncContext.config.mlVersion fileContext.oldMlFile?.mlVersion === syncContext.config.mlVersion
// TODO: reset mlversion of all files when user changes image source // TODO: reset mlversion of all files when user changes image source
) { ) {
console.log('======2===========');
return fileContext.oldMlFile; return fileContext.oldMlFile;
} }
const newMlFile = (fileContext.newMlFile = this.newMlData(enteFile.id)); const newMlFile = (fileContext.newMlFile = this.newMlData(enteFile.id));
@ -416,18 +429,24 @@ class MachineLearningService {
try { try {
await ReaderService.getImageBitmap(syncContext, fileContext); await ReaderService.getImageBitmap(syncContext, fileContext);
await Promise.all([ console.log('ReaderService done');
this.syncFaceDetections(syncContext, fileContext), // await this.syncFaceDetections(syncContext, fileContext);
ObjectService.syncFileObjectDetections( await ObjectService.syncFileObjectDetections(
syncContext, syncContext,
fileContext fileContext
), );
// TextService.syncFileTextDetections( // await Promise.all([
// syncContext, // this.syncFaceDetections(syncContext, fileContext),
// fileContext, // ObjectService.syncFileObjectDetections(
// textDetectionTimeoutIndex // syncContext,
// ), // fileContext
]); // ),
// // TextService.syncFileTextDetections(
// // syncContext,
// // fileContext,
// // textDetectionTimeoutIndex
// // ),
// ]);
newMlFile.errorCount = 0; newMlFile.errorCount = 0;
newMlFile.lastErrorMessage = undefined; newMlFile.lastErrorMessage = undefined;
await this.persistMLFileData(syncContext, newMlFile); await this.persistMLFileData(syncContext, newMlFile);

2
src/services/machineLearning/mlWorkManager.ts
View file

@ -217,7 +217,7 @@ class MLWorkManager {
const mlSyncResult = await jobWorkerProxy.sync(token); const mlSyncResult = await jobWorkerProxy.sync(token);
this.terminateSyncJobWorker(); // this.terminateSyncJobWorker();
const jobResult: MLSyncJobResult = { const jobResult: MLSyncJobResult = {
shouldBackoff: shouldBackoff:
!!mlSyncResult.error || mlSyncResult.nOutOfSyncFiles < 1, !!mlSyncResult.error || mlSyncResult.nOutOfSyncFiles < 1,