reuse same syncJobWorker instead of terminating
This commit is contained in:
parent
e38c663e1a
commit
5c93a7ccea
|
@ -42,6 +42,7 @@ class MachineLearningService {
|
||||||
// private clusteringService: ClusteringService;
|
// private clusteringService: ClusteringService;
|
||||||
|
|
||||||
private localSyncContext: Promise<MLSyncContext>;
|
private localSyncContext: Promise<MLSyncContext>;
|
||||||
|
private syncContext: Promise<MLSyncContext>;
|
||||||
|
|
||||||
public constructor() {
|
public constructor() {
|
||||||
// setWasmPaths('/js/tfjs/');
|
// setWasmPaths('/js/tfjs/');
|
||||||
|
@ -64,12 +65,7 @@ class MachineLearningService {
|
||||||
// needs to be cleaned using tf.dispose or tf.tidy
|
// needs to be cleaned using tf.dispose or tf.tidy
|
||||||
// tf.engine().startScope();
|
// tf.engine().startScope();
|
||||||
|
|
||||||
const mlSyncConfig = await getMLSyncConfig();
|
const syncContext = await this.getSyncContext(token);
|
||||||
const syncContext = MLFactory.getMLSyncContext(
|
|
||||||
token,
|
|
||||||
mlSyncConfig,
|
|
||||||
true
|
|
||||||
);
|
|
||||||
|
|
||||||
await this.syncLocalFiles(syncContext);
|
await this.syncLocalFiles(syncContext);
|
||||||
|
|
||||||
|
@ -81,13 +77,13 @@ class MachineLearningService {
|
||||||
|
|
||||||
// TODO: running index before all files are on latest ml version
|
// TODO: running index before all files are on latest ml version
|
||||||
// may be need to just take synced files on latest ml version for indexing
|
// may be need to just take synced files on latest ml version for indexing
|
||||||
if (
|
// if (
|
||||||
syncContext.outOfSyncFiles.length <= 0 ||
|
// syncContext.outOfSyncFiles.length <= 0 ||
|
||||||
(syncContext.nSyncedFiles === syncContext.config.batchSize &&
|
// (syncContext.nSyncedFiles === syncContext.config.batchSize &&
|
||||||
Math.random() < 0.2)
|
// Math.random() < 0.2)
|
||||||
) {
|
// ) {
|
||||||
await this.syncIndex(syncContext);
|
// await this.syncIndex(syncContext);
|
||||||
}
|
// }
|
||||||
|
|
||||||
// tf.engine().endScope();
|
// tf.engine().endScope();
|
||||||
|
|
||||||
|
@ -109,7 +105,7 @@ class MachineLearningService {
|
||||||
};
|
};
|
||||||
// console.log('[MLService] sync results: ', mlSyncResult);
|
// console.log('[MLService] sync results: ', mlSyncResult);
|
||||||
|
|
||||||
await syncContext.dispose();
|
// await syncContext.dispose();
|
||||||
console.log('Final TF Memory stats: ', tf.memory());
|
console.log('Final TF Memory stats: ', tf.memory());
|
||||||
|
|
||||||
return mlSyncResult;
|
return mlSyncResult;
|
||||||
|
@ -295,14 +291,28 @@ class MachineLearningService {
|
||||||
// await this.disposeMLModels();
|
// await this.disposeMLModels();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async getSyncContext(token: string) {
|
||||||
|
if (!this.syncContext) {
|
||||||
|
console.log('Creating syncContext');
|
||||||
|
|
||||||
|
this.syncContext = getMLSyncConfig().then((mlSyncConfig) =>
|
||||||
|
MLFactory.getMLSyncContext(token, mlSyncConfig, true)
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
console.log('reusing existing syncContext');
|
||||||
|
}
|
||||||
|
return this.syncContext;
|
||||||
|
}
|
||||||
|
|
||||||
private async getLocalSyncContext(token: string) {
|
private async getLocalSyncContext(token: string) {
|
||||||
if (!this.localSyncContext) {
|
if (!this.localSyncContext) {
|
||||||
console.log('Creating localSyncContext');
|
console.log('Creating localSyncContext');
|
||||||
this.localSyncContext = getMLSyncConfig().then((mlSyncConfig) =>
|
this.localSyncContext = getMLSyncConfig().then((mlSyncConfig) =>
|
||||||
MLFactory.getMLSyncContext(token, mlSyncConfig, false)
|
MLFactory.getMLSyncContext(token, mlSyncConfig, false)
|
||||||
);
|
);
|
||||||
|
} else {
|
||||||
|
console.log('reusing existing localSyncContext');
|
||||||
}
|
}
|
||||||
|
|
||||||
return this.localSyncContext;
|
return this.localSyncContext;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -396,14 +406,17 @@ class MachineLearningService {
|
||||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||||
textDetectionTimeoutIndex?: number
|
textDetectionTimeoutIndex?: number
|
||||||
) {
|
) {
|
||||||
|
console.log('syncFile called');
|
||||||
const fileContext: MLSyncFileContext = { enteFile, localFile };
|
const fileContext: MLSyncFileContext = { enteFile, localFile };
|
||||||
const oldMlFile =
|
const oldMlFile =
|
||||||
(fileContext.oldMlFile = await this.getMLFileData(enteFile.id)) ??
|
(fileContext.oldMlFile = await this.getMLFileData(enteFile.id)) ??
|
||||||
this.newMlData(enteFile.id);
|
this.newMlData(enteFile.id);
|
||||||
|
console.log('======1===========');
|
||||||
if (
|
if (
|
||||||
fileContext.oldMlFile?.mlVersion === syncContext.config.mlVersion
|
fileContext.oldMlFile?.mlVersion === syncContext.config.mlVersion
|
||||||
// TODO: reset mlversion of all files when user changes image source
|
// TODO: reset mlversion of all files when user changes image source
|
||||||
) {
|
) {
|
||||||
|
console.log('======2===========');
|
||||||
return fileContext.oldMlFile;
|
return fileContext.oldMlFile;
|
||||||
}
|
}
|
||||||
const newMlFile = (fileContext.newMlFile = this.newMlData(enteFile.id));
|
const newMlFile = (fileContext.newMlFile = this.newMlData(enteFile.id));
|
||||||
|
@ -416,18 +429,24 @@ class MachineLearningService {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await ReaderService.getImageBitmap(syncContext, fileContext);
|
await ReaderService.getImageBitmap(syncContext, fileContext);
|
||||||
await Promise.all([
|
console.log('ReaderService done');
|
||||||
this.syncFaceDetections(syncContext, fileContext),
|
// await this.syncFaceDetections(syncContext, fileContext);
|
||||||
ObjectService.syncFileObjectDetections(
|
await ObjectService.syncFileObjectDetections(
|
||||||
syncContext,
|
syncContext,
|
||||||
fileContext
|
fileContext
|
||||||
),
|
);
|
||||||
// TextService.syncFileTextDetections(
|
// await Promise.all([
|
||||||
|
// this.syncFaceDetections(syncContext, fileContext),
|
||||||
|
// ObjectService.syncFileObjectDetections(
|
||||||
// syncContext,
|
// syncContext,
|
||||||
// fileContext,
|
// fileContext
|
||||||
// textDetectionTimeoutIndex
|
|
||||||
// ),
|
// ),
|
||||||
]);
|
// // TextService.syncFileTextDetections(
|
||||||
|
// // syncContext,
|
||||||
|
// // fileContext,
|
||||||
|
// // textDetectionTimeoutIndex
|
||||||
|
// // ),
|
||||||
|
// ]);
|
||||||
newMlFile.errorCount = 0;
|
newMlFile.errorCount = 0;
|
||||||
newMlFile.lastErrorMessage = undefined;
|
newMlFile.lastErrorMessage = undefined;
|
||||||
await this.persistMLFileData(syncContext, newMlFile);
|
await this.persistMLFileData(syncContext, newMlFile);
|
||||||
|
|
|
@ -217,7 +217,7 @@ class MLWorkManager {
|
||||||
|
|
||||||
const mlSyncResult = await jobWorkerProxy.sync(token);
|
const mlSyncResult = await jobWorkerProxy.sync(token);
|
||||||
|
|
||||||
this.terminateSyncJobWorker();
|
// this.terminateSyncJobWorker();
|
||||||
const jobResult: MLSyncJobResult = {
|
const jobResult: MLSyncJobResult = {
|
||||||
shouldBackoff:
|
shouldBackoff:
|
||||||
!!mlSyncResult.error || mlSyncResult.nOutOfSyncFiles < 1,
|
!!mlSyncResult.error || mlSyncResult.nOutOfSyncFiles < 1,
|
||||||
|
|
Loading…
Reference in a new issue