beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

improve syncEmbeddings logic

Browse source
This commit is contained in:
Abhinav 2024-01-11 11:14:40 +05:30
parent 521474f952
commit f033605248
1 changed files with 77 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
apps/photos/src/services/embeddingService.ts
View file

@ -44,11 +44,19 @@ export const getLocalEmbeddings = async (model: Model) => {
return embeddings.filter((embedding) => embedding.model === model);
};
const getEmbeddingSyncTime = async () => {
return (await localForage.getItem<number>(EMBEDDING_SYNC_TIME_TABLE)) ?? 0;
const getModelEmbeddingSyncTime = async (model: Model) => {
return (
(await localForage.getItem<number>(
`${model}-${EMBEDDING_SYNC_TIME_TABLE}`
)) ?? 0
);
};
export const syncEmbeddings = async (model: Model = Model.ONNX_CLIP) => {
const setModelEmbeddingSyncTime = async (model: Model, time: number) => {
await localForage.setItem(`${model}-${EMBEDDING_SYNC_TIME_TABLE}`, time);
};
export const syncEmbeddings = async (models: Model[] = [Model.ONNX_CLIP]) => {
try {
let allEmbeddings = await getAllLocalEmbeddings();
const localFiles = await getAllLocalFiles();
@ -61,11 +69,14 @@ export const syncEmbeddings = async (model: Model = Model.ONNX_CLIP) => {
});
await cleanupDeletedEmbeddings(allLocalFiles, allEmbeddings);
addLogLine(`Syncing embeddings localCount: ${allEmbeddings.length}`);
let sinceTime = await getEmbeddingSyncTime();
addLogLine(`Syncing embeddings sinceTime: ${sinceTime}`);
for (const model of models) {
let modelLastSinceTime = await getModelEmbeddingSyncTime(model);
addLogLine(
`Syncing ${model} model's embeddings sinceTime: ${modelLastSinceTime}`
);
let response: GetEmbeddingDiffResponse;
do {
response = await getEmbeddingsDiff(sinceTime, model);
response = await getEmbeddingsDiff(modelLastSinceTime, model);
if (!response.diff?.length) {
return;
}
@ -77,8 +88,11 @@ export const syncEmbeddings = async (model: Model = Model.ONNX_CLIP) => {
decryptionHeader,
...rest
} = embedding;
const worker = await ComlinkCryptoWorker.getInstance();
const fileKey = fileIdToKeyMap.get(embedding.fileID);
const worker =
await ComlinkCryptoWorker.getInstance();
const fileKey = fileIdToKeyMap.get(
embedding.fileID
);
if (!fileKey) {
throw Error(CustomError.FILE_NOT_FOUND);
}
@ -95,12 +109,17 @@ export const syncEmbeddings = async (model: Model = Model.ONNX_CLIP) => {
} catch (e) {
let info: Record<string, unknown>;
if (e.message === CustomError.FILE_NOT_FOUND) {
const hasHiddenAlbums = hiddenAlbums?.length > 0;
const hasHiddenAlbums =
hiddenAlbums?.length > 0;
info = {
hasHiddenAlbums,
};
}
logError(e, 'decryptEmbedding failed for file', info);
logError(
e,
'decryptEmbedding failed for file',
info
);
}
})
);
@ -109,14 +128,15 @@ export const syncEmbeddings = async (model: Model = Model.ONNX_CLIP) => {
...newEmbeddings,
]);
if (response.diff.length) {
sinceTime = response.diff.slice(-1)[0].updatedAt;
modelLastSinceTime = response.diff.slice(-1)[0].updatedAt;
}
await localForage.setItem(EMBEDDINGS_TABLE, allEmbeddings);
await localForage.setItem(EMBEDDING_SYNC_TIME_TABLE, sinceTime);
await setModelEmbeddingSyncTime(model, modelLastSinceTime);
addLogLine(
`Syncing embeddings syncedEmbeddingsCount: ${allEmbeddings.length}`
);
} while (response.diff.length === DIFF_LIMIT);
}
} catch (e) {
logError(e, 'Sync embeddings failed');
}