beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add API batching (#971)

Browse source
This commit is contained in:
Abhinav Kumar 2023-03-06 15:23:45 +05:30 committed by GitHub
parent d1a7de9e42 57b487c2df
commit 07c51dd5d0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 105 additions and 111 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/constants/api/index.ts Normal file
View file

@ -0,0 +1 @@
export const REQUEST_BATCH_SIZE = 1000;

1
src/constants/file/index.ts
View file

@ -3,7 +3,6 @@ export const MAX_EDITED_CREATION_TIME = new Date();
export const MAX_EDITED_FILE_NAME_LENGTH = 100; export const MAX_EDITED_FILE_NAME_LENGTH = 100;
export const MAX_CAPTION_SIZE = 5000; export const MAX_CAPTION_SIZE = 5000;
export const MAX_TRASH_BATCH_SIZE = 1000;
export const TYPE_HEIC = 'heic'; export const TYPE_HEIC = 'heic';
export const TYPE_HEIF = 'heif'; export const TYPE_HEIF = 'heif';

121
src/services/collectionService.ts
View file

@ -66,6 +66,8 @@ import {
} from 'utils/collection'; } from 'utils/collection';
import ComlinkCryptoWorker from 'utils/comlink/ComlinkCryptoWorker'; import ComlinkCryptoWorker from 'utils/comlink/ComlinkCryptoWorker';
import { getLocalFiles } from './fileService'; import { getLocalFiles } from './fileService';
import { REQUEST_BATCH_SIZE } from 'constants/api';
import { batch } from 'utils/common';
const ENDPOINT = getEndpoint(); const ENDPOINT = getEndpoint();
const COLLECTION_TABLE = 'collections'; const COLLECTION_TABLE = 'collections';
@ -405,21 +407,24 @@ export const addToCollection = async (
) => { ) => {
try { try {
const token = getToken(); const token = getToken();
const fileKeysEncryptedWithNewCollection = const batchedFiles = batch(files, REQUEST_BATCH_SIZE);
await encryptWithNewCollectionKey(collection, files); for (const batch of batchedFiles) {
const fileKeysEncryptedWithNewCollection =
await encryptWithNewCollectionKey(collection, batch);
const requestBody: AddToCollectionRequest = { const requestBody: AddToCollectionRequest = {
collectionID: collection.id, collectionID: collection.id,
files: fileKeysEncryptedWithNewCollection, files: fileKeysEncryptedWithNewCollection,
}; };
await HTTPService.post( await HTTPService.post(
`${ENDPOINT}/collections/add-files`, `${ENDPOINT}/collections/add-files`,
requestBody, requestBody,
null, null,
{ {
'X-Auth-Token': token, 'X-Auth-Token': token,
} }
); );
}
} catch (e) { } catch (e) {
logError(e, 'Add to collection Failed '); logError(e, 'Add to collection Failed ');
throw e; throw e;
@ -432,21 +437,24 @@ export const restoreToCollection = async (
) => { ) => {
try { try {
const token = getToken(); const token = getToken();
const fileKeysEncryptedWithNewCollection = const batchedFiles = batch(files, REQUEST_BATCH_SIZE);
await encryptWithNewCollectionKey(collection, files); for (const batch of batchedFiles) {
const fileKeysEncryptedWithNewCollection =
await encryptWithNewCollectionKey(collection, batch);
const requestBody: AddToCollectionRequest = { const requestBody: AddToCollectionRequest = {
collectionID: collection.id, collectionID: collection.id,
files: fileKeysEncryptedWithNewCollection, files: fileKeysEncryptedWithNewCollection,
}; };
await HTTPService.post( await HTTPService.post(
`${ENDPOINT}/collections/restore-files`, `${ENDPOINT}/collections/restore-files`,
requestBody, requestBody,
null, null,
{ {
'X-Auth-Token': token, 'X-Auth-Token': token,
} }
); );
}
} catch (e) { } catch (e) {
logError(e, 'restore to collection Failed '); logError(e, 'restore to collection Failed ');
throw e; throw e;
@ -459,22 +467,25 @@ export const moveToCollection = async (
) => { ) => {
try { try {
const token = getToken(); const token = getToken();
const fileKeysEncryptedWithNewCollection = const batchedFiles = batch(files, REQUEST_BATCH_SIZE);
await encryptWithNewCollectionKey(toCollection, files); for (const batch of batchedFiles) {
const fileKeysEncryptedWithNewCollection =
await encryptWithNewCollectionKey(toCollection, batch);
const requestBody: MoveToCollectionRequest = { const requestBody: MoveToCollectionRequest = {
fromCollectionID: fromCollectionID, fromCollectionID: fromCollectionID,
toCollectionID: toCollection.id, toCollectionID: toCollection.id,
files: fileKeysEncryptedWithNewCollection, files: fileKeysEncryptedWithNewCollection,
}; };
await HTTPService.post( await HTTPService.post(
`${ENDPOINT}/collections/move-files`, `${ENDPOINT}/collections/move-files`,
requestBody, requestBody,
null, null,
{ {
'X-Auth-Token': token, 'X-Auth-Token': token,
} }
); );
}
} catch (e) { } catch (e) {
logError(e, 'move to collection Failed '); logError(e, 'move to collection Failed ');
throw e; throw e;
@ -605,18 +616,20 @@ export const removeNonUserFiles = async (
try { try {
const fileIDs = nonUserFiles.map((f) => f.id); const fileIDs = nonUserFiles.map((f) => f.id);
const token = getToken(); const token = getToken();
const batchedFileIDs = batch(fileIDs, REQUEST_BATCH_SIZE);
for (const batch of batchedFileIDs) {
const request: RemoveFromCollectionRequest = {
collectionID,
fileIDs: batch,
};
const request: RemoveFromCollectionRequest = { await HTTPService.post(
collectionID, `${ENDPOINT}/collections/v3/remove-files`,
fileIDs, request,
}; null,
{ 'X-Auth-Token': token }
await HTTPService.post( );
`${ENDPOINT}/collections/v3/remove-files`, }
request,
null,
{ 'X-Auth-Token': token }
);
} catch (e) { } catch (e) {
logError(e, 'remove non user files failed '); logError(e, 'remove non user files failed ');
throw e; throw e;

85
src/services/fileService.ts
View file

@ -14,7 +14,6 @@ import {
import { eventBus, Events } from './events'; import { eventBus, Events } from './events';
import { EnteFile, EncryptedEnteFile, TrashRequest } from 'types/file'; import { EnteFile, EncryptedEnteFile, TrashRequest } from 'types/file';
import { SetFiles } from 'types/gallery'; import { SetFiles } from 'types/gallery';
import { MAX_TRASH_BATCH_SIZE } from 'constants/file';
import { BulkUpdateMagicMetadataRequest } from 'types/magicMetadata'; import { BulkUpdateMagicMetadataRequest } from 'types/magicMetadata';
import { addLogLine } from 'utils/logging'; import { addLogLine } from 'utils/logging';
import { isCollectionHidden } from 'utils/collection'; import { isCollectionHidden } from 'utils/collection';
@ -24,6 +23,8 @@ import {
getCollectionLastSyncTime, getCollectionLastSyncTime,
setCollectionLastSyncTime, setCollectionLastSyncTime,
} from './collectionService'; } from './collectionService';
import { REQUEST_BATCH_SIZE } from 'constants/api';
import { batch } from 'utils/common';
const ENDPOINT = getEndpoint(); const ENDPOINT = getEndpoint();
const FILES_TABLE = 'files'; const FILES_TABLE = 'files';
@ -161,24 +162,22 @@ export const trashFiles = async (filesToTrash: EnteFile[]) => {
if (!token) { if (!token) {
return; return;
} }
const batchedFilesToTrash = batch(filesToTrash, REQUEST_BATCH_SIZE);
const trashBatch: TrashRequest = { for (const batch of batchedFilesToTrash) {
items: [], const trashRequest: TrashRequest = {
}; items: batch.map((file) => ({
fileID: file.id,
for (const file of filesToTrash) { collectionID: file.collectionID,
trashBatch.items.push({ })),
collectionID: file.collectionID, };
fileID: file.id, await HTTPService.post(
}); `${ENDPOINT}/files/trash`,
if (trashBatch.items.length >= MAX_TRASH_BATCH_SIZE) { trashRequest,
await trashFilesFromServer(trashBatch, token); null,
trashBatch.items = []; {
} 'X-Auth-Token': token,
} }
);
if (trashBatch.items.length > 0) {
await trashFilesFromServer(trashBatch, token);
} }
} catch (e) { } catch (e) {
logError(e, 'trash file failed'); logError(e, 'trash file failed');
@ -192,16 +191,17 @@ export const deleteFromTrash = async (filesToDelete: number[]) => {
if (!token) { if (!token) {
return; return;
} }
let deleteBatch: number[] = []; const batchedFilesToDelete = batch(filesToDelete, REQUEST_BATCH_SIZE);
for (const fileID of filesToDelete) {
deleteBatch.push(fileID); for (const batch of batchedFilesToDelete) {
if (deleteBatch.length >= MAX_TRASH_BATCH_SIZE) { await HTTPService.post(
await deleteBatchFromTrash(token, deleteBatch); `${ENDPOINT}/trash/delete`,
deleteBatch = []; { fileIDs: batch },
} null,
} {
if (deleteBatch.length > 0) { 'X-Auth-Token': token,
await deleteBatchFromTrash(token, deleteBatch); }
);
} }
} catch (e) { } catch (e) {
logError(e, 'deleteFromTrash failed'); logError(e, 'deleteFromTrash failed');
@ -209,22 +209,6 @@ export const deleteFromTrash = async (filesToDelete: number[]) => {
} }
}; };
const deleteBatchFromTrash = async (token: string, deleteBatch: number[]) => {
try {
await HTTPService.post(
`${ENDPOINT}/trash/delete`,
{ fileIDs: deleteBatch },
null,
{
'X-Auth-Token': token,
}
);
} catch (e) {
logError(e, 'deleteBatchFromTrash failed');
throw e;
}
};
export const updateFileMagicMetadata = async (files: EnteFile[]) => { export const updateFileMagicMetadata = async (files: EnteFile[]) => {
const token = getToken(); const token = getToken();
if (!token) { if (!token) {
@ -303,14 +287,3 @@ export const updateFilePublicMagicMetadata = async (files: EnteFile[]) => {
}) })
); );
}; };
async function trashFilesFromServer(trashBatch: TrashRequest, token: any) {
try {
await HTTPService.post(`${ENDPOINT}/files/trash`, trashBatch, null, {
'X-Auth-Token': token,
});
} catch (e) {
logError(e, 'trash files from server failed');
throw e;
}
}

8
src/utils/common/index.ts
View file

@ -125,3 +125,11 @@ function isPromise(p: any) {
export function isClipboardItemPresent() { export function isClipboardItemPresent() {
return typeof ClipboardItem !== 'undefined'; return typeof ClipboardItem !== 'undefined';
} }
export function batch<T>(arr: T[], batchSize: number): T[][] {
const batches: T[][] = [];
for (let i = 0; i < arr.length; i += batchSize) {
batches.push(arr.slice(i, i + batchSize));
}
return batches;
}