From 3ba38ac71a058058657af4ef7d1a6619ab7ce128 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 6 Mar 2023 14:49:11 +0530 Subject: [PATCH 1/3] add api batch size constant and batch util --- src/constants/api/index.ts | 1 + src/constants/file/index.ts | 1 - src/utils/common/index.ts | 8 ++++++++ 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 src/constants/api/index.ts diff --git a/src/constants/api/index.ts b/src/constants/api/index.ts new file mode 100644 index 000000000..17571f7f3 --- /dev/null +++ b/src/constants/api/index.ts @@ -0,0 +1 @@ +export const REQUEST_BATCH_SIZE = 1000; diff --git a/src/constants/file/index.ts b/src/constants/file/index.ts index 57c5efab1..3054fb68d 100644 --- a/src/constants/file/index.ts +++ b/src/constants/file/index.ts @@ -3,7 +3,6 @@ export const MAX_EDITED_CREATION_TIME = new Date(); export const MAX_EDITED_FILE_NAME_LENGTH = 100; export const MAX_CAPTION_SIZE = 5000; -export const MAX_TRASH_BATCH_SIZE = 1000; export const TYPE_HEIC = 'heic'; export const TYPE_HEIF = 'heif'; diff --git a/src/utils/common/index.ts b/src/utils/common/index.ts index 2da2a0792..81a1280ae 100644 --- a/src/utils/common/index.ts +++ b/src/utils/common/index.ts @@ -125,3 +125,11 @@ function isPromise(p: any) { export function isClipboardItemPresent() { return typeof ClipboardItem !== 'undefined'; } + +export function batch(arr: T[], batchSize: number): T[][] { + const batches: T[][] = []; + for (let i = 0; i < arr.length; i += batchSize) { + batches.push(arr.slice(i, i + batchSize)); + } + return batches; +} From 882003eccecd4606f4d8c7ca9fad18ef1416d9e1 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 6 Mar 2023 14:55:43 +0530 Subject: [PATCH 2/3] update needed API to use batch util --- src/services/collectionService.ts | 121 +++++++++++++++++------------- src/services/fileService.ts | 73 +++++------------- 2 files changed, 84 insertions(+), 110 deletions(-) diff --git a/src/services/collectionService.ts b/src/services/collectionService.ts index b4f627313..902f614df 100644 --- a/src/services/collectionService.ts +++ b/src/services/collectionService.ts @@ -66,6 +66,8 @@ import { } from 'utils/collection'; import ComlinkCryptoWorker from 'utils/comlink/ComlinkCryptoWorker'; import { getLocalFiles } from './fileService'; +import { REQUEST_BATCH_SIZE } from 'constants/api'; +import { batch } from 'utils/common'; const ENDPOINT = getEndpoint(); const COLLECTION_TABLE = 'collections'; @@ -405,21 +407,24 @@ export const addToCollection = async ( ) => { try { const token = getToken(); - const fileKeysEncryptedWithNewCollection = - await encryptWithNewCollectionKey(collection, files); + const batchedFiles = batch(files, REQUEST_BATCH_SIZE); + for (const batch of batchedFiles) { + const fileKeysEncryptedWithNewCollection = + await encryptWithNewCollectionKey(collection, batch); - const requestBody: AddToCollectionRequest = { - collectionID: collection.id, - files: fileKeysEncryptedWithNewCollection, - }; - await HTTPService.post( - `${ENDPOINT}/collections/add-files`, - requestBody, - null, - { - 'X-Auth-Token': token, - } - ); + const requestBody: AddToCollectionRequest = { + collectionID: collection.id, + files: fileKeysEncryptedWithNewCollection, + }; + await HTTPService.post( + `${ENDPOINT}/collections/add-files`, + requestBody, + null, + { + 'X-Auth-Token': token, + } + ); + } } catch (e) { logError(e, 'Add to collection Failed '); throw e; @@ -432,21 +437,24 @@ export const restoreToCollection = async ( ) => { try { const token = getToken(); - const fileKeysEncryptedWithNewCollection = - await encryptWithNewCollectionKey(collection, files); + const batchedFiles = batch(files, REQUEST_BATCH_SIZE); + for (const batch of batchedFiles) { + const fileKeysEncryptedWithNewCollection = + await encryptWithNewCollectionKey(collection, batch); - const requestBody: AddToCollectionRequest = { - collectionID: collection.id, - files: fileKeysEncryptedWithNewCollection, - }; - await HTTPService.post( - `${ENDPOINT}/collections/restore-files`, - requestBody, - null, - { - 'X-Auth-Token': token, - } - ); + const requestBody: AddToCollectionRequest = { + collectionID: collection.id, + files: fileKeysEncryptedWithNewCollection, + }; + await HTTPService.post( + `${ENDPOINT}/collections/restore-files`, + requestBody, + null, + { + 'X-Auth-Token': token, + } + ); + } } catch (e) { logError(e, 'restore to collection Failed '); throw e; @@ -459,22 +467,25 @@ export const moveToCollection = async ( ) => { try { const token = getToken(); - const fileKeysEncryptedWithNewCollection = - await encryptWithNewCollectionKey(toCollection, files); + const batchedFiles = batch(files, REQUEST_BATCH_SIZE); + for (const batch of batchedFiles) { + const fileKeysEncryptedWithNewCollection = + await encryptWithNewCollectionKey(toCollection, batch); - const requestBody: MoveToCollectionRequest = { - fromCollectionID: fromCollectionID, - toCollectionID: toCollection.id, - files: fileKeysEncryptedWithNewCollection, - }; - await HTTPService.post( - `${ENDPOINT}/collections/move-files`, - requestBody, - null, - { - 'X-Auth-Token': token, - } - ); + const requestBody: MoveToCollectionRequest = { + fromCollectionID: fromCollectionID, + toCollectionID: toCollection.id, + files: fileKeysEncryptedWithNewCollection, + }; + await HTTPService.post( + `${ENDPOINT}/collections/move-files`, + requestBody, + null, + { + 'X-Auth-Token': token, + } + ); + } } catch (e) { logError(e, 'move to collection Failed '); throw e; @@ -605,18 +616,20 @@ export const removeNonUserFiles = async ( try { const fileIDs = nonUserFiles.map((f) => f.id); const token = getToken(); + const batchedFileIDs = batch(fileIDs, REQUEST_BATCH_SIZE); + for (const batch of batchedFileIDs) { + const request: RemoveFromCollectionRequest = { + collectionID, + fileIDs: batch, + }; - const request: RemoveFromCollectionRequest = { - collectionID, - fileIDs, - }; - - await HTTPService.post( - `${ENDPOINT}/collections/v3/remove-files`, - request, - null, - { 'X-Auth-Token': token } - ); + await HTTPService.post( + `${ENDPOINT}/collections/v3/remove-files`, + request, + null, + { 'X-Auth-Token': token } + ); + } } catch (e) { logError(e, 'remove non user files failed '); throw e; diff --git a/src/services/fileService.ts b/src/services/fileService.ts index 763512e00..a11f9eb93 100644 --- a/src/services/fileService.ts +++ b/src/services/fileService.ts @@ -12,9 +12,8 @@ import { sortFiles, } from 'utils/file'; import { eventBus, Events } from './events'; -import { EnteFile, EncryptedEnteFile, TrashRequest } from 'types/file'; +import { EnteFile, EncryptedEnteFile } from 'types/file'; import { SetFiles } from 'types/gallery'; -import { MAX_TRASH_BATCH_SIZE } from 'constants/file'; import { BulkUpdateMagicMetadataRequest } from 'types/magicMetadata'; import { addLogLine } from 'utils/logging'; import { isCollectionHidden } from 'utils/collection'; @@ -24,6 +23,8 @@ import { getCollectionLastSyncTime, setCollectionLastSyncTime, } from './collectionService'; +import { REQUEST_BATCH_SIZE } from 'constants/api'; +import { batch } from 'utils/common'; const ENDPOINT = getEndpoint(); const FILES_TABLE = 'files'; @@ -161,24 +162,11 @@ export const trashFiles = async (filesToTrash: EnteFile[]) => { if (!token) { return; } - - const trashBatch: TrashRequest = { - items: [], - }; - - for (const file of filesToTrash) { - trashBatch.items.push({ - collectionID: file.collectionID, - fileID: file.id, + const batchedFilesToTrash = batch(filesToTrash, REQUEST_BATCH_SIZE); + for (const batch of batchedFilesToTrash) { + await HTTPService.post(`${ENDPOINT}/files/trash`, batch, null, { + 'X-Auth-Token': token, }); - if (trashBatch.items.length >= MAX_TRASH_BATCH_SIZE) { - await trashFilesFromServer(trashBatch, token); - trashBatch.items = []; - } - } - - if (trashBatch.items.length > 0) { - await trashFilesFromServer(trashBatch, token); } } catch (e) { logError(e, 'trash file failed'); @@ -192,16 +180,16 @@ export const deleteFromTrash = async (filesToDelete: number[]) => { if (!token) { return; } - let deleteBatch: number[] = []; - for (const fileID of filesToDelete) { - deleteBatch.push(fileID); - if (deleteBatch.length >= MAX_TRASH_BATCH_SIZE) { - await deleteBatchFromTrash(token, deleteBatch); - deleteBatch = []; - } - } - if (deleteBatch.length > 0) { - await deleteBatchFromTrash(token, deleteBatch); + const batchedFilesToDelete = batch(filesToDelete, REQUEST_BATCH_SIZE); + for (const batch of batchedFilesToDelete) { + await HTTPService.post( + `${ENDPOINT}/trash/delete`, + { fileIDs: batch }, + null, + { + 'X-Auth-Token': token, + } + ); } } catch (e) { logError(e, 'deleteFromTrash failed'); @@ -209,22 +197,6 @@ export const deleteFromTrash = async (filesToDelete: number[]) => { } }; -const deleteBatchFromTrash = async (token: string, deleteBatch: number[]) => { - try { - await HTTPService.post( - `${ENDPOINT}/trash/delete`, - { fileIDs: deleteBatch }, - null, - { - 'X-Auth-Token': token, - } - ); - } catch (e) { - logError(e, 'deleteBatchFromTrash failed'); - throw e; - } -}; - export const updateFileMagicMetadata = async (files: EnteFile[]) => { const token = getToken(); if (!token) { @@ -303,14 +275,3 @@ export const updateFilePublicMagicMetadata = async (files: EnteFile[]) => { }) ); }; - -async function trashFilesFromServer(trashBatch: TrashRequest, token: any) { - try { - await HTTPService.post(`${ENDPOINT}/files/trash`, trashBatch, null, { - 'X-Auth-Token': token, - }); - } catch (e) { - logError(e, 'trash files from server failed'); - throw e; - } -} From 57b487c2dfa2a9ffed7273bb51ed716fd9e22b88 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 6 Mar 2023 15:10:13 +0530 Subject: [PATCH 3/3] fix trashFiles --- src/services/fileService.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/services/fileService.ts b/src/services/fileService.ts index a11f9eb93..68d8b7b19 100644 --- a/src/services/fileService.ts +++ b/src/services/fileService.ts @@ -12,7 +12,7 @@ import { sortFiles, } from 'utils/file'; import { eventBus, Events } from './events'; -import { EnteFile, EncryptedEnteFile } from 'types/file'; +import { EnteFile, EncryptedEnteFile, TrashRequest } from 'types/file'; import { SetFiles } from 'types/gallery'; import { BulkUpdateMagicMetadataRequest } from 'types/magicMetadata'; import { addLogLine } from 'utils/logging'; @@ -164,9 +164,20 @@ export const trashFiles = async (filesToTrash: EnteFile[]) => { } const batchedFilesToTrash = batch(filesToTrash, REQUEST_BATCH_SIZE); for (const batch of batchedFilesToTrash) { - await HTTPService.post(`${ENDPOINT}/files/trash`, batch, null, { - 'X-Auth-Token': token, - }); + const trashRequest: TrashRequest = { + items: batch.map((file) => ({ + fileID: file.id, + collectionID: file.collectionID, + })), + }; + await HTTPService.post( + `${ENDPOINT}/files/trash`, + trashRequest, + null, + { + 'X-Auth-Token': token, + } + ); } } catch (e) { logError(e, 'trash file failed'); @@ -181,6 +192,7 @@ export const deleteFromTrash = async (filesToDelete: number[]) => { return; } const batchedFilesToDelete = batch(filesToDelete, REQUEST_BATCH_SIZE); + for (const batch of batchedFilesToDelete) { await HTTPService.post( `${ENDPOINT}/trash/delete`,