Add API batching (#971)

src/constants/api/index.ts

@@ -0,0 +1 @@
+export const REQUEST_BATCH_SIZE = 1000;

src/constants/file/index.ts

@@ -3,7 +3,6 @@ export const MAX_EDITED_CREATION_TIME = new Date();
 
 export const MAX_EDITED_FILE_NAME_LENGTH = 100;
 export const MAX_CAPTION_SIZE = 5000;
-export const MAX_TRASH_BATCH_SIZE = 1000;
 
 export const TYPE_HEIC = 'heic';
 export const TYPE_HEIF = 'heif';

src/services/collectionService.ts

@@ -66,6 +66,8 @@ import {
 } from 'utils/collection';
 import ComlinkCryptoWorker from 'utils/comlink/ComlinkCryptoWorker';
 import { getLocalFiles } from './fileService';
+import { REQUEST_BATCH_SIZE } from 'constants/api';
+import { batch } from 'utils/common';
 
 const ENDPOINT = getEndpoint();
 const COLLECTION_TABLE = 'collections';
@@ -405,8 +407,10 @@ export const addToCollection = async (
 ) => {
     try {
         const token = getToken();
+        const batchedFiles = batch(files, REQUEST_BATCH_SIZE);
+        for (const batch of batchedFiles) {
             const fileKeysEncryptedWithNewCollection =
-            await encryptWithNewCollectionKey(collection, files);
+                await encryptWithNewCollectionKey(collection, batch);
 
             const requestBody: AddToCollectionRequest = {
                 collectionID: collection.id,
@@ -420,6 +424,7 @@ export const addToCollection = async (
                     'X-Auth-Token': token,
                 }
             );
+        }
     } catch (e) {
         logError(e, 'Add to collection Failed ');
         throw e;
@@ -432,8 +437,10 @@ export const restoreToCollection = async (
 ) => {
     try {
         const token = getToken();
+        const batchedFiles = batch(files, REQUEST_BATCH_SIZE);
+        for (const batch of batchedFiles) {
             const fileKeysEncryptedWithNewCollection =
-            await encryptWithNewCollectionKey(collection, files);
+                await encryptWithNewCollectionKey(collection, batch);
 
             const requestBody: AddToCollectionRequest = {
                 collectionID: collection.id,
@@ -447,6 +454,7 @@ export const restoreToCollection = async (
                     'X-Auth-Token': token,
                 }
             );
+        }
     } catch (e) {
         logError(e, 'restore to collection Failed ');
         throw e;
@@ -459,8 +467,10 @@ export const moveToCollection = async (
 ) => {
     try {
         const token = getToken();
+        const batchedFiles = batch(files, REQUEST_BATCH_SIZE);
+        for (const batch of batchedFiles) {
             const fileKeysEncryptedWithNewCollection =
-            await encryptWithNewCollectionKey(toCollection, files);
+                await encryptWithNewCollectionKey(toCollection, batch);
 
             const requestBody: MoveToCollectionRequest = {
                 fromCollectionID: fromCollectionID,
@@ -475,6 +485,7 @@ export const moveToCollection = async (
                     'X-Auth-Token': token,
                 }
             );
+        }
     } catch (e) {
         logError(e, 'move to collection Failed ');
         throw e;
@@ -605,10 +616,11 @@ export const removeNonUserFiles = async (
     try {
         const fileIDs = nonUserFiles.map((f) => f.id);
         const token = getToken();
-
+        const batchedFileIDs = batch(fileIDs, REQUEST_BATCH_SIZE);
+        for (const batch of batchedFileIDs) {
             const request: RemoveFromCollectionRequest = {
                 collectionID,
-            fileIDs,
+                fileIDs: batch,
             };
 
             await HTTPService.post(
@@ -617,6 +629,7 @@ export const removeNonUserFiles = async (
                 null,
                 { 'X-Auth-Token': token }
             );
+        }
     } catch (e) {
         logError(e, 'remove non user files failed ');
         throw e;

src/services/fileService.ts

@@ -14,7 +14,6 @@ import {
 import { eventBus, Events } from './events';
 import { EnteFile, EncryptedEnteFile, TrashRequest } from 'types/file';
 import { SetFiles } from 'types/gallery';
-import { MAX_TRASH_BATCH_SIZE } from 'constants/file';
 import { BulkUpdateMagicMetadataRequest } from 'types/magicMetadata';
 import { addLogLine } from 'utils/logging';
 import { isCollectionHidden } from 'utils/collection';
@@ -24,6 +23,8 @@ import {
     getCollectionLastSyncTime,
     setCollectionLastSyncTime,
 } from './collectionService';
+import { REQUEST_BATCH_SIZE } from 'constants/api';
+import { batch } from 'utils/common';
 
 const ENDPOINT = getEndpoint();
 const FILES_TABLE = 'files';
@@ -161,24 +162,22 @@ export const trashFiles = async (filesToTrash: EnteFile[]) => {
         if (!token) {
             return;
         }
-
+        const batchedFilesToTrash = batch(filesToTrash, REQUEST_BATCH_SIZE);
-        const trashBatch: TrashRequest = {
+        for (const batch of batchedFilesToTrash) {
-            items: [],
+            const trashRequest: TrashRequest = {
-        };
+                items: batch.map((file) => ({
-
-        for (const file of filesToTrash) {
-            trashBatch.items.push({
-                collectionID: file.collectionID,
                     fileID: file.id,
-            });
+                    collectionID: file.collectionID,
-            if (trashBatch.items.length >= MAX_TRASH_BATCH_SIZE) {
+                })),
-                await trashFilesFromServer(trashBatch, token);
+            };
-                trashBatch.items = [];
+            await HTTPService.post(
+                `${ENDPOINT}/files/trash`,
+                trashRequest,
+                null,
+                {
+                    'X-Auth-Token': token,
                 }
-        }
+            );
-
-        if (trashBatch.items.length > 0) {
-            await trashFilesFromServer(trashBatch, token);
         }
     } catch (e) {
         logError(e, 'trash file failed');
@@ -192,35 +191,20 @@ export const deleteFromTrash = async (filesToDelete: number[]) => {
         if (!token) {
             return;
         }
-        let deleteBatch: number[] = [];
+        const batchedFilesToDelete = batch(filesToDelete, REQUEST_BATCH_SIZE);
-        for (const fileID of filesToDelete) {
-            deleteBatch.push(fileID);
-            if (deleteBatch.length >= MAX_TRASH_BATCH_SIZE) {
-                await deleteBatchFromTrash(token, deleteBatch);
-                deleteBatch = [];
-            }
-        }
-        if (deleteBatch.length > 0) {
-            await deleteBatchFromTrash(token, deleteBatch);
-        }
-    } catch (e) {
-        logError(e, 'deleteFromTrash failed');
-        throw e;
-    }
-};
 
-const deleteBatchFromTrash = async (token: string, deleteBatch: number[]) => {
+        for (const batch of batchedFilesToDelete) {
-    try {
             await HTTPService.post(
                 `${ENDPOINT}/trash/delete`,
-            { fileIDs: deleteBatch },
+                { fileIDs: batch },
                 null,
                 {
                     'X-Auth-Token': token,
                 }
             );
+        }
     } catch (e) {
-        logError(e, 'deleteBatchFromTrash failed');
+        logError(e, 'deleteFromTrash failed');
         throw e;
     }
 };
@@ -303,14 +287,3 @@ export const updateFilePublicMagicMetadata = async (files: EnteFile[]) => {
         })
     );
 };
-
-async function trashFilesFromServer(trashBatch: TrashRequest, token: any) {
-    try {
-        await HTTPService.post(`${ENDPOINT}/files/trash`, trashBatch, null, {
-            'X-Auth-Token': token,
-        });
-    } catch (e) {
-        logError(e, 'trash files from server failed');
-        throw e;
-    }
-}

src/utils/common/index.ts

@@ -125,3 +125,11 @@ function isPromise(p: any) {
 export function isClipboardItemPresent() {
     return typeof ClipboardItem !== 'undefined';
 }
+
+export function batch<T>(arr: T[], batchSize: number): T[][] {
+    const batches: T[][] = [];
+    for (let i = 0; i < arr.length; i += batchSize) {
+        batches.push(arr.slice(i, i + batchSize));
+    }
+    return batches;
+}