moved UploadURl logic to uploadService

src/services/upload/networkClient.ts

@@ -10,24 +10,20 @@ import { retryAsyncFunction } from 'utils/network';
 const ENDPOINT = getEndpoint();
 const MAX_URL_REQUESTS = 50;
 
-
 class NetworkClient {
-    private uploadURLFetchInProgress=null;
+    private uploadURLFetchInProgress = null;
 
-    async uploadFile(uploadFile: UploadFile):Promise<File> {
+    async uploadFile(uploadFile: UploadFile): Promise<File> {
         try {
             const token = getToken();
             if (!token) {
                 return;
             }
-            const response = await retryAsyncFunction(()=>HTTPService.post(
-                `${ENDPOINT}/files`,
-                uploadFile,
-                null,
-                {
+            const response = await retryAsyncFunction(() =>
+                HTTPService.post(`${ENDPOINT}/files`, uploadFile, null, {
                     'X-Auth-Token': token,
-                },
-            ));
+                }),
+            );
             return response.data;
         } catch (e) {
             logError(e, 'upload Files Failed');
@@ -35,7 +31,7 @@ class NetworkClient {
         }
     }
 
-    async fetchUploadURLs(count:number, urlStore:UploadURL[]): Promise<void> {
+    async fetchUploadURLs(count: number, urlStore: UploadURL[]): Promise<void> {
         try {
             if (!this.uploadURLFetchInProgress) {
                 try {
@@ -46,10 +42,7 @@ class NetworkClient {
                     this.uploadURLFetchInProgress = HTTPService.get(
                         `${ENDPOINT}/files/upload-urls`,
                         {
-                            count: Math.min(
-                                MAX_URL_REQUESTS,
-                                count * 2,
-                            ),
+                            count: Math.min(MAX_URL_REQUESTS, count * 2),
                         },
                         { 'X-Auth-Token': token },
                     );
@@ -92,16 +85,16 @@ class NetworkClient {
     async putFile(
         fileUploadURL: UploadURL,
         file: Uint8Array,
-        progressTracker:()=>any,
+        progressTracker,
     ): Promise<string> {
         try {
-            await retryAsyncFunction(()=>
+            await retryAsyncFunction(() =>
                 HTTPService.put(
                     fileUploadURL.url,
                     file,
                     null,
                     null,
-                    progressTracker(),
+                    progressTracker,
                 ),
             );
             return fileUploadURL.objectKey;
@@ -111,15 +104,14 @@ class NetworkClient {
         }
     }
 
-
     async putFilePart(
         partUploadURL: string,
         filePart: Uint8Array,
         progressTracker,
     ) {
         try {
-            const response=await retryAsyncFunction(async ()=>{
-                const resp =await HTTPService.put(
+            const response = await retryAsyncFunction(async () => {
+                const resp = await HTTPService.put(
                     partUploadURL,
                     filePart,
                     null,
@@ -127,7 +119,7 @@ class NetworkClient {
                     progressTracker(),
                 );
                 if (!resp?.headers?.etag) {
-                    const err=Error(CustomError.ETAG_MISSING);
+                    const err = Error(CustomError.ETAG_MISSING);
                     logError(err);
                     throw err;
                 }
@@ -140,9 +132,9 @@ class NetworkClient {
         }
     }
 
-    async completeMultipartUpload(completeURL:string, reqBody:any) {
+    async completeMultipartUpload(completeURL: string, reqBody: any) {
         try {
-            await retryAsyncFunction(()=>
+            await retryAsyncFunction(() =>
                 HTTPService.post(completeURL, reqBody, null, {
                     'content-type': 'text/xml',
                 }),
@@ -155,5 +147,3 @@ class NetworkClient {
 }
 
 export default new NetworkClient();
-
-

src/services/upload/uploadManager.ts

@@ -10,12 +10,12 @@ import {
 } from 'utils/file';
 import { logError } from 'utils/sentry';
 import localForage from 'utils/storage/localForage';
-import NetworkClient from './networkClient';
 import { ParsedMetaDataJSON, parseMetadataJSON } from './metadataService';
 import { segregateFiles } from 'utils/upload';
 import { ProgressUpdater } from 'components/pages/gallery/Upload';
 import uploader from './uploader';
 import uiService from './uiService';
+import uploadService from './uploadService';
 
 const MAX_CONCURRENT_UPLOADS = 4;
 const FILE_UPLOAD_COMPLETED = 100;
@@ -85,7 +85,7 @@ class UploadManager {
                 uiService.setUploadStage(
                     UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES,
                 );
-                await this.processMetadataFiles(metadataFiles);
+                await this.seedMetadataMap(metadataFiles);
             }
             if (mediaFiles.length) {
                 uiService.setUploadStage(UPLOAD_STAGES.START);
@@ -104,20 +104,7 @@ class UploadManager {
         }
     }
 
-    private async preFetchUploadURLs(count: number) {
-        try {
-            // checking for any subscription related errors
-            await NetworkClient.fetchUploadURLs(count, this.uploadURLs);
-        } catch (e) {
-            logError(e, 'error fetching uploadURLs');
-            const { parsedError, parsed } = parseError(e);
-            if (parsed) {
-                throw parsedError;
-            }
-        }
-    }
-
-    private async processMetadataFiles(metadataFiles: globalThis.File[]) {
+    private async seedMetadataMap(metadataFiles: globalThis.File[]) {
         uiService.reset(metadataFiles.length);
 
         for (const rawFile of metadataFiles) {
@@ -152,8 +139,22 @@ class UploadManager {
         await Promise.all(uploadProcesses);
     }
 
+    private async preFetchUploadURLs(count: number) {
+        try {
+            uploadService.setPendingUploadCount(count);
+            uploadService.preFetchUploadURLs();
+            // checking for any subscription related errors
+        } catch (e) {
+            logError(e, 'error fetching uploadURLs');
+            const { parsedError, parsed } = parseError(e);
+            if (parsed) {
+                throw parsedError;
+            }
+        }
+    }
+
     private async uploadNextFileInQueue(worker: any, fileReader: FileReader) {
-        if (this.filesToBeUploaded.length > 0) {
+        while (this.filesToBeUploaded.length > 0) {
             const fileWithCollection = this.filesToBeUploaded.pop();
             const { fileUploadResult, file } = await uploader(
                 worker,

src/services/upload/uploadService.ts

@@ -13,6 +13,7 @@ import {
 import { encryptFiledata } from './encryptionService';
 import { ENCRYPTION_CHUNK_SIZE } from 'types';
 import { uploadStreamUsingMultipart } from './s3Service';
+import uiService from './uiService';
 
 export const RANDOM_PERCENTAGE_PROGRESS_FOR_PUT = () => 90 + 10 * Math.random();
 export const MIN_STREAM_FILE_SIZE = 20 * 1024 * 1024;
@@ -106,8 +107,12 @@ export enum UPLOAD_STAGES {
 
 class UploadService {
     private uploadURLs: UploadURL[] = [];
-    private pendingFilesUploads: number;
     private metadataMap: Map<string, ParsedMetaDataJSON>;
+    private pendingUploadCount: number = 0;
+
+    setPendingUploadCount(count: number) {
+        this.pendingUploadCount = count;
+    }
 
     async readFile(reader: FileReader, receivedFile: globalThis.File) {
         try {
@@ -140,7 +145,7 @@ class UploadService {
                 filedata,
                 thumbnail,
                 metadata,
-            };
+            } as FileInMemory;
         } catch (e) {
             logError(e, 'error reading files');
             throw e;
@@ -182,22 +187,16 @@ class UploadService {
         }
     }
 
-    async uploadToBucket(
-        file: ProcessedFile,
-        trackUploadProgress,
-    ): Promise<BackupedFile> {
+    async uploadToBucket(file: ProcessedFile): Promise<BackupedFile> {
         try {
             let fileObjectKey: string = null;
             if (isDataStream(file.file.encryptedData)) {
-                const progressTracker = trackUploadProgress;
                 fileObjectKey = await uploadStreamUsingMultipart(
                     file.filename,
                     file.file.encryptedData,
-                    progressTracker,
                 );
             } else {
-                const progressTracker = trackUploadProgress.bind(
-                    null,
+                const progressTracker = uiService.trackUploadProgress(
                     file.filename,
                 );
                 const fileUploadURL = await this.getUploadURL();
@@ -211,7 +210,7 @@ class UploadService {
             const thumbnailObjectKey = await NetworkClient.putFile(
                 thumbnailUploadURL,
                 file.thumbnail.encryptedData as Uint8Array,
-                () => null,
+                null,
             );
 
             const backupedFile: BackupedFile = {
@@ -248,14 +247,22 @@ class UploadService {
     }
 
     private async getUploadURL() {
-        if (this.uploadURLs.length === 0) {
-            await NetworkClient.fetchUploadURLs(
-                this.pendingFilesUploads,
-                this.uploadURLs,
-            );
+        if (this.uploadURLs.length === 0 && this.pendingUploadCount) {
+            await this.fetchUploadURLs();
         }
         return this.uploadURLs.pop();
     }
+
+    public async preFetchUploadURLs() {
+        await this.fetchUploadURLs();
+    }
+
+    private async fetchUploadURLs() {
+        await NetworkClient.fetchUploadURLs(
+            this.pendingUploadCount,
+            this.uploadURLs,
+        );
+    }
 }
 
 export default new UploadService();