From b95a560072f1ce545328f4905ae60c4fec51b6da Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Mon, 9 Aug 2021 20:45:11 +0530 Subject: [PATCH] created s3Service layer --- src/services/upload/networkClient.ts | 90 +++++++++----------------- src/services/upload/readFileService.ts | 2 +- src/services/upload/s3Service.ts | 87 +++++++++++++++++++++++++ src/services/upload/uploadService.ts | 34 +++++----- 4 files changed, 134 insertions(+), 79 deletions(-) create mode 100644 src/services/upload/s3Service.ts diff --git a/src/services/upload/networkClient.ts b/src/services/upload/networkClient.ts index 87f32c65e..7a7f3b9a0 100644 --- a/src/services/upload/networkClient.ts +++ b/src/services/upload/networkClient.ts @@ -3,8 +3,7 @@ import { retryAsyncFunction } from 'utils/common'; import { getEndpoint } from 'utils/common/apiUtil'; import { getToken } from 'utils/common/key'; import { logError } from 'utils/sentry'; -import { CHUNKS_COMBINED_FOR_UPLOAD, MultipartUploadURLs, RANDOM_PERCENTAGE_PROGRESS_FOR_PUT, UploadFile } from './uploadService'; -import * as convert from 'xml-js'; +import { MultipartUploadURLs, UploadFile, UploadURL } from './uploadService'; import { File } from '../fileService'; import { CustomError } from 'utils/common/errorUtil'; @@ -12,10 +11,6 @@ const ENDPOINT = getEndpoint(); const MAX_URL_REQUESTS = 50; -export interface UploadURL { - url: string; - objectKey: string; -} class NetworkClient { private uploadURLFetchInProgress=null; @@ -116,72 +111,49 @@ class NetworkClient { } } - async putFileInParts( - multipartUploadURLs: MultipartUploadURLs, - file: ReadableStream, - filename: string, - uploadPartCount: number, - trackUploadProgress, + + async putFilePart( + partUploadURL: string, + filePart: Uint8Array, + progressTracker, ) { try { - const streamEncryptedFileReader = file.getReader(); - const percentPerPart = Math.round( - RANDOM_PERCENTAGE_PROGRESS_FOR_PUT() / uploadPartCount, - ); - const resParts = []; - for (const [ - index, - fileUploadURL, - ] of multipartUploadURLs.partURLs.entries()) { - const combinedChunks = []; - for (let i = 0; i < CHUNKS_COMBINED_FOR_UPLOAD; i++) { - const { done, value: chunk } = - await streamEncryptedFileReader.read(); - if (done) { - break; - } - for (let index = 0; index < chunk.length; index++) { - combinedChunks.push(chunk[index]); - } + const response=await retryAsyncFunction(async ()=>{ + const resp =await HTTPService.put( + partUploadURL, + filePart, + null, + null, + progressTracker(), + ); + if (!resp?.headers?.etag) { + const err=Error(CustomError.ETAG_MISSING); + logError(err); + throw err; } - const uploadChunk = Uint8Array.from(combinedChunks); - const response=await retryAsyncFunction(async ()=>{ - const resp =await HTTPService.put( - fileUploadURL, - uploadChunk, - null, - null, - trackUploadProgress(filename, percentPerPart, index), - ); - if (!resp?.headers?.etag) { - const err=Error(CustomError.ETAG_MISSING); - logError(err); - throw err; - } - return resp; - }); - resParts.push({ - PartNumber: index + 1, - ETag: response.headers.etag, - }); - } - const options = { compact: true, ignoreComment: true, spaces: 4 }; - const body = convert.js2xml( - { CompleteMultipartUpload: { Part: resParts } }, - options, - ); + return resp; + }); + return response.headers.etag; + } catch (e) { + logError(e, 'put filePart failed'); + throw e; + } + } + + async completeMultipartUpload(completeURL:string, reqBody:any) { + try { await retryAsyncFunction(()=> - HTTPService.post(multipartUploadURLs.completeURL, body, null, { + HTTPService.post(completeURL, reqBody, null, { 'content-type': 'text/xml', }), ); - return multipartUploadURLs.objectKey; } catch (e) { logError(e, 'put file in parts failed'); throw e; } } } + export default new NetworkClient(); diff --git a/src/services/upload/readFileService.ts b/src/services/upload/readFileService.ts index 7a56c1dbe..8eb0a2ae7 100644 --- a/src/services/upload/readFileService.ts +++ b/src/services/upload/readFileService.ts @@ -1,11 +1,11 @@ import { FILE_TYPE } from 'pages/gallery'; import { ENCRYPTION_CHUNK_SIZE } from 'types'; import { logError } from 'utils/sentry'; +import { MIN_STREAM_FILE_SIZE } from './uploadService'; const TYPE_VIDEO = 'video'; const TYPE_HEIC = 'HEIC'; export const TYPE_IMAGE = 'image'; -const MIN_STREAM_FILE_SIZE = 20 * 1024 * 1024; const EDITED_FILE_SUFFIX = '-edited'; diff --git a/src/services/upload/s3Service.ts b/src/services/upload/s3Service.ts new file mode 100644 index 000000000..9dc0a4450 --- /dev/null +++ b/src/services/upload/s3Service.ts @@ -0,0 +1,87 @@ +import { CHUNKS_COMBINED_FOR_A_UPLOAD_PART, DataStream, MultipartUploadURLs, RANDOM_PERCENTAGE_PROGRESS_FOR_PUT } from './uploadService'; +import NetworkClient from './networkClient'; +import * as convert from 'xml-js'; + + +interface PartEtag{ + PartNumber:number; + Etag:string; +} +export function calculatePartCount(encryptedChunkCount: number) { + const partCount = Math.ceil( + encryptedChunkCount / CHUNKS_COMBINED_FOR_A_UPLOAD_PART, + ); + return partCount; +} +export async function uploadStreamUsingMultipart(filename:string, encryptedData:DataStream, progressTracker) { + const { chunkCount, stream } = encryptedData; + const uploadPartCount = calculatePartCount(chunkCount); + const filePartUploadURLs = await NetworkClient.fetchMultipartUploadURLs( + uploadPartCount, + ); + const fileObjectKey = await uploadStreamInParts( + filePartUploadURLs, + stream, + filename, + uploadPartCount, + progressTracker, + ); + return fileObjectKey; +} + +export async function uploadStreamInParts( + multipartUploadURLs: MultipartUploadURLs, + file: ReadableStream, + filename: string, + uploadPartCount: number, + progressTracker, +) { + const encryptedFileStreamReader = file.getReader(); + const percentPerPart = getRandomProgressPerPartUpload(uploadPartCount); + + const partEtags:PartEtag[] = []; + for (const [ + index, + fileUploadURL, + ] of multipartUploadURLs.partURLs.entries()) { + const uploadChunk = await combineChunksToFormUploadPart(encryptedFileStreamReader); + const eTag= await NetworkClient.putFilePart(fileUploadURL, uploadChunk, progressTracker.bind(null, filename, percentPerPart, index)); + partEtags.push({ PartNumber: index+1, Etag: eTag }); + } + await completeMultipartUpload(partEtags, multipartUploadURLs.completeURL); + return multipartUploadURLs.objectKey; +} + + +export function getRandomProgressPerPartUpload(uploadPartCount:number) { + const percentPerPart = Math.round( + RANDOM_PERCENTAGE_PROGRESS_FOR_PUT() / uploadPartCount, + ); + return percentPerPart; +} + + +export async function combineChunksToFormUploadPart(dataStreamReader:ReadableStreamDefaultReader) { + const combinedChunks = []; + for (let i = 0; i < CHUNKS_COMBINED_FOR_A_UPLOAD_PART; i++) { + const { done, value: chunk } = + await dataStreamReader.read(); + if (done) { + break; + } + for (let index = 0; index < chunk.length; index++) { + combinedChunks.push(chunk[index]); + } + } + return Uint8Array.from(combinedChunks); +} + + +async function completeMultipartUpload(partEtags:PartEtag[], completeURL:string) { + const options = { compact: true, ignoreComment: true, spaces: 4 }; + const body = convert.js2xml( + { CompleteMultipartUpload: { Part: partEtags } }, + options, + ); + await NetworkClient.completeMultipartUpload(completeURL, body); +} diff --git a/src/services/upload/uploadService.ts b/src/services/upload/uploadService.ts index f1432c1b5..e47f3ef3b 100644 --- a/src/services/upload/uploadService.ts +++ b/src/services/upload/uploadService.ts @@ -16,11 +16,13 @@ import { import { logError } from 'utils/sentry'; import localForage from 'utils/storage/localForage'; import { sleep } from 'utils/common'; -import NetworkClient, { UploadURL } from './networkClient'; +import NetworkClient from './networkClient'; import { extractMetatdata, ParsedMetaDataJSON, parseMetadataJSON } from './metadataService'; import { generateThumbnail } from './thumbnailService'; import { getFileType, getFileOriginalName, getFileData } from './readFileService'; import { encryptFiledata } from './encryptionService'; +import { ENCRYPTION_CHUNK_SIZE } from 'types'; +import { uploadStreamUsingMultipart } from './s3Service'; const MAX_CONCURRENT_UPLOADS = 4; @@ -28,7 +30,8 @@ const TYPE_JSON = 'json'; const FILE_UPLOAD_COMPLETED = 100; const TwoSecondInMillSeconds = 2000; export const RANDOM_PERCENTAGE_PROGRESS_FOR_PUT = () => 90 + 10 * Math.random(); -export const CHUNKS_COMBINED_FOR_UPLOAD = 5; +export const MIN_STREAM_FILE_SIZE = 20 * 1024 * 1024; +export const CHUNKS_COMBINED_FOR_A_UPLOAD_PART = Math.floor(MIN_STREAM_FILE_SIZE/ENCRYPTION_CHUNK_SIZE); export enum FileUploadResults { FAILED = -1, @@ -38,6 +41,11 @@ export enum FileUploadResults { UPLOADED = 100, } +export interface UploadURL { + url: string; + objectKey: string; +} + export interface FileWithCollection { file: globalThis.File; collection: Collection; @@ -87,7 +95,7 @@ interface EncryptedFile { file: ProcessedFile; fileKey: B64EncryptionResult; } -interface ProcessedFile { +export interface ProcessedFile { file: fileAttribute; thumbnail: fileAttribute; metadata: fileAttribute; @@ -420,25 +428,13 @@ class UploadService { private async uploadToBucket(file: ProcessedFile): Promise { try { - let fileObjectKey; + let fileObjectKey:string=null; if (isDataStream(file.file.encryptedData)) { - const { chunkCount, stream } = file.file.encryptedData; - const uploadPartCount = Math.ceil( - chunkCount / CHUNKS_COMBINED_FOR_UPLOAD, - ); - const filePartUploadURLs = await NetworkClient.fetchMultipartUploadURLs( - uploadPartCount, - ); - fileObjectKey = await NetworkClient.putFileInParts( - filePartUploadURLs, - stream, - file.filename, - uploadPartCount, - this.trackUploadProgress.bind(this), - ); + const progressTracker=this.trackUploadProgress.bind(this); + fileObjectKey=await uploadStreamUsingMultipart(file.filename, file.file.encryptedData, progressTracker); } else { - const fileUploadURL = await this.getUploadURL(); const progressTracker=this.trackUploadProgress.bind(this, file.filename); + const fileUploadURL = await this.getUploadURL(); fileObjectKey = await NetworkClient.putFile( fileUploadURL, file.file.encryptedData,