From 6d8205febfd6c483b40a85789a15cabc05697679 Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Wed, 28 Jul 2021 21:15:30 +0530 Subject: [PATCH 001/231] added redirect to payments page --- src/services/billingService.ts | 103 +++++++++++++++++++-------------- src/services/userService.ts | 13 +++++ src/utils/common/apiUtil.ts | 11 +++- 3 files changed, 82 insertions(+), 45 deletions(-) diff --git a/src/services/billingService.ts b/src/services/billingService.ts index 189e550e0..239e73e2b 100644 --- a/src/services/billingService.ts +++ b/src/services/billingService.ts @@ -1,12 +1,12 @@ -import { getEndpoint } from 'utils/common/apiUtil'; +import { getEndpoint, getPaymentsUrl } from 'utils/common/apiUtil'; import { getStripePublishableKey, getToken } from 'utils/common/key'; import { checkConnectivity, runningInBrowser } from 'utils/common/'; import { setData, LS_KEYS } from 'utils/storage/localStorage'; import { convertToHumanReadable } from 'utils/billingUtil'; import { loadStripe, Stripe } from '@stripe/stripe-js'; -import { SUBSCRIPTION_VERIFICATION_ERROR } from 'utils/common/errorUtil'; import HTTPService from './HTTPService'; import { logError } from 'utils/sentry'; +import { getPaymentToken } from './userService'; const ENDPOINT = getEndpoint(); @@ -15,6 +15,10 @@ export enum PAYMENT_INTENT_STATUS { REQUIRE_ACTION = 'requires_action', REQUIRE_PAYMENT_METHOD = 'requires_payment_method', } +enum PaymentActionType{ + Buy='buy', + Update='update' +} export interface Subscription { id: number; userID: number; @@ -90,10 +94,12 @@ class billingService { public async buyPaidSubscription(productID) { try { - const response = await this.createCheckoutSession(productID); - await this.stripe.redirectToCheckout({ - sessionId: response.data.sessionID, - }); + // const response = await this.createCheckoutSession(productID); + // await this.stripe.redirectToCheckout({ + // sessionId: response.data.sessionID, + // }); + const paymentToken =await getPaymentToken(); + await this.redirectToPayments(paymentToken, productID, PaymentActionType.Buy); } catch (e) { logError(e, 'unable to buy subscription'); throw e; @@ -102,46 +108,48 @@ class billingService { public async updateSubscription(productID) { try { - const response = await HTTPService.post( - `${ENDPOINT}/billing/stripe/update-subscription`, - { - productID, - }, - null, - { - 'X-Auth-Token': getToken(), - }, - ); - const { result } = response.data; - switch (result.status) { - case PAYMENT_INTENT_STATUS.SUCCESS: - // subscription updated successfully - // no-op required - break; - case PAYMENT_INTENT_STATUS.REQUIRE_PAYMENT_METHOD: - throw new Error( - PAYMENT_INTENT_STATUS.REQUIRE_PAYMENT_METHOD, - ); - case PAYMENT_INTENT_STATUS.REQUIRE_ACTION: - { - const { error } = await this.stripe.confirmCardPayment( - result.clientSecret, - ); - if (error) { - throw error; - } - } - break; - } + // const response = await HTTPService.post( + // `${ENDPOINT}/billing/stripe/update-subscription`, + // { + // productID, + // }, + // null, + // { + // 'X-Auth-Token': getToken(), + // }, + // ); + // const { result } = response.data; + // switch (result.status) { + // case PAYMENT_INTENT_STATUS.SUCCESS: + // // subscription updated successfully + // // no-op required + // break; + // case PAYMENT_INTENT_STATUS.REQUIRE_PAYMENT_METHOD: + // throw new Error( + // PAYMENT_INTENT_STATUS.REQUIRE_PAYMENT_METHOD, + // ); + // case PAYMENT_INTENT_STATUS.REQUIRE_ACTION: + // { + // const { error } = await this.stripe.confirmCardPayment( + // result.clientSecret, + // ); + // if (error) { + // throw error; + // } + // } + // break; + // } + const paymentToken =await getPaymentToken(); + await this.redirectToPayments(paymentToken, productID, PaymentActionType.Update); } catch (e) { - logError(e); + logError(e, 'subscription update failed'); throw e; } - try { - await this.verifySubscription(); - } catch (e) { - throw new Error(SUBSCRIPTION_VERIFICATION_ERROR); - } + // try { + // await this.verifySubscription(); + // } catch (e) { + // throw new Error(SUBSCRIPTION_VERIFICATION_ERROR); + // } } public async cancelSubscription() { @@ -217,6 +225,15 @@ class billingService { } } + public async redirectToPayments(paymentToken:string, productID:string, action:string) { + try { + window.location.href =`${getPaymentsUrl()}?productID=${productID}&payment-token=${paymentToken}&action=${action}`; + } catch (e) { + logError(e, 'unable to get payments url'); + throw e; + } + } + public async redirectToCustomerPortal() { try { const response = await HTTPService.get( diff --git a/src/services/userService.ts b/src/services/userService.ts index fbff2e856..1adb0f3f0 100644 --- a/src/services/userService.ts +++ b/src/services/userService.ts @@ -72,6 +72,19 @@ export const getPublicKey = async (email: string) => { return resp.data.publicKey; }; +export const getPaymentToken = async () => { + const token = getToken(); + + const resp = await HTTPService.get( + `${ENDPOINT}/users/payment-token`, + null, + { + 'X-Auth-Token': token, + }, + ); + return resp.data['paymentToken']; +}; + export const verifyOtt = (email: string, ott: string) => HTTPService.post(`${ENDPOINT}/users/verify-email`, { email, ott }); export const putAttributes = (token: string, keyAttributes: KeyAttributes) => HTTPService.put( diff --git a/src/utils/common/apiUtil.ts b/src/utils/common/apiUtil.ts index 6478a9ec1..b2a190ec6 100644 --- a/src/utils/common/apiUtil.ts +++ b/src/utils/common/apiUtil.ts @@ -5,14 +5,14 @@ export const getEndpoint = () => { export const getFileUrl = (id: number) => { if (process.env.NEXT_PUBLIC_ENTE_ENDPOINT !== undefined) { - return `${process.env.NEXT_PUBLIC_ENTE_ENDPOINT}/files/download/${id}` ?? 'https://api.ente.io'; + return `${process.env.NEXT_PUBLIC_ENTE_ENDPOINT}/files/download/${id}` ?? `https://api.ente.io/files/download/${id}`; } return `https://files.ente.workers.dev/?fileID=${id}`; }; export const getThumbnailUrl = (id: number) => { if (process.env.NEXT_PUBLIC_ENTE_ENDPOINT !== undefined) { - return `${process.env.NEXT_PUBLIC_ENTE_ENDPOINT}/files/preview/${id}` ?? 'https://api.ente.io'; + return `${process.env.NEXT_PUBLIC_ENTE_ENDPOINT}/files/preview/${id}` ?? `https://api.ente.io/files/preview/${id}`; } return `https://thumbnails.ente.workers.dev/?fileID=${id}`; }; @@ -20,3 +20,10 @@ export const getThumbnailUrl = (id: number) => { export const getSentryTunnelUrl = () => { return `https://sentry-reporter.ente.workers.dev`; }; + +export const getPaymentsUrl = () => { + if (process.env.NEXT_PUBLIC_ENTE_ENDPOINT !== undefined) { + return process.env.NEXT_PUBLIC_ENTE_PAYMENT_ENDPOINT; + } + return `https://payments.ente.io`; +}; From 3d7a8c4106b9cf057f3cd96a3cd7d5f8c430099a Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Thu, 5 Aug 2021 17:34:47 +0530 Subject: [PATCH 002/231] moved uploadService to upload sub folder --- src/components/pages/gallery/Upload.tsx | 2 +- src/components/pages/gallery/UploadProgress.tsx | 2 +- src/pages/change-password/index.tsx | 2 +- src/pages/two-factor/recover/index.tsx | 2 +- src/pages/two-factor/setup/index.tsx | 2 +- src/services/collectionService.ts | 2 +- src/services/fileService.ts | 2 +- src/services/{ => upload}/uploadService.ts | 6 +++--- src/services/userService.ts | 2 +- src/utils/common/key.ts | 2 +- src/utils/crypto/index.ts | 2 +- src/utils/export/index.ts | 2 +- 12 files changed, 14 insertions(+), 14 deletions(-) rename src/services/{ => upload}/uploadService.ts (99%) diff --git a/src/components/pages/gallery/Upload.tsx b/src/components/pages/gallery/Upload.tsx index 082ec8668..dad4a69e1 100644 --- a/src/components/pages/gallery/Upload.tsx +++ b/src/components/pages/gallery/Upload.tsx @@ -1,5 +1,5 @@ import React, { useContext, useEffect, useState } from 'react'; -import UploadService, { FileWithCollection, UPLOAD_STAGES } from 'services/uploadService'; +import UploadService, { FileWithCollection, UPLOAD_STAGES } from 'services/upload/uploadService'; import { createAlbum } from 'services/collectionService'; import { getLocalFiles } from 'services/fileService'; import constants from 'utils/strings/constants'; diff --git a/src/components/pages/gallery/UploadProgress.tsx b/src/components/pages/gallery/UploadProgress.tsx index d76d0b226..a279cc419 100644 --- a/src/components/pages/gallery/UploadProgress.tsx +++ b/src/components/pages/gallery/UploadProgress.tsx @@ -3,7 +3,7 @@ import { Alert, Button, Modal, ProgressBar, } from 'react-bootstrap'; import { FileRejection } from 'react-dropzone'; -import { UPLOAD_STAGES, FileUploadErrorCode } from 'services/uploadService'; +import { UPLOAD_STAGES, FileUploadErrorCode } from 'services/upload/uploadService'; import constants from 'utils/strings/constants'; interface Props { diff --git a/src/pages/change-password/index.tsx b/src/pages/change-password/index.tsx index 14aa0b07f..a38810f71 100644 --- a/src/pages/change-password/index.tsx +++ b/src/pages/change-password/index.tsx @@ -2,7 +2,7 @@ import React, { useState, useEffect, useContext } from 'react'; import constants from 'utils/strings/constants'; import { getData, LS_KEYS, setData } from 'utils/storage/localStorage'; import { useRouter } from 'next/router'; -import { B64EncryptionResult } from 'services/uploadService'; +import { B64EncryptionResult } from 'services/upload/uploadService'; import CryptoWorker, { setSessionKeys, generateAndSaveIntermediateKeyAttributes, diff --git a/src/pages/two-factor/recover/index.tsx b/src/pages/two-factor/recover/index.tsx index 6c5634a9f..444b5762a 100644 --- a/src/pages/two-factor/recover/index.tsx +++ b/src/pages/two-factor/recover/index.tsx @@ -9,7 +9,7 @@ import Container from 'components/Container'; import { Card, Button } from 'react-bootstrap'; import LogoImg from 'components/LogoImg'; import { logError } from 'utils/sentry'; -import { B64EncryptionResult } from 'services/uploadService'; +import { B64EncryptionResult } from 'services/upload/uploadService'; import { recoverTwoFactor, removeTwoFactor } from 'services/userService'; export default function Recover() { diff --git a/src/pages/two-factor/setup/index.tsx b/src/pages/two-factor/setup/index.tsx index a9bab920b..aaeea34b0 100644 --- a/src/pages/two-factor/setup/index.tsx +++ b/src/pages/two-factor/setup/index.tsx @@ -10,7 +10,7 @@ import constants from 'utils/strings/constants'; import Container from 'components/Container'; import { useRouter } from 'next/router'; import VerifyTwoFactor from 'components/VerifyTwoFactor'; -import { B64EncryptionResult } from 'services/uploadService'; +import { B64EncryptionResult } from 'services/upload/uploadService'; import { encryptWithRecoveryKey } from 'utils/crypto'; import { setData, LS_KEYS, getData } from 'utils/storage/localStorage'; import { AppContext } from 'pages/_app'; diff --git a/src/services/collectionService.ts b/src/services/collectionService.ts index dfd9ade38..d43c4de02 100644 --- a/src/services/collectionService.ts +++ b/src/services/collectionService.ts @@ -7,7 +7,7 @@ import CryptoWorker from 'utils/crypto'; import { SetDialogMessage } from 'components/MessageDialog'; import constants from 'utils/strings/constants'; import { getPublicKey, User } from './userService'; -import { B64EncryptionResult } from './uploadService'; +import { B64EncryptionResult } from './upload/uploadService'; import HTTPService from './HTTPService'; import { File } from './fileService'; import { logError } from 'utils/sentry'; diff --git a/src/services/fileService.ts b/src/services/fileService.ts index b162509df..771efa603 100644 --- a/src/services/fileService.ts +++ b/src/services/fileService.ts @@ -2,7 +2,7 @@ import { getEndpoint } from 'utils/common/apiUtil'; import localForage from 'utils/storage/localForage'; import { getToken } from 'utils/common/key'; -import { DataStream, MetadataObject } from './uploadService'; +import { DataStream, MetadataObject } from './upload/uploadService'; import { Collection } from './collectionService'; import HTTPService from './HTTPService'; import { logError } from 'utils/sentry'; diff --git a/src/services/uploadService.ts b/src/services/upload/uploadService.ts similarity index 99% rename from src/services/uploadService.ts rename to src/services/upload/uploadService.ts index ff20969f8..45710a721 100644 --- a/src/services/uploadService.ts +++ b/src/services/upload/uploadService.ts @@ -1,8 +1,8 @@ import { getEndpoint } from 'utils/common/apiUtil'; -import HTTPService from './HTTPService'; +import HTTPService from '../HTTPService'; import EXIF from 'exif-js'; -import { File, fileAttribute } from './fileService'; -import { Collection } from './collectionService'; +import { File, fileAttribute } from '../fileService'; +import { Collection } from '../collectionService'; import { FILE_TYPE, SetFiles } from 'pages/gallery'; import { retryAsyncFunction, sleep } from 'utils/common'; import { diff --git a/src/services/userService.ts b/src/services/userService.ts index 353269002..d2a09dc5a 100644 --- a/src/services/userService.ts +++ b/src/services/userService.ts @@ -6,7 +6,7 @@ import { clearData } from 'utils/storage/localStorage'; import localForage from 'utils/storage/localForage'; import { getToken } from 'utils/common/key'; import HTTPService from './HTTPService'; -import { B64EncryptionResult } from './uploadService'; +import { B64EncryptionResult } from './upload/uploadService'; import { logError } from 'utils/sentry'; import { Subscription } from './billingService'; diff --git a/src/utils/common/key.ts b/src/utils/common/key.ts index c70d20475..28d81e3b6 100644 --- a/src/utils/common/key.ts +++ b/src/utils/common/key.ts @@ -1,4 +1,4 @@ -import { B64EncryptionResult } from 'services/uploadService'; +import { B64EncryptionResult } from 'services/upload/uploadService'; import CryptoWorker from 'utils/crypto'; import { getData, LS_KEYS } from 'utils/storage/localStorage'; import { getKey, SESSION_KEYS } from 'utils/storage/sessionStorage'; diff --git a/src/utils/crypto/index.ts b/src/utils/crypto/index.ts index ff9223f94..42c152ed9 100644 --- a/src/utils/crypto/index.ts +++ b/src/utils/crypto/index.ts @@ -1,5 +1,5 @@ import { KEK } from 'pages/generate'; -import { B64EncryptionResult } from 'services/uploadService'; +import { B64EncryptionResult } from 'services/upload/uploadService'; import { KeyAttributes } from 'types'; import * as Comlink from 'comlink'; import { runningInBrowser } from 'utils/common'; diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index ba2f936a3..131a96cc2 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -1,6 +1,6 @@ import { ExportRecord } from 'services/exportService'; import { File } from 'services/fileService'; -import { MetadataObject } from 'services/uploadService'; +import { MetadataObject } from 'services/upload/uploadService'; import { formatDate } from 'utils/file'; From 64042128b761af79e873839330d8d0edac7e5d67 Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Thu, 5 Aug 2021 19:18:25 +0530 Subject: [PATCH 003/231] refactor network logic to networkClient --- src/services/upload/networkClient.ts | 187 ++++++++++++++++++ src/services/upload/uploadService.ts | 283 ++++++--------------------- 2 files changed, 246 insertions(+), 224 deletions(-) create mode 100644 src/services/upload/networkClient.ts diff --git a/src/services/upload/networkClient.ts b/src/services/upload/networkClient.ts new file mode 100644 index 000000000..e023679e1 --- /dev/null +++ b/src/services/upload/networkClient.ts @@ -0,0 +1,187 @@ +import HTTPService from 'services/HTTPService'; +import { retryAsyncFunction } from 'utils/common'; +import { getEndpoint } from 'utils/common/apiUtil'; +import { getToken } from 'utils/common/key'; +import { logError } from 'utils/sentry'; +import { CHUNKS_COMBINED_FOR_UPLOAD, MultipartUploadURLs, RANDOM_PERCENTAGE_PROGRESS_FOR_PUT, UploadFile } from './uploadService'; +import * as convert from 'xml-js'; +import { File } from '../fileService'; + +const ENDPOINT = getEndpoint(); +const MAX_URL_REQUESTS = 50; + + +export interface UploadURL { + url: string; + objectKey: string; +} +class NetworkClient { + private uploadURLFetchInProgress=null; + + async uploadFile(uploadFile: UploadFile):Promise { + try { + const token = getToken(); + if (!token) { + return; + } + const response = await retryAsyncFunction(()=>HTTPService.post( + `${ENDPOINT}/files`, + uploadFile, + null, + { + 'X-Auth-Token': token, + }, + )); + return response.data; + } catch (e) { + logError(e, 'upload Files Failed'); + throw e; + } + } + + async fetchUploadURLs(count:number, urlStore:UploadURL[]): Promise { + try { + if (!this.uploadURLFetchInProgress) { + try { + const token = getToken(); + if (!token) { + return; + } + this.uploadURLFetchInProgress = HTTPService.get( + `${ENDPOINT}/files/upload-urls`, + { + count: Math.min( + MAX_URL_REQUESTS, + count * 2, + ), + }, + { 'X-Auth-Token': token }, + ); + const response = await this.uploadURLFetchInProgress; + urlStore.push(...response.data['urls']); + } finally { + this.uploadURLFetchInProgress = null; + } + } + return this.uploadURLFetchInProgress; + } catch (e) { + logError(e, 'fetch upload-url failed '); + throw e; + } + } + + async fetchMultipartUploadURLs( + count: number, + ): Promise { + try { + const token = getToken(); + if (!token) { + return; + } + const response = await HTTPService.get( + `${ENDPOINT}/files/multipart-upload-urls`, + { + count, + }, + { 'X-Auth-Token': token }, + ); + + return response.data['urls']; + } catch (e) { + logError(e, 'fetch multipart-upload-url failed'); + throw e; + } + } + + async putFile( + fileUploadURL: UploadURL, + file: Uint8Array, + progressTracker:()=>any, + ): Promise { + try { + console.log(fileUploadURL, file); + await retryAsyncFunction(()=> + HTTPService.put( + fileUploadURL.url, + file, + null, + null, + progressTracker(), + ), + ); + return fileUploadURL.objectKey; + } catch (e) { + logError(e, 'putFile to dataStore failed '); + throw e; + } + } + + async putFileInParts( + multipartUploadURLs: MultipartUploadURLs, + file: ReadableStream, + filename: string, + uploadPartCount: number, + trackUploadProgress, + ) { + try { + const streamEncryptedFileReader = file.getReader(); + const percentPerPart = Math.round( + RANDOM_PERCENTAGE_PROGRESS_FOR_PUT() / uploadPartCount, + ); + const resParts = []; + for (const [ + index, + fileUploadURL, + ] of multipartUploadURLs.partURLs.entries()) { + const combinedChunks = []; + for (let i = 0; i < CHUNKS_COMBINED_FOR_UPLOAD; i++) { + const { done, value: chunk } = + await streamEncryptedFileReader.read(); + if (done) { + break; + } + for (let index = 0; index < chunk.length; index++) { + combinedChunks.push(chunk[index]); + } + } + const uploadChunk = Uint8Array.from(combinedChunks); + const response=await retryAsyncFunction(async ()=>{ + const resp =await HTTPService.put( + fileUploadURL, + uploadChunk, + null, + null, + trackUploadProgress(filename, percentPerPart, index), + ); + if (!resp?.headers?.etag) { + const err=Error('no header/etag present in response body'); + logError(err); + throw err; + } + return resp; + }); + resParts.push({ + PartNumber: index + 1, + ETag: response.headers.etag, + }); + } + const options = { compact: true, ignoreComment: true, spaces: 4 }; + const body = convert.js2xml( + { CompleteMultipartUpload: { Part: resParts } }, + options, + ); + await retryAsyncFunction(()=> + HTTPService.post(multipartUploadURLs.completeURL, body, null, { + 'content-type': 'text/xml', + }), + ); + return multipartUploadURLs.objectKey; + } catch (e) { + logError(e, 'put file in parts failed'); + throw e; + } + } +} +export default new NetworkClient(); + + diff --git a/src/services/upload/uploadService.ts b/src/services/upload/uploadService.ts index 45710a721..eee918c23 100644 --- a/src/services/upload/uploadService.ts +++ b/src/services/upload/uploadService.ts @@ -1,19 +1,14 @@ -import { getEndpoint } from 'utils/common/apiUtil'; -import HTTPService from '../HTTPService'; import EXIF from 'exif-js'; import { File, fileAttribute } from '../fileService'; import { Collection } from '../collectionService'; import { FILE_TYPE, SetFiles } from 'pages/gallery'; -import { retryAsyncFunction, sleep } from 'utils/common'; import { handleError, parseError, THUMBNAIL_GENERATION_FAILED, } from 'utils/common/errorUtil'; import { ComlinkWorker, getDedicatedCryptoWorker } from 'utils/crypto'; -import * as convert from 'xml-js'; import { ENCRYPTION_CHUNK_SIZE } from 'types'; -import { getToken } from 'utils/common/key'; import { fileIsHEIC, convertHEIC2JPEG, @@ -24,10 +19,11 @@ import { } from 'utils/file'; import { logError } from 'utils/sentry'; import localForage from 'utils/storage/localForage'; -const ENDPOINT = getEndpoint(); +import { sleep } from 'utils/common'; +import NetworkClient, { UploadURL } from './networkClient'; + const THUMBNAIL_HEIGHT = 720; -const MAX_URL_REQUESTS = 50; const MAX_ATTEMPTS = 3; const MIN_THUMBNAIL_SIZE = 50000; const MAX_CONCURRENT_UPLOADS = 4; @@ -38,13 +34,13 @@ const TYPE_JSON = 'json'; const SOUTH_DIRECTION = 'S'; const WEST_DIRECTION = 'W'; const MIN_STREAM_FILE_SIZE = 20 * 1024 * 1024; -const CHUNKS_COMBINED_FOR_UPLOAD = 5; -const RANDOM_PERCENTAGE_PROGRESS_FOR_PUT = () => 90 + 10 * Math.random(); const NULL_LOCATION: Location = { latitude: null, longitude: null }; const WAIT_TIME_THUMBNAIL_GENERATION = 10 * 1000; const FILE_UPLOAD_COMPLETED = 100; const EDITED_FILE_SUFFIX = '-edited'; const TwoSecondInMillSeconds = 2000; +export const RANDOM_PERCENTAGE_PROGRESS_FOR_PUT = () => 90 + 10 * Math.random(); +export const CHUNKS_COMBINED_FOR_UPLOAD = 5; export enum FileUploadErrorCode { FAILED = -1, @@ -82,12 +78,8 @@ export interface B64EncryptionResult { nonce: string; } -interface UploadURL { - url: string; - objectKey: string; -} -interface MultipartUploadURLs { +export interface MultipartUploadURLs { objectKey: string; partURLs: string[]; completeURL: string; @@ -121,7 +113,7 @@ interface ProcessedFile { } interface BackupedFile extends Omit { } -interface uploadFile extends BackupedFile { +export interface UploadFile extends BackupedFile { collectionID: number; encryptedKey: string; keyDecryptionNonce: string; @@ -202,7 +194,8 @@ class UploadService { this.filesCompleted = 0; this.updateProgressBarUI(); try { - await this.fetchUploadURLs(); + // checking for any subscription related errors + await NetworkClient.fetchUploadURLs(this.totalFileCount, this.uploadURLs); } catch (e) { logError(e, 'error fetching uploadURLs'); const { parsedError, parsed } = parseError(e); @@ -271,14 +264,14 @@ class UploadService { encryptedFile.file, ); - let uploadFile: uploadFile = this.getUploadFile( + let uploadFile: UploadFile = this.getUploadFile( collection, backupedFile, encryptedFile.fileKey, ); - const uploadedFile =await this.uploadFile(uploadFile); + const uploadedFile =await NetworkClient.uploadFile(uploadFile); const decryptedFile=await decryptFile(uploadedFile, collection); this.existingFiles.push(decryptedFile); @@ -510,28 +503,30 @@ class UploadService { const uploadPartCount = Math.ceil( chunkCount / CHUNKS_COMBINED_FOR_UPLOAD, ); - const filePartUploadURLs = await this.fetchMultipartUploadURLs( + const filePartUploadURLs = await NetworkClient.fetchMultipartUploadURLs( uploadPartCount, ); - fileObjectKey = await this.putFileInParts( + fileObjectKey = await NetworkClient.putFileInParts( filePartUploadURLs, stream, file.filename, uploadPartCount, + this.trackUploadProgress, ); } else { const fileUploadURL = await this.getUploadURL(); - fileObjectKey = await this.putFile( + const progressTracker=this.trackUploadProgress.bind(this, file.filename); + fileObjectKey = await NetworkClient.putFile( fileUploadURL, file.file.encryptedData, - file.filename, + progressTracker, ); } const thumbnailUploadURL = await this.getUploadURL(); - const thumbnailObjectKey = await this.putFile( + const thumbnailObjectKey = await NetworkClient.putFile( thumbnailUploadURL, file.thumbnail.encryptedData as Uint8Array, - null, + ()=>null, ); const backupedFile: BackupedFile = { @@ -556,8 +551,8 @@ class UploadService { collection: Collection, backupedFile: BackupedFile, fileKey: B64EncryptionResult, - ): uploadFile { - const uploadFile: uploadFile = { + ): UploadFile { + const uploadFile: UploadFile = { collectionID: collection.id, encryptedKey: fileKey.encryptedData, keyDecryptionNonce: fileKey.nonce, @@ -567,26 +562,6 @@ class UploadService { return uploadFile; } - private async uploadFile(uploadFile: uploadFile):Promise { - try { - const token = getToken(); - if (!token) { - return; - } - const response = await retryAsyncFunction(()=>HTTPService.post( - `${ENDPOINT}/files`, - uploadFile, - null, - { - 'X-Auth-Token': token, - }, - )); - return response.data; - } catch (e) { - logError(e, 'upload Files Failed'); - throw e; - } - } private async seedMetadataMap(receivedFile: globalThis.File) { try { @@ -836,189 +811,12 @@ class UploadService { private async getUploadURL() { if (this.uploadURLs.length === 0) { - await this.fetchUploadURLs(); + await NetworkClient.fetchUploadURLs(this.totalFileCount-this.filesCompleted, this.uploadURLs); } return this.uploadURLs.pop(); } - private async fetchUploadURLs(): Promise { - try { - if (!this.uploadURLFetchInProgress) { - try { - const token = getToken(); - if (!token) { - return; - } - this.uploadURLFetchInProgress = HTTPService.get( - `${ENDPOINT}/files/upload-urls`, - { - count: Math.min( - MAX_URL_REQUESTS, - (this.totalFileCount - this.filesCompleted) * 2, - ), - }, - { 'X-Auth-Token': token }, - ); - const response = await this.uploadURLFetchInProgress; - this.uploadURLs.push(...response.data['urls']); - } finally { - this.uploadURLFetchInProgress = null; - } - } - return this.uploadURLFetchInProgress; - } catch (e) { - logError(e, 'fetch upload-url failed '); - throw e; - } - } - private async fetchMultipartUploadURLs( - count: number, - ): Promise { - try { - const token = getToken(); - if (!token) { - return; - } - const response = await HTTPService.get( - `${ENDPOINT}/files/multipart-upload-urls`, - { - count, - }, - { 'X-Auth-Token': token }, - ); - - return response.data['urls']; - } catch (e) { - logError(e, 'fetch multipart-upload-url failed'); - throw e; - } - } - - private async putFile( - fileUploadURL: UploadURL, - file: Uint8Array, - filename: string, - ): Promise { - try { - await retryAsyncFunction(()=> - HTTPService.put( - fileUploadURL.url, - file, - null, - null, - this.trackUploadProgress(filename), - ), - ); - return fileUploadURL.objectKey; - } catch (e) { - logError(e, 'putFile to dataStore failed '); - throw e; - } - } - - private async putFileInParts( - multipartUploadURLs: MultipartUploadURLs, - file: ReadableStream, - filename: string, - uploadPartCount: number, - ) { - try { - const streamEncryptedFileReader = file.getReader(); - const percentPerPart = Math.round( - RANDOM_PERCENTAGE_PROGRESS_FOR_PUT() / uploadPartCount, - ); - const resParts = []; - for (const [ - index, - fileUploadURL, - ] of multipartUploadURLs.partURLs.entries()) { - const combinedChunks = []; - for (let i = 0; i < CHUNKS_COMBINED_FOR_UPLOAD; i++) { - const { done, value: chunk } = - await streamEncryptedFileReader.read(); - if (done) { - break; - } - for (let index = 0; index < chunk.length; index++) { - combinedChunks.push(chunk[index]); - } - } - const uploadChunk = Uint8Array.from(combinedChunks); - const response=await retryAsyncFunction(async ()=>{ - const resp =await HTTPService.put( - fileUploadURL, - uploadChunk, - null, - null, - this.trackUploadProgress(filename, percentPerPart, index), - ); - if (!resp?.headers?.etag) { - const err=Error('no header/etag present in response body'); - logError(err); - throw err; - } - return resp; - }); - resParts.push({ - PartNumber: index + 1, - ETag: response.headers.etag, - }); - } - const options = { compact: true, ignoreComment: true, spaces: 4 }; - const body = convert.js2xml( - { CompleteMultipartUpload: { Part: resParts } }, - options, - ); - await retryAsyncFunction(()=> - HTTPService.post(multipartUploadURLs.completeURL, body, null, { - 'content-type': 'text/xml', - }), - ); - return multipartUploadURLs.objectKey; - } catch (e) { - logError(e, 'put file in parts failed'); - throw e; - } - } - - private trackUploadProgress( - filename, - percentPerPart = RANDOM_PERCENTAGE_PROGRESS_FOR_PUT(), - index = 0, - ) { - const cancel={ exec: null }; - let timeout=null; - const resetTimeout=()=>{ - if (timeout) { - clearTimeout(timeout); - } - timeout=setTimeout(()=>cancel.exec(), 30*1000); - }; - return { - cancel, - onUploadProgress: (event) => { - filename && - this.fileProgress.set( - filename, - Math.min( - Math.round( - percentPerPart * index + - (percentPerPart * event.loaded) / - event.total, - ), - 98, - ), - ); - this.updateProgressBarUI(); - if (event.loaded===event.total) { - clearTimeout(timeout); - } else { - resetTimeout(); - } - }, - }; - } private async getExifData( reader: FileReader, receivedFile: globalThis.File, @@ -1102,6 +900,43 @@ class UploadService { return dd; } + private trackUploadProgress( + filename:string, + percentPerPart = RANDOM_PERCENTAGE_PROGRESS_FOR_PUT(), + index = 0, + ) { + const cancel={ exec: null }; + let timeout=null; + const resetTimeout=()=>{ + if (timeout) { + clearTimeout(timeout); + } + timeout=setTimeout(()=>cancel.exec(), 30*1000); + }; + return { + cancel, + onUploadProgress: (event) => { + filename && + this.fileProgress.set( + filename, + Math.min( + Math.round( + percentPerPart * index + + (percentPerPart * event.loaded) / + event.total, + ), + 98, + ), + ); + this.updateProgressBarUI(); + if (event.loaded===event.total) { + clearTimeout(timeout); + } else { + resetTimeout(); + } + }, + }; + } } export default new UploadService(); From 06ed9bc54a183c8f61b454e3023a9505620b7cfa Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Fri, 6 Aug 2021 10:14:48 +0530 Subject: [PATCH 004/231] created exif service --- src/services/upload/exifService.ts | 98 ++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 src/services/upload/exifService.ts diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts new file mode 100644 index 000000000..96f8f95b2 --- /dev/null +++ b/src/services/upload/exifService.ts @@ -0,0 +1,98 @@ +import EXIF from 'exif-js'; +import { FILE_TYPE } from 'pages/gallery'; +import { logError } from 'utils/sentry'; +import { NULL_LOCATION, Location } from './fileReadClient'; + +const SOUTH_DIRECTION = 'S'; +const WEST_DIRECTION = 'W'; + +interface ParsedEXIFData { + location: Location; + creationTime: number; +} + + +export async function getExifData( + reader: FileReader, + receivedFile: globalThis.File, + fileType: FILE_TYPE, +): Promise { + try { + if (fileType === FILE_TYPE.VIDEO) { + // Todo extract exif data from videos + return { location: NULL_LOCATION, creationTime: null }; + } + const exifData: any = await new Promise((resolve) => { + reader.onload = () => { + resolve(EXIF.readFromBinaryFile(reader.result)); + }; + reader.readAsArrayBuffer(receivedFile); + }); + if (!exifData) { + return { location: NULL_LOCATION, creationTime: null }; + } + return { + location: getEXIFLocation(exifData), + creationTime: getUNIXTime(exifData), + }; + } catch (e) { + logError(e, 'error reading exif data'); + throw e; + } +} + +function getUNIXTime(exifData: any) { + const dateString: string = exifData.DateTimeOriginal || exifData.DateTime; + if (!dateString || dateString==='0000:00:00 00:00:00') { + return null; + } + const parts = dateString.split(' ')[0].split(':'); + const date = new Date( + Number(parts[0]), + Number(parts[1]) - 1, + Number(parts[2]), + ); + return date.getTime() * 1000; +} + +function getEXIFLocation(exifData): Location { + if (!exifData.GPSLatitude) { + return NULL_LOCATION; + } + + const latDegree = exifData.GPSLatitude[0]; + const latMinute = exifData.GPSLatitude[1]; + const latSecond = exifData.GPSLatitude[2]; + + const lonDegree = exifData.GPSLongitude[0]; + const lonMinute = exifData.GPSLongitude[1]; + const lonSecond = exifData.GPSLongitude[2]; + + const latDirection = exifData.GPSLatitudeRef; + const lonDirection = exifData.GPSLongitudeRef; + + const latFinal = convertDMSToDD( + latDegree, + latMinute, + latSecond, + latDirection, + ); + + const lonFinal = convertDMSToDD( + lonDegree, + lonMinute, + lonSecond, + lonDirection, + ); + return { latitude: latFinal * 1.0, longitude: lonFinal * 1.0 }; +} + +function convertDMSToDD(degrees, minutes, seconds, direction) { + let dd = degrees + minutes / 60 + seconds / 3600; + + if (direction === SOUTH_DIRECTION || direction === WEST_DIRECTION) { + dd = dd * -1; + } + + return dd; +} From b2be864bc72e856844d69f8759d435ca13910795 Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Fri, 6 Aug 2021 10:16:38 +0530 Subject: [PATCH 005/231] created fileReadClient --- src/services/upload/fileReadClient.ts | 350 ++++++++++++++++++++++++++ 1 file changed, 350 insertions(+) create mode 100644 src/services/upload/fileReadClient.ts diff --git a/src/services/upload/fileReadClient.ts b/src/services/upload/fileReadClient.ts new file mode 100644 index 000000000..f9d12dfd1 --- /dev/null +++ b/src/services/upload/fileReadClient.ts @@ -0,0 +1,350 @@ +import { FILE_TYPE } from 'pages/gallery'; +import { ENCRYPTION_CHUNK_SIZE } from 'types'; +import { THUMBNAIL_GENERATION_FAILED } from 'utils/common/errorUtil'; +import { fileIsHEIC, convertHEIC2JPEG } from 'utils/file'; +import { logError } from 'utils/sentry'; +import { getExifData } from './exifService'; + +const TYPE_VIDEO = 'video'; +const TYPE_HEIC = 'HEIC'; +export const TYPE_IMAGE = 'image'; +const MIN_STREAM_FILE_SIZE = 20 * 1024 * 1024; +const EDITED_FILE_SUFFIX = '-edited'; +const THUMBNAIL_HEIGHT = 720; +const MAX_ATTEMPTS = 3; +const MIN_THUMBNAIL_SIZE = 50000; + + +export const NULL_LOCATION: Location = { latitude: null, longitude: null }; +const WAIT_TIME_THUMBNAIL_GENERATION = 10 * 1000; +const NULL_PARSED_METADATA_JSON:ParsedMetaDataJSON={ title: null, creationTime: null, modificationTime: null, location: NULL_LOCATION }; + +export interface Location { + latitude: number; + longitude: number; +} + + +export interface ParsedMetaDataJSON{ + title:string; + creationTime:number; + modificationTime:number; + location:Location; +} + +export default async function readFile(reader: FileReader, receivedFile: globalThis.File, metadataMap:Map) { + try { + const { thumbnail, hasStaticThumbnail } = await generateThumbnail( + reader, + receivedFile, + ); + + let fileType: FILE_TYPE; + switch (receivedFile.type.split('/')[0]) { + case TYPE_IMAGE: + fileType = FILE_TYPE.IMAGE; + break; + case TYPE_VIDEO: + fileType = FILE_TYPE.VIDEO; + break; + default: + fileType = FILE_TYPE.OTHERS; + } + if ( + fileType === FILE_TYPE.OTHERS && + receivedFile.type.length === 0 && + receivedFile.name.endsWith(TYPE_HEIC) + ) { + fileType = FILE_TYPE.IMAGE; + } + + const { location, creationTime } = await getExifData( + reader, + receivedFile, + fileType, + ); + let receivedFileOriginalName = receivedFile.name; + if (receivedFile.name.endsWith(EDITED_FILE_SUFFIX)) { + receivedFileOriginalName = receivedFile.name.slice( + 0, + -1 * EDITED_FILE_SUFFIX.length, + ); + } + const metadata = Object.assign( + { + title: receivedFile.name, + creationTime: + creationTime || receivedFile.lastModified * 1000, + modificationTime: receivedFile.lastModified * 1000, + latitude: location?.latitude, + longitude: location?.latitude, + fileType, + }, + metadataMap.get(receivedFileOriginalName), + ); + if (hasStaticThumbnail) { + metadata['hasStaticThumbnail'] = hasStaticThumbnail; + } + const filedata = + receivedFile.size > MIN_STREAM_FILE_SIZE ? + getFileStream(reader, receivedFile) : + await getUint8ArrayView(reader, receivedFile); + + return { + filedata, + thumbnail, + metadata, + }; + } catch (e) { + logError(e, 'error reading files'); + throw e; + } +} + +async function generateThumbnail( + reader: FileReader, + file: globalThis.File, +): Promise<{ thumbnail: Uint8Array, hasStaticThumbnail: boolean }> { + try { + let hasStaticThumbnail = false; + const canvas = document.createElement('canvas'); + // eslint-disable-next-line camelcase + const canvas_CTX = canvas.getContext('2d'); + let imageURL = null; + let timeout = null; + try { + if (file.type.match(TYPE_IMAGE) || fileIsHEIC(file.name)) { + if (fileIsHEIC(file.name)) { + file = new globalThis.File( + [await convertHEIC2JPEG(file)], + null, + null, + ); + } + let image = new Image(); + imageURL = URL.createObjectURL(file); + image.setAttribute('src', imageURL); + await new Promise((resolve, reject) => { + image.onload = () => { + try { + const thumbnailWidth = + (image.width * THUMBNAIL_HEIGHT) / image.height; + canvas.width = thumbnailWidth; + canvas.height = THUMBNAIL_HEIGHT; + canvas_CTX.drawImage( + image, + 0, + 0, + thumbnailWidth, + THUMBNAIL_HEIGHT, + ); + image = null; + clearTimeout(timeout); + resolve(null); + } catch (e) { + reject(e); + logError(e); + reject(Error(`${THUMBNAIL_GENERATION_FAILED} err: ${e}`)); + } + }; + timeout = setTimeout( + () => + reject( + Error(`wait time exceeded for format ${file.name.split('.').slice(-1)[0]}`), + ), + WAIT_TIME_THUMBNAIL_GENERATION, + ); + }); + } else { + await new Promise((resolve, reject) => { + let video = document.createElement('video'); + imageURL = URL.createObjectURL(file); + video.addEventListener('timeupdate', function () { + try { + if (!video) { + return; + } + const thumbnailWidth = + (video.videoWidth * THUMBNAIL_HEIGHT) / + video.videoHeight; + canvas.width = thumbnailWidth; + canvas.height = THUMBNAIL_HEIGHT; + canvas_CTX.drawImage( + video, + 0, + 0, + thumbnailWidth, + THUMBNAIL_HEIGHT, + ); + video = null; + clearTimeout(timeout); + resolve(null); + } catch (e) { + reject(e); + logError(e); + reject(Error(`${THUMBNAIL_GENERATION_FAILED} err: ${e}`)); + } + }); + video.preload = 'metadata'; + video.src = imageURL; + video.currentTime = 3; + setTimeout( + () => + reject(Error(`wait time exceeded for format ${file.name.split('.').slice(-1)[0]}`)), + WAIT_TIME_THUMBNAIL_GENERATION, + ); + }); + } + URL.revokeObjectURL(imageURL); + } catch (e) { + logError(e); + // ignore and set staticThumbnail + hasStaticThumbnail = true; + } + let thumbnailBlob = null; + let attempts = 0; + let quality = 1; + + do { + attempts++; + quality /= 2; + thumbnailBlob = await new Promise((resolve) => { + canvas.toBlob( + function (blob) { + resolve(blob); + }, + 'image/jpeg', + quality, + ); + }); + thumbnailBlob = thumbnailBlob ?? new Blob([]); + } while ( + thumbnailBlob.size > MIN_THUMBNAIL_SIZE && + attempts <= MAX_ATTEMPTS + ); + const thumbnail = await getUint8ArrayView( + reader, + thumbnailBlob, + ); + return { thumbnail, hasStaticThumbnail }; + } catch (e) { + logError(e, 'Error generating thumbnail'); + throw e; + } +} + +function getFileStream(reader: FileReader, file: globalThis.File) { + const self = this; + const fileChunkReader = (async function* fileChunkReaderMaker( + fileSize, + self, + ) { + let offset = 0; + while (offset < fileSize) { + const blob = file.slice(offset, ENCRYPTION_CHUNK_SIZE + offset); + const fileChunk = await self.getUint8ArrayView(reader, blob); + yield fileChunk; + offset += ENCRYPTION_CHUNK_SIZE; + } + return null; + })(file.size, self); + return { + stream: new ReadableStream({ + async pull(controller: ReadableStreamDefaultController) { + const chunk = await fileChunkReader.next(); + if (chunk.done) { + controller.close(); + } else { + controller.enqueue(chunk.value); + } + }, + }), + chunkCount: Math.ceil(file.size / ENCRYPTION_CHUNK_SIZE), + }; +} +async function getUint8ArrayView( + reader: FileReader, + file: Blob, +): Promise { + try { + return await new Promise((resolve, reject) => { + reader.onabort = () => reject(Error('file reading was aborted')); + reader.onerror = () => reject(Error('file reading has failed')); + reader.onload = () => { + // Do whatever you want with the file contents + const result = + typeof reader.result === 'string' ? + new TextEncoder().encode(reader.result) : + new Uint8Array(reader.result); + resolve(result); + }; + reader.readAsArrayBuffer(file); + }); + } catch (e) { + logError(e, 'error reading file to byte-array'); + throw e; + } +} + + +export async function parseMetadataJSON(receivedFile: globalThis.File) { + try { + const metadataJSON: object = await new Promise( + (resolve, reject) => { + const reader = new FileReader(); + reader.onabort = () => reject(Error('file reading was aborted')); + reader.onerror = () => reject(Error('file reading has failed')); + reader.onload = () => { + const result = + typeof reader.result !== 'string' ? + new TextDecoder().decode(reader.result) : + reader.result; + resolve(JSON.parse(result)); + }; + reader.readAsText(receivedFile); + }, + ); + + const parsedMetaDataJSON:ParsedMetaDataJSON = NULL_PARSED_METADATA_JSON; + if (!metadataJSON || !metadataJSON['title']) { + return; + } + + parsedMetaDataJSON.title=metadataJSON['title']; + if ( + metadataJSON['photoTakenTime'] && + metadataJSON['photoTakenTime']['timestamp'] + ) { + parsedMetaDataJSON.creationTime = + metadataJSON['photoTakenTime']['timestamp'] * 1000000; + } + if ( + metadataJSON['modificationTime'] && + metadataJSON['modificationTime']['timestamp'] + ) { + parsedMetaDataJSON.modificationTime = + metadataJSON['modificationTime']['timestamp'] * 1000000; + } + let locationData:Location = NULL_LOCATION; + if ( + metadataJSON['geoData'] && + (metadataJSON['geoData']['latitude'] !== 0.0 || + metadataJSON['geoData']['longitude'] !== 0.0) + ) { + locationData = metadataJSON['geoData']; + } else if ( + metadataJSON['geoDataExif'] && + (metadataJSON['geoDataExif']['latitude'] !== 0.0 || + metadataJSON['geoDataExif']['longitude'] !== 0.0) + ) { + locationData = metadataJSON['geoDataExif']; + } + if (locationData !== null) { + parsedMetaDataJSON.location=locationData; + } + return parsedMetaDataJSON; + } catch (e) { + logError(e); + // ignore + } +} From 2d8f16742b1d97647344e4cbdab607b26762cb3a Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Fri, 6 Aug 2021 10:17:08 +0530 Subject: [PATCH 006/231] refactored code to use fileReadClient --- src/services/upload/uploadService.ts | 435 +-------------------------- 1 file changed, 8 insertions(+), 427 deletions(-) diff --git a/src/services/upload/uploadService.ts b/src/services/upload/uploadService.ts index eee918c23..9e7923ceb 100644 --- a/src/services/upload/uploadService.ts +++ b/src/services/upload/uploadService.ts @@ -1,17 +1,12 @@ -import EXIF from 'exif-js'; import { File, fileAttribute } from '../fileService'; import { Collection } from '../collectionService'; import { FILE_TYPE, SetFiles } from 'pages/gallery'; import { handleError, parseError, - THUMBNAIL_GENERATION_FAILED, } from 'utils/common/errorUtil'; import { ComlinkWorker, getDedicatedCryptoWorker } from 'utils/crypto'; -import { ENCRYPTION_CHUNK_SIZE } from 'types'; import { - fileIsHEIC, - convertHEIC2JPEG, sortFilesIntoCollections, sortFiles, decryptFile, @@ -21,23 +16,12 @@ import { logError } from 'utils/sentry'; import localForage from 'utils/storage/localForage'; import { sleep } from 'utils/common'; import NetworkClient, { UploadURL } from './networkClient'; +import readFile, { ParsedMetaDataJSON, parseMetadataJSON } from './fileReadClient'; -const THUMBNAIL_HEIGHT = 720; -const MAX_ATTEMPTS = 3; -const MIN_THUMBNAIL_SIZE = 50000; const MAX_CONCURRENT_UPLOADS = 4; -const TYPE_IMAGE = 'image'; -const TYPE_VIDEO = 'video'; -const TYPE_HEIC = 'HEIC'; const TYPE_JSON = 'json'; -const SOUTH_DIRECTION = 'S'; -const WEST_DIRECTION = 'W'; -const MIN_STREAM_FILE_SIZE = 20 * 1024 * 1024; -const NULL_LOCATION: Location = { latitude: null, longitude: null }; -const WAIT_TIME_THUMBNAIL_GENERATION = 10 * 1000; const FILE_UPLOAD_COMPLETED = 100; -const EDITED_FILE_SUFFIX = '-edited'; const TwoSecondInMillSeconds = 2000; export const RANDOM_PERCENTAGE_PROGRESS_FOR_PUT = () => 90 + 10 * Math.random(); export const CHUNKS_COMBINED_FOR_UPLOAD = 5; @@ -48,14 +32,6 @@ export enum FileUploadErrorCode { UNSUPPORTED = -3, } -interface Location { - latitude: number; - longitude: number; -} -interface ParsedEXIFData { - location: Location; - creationTime: number; -} export interface FileWithCollection { file: globalThis.File; collection: Collection; @@ -113,6 +89,8 @@ interface ProcessedFile { } interface BackupedFile extends Omit { } +export type MetadataMap = Map + export interface UploadFile extends BackupedFile { collectionID: number; encryptedKey: string; @@ -129,12 +107,11 @@ export enum UPLOAD_STAGES { class UploadService { private cryptoWorkers = new Array(MAX_CONCURRENT_UPLOADS); private uploadURLs: UploadURL[] = []; - private uploadURLFetchInProgress: Promise = null; private perFileProgress: number; private filesCompleted: number; private totalFileCount: number; private fileProgress: Map; - private metadataMap: Map; + private metadataMap: Map; private filesToBeUploaded: FileWithCollection[]; private progressBarProps; private failedFiles: FileWithCollection[]; @@ -153,7 +130,7 @@ class UploadService { this.filesCompleted = 0; this.fileProgress = new Map(); this.failedFiles = []; - this.metadataMap = new Map(); + this.metadataMap = new Map(); this.progressBarProps = progressBarProps; this.existingFiles=existingFiles; this.existingFilesCollectionWise = sortFilesIntoCollections(existingFiles); @@ -183,7 +160,8 @@ class UploadService { this.filesCompleted = 0; for (const rawFile of metadataFiles) { - await this.seedMetadataMap(rawFile); + const parsedMetaDataJSON=await parseMetadataJSON(rawFile); + this.metadataMap.set(parsedMetaDataJSON.title, parsedMetaDataJSON); this.filesCompleted++; this.updateProgressBarUI(); } @@ -248,7 +226,7 @@ class UploadService { let encryptedFile: EncryptedFile=null; try { // read the file into memory - file = await this.readFile(reader, rawFile); + file = await readFile(reader, rawFile, this.metadataMap); if (this.fileAlreadyInCollection(file, collection)) { // set progress to -2 indicating that file upload was skipped @@ -359,74 +337,6 @@ class UploadService { } } - private async readFile(reader: FileReader, receivedFile: globalThis.File) { - try { - const { thumbnail, hasStaticThumbnail } = await this.generateThumbnail( - reader, - receivedFile, - ); - - let fileType: FILE_TYPE; - switch (receivedFile.type.split('/')[0]) { - case TYPE_IMAGE: - fileType = FILE_TYPE.IMAGE; - break; - case TYPE_VIDEO: - fileType = FILE_TYPE.VIDEO; - break; - default: - fileType = FILE_TYPE.OTHERS; - } - if ( - fileType === FILE_TYPE.OTHERS && - receivedFile.type.length === 0 && - receivedFile.name.endsWith(TYPE_HEIC) - ) { - fileType = FILE_TYPE.IMAGE; - } - - const { location, creationTime } = await this.getExifData( - reader, - receivedFile, - fileType, - ); - let receivedFileOriginalName = receivedFile.name; - if (receivedFile.name.endsWith(EDITED_FILE_SUFFIX)) { - receivedFileOriginalName = receivedFile.name.slice( - 0, - -1 * EDITED_FILE_SUFFIX.length, - ); - } - const metadata = Object.assign( - { - title: receivedFile.name, - creationTime: - creationTime || receivedFile.lastModified * 1000, - modificationTime: receivedFile.lastModified * 1000, - latitude: location?.latitude, - longitude: location?.latitude, - fileType, - }, - this.metadataMap.get(receivedFileOriginalName), - ); - if (hasStaticThumbnail) { - metadata['hasStaticThumbnail'] = hasStaticThumbnail; - } - const filedata = - receivedFile.size > MIN_STREAM_FILE_SIZE ? - this.getFileStream(reader, receivedFile) : - await this.getUint8ArrayView(reader, receivedFile); - - return { - filedata, - thumbnail, - metadata, - }; - } catch (e) { - logError(e, 'error reading files'); - throw e; - } - } private async encryptFile( worker: any, @@ -563,252 +473,6 @@ class UploadService { } - private async seedMetadataMap(receivedFile: globalThis.File) { - try { - const metadataJSON: object = await new Promise( - (resolve, reject) => { - const reader = new FileReader(); - reader.onabort = () => reject(Error('file reading was aborted')); - reader.onerror = () => reject(Error('file reading has failed')); - reader.onload = () => { - const result = - typeof reader.result !== 'string' ? - new TextDecoder().decode(reader.result) : - reader.result; - resolve(JSON.parse(result)); - }; - reader.readAsText(receivedFile); - }, - ); - - const metaDataObject = {}; - if (!metadataJSON) { - return; - } - if ( - metadataJSON['photoTakenTime'] && - metadataJSON['photoTakenTime']['timestamp'] - ) { - metaDataObject['creationTime'] = - metadataJSON['photoTakenTime']['timestamp'] * 1000000; - } - if ( - metadataJSON['modificationTime'] && - metadataJSON['modificationTime']['timestamp'] - ) { - metaDataObject['modificationTime'] = - metadataJSON['modificationTime']['timestamp'] * 1000000; - } - let locationData = null; - if ( - metadataJSON['geoData'] && - (metadataJSON['geoData']['latitude'] !== 0.0 || - metadataJSON['geoData']['longitude'] !== 0.0) - ) { - locationData = metadataJSON['geoData']; - } else if ( - metadataJSON['geoDataExif'] && - (metadataJSON['geoDataExif']['latitude'] !== 0.0 || - metadataJSON['geoDataExif']['longitude'] !== 0.0) - ) { - locationData = metadataJSON['geoDataExif']; - } - if (locationData !== null) { - metaDataObject['latitude'] = locationData['latitude']; - metaDataObject['longitude'] = locationData['longitude']; - } - this.metadataMap.set(metadataJSON['title'], metaDataObject); - } catch (e) { - logError(e); - // ignore - } - } - private async generateThumbnail( - reader: FileReader, - file: globalThis.File, - ): Promise<{ thumbnail: Uint8Array, hasStaticThumbnail: boolean }> { - try { - let hasStaticThumbnail = false; - const canvas = document.createElement('canvas'); - // eslint-disable-next-line camelcase - const canvas_CTX = canvas.getContext('2d'); - let imageURL = null; - let timeout = null; - try { - if (file.type.match(TYPE_IMAGE) || fileIsHEIC(file.name)) { - if (fileIsHEIC(file.name)) { - file = new globalThis.File( - [await convertHEIC2JPEG(file)], - null, - null, - ); - } - let image = new Image(); - imageURL = URL.createObjectURL(file); - image.setAttribute('src', imageURL); - await new Promise((resolve, reject) => { - image.onload = () => { - try { - const thumbnailWidth = - (image.width * THUMBNAIL_HEIGHT) / image.height; - canvas.width = thumbnailWidth; - canvas.height = THUMBNAIL_HEIGHT; - canvas_CTX.drawImage( - image, - 0, - 0, - thumbnailWidth, - THUMBNAIL_HEIGHT, - ); - image = null; - clearTimeout(timeout); - resolve(null); - } catch (e) { - reject(e); - logError(e); - reject(Error(`${THUMBNAIL_GENERATION_FAILED} err: ${e}`)); - } - }; - timeout = setTimeout( - () => - reject( - Error(`wait time exceeded for format ${file.name.split('.').slice(-1)[0]}`), - ), - WAIT_TIME_THUMBNAIL_GENERATION, - ); - }); - } else { - await new Promise((resolve, reject) => { - let video = document.createElement('video'); - imageURL = URL.createObjectURL(file); - video.addEventListener('timeupdate', function () { - try { - if (!video) { - return; - } - const thumbnailWidth = - (video.videoWidth * THUMBNAIL_HEIGHT) / - video.videoHeight; - canvas.width = thumbnailWidth; - canvas.height = THUMBNAIL_HEIGHT; - canvas_CTX.drawImage( - video, - 0, - 0, - thumbnailWidth, - THUMBNAIL_HEIGHT, - ); - video = null; - clearTimeout(timeout); - resolve(null); - } catch (e) { - reject(e); - logError(e); - reject(Error(`${THUMBNAIL_GENERATION_FAILED} err: ${e}`)); - } - }); - video.preload = 'metadata'; - video.src = imageURL; - video.currentTime = 3; - setTimeout( - () => - reject(Error(`wait time exceeded for format ${file.name.split('.').slice(-1)[0]}`)), - WAIT_TIME_THUMBNAIL_GENERATION, - ); - }); - } - URL.revokeObjectURL(imageURL); - } catch (e) { - logError(e); - // ignore and set staticThumbnail - hasStaticThumbnail = true; - } - let thumbnailBlob = null; - let attempts = 0; - let quality = 1; - - do { - attempts++; - quality /= 2; - thumbnailBlob = await new Promise((resolve) => { - canvas.toBlob( - function (blob) { - resolve(blob); - }, - 'image/jpeg', - quality, - ); - }); - thumbnailBlob = thumbnailBlob ?? new Blob([]); - } while ( - thumbnailBlob.size > MIN_THUMBNAIL_SIZE && - attempts <= MAX_ATTEMPTS - ); - const thumbnail = await this.getUint8ArrayView( - reader, - thumbnailBlob, - ); - return { thumbnail, hasStaticThumbnail }; - } catch (e) { - logError(e, 'Error generating thumbnail'); - throw e; - } - } - - private getFileStream(reader: FileReader, file: globalThis.File) { - const self = this; - const fileChunkReader = (async function* fileChunkReaderMaker( - fileSize, - self, - ) { - let offset = 0; - while (offset < fileSize) { - const blob = file.slice(offset, ENCRYPTION_CHUNK_SIZE + offset); - const fileChunk = await self.getUint8ArrayView(reader, blob); - yield fileChunk; - offset += ENCRYPTION_CHUNK_SIZE; - } - return null; - })(file.size, self); - return { - stream: new ReadableStream({ - async pull(controller: ReadableStreamDefaultController) { - const chunk = await fileChunkReader.next(); - if (chunk.done) { - controller.close(); - } else { - controller.enqueue(chunk.value); - } - }, - }), - chunkCount: Math.ceil(file.size / ENCRYPTION_CHUNK_SIZE), - }; - } - - private async getUint8ArrayView( - reader: FileReader, - file: Blob, - ): Promise { - try { - return await new Promise((resolve, reject) => { - reader.onabort = () => reject(Error('file reading was aborted')); - reader.onerror = () => reject(Error('file reading has failed')); - reader.onload = () => { - // Do whatever you want with the file contents - const result = - typeof reader.result === 'string' ? - new TextEncoder().encode(reader.result) : - new Uint8Array(reader.result); - resolve(result); - }; - reader.readAsArrayBuffer(file); - }); - } catch (e) { - logError(e, 'error reading file to byte-array'); - throw e; - } - } - private async getUploadURL() { if (this.uploadURLs.length === 0) { await NetworkClient.fetchUploadURLs(this.totalFileCount-this.filesCompleted, this.uploadURLs); @@ -817,89 +481,6 @@ class UploadService { } - private async getExifData( - reader: FileReader, - receivedFile: globalThis.File, - fileType: FILE_TYPE, - ): Promise { - try { - if (fileType === FILE_TYPE.VIDEO) { - // Todo extract exif data from videos - return { location: NULL_LOCATION, creationTime: null }; - } - const exifData: any = await new Promise((resolve) => { - reader.onload = () => { - resolve(EXIF.readFromBinaryFile(reader.result)); - }; - reader.readAsArrayBuffer(receivedFile); - }); - if (!exifData) { - return { location: NULL_LOCATION, creationTime: null }; - } - return { - location: this.getEXIFLocation(exifData), - creationTime: this.getUNIXTime(exifData), - }; - } catch (e) { - logError(e, 'error reading exif data'); - throw e; - } - } - private getUNIXTime(exifData: any) { - const dateString: string = exifData.DateTimeOriginal || exifData.DateTime; - if (!dateString || dateString==='0000:00:00 00:00:00') { - return null; - } - const parts = dateString.split(' ')[0].split(':'); - const date = new Date( - Number(parts[0]), - Number(parts[1]) - 1, - Number(parts[2]), - ); - return date.getTime() * 1000; - } - - private getEXIFLocation(exifData): Location { - if (!exifData.GPSLatitude) { - return NULL_LOCATION; - } - - const latDegree = exifData.GPSLatitude[0]; - const latMinute = exifData.GPSLatitude[1]; - const latSecond = exifData.GPSLatitude[2]; - - const lonDegree = exifData.GPSLongitude[0]; - const lonMinute = exifData.GPSLongitude[1]; - const lonSecond = exifData.GPSLongitude[2]; - - const latDirection = exifData.GPSLatitudeRef; - const lonDirection = exifData.GPSLongitudeRef; - - const latFinal = this.convertDMSToDD( - latDegree, - latMinute, - latSecond, - latDirection, - ); - - const lonFinal = this.convertDMSToDD( - lonDegree, - lonMinute, - lonSecond, - lonDirection, - ); - return { latitude: latFinal * 1.0, longitude: lonFinal * 1.0 }; - } - - private convertDMSToDD(degrees, minutes, seconds, direction) { - let dd = degrees + minutes / 60 + seconds / 3600; - - if (direction === SOUTH_DIRECTION || direction === WEST_DIRECTION) { - dd = dd * -1; - } - - return dd; - } private trackUploadProgress( filename:string, percentPerPart = RANDOM_PERCENTAGE_PROGRESS_FOR_PUT(), From 5891a19547566678ffc3afdc9aeb0720962a4b46 Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Fri, 6 Aug 2021 10:28:32 +0530 Subject: [PATCH 007/231] remove console logs --- src/services/upload/networkClient.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/services/upload/networkClient.ts b/src/services/upload/networkClient.ts index e023679e1..c7e2d0b9c 100644 --- a/src/services/upload/networkClient.ts +++ b/src/services/upload/networkClient.ts @@ -99,7 +99,6 @@ class NetworkClient { progressTracker:()=>any, ): Promise { try { - console.log(fileUploadURL, file); await retryAsyncFunction(()=> HTTPService.put( fileUploadURL.url, From d83462b3eaa5b38d2cadcb7721b715a819f98427 Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Fri, 6 Aug 2021 10:46:33 +0530 Subject: [PATCH 008/231] refactored out fileChunkReaderMaker --- src/services/upload/fileReadClient.ts | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/services/upload/fileReadClient.ts b/src/services/upload/fileReadClient.ts index f9d12dfd1..237c99ba6 100644 --- a/src/services/upload/fileReadClient.ts +++ b/src/services/upload/fileReadClient.ts @@ -234,20 +234,7 @@ async function generateThumbnail( } function getFileStream(reader: FileReader, file: globalThis.File) { - const self = this; - const fileChunkReader = (async function* fileChunkReaderMaker( - fileSize, - self, - ) { - let offset = 0; - while (offset < fileSize) { - const blob = file.slice(offset, ENCRYPTION_CHUNK_SIZE + offset); - const fileChunk = await self.getUint8ArrayView(reader, blob); - yield fileChunk; - offset += ENCRYPTION_CHUNK_SIZE; - } - return null; - })(file.size, self); + const fileChunkReader = fileChunkReaderMaker(reader, file); return { stream: new ReadableStream({ async pull(controller: ReadableStreamDefaultController) { @@ -262,6 +249,18 @@ function getFileStream(reader: FileReader, file: globalThis.File) { chunkCount: Math.ceil(file.size / ENCRYPTION_CHUNK_SIZE), }; } + +async function* fileChunkReaderMaker(reader:FileReader, file:globalThis.File) { + let offset = 0; + while (offset < file.size) { + const blob = file.slice(offset, ENCRYPTION_CHUNK_SIZE + offset); + const fileChunk = await getUint8ArrayView(reader, blob); + yield fileChunk; + offset += ENCRYPTION_CHUNK_SIZE; + } + return null; +} + async function getUint8ArrayView( reader: FileReader, file: Blob, From 5fde5ada71175ed8228112e7af9352f37293e5ff Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Mon, 9 Aug 2021 14:37:47 +0530 Subject: [PATCH 009/231] created new services for readFile and encryption logic --- src/services/upload/encryptionService.ts | 37 +++ src/services/upload/exifService.ts | 2 +- src/services/upload/fileReadClient.ts | 349 ----------------------- src/services/upload/metadataService.ts | 110 +++++++ src/services/upload/readFileService.ts | 108 +++++++ src/services/upload/thumbnailService.ts | 169 +++++++++++ 6 files changed, 425 insertions(+), 350 deletions(-) create mode 100644 src/services/upload/encryptionService.ts delete mode 100644 src/services/upload/fileReadClient.ts create mode 100644 src/services/upload/metadataService.ts create mode 100644 src/services/upload/readFileService.ts create mode 100644 src/services/upload/thumbnailService.ts diff --git a/src/services/upload/encryptionService.ts b/src/services/upload/encryptionService.ts new file mode 100644 index 000000000..0802fddeb --- /dev/null +++ b/src/services/upload/encryptionService.ts @@ -0,0 +1,37 @@ +import { DataStream, EncryptionResult, isDataStream } from './uploadService'; + +async function encryptFileStream(worker, fileData: DataStream) { + const { stream, chunkCount } = fileData; + const fileStreamReader = stream.getReader(); + const { key, decryptionHeader, pushState } = + await worker.initChunkEncryption(); + const ref = { pullCount: 1 }; + const encryptedFileStream = new ReadableStream({ + async pull(controller) { + const { value } = await fileStreamReader.read(); + const encryptedFileChunk = await worker.encryptFileChunk( + value, + pushState, + ref.pullCount === chunkCount, + ); + controller.enqueue(encryptedFileChunk); + if (ref.pullCount === chunkCount) { + controller.close(); + } + ref.pullCount++; + }, + }); + return { + key, + file: { + decryptionHeader, + encryptedData: { stream: encryptedFileStream, chunkCount }, + }, + }; +} + +export async function encryptFiledata(worker, filedata:Uint8Array | DataStream) :Promise { + return isDataStream(filedata) ? + await encryptFileStream(worker, filedata) : + await worker.encryptFile(filedata); +} diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts index 96f8f95b2..d49580981 100644 --- a/src/services/upload/exifService.ts +++ b/src/services/upload/exifService.ts @@ -1,7 +1,7 @@ import EXIF from 'exif-js'; import { FILE_TYPE } from 'pages/gallery'; import { logError } from 'utils/sentry'; -import { NULL_LOCATION, Location } from './fileReadClient'; +import { NULL_LOCATION, Location } from './metadataService'; const SOUTH_DIRECTION = 'S'; const WEST_DIRECTION = 'W'; diff --git a/src/services/upload/fileReadClient.ts b/src/services/upload/fileReadClient.ts deleted file mode 100644 index 237c99ba6..000000000 --- a/src/services/upload/fileReadClient.ts +++ /dev/null @@ -1,349 +0,0 @@ -import { FILE_TYPE } from 'pages/gallery'; -import { ENCRYPTION_CHUNK_SIZE } from 'types'; -import { THUMBNAIL_GENERATION_FAILED } from 'utils/common/errorUtil'; -import { fileIsHEIC, convertHEIC2JPEG } from 'utils/file'; -import { logError } from 'utils/sentry'; -import { getExifData } from './exifService'; - -const TYPE_VIDEO = 'video'; -const TYPE_HEIC = 'HEIC'; -export const TYPE_IMAGE = 'image'; -const MIN_STREAM_FILE_SIZE = 20 * 1024 * 1024; -const EDITED_FILE_SUFFIX = '-edited'; -const THUMBNAIL_HEIGHT = 720; -const MAX_ATTEMPTS = 3; -const MIN_THUMBNAIL_SIZE = 50000; - - -export const NULL_LOCATION: Location = { latitude: null, longitude: null }; -const WAIT_TIME_THUMBNAIL_GENERATION = 10 * 1000; -const NULL_PARSED_METADATA_JSON:ParsedMetaDataJSON={ title: null, creationTime: null, modificationTime: null, location: NULL_LOCATION }; - -export interface Location { - latitude: number; - longitude: number; -} - - -export interface ParsedMetaDataJSON{ - title:string; - creationTime:number; - modificationTime:number; - location:Location; -} - -export default async function readFile(reader: FileReader, receivedFile: globalThis.File, metadataMap:Map) { - try { - const { thumbnail, hasStaticThumbnail } = await generateThumbnail( - reader, - receivedFile, - ); - - let fileType: FILE_TYPE; - switch (receivedFile.type.split('/')[0]) { - case TYPE_IMAGE: - fileType = FILE_TYPE.IMAGE; - break; - case TYPE_VIDEO: - fileType = FILE_TYPE.VIDEO; - break; - default: - fileType = FILE_TYPE.OTHERS; - } - if ( - fileType === FILE_TYPE.OTHERS && - receivedFile.type.length === 0 && - receivedFile.name.endsWith(TYPE_HEIC) - ) { - fileType = FILE_TYPE.IMAGE; - } - - const { location, creationTime } = await getExifData( - reader, - receivedFile, - fileType, - ); - let receivedFileOriginalName = receivedFile.name; - if (receivedFile.name.endsWith(EDITED_FILE_SUFFIX)) { - receivedFileOriginalName = receivedFile.name.slice( - 0, - -1 * EDITED_FILE_SUFFIX.length, - ); - } - const metadata = Object.assign( - { - title: receivedFile.name, - creationTime: - creationTime || receivedFile.lastModified * 1000, - modificationTime: receivedFile.lastModified * 1000, - latitude: location?.latitude, - longitude: location?.latitude, - fileType, - }, - metadataMap.get(receivedFileOriginalName), - ); - if (hasStaticThumbnail) { - metadata['hasStaticThumbnail'] = hasStaticThumbnail; - } - const filedata = - receivedFile.size > MIN_STREAM_FILE_SIZE ? - getFileStream(reader, receivedFile) : - await getUint8ArrayView(reader, receivedFile); - - return { - filedata, - thumbnail, - metadata, - }; - } catch (e) { - logError(e, 'error reading files'); - throw e; - } -} - -async function generateThumbnail( - reader: FileReader, - file: globalThis.File, -): Promise<{ thumbnail: Uint8Array, hasStaticThumbnail: boolean }> { - try { - let hasStaticThumbnail = false; - const canvas = document.createElement('canvas'); - // eslint-disable-next-line camelcase - const canvas_CTX = canvas.getContext('2d'); - let imageURL = null; - let timeout = null; - try { - if (file.type.match(TYPE_IMAGE) || fileIsHEIC(file.name)) { - if (fileIsHEIC(file.name)) { - file = new globalThis.File( - [await convertHEIC2JPEG(file)], - null, - null, - ); - } - let image = new Image(); - imageURL = URL.createObjectURL(file); - image.setAttribute('src', imageURL); - await new Promise((resolve, reject) => { - image.onload = () => { - try { - const thumbnailWidth = - (image.width * THUMBNAIL_HEIGHT) / image.height; - canvas.width = thumbnailWidth; - canvas.height = THUMBNAIL_HEIGHT; - canvas_CTX.drawImage( - image, - 0, - 0, - thumbnailWidth, - THUMBNAIL_HEIGHT, - ); - image = null; - clearTimeout(timeout); - resolve(null); - } catch (e) { - reject(e); - logError(e); - reject(Error(`${THUMBNAIL_GENERATION_FAILED} err: ${e}`)); - } - }; - timeout = setTimeout( - () => - reject( - Error(`wait time exceeded for format ${file.name.split('.').slice(-1)[0]}`), - ), - WAIT_TIME_THUMBNAIL_GENERATION, - ); - }); - } else { - await new Promise((resolve, reject) => { - let video = document.createElement('video'); - imageURL = URL.createObjectURL(file); - video.addEventListener('timeupdate', function () { - try { - if (!video) { - return; - } - const thumbnailWidth = - (video.videoWidth * THUMBNAIL_HEIGHT) / - video.videoHeight; - canvas.width = thumbnailWidth; - canvas.height = THUMBNAIL_HEIGHT; - canvas_CTX.drawImage( - video, - 0, - 0, - thumbnailWidth, - THUMBNAIL_HEIGHT, - ); - video = null; - clearTimeout(timeout); - resolve(null); - } catch (e) { - reject(e); - logError(e); - reject(Error(`${THUMBNAIL_GENERATION_FAILED} err: ${e}`)); - } - }); - video.preload = 'metadata'; - video.src = imageURL; - video.currentTime = 3; - setTimeout( - () => - reject(Error(`wait time exceeded for format ${file.name.split('.').slice(-1)[0]}`)), - WAIT_TIME_THUMBNAIL_GENERATION, - ); - }); - } - URL.revokeObjectURL(imageURL); - } catch (e) { - logError(e); - // ignore and set staticThumbnail - hasStaticThumbnail = true; - } - let thumbnailBlob = null; - let attempts = 0; - let quality = 1; - - do { - attempts++; - quality /= 2; - thumbnailBlob = await new Promise((resolve) => { - canvas.toBlob( - function (blob) { - resolve(blob); - }, - 'image/jpeg', - quality, - ); - }); - thumbnailBlob = thumbnailBlob ?? new Blob([]); - } while ( - thumbnailBlob.size > MIN_THUMBNAIL_SIZE && - attempts <= MAX_ATTEMPTS - ); - const thumbnail = await getUint8ArrayView( - reader, - thumbnailBlob, - ); - return { thumbnail, hasStaticThumbnail }; - } catch (e) { - logError(e, 'Error generating thumbnail'); - throw e; - } -} - -function getFileStream(reader: FileReader, file: globalThis.File) { - const fileChunkReader = fileChunkReaderMaker(reader, file); - return { - stream: new ReadableStream({ - async pull(controller: ReadableStreamDefaultController) { - const chunk = await fileChunkReader.next(); - if (chunk.done) { - controller.close(); - } else { - controller.enqueue(chunk.value); - } - }, - }), - chunkCount: Math.ceil(file.size / ENCRYPTION_CHUNK_SIZE), - }; -} - -async function* fileChunkReaderMaker(reader:FileReader, file:globalThis.File) { - let offset = 0; - while (offset < file.size) { - const blob = file.slice(offset, ENCRYPTION_CHUNK_SIZE + offset); - const fileChunk = await getUint8ArrayView(reader, blob); - yield fileChunk; - offset += ENCRYPTION_CHUNK_SIZE; - } - return null; -} - -async function getUint8ArrayView( - reader: FileReader, - file: Blob, -): Promise { - try { - return await new Promise((resolve, reject) => { - reader.onabort = () => reject(Error('file reading was aborted')); - reader.onerror = () => reject(Error('file reading has failed')); - reader.onload = () => { - // Do whatever you want with the file contents - const result = - typeof reader.result === 'string' ? - new TextEncoder().encode(reader.result) : - new Uint8Array(reader.result); - resolve(result); - }; - reader.readAsArrayBuffer(file); - }); - } catch (e) { - logError(e, 'error reading file to byte-array'); - throw e; - } -} - - -export async function parseMetadataJSON(receivedFile: globalThis.File) { - try { - const metadataJSON: object = await new Promise( - (resolve, reject) => { - const reader = new FileReader(); - reader.onabort = () => reject(Error('file reading was aborted')); - reader.onerror = () => reject(Error('file reading has failed')); - reader.onload = () => { - const result = - typeof reader.result !== 'string' ? - new TextDecoder().decode(reader.result) : - reader.result; - resolve(JSON.parse(result)); - }; - reader.readAsText(receivedFile); - }, - ); - - const parsedMetaDataJSON:ParsedMetaDataJSON = NULL_PARSED_METADATA_JSON; - if (!metadataJSON || !metadataJSON['title']) { - return; - } - - parsedMetaDataJSON.title=metadataJSON['title']; - if ( - metadataJSON['photoTakenTime'] && - metadataJSON['photoTakenTime']['timestamp'] - ) { - parsedMetaDataJSON.creationTime = - metadataJSON['photoTakenTime']['timestamp'] * 1000000; - } - if ( - metadataJSON['modificationTime'] && - metadataJSON['modificationTime']['timestamp'] - ) { - parsedMetaDataJSON.modificationTime = - metadataJSON['modificationTime']['timestamp'] * 1000000; - } - let locationData:Location = NULL_LOCATION; - if ( - metadataJSON['geoData'] && - (metadataJSON['geoData']['latitude'] !== 0.0 || - metadataJSON['geoData']['longitude'] !== 0.0) - ) { - locationData = metadataJSON['geoData']; - } else if ( - metadataJSON['geoDataExif'] && - (metadataJSON['geoDataExif']['latitude'] !== 0.0 || - metadataJSON['geoDataExif']['longitude'] !== 0.0) - ) { - locationData = metadataJSON['geoDataExif']; - } - if (locationData !== null) { - parsedMetaDataJSON.location=locationData; - } - return parsedMetaDataJSON; - } catch (e) { - logError(e); - // ignore - } -} diff --git a/src/services/upload/metadataService.ts b/src/services/upload/metadataService.ts new file mode 100644 index 000000000..0eedbcee8 --- /dev/null +++ b/src/services/upload/metadataService.ts @@ -0,0 +1,110 @@ +import { FILE_TYPE } from 'pages/gallery'; +import { logError } from 'utils/sentry'; +import { getExifData } from './exifService'; +import { MetadataObject } from './uploadService'; + + +export interface Location { + latitude: number; + longitude: number; +} + +export interface ParsedMetaDataJSON{ + title:string; + creationTime:number; + modificationTime:number; + location:Location; +} + +export const NULL_LOCATION: Location = +{ latitude: null, + longitude: null }; + +const NULL_PARSED_METADATA_JSON:ParsedMetaDataJSON= +{ title: null, + creationTime: null, + modificationTime: null, + location: NULL_LOCATION, +}; + +export async function extractMetatdata(reader:FileReader, receivedFile:globalThis.File, fileType:FILE_TYPE) { + const { location, creationTime } = await getExifData( + reader, + receivedFile, + fileType, + ); + + const extractedMetadata:MetadataObject = { + title: receivedFile.name, + creationTime: + creationTime || receivedFile.lastModified * 1000, + modificationTime: receivedFile.lastModified * 1000, + latitude: location?.latitude, + longitude: location?.latitude, + fileType, + }; + return extractedMetadata; +} + + +export async function parseMetadataJSON(receivedFile: globalThis.File) { + try { + const metadataJSON: object = await new Promise( + (resolve, reject) => { + const reader = new FileReader(); + reader.onabort = () => reject(Error('file reading was aborted')); + reader.onerror = () => reject(Error('file reading has failed')); + reader.onload = () => { + const result = + typeof reader.result !== 'string' ? + new TextDecoder().decode(reader.result) : + reader.result; + resolve(JSON.parse(result)); + }; + reader.readAsText(receivedFile); + }, + ); + + const parsedMetaDataJSON:ParsedMetaDataJSON = NULL_PARSED_METADATA_JSON; + if (!metadataJSON || !metadataJSON['title']) { + return; + } + + parsedMetaDataJSON.title=metadataJSON['title']; + if ( + metadataJSON['photoTakenTime'] && + metadataJSON['photoTakenTime']['timestamp'] + ) { + parsedMetaDataJSON.creationTime = + metadataJSON['photoTakenTime']['timestamp'] * 1000000; + } + if ( + metadataJSON['modificationTime'] && + metadataJSON['modificationTime']['timestamp'] + ) { + parsedMetaDataJSON.modificationTime = + metadataJSON['modificationTime']['timestamp'] * 1000000; + } + let locationData:Location = NULL_LOCATION; + if ( + metadataJSON['geoData'] && + (metadataJSON['geoData']['latitude'] !== 0.0 || + metadataJSON['geoData']['longitude'] !== 0.0) + ) { + locationData = metadataJSON['geoData']; + } else if ( + metadataJSON['geoDataExif'] && + (metadataJSON['geoDataExif']['latitude'] !== 0.0 || + metadataJSON['geoDataExif']['longitude'] !== 0.0) + ) { + locationData = metadataJSON['geoDataExif']; + } + if (locationData !== null) { + parsedMetaDataJSON.location=locationData; + } + return parsedMetaDataJSON; + } catch (e) { + logError(e); + // ignore + } +} diff --git a/src/services/upload/readFileService.ts b/src/services/upload/readFileService.ts new file mode 100644 index 000000000..7a56c1dbe --- /dev/null +++ b/src/services/upload/readFileService.ts @@ -0,0 +1,108 @@ +import { FILE_TYPE } from 'pages/gallery'; +import { ENCRYPTION_CHUNK_SIZE } from 'types'; +import { logError } from 'utils/sentry'; + +const TYPE_VIDEO = 'video'; +const TYPE_HEIC = 'HEIC'; +export const TYPE_IMAGE = 'image'; +const MIN_STREAM_FILE_SIZE = 20 * 1024 * 1024; +const EDITED_FILE_SUFFIX = '-edited'; + + +export async function getFileData(reader:FileReader, file:globalThis.File) { + return file.size > MIN_STREAM_FILE_SIZE ? + getFileStream(reader, file) : + await getUint8ArrayView(reader, file); +} + +export function getFileType(receivedFile:globalThis.File) { + let fileType: FILE_TYPE; + switch (receivedFile.type.split('/')[0]) { + case TYPE_IMAGE: + fileType = FILE_TYPE.IMAGE; + break; + case TYPE_VIDEO: + fileType = FILE_TYPE.VIDEO; + break; + default: + fileType = FILE_TYPE.OTHERS; + } + if ( + fileType === FILE_TYPE.OTHERS && + receivedFile.type.length === 0 && + receivedFile.name.endsWith(TYPE_HEIC) + ) { + fileType = FILE_TYPE.IMAGE; + } + return fileType; +} + + +export function getFileOriginalName(file:globalThis.File) { + let originalName:string=null; + + const isEditedFile=file.name.endsWith(EDITED_FILE_SUFFIX); + if (isEditedFile) { + originalName = file.name.slice( + 0, + -1 * EDITED_FILE_SUFFIX.length, + ); + } else { + originalName=file.name; + } + return originalName; +} + +function getFileStream(reader: FileReader, file: globalThis.File) { + const fileChunkReader = fileChunkReaderMaker(reader, file); + return { + stream: new ReadableStream({ + async pull(controller: ReadableStreamDefaultController) { + const chunk = await fileChunkReader.next(); + if (chunk.done) { + controller.close(); + } else { + controller.enqueue(chunk.value); + } + }, + }), + chunkCount: Math.ceil(file.size / ENCRYPTION_CHUNK_SIZE), + }; +} + +async function* fileChunkReaderMaker(reader:FileReader, file:globalThis.File) { + let offset = 0; + while (offset < file.size) { + const blob = file.slice(offset, ENCRYPTION_CHUNK_SIZE + offset); + const fileChunk = await getUint8ArrayView(reader, blob); + yield fileChunk; + offset += ENCRYPTION_CHUNK_SIZE; + } + return null; +} + +export async function getUint8ArrayView( + reader: FileReader, + file: Blob, +): Promise { + try { + return await new Promise((resolve, reject) => { + reader.onabort = () => reject(Error('file reading was aborted')); + reader.onerror = () => reject(Error('file reading has failed')); + reader.onload = () => { + // Do whatever you want with the file contents + const result = + typeof reader.result === 'string' ? + new TextEncoder().encode(reader.result) : + new Uint8Array(reader.result); + resolve(result); + }; + reader.readAsArrayBuffer(file); + }); + } catch (e) { + logError(e, 'error reading file to byte-array'); + throw e; + } +} + + diff --git a/src/services/upload/thumbnailService.ts b/src/services/upload/thumbnailService.ts new file mode 100644 index 000000000..0420d797f --- /dev/null +++ b/src/services/upload/thumbnailService.ts @@ -0,0 +1,169 @@ +import { FILE_TYPE } from 'pages/gallery'; +import { THUMBNAIL_GENERATION_FAILED } from 'utils/common/errorUtil'; +import { fileIsHEIC, convertHEIC2JPEG } from 'utils/file'; +import { logError } from 'utils/sentry'; +import { getUint8ArrayView } from './readFileService'; + +export const TYPE_IMAGE = 'image'; +const THUMBNAIL_HEIGHT = 720; +const MAX_ATTEMPTS = 3; +const MIN_THUMBNAIL_SIZE = 50000; + + +const WAIT_TIME_THUMBNAIL_GENERATION = 10 * 1000; + + +export async function generateThumbnail( + reader: FileReader, + file: globalThis.File, + fileType:FILE_TYPE, +): Promise<{ thumbnail: Uint8Array, hasStaticThumbnail: boolean }> { + try { + let hasStaticThumbnail = false; + let canvas = null; + try { + if (fileType===FILE_TYPE.IMAGE) { + canvas=await generateImageThumbnail(file); + } else { + canvas=await generateVideoThumbnail(file); + } + } catch (e) { + logError(e); + // ignore and set staticThumbnail + hasStaticThumbnail = true; + } + const thumbnailBlob=await thumbnailCanvasToBlob(canvas); + const thumbnail = await getUint8ArrayView( + reader, + thumbnailBlob, + ); + return { thumbnail, hasStaticThumbnail }; + } catch (e) { + logError(e, 'Error generating thumbnail'); + throw e; + } +} + +export async function generateImageThumbnail(file:globalThis.File) { + const canvas = document.createElement('canvas'); + const canvasCTX = canvas.getContext('2d'); + + let imageURL=null; + let timeout = null; + + if (fileIsHEIC(file.name)) { + file = new globalThis.File( + [await convertHEIC2JPEG(file)], + null, + null, + ); + } + let image = new Image(); + imageURL = URL.createObjectURL(file); + image.setAttribute('src', imageURL); + await new Promise((resolve, reject) => { + image.onload = () => { + try { + const thumbnailWidth = + (image.width * THUMBNAIL_HEIGHT) / image.height; + canvas.width = thumbnailWidth; + canvas.height = THUMBNAIL_HEIGHT; + canvasCTX.drawImage( + image, + 0, + 0, + thumbnailWidth, + THUMBNAIL_HEIGHT, + ); + image = null; + clearTimeout(timeout); + resolve(null); + } catch (e) { + reject(e); + logError(e); + reject(Error(`${THUMBNAIL_GENERATION_FAILED} err: ${e}`)); + } + }; + timeout = setTimeout( + () => + reject( + Error(`wait time exceeded for format ${file.name.split('.').slice(-1)[0]}`), + ), + WAIT_TIME_THUMBNAIL_GENERATION, + ); + }); + return canvas; +} + +export async function generateVideoThumbnail(file:globalThis.File) { + const canvas = document.createElement('canvas'); + const canvasCTX = canvas.getContext('2d'); + + let videoURL=null; + let timeout=null; + + await new Promise((resolve, reject) => { + let video = document.createElement('video'); + videoURL = URL.createObjectURL(file); + video.addEventListener('timeupdate', function () { + try { + if (!video) { + return; + } + const thumbnailWidth = + (video.videoWidth * THUMBNAIL_HEIGHT) / + video.videoHeight; + canvas.width = thumbnailWidth; + canvas.height = THUMBNAIL_HEIGHT; + canvasCTX.drawImage( + video, + 0, + 0, + thumbnailWidth, + THUMBNAIL_HEIGHT, + ); + video = null; + clearTimeout(timeout); + resolve(null); + } catch (e) { + reject(e); + logError(e); + reject(Error(`${THUMBNAIL_GENERATION_FAILED} err: ${e}`)); + } + }); + video.preload = 'metadata'; + video.src = videoURL; + video.currentTime = 3; + timeout=setTimeout( + () => + reject(Error(`wait time exceeded for format ${file.name.split('.').slice(-1)[0]}`)), + WAIT_TIME_THUMBNAIL_GENERATION, + ); + }); +} + +export async function thumbnailCanvasToBlob(canvas:HTMLCanvasElement) { + let thumbnailBlob = null; + let attempts = 0; + let quality = 1; + + do { + attempts++; + quality /= 2; + thumbnailBlob = await new Promise((resolve) => { + canvas.toBlob( + function (blob) { + resolve(blob); + }, + 'image/jpeg', + quality, + ); + }); + thumbnailBlob = thumbnailBlob ?? new Blob([]); + } while ( + thumbnailBlob.size > MIN_THUMBNAIL_SIZE && + attempts <= MAX_ATTEMPTS + ); + + return thumbnailBlob; +} From ff98eefe67f53508074c7edc1dc1815ee11d988f Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Mon, 9 Aug 2021 14:39:20 +0530 Subject: [PATCH 010/231] refactor readFile and encrypt logic --- src/services/upload/uploadService.ts | 76 ++++++++++++++-------------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/src/services/upload/uploadService.ts b/src/services/upload/uploadService.ts index 9e7923ceb..ae7e5bd07 100644 --- a/src/services/upload/uploadService.ts +++ b/src/services/upload/uploadService.ts @@ -16,7 +16,10 @@ import { logError } from 'utils/sentry'; import localForage from 'utils/storage/localForage'; import { sleep } from 'utils/common'; import NetworkClient, { UploadURL } from './networkClient'; -import readFile, { ParsedMetaDataJSON, parseMetadataJSON } from './fileReadClient'; +import { extractMetatdata, ParsedMetaDataJSON, parseMetadataJSON } from './metadataService'; +import { generateThumbnail } from './thumbnailService'; +import { getFileType, getFileOriginalName, getFileData } from './readFileService'; +import { encryptFiledata } from './encryptionService'; const MAX_CONCURRENT_UPLOADS = 4; @@ -41,10 +44,10 @@ export interface DataStream { chunkCount: number; } -function isDataStream(object: any): object is DataStream { +export function isDataStream(object: any): object is DataStream { return 'stream' in object; } -interface EncryptionResult { +export interface EncryptionResult { file: fileAttribute; key: string; } @@ -226,7 +229,7 @@ class UploadService { let encryptedFile: EncryptedFile=null; try { // read the file into memory - file = await readFile(reader, rawFile, this.metadataMap); + file = await this.readFile(reader, rawFile, this.metadataMap); if (this.fileAlreadyInCollection(file, collection)) { // set progress to -2 indicating that file upload was skipped @@ -308,6 +311,37 @@ class UploadService { setFileProgress(this.fileProgress); } + async readFile(reader: FileReader, receivedFile: globalThis.File, metadataMap:Map) { + try { + const fileType=getFileType(receivedFile); + + const { thumbnail, hasStaticThumbnail } = await generateThumbnail( + reader, + receivedFile, + fileType, + ); + + const originalName=getFileOriginalName(receivedFile); + const googleMetadata=metadataMap.get(originalName); + const extractedMetadata:MetadataObject =await extractMetatdata(reader, receivedFile, fileType); + if (hasStaticThumbnail) { + extractedMetadata.hasStaticThumbnail=true; + } + const metadata:MetadataObject={ ...extractedMetadata, ...googleMetadata }; + + const filedata = await getFileData(reader, receivedFile); + + return { + filedata, + thumbnail, + metadata, + }; + } catch (e) { + logError(e, 'error reading files'); + throw e; + } + } + private fileAlreadyInCollection( newFile: FileInMemory, collection: Collection, @@ -344,10 +378,7 @@ class UploadService { encryptionKey: string, ): Promise { try { - const { key: fileKey, file: encryptedFiledata }: EncryptionResult = - isDataStream(file.filedata) ? - await this.encryptFileStream(worker, file.filedata) : - await worker.encryptFile(file.filedata); + const { key: fileKey, file: encryptedFiledata } = await encryptFiledata(worker, file.filedata); const { file: encryptedThumbnail }: EncryptionResult = await worker.encryptThumbnail(file.thumbnail, fileKey); @@ -375,35 +406,6 @@ class UploadService { } } - private async encryptFileStream(worker, fileData: DataStream) { - const { stream, chunkCount } = fileData; - const fileStreamReader = stream.getReader(); - const { key, decryptionHeader, pushState } = - await worker.initChunkEncryption(); - const ref = { pullCount: 1 }; - const encryptedFileStream = new ReadableStream({ - async pull(controller) { - const { value } = await fileStreamReader.read(); - const encryptedFileChunk = await worker.encryptFileChunk( - value, - pushState, - ref.pullCount === chunkCount, - ); - controller.enqueue(encryptedFileChunk); - if (ref.pullCount === chunkCount) { - controller.close(); - } - ref.pullCount++; - }, - }); - return { - key, - file: { - decryptionHeader, - encryptedData: { stream: encryptedFileStream, chunkCount }, - }, - }; - } private async uploadToBucket(file: ProcessedFile): Promise { try { From a033ee383c8796a30f123f93a46e168ca1e358bf Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Mon, 9 Aug 2021 16:55:06 +0530 Subject: [PATCH 011/231] return canvas from videoThumbnail generator --- src/services/upload/thumbnailService.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/services/upload/thumbnailService.ts b/src/services/upload/thumbnailService.ts index f058c51c8..4002a1b21 100644 --- a/src/services/upload/thumbnailService.ts +++ b/src/services/upload/thumbnailService.ts @@ -126,9 +126,9 @@ export async function generateVideoThumbnail(file:globalThis.File) { clearTimeout(timeout); resolve(null); } catch (e) { - reject(e); - logError(e); - reject(Error(`${CustomError.THUMBNAIL_GENERATION_FAILED} err: ${e}`)); + const err=Error(`${CustomError.THUMBNAIL_GENERATION_FAILED} err: ${e}`); + logError(err); + reject(err); } }); video.preload = 'metadata'; @@ -140,6 +140,7 @@ export async function generateVideoThumbnail(file:globalThis.File) { WAIT_TIME_THUMBNAIL_GENERATION, ); }); + return canvas; } export async function thumbnailCanvasToBlob(canvas:HTMLCanvasElement) { From 141924bbf6edea3477d1f97055c7d366e4845faa Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Mon, 9 Aug 2021 17:02:45 +0530 Subject: [PATCH 012/231] fix progress bar for multipart upload --- src/services/upload/uploadService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/upload/uploadService.ts b/src/services/upload/uploadService.ts index 0d771b933..f1432c1b5 100644 --- a/src/services/upload/uploadService.ts +++ b/src/services/upload/uploadService.ts @@ -434,7 +434,7 @@ class UploadService { stream, file.filename, uploadPartCount, - this.trackUploadProgress, + this.trackUploadProgress.bind(this), ); } else { const fileUploadURL = await this.getUploadURL(); From b95a560072f1ce545328f4905ae60c4fec51b6da Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Mon, 9 Aug 2021 20:45:11 +0530 Subject: [PATCH 013/231] created s3Service layer --- src/services/upload/networkClient.ts | 90 +++++++++----------------- src/services/upload/readFileService.ts | 2 +- src/services/upload/s3Service.ts | 87 +++++++++++++++++++++++++ src/services/upload/uploadService.ts | 34 +++++----- 4 files changed, 134 insertions(+), 79 deletions(-) create mode 100644 src/services/upload/s3Service.ts diff --git a/src/services/upload/networkClient.ts b/src/services/upload/networkClient.ts index 87f32c65e..7a7f3b9a0 100644 --- a/src/services/upload/networkClient.ts +++ b/src/services/upload/networkClient.ts @@ -3,8 +3,7 @@ import { retryAsyncFunction } from 'utils/common'; import { getEndpoint } from 'utils/common/apiUtil'; import { getToken } from 'utils/common/key'; import { logError } from 'utils/sentry'; -import { CHUNKS_COMBINED_FOR_UPLOAD, MultipartUploadURLs, RANDOM_PERCENTAGE_PROGRESS_FOR_PUT, UploadFile } from './uploadService'; -import * as convert from 'xml-js'; +import { MultipartUploadURLs, UploadFile, UploadURL } from './uploadService'; import { File } from '../fileService'; import { CustomError } from 'utils/common/errorUtil'; @@ -12,10 +11,6 @@ const ENDPOINT = getEndpoint(); const MAX_URL_REQUESTS = 50; -export interface UploadURL { - url: string; - objectKey: string; -} class NetworkClient { private uploadURLFetchInProgress=null; @@ -116,72 +111,49 @@ class NetworkClient { } } - async putFileInParts( - multipartUploadURLs: MultipartUploadURLs, - file: ReadableStream, - filename: string, - uploadPartCount: number, - trackUploadProgress, + + async putFilePart( + partUploadURL: string, + filePart: Uint8Array, + progressTracker, ) { try { - const streamEncryptedFileReader = file.getReader(); - const percentPerPart = Math.round( - RANDOM_PERCENTAGE_PROGRESS_FOR_PUT() / uploadPartCount, - ); - const resParts = []; - for (const [ - index, - fileUploadURL, - ] of multipartUploadURLs.partURLs.entries()) { - const combinedChunks = []; - for (let i = 0; i < CHUNKS_COMBINED_FOR_UPLOAD; i++) { - const { done, value: chunk } = - await streamEncryptedFileReader.read(); - if (done) { - break; - } - for (let index = 0; index < chunk.length; index++) { - combinedChunks.push(chunk[index]); - } + const response=await retryAsyncFunction(async ()=>{ + const resp =await HTTPService.put( + partUploadURL, + filePart, + null, + null, + progressTracker(), + ); + if (!resp?.headers?.etag) { + const err=Error(CustomError.ETAG_MISSING); + logError(err); + throw err; } - const uploadChunk = Uint8Array.from(combinedChunks); - const response=await retryAsyncFunction(async ()=>{ - const resp =await HTTPService.put( - fileUploadURL, - uploadChunk, - null, - null, - trackUploadProgress(filename, percentPerPart, index), - ); - if (!resp?.headers?.etag) { - const err=Error(CustomError.ETAG_MISSING); - logError(err); - throw err; - } - return resp; - }); - resParts.push({ - PartNumber: index + 1, - ETag: response.headers.etag, - }); - } - const options = { compact: true, ignoreComment: true, spaces: 4 }; - const body = convert.js2xml( - { CompleteMultipartUpload: { Part: resParts } }, - options, - ); + return resp; + }); + return response.headers.etag; + } catch (e) { + logError(e, 'put filePart failed'); + throw e; + } + } + + async completeMultipartUpload(completeURL:string, reqBody:any) { + try { await retryAsyncFunction(()=> - HTTPService.post(multipartUploadURLs.completeURL, body, null, { + HTTPService.post(completeURL, reqBody, null, { 'content-type': 'text/xml', }), ); - return multipartUploadURLs.objectKey; } catch (e) { logError(e, 'put file in parts failed'); throw e; } } } + export default new NetworkClient(); diff --git a/src/services/upload/readFileService.ts b/src/services/upload/readFileService.ts index 7a56c1dbe..8eb0a2ae7 100644 --- a/src/services/upload/readFileService.ts +++ b/src/services/upload/readFileService.ts @@ -1,11 +1,11 @@ import { FILE_TYPE } from 'pages/gallery'; import { ENCRYPTION_CHUNK_SIZE } from 'types'; import { logError } from 'utils/sentry'; +import { MIN_STREAM_FILE_SIZE } from './uploadService'; const TYPE_VIDEO = 'video'; const TYPE_HEIC = 'HEIC'; export const TYPE_IMAGE = 'image'; -const MIN_STREAM_FILE_SIZE = 20 * 1024 * 1024; const EDITED_FILE_SUFFIX = '-edited'; diff --git a/src/services/upload/s3Service.ts b/src/services/upload/s3Service.ts new file mode 100644 index 000000000..9dc0a4450 --- /dev/null +++ b/src/services/upload/s3Service.ts @@ -0,0 +1,87 @@ +import { CHUNKS_COMBINED_FOR_A_UPLOAD_PART, DataStream, MultipartUploadURLs, RANDOM_PERCENTAGE_PROGRESS_FOR_PUT } from './uploadService'; +import NetworkClient from './networkClient'; +import * as convert from 'xml-js'; + + +interface PartEtag{ + PartNumber:number; + Etag:string; +} +export function calculatePartCount(encryptedChunkCount: number) { + const partCount = Math.ceil( + encryptedChunkCount / CHUNKS_COMBINED_FOR_A_UPLOAD_PART, + ); + return partCount; +} +export async function uploadStreamUsingMultipart(filename:string, encryptedData:DataStream, progressTracker) { + const { chunkCount, stream } = encryptedData; + const uploadPartCount = calculatePartCount(chunkCount); + const filePartUploadURLs = await NetworkClient.fetchMultipartUploadURLs( + uploadPartCount, + ); + const fileObjectKey = await uploadStreamInParts( + filePartUploadURLs, + stream, + filename, + uploadPartCount, + progressTracker, + ); + return fileObjectKey; +} + +export async function uploadStreamInParts( + multipartUploadURLs: MultipartUploadURLs, + file: ReadableStream, + filename: string, + uploadPartCount: number, + progressTracker, +) { + const encryptedFileStreamReader = file.getReader(); + const percentPerPart = getRandomProgressPerPartUpload(uploadPartCount); + + const partEtags:PartEtag[] = []; + for (const [ + index, + fileUploadURL, + ] of multipartUploadURLs.partURLs.entries()) { + const uploadChunk = await combineChunksToFormUploadPart(encryptedFileStreamReader); + const eTag= await NetworkClient.putFilePart(fileUploadURL, uploadChunk, progressTracker.bind(null, filename, percentPerPart, index)); + partEtags.push({ PartNumber: index+1, Etag: eTag }); + } + await completeMultipartUpload(partEtags, multipartUploadURLs.completeURL); + return multipartUploadURLs.objectKey; +} + + +export function getRandomProgressPerPartUpload(uploadPartCount:number) { + const percentPerPart = Math.round( + RANDOM_PERCENTAGE_PROGRESS_FOR_PUT() / uploadPartCount, + ); + return percentPerPart; +} + + +export async function combineChunksToFormUploadPart(dataStreamReader:ReadableStreamDefaultReader) { + const combinedChunks = []; + for (let i = 0; i < CHUNKS_COMBINED_FOR_A_UPLOAD_PART; i++) { + const { done, value: chunk } = + await dataStreamReader.read(); + if (done) { + break; + } + for (let index = 0; index < chunk.length; index++) { + combinedChunks.push(chunk[index]); + } + } + return Uint8Array.from(combinedChunks); +} + + +async function completeMultipartUpload(partEtags:PartEtag[], completeURL:string) { + const options = { compact: true, ignoreComment: true, spaces: 4 }; + const body = convert.js2xml( + { CompleteMultipartUpload: { Part: partEtags } }, + options, + ); + await NetworkClient.completeMultipartUpload(completeURL, body); +} diff --git a/src/services/upload/uploadService.ts b/src/services/upload/uploadService.ts index f1432c1b5..e47f3ef3b 100644 --- a/src/services/upload/uploadService.ts +++ b/src/services/upload/uploadService.ts @@ -16,11 +16,13 @@ import { import { logError } from 'utils/sentry'; import localForage from 'utils/storage/localForage'; import { sleep } from 'utils/common'; -import NetworkClient, { UploadURL } from './networkClient'; +import NetworkClient from './networkClient'; import { extractMetatdata, ParsedMetaDataJSON, parseMetadataJSON } from './metadataService'; import { generateThumbnail } from './thumbnailService'; import { getFileType, getFileOriginalName, getFileData } from './readFileService'; import { encryptFiledata } from './encryptionService'; +import { ENCRYPTION_CHUNK_SIZE } from 'types'; +import { uploadStreamUsingMultipart } from './s3Service'; const MAX_CONCURRENT_UPLOADS = 4; @@ -28,7 +30,8 @@ const TYPE_JSON = 'json'; const FILE_UPLOAD_COMPLETED = 100; const TwoSecondInMillSeconds = 2000; export const RANDOM_PERCENTAGE_PROGRESS_FOR_PUT = () => 90 + 10 * Math.random(); -export const CHUNKS_COMBINED_FOR_UPLOAD = 5; +export const MIN_STREAM_FILE_SIZE = 20 * 1024 * 1024; +export const CHUNKS_COMBINED_FOR_A_UPLOAD_PART = Math.floor(MIN_STREAM_FILE_SIZE/ENCRYPTION_CHUNK_SIZE); export enum FileUploadResults { FAILED = -1, @@ -38,6 +41,11 @@ export enum FileUploadResults { UPLOADED = 100, } +export interface UploadURL { + url: string; + objectKey: string; +} + export interface FileWithCollection { file: globalThis.File; collection: Collection; @@ -87,7 +95,7 @@ interface EncryptedFile { file: ProcessedFile; fileKey: B64EncryptionResult; } -interface ProcessedFile { +export interface ProcessedFile { file: fileAttribute; thumbnail: fileAttribute; metadata: fileAttribute; @@ -420,25 +428,13 @@ class UploadService { private async uploadToBucket(file: ProcessedFile): Promise { try { - let fileObjectKey; + let fileObjectKey:string=null; if (isDataStream(file.file.encryptedData)) { - const { chunkCount, stream } = file.file.encryptedData; - const uploadPartCount = Math.ceil( - chunkCount / CHUNKS_COMBINED_FOR_UPLOAD, - ); - const filePartUploadURLs = await NetworkClient.fetchMultipartUploadURLs( - uploadPartCount, - ); - fileObjectKey = await NetworkClient.putFileInParts( - filePartUploadURLs, - stream, - file.filename, - uploadPartCount, - this.trackUploadProgress.bind(this), - ); + const progressTracker=this.trackUploadProgress.bind(this); + fileObjectKey=await uploadStreamUsingMultipart(file.filename, file.file.encryptedData, progressTracker); } else { - const fileUploadURL = await this.getUploadURL(); const progressTracker=this.trackUploadProgress.bind(this, file.filename); + const fileUploadURL = await this.getUploadURL(); fileObjectKey = await NetworkClient.putFile( fileUploadURL, file.file.encryptedData, From befcdedd082f1703d22c735df58399a507a72cde Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Tue, 10 Aug 2021 11:05:45 +0530 Subject: [PATCH 014/231] moved retryAsyncFunction to network util --- src/services/upload/networkClient.ts | 4 ++-- src/utils/common/index.ts | 14 -------------- src/utils/network/index.ts | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 16 deletions(-) create mode 100644 src/utils/network/index.ts diff --git a/src/services/upload/networkClient.ts b/src/services/upload/networkClient.ts index 7a7f3b9a0..fdfb3d4da 100644 --- a/src/services/upload/networkClient.ts +++ b/src/services/upload/networkClient.ts @@ -1,11 +1,11 @@ import HTTPService from 'services/HTTPService'; -import { retryAsyncFunction } from 'utils/common'; import { getEndpoint } from 'utils/common/apiUtil'; import { getToken } from 'utils/common/key'; import { logError } from 'utils/sentry'; import { MultipartUploadURLs, UploadFile, UploadURL } from './uploadService'; import { File } from '../fileService'; import { CustomError } from 'utils/common/errorUtil'; +import { retryAsyncFunction } from 'utils/network'; const ENDPOINT = getEndpoint(); const MAX_URL_REQUESTS = 50; @@ -133,7 +133,7 @@ class NetworkClient { } return resp; }); - return response.headers.etag; + return response.headers.etag as string; } catch (e) { logError(e, 'put filePart failed'); throw e; diff --git a/src/utils/common/index.ts b/src/utils/common/index.ts index d6785c83b..467ac4ef8 100644 --- a/src/utils/common/index.ts +++ b/src/utils/common/index.ts @@ -2,7 +2,6 @@ import constants from 'utils/strings/constants'; export const DESKTOP_APP_DOWNLOAD_URL = 'https://github.com/ente-io/bhari-frame/releases/latest'; -const retrySleepTime = [2000, 5000, 10000]; export function checkConnectivity() { if (navigator.onLine) { @@ -32,16 +31,3 @@ export function reverseString(title: string) { .reduce((reversedString, currWord) => `${currWord} ${reversedString}`); } -export async function retryAsyncFunction(func: ()=>Promise, retryCount: number = 3) { - try { - const resp = await func(); - return resp; - } catch (e) { - if (retryCount > 0) { - await sleep(retrySleepTime[3 - retryCount]); - await retryAsyncFunction(func, retryCount - 1); - } else { - throw e; - } - } -} diff --git a/src/utils/network/index.ts b/src/utils/network/index.ts new file mode 100644 index 000000000..0dd0e137b --- /dev/null +++ b/src/utils/network/index.ts @@ -0,0 +1,17 @@ +import { sleep } from 'utils/upload'; + +const retrySleepTime = [2000, 5000, 10000]; + +export async function retryAsyncFunction(func: ()=>Promise, retryCount: number = 3) { + try { + const resp = await func(); + return resp; + } catch (e) { + if (retryCount > 0) { + await sleep(retrySleepTime[3 - retryCount]); + await retryAsyncFunction(func, retryCount - 1); + } else { + throw e; + } + } +} From 8bd10f4cf2473d42da5b10401c3f0b499a340001 Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Tue, 10 Aug 2021 11:06:03 +0530 Subject: [PATCH 015/231] better names --- src/services/upload/s3Service.ts | 25 ++++++++++++------------- src/utils/network/index.ts | 2 +- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/services/upload/s3Service.ts b/src/services/upload/s3Service.ts index 9dc0a4450..abd1f65d9 100644 --- a/src/services/upload/s3Service.ts +++ b/src/services/upload/s3Service.ts @@ -7,21 +7,20 @@ interface PartEtag{ PartNumber:number; Etag:string; } -export function calculatePartCount(encryptedChunkCount: number) { +export function calculatePartCount(chunkCount: number) { const partCount = Math.ceil( - encryptedChunkCount / CHUNKS_COMBINED_FOR_A_UPLOAD_PART, + chunkCount / CHUNKS_COMBINED_FOR_A_UPLOAD_PART, ); return partCount; } -export async function uploadStreamUsingMultipart(filename:string, encryptedData:DataStream, progressTracker) { - const { chunkCount, stream } = encryptedData; - const uploadPartCount = calculatePartCount(chunkCount); - const filePartUploadURLs = await NetworkClient.fetchMultipartUploadURLs( +export async function uploadStreamUsingMultipart(filename:string, dataStream:DataStream, progressTracker) { + const uploadPartCount = calculatePartCount(dataStream.chunkCount); + const multipartUploadURLs = await NetworkClient.fetchMultipartUploadURLs( uploadPartCount, ); const fileObjectKey = await uploadStreamInParts( - filePartUploadURLs, - stream, + multipartUploadURLs, + dataStream.stream, filename, uploadPartCount, progressTracker, @@ -31,12 +30,12 @@ export async function uploadStreamUsingMultipart(filename:string, encryptedData: export async function uploadStreamInParts( multipartUploadURLs: MultipartUploadURLs, - file: ReadableStream, + dataStream: ReadableStream, filename: string, uploadPartCount: number, progressTracker, ) { - const encryptedFileStreamReader = file.getReader(); + const streamReader = dataStream.getReader(); const percentPerPart = getRandomProgressPerPartUpload(uploadPartCount); const partEtags:PartEtag[] = []; @@ -44,7 +43,7 @@ export async function uploadStreamInParts( index, fileUploadURL, ] of multipartUploadURLs.partURLs.entries()) { - const uploadChunk = await combineChunksToFormUploadPart(encryptedFileStreamReader); + const uploadChunk = await combineChunksToFormUploadPart(streamReader); const eTag= await NetworkClient.putFilePart(fileUploadURL, uploadChunk, progressTracker.bind(null, filename, percentPerPart, index)); partEtags.push({ PartNumber: index+1, Etag: eTag }); } @@ -61,11 +60,11 @@ export function getRandomProgressPerPartUpload(uploadPartCount:number) { } -export async function combineChunksToFormUploadPart(dataStreamReader:ReadableStreamDefaultReader) { +export async function combineChunksToFormUploadPart(streamReader:ReadableStreamDefaultReader) { const combinedChunks = []; for (let i = 0; i < CHUNKS_COMBINED_FOR_A_UPLOAD_PART; i++) { const { done, value: chunk } = - await dataStreamReader.read(); + await streamReader.read(); if (done) { break; } diff --git a/src/utils/network/index.ts b/src/utils/network/index.ts index 0dd0e137b..5d68825fe 100644 --- a/src/utils/network/index.ts +++ b/src/utils/network/index.ts @@ -1,4 +1,4 @@ -import { sleep } from 'utils/upload'; +import { sleep } from 'utils/common'; const retrySleepTime = [2000, 5000, 10000]; From cf67cd7b3b2eea4f87fcdfcef6e463a6b175fa25 Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Tue, 10 Aug 2021 11:19:58 +0530 Subject: [PATCH 016/231] moved to utils to util file --- src/services/upload/uploadService.ts | 42 +++------------------------- src/utils/upload/index.ts | 32 +++++++++++++++++++++ 2 files changed, 36 insertions(+), 38 deletions(-) create mode 100644 src/utils/upload/index.ts diff --git a/src/services/upload/uploadService.ts b/src/services/upload/uploadService.ts index e47f3ef3b..f20fc50c5 100644 --- a/src/services/upload/uploadService.ts +++ b/src/services/upload/uploadService.ts @@ -23,6 +23,7 @@ import { getFileType, getFileOriginalName, getFileData } from './readFileService import { encryptFiledata } from './encryptionService'; import { ENCRYPTION_CHUNK_SIZE } from 'types'; import { uploadStreamUsingMultipart } from './s3Service'; +import { fileAlreadyInCollection } from 'utils/upload'; const MAX_CONCURRENT_UPLOADS = 4; @@ -85,7 +86,7 @@ export interface MetadataObject { hasStaticThumbnail?: boolean; } -interface FileInMemory { +export interface FileInMemory { filedata: Uint8Array | DataStream; thumbnail: Uint8Array; metadata: MetadataObject; @@ -244,7 +245,7 @@ class UploadService { // read the file into memory file = await this.readFile(reader, rawFile, this.metadataMap); - if (this.fileAlreadyInCollection(file, collection)) { + if (fileAlreadyInCollection(this.existingFilesCollectionWise, file, collection)) { // set progress to -2 indicating that file upload was skipped this.fileProgress.set(rawFile.name, FileUploadResults.SKIPPED); this.updateProgressBarUI(); @@ -256,7 +257,7 @@ class UploadService { encryptedFile.file, ); - let uploadFile: UploadFile = this.getUploadFile( + const uploadFile: UploadFile = this.getUploadFile( collection, backupedFile, encryptedFile.fileKey, @@ -270,15 +271,10 @@ class UploadService { this.existingFiles=sortFiles(this.existingFiles); await localForage.setItem('files', removeUnneccessaryFileProps(this.existingFiles)); this.setFiles(this.existingFiles); - - uploadFile = null; - this.fileProgress.set(rawFile.name, FileUploadResults.UPLOADED); - this.filesCompleted++; } } catch (e) { - logError(e, 'file upload failed'); logError(e, 'file upload failed'); handleError(e); this.failedFiles.push(fileWithCollection); @@ -361,36 +357,6 @@ class UploadService { } } - private fileAlreadyInCollection( - newFile: FileInMemory, - collection: Collection, - ): boolean { - const collectionFiles = - this.existingFilesCollectionWise.get(collection.id) ?? []; - for (const existingFile of collectionFiles) { - if (this.areFilesSame(existingFile.metadata, newFile.metadata)) { - return true; - } - } - return false; - } - private areFilesSame( - existingFile: MetadataObject, - newFile: MetadataObject, - ): boolean { - if ( - existingFile.fileType === newFile.fileType && - existingFile.creationTime === newFile.creationTime && - existingFile.modificationTime === newFile.modificationTime && - existingFile.title === newFile.title - ) { - return true; - } else { - return false; - } - } - - private async encryptFile( worker: any, file: FileInMemory, diff --git a/src/utils/upload/index.ts b/src/utils/upload/index.ts new file mode 100644 index 000000000..70603f01b --- /dev/null +++ b/src/utils/upload/index.ts @@ -0,0 +1,32 @@ +import { Collection } from 'services/collectionService'; +import { FileInMemory, MetadataObject } from 'services/upload/uploadService'; + +export function fileAlreadyInCollection( + existingFilesCollectionWise, + newFile: FileInMemory, + collection: Collection, +): boolean { + const collectionFiles = + existingFilesCollectionWise.get(collection.id) ?? []; + for (const existingFile of collectionFiles) { + if (areFilesSame(existingFile.metadata, newFile.metadata)) { + return true; + } + } + return false; +} +export function areFilesSame( + existingFile: MetadataObject, + newFile: MetadataObject, +): boolean { + if ( + existingFile.fileType === newFile.fileType && + existingFile.creationTime === newFile.creationTime && + existingFile.modificationTime === newFile.modificationTime && + existingFile.title === newFile.title + ) { + return true; + } else { + return false; + } +} From 285946bd9c6771b8851a45ac2cc09c3e14af1f39 Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Tue, 10 Aug 2021 13:24:16 +0530 Subject: [PATCH 017/231] updated operator line breka elsint prop --- .eslintrc.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.eslintrc.json b/.eslintrc.json index 5e580ced6..2918ec78f 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -48,7 +48,8 @@ "error", "always" ], - "space-before-function-paren": "off" + "space-before-function-paren": "off", + "operator-linebreak":["error","after", { "overrides": { "?": "before", ":": "before" } }] }, "settings": { "react": { From 5e98981c058a291f70d819c1bfa68076194c925b Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Tue, 10 Aug 2021 17:59:57 +0530 Subject: [PATCH 018/231] refactored out uploadManager --- src/services/upload/uiService.ts | 119 +++++++++ src/services/upload/uploadManager.ts | 184 ++++++++++++++ src/services/upload/uploadService.ts | 347 +++++---------------------- src/services/upload/uploader.ts | 92 +++++++ src/utils/upload/index.ts | 23 ++ 5 files changed, 473 insertions(+), 292 deletions(-) create mode 100644 src/services/upload/uiService.ts create mode 100644 src/services/upload/uploadManager.ts create mode 100644 src/services/upload/uploader.ts diff --git a/src/services/upload/uiService.ts b/src/services/upload/uiService.ts new file mode 100644 index 000000000..81a4aeab1 --- /dev/null +++ b/src/services/upload/uiService.ts @@ -0,0 +1,119 @@ +import { ProgressUpdater } from 'components/pages/gallery/Upload'; + +export const RANDOM_PERCENTAGE_PROGRESS_FOR_PUT = () => 90 + 10 * Math.random(); + +class UIService { + private perFileProgress: number; + private filesUploaded: number; + private totalFileCount: number; + private fileProgress: Map; + private uploadResult: Map; + private progressUpdater: ProgressUpdater; + + init(progressUpdater: ProgressUpdater) { + this.progressUpdater = progressUpdater; + } + + reset(count: number) { + this.setTotalFileCount(count); + this.filesUploaded = 0; + } + + setTotalFileCount(count: number) { + this.totalFileCount = count; + this.perFileProgress = 100 / this.totalFileCount; + } + + setFileProgress(filename: string, progress: number) { + this.fileProgress.set(filename, progress); + this.updateProgressBarUI(); + } + + setUploadStage(stage) { + this.progressUpdater.setUploadStage(stage); + this.updateProgressBarUI(); + } + + setPercentComplete(percent) { + this.progressUpdater.setPercentComplete(percent); + this.updateProgressBarUI(); + } + + increaseFileUploaded() { + this.filesUploaded++; + this.updateProgressBarUI(); + } + + moveFileToResultList(filename: string) { + this.uploadResult.set(filename, this.fileProgress.get(filename)); + this.fileProgress.delete(filename); + this.updateProgressBarUI(); + } + + updateProgressBarUI() { + const { + setPercentComplete, + setFileCounter, + setFileProgress, + setUploadResult, + } = this.progressUpdater; + setFileCounter({ + finished: this.filesUploaded, + total: this.totalFileCount, + }); + let percentComplete = this.perFileProgress * this.filesUploaded; + if (this.fileProgress) { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for (const [_, progress] of this.fileProgress) { + // filter negative indicator values during percentComplete calculation + if (progress < 0) { + continue; + } + percentComplete += (this.perFileProgress * progress) / 100; + } + } + setPercentComplete(percentComplete); + setFileProgress(this.fileProgress); + setUploadResult(this.uploadResult); + } + + private trackUploadProgress( + filename: string, + percentPerPart = RANDOM_PERCENTAGE_PROGRESS_FOR_PUT(), + index = 0, + ) { + const cancel = { exec: null }; + let timeout = null; + const resetTimeout = () => { + if (timeout) { + clearTimeout(timeout); + } + timeout = setTimeout(() => cancel.exec(), 30 * 1000); + }; + return { + cancel, + onUploadProgress: (event) => { + filename && + this.fileProgress.set( + filename, + Math.min( + Math.round( + percentPerPart * index + + (percentPerPart * event.loaded) / + event.total, + ), + 98, + ), + ); + this.updateProgressBarUI(); + if (event.loaded === event.total) { + clearTimeout(timeout); + } else { + resetTimeout(); + } + }, + }; + } +} + +export default new UIService(); diff --git a/src/services/upload/uploadManager.ts b/src/services/upload/uploadManager.ts new file mode 100644 index 000000000..a7203a9f5 --- /dev/null +++ b/src/services/upload/uploadManager.ts @@ -0,0 +1,184 @@ +import { File, getLocalFiles } from '../fileService'; +import { Collection } from '../collectionService'; +import { SetFiles } from 'pages/gallery'; +import { parseError } from 'utils/common/errorUtil'; +import { ComlinkWorker, getDedicatedCryptoWorker } from 'utils/crypto'; +import { + sortFilesIntoCollections, + sortFiles, + removeUnneccessaryFileProps, +} from 'utils/file'; +import { logError } from 'utils/sentry'; +import localForage from 'utils/storage/localForage'; +import NetworkClient from './networkClient'; +import { ParsedMetaDataJSON, parseMetadataJSON } from './metadataService'; +import { segregateFiles } from 'utils/upload'; +import { ProgressUpdater } from 'components/pages/gallery/Upload'; +import uploader from './uploader'; +import uiService from './uiService'; + +const MAX_CONCURRENT_UPLOADS = 4; +const FILE_UPLOAD_COMPLETED = 100; + +export enum FileUploadResults { + FAILED = -1, + SKIPPED = -2, + UNSUPPORTED = -3, + BLOCKED = -4, + UPLOADED = 100, +} + +export interface UploadURL { + url: string; + objectKey: string; +} + +export interface FileWithCollection { + file: globalThis.File; + collection: Collection; +} + +export enum UPLOAD_STAGES { + START, + READING_GOOGLE_METADATA_FILES, + UPLOADING, + FINISH, +} + +class UploadManager { + private cryptoWorkers = new Array(MAX_CONCURRENT_UPLOADS); + private uploadURLs: UploadURL[] = []; + private metadataMap: Map; + private filesToBeUploaded: FileWithCollection[]; + private failedFiles: FileWithCollection[]; + private existingFilesCollectionWise: Map; + private existingFiles: File[]; + private setFiles: SetFiles; + + public async initUploader( + progressUpdater: ProgressUpdater, + setFiles: SetFiles, + ) { + uiService.init(progressUpdater); + this.setFiles = setFiles; + } + + private async init() { + this.failedFiles = []; + this.metadataMap = new Map(); + this.existingFiles = await getLocalFiles(); + this.existingFilesCollectionWise = sortFilesIntoCollections( + this.existingFiles, + ); + } + + public async queueFilesForUpload( + filesWithCollectionToUpload: FileWithCollection[], + ) { + try { + await this.init(); + + const { metadataFiles, mediaFiles } = segregateFiles( + filesWithCollectionToUpload, + ); + if (metadataFiles.length) { + uiService.setUploadStage( + UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES, + ); + await this.processMetadataFiles(metadataFiles); + } + if (mediaFiles.length) { + uiService.setUploadStage(UPLOAD_STAGES.START); + await this.uploadMediaFiles(mediaFiles); + } + uiService.setUploadStage(UPLOAD_STAGES.FINISH); + uiService.setPercentComplete(FILE_UPLOAD_COMPLETED); + } catch (e) { + logError(e, 'uploading failed with error'); + this.filesToBeUploaded = []; + throw e; + } finally { + for (let i = 0; i < MAX_CONCURRENT_UPLOADS; i++) { + this.cryptoWorkers[i]?.worker.terminate(); + } + } + } + + private async preFetchUploadURLs(count: number) { + try { + // checking for any subscription related errors + await NetworkClient.fetchUploadURLs(count, this.uploadURLs); + } catch (e) { + logError(e, 'error fetching uploadURLs'); + const { parsedError, parsed } = parseError(e); + if (parsed) { + throw parsedError; + } + } + } + + private async processMetadataFiles(metadataFiles: globalThis.File[]) { + uiService.reset(metadataFiles.length); + + for (const rawFile of metadataFiles) { + const parsedMetaDataJSON = await parseMetadataJSON(rawFile); + this.metadataMap.set(parsedMetaDataJSON.title, parsedMetaDataJSON); + uiService.increaseFileUploaded(); + } + } + + private async uploadMediaFiles(mediaFiles: FileWithCollection[]) { + this.filesToBeUploaded.push(...mediaFiles); + uiService.reset(mediaFiles.length); + + this.preFetchUploadURLs(mediaFiles.length); + + uiService.setUploadStage(UPLOAD_STAGES.UPLOADING); + + const uploadProcesses = []; + for ( + let i = 0; + i < Math.min(MAX_CONCURRENT_UPLOADS, this.filesToBeUploaded.length); + i++ + ) { + this.cryptoWorkers[i] = getDedicatedCryptoWorker(); + uploadProcesses.push( + this.uploadNextFileInQueue( + await new this.cryptoWorkers[i].comlink(), + new FileReader(), + ), + ); + } + await Promise.all(uploadProcesses); + } + + private async uploadNextFileInQueue(worker: any, fileReader: FileReader) { + if (this.filesToBeUploaded.length > 0) { + const fileWithCollection = this.filesToBeUploaded.pop(); + const { fileUploadResult, file } = await uploader( + worker, + fileReader, + fileWithCollection, + this.existingFilesCollectionWise, + ); + + if (fileUploadResult === FileUploadResults.UPLOADED) { + this.existingFiles.push(file); + this.existingFiles = sortFiles(this.existingFiles); + await localForage.setItem( + 'files', + removeUnneccessaryFileProps(this.existingFiles), + ); + this.setFiles(this.existingFiles); + } + + uiService.moveFileToResultList(fileWithCollection.file.name); + } + } + + async retryFailedFiles() { + await this.queueFilesForUpload(this.failedFiles); + } +} + +export default new UploadManager(); diff --git a/src/services/upload/uploadService.ts b/src/services/upload/uploadService.ts index f20fc50c5..8426ace32 100644 --- a/src/services/upload/uploadService.ts +++ b/src/services/upload/uploadService.ts @@ -1,44 +1,30 @@ -import { File, fileAttribute } from '../fileService'; +import { fileAttribute } from '../fileService'; import { Collection } from '../collectionService'; -import { FILE_TYPE, SetFiles } from 'pages/gallery'; -import { - CustomError, - handleError, - parseError, -} from 'utils/common/errorUtil'; -import { ComlinkWorker, getDedicatedCryptoWorker } from 'utils/crypto'; -import { - sortFilesIntoCollections, - sortFiles, - decryptFile, - removeUnneccessaryFileProps, -} from 'utils/file'; +import { FILE_TYPE } from 'pages/gallery'; import { logError } from 'utils/sentry'; -import localForage from 'utils/storage/localForage'; -import { sleep } from 'utils/common'; import NetworkClient from './networkClient'; -import { extractMetatdata, ParsedMetaDataJSON, parseMetadataJSON } from './metadataService'; +import { extractMetatdata, ParsedMetaDataJSON } from './metadataService'; import { generateThumbnail } from './thumbnailService'; -import { getFileType, getFileOriginalName, getFileData } from './readFileService'; +import { + getFileType, + getFileOriginalName, + getFileData, +} from './readFileService'; import { encryptFiledata } from './encryptionService'; import { ENCRYPTION_CHUNK_SIZE } from 'types'; import { uploadStreamUsingMultipart } from './s3Service'; -import { fileAlreadyInCollection } from 'utils/upload'; - -const MAX_CONCURRENT_UPLOADS = 4; -const TYPE_JSON = 'json'; -const FILE_UPLOAD_COMPLETED = 100; -const TwoSecondInMillSeconds = 2000; export const RANDOM_PERCENTAGE_PROGRESS_FOR_PUT = () => 90 + 10 * Math.random(); export const MIN_STREAM_FILE_SIZE = 20 * 1024 * 1024; -export const CHUNKS_COMBINED_FOR_A_UPLOAD_PART = Math.floor(MIN_STREAM_FILE_SIZE/ENCRYPTION_CHUNK_SIZE); +export const CHUNKS_COMBINED_FOR_A_UPLOAD_PART = Math.floor( + MIN_STREAM_FILE_SIZE / ENCRYPTION_CHUNK_SIZE, +); export enum FileUploadResults { FAILED = -1, SKIPPED = -2, UNSUPPORTED = -3, - BLOCKED=-4, + BLOCKED = -4, UPLOADED = 100, } @@ -69,7 +55,6 @@ export interface B64EncryptionResult { nonce: string; } - export interface MultipartUploadURLs { objectKey: string; partURLs: string[]; @@ -92,7 +77,7 @@ export interface FileInMemory { metadata: MetadataObject; } -interface EncryptedFile { +export interface EncryptedFile { file: ProcessedFile; fileKey: B64EncryptionResult; } @@ -102,9 +87,9 @@ export interface ProcessedFile { metadata: fileAttribute; filename: string; } -interface BackupedFile extends Omit { } +export interface BackupedFile extends Omit {} -export type MetadataMap = Map +export type MetadataMap = Map; export interface UploadFile extends BackupedFile { collectionID: number; @@ -120,215 +105,13 @@ export enum UPLOAD_STAGES { } class UploadService { - private cryptoWorkers = new Array(MAX_CONCURRENT_UPLOADS); private uploadURLs: UploadURL[] = []; - private perFileProgress: number; - private filesCompleted: number; - private totalFileCount: number; - private fileProgress: Map; - private uploadResult:Map; + private pendingFilesUploads: number; private metadataMap: Map; - private filesToBeUploaded: FileWithCollection[]; - private progressBarProps; - private failedFiles: FileWithCollection[]; - private existingFilesCollectionWise: Map; - private existingFiles: File[]; - private setFiles:SetFiles; - public async uploadFiles( - filesWithCollectionToUpload: FileWithCollection[], - existingFiles: File[], - progressBarProps, - setFiles:SetFiles, - ) { + + async readFile(reader: FileReader, receivedFile: globalThis.File) { try { - progressBarProps.setUploadStage(UPLOAD_STAGES.START); - - this.filesCompleted = 0; - this.fileProgress = new Map(); - this.uploadResult = new Map(); - this.failedFiles = []; - this.metadataMap = new Map(); - this.progressBarProps = progressBarProps; - this.existingFiles=existingFiles; - this.existingFilesCollectionWise = sortFilesIntoCollections(existingFiles); - this.updateProgressBarUI(); - this.setFiles=setFiles; - const metadataFiles: globalThis.File[] = []; - const actualFiles: FileWithCollection[] = []; - filesWithCollectionToUpload.forEach((fileWithCollection) => { - const file = fileWithCollection.file; - if (file?.name.substr(0, 1) === '.') { - // ignore files with name starting with . (hidden files) - return; - } - if (file.name.slice(-4) === TYPE_JSON) { - metadataFiles.push(fileWithCollection.file); - } else { - actualFiles.push(fileWithCollection); - } - }); - this.filesToBeUploaded = actualFiles; - - progressBarProps.setUploadStage( - UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES, - ); - this.totalFileCount = metadataFiles.length; - this.perFileProgress = 100 / metadataFiles.length; - this.filesCompleted = 0; - - for (const rawFile of metadataFiles) { - const parsedMetaDataJSON=await parseMetadataJSON(rawFile); - this.metadataMap.set(parsedMetaDataJSON.title, parsedMetaDataJSON); - this.filesCompleted++; - this.updateProgressBarUI(); - } - - progressBarProps.setUploadStage(UPLOAD_STAGES.START); - this.totalFileCount = actualFiles.length; - this.perFileProgress = 100 / actualFiles.length; - this.filesCompleted = 0; - this.updateProgressBarUI(); - try { - // checking for any subscription related errors - await NetworkClient.fetchUploadURLs(this.totalFileCount, this.uploadURLs); - } catch (e) { - logError(e, 'error fetching uploadURLs'); - const { parsedError, parsed } = parseError(e); - if (parsed) { - throw parsedError; - } - } - const uploadProcesses = []; - for ( - let i = 0; - i < MAX_CONCURRENT_UPLOADS; - i++ - ) { - if (this.filesToBeUploaded.length>0) { - const fileWithCollection= this.filesToBeUploaded.pop(); - this.cryptoWorkers[i] = getDedicatedCryptoWorker(); - uploadProcesses.push( - this.uploader( - await new this.cryptoWorkers[i].comlink(), - new FileReader(), - fileWithCollection, - ), - ); - } - } - progressBarProps.setUploadStage(UPLOAD_STAGES.UPLOADING); - await Promise.all(uploadProcesses); - progressBarProps.setUploadStage(UPLOAD_STAGES.FINISH); - progressBarProps.setPercentComplete(FILE_UPLOAD_COMPLETED); - } catch (e) { - logError(e, 'uploading failed with error'); - this.filesToBeUploaded = []; - throw e; - } finally { - for (let i = 0; i < MAX_CONCURRENT_UPLOADS; i++) { - this.cryptoWorkers[i]?.worker.terminate(); - } - } - } - - private async uploader( - worker: any, - reader: FileReader, - fileWithCollection: FileWithCollection, - ) { - const { file: rawFile, collection } = fileWithCollection; - this.fileProgress.set(rawFile.name, 0); - this.updateProgressBarUI(); - let file:FileInMemory=null; - let encryptedFile: EncryptedFile=null; - try { - // read the file into memory - file = await this.readFile(reader, rawFile, this.metadataMap); - - if (fileAlreadyInCollection(this.existingFilesCollectionWise, file, collection)) { - // set progress to -2 indicating that file upload was skipped - this.fileProgress.set(rawFile.name, FileUploadResults.SKIPPED); - this.updateProgressBarUI(); - await sleep(TwoSecondInMillSeconds); - } else { - encryptedFile = await this.encryptFile(worker, file, collection.key); - - const backupedFile: BackupedFile = await this.uploadToBucket( - encryptedFile.file, - ); - - const uploadFile: UploadFile = this.getUploadFile( - collection, - backupedFile, - encryptedFile.fileKey, - ); - - - const uploadedFile =await NetworkClient.uploadFile(uploadFile); - const decryptedFile=await decryptFile(uploadedFile, collection); - - this.existingFiles.push(decryptedFile); - this.existingFiles=sortFiles(this.existingFiles); - await localForage.setItem('files', removeUnneccessaryFileProps(this.existingFiles)); - this.setFiles(this.existingFiles); - this.fileProgress.set(rawFile.name, FileUploadResults.UPLOADED); - this.filesCompleted++; - } - } catch (e) { - logError(e, 'file upload failed'); - handleError(e); - this.failedFiles.push(fileWithCollection); - if (e.message ===CustomError.ETAG_MISSING) { - this.fileProgress.set(rawFile.name, FileUploadResults.BLOCKED); - } else { - this.fileProgress.set(rawFile.name, FileUploadResults.FAILED); - } - } finally { - file=null; - encryptedFile=null; - } - this.uploadResult.set(rawFile.name, this.fileProgress.get(rawFile.name)); - this.fileProgress.delete(rawFile.name); - this.updateProgressBarUI(); - - if (this.filesToBeUploaded.length > 0) { - await this.uploader( - worker, - reader, - this.filesToBeUploaded.pop(), - ); - } - } - async retryFailedFiles(localFiles:File[]) { - await this.uploadFiles(this.failedFiles, localFiles, this.progressBarProps, this.setFiles); - } - - private updateProgressBarUI() { - const { setPercentComplete, setFileCounter, setFileProgress, setUploadResult } = - this.progressBarProps; - setFileCounter({ - finished: this.filesCompleted, - total: this.totalFileCount, - }); - let percentComplete = this.perFileProgress * this.filesCompleted; - if (this.fileProgress) { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - for (const [_, progress] of this.fileProgress) { - // filter negative indicator values during percentComplete calculation - if (progress < 0) { - continue; - } - percentComplete += (this.perFileProgress * progress) / 100; - } - } - setPercentComplete(percentComplete); - setFileProgress(this.fileProgress); - setUploadResult(this.uploadResult); - } - - async readFile(reader: FileReader, receivedFile: globalThis.File, metadataMap:Map) { - try { - const fileType=getFileType(receivedFile); + const fileType = getFileType(receivedFile); const { thumbnail, hasStaticThumbnail } = await generateThumbnail( reader, @@ -336,13 +119,20 @@ class UploadService { fileType, ); - const originalName=getFileOriginalName(receivedFile); - const googleMetadata=metadataMap.get(originalName); - const extractedMetadata:MetadataObject =await extractMetatdata(reader, receivedFile, fileType); + const originalName = getFileOriginalName(receivedFile); + const googleMetadata = this.metadataMap.get(originalName); + const extractedMetadata: MetadataObject = await extractMetatdata( + reader, + receivedFile, + fileType, + ); if (hasStaticThumbnail) { - extractedMetadata.hasStaticThumbnail=true; + extractedMetadata.hasStaticThumbnail = true; } - const metadata:MetadataObject={ ...extractedMetadata, ...googleMetadata }; + const metadata: MetadataObject = { + ...extractedMetadata, + ...googleMetadata, + }; const filedata = await getFileData(reader, receivedFile); @@ -357,13 +147,14 @@ class UploadService { } } - private async encryptFile( + async encryptFile( worker: any, file: FileInMemory, encryptionKey: string, ): Promise { try { - const { key: fileKey, file: encryptedFiledata } = await encryptFiledata(worker, file.filedata); + const { key: fileKey, file: encryptedFiledata } = + await encryptFiledata(worker, file.filedata); const { file: encryptedThumbnail }: EncryptionResult = await worker.encryptThumbnail(file.thumbnail, fileKey); @@ -391,15 +182,24 @@ class UploadService { } } - - private async uploadToBucket(file: ProcessedFile): Promise { + async uploadToBucket( + file: ProcessedFile, + trackUploadProgress, + ): Promise { try { - let fileObjectKey:string=null; + let fileObjectKey: string = null; if (isDataStream(file.file.encryptedData)) { - const progressTracker=this.trackUploadProgress.bind(this); - fileObjectKey=await uploadStreamUsingMultipart(file.filename, file.file.encryptedData, progressTracker); + const progressTracker = trackUploadProgress; + fileObjectKey = await uploadStreamUsingMultipart( + file.filename, + file.file.encryptedData, + progressTracker, + ); } else { - const progressTracker=this.trackUploadProgress.bind(this, file.filename); + const progressTracker = trackUploadProgress.bind( + null, + file.filename, + ); const fileUploadURL = await this.getUploadURL(); fileObjectKey = await NetworkClient.putFile( fileUploadURL, @@ -411,7 +211,7 @@ class UploadService { const thumbnailObjectKey = await NetworkClient.putFile( thumbnailUploadURL, file.thumbnail.encryptedData as Uint8Array, - ()=>null, + () => null, ); const backupedFile: BackupedFile = { @@ -432,7 +232,7 @@ class UploadService { } } - private getUploadFile( + getUploadFile( collection: Collection, backupedFile: BackupedFile, fileKey: B64EncryptionResult, @@ -447,52 +247,15 @@ class UploadService { return uploadFile; } - private async getUploadURL() { if (this.uploadURLs.length === 0) { - await NetworkClient.fetchUploadURLs(this.totalFileCount-this.filesCompleted, this.uploadURLs); + await NetworkClient.fetchUploadURLs( + this.pendingFilesUploads, + this.uploadURLs, + ); } return this.uploadURLs.pop(); } - - - private trackUploadProgress( - filename:string, - percentPerPart = RANDOM_PERCENTAGE_PROGRESS_FOR_PUT(), - index = 0, - ) { - const cancel={ exec: null }; - let timeout=null; - const resetTimeout=()=>{ - if (timeout) { - clearTimeout(timeout); - } - timeout=setTimeout(()=>cancel.exec(), 30*1000); - }; - return { - cancel, - onUploadProgress: (event) => { - filename && - this.fileProgress.set( - filename, - Math.min( - Math.round( - percentPerPart * index + - (percentPerPart * event.loaded) / - event.total, - ), - 98, - ), - ); - this.updateProgressBarUI(); - if (event.loaded===event.total) { - clearTimeout(timeout); - } else { - resetTimeout(); - } - }, - }; - } } export default new UploadService(); diff --git a/src/services/upload/uploader.ts b/src/services/upload/uploader.ts new file mode 100644 index 000000000..df51482a7 --- /dev/null +++ b/src/services/upload/uploader.ts @@ -0,0 +1,92 @@ +import { File } from 'services/fileService'; +import { sleep } from 'utils/common'; +import { handleError, CustomError } from 'utils/common/errorUtil'; +import { decryptFile } from 'utils/file'; +import { logError } from 'utils/sentry'; +import { fileAlreadyInCollection } from 'utils/upload'; +import NetworkClient from './networkClient'; +import uiService from './uiService'; +import UploadService, { + BackupedFile, + EncryptedFile, + FileInMemory, + FileUploadResults, + FileWithCollection, + UploadFile, +} from './uploadService'; + +const TwoSecondInMillSeconds = 2000; + +interface UploadResponse { + fileUploadResult: FileUploadResults; + file?: File; +} +export default async function uploader( + worker: any, + reader: FileReader, + fileWithCollection: FileWithCollection, + existingFilesCollectionWise: Map, +): Promise { + const { file: rawFile, collection } = fileWithCollection; + uiService.setFileProgress(rawFile.name, 0); + let file: FileInMemory = null; + let encryptedFile: EncryptedFile = null; + try { + // read the file into memory + file = await UploadService.readFile(reader, rawFile); + + if ( + fileAlreadyInCollection( + existingFilesCollectionWise, + file, + collection, + ) + ) { + // set progress to -2 indicating that file upload was skipped + uiService.setFileProgress(rawFile.name, FileUploadResults.SKIPPED); + await sleep(TwoSecondInMillSeconds); + return { fileUploadResult: FileUploadResults.SKIPPED }; + } + + encryptedFile = await UploadService.encryptFile( + worker, + file, + collection.key, + ); + + const backupedFile: BackupedFile = await UploadService.uploadToBucket( + encryptedFile.file, + this.trackUploadProgress.bind(this), + ); + + const uploadFile: UploadFile = UploadService.getUploadFile( + collection, + backupedFile, + encryptedFile.fileKey, + ); + + const uploadedFile = await NetworkClient.uploadFile(uploadFile); + const decryptedFile = await decryptFile(uploadedFile, collection); + + uiService.setFileProgress(rawFile.name, FileUploadResults.UPLOADED); + uiService.increaseFileUploaded(); + return { + fileUploadResult: FileUploadResults.UPLOADED, + file: decryptedFile, + }; + } catch (e) { + logError(e, 'file upload failed'); + handleError(e); + this.failedFiles.push(fileWithCollection); + if (e.message === CustomError.ETAG_MISSING) { + uiService.setFileProgress(rawFile.name, FileUploadResults.BLOCKED); + return { fileUploadResult: FileUploadResults.BLOCKED }; + } else { + uiService.setFileProgress(rawFile.name, FileUploadResults.FAILED); + return { fileUploadResult: FileUploadResults.FAILED }; + } + } finally { + file = null; + encryptedFile = null; + } +} diff --git a/src/utils/upload/index.ts b/src/utils/upload/index.ts index 70603f01b..11881a108 100644 --- a/src/utils/upload/index.ts +++ b/src/utils/upload/index.ts @@ -1,6 +1,9 @@ import { Collection } from 'services/collectionService'; +import { FileWithCollection } from 'services/upload/uploadManager'; import { FileInMemory, MetadataObject } from 'services/upload/uploadService'; +const TYPE_JSON = 'json'; + export function fileAlreadyInCollection( existingFilesCollectionWise, newFile: FileInMemory, @@ -30,3 +33,23 @@ export function areFilesSame( return false; } } + +export function segregateFiles( + filesWithCollectionToUpload: FileWithCollection[], +) { + const metadataFiles: globalThis.File[] = []; + const mediaFiles: FileWithCollection[] = []; + filesWithCollectionToUpload.forEach((fileWithCollection) => { + const file = fileWithCollection.file; + if (file?.name.substr(0, 1) === '.') { + // ignore files with name starting with . (hidden files) + return; + } + if (file.name.slice(-4) === TYPE_JSON) { + metadataFiles.push(fileWithCollection.file); + } else { + mediaFiles.push(fileWithCollection); + } + }); + return { mediaFiles, metadataFiles }; +} From 0802370f353b0409c7071f6ae55587de0f6b8eab Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Tue, 10 Aug 2021 18:02:37 +0530 Subject: [PATCH 019/231] updated uploadComponent to use the new UploadManager --- src/components/pages/gallery/Upload.tsx | 88 +++++++++++++++---------- 1 file changed, 55 insertions(+), 33 deletions(-) diff --git a/src/components/pages/gallery/Upload.tsx b/src/components/pages/gallery/Upload.tsx index b91ccf5e9..a184b91d0 100644 --- a/src/components/pages/gallery/Upload.tsx +++ b/src/components/pages/gallery/Upload.tsx @@ -1,7 +1,6 @@ import React, { useContext, useEffect, useState } from 'react'; -import UploadService, { FileWithCollection, UPLOAD_STAGES } from 'services/upload/uploadService'; + import { createAlbum } from 'services/collectionService'; -import { getLocalFiles } from 'services/fileService'; import constants from 'utils/strings/constants'; import { SetDialogMessage } from 'components/MessageDialog'; import UploadProgress from './UploadProgress'; @@ -13,6 +12,11 @@ import { SetFiles, SetLoading } from 'pages/gallery'; import { AppContext } from 'pages/_app'; import { logError } from 'utils/sentry'; import { FileRejection } from 'react-dropzone'; +import UploadManager, { + FileWithCollection, + UPLOAD_STAGES, +} from 'services/upload/uploadManager'; +import uploadManager from 'services/upload/uploadManager'; interface Props { syncWithRemote: (force?: boolean, silent?: boolean) => Promise; @@ -25,8 +29,8 @@ interface Props { setDialogMessage: SetDialogMessage; setUploadInProgress: any; showCollectionSelector: () => void; - fileRejections:FileRejection[]; - setFiles:SetFiles; + fileRejections: FileRejection[]; + setFiles: SetFiles; } export enum UPLOAD_STRATEGY { @@ -38,22 +42,50 @@ interface AnalysisResult { suggestedCollectionName: string; multipleFolders: boolean; } +export interface ProgressUpdater { + setPercentComplete: React.Dispatch>; + setFileCounter: React.Dispatch< + React.SetStateAction<{ + finished: number; + total: number; + }> + >; + setUploadStage: React.Dispatch>; + setFileProgress: React.Dispatch>>; + setUploadResult: React.Dispatch>>; +} export default function Upload(props: Props) { const [progressView, setProgressView] = useState(false); const [uploadStage, setUploadStage] = useState( UPLOAD_STAGES.START, ); - const [fileCounter, setFileCounter] = useState({ current: 0, total: 0 }); + const [fileCounter, setFileCounter] = useState({ finished: 0, total: 0 }); const [fileProgress, setFileProgress] = useState(new Map()); - const [uploadResult, setUploadResult]=useState(new Map()); + const [uploadResult, setUploadResult] = useState(new Map()); const [percentComplete, setPercentComplete] = useState(0); const [choiceModalView, setChoiceModalView] = useState(false); - const [fileAnalysisResult, setFileAnalysisResult] = useState(null); + const [fileAnalysisResult, setFileAnalysisResult] = + useState(null); const appContext = useContext(AppContext); useEffect(() => { - if (props.acceptedFiles?.length > 0 || appContext.sharedFiles?.length > 0) { + UploadManager.initUploader( + { + setPercentComplete, + setFileCounter, + setFileProgress, + setUploadResult, + setUploadStage, + }, + props.setFiles, + ); + }); + useEffect(() => { + if ( + props.acceptedFiles?.length > 0 || + appContext.sharedFiles?.length > 0 + ) { props.setLoading(true); let fileAnalysisResult; @@ -77,7 +109,7 @@ export default function Upload(props: Props) { const uploadInit = function () { setUploadStage(UPLOAD_STAGES.START); - setFileCounter({ current: 0, total: 0 }); + setFileCounter({ finished: 0, total: 0 }); setFileProgress(new Map()); setUploadResult(new Map()); setPercentComplete(0); @@ -99,9 +131,9 @@ export default function Upload(props: Props) { }; const nextModal = (fileAnalysisResult: AnalysisResult) => { - fileAnalysisResult?.multipleFolders ? - setChoiceModalView(true) : - showCreateCollectionModal(fileAnalysisResult); + fileAnalysisResult?.multipleFolders + ? setChoiceModalView(true) + : showCreateCollectionModal(fileAnalysisResult); }; function analyseUploadFiles(): AnalysisResult { @@ -148,10 +180,11 @@ export default function Upload(props: Props) { const uploadFilesToExistingCollection = async (collection) => { try { uploadInit(); - const filesWithCollectionToUpload: FileWithCollection[] = props.acceptedFiles.map((file) => ({ - file, - collection, - })); + const filesWithCollectionToUpload: FileWithCollection[] = + props.acceptedFiles.map((file) => ({ + file, + collection, + })); await uploadFiles(filesWithCollectionToUpload); } catch (e) { logError(e, 'Failed to upload files to existing collections'); @@ -203,18 +236,8 @@ export default function Upload(props: Props) { props.setUploadInProgress(true); props.closeCollectionSelector(); await props.syncWithRemote(true, true); - const localFiles= await getLocalFiles(); - await UploadService.uploadFiles( + await uploadManager.queueFilesForUpload( filesWithCollectionToUpload, - localFiles, - { - setPercentComplete, - setFileCounter, - setUploadStage, - setFileProgress, - setUploadResult, - }, - props.setFiles, ); } catch (err) { props.setBannerMessage(err.message); @@ -226,14 +249,12 @@ export default function Upload(props: Props) { props.syncWithRemote(); } }; - const retryFailed = async ( - ) => { + const retryFailed = async () => { try { props.setUploadInProgress(true); uploadInit(); await props.syncWithRemote(true, true); - const localFiles= await getLocalFiles(); - await UploadService.retryFailedFiles(localFiles); + await uploadManager.retryFailedFiles(); } catch (err) { props.setBannerMessage(err.message); setProgressView(false); @@ -244,14 +265,15 @@ export default function Upload(props: Props) { } }; - return ( <> setChoiceModalView(false)} uploadFiles={uploadFilesToNewCollections} - showCollectionCreateModal={() => showCreateCollectionModal(fileAnalysisResult)} + showCollectionCreateModal={() => + showCreateCollectionModal(fileAnalysisResult) + } /> Date: Wed, 11 Aug 2021 10:15:19 +0530 Subject: [PATCH 020/231] moved progressTracking logic to uiService --- src/services/upload/s3Service.ts | 59 ++++++++++++++++++++------------ src/services/upload/uiService.ts | 2 +- src/services/upload/uploader.ts | 2 -- 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/src/services/upload/s3Service.ts b/src/services/upload/s3Service.ts index abd1f65d9..52c369502 100644 --- a/src/services/upload/s3Service.ts +++ b/src/services/upload/s3Service.ts @@ -1,19 +1,25 @@ -import { CHUNKS_COMBINED_FOR_A_UPLOAD_PART, DataStream, MultipartUploadURLs, RANDOM_PERCENTAGE_PROGRESS_FOR_PUT } from './uploadService'; +import { + CHUNKS_COMBINED_FOR_A_UPLOAD_PART, + DataStream, + MultipartUploadURLs, + RANDOM_PERCENTAGE_PROGRESS_FOR_PUT, +} from './uploadService'; import NetworkClient from './networkClient'; import * as convert from 'xml-js'; +import uiService from './uiService'; - -interface PartEtag{ - PartNumber:number; - Etag:string; +interface PartEtag { + PartNumber: number; + Etag: string; } export function calculatePartCount(chunkCount: number) { - const partCount = Math.ceil( - chunkCount / CHUNKS_COMBINED_FOR_A_UPLOAD_PART, - ); + const partCount = Math.ceil(chunkCount / CHUNKS_COMBINED_FOR_A_UPLOAD_PART); return partCount; } -export async function uploadStreamUsingMultipart(filename:string, dataStream:DataStream, progressTracker) { +export async function uploadStreamUsingMultipart( + filename: string, + dataStream: DataStream, +) { const uploadPartCount = calculatePartCount(dataStream.chunkCount); const multipartUploadURLs = await NetworkClient.fetchMultipartUploadURLs( uploadPartCount, @@ -23,7 +29,6 @@ export async function uploadStreamUsingMultipart(filename:string, dataStream:Dat dataStream.stream, filename, uploadPartCount, - progressTracker, ); return fileObjectKey; } @@ -33,38 +38,46 @@ export async function uploadStreamInParts( dataStream: ReadableStream, filename: string, uploadPartCount: number, - progressTracker, ) { const streamReader = dataStream.getReader(); const percentPerPart = getRandomProgressPerPartUpload(uploadPartCount); - const partEtags:PartEtag[] = []; + const partEtags: PartEtag[] = []; for (const [ index, fileUploadURL, ] of multipartUploadURLs.partURLs.entries()) { const uploadChunk = await combineChunksToFormUploadPart(streamReader); - const eTag= await NetworkClient.putFilePart(fileUploadURL, uploadChunk, progressTracker.bind(null, filename, percentPerPart, index)); - partEtags.push({ PartNumber: index+1, Etag: eTag }); + const progressTracker = uiService.trackUploadProgress( + filename, + percentPerPart, + index, + ); + + const eTag = await NetworkClient.putFilePart( + fileUploadURL, + uploadChunk, + progressTracker, + ); + partEtags.push({ PartNumber: index + 1, Etag: eTag }); } await completeMultipartUpload(partEtags, multipartUploadURLs.completeURL); return multipartUploadURLs.objectKey; } - -export function getRandomProgressPerPartUpload(uploadPartCount:number) { +export function getRandomProgressPerPartUpload(uploadPartCount: number) { const percentPerPart = Math.round( RANDOM_PERCENTAGE_PROGRESS_FOR_PUT() / uploadPartCount, ); return percentPerPart; } - -export async function combineChunksToFormUploadPart(streamReader:ReadableStreamDefaultReader) { +export async function combineChunksToFormUploadPart( + streamReader: ReadableStreamDefaultReader, +) { const combinedChunks = []; for (let i = 0; i < CHUNKS_COMBINED_FOR_A_UPLOAD_PART; i++) { - const { done, value: chunk } = - await streamReader.read(); + const { done, value: chunk } = await streamReader.read(); if (done) { break; } @@ -75,8 +88,10 @@ export async function combineChunksToFormUploadPart(streamReader:ReadableStreamD return Uint8Array.from(combinedChunks); } - -async function completeMultipartUpload(partEtags:PartEtag[], completeURL:string) { +async function completeMultipartUpload( + partEtags: PartEtag[], + completeURL: string, +) { const options = { compact: true, ignoreComment: true, spaces: 4 }; const body = convert.js2xml( { CompleteMultipartUpload: { Part: partEtags } }, diff --git a/src/services/upload/uiService.ts b/src/services/upload/uiService.ts index 81a4aeab1..4942df12b 100644 --- a/src/services/upload/uiService.ts +++ b/src/services/upload/uiService.ts @@ -77,7 +77,7 @@ class UIService { setUploadResult(this.uploadResult); } - private trackUploadProgress( + trackUploadProgress( filename: string, percentPerPart = RANDOM_PERCENTAGE_PROGRESS_FOR_PUT(), index = 0, diff --git a/src/services/upload/uploader.ts b/src/services/upload/uploader.ts index df51482a7..917d4450e 100644 --- a/src/services/upload/uploader.ts +++ b/src/services/upload/uploader.ts @@ -32,7 +32,6 @@ export default async function uploader( let file: FileInMemory = null; let encryptedFile: EncryptedFile = null; try { - // read the file into memory file = await UploadService.readFile(reader, rawFile); if ( @@ -56,7 +55,6 @@ export default async function uploader( const backupedFile: BackupedFile = await UploadService.uploadToBucket( encryptedFile.file, - this.trackUploadProgress.bind(this), ); const uploadFile: UploadFile = UploadService.getUploadFile( From e59be3e17463eff21b910d6658b6452c2a0700ff Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Wed, 11 Aug 2021 10:16:06 +0530 Subject: [PATCH 021/231] moved UploadURl logic to uploadService --- src/services/upload/networkClient.ts | 42 +++++++++++----------------- src/services/upload/uploadManager.ts | 35 ++++++++++++----------- src/services/upload/uploadService.ts | 39 +++++++++++++++----------- 3 files changed, 57 insertions(+), 59 deletions(-) diff --git a/src/services/upload/networkClient.ts b/src/services/upload/networkClient.ts index fdfb3d4da..2c488260c 100644 --- a/src/services/upload/networkClient.ts +++ b/src/services/upload/networkClient.ts @@ -10,24 +10,20 @@ import { retryAsyncFunction } from 'utils/network'; const ENDPOINT = getEndpoint(); const MAX_URL_REQUESTS = 50; - class NetworkClient { - private uploadURLFetchInProgress=null; + private uploadURLFetchInProgress = null; - async uploadFile(uploadFile: UploadFile):Promise { + async uploadFile(uploadFile: UploadFile): Promise { try { const token = getToken(); if (!token) { return; } - const response = await retryAsyncFunction(()=>HTTPService.post( - `${ENDPOINT}/files`, - uploadFile, - null, - { + const response = await retryAsyncFunction(() => + HTTPService.post(`${ENDPOINT}/files`, uploadFile, null, { 'X-Auth-Token': token, - }, - )); + }), + ); return response.data; } catch (e) { logError(e, 'upload Files Failed'); @@ -35,7 +31,7 @@ class NetworkClient { } } - async fetchUploadURLs(count:number, urlStore:UploadURL[]): Promise { + async fetchUploadURLs(count: number, urlStore: UploadURL[]): Promise { try { if (!this.uploadURLFetchInProgress) { try { @@ -46,10 +42,7 @@ class NetworkClient { this.uploadURLFetchInProgress = HTTPService.get( `${ENDPOINT}/files/upload-urls`, { - count: Math.min( - MAX_URL_REQUESTS, - count * 2, - ), + count: Math.min(MAX_URL_REQUESTS, count * 2), }, { 'X-Auth-Token': token }, ); @@ -92,16 +85,16 @@ class NetworkClient { async putFile( fileUploadURL: UploadURL, file: Uint8Array, - progressTracker:()=>any, + progressTracker, ): Promise { try { - await retryAsyncFunction(()=> + await retryAsyncFunction(() => HTTPService.put( fileUploadURL.url, file, null, null, - progressTracker(), + progressTracker, ), ); return fileUploadURL.objectKey; @@ -111,15 +104,14 @@ class NetworkClient { } } - async putFilePart( partUploadURL: string, filePart: Uint8Array, progressTracker, ) { try { - const response=await retryAsyncFunction(async ()=>{ - const resp =await HTTPService.put( + const response = await retryAsyncFunction(async () => { + const resp = await HTTPService.put( partUploadURL, filePart, null, @@ -127,7 +119,7 @@ class NetworkClient { progressTracker(), ); if (!resp?.headers?.etag) { - const err=Error(CustomError.ETAG_MISSING); + const err = Error(CustomError.ETAG_MISSING); logError(err); throw err; } @@ -140,9 +132,9 @@ class NetworkClient { } } - async completeMultipartUpload(completeURL:string, reqBody:any) { + async completeMultipartUpload(completeURL: string, reqBody: any) { try { - await retryAsyncFunction(()=> + await retryAsyncFunction(() => HTTPService.post(completeURL, reqBody, null, { 'content-type': 'text/xml', }), @@ -155,5 +147,3 @@ class NetworkClient { } export default new NetworkClient(); - - diff --git a/src/services/upload/uploadManager.ts b/src/services/upload/uploadManager.ts index a7203a9f5..6573da37a 100644 --- a/src/services/upload/uploadManager.ts +++ b/src/services/upload/uploadManager.ts @@ -10,12 +10,12 @@ import { } from 'utils/file'; import { logError } from 'utils/sentry'; import localForage from 'utils/storage/localForage'; -import NetworkClient from './networkClient'; import { ParsedMetaDataJSON, parseMetadataJSON } from './metadataService'; import { segregateFiles } from 'utils/upload'; import { ProgressUpdater } from 'components/pages/gallery/Upload'; import uploader from './uploader'; import uiService from './uiService'; +import uploadService from './uploadService'; const MAX_CONCURRENT_UPLOADS = 4; const FILE_UPLOAD_COMPLETED = 100; @@ -85,7 +85,7 @@ class UploadManager { uiService.setUploadStage( UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES, ); - await this.processMetadataFiles(metadataFiles); + await this.seedMetadataMap(metadataFiles); } if (mediaFiles.length) { uiService.setUploadStage(UPLOAD_STAGES.START); @@ -104,20 +104,7 @@ class UploadManager { } } - private async preFetchUploadURLs(count: number) { - try { - // checking for any subscription related errors - await NetworkClient.fetchUploadURLs(count, this.uploadURLs); - } catch (e) { - logError(e, 'error fetching uploadURLs'); - const { parsedError, parsed } = parseError(e); - if (parsed) { - throw parsedError; - } - } - } - - private async processMetadataFiles(metadataFiles: globalThis.File[]) { + private async seedMetadataMap(metadataFiles: globalThis.File[]) { uiService.reset(metadataFiles.length); for (const rawFile of metadataFiles) { @@ -152,8 +139,22 @@ class UploadManager { await Promise.all(uploadProcesses); } + private async preFetchUploadURLs(count: number) { + try { + uploadService.setPendingUploadCount(count); + uploadService.preFetchUploadURLs(); + // checking for any subscription related errors + } catch (e) { + logError(e, 'error fetching uploadURLs'); + const { parsedError, parsed } = parseError(e); + if (parsed) { + throw parsedError; + } + } + } + private async uploadNextFileInQueue(worker: any, fileReader: FileReader) { - if (this.filesToBeUploaded.length > 0) { + while (this.filesToBeUploaded.length > 0) { const fileWithCollection = this.filesToBeUploaded.pop(); const { fileUploadResult, file } = await uploader( worker, diff --git a/src/services/upload/uploadService.ts b/src/services/upload/uploadService.ts index 8426ace32..55263adcc 100644 --- a/src/services/upload/uploadService.ts +++ b/src/services/upload/uploadService.ts @@ -13,6 +13,7 @@ import { import { encryptFiledata } from './encryptionService'; import { ENCRYPTION_CHUNK_SIZE } from 'types'; import { uploadStreamUsingMultipart } from './s3Service'; +import uiService from './uiService'; export const RANDOM_PERCENTAGE_PROGRESS_FOR_PUT = () => 90 + 10 * Math.random(); export const MIN_STREAM_FILE_SIZE = 20 * 1024 * 1024; @@ -106,8 +107,12 @@ export enum UPLOAD_STAGES { class UploadService { private uploadURLs: UploadURL[] = []; - private pendingFilesUploads: number; private metadataMap: Map; + private pendingUploadCount: number = 0; + + setPendingUploadCount(count: number) { + this.pendingUploadCount = count; + } async readFile(reader: FileReader, receivedFile: globalThis.File) { try { @@ -140,7 +145,7 @@ class UploadService { filedata, thumbnail, metadata, - }; + } as FileInMemory; } catch (e) { logError(e, 'error reading files'); throw e; @@ -182,22 +187,16 @@ class UploadService { } } - async uploadToBucket( - file: ProcessedFile, - trackUploadProgress, - ): Promise { + async uploadToBucket(file: ProcessedFile): Promise { try { let fileObjectKey: string = null; if (isDataStream(file.file.encryptedData)) { - const progressTracker = trackUploadProgress; fileObjectKey = await uploadStreamUsingMultipart( file.filename, file.file.encryptedData, - progressTracker, ); } else { - const progressTracker = trackUploadProgress.bind( - null, + const progressTracker = uiService.trackUploadProgress( file.filename, ); const fileUploadURL = await this.getUploadURL(); @@ -211,7 +210,7 @@ class UploadService { const thumbnailObjectKey = await NetworkClient.putFile( thumbnailUploadURL, file.thumbnail.encryptedData as Uint8Array, - () => null, + null, ); const backupedFile: BackupedFile = { @@ -248,14 +247,22 @@ class UploadService { } private async getUploadURL() { - if (this.uploadURLs.length === 0) { - await NetworkClient.fetchUploadURLs( - this.pendingFilesUploads, - this.uploadURLs, - ); + if (this.uploadURLs.length === 0 && this.pendingUploadCount) { + await this.fetchUploadURLs(); } return this.uploadURLs.pop(); } + + public async preFetchUploadURLs() { + await this.fetchUploadURLs(); + } + + private async fetchUploadURLs() { + await NetworkClient.fetchUploadURLs( + this.pendingUploadCount, + this.uploadURLs, + ); + } } export default new UploadService(); From c3693a2eb3d685c1927347e7707edab6d5073c8a Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Wed, 11 Aug 2021 10:54:07 +0530 Subject: [PATCH 022/231] minor bug fixes --- src/services/upload/uiService.ts | 3 +++ src/services/upload/uploadManager.ts | 27 ++++----------------------- src/services/upload/uploadService.ts | 16 ++++++++++++++-- src/services/upload/uploader.ts | 1 - 4 files changed, 21 insertions(+), 26 deletions(-) diff --git a/src/services/upload/uiService.ts b/src/services/upload/uiService.ts index 4942df12b..eaa0da539 100644 --- a/src/services/upload/uiService.ts +++ b/src/services/upload/uiService.ts @@ -17,6 +17,9 @@ class UIService { reset(count: number) { this.setTotalFileCount(count); this.filesUploaded = 0; + this.fileProgress = new Map(); + this.uploadResult = new Map(); + this.updateProgressBarUI(); } setTotalFileCount(count: number) { diff --git a/src/services/upload/uploadManager.ts b/src/services/upload/uploadManager.ts index 6573da37a..1a878ccf7 100644 --- a/src/services/upload/uploadManager.ts +++ b/src/services/upload/uploadManager.ts @@ -1,7 +1,6 @@ import { File, getLocalFiles } from '../fileService'; import { Collection } from '../collectionService'; import { SetFiles } from 'pages/gallery'; -import { parseError } from 'utils/common/errorUtil'; import { ComlinkWorker, getDedicatedCryptoWorker } from 'utils/crypto'; import { sortFilesIntoCollections, @@ -47,7 +46,6 @@ export enum UPLOAD_STAGES { class UploadManager { private cryptoWorkers = new Array(MAX_CONCURRENT_UPLOADS); - private uploadURLs: UploadURL[] = []; private metadataMap: Map; private filesToBeUploaded: FileWithCollection[]; private failedFiles: FileWithCollection[]; @@ -55,15 +53,13 @@ class UploadManager { private existingFiles: File[]; private setFiles: SetFiles; - public async initUploader( - progressUpdater: ProgressUpdater, - setFiles: SetFiles, - ) { + public initUploader(progressUpdater: ProgressUpdater, setFiles: SetFiles) { uiService.init(progressUpdater); this.setFiles = setFiles; } private async init() { + this.filesToBeUploaded = []; this.failedFiles = []; this.metadataMap = new Map(); this.existingFiles = await getLocalFiles(); @@ -95,7 +91,6 @@ class UploadManager { uiService.setPercentComplete(FILE_UPLOAD_COMPLETED); } catch (e) { logError(e, 'uploading failed with error'); - this.filesToBeUploaded = []; throw e; } finally { for (let i = 0; i < MAX_CONCURRENT_UPLOADS; i++) { @@ -118,14 +113,14 @@ class UploadManager { this.filesToBeUploaded.push(...mediaFiles); uiService.reset(mediaFiles.length); - this.preFetchUploadURLs(mediaFiles.length); + uploadService.init(mediaFiles.length, this.metadataMap); uiService.setUploadStage(UPLOAD_STAGES.UPLOADING); const uploadProcesses = []; for ( let i = 0; - i < Math.min(MAX_CONCURRENT_UPLOADS, this.filesToBeUploaded.length); + i < MAX_CONCURRENT_UPLOADS && this.filesToBeUploaded.length > 0; i++ ) { this.cryptoWorkers[i] = getDedicatedCryptoWorker(); @@ -139,20 +134,6 @@ class UploadManager { await Promise.all(uploadProcesses); } - private async preFetchUploadURLs(count: number) { - try { - uploadService.setPendingUploadCount(count); - uploadService.preFetchUploadURLs(); - // checking for any subscription related errors - } catch (e) { - logError(e, 'error fetching uploadURLs'); - const { parsedError, parsed } = parseError(e); - if (parsed) { - throw parsedError; - } - } - } - private async uploadNextFileInQueue(worker: any, fileReader: FileReader) { while (this.filesToBeUploaded.length > 0) { const fileWithCollection = this.filesToBeUploaded.pop(); diff --git a/src/services/upload/uploadService.ts b/src/services/upload/uploadService.ts index 55263adcc..1bf190365 100644 --- a/src/services/upload/uploadService.ts +++ b/src/services/upload/uploadService.ts @@ -14,6 +14,7 @@ import { encryptFiledata } from './encryptionService'; import { ENCRYPTION_CHUNK_SIZE } from 'types'; import { uploadStreamUsingMultipart } from './s3Service'; import uiService from './uiService'; +import { parseError } from 'utils/common/errorUtil'; export const RANDOM_PERCENTAGE_PROGRESS_FOR_PUT = () => 90 + 10 * Math.random(); export const MIN_STREAM_FILE_SIZE = 20 * 1024 * 1024; @@ -110,8 +111,10 @@ class UploadService { private metadataMap: Map; private pendingUploadCount: number = 0; - setPendingUploadCount(count: number) { + init(count: number, metadataMap: MetadataMap) { this.pendingUploadCount = count; + this.metadataMap = metadataMap; + this.preFetchUploadURLs(); } async readFile(reader: FileReader, receivedFile: globalThis.File) { @@ -254,7 +257,16 @@ class UploadService { } public async preFetchUploadURLs() { - await this.fetchUploadURLs(); + try { + await this.fetchUploadURLs(); + // checking for any subscription related errors + } catch (e) { + logError(e, 'error fetching uploadURLs'); + const { parsedError, parsed } = parseError(e); + if (parsed) { + throw parsedError; + } + } } private async fetchUploadURLs() { diff --git a/src/services/upload/uploader.ts b/src/services/upload/uploader.ts index 917d4450e..d57f3dd46 100644 --- a/src/services/upload/uploader.ts +++ b/src/services/upload/uploader.ts @@ -75,7 +75,6 @@ export default async function uploader( } catch (e) { logError(e, 'file upload failed'); handleError(e); - this.failedFiles.push(fileWithCollection); if (e.message === CustomError.ETAG_MISSING) { uiService.setFileProgress(rawFile.name, FileUploadResults.BLOCKED); return { fileUploadResult: FileUploadResults.BLOCKED }; From 74340329569ff1e8d8dd12caca6d0f0fcdc3bbbf Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Wed, 11 Aug 2021 10:56:40 +0530 Subject: [PATCH 023/231] corrected case --- src/services/upload/s3Service.ts | 4 ++-- src/services/upload/uploadManager.ts | 26 +++++++++++++------------- src/services/upload/uploadService.ts | 4 ++-- src/services/upload/uploader.ts | 14 +++++++------- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/services/upload/s3Service.ts b/src/services/upload/s3Service.ts index 52c369502..ae9aa0650 100644 --- a/src/services/upload/s3Service.ts +++ b/src/services/upload/s3Service.ts @@ -6,7 +6,7 @@ import { } from './uploadService'; import NetworkClient from './networkClient'; import * as convert from 'xml-js'; -import uiService from './uiService'; +import UIService from './uiService'; interface PartEtag { PartNumber: number; @@ -48,7 +48,7 @@ export async function uploadStreamInParts( fileUploadURL, ] of multipartUploadURLs.partURLs.entries()) { const uploadChunk = await combineChunksToFormUploadPart(streamReader); - const progressTracker = uiService.trackUploadProgress( + const progressTracker = UIService.trackUploadProgress( filename, percentPerPart, index, diff --git a/src/services/upload/uploadManager.ts b/src/services/upload/uploadManager.ts index 1a878ccf7..47c82a4c3 100644 --- a/src/services/upload/uploadManager.ts +++ b/src/services/upload/uploadManager.ts @@ -13,8 +13,8 @@ import { ParsedMetaDataJSON, parseMetadataJSON } from './metadataService'; import { segregateFiles } from 'utils/upload'; import { ProgressUpdater } from 'components/pages/gallery/Upload'; import uploader from './uploader'; -import uiService from './uiService'; -import uploadService from './uploadService'; +import UIService from './uiService'; +import UploadService from './uploadService'; const MAX_CONCURRENT_UPLOADS = 4; const FILE_UPLOAD_COMPLETED = 100; @@ -54,7 +54,7 @@ class UploadManager { private setFiles: SetFiles; public initUploader(progressUpdater: ProgressUpdater, setFiles: SetFiles) { - uiService.init(progressUpdater); + UIService.init(progressUpdater); this.setFiles = setFiles; } @@ -78,17 +78,17 @@ class UploadManager { filesWithCollectionToUpload, ); if (metadataFiles.length) { - uiService.setUploadStage( + UIService.setUploadStage( UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES, ); await this.seedMetadataMap(metadataFiles); } if (mediaFiles.length) { - uiService.setUploadStage(UPLOAD_STAGES.START); + UIService.setUploadStage(UPLOAD_STAGES.START); await this.uploadMediaFiles(mediaFiles); } - uiService.setUploadStage(UPLOAD_STAGES.FINISH); - uiService.setPercentComplete(FILE_UPLOAD_COMPLETED); + UIService.setUploadStage(UPLOAD_STAGES.FINISH); + UIService.setPercentComplete(FILE_UPLOAD_COMPLETED); } catch (e) { logError(e, 'uploading failed with error'); throw e; @@ -100,22 +100,22 @@ class UploadManager { } private async seedMetadataMap(metadataFiles: globalThis.File[]) { - uiService.reset(metadataFiles.length); + UIService.reset(metadataFiles.length); for (const rawFile of metadataFiles) { const parsedMetaDataJSON = await parseMetadataJSON(rawFile); this.metadataMap.set(parsedMetaDataJSON.title, parsedMetaDataJSON); - uiService.increaseFileUploaded(); + UIService.increaseFileUploaded(); } } private async uploadMediaFiles(mediaFiles: FileWithCollection[]) { this.filesToBeUploaded.push(...mediaFiles); - uiService.reset(mediaFiles.length); + UIService.reset(mediaFiles.length); - uploadService.init(mediaFiles.length, this.metadataMap); + UploadService.init(mediaFiles.length, this.metadataMap); - uiService.setUploadStage(UPLOAD_STAGES.UPLOADING); + UIService.setUploadStage(UPLOAD_STAGES.UPLOADING); const uploadProcesses = []; for ( @@ -154,7 +154,7 @@ class UploadManager { this.setFiles(this.existingFiles); } - uiService.moveFileToResultList(fileWithCollection.file.name); + UIService.moveFileToResultList(fileWithCollection.file.name); } } diff --git a/src/services/upload/uploadService.ts b/src/services/upload/uploadService.ts index 1bf190365..f88d38a40 100644 --- a/src/services/upload/uploadService.ts +++ b/src/services/upload/uploadService.ts @@ -13,7 +13,7 @@ import { import { encryptFiledata } from './encryptionService'; import { ENCRYPTION_CHUNK_SIZE } from 'types'; import { uploadStreamUsingMultipart } from './s3Service'; -import uiService from './uiService'; +import UIService from './uiService'; import { parseError } from 'utils/common/errorUtil'; export const RANDOM_PERCENTAGE_PROGRESS_FOR_PUT = () => 90 + 10 * Math.random(); @@ -199,7 +199,7 @@ class UploadService { file.file.encryptedData, ); } else { - const progressTracker = uiService.trackUploadProgress( + const progressTracker = UIService.trackUploadProgress( file.filename, ); const fileUploadURL = await this.getUploadURL(); diff --git a/src/services/upload/uploader.ts b/src/services/upload/uploader.ts index d57f3dd46..14e844ba9 100644 --- a/src/services/upload/uploader.ts +++ b/src/services/upload/uploader.ts @@ -5,7 +5,7 @@ import { decryptFile } from 'utils/file'; import { logError } from 'utils/sentry'; import { fileAlreadyInCollection } from 'utils/upload'; import NetworkClient from './networkClient'; -import uiService from './uiService'; +import UIService from './uiService'; import UploadService, { BackupedFile, EncryptedFile, @@ -28,7 +28,7 @@ export default async function uploader( existingFilesCollectionWise: Map, ): Promise { const { file: rawFile, collection } = fileWithCollection; - uiService.setFileProgress(rawFile.name, 0); + UIService.setFileProgress(rawFile.name, 0); let file: FileInMemory = null; let encryptedFile: EncryptedFile = null; try { @@ -42,7 +42,7 @@ export default async function uploader( ) ) { // set progress to -2 indicating that file upload was skipped - uiService.setFileProgress(rawFile.name, FileUploadResults.SKIPPED); + UIService.setFileProgress(rawFile.name, FileUploadResults.SKIPPED); await sleep(TwoSecondInMillSeconds); return { fileUploadResult: FileUploadResults.SKIPPED }; } @@ -66,8 +66,8 @@ export default async function uploader( const uploadedFile = await NetworkClient.uploadFile(uploadFile); const decryptedFile = await decryptFile(uploadedFile, collection); - uiService.setFileProgress(rawFile.name, FileUploadResults.UPLOADED); - uiService.increaseFileUploaded(); + UIService.setFileProgress(rawFile.name, FileUploadResults.UPLOADED); + UIService.increaseFileUploaded(); return { fileUploadResult: FileUploadResults.UPLOADED, file: decryptedFile, @@ -76,10 +76,10 @@ export default async function uploader( logError(e, 'file upload failed'); handleError(e); if (e.message === CustomError.ETAG_MISSING) { - uiService.setFileProgress(rawFile.name, FileUploadResults.BLOCKED); + UIService.setFileProgress(rawFile.name, FileUploadResults.BLOCKED); return { fileUploadResult: FileUploadResults.BLOCKED }; } else { - uiService.setFileProgress(rawFile.name, FileUploadResults.FAILED); + UIService.setFileProgress(rawFile.name, FileUploadResults.FAILED); return { fileUploadResult: FileUploadResults.FAILED }; } } finally { From 01827e268e0f08e7ef636fc08ea3b05ed9ebd7e6 Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Wed, 11 Aug 2021 11:09:16 +0530 Subject: [PATCH 024/231] prevent progressbar from going back --- src/services/upload/uiService.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/services/upload/uiService.ts b/src/services/upload/uiService.ts index eaa0da539..19595e5c1 100644 --- a/src/services/upload/uiService.ts +++ b/src/services/upload/uiService.ts @@ -1,4 +1,5 @@ import { ProgressUpdater } from 'components/pages/gallery/Upload'; +import { UPLOAD_STAGES } from './uploadManager'; export const RANDOM_PERCENTAGE_PROGRESS_FOR_PUT = () => 90 + 10 * Math.random(); @@ -32,14 +33,12 @@ class UIService { this.updateProgressBarUI(); } - setUploadStage(stage) { + setUploadStage(stage: UPLOAD_STAGES) { this.progressUpdater.setUploadStage(stage); - this.updateProgressBarUI(); } - setPercentComplete(percent) { + setPercentComplete(percent: number) { this.progressUpdater.setPercentComplete(percent); - this.updateProgressBarUI(); } increaseFileUploaded() { @@ -64,7 +63,7 @@ class UIService { finished: this.filesUploaded, total: this.totalFileCount, }); - let percentComplete = this.perFileProgress * this.filesUploaded; + let percentComplete = this.perFileProgress * this.uploadResult.size; if (this.fileProgress) { // eslint-disable-next-line @typescript-eslint/no-unused-vars for (const [_, progress] of this.fileProgress) { From 07ce0f4dcb9a3a1cb4cf9afce0943438ff0e6138 Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Wed, 11 Aug 2021 11:25:18 +0530 Subject: [PATCH 025/231] add failed files to failed upload list --- src/services/upload/uploadManager.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/services/upload/uploadManager.ts b/src/services/upload/uploadManager.ts index 47c82a4c3..dc242768d 100644 --- a/src/services/upload/uploadManager.ts +++ b/src/services/upload/uploadManager.ts @@ -153,6 +153,12 @@ class UploadManager { ); this.setFiles(this.existingFiles); } + if ( + fileUploadResult === FileUploadResults.BLOCKED || + FileUploadResults.FAILED + ) { + this.failedFiles.push(fileWithCollection); + } UIService.moveFileToResultList(fileWithCollection.file.name); } From 3a3de95df9016c42d86543e12685aeb95492f470 Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Wed, 11 Aug 2021 11:40:53 +0530 Subject: [PATCH 026/231] added await to catch preFetchUploadURLs error --- src/services/upload/uploadManager.ts | 2 +- src/services/upload/uploadService.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/upload/uploadManager.ts b/src/services/upload/uploadManager.ts index dc242768d..9af515a00 100644 --- a/src/services/upload/uploadManager.ts +++ b/src/services/upload/uploadManager.ts @@ -113,7 +113,7 @@ class UploadManager { this.filesToBeUploaded.push(...mediaFiles); UIService.reset(mediaFiles.length); - UploadService.init(mediaFiles.length, this.metadataMap); + await UploadService.init(mediaFiles.length, this.metadataMap); UIService.setUploadStage(UPLOAD_STAGES.UPLOADING); diff --git a/src/services/upload/uploadService.ts b/src/services/upload/uploadService.ts index f88d38a40..3c9820ce5 100644 --- a/src/services/upload/uploadService.ts +++ b/src/services/upload/uploadService.ts @@ -111,10 +111,10 @@ class UploadService { private metadataMap: Map; private pendingUploadCount: number = 0; - init(count: number, metadataMap: MetadataMap) { + async init(count: number, metadataMap: MetadataMap) { this.pendingUploadCount = count; this.metadataMap = metadataMap; - this.preFetchUploadURLs(); + await this.preFetchUploadURLs(); } async readFile(reader: FileReader, receivedFile: globalThis.File) { From 82e70a343ddb3f86de0c208df11b60decce7463d Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Wed, 11 Aug 2021 11:41:09 +0530 Subject: [PATCH 027/231] formatted metadataService --- src/services/upload/metadataService.ts | 68 +++++++++++++------------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/src/services/upload/metadataService.ts b/src/services/upload/metadataService.ts index 0eedbcee8..d631507b4 100644 --- a/src/services/upload/metadataService.ts +++ b/src/services/upload/metadataService.ts @@ -3,41 +3,41 @@ import { logError } from 'utils/sentry'; import { getExifData } from './exifService'; import { MetadataObject } from './uploadService'; - export interface Location { latitude: number; longitude: number; } -export interface ParsedMetaDataJSON{ - title:string; - creationTime:number; - modificationTime:number; - location:Location; +export interface ParsedMetaDataJSON { + title: string; + creationTime: number; + modificationTime: number; + location: Location; } -export const NULL_LOCATION: Location = -{ latitude: null, - longitude: null }; +export const NULL_LOCATION: Location = { latitude: null, longitude: null }; -const NULL_PARSED_METADATA_JSON:ParsedMetaDataJSON= -{ title: null, +const NULL_PARSED_METADATA_JSON: ParsedMetaDataJSON = { + title: null, creationTime: null, modificationTime: null, location: NULL_LOCATION, }; -export async function extractMetatdata(reader:FileReader, receivedFile:globalThis.File, fileType:FILE_TYPE) { +export async function extractMetatdata( + reader: FileReader, + receivedFile: globalThis.File, + fileType: FILE_TYPE, +) { const { location, creationTime } = await getExifData( reader, receivedFile, fileType, ); - const extractedMetadata:MetadataObject = { + const extractedMetadata: MetadataObject = { title: receivedFile.name, - creationTime: - creationTime || receivedFile.lastModified * 1000, + creationTime: creationTime || receivedFile.lastModified * 1000, modificationTime: receivedFile.lastModified * 1000, latitude: location?.latitude, longitude: location?.latitude, @@ -46,31 +46,29 @@ export async function extractMetatdata(reader:FileReader, receivedFile:globalThi return extractedMetadata; } - export async function parseMetadataJSON(receivedFile: globalThis.File) { try { - const metadataJSON: object = await new Promise( - (resolve, reject) => { - const reader = new FileReader(); - reader.onabort = () => reject(Error('file reading was aborted')); - reader.onerror = () => reject(Error('file reading has failed')); - reader.onload = () => { - const result = - typeof reader.result !== 'string' ? - new TextDecoder().decode(reader.result) : - reader.result; - resolve(JSON.parse(result)); - }; - reader.readAsText(receivedFile); - }, - ); + const metadataJSON: object = await new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onabort = () => reject(Error('file reading was aborted')); + reader.onerror = () => reject(Error('file reading has failed')); + reader.onload = () => { + const result = + typeof reader.result !== 'string' + ? new TextDecoder().decode(reader.result) + : reader.result; + resolve(JSON.parse(result)); + }; + reader.readAsText(receivedFile); + }); - const parsedMetaDataJSON:ParsedMetaDataJSON = NULL_PARSED_METADATA_JSON; + const parsedMetaDataJSON: ParsedMetaDataJSON = + NULL_PARSED_METADATA_JSON; if (!metadataJSON || !metadataJSON['title']) { return; } - parsedMetaDataJSON.title=metadataJSON['title']; + parsedMetaDataJSON.title = metadataJSON['title']; if ( metadataJSON['photoTakenTime'] && metadataJSON['photoTakenTime']['timestamp'] @@ -85,7 +83,7 @@ export async function parseMetadataJSON(receivedFile: globalThis.File) { parsedMetaDataJSON.modificationTime = metadataJSON['modificationTime']['timestamp'] * 1000000; } - let locationData:Location = NULL_LOCATION; + let locationData: Location = NULL_LOCATION; if ( metadataJSON['geoData'] && (metadataJSON['geoData']['latitude'] !== 0.0 || @@ -100,7 +98,7 @@ export async function parseMetadataJSON(receivedFile: globalThis.File) { locationData = metadataJSON['geoDataExif']; } if (locationData !== null) { - parsedMetaDataJSON.location=locationData; + parsedMetaDataJSON.location = locationData; } return parsedMetaDataJSON; } catch (e) { From 9aef8246030f441abb95dd36a0c8e3944f8e3dfb Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Wed, 11 Aug 2021 12:41:57 +0530 Subject: [PATCH 028/231] send only collectionId in FileWithCollections --- src/components/pages/gallery/Upload.tsx | 17 ++++++--- src/services/upload/uploadManager.ts | 50 ++++++++++++++++++------- src/services/upload/uploader.ts | 8 ++-- 3 files changed, 53 insertions(+), 22 deletions(-) diff --git a/src/components/pages/gallery/Upload.tsx b/src/components/pages/gallery/Upload.tsx index a184b91d0..72d39e1e0 100644 --- a/src/components/pages/gallery/Upload.tsx +++ b/src/components/pages/gallery/Upload.tsx @@ -1,6 +1,6 @@ import React, { useContext, useEffect, useState } from 'react'; -import { createAlbum } from 'services/collectionService'; +import { Collection, createAlbum } from 'services/collectionService'; import constants from 'utils/strings/constants'; import { SetDialogMessage } from 'components/MessageDialog'; import UploadProgress from './UploadProgress'; @@ -183,7 +183,7 @@ export default function Upload(props: Props) { const filesWithCollectionToUpload: FileWithCollection[] = props.acceptedFiles.map((file) => ({ file, - collection, + collectionID: collection.id, })); await uploadFiles(filesWithCollectionToUpload); } catch (e) { @@ -198,7 +198,8 @@ export default function Upload(props: Props) { try { uploadInit(); - const filesWithCollectionToUpload = []; + const filesWithCollectionToUpload: FileWithCollection[] = []; + const collections: Collection[] = []; let collectionWiseFiles = new Map(); if (strategy === UPLOAD_STRATEGY.SINGLE_COLLECTION) { collectionWiseFiles.set(collectionName, props.acceptedFiles); @@ -208,8 +209,12 @@ export default function Upload(props: Props) { try { for (const [collectionName, files] of collectionWiseFiles) { const collection = await createAlbum(collectionName); + collections.push(collection); for (const file of files) { - filesWithCollectionToUpload.push({ collection, file }); + filesWithCollectionToUpload.push({ + collectionID: collection.id, + file, + }); } } } catch (e) { @@ -223,7 +228,7 @@ export default function Upload(props: Props) { }); throw e; } - await uploadFiles(filesWithCollectionToUpload); + await uploadFiles(filesWithCollectionToUpload, collections); } catch (e) { logError(e, 'Failed to upload files to new collections'); } @@ -231,6 +236,7 @@ export default function Upload(props: Props) { const uploadFiles = async ( filesWithCollectionToUpload: FileWithCollection[], + collections?: Collection[], ) => { try { props.setUploadInProgress(true); @@ -238,6 +244,7 @@ export default function Upload(props: Props) { await props.syncWithRemote(true, true); await uploadManager.queueFilesForUpload( filesWithCollectionToUpload, + collections, ); } catch (err) { props.setBannerMessage(err.message); diff --git a/src/services/upload/uploadManager.ts b/src/services/upload/uploadManager.ts index 9af515a00..24a699b5d 100644 --- a/src/services/upload/uploadManager.ts +++ b/src/services/upload/uploadManager.ts @@ -1,5 +1,5 @@ import { File, getLocalFiles } from '../fileService'; -import { Collection } from '../collectionService'; +import { Collection, getLocalCollections } from '../collectionService'; import { SetFiles } from 'pages/gallery'; import { ComlinkWorker, getDedicatedCryptoWorker } from 'utils/crypto'; import { @@ -9,7 +9,11 @@ import { } from 'utils/file'; import { logError } from 'utils/sentry'; import localForage from 'utils/storage/localForage'; -import { ParsedMetaDataJSON, parseMetadataJSON } from './metadataService'; +import { + getMetadataKey, + ParsedMetaDataJSON, + parseMetadataJSON, +} from './metadataService'; import { segregateFiles } from 'utils/upload'; import { ProgressUpdater } from 'components/pages/gallery/Upload'; import uploader from './uploader'; @@ -34,7 +38,7 @@ export interface UploadURL { export interface FileWithCollection { file: globalThis.File; - collection: Collection; + collectionID: number; } export enum UPLOAD_STAGES { @@ -44,21 +48,23 @@ export enum UPLOAD_STAGES { FINISH, } +export type MetadataMap = Map; + class UploadManager { private cryptoWorkers = new Array(MAX_CONCURRENT_UPLOADS); - private metadataMap: Map; + private metadataMap: MetadataMap; private filesToBeUploaded: FileWithCollection[]; private failedFiles: FileWithCollection[]; private existingFilesCollectionWise: Map; private existingFiles: File[]; private setFiles: SetFiles; - + private collections: Map; public initUploader(progressUpdater: ProgressUpdater, setFiles: SetFiles) { UIService.init(progressUpdater); this.setFiles = setFiles; } - private async init() { + private async init(newCollections?: Collection[]) { this.filesToBeUploaded = []; this.failedFiles = []; this.metadataMap = new Map(); @@ -66,16 +72,23 @@ class UploadManager { this.existingFilesCollectionWise = sortFilesIntoCollections( this.existingFiles, ); + const collections = await getLocalCollections(); + if (newCollections) { + collections.push(...newCollections); + } + this.collections = new Map( + collections.map((collection) => [collection.id, collection]), + ); } public async queueFilesForUpload( - filesWithCollectionToUpload: FileWithCollection[], + fileWithCollectionTobeUploaded: FileWithCollection[], + collections?: Collection[], ) { try { - await this.init(); - + await this.init(collections); const { metadataFiles, mediaFiles } = segregateFiles( - filesWithCollectionToUpload, + fileWithCollectionTobeUploaded, ); if (metadataFiles.length) { UIService.setUploadStage( @@ -99,12 +112,20 @@ class UploadManager { } } - private async seedMetadataMap(metadataFiles: globalThis.File[]) { + private async seedMetadataMap(metadataFiles: FileWithCollection[]) { UIService.reset(metadataFiles.length); - for (const rawFile of metadataFiles) { - const parsedMetaDataJSON = await parseMetadataJSON(rawFile); - this.metadataMap.set(parsedMetaDataJSON.title, parsedMetaDataJSON); + for (const fileWithCollection of metadataFiles) { + const parsedMetaDataJSON = await parseMetadataJSON( + fileWithCollection.file, + ); + this.metadataMap.set( + getMetadataKey( + fileWithCollection.collectionID, + parsedMetaDataJSON.title, + ), + parsedMetaDataJSON, + ); UIService.increaseFileUploaded(); } } @@ -142,6 +163,7 @@ class UploadManager { fileReader, fileWithCollection, this.existingFilesCollectionWise, + this.collections, ); if (fileUploadResult === FileUploadResults.UPLOADED) { diff --git a/src/services/upload/uploader.ts b/src/services/upload/uploader.ts index 14e844ba9..afeca5f85 100644 --- a/src/services/upload/uploader.ts +++ b/src/services/upload/uploader.ts @@ -1,3 +1,4 @@ +import { Collection } from 'services/collectionService'; import { File } from 'services/fileService'; import { sleep } from 'utils/common'; import { handleError, CustomError } from 'utils/common/errorUtil'; @@ -6,12 +7,11 @@ import { logError } from 'utils/sentry'; import { fileAlreadyInCollection } from 'utils/upload'; import NetworkClient from './networkClient'; import UIService from './uiService'; +import { FileUploadResults, FileWithCollection } from './uploadManager'; import UploadService, { BackupedFile, EncryptedFile, FileInMemory, - FileUploadResults, - FileWithCollection, UploadFile, } from './uploadService'; @@ -26,8 +26,10 @@ export default async function uploader( reader: FileReader, fileWithCollection: FileWithCollection, existingFilesCollectionWise: Map, + collections: Map, ): Promise { - const { file: rawFile, collection } = fileWithCollection; + const { file: rawFile, collectionID } = fileWithCollection; + const collection = collections.get(collectionID); UIService.setFileProgress(rawFile.name, 0); let file: FileInMemory = null; let encryptedFile: EncryptedFile = null; From ba065405eb48780b3b287ed63c06797e2676695c Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Wed, 11 Aug 2021 12:50:26 +0530 Subject: [PATCH 029/231] has collection identifier with metadat --- .../pages/gallery/UploadProgress.tsx | 271 ++++++++++-------- src/services/upload/metadataService.ts | 3 + src/services/upload/readFileService.ts | 38 ++- src/services/upload/s3Service.ts | 16 +- src/services/upload/uploadService.ts | 44 +-- src/services/upload/uploader.ts | 2 +- src/utils/upload/index.ts | 4 +- 7 files changed, 199 insertions(+), 179 deletions(-) diff --git a/src/components/pages/gallery/UploadProgress.tsx b/src/components/pages/gallery/UploadProgress.tsx index 01ac49b1b..26ff39711 100644 --- a/src/components/pages/gallery/UploadProgress.tsx +++ b/src/components/pages/gallery/UploadProgress.tsx @@ -1,11 +1,12 @@ import ExpandLess from 'components/icons/ExpandLess'; import ExpandMore from 'components/icons/ExpandMore'; import React, { useState } from 'react'; -import { - Button, Modal, ProgressBar, -} from 'react-bootstrap'; +import { Button, Modal, ProgressBar } from 'react-bootstrap'; import { FileRejection } from 'react-dropzone'; -import { FileUploadResults, UPLOAD_STAGES } from 'services/upload/uploadService'; +import { + FileUploadResults, + UPLOAD_STAGES, +} from 'services/upload/uploadManager'; import styled from 'styled-components'; import { DESKTOP_APP_DOWNLOAD_URL } from 'utils/common'; import constants from 'utils/strings/constants'; @@ -19,77 +20,85 @@ interface Props { retryFailed; fileProgress: Map; show; - fileRejections:FileRejection[] - uploadResult:Map; + fileRejections: FileRejection[]; + uploadResult: Map; } -interface FileProgresses{ - fileName:string; - progress:number; +interface FileProgresses { + fileName: string; + progress: number; } -const Content =styled.div<{collapsed:boolean, sm?:boolean, height?:number}>` +const Content = styled.div<{ + collapsed: boolean; + sm?: boolean; + height?: number; +}>` overflow: hidden; - height:${(props)=> props.collapsed?'0px':props.height+'px'}; - transition:${(props)=> 'height '+0.001*props.height+'s ease-out'}; + height: ${(props) => (props.collapsed ? '0px' : props.height + 'px')}; + transition: ${(props) => 'height ' + 0.001 * props.height + 's ease-out'}; margin-bottom: 20px; - & >p { - padding-left:35px; - margin:0; + & > p { + padding-left: 35px; + margin: 0; } `; -const FileList =styled.ul` - padding-left:50px; - margin-top:5px; +const FileList = styled.ul` + padding-left: 50px; + margin-top: 5px; & > li { - padding-left:10px; - margin-bottom:10px; - color:#ccc; + padding-left: 10px; + margin-bottom: 10px; + color: #ccc; } `; -const SectionTitle =styled.div` - display:flex; - justify-content:space-between; - padding:0 20px; - color:#eee; - font-size:20px; - cursor:pointer; +const SectionTitle = styled.div` + display: flex; + justify-content: space-between; + padding: 0 20px; + color: #eee; + font-size: 20px; + cursor: pointer; `; - -interface ResultSectionProps{ +interface ResultSectionProps { fileUploadResultMap: Map; - fileUploadResult:FileUploadResults; + fileUploadResult: FileUploadResults; sectionTitle; sectionInfo; - infoHeight:number; + infoHeight: number; } -const ResultSection =(props:ResultSectionProps)=>{ - const [listView, setListView]=useState(false); - const fileList=props.fileUploadResultMap?.get(props.fileUploadResult); +const ResultSection = (props: ResultSectionProps) => { + const [listView, setListView] = useState(false); + const fileList = props.fileUploadResultMap?.get(props.fileUploadResult); if (!fileList?.length) { return <>; } - return (<> - setListView(!listView)} > {props.sectionTitle} {listView?:} - -

{props.sectionInfo}

- - {fileList.map((fileName) => ( - -
  • - {fileName} -
    • - ))} -     -     - ); + return ( + <> + setListView(!listView)}> + {' '} + {props.sectionTitle}{' '} + {listView ? : } + + +

    {props.sectionInfo}

    + + {fileList.map((fileName) => ( +
  • {fileName}
    • + ))} +     +     + + ); }; export default function UploadProgress(props: Props) { const fileProgressStatuses = [] as FileProgresses[]; const fileUploadResultMap = new Map(); - let filesNotUploaded=false; + let filesNotUploaded = false; if (props.fileProgress) { for (const [fileName, progress] of props.fileProgress) { @@ -101,10 +110,10 @@ export default function UploadProgress(props: Props) { if (!fileUploadResultMap.has(progress)) { fileUploadResultMap.set(progress, []); } - if (progress<0) { - filesNotUploaded=true; + if (progress < 0) { + filesNotUploaded = true; } - const fileList= fileUploadResultMap.get(progress); + const fileList = fileUploadResultMap.get(progress); fileUploadResultMap.set(progress, [...fileList, fileName]); } } @@ -113,90 +122,120 @@ export default function UploadProgress(props: Props) { null : - props.closeModal + props.uploadStage !== UPLOAD_STAGES.FINISH + ? () => null + : props.closeModal } aria-labelledby="contained-modal-title-vcenter" centered - backdrop={ - fileProgressStatuses?.length !== 0 ? 'static' : 'true' - } - > + backdrop={fileProgressStatuses?.length !== 0 ? 'static' : 'true'}> - + paddingBottom: '0px', + }} + closeButton={props.uploadStage === UPLOAD_STAGES.FINISH}>

    - {props.uploadStage === UPLOAD_STAGES.UPLOADING ? - constants.UPLOAD[props.uploadStage]( - props.fileCounter, - ) : - constants.UPLOAD[props.uploadStage]} + {props.uploadStage === UPLOAD_STAGES.UPLOADING + ? constants.UPLOAD[props.uploadStage](props.fileCounter) + : constants.UPLOAD[props.uploadStage]}

    - {(props.uploadStage === UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES || - props.uploadStage === UPLOAD_STAGES.UPLOADING) && - ( - < ProgressBar - now={props.now} - animated - variant="upload-progress-bar" - /> - ) - } - {fileProgressStatuses.length>0 && - - {fileProgressStatuses.map(({ fileName, progress }) => ( -
  • - {props.uploadStage===UPLOAD_STAGES.FINISH ? - fileName : - constants.FILE_UPLOAD_PROGRESS( - fileName, - progress, - ) - } -
    • - ))} -     } + {(props.uploadStage === + UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES || + props.uploadStage === UPLOAD_STAGES.UPLOADING) && ( + + )} + {fileProgressStatuses.length > 0 && ( + + {fileProgressStatuses.map(({ fileName, progress }) => ( +
  • + {props.uploadStage === UPLOAD_STAGES.FINISH + ? fileName + : constants.FILE_UPLOAD_PROGRESS( + fileName, + progress, + )} +
    • + ))} +     + )} - + - {props.uploadStage === UPLOAD_STAGES.FINISH && filesNotUploaded && ( + {props.uploadStage === UPLOAD_STAGES.FINISH && + filesNotUploaded && ( {constants.FILE_NOT_UPLOADED_LIST} )} - - - - + + + + {props.uploadStage === UPLOAD_STAGES.FINISH && ( - {props.uploadStage===UPLOAD_STAGES.FINISH && ((fileUploadResultMap?.get(FileUploadResults.FAILED)?.length>0 || fileUploadResultMap?.get(FileUploadResults.BLOCKED)?.length>0)? ( - - ) : ( - ))} + {props.uploadStage === UPLOAD_STAGES.FINISH && + (fileUploadResultMap?.get(FileUploadResults.FAILED) + ?.length > 0 || + fileUploadResultMap?.get(FileUploadResults.BLOCKED) + ?.length > 0 ? ( + + ) : ( + + ))} )}     diff --git a/src/services/upload/metadataService.ts b/src/services/upload/metadataService.ts index d631507b4..8e277da16 100644 --- a/src/services/upload/metadataService.ts +++ b/src/services/upload/metadataService.ts @@ -46,6 +46,9 @@ export async function extractMetatdata( return extractedMetadata; } +export const getMetadataKey = (fileWithCollection: number, title: string) => + `${fileWithCollection}_${title}`; + export async function parseMetadataJSON(receivedFile: globalThis.File) { try { const metadataJSON: object = await new Promise((resolve, reject) => { diff --git a/src/services/upload/readFileService.ts b/src/services/upload/readFileService.ts index 8eb0a2ae7..6393baf2a 100644 --- a/src/services/upload/readFileService.ts +++ b/src/services/upload/readFileService.ts @@ -8,14 +8,13 @@ const TYPE_HEIC = 'HEIC'; export const TYPE_IMAGE = 'image'; const EDITED_FILE_SUFFIX = '-edited'; - -export async function getFileData(reader:FileReader, file:globalThis.File) { - return file.size > MIN_STREAM_FILE_SIZE ? - getFileStream(reader, file) : - await getUint8ArrayView(reader, file); +export async function getFileData(reader: FileReader, file: globalThis.File) { + return file.size > MIN_STREAM_FILE_SIZE + ? getFileStream(reader, file) + : await getUint8ArrayView(reader, file); } -export function getFileType(receivedFile:globalThis.File) { +export function getFileType(receivedFile: globalThis.File) { let fileType: FILE_TYPE; switch (receivedFile.type.split('/')[0]) { case TYPE_IMAGE: @@ -37,18 +36,14 @@ export function getFileType(receivedFile:globalThis.File) { return fileType; } +export function getFileOriginalName(file: globalThis.File) { + let originalName: string = null; -export function getFileOriginalName(file:globalThis.File) { - let originalName:string=null; - - const isEditedFile=file.name.endsWith(EDITED_FILE_SUFFIX); + const isEditedFile = file.name.endsWith(EDITED_FILE_SUFFIX); if (isEditedFile) { - originalName = file.name.slice( - 0, - -1 * EDITED_FILE_SUFFIX.length, - ); + originalName = file.name.slice(0, -1 * EDITED_FILE_SUFFIX.length); } else { - originalName=file.name; + originalName = file.name; } return originalName; } @@ -70,7 +65,10 @@ function getFileStream(reader: FileReader, file: globalThis.File) { }; } -async function* fileChunkReaderMaker(reader:FileReader, file:globalThis.File) { +async function* fileChunkReaderMaker( + reader: FileReader, + file: globalThis.File, +) { let offset = 0; while (offset < file.size) { const blob = file.slice(offset, ENCRYPTION_CHUNK_SIZE + offset); @@ -92,9 +90,9 @@ export async function getUint8ArrayView( reader.onload = () => { // Do whatever you want with the file contents const result = - typeof reader.result === 'string' ? - new TextEncoder().encode(reader.result) : - new Uint8Array(reader.result); + typeof reader.result === 'string' + ? new TextEncoder().encode(reader.result) + : new Uint8Array(reader.result); resolve(result); }; reader.readAsArrayBuffer(file); @@ -104,5 +102,3 @@ export async function getUint8ArrayView( throw e; } } - - diff --git a/src/services/upload/s3Service.ts b/src/services/upload/s3Service.ts index ae9aa0650..8028b0683 100644 --- a/src/services/upload/s3Service.ts +++ b/src/services/upload/s3Service.ts @@ -1,17 +1,19 @@ -import { - CHUNKS_COMBINED_FOR_A_UPLOAD_PART, - DataStream, - MultipartUploadURLs, - RANDOM_PERCENTAGE_PROGRESS_FOR_PUT, -} from './uploadService'; +import { CHUNKS_COMBINED_FOR_A_UPLOAD_PART, DataStream } from './uploadService'; import NetworkClient from './networkClient'; import * as convert from 'xml-js'; -import UIService from './uiService'; +import UIService, { RANDOM_PERCENTAGE_PROGRESS_FOR_PUT } from './uiService'; interface PartEtag { PartNumber: number; Etag: string; } + +export interface MultipartUploadURLs { + objectKey: string; + partURLs: string[]; + completeURL: string; +} + export function calculatePartCount(chunkCount: number) { const partCount = Math.ceil(chunkCount / CHUNKS_COMBINED_FOR_A_UPLOAD_PART); return partCount; diff --git a/src/services/upload/uploadService.ts b/src/services/upload/uploadService.ts index 3c9820ce5..ebefb9580 100644 --- a/src/services/upload/uploadService.ts +++ b/src/services/upload/uploadService.ts @@ -3,7 +3,11 @@ import { Collection } from '../collectionService'; import { FILE_TYPE } from 'pages/gallery'; import { logError } from 'utils/sentry'; import NetworkClient from './networkClient'; -import { extractMetatdata, ParsedMetaDataJSON } from './metadataService'; +import { + extractMetatdata, + getMetadataKey, + ParsedMetaDataJSON, +} from './metadataService'; import { generateThumbnail } from './thumbnailService'; import { getFileType, @@ -15,30 +19,18 @@ import { ENCRYPTION_CHUNK_SIZE } from 'types'; import { uploadStreamUsingMultipart } from './s3Service'; import UIService from './uiService'; import { parseError } from 'utils/common/errorUtil'; +import { FileWithCollection, MetadataMap } from './uploadManager'; -export const RANDOM_PERCENTAGE_PROGRESS_FOR_PUT = () => 90 + 10 * Math.random(); export const MIN_STREAM_FILE_SIZE = 20 * 1024 * 1024; export const CHUNKS_COMBINED_FOR_A_UPLOAD_PART = Math.floor( MIN_STREAM_FILE_SIZE / ENCRYPTION_CHUNK_SIZE, ); -export enum FileUploadResults { - FAILED = -1, - SKIPPED = -2, - UNSUPPORTED = -3, - BLOCKED = -4, - UPLOADED = 100, -} - export interface UploadURL { url: string; objectKey: string; } -export interface FileWithCollection { - file: globalThis.File; - collection: Collection; -} export interface DataStream { stream: ReadableStream; chunkCount: number; @@ -57,12 +49,6 @@ export interface B64EncryptionResult { nonce: string; } -export interface MultipartUploadURLs { - objectKey: string; - partURLs: string[]; - completeURL: string; -} - export interface MetadataObject { title: string; creationTime: number; @@ -91,21 +77,12 @@ export interface ProcessedFile { } export interface BackupedFile extends Omit {} -export type MetadataMap = Map; - export interface UploadFile extends BackupedFile { collectionID: number; encryptedKey: string; keyDecryptionNonce: string; } -export enum UPLOAD_STAGES { - START, - READING_GOOGLE_METADATA_FILES, - UPLOADING, - FINISH, -} - class UploadService { private uploadURLs: UploadURL[] = []; private metadataMap: Map; @@ -117,9 +94,10 @@ class UploadService { await this.preFetchUploadURLs(); } - async readFile(reader: FileReader, receivedFile: globalThis.File) { + async readFile(reader: FileReader, fileWithCollection: FileWithCollection) { try { - const fileType = getFileType(receivedFile); + const { file: receivedFile, collectionID } = fileWithCollection; + const fileType = getFileType(fileWithCollection.file); const { thumbnail, hasStaticThumbnail } = await generateThumbnail( reader, @@ -128,7 +106,9 @@ class UploadService { ); const originalName = getFileOriginalName(receivedFile); - const googleMetadata = this.metadataMap.get(originalName); + const googleMetadata = this.metadataMap.get( + getMetadataKey(collectionID, originalName), + ); const extractedMetadata: MetadataObject = await extractMetatdata( reader, receivedFile, diff --git a/src/services/upload/uploader.ts b/src/services/upload/uploader.ts index afeca5f85..da750d1de 100644 --- a/src/services/upload/uploader.ts +++ b/src/services/upload/uploader.ts @@ -34,7 +34,7 @@ export default async function uploader( let file: FileInMemory = null; let encryptedFile: EncryptedFile = null; try { - file = await UploadService.readFile(reader, rawFile); + file = await UploadService.readFile(reader, fileWithCollection); if ( fileAlreadyInCollection( diff --git a/src/utils/upload/index.ts b/src/utils/upload/index.ts index 11881a108..226adb437 100644 --- a/src/utils/upload/index.ts +++ b/src/utils/upload/index.ts @@ -37,7 +37,7 @@ export function areFilesSame( export function segregateFiles( filesWithCollectionToUpload: FileWithCollection[], ) { - const metadataFiles: globalThis.File[] = []; + const metadataFiles: FileWithCollection[] = []; const mediaFiles: FileWithCollection[] = []; filesWithCollectionToUpload.forEach((fileWithCollection) => { const file = fileWithCollection.file; @@ -46,7 +46,7 @@ export function segregateFiles( return; } if (file.name.slice(-4) === TYPE_JSON) { - metadataFiles.push(fileWithCollection.file); + metadataFiles.push(fileWithCollection); } else { mediaFiles.push(fileWithCollection); } From a6652c105a6265bb4a743a5e42d23f4259ba756b Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Wed, 11 Aug 2021 12:54:56 +0530 Subject: [PATCH 030/231] only use the lastFolder as suggested name --- src/components/pages/gallery/Upload.tsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/components/pages/gallery/Upload.tsx b/src/components/pages/gallery/Upload.tsx index 72d39e1e0..04a73779e 100644 --- a/src/components/pages/gallery/Upload.tsx +++ b/src/components/pages/gallery/Upload.tsx @@ -155,6 +155,11 @@ export default function Upload(props: Props) { 1, commonPathPrefix.lastIndexOf('/') - 1, ); + if (commonPathPrefix) { + commonPathPrefix = commonPathPrefix.substr( + commonPathPrefix.lastIndexOf('/') + 1, + ); + } } return { suggestedCollectionName: commonPathPrefix, From d65f6cb494151ec7ea41cb7a3477bc997315f796 Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Wed, 11 Aug 2021 13:30:59 +0530 Subject: [PATCH 031/231] fix build --- src/components/ChangeEmail.tsx | 82 ++++--- src/components/ExportFinished.tsx | 90 +++++--- src/components/ExportInProgress.tsx | 77 +++++-- src/components/ExportModal.tsx | 141 ++++++++---- src/components/PhotoFrame.tsx | 210 +++++++++++------- src/components/Sidebar.tsx | 119 +++++----- src/components/TwoFactorModal.tsx | 109 ++++++--- src/components/pages/gallery/PlanSelector.tsx | 176 ++++++++------- src/pages/_app.tsx | 83 ++++--- src/pages/change-email/index.tsx | 76 +++---- src/pages/change-password/index.tsx | 9 +- src/pages/index.tsx | 131 ++++++----- src/pages/login/index.tsx | 25 ++- src/pages/signup/index.tsx | 9 +- src/pages/two-factor/setup/index.tsx | 126 ++++++++--- src/services/exportService.ts | 145 +++++++----- src/services/upload/encryptionService.ts | 11 +- src/services/upload/networkClient.ts | 5 +- src/utils/crypto/libsodium.ts | 39 ++-- 19 files changed, 1034 insertions(+), 629 deletions(-) diff --git a/src/components/ChangeEmail.tsx b/src/components/ChangeEmail.tsx index c2546a415..26e187a4a 100644 --- a/src/components/ChangeEmail.tsx +++ b/src/components/ChangeEmail.tsx @@ -1,4 +1,3 @@ - import { Formik, FormikHelpers } from 'formik'; import React, { useContext, useEffect, useRef, useState } from 'react'; import { Button, Col, Form, FormControl } from 'react-bootstrap'; @@ -13,10 +12,10 @@ import { getData, LS_KEYS, setData } from 'utils/storage/localStorage'; interface formValues { email: string; - ott?:string; + ott?: string; } -const EmailRow =styled.div` +const EmailRow = styled.div` display: flex; flex-wrap: wrap; border: 1px solid grey; @@ -26,15 +25,15 @@ const EmailRow =styled.div` color: #fff; `; -interface Props{ -showMessage:(value:boolean)=>void; -setEmail:(email:string)=>void; +interface Props { + showMessage: (value: boolean) => void; + setEmail: (email: string) => void; } -function ChangeEmailForm(props:Props) { - const [loading, setLoading]=useState(false); - const [ottInputVisible, setShowOttInputVisibility]=useState(false); +function ChangeEmailForm(props: Props) { + const [loading, setLoading] = useState(false); + const [ottInputVisible, setShowOttInputVisibility] = useState(false); const emailInputElement = useRef(null); - const ottInputRef=useRef(null); + const ottInputRef = useRef(null); const appContext = useContext(AppContext); useEffect(() => { @@ -43,14 +42,16 @@ function ChangeEmailForm(props:Props) { }, 250); }, []); - useEffect(()=>{ + useEffect(() => { if (!ottInputVisible) { props.showMessage(false); } }, [ottInputVisible]); - const requestOTT= async( { email }: formValues, - { setFieldError }: FormikHelpers)=>{ + const requestOTT = async ( + { email }: formValues, + { setFieldError }: FormikHelpers, + ) => { try { setLoading(true); await getOTTForEmailChange(email); @@ -66,14 +67,18 @@ function ChangeEmailForm(props:Props) { setLoading(false); }; - - const requestEmailChange= async( { email, ott }: formValues, - { setFieldError }: FormikHelpers)=>{ + const requestEmailChange = async ( + { email, ott }: formValues, + { setFieldError }: FormikHelpers, + ) => { try { setLoading(true); await changeEmail(email, ott); setData(LS_KEYS.USER, { ...getData(LS_KEYS.USER), email }); - appContext.setDisappearingFlashMessage({ message: constants.EMAIL_UDPATE_SUCCESSFUL, severity: 'success' }); + appContext.setDisappearingFlashMessage({ + message: constants.EMAIL_UDPATE_SUCCESSFUL, + severity: 'success', + }); router.push('/gallery'); } catch (e) { setFieldError('ott', `${constants.INCORRECT_CODE}`); @@ -91,17 +96,10 @@ function ChangeEmailForm(props:Props) { })} validateOnChange={false} validateOnBlur={false} - onSubmit={!ottInputVisible?requestOTT:requestEmailChange} - > - {({ - values, - errors, - touched, - handleChange, - handleSubmit, - }) => ( + onSubmit={!ottInputVisible ? requestOTT : requestEmailChange}> + {({ values, errors, touched, handleChange, handleSubmit }) => (
    - {!ottInputVisible ? + {!ottInputVisible ? ( {errors.email} - : + + ) : ( <> - - {values.email} - - - @@ -146,14 +147,23 @@ function ChangeEmailForm(props:Props) { {errors.ott} - } + + )}
    - diff --git a/src/components/ExportFinished.tsx b/src/components/ExportFinished.tsx index f2f0faf97..dedae4d34 100644 --- a/src/components/ExportFinished.tsx +++ b/src/components/ExportFinished.tsx @@ -6,14 +6,13 @@ import constants from 'utils/strings/constants'; import { Label, Row, Value } from './Container'; import { ComfySpan } from './ExportInProgress'; - interface Props { - show: boolean - onHide: () => void - exportFolder: string - exportSize: string - lastExportTime: number - exportStats: ExportStats + show: boolean; + onHide: () => void; + exportFolder: string; + exportSize: string; + lastExportTime: number; + exportStats: ExportStats; updateExportFolder: (newFolder: string) => void; exportFiles: () => void; retryFailed: () => void; @@ -23,30 +22,69 @@ export default function ExportFinished(props: Props) { const totalFiles = props.exportStats.failed + props.exportStats.success; return ( <> -
    +
    - {formatDateTime(props.lastExportTime)} - - - - {props.exportStats.success} / {totalFiles} - - {props.exportStats.failed>0 && - - - - {props.exportStats.failed} / {totalFiles} + + {formatDateTime(props.lastExportTime)} - } + + + + + + {props.exportStats.success} / {totalFiles} + + + + {props.exportStats.failed > 0 && ( + + + + + {props.exportStats.failed} / {totalFiles} + + + + )}
    -
    - +
    +
    - {props.exportStats.failed !== 0 ? - : - - } + {props.exportStats.failed !== 0 ? ( + + ) : ( + + )}
    ); diff --git a/src/components/ExportInProgress.tsx b/src/components/ExportInProgress.tsx index 1083d907b..2cebfbd42 100644 --- a/src/components/ExportInProgress.tsx +++ b/src/components/ExportInProgress.tsx @@ -5,42 +5,83 @@ import styled from 'styled-components'; import constants from 'utils/strings/constants'; export const ComfySpan = styled.span` - word-spacing:1rem; - color:#ddd; + word-spacing: 1rem; + color: #ddd; `; interface Props { - show: boolean - onHide: () => void - exportFolder: string - exportSize: string - exportStage: ExportStage - exportProgress: ExportProgress + show: boolean; + onHide: () => void; + exportFolder: string; + exportSize: string; + exportStage: ExportStage; + exportProgress: ExportProgress; resumeExport: () => void; - cancelExport: () => void + cancelExport: () => void; pauseExport: () => void; } + export default function ExportInProgress(props: Props) { return ( <> -
    +
    - {props.exportProgress.current} / {props.exportProgress.total} files exported {props.exportStage === ExportStage.PAUSED && `(paused)`} + + {' '} + {props.exportProgress.current} /{' '} + {props.exportProgress.total}{' '} + {' '} + + {' '} + files exported{' '} + {props.exportStage === ExportStage.PAUSED && `(paused)`} +
    -
    - {props.exportStage === ExportStage.PAUSED ? - : - - } +
    + {props.exportStage === ExportStage.PAUSED ? ( + + ) : ( + + )}
    - +
    diff --git a/src/components/ExportModal.tsx b/src/components/ExportModal.tsx index c258d91dd..54928960b 100644 --- a/src/components/ExportModal.tsx +++ b/src/components/ExportModal.tsx @@ -1,7 +1,12 @@ import isElectron from 'is-electron'; import React, { useEffect, useState } from 'react'; import { Button } from 'react-bootstrap'; -import exportService, { ExportProgress, ExportStage, ExportStats, ExportType } from 'services/exportService'; +import exportService, { + ExportProgress, + ExportStage, + ExportStats, + ExportType, +} from 'services/exportService'; import { getLocalFiles } from 'services/fileService'; import styled from 'styled-components'; import { sleep } from 'utils/common'; @@ -18,38 +23,44 @@ import MessageDialog from './MessageDialog'; const FolderIconWrapper = styled.div` width: 15%; - margin-left: 10px; - cursor: pointer; + margin-left: 10px; + cursor: pointer; padding: 3px; border: 1px solid #444; - border-radius:15%; - &:hover{ - background-color:#444; + border-radius: 15%; + &:hover { + background-color: #444; } `; -const ExportFolderPathContainer =styled.span` - white-space: nowrap; +const ExportFolderPathContainer = styled.span` + white-space: nowrap; overflow: hidden; - text-overflow: ellipsis; + text-overflow: ellipsis; width: 200px; - + /* Beginning of string */ direction: rtl; text-align: left; `; interface Props { - show: boolean - onHide: () => void - usage: string + show: boolean; + onHide: () => void; + usage: string; } export default function ExportModal(props: Props) { const [exportStage, setExportStage] = useState(ExportStage.INIT); const [exportFolder, setExportFolder] = useState(''); const [exportSize, setExportSize] = useState(''); - const [exportProgress, setExportProgress] = useState({ current: 0, total: 0 }); - const [exportStats, setExportStats] = useState({ failed: 0, success: 0 }); + const [exportProgress, setExportProgress] = useState({ + current: 0, + total: 0, + }); + const [exportStats, setExportStats] = useState({ + failed: 0, + success: 0, + }); const [lastExportTime, setLastExportTime] = useState(0); // ==================== @@ -64,7 +75,9 @@ export default function ExportModal(props: Props) { exportService.ElectronAPIs.registerStopExportListener(stopExport); exportService.ElectronAPIs.registerPauseExportListener(pauseExport); exportService.ElectronAPIs.registerResumeExportListener(resumeExport); - exportService.ElectronAPIs.registerRetryFailedExportListener(retryFailedExport); + exportService.ElectronAPIs.registerRetryFailedExportListener( + retryFailedExport, + ); }, []); useEffect(() => { @@ -76,7 +89,10 @@ export default function ExportModal(props: Props) { setExportStage(exportInfo?.stage ?? ExportStage.INIT); setLastExportTime(exportInfo?.lastAttemptTimestamp); setExportProgress(exportInfo?.progress ?? { current: 0, total: 0 }); - setExportStats({ success: exportInfo?.exportedFiles?.length ?? 0, failed: exportInfo?.failedFiles?.length ?? 0 }); + setExportStats({ + success: exportInfo?.exportedFiles?.length ?? 0, + failed: exportInfo?.failedFiles?.length ?? 0, + }); if (exportInfo?.stage === ExportStage.INPROGRESS) { resumeExport(); } @@ -96,10 +112,21 @@ export default function ExportModal(props: Props) { const failedFilesCnt = exportRecord.failedFiles.length; const syncedFilesCnt = localFiles.length; if (syncedFilesCnt > exportedFileCnt + failedFilesCnt) { - updateExportProgress({ current: exportedFileCnt + failedFilesCnt, total: syncedFilesCnt }); - const exportFileUIDs = new Set([...exportRecord.exportedFiles, ...exportRecord.failedFiles]); - const unExportedFiles = localFiles.filter((file) => !exportFileUIDs.has(getFileUID(file))); - exportService.addFilesQueuedRecord(exportFolder, unExportedFiles); + updateExportProgress({ + current: exportedFileCnt + failedFilesCnt, + total: syncedFilesCnt, + }); + const exportFileUIDs = new Set([ + ...exportRecord.exportedFiles, + ...exportRecord.failedFiles, + ]); + const unExportedFiles = localFiles.filter( + (file) => !exportFileUIDs.has(getFileUID(file)), + ); + exportService.addFilesQueuedRecord( + exportFolder, + unExportedFiles, + ); updateExportStage(ExportStage.PAUSED); } } @@ -107,7 +134,6 @@ export default function ExportModal(props: Props) { main(); }, [props.show]); - useEffect(() => { setExportSize(props.usage); }, [props.usage]); @@ -162,7 +188,10 @@ export default function ExportModal(props: Props) { const startExport = async () => { await preExportRun(); updateExportProgress({ current: 0, total: 0 }); - const { paused } = await exportService.exportFiles(updateExportProgress, ExportType.NEW); + const { paused } = await exportService.exportFiles( + updateExportProgress, + ExportType.NEW, + ); await postExportRun(paused); }; @@ -184,13 +213,15 @@ export default function ExportModal(props: Props) { const pausedStageProgress = exportRecord.progress; setExportProgress(pausedStageProgress); - const updateExportStatsWithOffset = ((progress: ExportProgress) => updateExportProgress( - { + const updateExportStatsWithOffset = (progress: ExportProgress) => + updateExportProgress({ current: pausedStageProgress.current + progress.current, total: pausedStageProgress.current + progress.total, - }, - )); - const { paused } = await exportService.exportFiles(updateExportStatsWithOffset, ExportType.PENDING); + }); + const { paused } = await exportService.exportFiles( + updateExportStatsWithOffset, + ExportType.PENDING, + ); await postExportRun(paused); }; @@ -199,7 +230,10 @@ export default function ExportModal(props: Props) { await preExportRun(); updateExportProgress({ current: 0, total: exportStats.failed }); - const { paused } = await exportService.exportFiles(updateExportProgress, ExportType.RETRY_FAILED); + const { paused } = await exportService.exportFiles( + updateExportProgress, + ExportType.RETRY_FAILED, + ); await postExportRun(paused); }; @@ -224,7 +258,8 @@ export default function ExportModal(props: Props) { switch (exportStage) { case ExportStage.INIT: return ( - ); - default: return (<>); + default: + return <>; } }; @@ -269,34 +306,50 @@ export default function ExportModal(props: Props) { onHide={props.onHide} attributes={{ title: constants.EXPORT_DATA, - }} - > -
    + }}> +
    - {!exportFolder ? - () : - (<> + {!exportFolder ? ( + + ) : ( + <> {/* */} {exportFolder} {/* */} - {(exportStage === ExportStage.FINISHED || exportStage === ExportStage.INIT) && ( - + {(exportStage === ExportStage.FINISHED || + exportStage === ExportStage.INIT) && ( + )} - ) - } + + )} - {exportSize ? `${exportSize}` : } + + + {exportSize ? `${exportSize}` : } +
    - + ); } diff --git a/src/components/PhotoFrame.tsx b/src/components/PhotoFrame.tsx index dc26d43bc..33bee68e2 100644 --- a/src/components/PhotoFrame.tsx +++ b/src/components/PhotoFrame.tsx @@ -21,8 +21,12 @@ import { isInsideBox, isSameDay as isSameDayAnyYear } from 'utils/search'; import { SetDialogMessage } from './MessageDialog'; import { CustomError } from 'utils/common/errorUtil'; import { - GAP_BTW_TILES, DATE_CONTAINER_HEIGHT, IMAGE_CONTAINER_MAX_HEIGHT, - IMAGE_CONTAINER_MAX_WIDTH, MIN_COLUMNS, SPACE_BTW_DATES, + GAP_BTW_TILES, + DATE_CONTAINER_HEIGHT, + IMAGE_CONTAINER_MAX_HEIGHT, + IMAGE_CONTAINER_MAX_WIDTH, + MIN_COLUMNS, + SPACE_BTW_DATES, } from 'types'; const NO_OF_PAGES = 2; @@ -68,21 +72,24 @@ const getTemplateColumns = (columns: number, groups?: number[]): string => { if (sum < columns) { groups[groups.length - 1] += columns - sum; } - return groups.map((x) => `repeat(${x}, 1fr)`).join(` ${SPACE_BTW_DATES}px `); + return groups + .map((x) => `repeat(${x}, 1fr)`) + .join(` ${SPACE_BTW_DATES}px `); } else { return `repeat(${columns}, 1fr)`; } }; -const ListContainer = styled.div<{ columns: number, groups?: number[] }>` +const ListContainer = styled.div<{ columns: number; groups?: number[] }>` display: grid; - grid-template-columns: ${({ columns, groups }) => getTemplateColumns(columns, groups)}; + grid-template-columns: ${({ columns, groups }) => + getTemplateColumns(columns, groups)}; grid-column-gap: ${GAP_BTW_TILES}px; padding: 0 24px; width: 100%; color: #fff; - @media(max-width: ${IMAGE_CONTAINER_MAX_WIDTH * 4}px) { + @media (max-width: ${IMAGE_CONTAINER_MAX_WIDTH * 4}px) { padding: 0 4px; } `; @@ -139,7 +146,7 @@ interface Props { search: Search; setSearchStats: setSearchStats; deleted?: number[]; - setDialogMessage: SetDialogMessage + setDialogMessage: SetDialogMessage; } const PhotoFrame = ({ @@ -303,14 +310,13 @@ const PhotoFrame = ({ video.preload = 'metadata'; video.src = url; video.currentTime = 3; - const t = setTimeout( - () => { - reject( - Error(`${CustomError.VIDEO_PLAYBACK_FAILED} err: wait time exceeded`), - ); - }, - WAIT_FOR_VIDEO_PLAYBACK, - ); + const t = setTimeout(() => { + reject( + Error( + `${CustomError.VIDEO_PLAYBACK_FAILED} err: wait time exceeded`, + ), + ); + }, WAIT_FOR_VIDEO_PLAYBACK); }); item.html = `