moved UploadURl logic to uploadService
This commit is contained in:
parent
e39583efdf
commit
e59be3e174
|
@ -10,7 +10,6 @@ import { retryAsyncFunction } from 'utils/network';
|
|||
const ENDPOINT = getEndpoint();
|
||||
const MAX_URL_REQUESTS = 50;
|
||||
|
||||
|
||||
class NetworkClient {
|
||||
private uploadURLFetchInProgress = null;
|
||||
|
||||
|
@ -20,14 +19,11 @@ class NetworkClient {
|
|||
if (!token) {
|
||||
return;
|
||||
}
|
||||
const response = await retryAsyncFunction(()=>HTTPService.post(
|
||||
`${ENDPOINT}/files`,
|
||||
uploadFile,
|
||||
null,
|
||||
{
|
||||
const response = await retryAsyncFunction(() =>
|
||||
HTTPService.post(`${ENDPOINT}/files`, uploadFile, null, {
|
||||
'X-Auth-Token': token,
|
||||
},
|
||||
));
|
||||
}),
|
||||
);
|
||||
return response.data;
|
||||
} catch (e) {
|
||||
logError(e, 'upload Files Failed');
|
||||
|
@ -46,10 +42,7 @@ class NetworkClient {
|
|||
this.uploadURLFetchInProgress = HTTPService.get(
|
||||
`${ENDPOINT}/files/upload-urls`,
|
||||
{
|
||||
count: Math.min(
|
||||
MAX_URL_REQUESTS,
|
||||
count * 2,
|
||||
),
|
||||
count: Math.min(MAX_URL_REQUESTS, count * 2),
|
||||
},
|
||||
{ 'X-Auth-Token': token },
|
||||
);
|
||||
|
@ -92,7 +85,7 @@ class NetworkClient {
|
|||
async putFile(
|
||||
fileUploadURL: UploadURL,
|
||||
file: Uint8Array,
|
||||
progressTracker:()=>any,
|
||||
progressTracker,
|
||||
): Promise<string> {
|
||||
try {
|
||||
await retryAsyncFunction(() =>
|
||||
|
@ -101,7 +94,7 @@ class NetworkClient {
|
|||
file,
|
||||
null,
|
||||
null,
|
||||
progressTracker(),
|
||||
progressTracker,
|
||||
),
|
||||
);
|
||||
return fileUploadURL.objectKey;
|
||||
|
@ -111,7 +104,6 @@ class NetworkClient {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
async putFilePart(
|
||||
partUploadURL: string,
|
||||
filePart: Uint8Array,
|
||||
|
@ -155,5 +147,3 @@ class NetworkClient {
|
|||
}
|
||||
|
||||
export default new NetworkClient();
|
||||
|
||||
|
||||
|
|
|
@ -10,12 +10,12 @@ import {
|
|||
} from 'utils/file';
|
||||
import { logError } from 'utils/sentry';
|
||||
import localForage from 'utils/storage/localForage';
|
||||
import NetworkClient from './networkClient';
|
||||
import { ParsedMetaDataJSON, parseMetadataJSON } from './metadataService';
|
||||
import { segregateFiles } from 'utils/upload';
|
||||
import { ProgressUpdater } from 'components/pages/gallery/Upload';
|
||||
import uploader from './uploader';
|
||||
import uiService from './uiService';
|
||||
import uploadService from './uploadService';
|
||||
|
||||
const MAX_CONCURRENT_UPLOADS = 4;
|
||||
const FILE_UPLOAD_COMPLETED = 100;
|
||||
|
@ -85,7 +85,7 @@ class UploadManager {
|
|||
uiService.setUploadStage(
|
||||
UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES,
|
||||
);
|
||||
await this.processMetadataFiles(metadataFiles);
|
||||
await this.seedMetadataMap(metadataFiles);
|
||||
}
|
||||
if (mediaFiles.length) {
|
||||
uiService.setUploadStage(UPLOAD_STAGES.START);
|
||||
|
@ -104,20 +104,7 @@ class UploadManager {
|
|||
}
|
||||
}
|
||||
|
||||
private async preFetchUploadURLs(count: number) {
|
||||
try {
|
||||
// checking for any subscription related errors
|
||||
await NetworkClient.fetchUploadURLs(count, this.uploadURLs);
|
||||
} catch (e) {
|
||||
logError(e, 'error fetching uploadURLs');
|
||||
const { parsedError, parsed } = parseError(e);
|
||||
if (parsed) {
|
||||
throw parsedError;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async processMetadataFiles(metadataFiles: globalThis.File[]) {
|
||||
private async seedMetadataMap(metadataFiles: globalThis.File[]) {
|
||||
uiService.reset(metadataFiles.length);
|
||||
|
||||
for (const rawFile of metadataFiles) {
|
||||
|
@ -152,8 +139,22 @@ class UploadManager {
|
|||
await Promise.all(uploadProcesses);
|
||||
}
|
||||
|
||||
private async preFetchUploadURLs(count: number) {
|
||||
try {
|
||||
uploadService.setPendingUploadCount(count);
|
||||
uploadService.preFetchUploadURLs();
|
||||
// checking for any subscription related errors
|
||||
} catch (e) {
|
||||
logError(e, 'error fetching uploadURLs');
|
||||
const { parsedError, parsed } = parseError(e);
|
||||
if (parsed) {
|
||||
throw parsedError;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async uploadNextFileInQueue(worker: any, fileReader: FileReader) {
|
||||
if (this.filesToBeUploaded.length > 0) {
|
||||
while (this.filesToBeUploaded.length > 0) {
|
||||
const fileWithCollection = this.filesToBeUploaded.pop();
|
||||
const { fileUploadResult, file } = await uploader(
|
||||
worker,
|
||||
|
|
|
@ -13,6 +13,7 @@ import {
|
|||
import { encryptFiledata } from './encryptionService';
|
||||
import { ENCRYPTION_CHUNK_SIZE } from 'types';
|
||||
import { uploadStreamUsingMultipart } from './s3Service';
|
||||
import uiService from './uiService';
|
||||
|
||||
export const RANDOM_PERCENTAGE_PROGRESS_FOR_PUT = () => 90 + 10 * Math.random();
|
||||
export const MIN_STREAM_FILE_SIZE = 20 * 1024 * 1024;
|
||||
|
@ -106,8 +107,12 @@ export enum UPLOAD_STAGES {
|
|||
|
||||
class UploadService {
|
||||
private uploadURLs: UploadURL[] = [];
|
||||
private pendingFilesUploads: number;
|
||||
private metadataMap: Map<string, ParsedMetaDataJSON>;
|
||||
private pendingUploadCount: number = 0;
|
||||
|
||||
setPendingUploadCount(count: number) {
|
||||
this.pendingUploadCount = count;
|
||||
}
|
||||
|
||||
async readFile(reader: FileReader, receivedFile: globalThis.File) {
|
||||
try {
|
||||
|
@ -140,7 +145,7 @@ class UploadService {
|
|||
filedata,
|
||||
thumbnail,
|
||||
metadata,
|
||||
};
|
||||
} as FileInMemory;
|
||||
} catch (e) {
|
||||
logError(e, 'error reading files');
|
||||
throw e;
|
||||
|
@ -182,22 +187,16 @@ class UploadService {
|
|||
}
|
||||
}
|
||||
|
||||
async uploadToBucket(
|
||||
file: ProcessedFile,
|
||||
trackUploadProgress,
|
||||
): Promise<BackupedFile> {
|
||||
async uploadToBucket(file: ProcessedFile): Promise<BackupedFile> {
|
||||
try {
|
||||
let fileObjectKey: string = null;
|
||||
if (isDataStream(file.file.encryptedData)) {
|
||||
const progressTracker = trackUploadProgress;
|
||||
fileObjectKey = await uploadStreamUsingMultipart(
|
||||
file.filename,
|
||||
file.file.encryptedData,
|
||||
progressTracker,
|
||||
);
|
||||
} else {
|
||||
const progressTracker = trackUploadProgress.bind(
|
||||
null,
|
||||
const progressTracker = uiService.trackUploadProgress(
|
||||
file.filename,
|
||||
);
|
||||
const fileUploadURL = await this.getUploadURL();
|
||||
|
@ -211,7 +210,7 @@ class UploadService {
|
|||
const thumbnailObjectKey = await NetworkClient.putFile(
|
||||
thumbnailUploadURL,
|
||||
file.thumbnail.encryptedData as Uint8Array,
|
||||
() => null,
|
||||
null,
|
||||
);
|
||||
|
||||
const backupedFile: BackupedFile = {
|
||||
|
@ -248,14 +247,22 @@ class UploadService {
|
|||
}
|
||||
|
||||
private async getUploadURL() {
|
||||
if (this.uploadURLs.length === 0) {
|
||||
await NetworkClient.fetchUploadURLs(
|
||||
this.pendingFilesUploads,
|
||||
this.uploadURLs,
|
||||
);
|
||||
if (this.uploadURLs.length === 0 && this.pendingUploadCount) {
|
||||
await this.fetchUploadURLs();
|
||||
}
|
||||
return this.uploadURLs.pop();
|
||||
}
|
||||
|
||||
public async preFetchUploadURLs() {
|
||||
await this.fetchUploadURLs();
|
||||
}
|
||||
|
||||
private async fetchUploadURLs() {
|
||||
await NetworkClient.fetchUploadURLs(
|
||||
this.pendingUploadCount,
|
||||
this.uploadURLs,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export default new UploadService();
|
||||
|
|
Loading…
Reference in a new issue