beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

improved file-upload service

Browse source
This commit is contained in:
Abhinav-grd 2021-02-16 15:14:03 +05:30
parent 2cf099132e
commit 2f161f9eda
1 changed files with 165 additions and 192 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

357
src/services/uploadService.ts
View file

@ -15,72 +15,69 @@ const THUMBNAIL_HEIGHT = 1080;
const MAX_ATTEMPTS = 3; const MAX_ATTEMPTS = 3;
const MIN_THUMBNAIL_SIZE = 50000; const MIN_THUMBNAIL_SIZE = 50000;
interface encryptionResult { interface EncryptionResult {
file: fileAttribute; file: fileAttribute;
key: string; key: string;
} }
export interface keyEncryptionResult { export interface KeyEncryptionResult {
encryptedData: string; encryptedData: string;
key: string; key: string;
nonce: string; nonce: string;
} }
interface uploadURL { interface UploadURL {
url: string; url: string;
objectKey: string; objectKey: string;
} }
export interface MetadataObject {
title: string;
creationTime: number;
modificationTime: number;
latitude: number;
longitude: number;
fileType: FILE_TYPE;
}
interface FileinMemory { interface FileinMemory {
filedata: Uint8Array; filedata: Uint8Array;
thumbnail: Uint8Array; thumbnail: Uint8Array;
filename: string; metadata: MetadataObject;
} }
interface encryptedFile { interface EncryptedFile {
filedata: fileAttribute; file: ProcessedFile;
fileKey: KeyEncryptionResult;
}
interface ProcessedFile {
file: fileAttribute;
thumbnail: fileAttribute; thumbnail: fileAttribute;
fileKey: keyEncryptionResult; metadata: fileAttribute;
} }
interface BackupedFile extends ProcessedFile {}
interface objectKey { interface uploadFile extends BackupedFile {
objectKey: string;
decryptionHeader: string;
}
interface objectKeys {
file: objectKey;
thumbnail: objectKey;
}
interface uploadFile extends objectKeys {
collectionID: number; collectionID: number;
encryptedKey: string; encryptedKey: string;
keyDecryptionNonce: string; keyDecryptionNonce: string;
metadata?: {
encryptedData: string | Uint8Array;
decryptionHeader: string;
};
}
interface UploadFileWithoutMetaData {
tempUploadFile: uploadFile;
encryptedFileKey: keyEncryptionResult;
fileName: string;
} }
export enum UPLOAD_STAGES { export enum UPLOAD_STAGES {
START, START,
ENCRYPTION, READING_GOOGLE_METADATA_FILES,
UPLOAD, UPLOADING,
FINISH, FINISH,
} }
class UploadService { class UploadService {
private uploadURLs: uploadURL[]; private uploadURLs: UploadURL[] = [];
private uploadURLFetchInProgress: Promise<any>; private uploadURLFetchInProgress: Promise<any> = null;
private perStepProgress: number; private perFileProgress: number;
private stepsCompleted: number; private filesCompleted: number;
private totalFilesCount: number; private totalFileCount: number;
private metadataMap: Map<string, Object>; private metadataMap: Map<string, Object>;
private filesToBeUploaded: File[];
private progressBarProps;
public async uploadFiles( public async uploadFiles(
recievedFiles: File[], recievedFiles: File[],
@ -89,11 +86,11 @@ class UploadService {
progressBarProps progressBarProps
) { ) {
try { try {
const worker = await new CryptoWorker(); progressBarProps.setUploadStage(UPLOAD_STAGES.START);
this.stepsCompleted = 0;
this.filesCompleted = 0;
this.metadataMap = new Map<string, object>(); this.metadataMap = new Map<string, object>();
this.uploadURLs = []; this.progressBarProps = progressBarProps;
this.uploadURLFetchInProgress = null;
let metadataFiles: File[] = []; let metadataFiles: File[] = [];
let actualFiles: File[] = []; let actualFiles: File[] = [];
@ -108,67 +105,33 @@ class UploadService {
metadataFiles.push(file); metadataFiles.push(file);
} }
}); });
this.totalFilesCount = actualFiles.length; this.totalFileCount = actualFiles.length;
this.perStepProgress = 100 / (3 * actualFiles.length); this.perFileProgress = 100 / actualFiles.length;
this.filesToBeUploaded = actualFiles;
progressBarProps.setUploadStage(UPLOAD_STAGES.START); progressBarProps.setUploadStage(
this.changeProgressBarProps(progressBarProps); UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES
);
const uploadFilesWithoutMetaData: UploadFileWithoutMetaData[] = [];
while (actualFiles.length > 0) {
var promises = [];
for (var i = 0; i < 5 && actualFiles.length > 0; i++)
promises.push(
this.uploadHelper(
progressBarProps,
actualFiles.pop(),
collectionAndItsLatestFile.collection,
token
)
);
uploadFilesWithoutMetaData.push(
...(await Promise.all(promises))
);
}
for await (const rawFile of metadataFiles) { for await (const rawFile of metadataFiles) {
await this.updateMetadata(rawFile); await this.seedMetadataMap(rawFile);
} }
progressBarProps.setUploadStage(UPLOAD_STAGES.ENCRYPTION); progressBarProps.setUploadStage(UPLOAD_STAGES.UPLOADING);
const completeUploadFiles: uploadFile[] = await Promise.all( this.changeProgressBarProps();
uploadFilesWithoutMetaData.map(
async (file: UploadFileWithoutMetaData) => {
const {
file: encryptedMetaData,
} = await this.encryptMetadata(
worker,
file.fileName,
file.encryptedFileKey
);
const completeUploadFile = {
...file.tempUploadFile,
metadata: {
encryptedData: encryptedMetaData.encryptedData,
decryptionHeader:
encryptedMetaData.decryptionHeader,
},
};
this.changeProgressBarProps(progressBarProps);
return completeUploadFile;
}
)
);
progressBarProps.setUploadStage(UPLOAD_STAGES.UPLOAD);
await Promise.all(
completeUploadFiles.map(async (uploadFile: uploadFile) => {
await this.uploadFile(uploadFile, token);
this.changeProgressBarProps(progressBarProps);
})
);
const uploadProcesses = [];
for (let i = 0; i < Math.min(5, this.totalFileCount); i++) {
uploadProcesses.push(
this.uploader(
await new CryptoWorker(),
this.filesToBeUploaded.pop(),
collectionAndItsLatestFile.collection,
token
)
);
}
await Promise.all(uploadProcesses);
progressBarProps.setUploadStage(UPLOAD_STAGES.FINISH); progressBarProps.setUploadStage(UPLOAD_STAGES.FINISH);
progressBarProps.setPercentComplete(100); progressBarProps.setPercentComplete(100);
} catch (e) { } catch (e) {
@ -176,43 +139,50 @@ class UploadService {
throw e; throw e;
} }
} }
private async uploadHelper(progressBarProps, rawFile, collection, token) { private async uploader(worker, rawFile, collection, token) {
try { try {
const worker = await new CryptoWorker();
let file: FileinMemory = await this.readFile(rawFile); let file: FileinMemory = await this.readFile(rawFile);
let encryptedFile: encryptedFile = await this.encryptFile(
let encryptedFile: EncryptedFile = await this.encryptFile(
worker, worker,
file, file,
collection.key collection.key
); );
let objectKeys = await this.uploadtoBucket( let backupedFile: BackupedFile = await this.uploadtoBucket(
encryptedFile, encryptedFile.file,
token, token
2 * this.totalFilesCount
); );
let uploadFileWithoutMetaData: uploadFile = this.getuploadFile( let uploadFile: uploadFile = this.getuploadFile(
collection, collection,
encryptedFile.fileKey, backupedFile,
objectKeys encryptedFile.fileKey
); );
this.changeProgressBarProps(progressBarProps); await this.uploadFile(uploadFile, token);
return { this.changeProgressBarProps();
tempUploadFile: uploadFileWithoutMetaData,
encryptedFileKey: encryptedFile.fileKey, if (this.filesToBeUploaded.length > 0) {
fileName: file.filename, await this.uploader(
}; worker,
this.filesToBeUploaded.pop(),
collection,
token
);
}
} catch (e) { } catch (e) {
console.log(e); console.log(e);
throw e; throw e;
} }
} }
private changeProgressBarProps({ setPercentComplete, setFileCounter }) { private changeProgressBarProps() {
this.stepsCompleted++; const { setPercentComplete, setFileCounter } = this.progressBarProps;
const fileCompleted = this.stepsCompleted % this.totalFilesCount; setFileCounter({
setFileCounter({ current: fileCompleted, total: this.totalFilesCount }); current: this.filesCompleted + 1,
setPercentComplete(this.perStepProgress * this.stepsCompleted); total: this.totalFileCount,
});
setPercentComplete(this.filesCompleted * this.perFileProgress);
this.filesCompleted++;
} }
private async readFile(recievedFile: File) { private async readFile(recievedFile: File) {
@ -220,7 +190,9 @@ class UploadService {
const filedata: Uint8Array = await this.getUint8ArrayView( const filedata: Uint8Array = await this.getUint8ArrayView(
recievedFile recievedFile
); );
let fileType; const thumbnail = await this.generateThumbnail(recievedFile);
let fileType: FILE_TYPE;
switch (recievedFile.type.split('/')[0]) { switch (recievedFile.type.split('/')[0]) {
case 'image': case 'image':
fileType = FILE_TYPE.IMAGE; fileType = FILE_TYPE.IMAGE;
@ -235,18 +207,22 @@ class UploadService {
const { location, creationTime } = await this.getExifData( const { location, creationTime } = await this.getExifData(
recievedFile recievedFile
); );
this.metadataMap.set(recievedFile.name, { const metadata = Object.assign(
title: recievedFile.name, this.metadataMap.get(recievedFile.name) ?? {},
creationTime: creationTime || recievedFile.lastModified * 1000, {
modificationTime: recievedFile.lastModified * 1000, title: recievedFile.name,
latitude: location?.latitude, creationTime:
longitude: location?.latitude, creationTime || recievedFile.lastModified * 1000,
fileType, modificationTime: recievedFile.lastModified * 1000,
}); latitude: location?.latitude,
longitude: location?.latitude,
fileType,
}
);
return { return {
filedata, filedata,
filename: recievedFile.name, thumbnail,
thumbnail: await this.generateThumbnail(recievedFile), metadata,
}; };
} catch (e) { } catch (e) {
console.log('error reading files ' + e); console.log('error reading files ' + e);
@ -256,28 +232,37 @@ class UploadService {
worker, worker,
file: FileinMemory, file: FileinMemory,
encryptionKey: string encryptionKey: string
): Promise<encryptedFile> { ): Promise<EncryptedFile> {
try { try {
const { const {
key: fileKey, key: fileKey,
file: encryptedFiledata, file: encryptedFiledata,
}: encryptionResult = await worker.encryptFile(file.filedata); }: EncryptionResult = await worker.encryptFile(file.filedata);
const { const {
file: encryptedThumbnail, file: encryptedThumbnail,
}: encryptionResult = await worker.encryptThumbnail( }: EncryptionResult = await worker.encryptThumbnail(
file.thumbnail, file.thumbnail,
fileKey fileKey
); );
const {
file: encryptedMetadata,
}: EncryptionResult = await worker.encryptMetadata(
file.metadata,
fileKey
);
const encryptedKey: keyEncryptionResult = await worker.encryptToB64( const encryptedKey: KeyEncryptionResult = await worker.encryptToB64(
fileKey, fileKey,
encryptionKey encryptionKey
); );
const result: encryptedFile = { const result: EncryptedFile = {
filedata: encryptedFiledata, file: {
thumbnail: encryptedThumbnail, file: encryptedFiledata,
thumbnail: encryptedThumbnail,
metadata: encryptedMetadata,
},
fileKey: encryptedKey, fileKey: encryptedKey,
}; };
return result; return result;
@ -286,52 +271,26 @@ class UploadService {
} }
} }
private async encryptMetadata(
worker: any,
fileName: string,
encryptedFileKey: keyEncryptionResult
) {
const metaData = this.metadataMap.get(fileName);
const fileKey = await worker.decryptB64(
encryptedFileKey.encryptedData,
encryptedFileKey.nonce,
encryptedFileKey.key
);
const encryptedMetaData = await worker.encryptMetadata(
metaData,
fileKey
);
return encryptedMetaData;
}
private async uploadtoBucket( private async uploadtoBucket(
file: encryptedFile, file: ProcessedFile,
token, token
count: number ): Promise<BackupedFile> {
): Promise<objectKeys> {
try { try {
const fileUploadURL = await this.getUploadURL(token, count); const fileUploadURL = await this.getUploadURL(token);
const fileObjectKey = await this.putFile( file.file.objectKey = await this.putFile(
fileUploadURL, fileUploadURL,
file.filedata.encryptedData file.file.encryptedData
); );
const thumbnailUploadURL = await this.getUploadURL(token, count); const thumbnailUploadURL = await this.getUploadURL(token);
const thumbnailObjectKey = await this.putFile( file.thumbnail.objectKey = await this.putFile(
thumbnailUploadURL, thumbnailUploadURL,
file.thumbnail.encryptedData file.thumbnail.encryptedData
); );
delete file.file.encryptedData;
delete file.thumbnail.encryptedData;
return { return file;
file: {
objectKey: fileObjectKey,
decryptionHeader: file.filedata.decryptionHeader,
},
thumbnail: {
objectKey: thumbnailObjectKey,
decryptionHeader: file.thumbnail.decryptionHeader,
},
};
} catch (e) { } catch (e) {
console.log('error uploading to bucket ' + e); console.log('error uploading to bucket ' + e);
throw e; throw e;
@ -340,14 +299,14 @@ class UploadService {
private getuploadFile( private getuploadFile(
collection: collection, collection: collection,
encryptedKey: keyEncryptionResult, backupedFile: BackupedFile,
objectKeys: objectKeys fileKey: KeyEncryptionResult
): uploadFile { ): uploadFile {
const uploadFile: uploadFile = { const uploadFile: uploadFile = {
collectionID: collection.id, collectionID: collection.id,
encryptedKey: encryptedKey.encryptedData, encryptedKey: fileKey.encryptedData,
keyDecryptionNonce: encryptedKey.nonce, keyDecryptionNonce: fileKey.nonce,
...objectKeys, ...backupedFile,
}; };
return uploadFile; return uploadFile;
} }
@ -367,7 +326,7 @@ class UploadService {
} }
} }
private async updateMetadata(recievedFile: File) { private async seedMetadataMap(recievedFile: File) {
try { try {
const metadataJSON: object = await new Promise( const metadataJSON: object = await new Promise(
(resolve, reject) => { (resolve, reject) => {
@ -520,20 +479,23 @@ class UploadService {
} }
} }
private async getUploadURL(token: string, count: number) { private async getUploadURL(token: string) {
if (this.uploadURLs.length == 0) { if (this.uploadURLs.length == 0) {
await this.fetchUploadURLs(token, count); await this.fetchUploadURLs(token);
} }
return this.uploadURLs.pop(); return this.uploadURLs.pop();
} }
private async fetchUploadURLs(token: string, count: number): Promise<void> { private async fetchUploadURLs(token: string): Promise<void> {
try { try {
if (!this.uploadURLFetchInProgress) { if (!this.uploadURLFetchInProgress) {
this.uploadURLFetchInProgress = HTTPService.get( this.uploadURLFetchInProgress = HTTPService.get(
`${ENDPOINT}/files/upload-urls`, `${ENDPOINT}/files/upload-urls`,
{ {
count: Math.min(50, count).toString(), //m4gic number count: Math.min(
50,
(this.filesToBeUploaded.length + 1) * 2
).toString(),
}, },
{ 'X-Auth-Token': token } { 'X-Auth-Token': token }
); );
@ -550,7 +512,7 @@ class UploadService {
} }
private async putFile( private async putFile(
fileUploadURL: uploadURL, fileUploadURL: UploadURL,
file: Uint8Array | string file: Uint8Array | string
): Promise<string> { ): Promise<string> {
try { try {
@ -586,10 +548,10 @@ class UploadService {
} }
} }
private getUNIXTime(exifData: any) { private getUNIXTime(exifData: any) {
if (!exifData.DateTimeOriginal) { let dateString: string = exifData.DateTimeOriginal || exifData.DateTime;
if (!dateString) {
return null; return null;
} }
let dateString: string = exifData.DateTimeOriginal;
var parts = dateString.split(' ')[0].split(':'); var parts = dateString.split(' ')[0].split(':');
var date = new Date( var date = new Date(
Number(parts[0]), Number(parts[0]),
@ -603,10 +565,28 @@ class UploadService {
if (!exifData.GPSLatitude) { if (!exifData.GPSLatitude) {
return null; return null;
} }
var latDegree = exifData.GPSLatitude[0].numerator;
var latMinute = exifData.GPSLatitude[1].numerator; let latDegree: number, latMinute: number, latSecond: number;
var latSecond = exifData.GPSLatitude[2].numerator; let lonDegree: number, lonMinute: number, lonSecond: number;
if (exifData.GPSLatitude[0].numerator) {
latDegree = exifData.GPSLatitude[0].numerator;
latMinute = exifData.GPSLatitude[1].numerator;
latSecond = exifData.GPSLatitude[2].numerator;
lonDegree = exifData.GPSLongitude[0].numerator;
lonMinute = exifData.GPSLongitude[1].numerator;
lonSecond = exifData.GPSLongitude[2].numerator;
} else {
latDegree = exifData.GPSLatitude[0];
latMinute = exifData.GPSLatitude[1];
latSecond = exifData.GPSLatitude[2];
lonDegree = exifData.GPSLongitude[0];
lonMinute = exifData.GPSLongitude[1];
lonSecond = exifData.GPSLongitude[2];
}
var latDirection = exifData.GPSLatitudeRef; var latDirection = exifData.GPSLatitudeRef;
var lonDirection = exifData.GPSLongitudeRef;
var latFinal = this.convertDMSToDD( var latFinal = this.convertDMSToDD(
latDegree, latDegree,
@ -615,19 +595,12 @@ class UploadService {
latDirection latDirection
); );
// Calculate longitude decimal
var lonDegree = exifData.GPSLongitude[0].numerator;
var lonMinute = exifData.GPSLongitude[1].numerator;
var lonSecond = exifData.GPSLongitude[2].numerator;
var lonDirection = exifData.GPSLongitudeRef;
var lonFinal = this.convertDMSToDD( var lonFinal = this.convertDMSToDD(
lonDegree, lonDegree,
lonMinute, lonMinute,
lonSecond, lonSecond,
lonDirection lonDirection
); );
return { latitude: latFinal * 1.0, longitude: lonFinal * 1.0 }; return { latitude: latFinal * 1.0, longitude: lonFinal * 1.0 };
} }