beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #20 from ente-io/improve-uploading

Browse source 
Improve uploading
This commit is contained in:
Vishnu Mohandas 2021-02-16 18:51:55 +05:30 committed by GitHub
parent d2a19c8cd8 e4ef214711
commit ebe2397463
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 225 additions and 248 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/pages/gallery/components/CollectionSelector.tsx
View file

@ -13,11 +13,9 @@ function CollectionSelector(props) {
collectionAndItsLatestFile,
...rest
} = props;
const CollectionIcons = collectionAndItsLatestFile?.map((item) => (
<div style={{ margin: '10px' }}>
<div style={{ margin: '10px' }} key={item.collection.id}>
<CollectionDropZone
key={item.collection.id}
{...rest}
closeModal={closeUploadModal}
showModal={showUploadModal}
@ -45,7 +43,9 @@ function CollectionSelector(props) {
style={{ maxWidth: '100%' }}
>
<Modal.Header closeButton>
<Modal.Title style={{ marginLeft: '12px' }}>{constants.SELECT_COLLECTION}</Modal.Title>
<Modal.Title style={{ marginLeft: '12px' }}>
{constants.SELECT_COLLECTION}
</Modal.Title>
</Modal.Header>
<Modal.Body
style={{

84
src/services/collectionService.ts
View file

@ -207,46 +207,50 @@ export const AddCollection = async (
collectionName: string,
type: CollectionType
) => {
const worker = await new CryptoWorker();
const encryptionKey = await getActualKey();
const token = getToken();
const collectionKey: string = await worker.generateMasterKey();
const {
encryptedData: encryptedKey,
nonce: keyDecryptionNonce,
}: B64EncryptionResult = await worker.encryptToB64(
collectionKey,
encryptionKey
);
const {
encryptedData: encryptedName,
nonce: nameDecryptionNonce,
}: B64EncryptionResult = await worker.encryptUTF8(
collectionName,
collectionKey
);
const newCollection: collection = {
id: null,
owner: null,
encryptedKey,
keyDecryptionNonce,
encryptedName,
nameDecryptionNonce,
type,
attributes: {},
sharees: null,
updationTime: null,
isDeleted: false,
};
let createdCollection: collection = await createCollection(
newCollection,
token
);
createdCollection = await getCollectionSecrets(
createdCollection,
encryptionKey
);
return createdCollection;
try {
const worker = await new CryptoWorker();
const encryptionKey = await getActualKey();
const token = getToken();
const collectionKey: string = await worker.generateMasterKey();
const {
encryptedData: encryptedKey,
nonce: keyDecryptionNonce,
}: B64EncryptionResult = await worker.encryptToB64(
collectionKey,
encryptionKey
);
const {
encryptedData: encryptedName,
nonce: nameDecryptionNonce,
}: B64EncryptionResult = await worker.encryptUTF8(
collectionName,
collectionKey
);
const newCollection: collection = {
id: null,
owner: null,
encryptedKey,
keyDecryptionNonce,
encryptedName,
nameDecryptionNonce,
type,
attributes: {},
sharees: null,
updationTime: null,
isDeleted: false,
};
let createdCollection: collection = await createCollection(
newCollection,
token
);
createdCollection = await getCollectionSecrets(
createdCollection,
encryptionKey
);
return createdCollection;
} catch (e) {
console.log('Add collection failed', e);
}
};
const createCollection = async (

8
src/services/fileService.ts
View file

@ -3,6 +3,7 @@ import HTTPService from './HTTPService';
import * as Comlink from 'comlink';
import localForage from 'localforage';
import { collection } from './collectionService';
import { MetadataObject } from './uploadService';
const CryptoWorker: any =
typeof window !== 'undefined' &&
@ -19,10 +20,9 @@ localForage.config({
const FILES = 'files';
export interface fileAttribute {
encryptedData: Uint8Array | string;
encryptedData?: Uint8Array;
objectKey?: string;
decryptionHeader: string;
creationTime: number;
fileType: number;
}
@ -31,7 +31,7 @@ export interface file {
collectionID: number;
file: fileAttribute;
thumbnail: fileAttribute;
metadata: fileAttribute;
metadata: MetadataObject;
encryptedKey: string;
keyDecryptionNonce: string;
key: string;

369
src/services/uploadService.ts
View file

@ -10,12 +10,11 @@ const CryptoWorker: any =
Comlink.wrap(new Worker('worker/crypto.worker.js', { type: 'module' }));
const ENDPOINT = getEndpoint();
const THUMBNAIL_WIDTH = 1920;
const THUMBNAIL_HEIGHT = 1080;
const THUMBNAIL_HEIGHT = 720;
const MAX_ATTEMPTS = 3;
const MIN_THUMBNAIL_SIZE = 50000;
interface encryptionResult {
interface EncryptionResult {
file: fileAttribute;
key: string;
}
@ -25,62 +24,59 @@ export interface B64EncryptionResult {
nonce: string;
}
interface uploadURL {
interface UploadURL {
url: string;
objectKey: string;
}
export interface MetadataObject {
title: string;
creationTime: number;
modificationTime: number;
latitude: number;
longitude: number;
fileType: FILE_TYPE;
}
interface FileinMemory {
filedata: Uint8Array;
thumbnail: Uint8Array;
filename: string;
metadata: MetadataObject;
}
interface encryptedFile {
filedata: fileAttribute;
thumbnail: fileAttribute;
interface EncryptedFile {
file: ProcessedFile;
fileKey: B64EncryptionResult;
}
interface objectKey {
objectKey: string;
decryptionHeader: string;
}
interface objectKeys {
file: objectKey;
thumbnail: objectKey;
interface ProcessedFile {
file: fileAttribute;
thumbnail: fileAttribute;
metadata: fileAttribute;
}
interface BackupedFile extends ProcessedFile {}
interface uploadFile extends objectKeys {
interface uploadFile extends BackupedFile {
collectionID: number;
encryptedKey: string;
keyDecryptionNonce: string;
metadata?: {
encryptedData: string | Uint8Array;
decryptionHeader: string;
};
}
interface UploadFileWithoutMetaData {
tempUploadFile: uploadFile;
encryptedFileKey: B64EncryptionResult;
fileName: string;
}
export enum UPLOAD_STAGES {
START,
ENCRYPTION,
UPLOAD,
READING_GOOGLE_METADATA_FILES,
UPLOADING,
FINISH,
}
class UploadService {
private uploadURLs: uploadURL[];
private uploadURLFetchInProgress: Promise<any>;
private perStepProgress: number;
private stepsCompleted: number;
private totalFilesCount: number;
private uploadURLs: UploadURL[] = [];
private uploadURLFetchInProgress: Promise<any> = null;
private perFileProgress: number;
private filesCompleted: number;
private totalFileCount: number;
private metadataMap: Map<string, Object>;
private filesToBeUploaded: File[];
private progressBarProps;
public async uploadFiles(
recievedFiles: File[],
@ -89,11 +85,11 @@ class UploadService {
progressBarProps
) {
try {
const worker = await new CryptoWorker();
this.stepsCompleted = 0;
progressBarProps.setUploadStage(UPLOAD_STAGES.START);
this.filesCompleted = 0;
this.metadataMap = new Map<string, object>();
this.uploadURLs = [];
this.uploadURLFetchInProgress = null;
this.progressBarProps = progressBarProps;
let metadataFiles: File[] = [];
let actualFiles: File[] = [];
@ -108,67 +104,33 @@ class UploadService {
metadataFiles.push(file);
}
});
this.totalFilesCount = actualFiles.length;
this.perStepProgress = 100 / (3 * actualFiles.length);
this.totalFileCount = actualFiles.length;
this.perFileProgress = 100 / actualFiles.length;
this.filesToBeUploaded = actualFiles;
progressBarProps.setUploadStage(UPLOAD_STAGES.START);
this.changeProgressBarProps(progressBarProps);
const uploadFilesWithoutMetaData: UploadFileWithoutMetaData[] = [];
while (actualFiles.length > 0) {
var promises = [];
for (var i = 0; i < 5 && actualFiles.length > 0; i++)
promises.push(
this.uploadHelper(
progressBarProps,
actualFiles.pop(),
collectionAndItsLatestFile.collection,
token
)
);
uploadFilesWithoutMetaData.push(
...(await Promise.all(promises))
);
}
progressBarProps.setUploadStage(
UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES
);
for await (const rawFile of metadataFiles) {
await this.updateMetadata(rawFile);
await this.seedMetadataMap(rawFile);
}
progressBarProps.setUploadStage(UPLOAD_STAGES.ENCRYPTION);
const completeUploadFiles: uploadFile[] = await Promise.all(
uploadFilesWithoutMetaData.map(
async (file: UploadFileWithoutMetaData) => {
const {
file: encryptedMetaData,
} = await this.encryptMetadata(
worker,
file.fileName,
file.encryptedFileKey
);
const completeUploadFile = {
...file.tempUploadFile,
metadata: {
encryptedData: encryptedMetaData.encryptedData,
decryptionHeader:
encryptedMetaData.decryptionHeader,
},
};
this.changeProgressBarProps(progressBarProps);
return completeUploadFile;
}
)
);
progressBarProps.setUploadStage(UPLOAD_STAGES.UPLOAD);
await Promise.all(
completeUploadFiles.map(async (uploadFile: uploadFile) => {
await this.uploadFile(uploadFile, token);
this.changeProgressBarProps(progressBarProps);
})
);
progressBarProps.setUploadStage(UPLOAD_STAGES.UPLOADING);
this.changeProgressBarProps();
const uploadProcesses = [];
for (let i = 0; i < Math.min(5, this.totalFileCount); i++) {
uploadProcesses.push(
this.uploader(
await new CryptoWorker(),
this.filesToBeUploaded.pop(),
collectionAndItsLatestFile.collection,
token
)
);
}
await Promise.all(uploadProcesses);
progressBarProps.setUploadStage(UPLOAD_STAGES.FINISH);
progressBarProps.setPercentComplete(100);
} catch (e) {
@ -176,43 +138,50 @@ class UploadService {
throw e;
}
}
private async uploadHelper(progressBarProps, rawFile, collection, token) {
private async uploader(worker, rawFile, collection, token) {
try {
const worker = await new CryptoWorker();
let file: FileinMemory = await this.readFile(rawFile);
let encryptedFile: encryptedFile = await this.encryptFile(
let encryptedFile: EncryptedFile = await this.encryptFile(
worker,
file,
collection.key
);
let objectKeys = await this.uploadtoBucket(
encryptedFile,
token,
2 * this.totalFilesCount
let backupedFile: BackupedFile = await this.uploadtoBucket(
encryptedFile.file,
token
);
let uploadFileWithoutMetaData: uploadFile = this.getuploadFile(
let uploadFile: uploadFile = this.getuploadFile(
collection,
encryptedFile.fileKey,
objectKeys
backupedFile,
encryptedFile.fileKey
);
this.changeProgressBarProps(progressBarProps);
await this.uploadFile(uploadFile, token);
return {
tempUploadFile: uploadFileWithoutMetaData,
encryptedFileKey: encryptedFile.fileKey,
fileName: file.filename,
};
this.changeProgressBarProps();
if (this.filesToBeUploaded.length > 0) {
await this.uploader(
worker,
this.filesToBeUploaded.pop(),
collection,
token
);
}
} catch (e) {
console.log(e);
throw e;
}
}
private changeProgressBarProps({ setPercentComplete, setFileCounter }) {
this.stepsCompleted++;
const fileCompleted = this.stepsCompleted % this.totalFilesCount;
setFileCounter({ current: fileCompleted, total: this.totalFilesCount });
setPercentComplete(this.perStepProgress * this.stepsCompleted);
private changeProgressBarProps() {
const { setPercentComplete, setFileCounter } = this.progressBarProps;
setFileCounter({
current: this.filesCompleted + 1,
total: this.totalFileCount,
});
setPercentComplete(this.filesCompleted * this.perFileProgress);
this.filesCompleted++;
}
private async readFile(recievedFile: File) {
@ -220,7 +189,9 @@ class UploadService {
const filedata: Uint8Array = await this.getUint8ArrayView(
recievedFile
);
let fileType;
const thumbnail = await this.generateThumbnail(recievedFile);
let fileType: FILE_TYPE;
switch (recievedFile.type.split('/')[0]) {
case 'image':
fileType = FILE_TYPE.IMAGE;
@ -235,18 +206,22 @@ class UploadService {
const { location, creationTime } = await this.getExifData(
recievedFile
);
this.metadataMap.set(recievedFile.name, {
title: recievedFile.name,
creationTime: creationTime || recievedFile.lastModified * 1000,
modificationTime: recievedFile.lastModified * 1000,
latitude: location?.latitude,
longitude: location?.latitude,
fileType,
});
const metadata = Object.assign(
this.metadataMap.get(recievedFile.name) ?? {},
{
title: recievedFile.name,
creationTime:
creationTime || recievedFile.lastModified * 1000,
modificationTime: recievedFile.lastModified * 1000,
latitude: location?.latitude,
longitude: location?.latitude,
fileType,
}
);
return {
filedata,
filename: recievedFile.name,
thumbnail: await this.generateThumbnail(recievedFile),
thumbnail,
metadata,
};
} catch (e) {
console.log('error reading files ' + e);
@ -256,28 +231,37 @@ class UploadService {
worker,
file: FileinMemory,
encryptionKey: string
): Promise<encryptedFile> {
): Promise<EncryptedFile> {
try {
const {
key: fileKey,
file: encryptedFiledata,
}: encryptionResult = await worker.encryptFile(file.filedata);
}: EncryptionResult = await worker.encryptFile(file.filedata);
const {
file: encryptedThumbnail,
}: encryptionResult = await worker.encryptThumbnail(
}: EncryptionResult = await worker.encryptThumbnail(
file.thumbnail,
fileKey
);
const {
file: encryptedMetadata,
}: EncryptionResult = await worker.encryptMetadata(
file.metadata,
fileKey
);
const encryptedKey: B64EncryptionResult = await worker.encryptB64(
const encryptedKey: B64EncryptionResult = await worker.encryptToB64(
fileKey,
encryptionKey
);
const result: encryptedFile = {
filedata: encryptedFiledata,
thumbnail: encryptedThumbnail,
const result: EncryptedFile = {
file: {
file: encryptedFiledata,
thumbnail: encryptedThumbnail,
metadata: encryptedMetadata,
},
fileKey: encryptedKey,
};
return result;
@ -286,52 +270,26 @@ class UploadService {
}
}
private async encryptMetadata(
worker: any,
fileName: string,
encryptedFileKey: B64EncryptionResult
) {
const metaData = this.metadataMap.get(fileName);
const fileKey = await worker.decryptB64(
encryptedFileKey.encryptedData,
encryptedFileKey.nonce,
encryptedFileKey.key
);
const encryptedMetaData = await worker.encryptMetadata(
metaData,
fileKey
);
return encryptedMetaData;
}
private async uploadtoBucket(
file: encryptedFile,
token,
count: number
): Promise<objectKeys> {
file: ProcessedFile,
token
): Promise<BackupedFile> {
try {
const fileUploadURL = await this.getUploadURL(token, count);
const fileObjectKey = await this.putFile(
const fileUploadURL = await this.getUploadURL(token);
file.file.objectKey = await this.putFile(
fileUploadURL,
file.filedata.encryptedData
file.file.encryptedData
);
const thumbnailUploadURL = await this.getUploadURL(token, count);
const thumbnailObjectKey = await this.putFile(
const thumbnailUploadURL = await this.getUploadURL(token);
file.thumbnail.objectKey = await this.putFile(
thumbnailUploadURL,
file.thumbnail.encryptedData
);
delete file.file.encryptedData;
delete file.thumbnail.encryptedData;
return {
file: {
objectKey: fileObjectKey,
decryptionHeader: file.filedata.decryptionHeader,
},
thumbnail: {
objectKey: thumbnailObjectKey,
decryptionHeader: file.thumbnail.decryptionHeader,
},
};
return file;
} catch (e) {
console.log('error uploading to bucket ' + e);
throw e;
@ -340,14 +298,14 @@ class UploadService {
private getuploadFile(
collection: collection,
encryptedKey: B64EncryptionResult,
objectKeys: objectKeys
backupedFile: BackupedFile,
fileKey: B64EncryptionResult
): uploadFile {
const uploadFile: uploadFile = {
collectionID: collection.id,
encryptedKey: encryptedKey.encryptedData,
keyDecryptionNonce: encryptedKey.nonce,
...objectKeys,
encryptedKey: fileKey.encryptedData,
keyDecryptionNonce: fileKey.nonce,
...backupedFile,
};
return uploadFile;
}
@ -367,7 +325,7 @@ class UploadService {
}
}
private async updateMetadata(recievedFile: File) {
private async seedMetadataMap(recievedFile: File) {
try {
const metadataJSON: object = await new Promise(
(resolve, reject) => {
@ -429,13 +387,15 @@ class UploadService {
image.setAttribute('src', imageURL);
await new Promise((resolve) => {
image.onload = () => {
canvas.width = THUMBNAIL_WIDTH;
const thumbnailWidth =
(image.width * THUMBNAIL_HEIGHT) / image.height;
canvas.width = thumbnailWidth;
canvas.height = THUMBNAIL_HEIGHT;
canvas_CTX.drawImage(
image,
0,
0,
THUMBNAIL_WIDTH,
thumbnailWidth,
THUMBNAIL_HEIGHT
);
image = undefined;
@ -447,13 +407,16 @@ class UploadService {
let video = document.createElement('video');
imageURL = URL.createObjectURL(file);
video.addEventListener('loadeddata', function () {
canvas.width = THUMBNAIL_WIDTH;
const thumbnailWidth =
(video.videoWidth * THUMBNAIL_HEIGHT) /
video.videoHeight;
canvas.width = thumbnailWidth;
canvas.height = THUMBNAIL_HEIGHT;
canvas_CTX.drawImage(
video,
0,
0,
THUMBNAIL_WIDTH,
thumbnailWidth,
THUMBNAIL_HEIGHT
);
resolve(null);
@ -520,20 +483,23 @@ class UploadService {
}
}
private async getUploadURL(token: string, count: number) {
private async getUploadURL(token: string) {
if (this.uploadURLs.length == 0) {
await this.fetchUploadURLs(token, count);
await this.fetchUploadURLs(token);
}
return this.uploadURLs.pop();
}
private async fetchUploadURLs(token: string, count: number): Promise<void> {
private async fetchUploadURLs(token: string): Promise<void> {
try {
if (!this.uploadURLFetchInProgress) {
this.uploadURLFetchInProgress = HTTPService.get(
`${ENDPOINT}/files/upload-urls`,
{
count: Math.min(50, count).toString(), //m4gic number
count: Math.min(
50,
(this.filesToBeUploaded.length + 1) * 2
).toString(),
},
{ 'X-Auth-Token': token }
);
@ -550,7 +516,7 @@ class UploadService {
}
private async putFile(
fileUploadURL: uploadURL,
fileUploadURL: UploadURL,
file: Uint8Array | string
): Promise<string> {
try {
@ -586,10 +552,10 @@ class UploadService {
}
}
private getUNIXTime(exifData: any) {
if (!exifData.DateTimeOriginal) {
let dateString: string = exifData.DateTimeOriginal || exifData.DateTime;
if (!dateString) {
return null;
}
let dateString: string = exifData.DateTimeOriginal;
var parts = dateString.split(' ')[0].split(':');
var date = new Date(
Number(parts[0]),
@ -603,10 +569,28 @@ class UploadService {
if (!exifData.GPSLatitude) {
return null;
}
var latDegree = exifData.GPSLatitude[0].numerator;
var latMinute = exifData.GPSLatitude[1].numerator;
var latSecond = exifData.GPSLatitude[2].numerator;
let latDegree: number, latMinute: number, latSecond: number;
let lonDegree: number, lonMinute: number, lonSecond: number;
if (exifData.GPSLatitude[0].numerator) {
latDegree = exifData.GPSLatitude[0].numerator;
latMinute = exifData.GPSLatitude[1].numerator;
latSecond = exifData.GPSLatitude[2].numerator;
lonDegree = exifData.GPSLongitude[0].numerator;
lonMinute = exifData.GPSLongitude[1].numerator;
lonSecond = exifData.GPSLongitude[2].numerator;
} else {
latDegree = exifData.GPSLatitude[0];
latMinute = exifData.GPSLatitude[1];
latSecond = exifData.GPSLatitude[2];
lonDegree = exifData.GPSLongitude[0];
lonMinute = exifData.GPSLongitude[1];
lonSecond = exifData.GPSLongitude[2];
}
var latDirection = exifData.GPSLatitudeRef;
var lonDirection = exifData.GPSLongitudeRef;
var latFinal = this.convertDMSToDD(
latDegree,
@ -615,19 +599,12 @@ class UploadService {
latDirection
);
// Calculate longitude decimal
var lonDegree = exifData.GPSLongitude[0].numerator;
var lonMinute = exifData.GPSLongitude[1].numerator;
var lonSecond = exifData.GPSLongitude[2].numerator;
var lonDirection = exifData.GPSLongitudeRef;
var lonFinal = this.convertDMSToDD(
lonDegree,
lonMinute,
lonSecond,
lonDirection
);
return { latitude: latFinal * 1.0, longitude: lonFinal * 1.0 };
}

4
src/utils/crypto/libsodium.ts
View file

@ -77,8 +77,6 @@ export async function encryptChaChaOneShot(data: Uint8Array, key?: string) {
file: {
encryptedData: pushResult,
decryptionHeader: await toB64(header),
creationTime: Date.now(),
fileType: 0,
},
};
}
@ -123,8 +121,6 @@ export async function encryptChaCha(data: Uint8Array, key?: string) {
file: {
encryptedData: new Uint8Array(encryptedData),
decryptionHeader: await toB64(header),
creationTime: Date.now(),
fileType: 0,
},
};
}