moved file reading ot worker
This commit is contained in:
parent
9270d80d24
commit
9b49e15d5c
|
@ -82,8 +82,11 @@ class DownloadManager {
|
||||||
|
|
||||||
const typeFromExtension =
|
const typeFromExtension =
|
||||||
file.metadata.title.split('.')[-1];
|
file.metadata.title.split('.')[-1];
|
||||||
|
const worker = await new CryptoWorker();
|
||||||
|
|
||||||
const mimeType =
|
const mimeType =
|
||||||
(await getMimeTypeFromBlob(
|
(await getMimeTypeFromBlob(
|
||||||
|
worker,
|
||||||
new FileReader(),
|
new FileReader(),
|
||||||
fileBlob
|
fileBlob
|
||||||
)) ?? typeFromExtension;
|
)) ?? typeFromExtension;
|
||||||
|
|
|
@ -8,20 +8,25 @@ const TYPE_IMAGE = 'image';
|
||||||
const EDITED_FILE_SUFFIX = '-edited';
|
const EDITED_FILE_SUFFIX = '-edited';
|
||||||
const CHUNK_SIZE_FOR_TYPE_DETECTION = 4100;
|
const CHUNK_SIZE_FOR_TYPE_DETECTION = 4100;
|
||||||
|
|
||||||
export async function getFileData(reader: FileReader, file: globalThis.File) {
|
export async function getFileData(
|
||||||
|
worker,
|
||||||
|
reader: FileReader,
|
||||||
|
file: globalThis.File
|
||||||
|
) {
|
||||||
if (file.size > MULTIPART_PART_SIZE) {
|
if (file.size > MULTIPART_PART_SIZE) {
|
||||||
return getFileStream(reader, file, FILE_READER_CHUNK_SIZE);
|
return getFileStream(worker, reader, file, FILE_READER_CHUNK_SIZE);
|
||||||
} else {
|
} else {
|
||||||
return await getUint8ArrayView(reader, file);
|
return await worker.getUint8ArrayView(file);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function getFileType(
|
export async function getFileType(
|
||||||
|
worker,
|
||||||
reader: FileReader,
|
reader: FileReader,
|
||||||
receivedFile: globalThis.File
|
receivedFile: globalThis.File
|
||||||
) {
|
) {
|
||||||
let fileType: FILE_TYPE;
|
let fileType: FILE_TYPE;
|
||||||
const mimeType = await getMimeType(reader, receivedFile);
|
const mimeType = await getMimeType(worker, reader, receivedFile);
|
||||||
const majorType = mimeType?.split('/')[0].toLowerCase();
|
const majorType = mimeType?.split('/')[0].toLowerCase();
|
||||||
switch (majorType) {
|
switch (majorType) {
|
||||||
case TYPE_IMAGE:
|
case TYPE_IMAGE:
|
||||||
|
@ -52,23 +57,33 @@ export function getFileOriginalName(file: globalThis.File) {
|
||||||
return originalName;
|
return originalName;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function getMimeType(reader: FileReader, file: globalThis.File) {
|
async function getMimeType(worker, reader: FileReader, file: globalThis.File) {
|
||||||
const fileChunkBlob = file.slice(0, CHUNK_SIZE_FOR_TYPE_DETECTION);
|
const fileChunkBlob = file.slice(0, CHUNK_SIZE_FOR_TYPE_DETECTION);
|
||||||
return getMimeTypeFromBlob(reader, fileChunkBlob);
|
return getMimeTypeFromBlob(worker, reader, fileChunkBlob);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function getMimeTypeFromBlob(reader: FileReader, fileBlob: Blob) {
|
export async function getMimeTypeFromBlob(
|
||||||
const initialFiledata = await getUint8ArrayView(reader, fileBlob);
|
worker,
|
||||||
|
reader: FileReader,
|
||||||
|
fileBlob: Blob
|
||||||
|
) {
|
||||||
|
const initialFiledata = await worker.getUint8ArrayView(fileBlob);
|
||||||
const result = await FileType.fromBuffer(initialFiledata);
|
const result = await FileType.fromBuffer(initialFiledata);
|
||||||
return result?.mime;
|
return result?.mime;
|
||||||
}
|
}
|
||||||
|
|
||||||
function getFileStream(
|
function getFileStream(
|
||||||
|
worker,
|
||||||
reader: FileReader,
|
reader: FileReader,
|
||||||
file: globalThis.File,
|
file: globalThis.File,
|
||||||
chunkSize: number
|
chunkSize: number
|
||||||
) {
|
) {
|
||||||
const fileChunkReader = fileChunkReaderMaker(reader, file, chunkSize);
|
const fileChunkReader = fileChunkReaderMaker(
|
||||||
|
worker,
|
||||||
|
reader,
|
||||||
|
file,
|
||||||
|
chunkSize
|
||||||
|
);
|
||||||
|
|
||||||
const stream = new ReadableStream<Uint8Array>({
|
const stream = new ReadableStream<Uint8Array>({
|
||||||
async pull(controller: ReadableStreamDefaultController) {
|
async pull(controller: ReadableStreamDefaultController) {
|
||||||
|
@ -88,6 +103,7 @@ function getFileStream(
|
||||||
}
|
}
|
||||||
|
|
||||||
async function* fileChunkReaderMaker(
|
async function* fileChunkReaderMaker(
|
||||||
|
worker,
|
||||||
reader: FileReader,
|
reader: FileReader,
|
||||||
file: globalThis.File,
|
file: globalThis.File,
|
||||||
chunkSize: number
|
chunkSize: number
|
||||||
|
@ -95,7 +111,7 @@ async function* fileChunkReaderMaker(
|
||||||
let offset = 0;
|
let offset = 0;
|
||||||
while (offset < file.size) {
|
while (offset < file.size) {
|
||||||
const blob = file.slice(offset, chunkSize + offset);
|
const blob = file.slice(offset, chunkSize + offset);
|
||||||
const fileChunk = await getUint8ArrayView(reader, blob);
|
const fileChunk = await worker.getUint8ArrayView(blob);
|
||||||
yield fileChunk;
|
yield fileChunk;
|
||||||
offset += chunkSize;
|
offset += chunkSize;
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,7 +2,6 @@ import { FILE_TYPE } from 'services/fileService';
|
||||||
import { CustomError } from 'utils/common/errorUtil';
|
import { CustomError } from 'utils/common/errorUtil';
|
||||||
import { convertHEIC2JPEG } from 'utils/file';
|
import { convertHEIC2JPEG } from 'utils/file';
|
||||||
import { logError } from 'utils/sentry';
|
import { logError } from 'utils/sentry';
|
||||||
import { getUint8ArrayView } from './readFileService';
|
|
||||||
|
|
||||||
const THUMBNAIL_HEIGHT = 720;
|
const THUMBNAIL_HEIGHT = 720;
|
||||||
const MAX_ATTEMPTS = 3;
|
const MAX_ATTEMPTS = 3;
|
||||||
|
@ -11,6 +10,7 @@ const MIN_THUMBNAIL_SIZE = 50000;
|
||||||
const WAIT_TIME_THUMBNAIL_GENERATION = 10 * 1000;
|
const WAIT_TIME_THUMBNAIL_GENERATION = 10 * 1000;
|
||||||
|
|
||||||
export async function generateThumbnail(
|
export async function generateThumbnail(
|
||||||
|
worker,
|
||||||
reader: FileReader,
|
reader: FileReader,
|
||||||
file: globalThis.File,
|
file: globalThis.File,
|
||||||
fileType: FILE_TYPE,
|
fileType: FILE_TYPE,
|
||||||
|
@ -31,7 +31,7 @@ export async function generateThumbnail(
|
||||||
hasStaticThumbnail = true;
|
hasStaticThumbnail = true;
|
||||||
}
|
}
|
||||||
const thumbnailBlob = await thumbnailCanvasToBlob(canvas);
|
const thumbnailBlob = await thumbnailCanvasToBlob(canvas);
|
||||||
const thumbnail = await getUint8ArrayView(reader, thumbnailBlob);
|
const thumbnail = await worker.getUint8ArrayView(thumbnailBlob);
|
||||||
return { thumbnail, hasStaticThumbnail };
|
return { thumbnail, hasStaticThumbnail };
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
logError(e, 'Error generating thumbnail');
|
logError(e, 'Error generating thumbnail');
|
||||||
|
|
|
@ -99,11 +99,16 @@ class UploadService {
|
||||||
}
|
}
|
||||||
|
|
||||||
async readFile(
|
async readFile(
|
||||||
|
worker,
|
||||||
reader: FileReader,
|
reader: FileReader,
|
||||||
rawFile: globalThis.File,
|
rawFile: globalThis.File,
|
||||||
collection: Collection
|
collection: Collection
|
||||||
): Promise<FileInMemory> {
|
): Promise<FileInMemory> {
|
||||||
const { fileType, mimeType } = await getFileType(reader, rawFile);
|
const { fileType, mimeType } = await getFileType(
|
||||||
|
worker,
|
||||||
|
reader,
|
||||||
|
rawFile
|
||||||
|
);
|
||||||
if (fileType === FILE_TYPE.OTHERS) {
|
if (fileType === FILE_TYPE.OTHERS) {
|
||||||
throw Error(CustomError.UNSUPPORTED_FILE_FORMAT);
|
throw Error(CustomError.UNSUPPORTED_FILE_FORMAT);
|
||||||
}
|
}
|
||||||
|
@ -111,6 +116,7 @@ class UploadService {
|
||||||
const isHEIC = fileIsHEIC(mimeType);
|
const isHEIC = fileIsHEIC(mimeType);
|
||||||
|
|
||||||
const { thumbnail, hasStaticThumbnail } = await generateThumbnail(
|
const { thumbnail, hasStaticThumbnail } = await generateThumbnail(
|
||||||
|
worker,
|
||||||
reader,
|
reader,
|
||||||
rawFile,
|
rawFile,
|
||||||
fileType,
|
fileType,
|
||||||
|
@ -137,7 +143,7 @@ class UploadService {
|
||||||
extractedMetadata[key] = value;
|
extractedMetadata[key] = value;
|
||||||
}
|
}
|
||||||
|
|
||||||
const filedata = await getFileData(reader, rawFile);
|
const filedata = await getFileData(worker, reader, rawFile);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
filedata,
|
filedata,
|
||||||
|
|
|
@ -33,7 +33,12 @@ export default async function uploader(
|
||||||
let file: FileInMemory = null;
|
let file: FileInMemory = null;
|
||||||
let encryptedFile: EncryptedFile = null;
|
let encryptedFile: EncryptedFile = null;
|
||||||
try {
|
try {
|
||||||
file = await UploadService.readFile(reader, rawFile, collection);
|
file = await UploadService.readFile(
|
||||||
|
worker,
|
||||||
|
reader,
|
||||||
|
rawFile,
|
||||||
|
collection
|
||||||
|
);
|
||||||
|
|
||||||
if (fileAlreadyInCollection(existingFilesInCollection, file)) {
|
if (fileAlreadyInCollection(existingFilesInCollection, file)) {
|
||||||
UIService.setFileProgress(rawFile.name, FileUploadResults.SKIPPED);
|
UIService.setFileProgress(rawFile.name, FileUploadResults.SKIPPED);
|
||||||
|
@ -68,6 +73,7 @@ export default async function uploader(
|
||||||
file: decryptedFile,
|
file: decryptedFile,
|
||||||
};
|
};
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
console.log(e);
|
||||||
logError(e, 'file upload failed');
|
logError(e, 'file upload failed');
|
||||||
handleError(e);
|
handleError(e);
|
||||||
switch (e.message) {
|
switch (e.message) {
|
||||||
|
|
|
@ -147,6 +147,29 @@ export class Crypto {
|
||||||
async fromHex(string) {
|
async fromHex(string) {
|
||||||
return libsodium.fromHex(string);
|
return libsodium.fromHex(string);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async getUint8ArrayView(file) {
|
||||||
|
try {
|
||||||
|
return await new Promise((resolve, reject) => {
|
||||||
|
const reader = new FileReader();
|
||||||
|
reader.onabort = () =>
|
||||||
|
reject(Error('file reading was aborted'));
|
||||||
|
reader.onerror = () => reject(Error('file reading has failed'));
|
||||||
|
reader.onload = () => {
|
||||||
|
// Do whatever you want with the file contents
|
||||||
|
const result =
|
||||||
|
typeof reader.result === 'string'
|
||||||
|
? new TextEncoder().encode(reader.result)
|
||||||
|
: new Uint8Array(reader.result);
|
||||||
|
resolve(result);
|
||||||
|
};
|
||||||
|
reader.readAsArrayBuffer(file);
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
console.log(e, 'error reading file to byte-array');
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Comlink.expose(Crypto);
|
Comlink.expose(Crypto);
|
||||||
|
|
Loading…
Reference in a new issue