beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

move to arrayBuffer instead of readAsArrayBuffer

Browse source
This commit is contained in:
Rushikesh Tote 2022-05-23 14:41:07 +05:30
parent a463a37456
commit a740ac2e36
14 changed files with 39 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/services/ffmpeg/ffmpegClient.ts
View file

@ -4,7 +4,6 @@ import { parseFFmpegExtractedMetadata } from 'utils/ffmpeg';
class FFmpegClient {
private ffmpeg: FFmpeg;
private fileReader: FileReader;
private ready: Promise<void> = null;
constructor() {
this.ffmpeg = createFFmpeg({
@ -19,9 +18,6 @@ class FFmpegClient {
if (!this.ffmpeg.isLoaded()) {
await this.ffmpeg.load();
}
if (!this.fileReader) {
this.fileReader = new FileReader();
}
}
async generateThumbnail(file: File) {
@ -31,7 +27,7 @@ class FFmpegClient {
this.ffmpeg.FS(
'writeFile',
inputFileName,
await getUint8ArrayView(this.fileReader, file)
await getUint8ArrayView(file)
);
let seekTime = 1.0;
let thumb = null;
@ -66,7 +62,7 @@ class FFmpegClient {
this.ffmpeg.FS(
'writeFile',
inputFileName,
await getUint8ArrayView(this.fileReader, file)
await getUint8ArrayView(file)
);
let metadata = null;

4
src/services/migrateThumbnailService.ts
View file

@ -44,7 +44,6 @@ export async function replaceThumbnail(
try {
const token = getToken();
const worker = await new CryptoWorker();
const reader = new FileReader();
const files = await getLocalFiles();
const trash = await getLocalTrash();
const trashFiles = getTrashedFiles(trash);
@ -77,9 +76,8 @@ export async function replaceThumbnail(
[originalThumbnail],
file.metadata.title
);
const fileTypeInfo = await getFileType(reader, dummyImageFile);
const fileTypeInfo = await getFileType(dummyImageFile);
const { thumbnail: newThumbnail } = await generateThumbnail(
reader,
dummyImageFile,
fileTypeInfo
);

49
src/services/readerService.ts
View file

@ -1,44 +1,11 @@
import { ElectronFile } from 'types/upload';
export async function getUint8ArrayView(
reader: FileReader,
file: Blob
): Promise<Uint8Array> {
return await new Promise((resolve, reject) => {
reader.onabort = () =>
reject(
Error(
`file reading was aborted, file size= ${convertBytesToHumanReadable(
file.size
)}`
)
);
reader.onerror = () =>
reject(
Error(
`file reading has failed, file size= ${convertBytesToHumanReadable(
file.size
)} , reason= ${reader.error}`
)
);
reader.onload = () => {
// Do whatever you want with the file contents
const result =
typeof reader.result === 'string'
? new TextEncoder().encode(reader.result)
: new Uint8Array(reader.result);
resolve(result);
};
reader.readAsArrayBuffer(file);
});
export async function getUint8ArrayView(file: Blob): Promise<Uint8Array> {
return new Uint8Array(await file.arrayBuffer());
}
export function getFileStream(
reader: FileReader,
file: File,
chunkSize: number
) {
const fileChunkReader = fileChunkReaderMaker(reader, file, chunkSize);
export function getFileStream(file: File, chunkSize: number) {
const fileChunkReader = fileChunkReaderMaker(file, chunkSize);
const stream = new ReadableStream<Uint8Array>({
async pull(controller: ReadableStreamDefaultController) {
@ -68,15 +35,11 @@ export async function getElectronFileStream(
};
}
async function* fileChunkReaderMaker(
reader: FileReader,
file: File,
chunkSize: number
) {
async function* fileChunkReaderMaker(file: File, chunkSize: number) {
let offset = 0;
while (offset < file.size) {
const blob = file.slice(offset, chunkSize + offset);
const fileChunk = await getUint8ArrayView(reader, blob);
const fileChunk = await getUint8ArrayView(blob);
yield fileChunk;
offset += chunkSize;
}

11
src/services/typeDetectionService.ts
View file

@ -12,7 +12,6 @@ const TYPE_IMAGE = 'image';
const CHUNK_SIZE_FOR_TYPE_DETECTION = 4100;
export async function getFileType(
reader: FileReader,
receivedFile: File | ElectronFile
): Promise<FileTypeInfo> {
try {
@ -20,7 +19,7 @@ export async function getFileType(
let typeResult: FileTypeResult;
if (receivedFile instanceof File) {
typeResult = await extractFileType(reader, receivedFile);
typeResult = await extractFileType(receivedFile);
} else {
typeResult = await extractElectronFileType(receivedFile);
}
@ -64,9 +63,9 @@ export async function getFileType(
}
}
async function extractFileType(reader: FileReader, file: File) {
async function extractFileType(file: File) {
const fileChunkBlob = file.slice(0, CHUNK_SIZE_FOR_TYPE_DETECTION);
return getFileTypeFromBlob(reader, fileChunkBlob);
return getFileTypeFromBlob(fileChunkBlob);
}
async function extractElectronFileType(file: ElectronFile) {
@ -77,9 +76,9 @@ async function extractElectronFileType(file: ElectronFile) {
return fileTypeResult;
}
async function getFileTypeFromBlob(reader: FileReader, fileBlob: Blob) {
async function getFileTypeFromBlob(fileBlob: Blob) {
try {
const initialFiledata = await getUint8ArrayView(reader, fileBlob);
const initialFiledata = await getUint8ArrayView(fileBlob);
return await FileType.fromBuffer(initialFiledata);
} catch (e) {
throw Error(CustomError.TYPE_DETECTION_FAILED);

3
src/services/updateCreationTimeWithExif.ts
View file

@ -38,8 +38,7 @@ export async function updateCreationTimeWithExif(
} else {
const fileURL = await downloadManager.getFile(file)[0];
const fileObject = await getFileFromURL(fileURL);
const reader = new FileReader();
const fileTypeInfo = await getFileType(reader, fileObject);
const fileTypeInfo = await getFileType(fileObject);
const exifData = await getRawExif(fileObject, fileTypeInfo);
if (fixOption === FIX_OPTIONS.DATE_TIME_ORIGINAL) {
correctCreationTime = getUnixTimeInMicroSeconds(

12
src/services/upload/fileService.ts
View file

@ -34,12 +34,10 @@ export function getFilename(file: File | ElectronFile) {
}
export async function readFile(
reader: FileReader,
fileTypeInfo: FileTypeInfo,
rawFile: File | ElectronFile
): Promise<FileInMemory> {
const { thumbnail, hasStaticThumbnail } = await generateThumbnail(
reader,
rawFile,
fileTypeInfo
);
@ -55,9 +53,9 @@ export async function readFile(
filedata = await rawFile.arrayBuffer();
}
} else if (rawFile.size > MULTIPART_PART_SIZE) {
filedata = getFileStream(reader, rawFile, FILE_READER_CHUNK_SIZE);
filedata = getFileStream(rawFile, FILE_READER_CHUNK_SIZE);
} else {
filedata = await getUint8ArrayView(reader, rawFile);
filedata = await getUint8ArrayView(rawFile);
}
logUploadInfo(`read file data successfully ${getFileNameSize(rawFile)} `);
@ -73,8 +71,7 @@ export async function extractFileMetadata(
parsedMetadataJSONMap: ParsedMetadataJSONMap,
rawFile: File | ElectronFile,
collectionID: number,
fileTypeInfo: FileTypeInfo,
reader: FileReader
fileTypeInfo: FileTypeInfo
) {
const originalName = getFileOriginalName(rawFile);
const googleMetadata =
@ -83,8 +80,7 @@ export async function extractFileMetadata(
) ?? {};
const extractedMetadata: Metadata = await extractMetadata(
rawFile,
fileTypeInfo,
reader
fileTypeInfo
);
for (const [key, value] of Object.entries(googleMetadata)) {

6
src/services/upload/livePhotoService.ts
View file

@ -72,12 +72,10 @@ export function getLivePhotoName(imageTitle: string) {
}
export async function readLivePhoto(
reader: FileReader,
fileTypeInfo: FileTypeInfo,
livePhotoAssets: LivePhotoAssets
) {
const { thumbnail, hasStaticThumbnail } = await generateThumbnail(
reader,
livePhotoAssets.image,
{
exactType: fileTypeInfo.imageType,
@ -87,12 +85,12 @@ export async function readLivePhoto(
const image =
livePhotoAssets.image instanceof File
? await getUint8ArrayView(reader, livePhotoAssets.image)
? await getUint8ArrayView(livePhotoAssets.image)
: await livePhotoAssets.image.arrayBuffer();
const video =
livePhotoAssets.video instanceof File
? await getUint8ArrayView(reader, livePhotoAssets.video)
? await getUint8ArrayView(livePhotoAssets.video)
: await livePhotoAssets.video.arrayBuffer();
return {

5
src/services/upload/metadataService.ts
View file

@ -33,8 +33,7 @@ const NULL_PARSED_METADATA_JSON: ParsedMetadataJSON = {
export async function extractMetadata(
receivedFile: File | ElectronFile,
fileTypeInfo: FileTypeInfo,
reader: FileReader
fileTypeInfo: FileTypeInfo
) {
let extractedMetadata: ParsedExtractedMetadata = NULL_EXTRACTED_METADATA;
if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) {
@ -51,7 +50,7 @@ export async function extractMetadata(
);
}
const fileHash = await getFileHash(receivedFile, reader);
const fileHash = await getFileHash(receivedFile);
const metadata: Metadata = {
title: receivedFile.name,

3
src/services/upload/thumbnailService.ts
View file

@ -24,7 +24,6 @@ interface Dimension {
}
export async function generateThumbnail(
reader: FileReader,
file: File | ElectronFile,
fileTypeInfo: FileTypeInfo
): Promise<{ thumbnail: Uint8Array; hasStaticThumbnail: boolean }> {
@ -72,7 +71,7 @@ export async function generateThumbnail(
}
}
const thumbnailBlob = await thumbnailCanvasToBlob(canvas);
thumbnail = await getUint8ArrayView(reader, thumbnailBlob);
thumbnail = await getUint8ArrayView(thumbnailBlob);
if (thumbnail.length === 0) {
throw Error('EMPTY THUMBNAIL');
}

11
src/services/upload/uploadManager.ts
View file

@ -201,7 +201,6 @@ class UploadManager {
try {
logUploadInfo(`extractMetadataFromFiles executed`);
UIService.reset(mediaFiles.length);
const reader = new FileReader();
for (const { file, localID, collectionID } of mediaFiles) {
try {
const { fileTypeInfo, metadata } = await (async () => {
@ -215,7 +214,6 @@ class UploadManager {
return { fileTypeInfo: null, metadata: null };
}
const fileTypeInfo = await UploadService.getFileType(
reader,
file
);
if (fileTypeInfo.fileType === FILE_TYPE.OTHERS) {
@ -233,8 +231,7 @@ class UploadManager {
(await UploadService.extractFileMetadata(
file,
collectionID,
fileTypeInfo,
reader
fileTypeInfo
)) || null;
return { fileTypeInfo, metadata };
})();
@ -291,15 +288,14 @@ class UploadManager {
this.cryptoWorkers[i] = cryptoWorker;
uploadProcesses.push(
this.uploadNextFileInQueue(
await new this.cryptoWorkers[i].comlink(),
new FileReader()
await new this.cryptoWorkers[i].comlink()
)
);
}
await Promise.all(uploadProcesses);
}
private async uploadNextFileInQueue(worker: any, reader: FileReader) {
private async uploadNextFileInQueue(worker: any) {
while (this.filesToBeUploaded.length > 0) {
let fileWithCollection = this.filesToBeUploaded.pop();
const { collectionID } = fileWithCollection;
@ -310,7 +306,6 @@ class UploadManager {
const { fileUploadResult, uploadedFile, skipDecryption } =
await uploader(
worker,
reader,
existingFilesInCollection,
this.existingFiles,
fileWithCollection

15
src/services/upload/uploadService.ts
View file

@ -76,32 +76,29 @@ class UploadService {
: getFilename(file);
}
async getFileType(reader: FileReader, file: File | ElectronFile) {
return getFileType(reader, file);
async getFileType(file: File | ElectronFile) {
return getFileType(file);
}
async readAsset(
reader: FileReader,
fileTypeInfo: FileTypeInfo,
{ isLivePhoto, file, livePhotoAssets }: UploadAsset
) {
return isLivePhoto
? await readLivePhoto(reader, fileTypeInfo, livePhotoAssets)
: await readFile(reader, fileTypeInfo, file);
? await readLivePhoto(fileTypeInfo, livePhotoAssets)
: await readFile(fileTypeInfo, file);
}
async extractFileMetadata(
file: File | ElectronFile,
collectionID: number,
fileTypeInfo: FileTypeInfo,
reader: FileReader
fileTypeInfo: FileTypeInfo
): Promise<Metadata> {
return extractFileMetadata(
this.parsedMetadataJSONMap,
file,
collectionID,
fileTypeInfo,
reader
fileTypeInfo
);
}

7
src/services/upload/uploader.ts
View file

@ -24,7 +24,6 @@ interface UploadResponse {
}
export default async function uploader(
worker: any,
reader: FileReader,
existingFilesInCollection: EnteFile[],
existingFiles: EnteFile[],
fileWithCollection: FileWithCollection
@ -87,11 +86,7 @@ export default async function uploader(
}
logUploadInfo(`reading asset ${fileNameSize}`);
const file = await UploadService.readAsset(
reader,
fileTypeInfo,
uploadAsset
);
const file = await UploadService.readAsset(fileTypeInfo, uploadAsset);
if (file.hasStaticThumbnail) {
metadata.hasStaticThumbnail = true;

7
src/utils/crypto/index.ts
View file

@ -201,13 +201,10 @@ export async function encryptWithRecoveryKey(key: string) {
}
export default CryptoWorker;
export async function getFileHash(
file: File | ElectronFile,
reader: FileReader
) {
export async function getFileHash(file: File | ElectronFile) {
let filedata: DataStream;
if (file instanceof File) {
filedata = getFileStream(reader, file, FILE_READER_CHUNK_SIZE);
filedata = getFileStream(file, FILE_READER_CHUNK_SIZE);
} else {
filedata = await getElectronFileStream(file, FILE_READER_CHUNK_SIZE);
}

8
src/utils/file/index.ts
View file

@ -77,7 +77,6 @@ export async function downloadFile(
}
const fileType = await getFileType(
fileReader,
new File([fileBlob], file.metadata.title)
);
if (
@ -98,12 +97,12 @@ export async function downloadFile(
const originalName = fileNameWithoutExtension(file.metadata.title);
const motionPhoto = await decodeMotionPhoto(fileBlob, originalName);
const image = new File([motionPhoto.image], motionPhoto.imageNameTitle);
const imageType = await getFileType(fileReader, image);
const imageType = await getFileType(image);
tempImageURL = URL.createObjectURL(
new Blob([motionPhoto.image], { type: imageType.mimeType })
);
const video = new File([motionPhoto.video], motionPhoto.videoNameTitle);
const videoType = await getFileType(fileReader, video);
const videoType = await getFileType(video);
tempVideoURL = URL.createObjectURL(
new Blob([motionPhoto.video], { type: videoType.mimeType })
);
@ -338,9 +337,8 @@ export async function convertForPreview(
fileBlob: Blob
): Promise<Blob[]> {
const convertIfHEIC = async (fileName: string, fileBlob: Blob) => {
const reader = new FileReader();
const mimeType = (
await getFileType(reader, new File([fileBlob], file.metadata.title))
await getFileType(new File([fileBlob], file.metadata.title))
).exactType;
if (isFileHEIC(mimeType)) {
fileBlob = await HEICConverter.convert(fileBlob);