beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #281 from ente-io/reuse-file-readers

Browse source 
reuse file readers
This commit is contained in:
abhinavkgrd 2022-01-09 18:00:51 +05:30 committed by GitHub
parent b504e072f3 d42af9cd1a
commit 71a97caa03
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 73 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
src/services/exportService.ts
View file

@ -62,6 +62,7 @@ class ExportService {
private stopExport: boolean = false;
private pauseExport: boolean = false;
private allElectronAPIsExist: boolean = false;
private fileReader: FileReader = null;
constructor() {
this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs'];
@ -438,7 +439,11 @@ class ExportService {
(fileType === TYPE_JPEG || fileType === TYPE_JPG)
) {
const fileBlob = await new Response(fileStream).blob();
if (!this.fileReader) {
this.fileReader = new FileReader();
}
const updatedFileBlob = await updateFileCreationDateInEXIF(
this.fileReader,
fileBlob,
new Date(file.pubMagicMetadata.data.editedTime / 1000)
);

19
src/services/ffmpegService.ts
View file

@ -7,6 +7,7 @@ import { getUint8ArrayView } from './upload/readFileService';
class FFmpegService {
private ffmpeg: FFmpeg = null;
private isLoading = null;
private fileReader: FileReader = null;
private generateThumbnailProcessor = new QueueProcessor<Uint8Array>(1);
async init() {
@ -29,11 +30,19 @@ class FFmpegService {
if (!this.ffmpeg) {
await this.init();
}
if (!this.fileReader) {
this.fileReader = new FileReader();
}
if (this.isLoading) {
await this.isLoading;
}
const response = this.generateThumbnailProcessor.queueUpRequest(
generateThumbnailHelper.bind(null, this.ffmpeg, file)
generateThumbnailHelper.bind(
null,
this.ffmpeg,
this.fileReader,
file
)
);
try {
return await response.promise;
@ -49,14 +58,18 @@ class FFmpegService {
}
}
async function generateThumbnailHelper(ffmpeg: FFmpeg, file: File) {
async function generateThumbnailHelper(
ffmpeg: FFmpeg,
reader: FileReader,
file: File
) {
try {
const inputFileName = `${Date.now().toString()}-${file.name}`;
const thumbFileName = `${Date.now().toString()}-thumb.jpeg`;
ffmpeg.FS(
'writeFile',
inputFileName,
await getUint8ArrayView(new FileReader(), file)
await getUint8ArrayView(reader, file)
);
let seekTime = 1.0;
let thumb = null;

4
src/services/migrateThumbnailService.ts
View file

@ -44,6 +44,7 @@ export async function replaceThumbnail(
try {
const token = getToken();
const worker = await new CryptoWorker();
const reader = new FileReader();
const files = await getLocalFiles();
const trash = await getLocalTrash();
const trashFiles = getTrashedFiles(trash);
@ -76,9 +77,10 @@ export async function replaceThumbnail(
[originalThumbnail],
file.metadata.title
);
const fileTypeInfo = await getFileType(worker, dummyImageFile);
const fileTypeInfo = await getFileType(reader, dummyImageFile);
const { thumbnail: newThumbnail } = await generateThumbnail(
worker,
reader,
dummyImageFile,
fileTypeInfo
);

5
src/services/updateCreationTimeWithExif.ts
View file

@ -1,6 +1,5 @@
import { FIX_OPTIONS } from 'components/FixCreationTime';
import { SetProgressTracker } from 'components/FixLargeThumbnail';
import CryptoWorker from 'utils/crypto';
import {
changeFileCreationTime,
getFileFromURL,
@ -38,8 +37,8 @@ export async function updateCreationTimeWithExif(
} else {
const fileURL = await downloadManager.getFile(file);
const fileObject = await getFileFromURL(fileURL);
const worker = await new CryptoWorker();
const fileTypeInfo = await getFileType(worker, fileObject);
const reader = new FileReader();
const fileTypeInfo = await getFileType(reader, fileObject);
const exifData = await getRawExif(fileObject, fileTypeInfo);
if (fixOption === FIX_OPTIONS.DATE_TIME_ORIGINAL) {
correctCreationTime = getUNIXTime(

6
src/services/upload/exifService.ts
View file

@ -57,12 +57,13 @@ export async function getExifData(
}
export async function updateFileCreationDateInEXIF(
reader: FileReader,
fileBlob: Blob,
updatedDate: Date
) {
try {
const fileURL = URL.createObjectURL(fileBlob);
let imageDataURL = await convertImageToDataURL(fileURL);
let imageDataURL = await convertImageToDataURL(reader, fileURL);
imageDataURL =
'data:image/jpeg;base64' +
imageDataURL.slice(imageDataURL.indexOf(','));
@ -82,10 +83,9 @@ export async function updateFileCreationDateInEXIF(
}
}
export async function convertImageToDataURL(url: string) {
export async function convertImageToDataURL(reader: FileReader, url: string) {
const blob = await fetch(url).then((r) => r.blob());
const dataUrl = await new Promise<string>((resolve) => {
const reader = new FileReader();
reader.onload = () => resolve(reader.result as string);
reader.readAsDataURL(blob);
});

6
src/services/upload/metadataService.ts
View file

@ -44,10 +44,12 @@ export async function extractMetadata(
export const getMetadataMapKey = (collectionID: number, title: string) =>
`${collectionID}_${title}`;
export async function parseMetadataJSON(receivedFile: File) {
export async function parseMetadataJSON(
reader: FileReader,
receivedFile: File
) {
try {
const metadataJSON: object = await new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onabort = () => reject(Error('file reading was aborted'));
reader.onerror = () => reject(Error('file reading has failed'));
reader.onload = () => {

30
src/services/upload/readFileService.ts
View file

@ -15,21 +15,21 @@ const TYPE_IMAGE = 'image';
const EDITED_FILE_SUFFIX = '-edited';
const CHUNK_SIZE_FOR_TYPE_DETECTION = 4100;
export async function getFileData(worker, file: File) {
export async function getFileData(reader: FileReader, file: File) {
if (file.size > MULTIPART_PART_SIZE) {
return getFileStream(worker, file, FILE_READER_CHUNK_SIZE);
return getFileStream(reader, file, FILE_READER_CHUNK_SIZE);
} else {
return await worker.getUint8ArrayView(file);
return await getUint8ArrayView(reader, file);
}
}
export async function getFileType(
worker,
reader: FileReader,
receivedFile: File
): Promise<FileTypeInfo> {
try {
let fileType: FILE_TYPE;
const mimeType = await getMimeType(worker, receivedFile);
const mimeType = await getMimeType(reader, receivedFile);
const typeParts = mimeType?.split('/');
if (typeParts?.length !== 2) {
throw Error(CustomError.TYPE_DETECTION_FAILED);
@ -85,14 +85,14 @@ export function getFileOriginalName(file: File) {
return originalName;
}
async function getMimeType(worker, file: File) {
async function getMimeType(reader: FileReader, file: File) {
const fileChunkBlob = file.slice(0, CHUNK_SIZE_FOR_TYPE_DETECTION);
return getMimeTypeFromBlob(worker, fileChunkBlob);
return getMimeTypeFromBlob(reader, fileChunkBlob);
}
export async function getMimeTypeFromBlob(worker, fileBlob: Blob) {
export async function getMimeTypeFromBlob(reader: FileReader, fileBlob: Blob) {
try {
const initialFiledata = await worker.getUint8ArrayView(fileBlob);
const initialFiledata = await getUint8ArrayView(reader, fileBlob);
const result = await FileType.fromBuffer(initialFiledata);
return result.mime;
} catch (e) {
@ -100,8 +100,8 @@ export async function getMimeTypeFromBlob(worker, fileBlob: Blob) {
}
}
function getFileStream(worker, file: File, chunkSize: number) {
const fileChunkReader = fileChunkReaderMaker(worker, file, chunkSize);
function getFileStream(reader: FileReader, file: File, chunkSize: number) {
const fileChunkReader = fileChunkReaderMaker(reader, file, chunkSize);
const stream = new ReadableStream<Uint8Array>({
async pull(controller: ReadableStreamDefaultController) {
@ -120,11 +120,15 @@ function getFileStream(worker, file: File, chunkSize: number) {
};
}
async function* fileChunkReaderMaker(worker, file: File, chunkSize: number) {
async function* fileChunkReaderMaker(
reader: FileReader,
file: File,
chunkSize: number
) {
let offset = 0;
while (offset < file.size) {
const blob = file.slice(offset, chunkSize + offset);
const fileChunk = await worker.getUint8ArrayView(blob);
const fileChunk = await getUint8ArrayView(reader, blob);
yield fileChunk;
offset += chunkSize;
}

4
src/services/upload/thumbnailService.ts
View file

@ -6,6 +6,7 @@ import FFmpegService from 'services/ffmpegService';
import { convertToHumanReadable } from 'utils/billing';
import { isFileHEIC } from 'utils/file';
import { FileTypeInfo } from 'types/upload';
import { getUint8ArrayView } from './readFileService';
const MAX_THUMBNAIL_DIMENSION = 720;
const MIN_COMPRESSION_PERCENTAGE_SIZE_DIFF = 10;
@ -22,6 +23,7 @@ interface Dimension {
export async function generateThumbnail(
worker,
reader: FileReader,
file: File,
fileTypeInfo: FileTypeInfo
): Promise<{ thumbnail: Uint8Array; hasStaticThumbnail: boolean }> {
@ -50,7 +52,7 @@ export async function generateThumbnail(
}
}
const thumbnailBlob = await thumbnailCanvasToBlob(canvas);
thumbnail = await worker.getUint8ArrayView(thumbnailBlob);
thumbnail = await getUint8ArrayView(reader, thumbnailBlob);
if (thumbnail.length === 0) {
throw Error('EMPTY THUMBNAIL');
}

9
src/services/upload/uploadManager.ts
View file

@ -92,9 +92,10 @@ class UploadManager {
private async seedMetadataMap(metadataFiles: FileWithCollection[]) {
try {
UIService.reset(metadataFiles.length);
const reader = new FileReader();
for (const fileWithCollection of metadataFiles) {
const parsedMetaDataJSONWithTitle = await parseMetadataJSON(
reader,
fileWithCollection.file
);
if (parsedMetaDataJSONWithTitle) {
@ -137,14 +138,15 @@ class UploadManager {
this.cryptoWorkers[i] = cryptoWorker;
uploadProcesses.push(
this.uploadNextFileInQueue(
await new this.cryptoWorkers[i].comlink()
await new this.cryptoWorkers[i].comlink(),
new FileReader()
)
);
}
await Promise.all(uploadProcesses);
}
private async uploadNextFileInQueue(worker: any) {
private async uploadNextFileInQueue(worker: any, reader: FileReader) {
while (this.filesToBeUploaded.length > 0) {
const fileWithCollection = this.filesToBeUploaded.pop();
const existingFilesInCollection =
@ -157,6 +159,7 @@ class UploadManager {
fileWithCollection.collection = collection;
const { fileUploadResult, file } = await uploader(
worker,
reader,
existingFilesInCollection,
fileWithCollection
);

4
src/services/upload/uploadService.ts
View file

@ -38,16 +38,18 @@ class UploadService {
async readFile(
worker: any,
reader: FileReader,
rawFile: File,
fileTypeInfo: FileTypeInfo
): Promise<FileInMemory> {
const { thumbnail, hasStaticThumbnail } = await generateThumbnail(
worker,
reader,
rawFile,
fileTypeInfo
);
const filedata = await getFileData(worker, rawFile);
const filedata = await getFileData(reader, rawFile);
return {
filedata,

10
src/services/upload/uploader.ts
View file

@ -30,6 +30,7 @@ interface UploadResponse {
}
export default async function uploader(
worker: any,
reader: FileReader,
existingFilesInCollection: EnteFile[],
fileWithCollection: FileWithCollection
): Promise<UploadResponse> {
@ -53,7 +54,7 @@ export default async function uploader(
await sleep(TwoSecondInMillSeconds);
return { fileUploadResult: FileUploadResults.TOO_LARGE };
}
fileTypeInfo = await getFileType(worker, rawFile);
fileTypeInfo = await getFileType(reader, rawFile);
if (fileTypeInfo.fileType === FILE_TYPE.OTHERS) {
throw Error(CustomError.UNSUPPORTED_FILE_FORMAT);
}
@ -70,7 +71,12 @@ export default async function uploader(
return { fileUploadResult: FileUploadResults.SKIPPED };
}
file = await UploadService.readFile(worker, rawFile, fileTypeInfo);
file = await UploadService.readFile(
worker,
reader,
rawFile,
fileTypeInfo
);
if (file.hasStaticThumbnail) {
metadata.hasStaticThumbnail = true;
}

4
src/utils/file/index.ts
View file

@ -60,6 +60,7 @@ export async function downloadFile(file: EnteFile) {
let fileBlob = await (await fetch(fileURL)).blob();
fileBlob = await updateFileCreationDateInEXIF(
new FileReader(),
fileBlob,
new Date(file.pubMagicMetadata.data.editedTime / 1000)
);
@ -295,9 +296,10 @@ export async function convertForPreview(file: EnteFile, fileBlob: Blob) {
const typeFromExtension = getFileExtension(file.metadata.title);
const worker = await new CryptoWorker();
const reader = new FileReader();
const mimeType =
(await getMimeTypeFromBlob(worker, fileBlob)) ?? typeFromExtension;
(await getMimeTypeFromBlob(reader, fileBlob)) ?? typeFromExtension;
if (isFileHEIC(mimeType)) {
fileBlob = await worker.convertHEIC2JPEG(fileBlob);
}

24
src/worker/crypto.worker.js
View file

@ -149,30 +149,6 @@ export class Crypto {
return libsodium.fromHex(string);
}
// temporary fix for https://github.com/vercel/next.js/issues/25484
async getUint8ArrayView(file) {
try {
return await new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onabort = () =>
reject(Error('file reading was aborted'));
reader.onerror = () => reject(Error('file reading has failed'));
reader.onload = () => {
// Do whatever you want with the file contents
const result =
typeof reader.result === 'string'
? new TextEncoder().encode(reader.result)
: new Uint8Array(reader.result);
resolve(result);
};
reader.readAsArrayBuffer(file);
});
} catch (e) {
console.log(e, 'error reading file to byte-array');
throw e;
}
}
async convertHEIC2JPEG(file) {
return convertHEIC2JPEG(file);
}