Merge pull request #281 from ente-io/reuse-file-readers
reuse file readers
This commit is contained in:
commit
71a97caa03
|
@ -62,6 +62,7 @@ class ExportService {
|
|||
private stopExport: boolean = false;
|
||||
private pauseExport: boolean = false;
|
||||
private allElectronAPIsExist: boolean = false;
|
||||
private fileReader: FileReader = null;
|
||||
|
||||
constructor() {
|
||||
this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs'];
|
||||
|
@ -438,7 +439,11 @@ class ExportService {
|
|||
(fileType === TYPE_JPEG || fileType === TYPE_JPG)
|
||||
) {
|
||||
const fileBlob = await new Response(fileStream).blob();
|
||||
if (!this.fileReader) {
|
||||
this.fileReader = new FileReader();
|
||||
}
|
||||
const updatedFileBlob = await updateFileCreationDateInEXIF(
|
||||
this.fileReader,
|
||||
fileBlob,
|
||||
new Date(file.pubMagicMetadata.data.editedTime / 1000)
|
||||
);
|
||||
|
|
|
@ -7,6 +7,7 @@ import { getUint8ArrayView } from './upload/readFileService';
|
|||
class FFmpegService {
|
||||
private ffmpeg: FFmpeg = null;
|
||||
private isLoading = null;
|
||||
private fileReader: FileReader = null;
|
||||
|
||||
private generateThumbnailProcessor = new QueueProcessor<Uint8Array>(1);
|
||||
async init() {
|
||||
|
@ -29,11 +30,19 @@ class FFmpegService {
|
|||
if (!this.ffmpeg) {
|
||||
await this.init();
|
||||
}
|
||||
if (!this.fileReader) {
|
||||
this.fileReader = new FileReader();
|
||||
}
|
||||
if (this.isLoading) {
|
||||
await this.isLoading;
|
||||
}
|
||||
const response = this.generateThumbnailProcessor.queueUpRequest(
|
||||
generateThumbnailHelper.bind(null, this.ffmpeg, file)
|
||||
generateThumbnailHelper.bind(
|
||||
null,
|
||||
this.ffmpeg,
|
||||
this.fileReader,
|
||||
file
|
||||
)
|
||||
);
|
||||
try {
|
||||
return await response.promise;
|
||||
|
@ -49,14 +58,18 @@ class FFmpegService {
|
|||
}
|
||||
}
|
||||
|
||||
async function generateThumbnailHelper(ffmpeg: FFmpeg, file: File) {
|
||||
async function generateThumbnailHelper(
|
||||
ffmpeg: FFmpeg,
|
||||
reader: FileReader,
|
||||
file: File
|
||||
) {
|
||||
try {
|
||||
const inputFileName = `${Date.now().toString()}-${file.name}`;
|
||||
const thumbFileName = `${Date.now().toString()}-thumb.jpeg`;
|
||||
ffmpeg.FS(
|
||||
'writeFile',
|
||||
inputFileName,
|
||||
await getUint8ArrayView(new FileReader(), file)
|
||||
await getUint8ArrayView(reader, file)
|
||||
);
|
||||
let seekTime = 1.0;
|
||||
let thumb = null;
|
||||
|
|
|
@ -44,6 +44,7 @@ export async function replaceThumbnail(
|
|||
try {
|
||||
const token = getToken();
|
||||
const worker = await new CryptoWorker();
|
||||
const reader = new FileReader();
|
||||
const files = await getLocalFiles();
|
||||
const trash = await getLocalTrash();
|
||||
const trashFiles = getTrashedFiles(trash);
|
||||
|
@ -76,9 +77,10 @@ export async function replaceThumbnail(
|
|||
[originalThumbnail],
|
||||
file.metadata.title
|
||||
);
|
||||
const fileTypeInfo = await getFileType(worker, dummyImageFile);
|
||||
const fileTypeInfo = await getFileType(reader, dummyImageFile);
|
||||
const { thumbnail: newThumbnail } = await generateThumbnail(
|
||||
worker,
|
||||
reader,
|
||||
dummyImageFile,
|
||||
fileTypeInfo
|
||||
);
|
||||
|
|
|
@ -1,6 +1,5 @@
|
|||
import { FIX_OPTIONS } from 'components/FixCreationTime';
|
||||
import { SetProgressTracker } from 'components/FixLargeThumbnail';
|
||||
import CryptoWorker from 'utils/crypto';
|
||||
import {
|
||||
changeFileCreationTime,
|
||||
getFileFromURL,
|
||||
|
@ -38,8 +37,8 @@ export async function updateCreationTimeWithExif(
|
|||
} else {
|
||||
const fileURL = await downloadManager.getFile(file);
|
||||
const fileObject = await getFileFromURL(fileURL);
|
||||
const worker = await new CryptoWorker();
|
||||
const fileTypeInfo = await getFileType(worker, fileObject);
|
||||
const reader = new FileReader();
|
||||
const fileTypeInfo = await getFileType(reader, fileObject);
|
||||
const exifData = await getRawExif(fileObject, fileTypeInfo);
|
||||
if (fixOption === FIX_OPTIONS.DATE_TIME_ORIGINAL) {
|
||||
correctCreationTime = getUNIXTime(
|
||||
|
|
|
@ -57,12 +57,13 @@ export async function getExifData(
|
|||
}
|
||||
|
||||
export async function updateFileCreationDateInEXIF(
|
||||
reader: FileReader,
|
||||
fileBlob: Blob,
|
||||
updatedDate: Date
|
||||
) {
|
||||
try {
|
||||
const fileURL = URL.createObjectURL(fileBlob);
|
||||
let imageDataURL = await convertImageToDataURL(fileURL);
|
||||
let imageDataURL = await convertImageToDataURL(reader, fileURL);
|
||||
imageDataURL =
|
||||
'data:image/jpeg;base64' +
|
||||
imageDataURL.slice(imageDataURL.indexOf(','));
|
||||
|
@ -82,10 +83,9 @@ export async function updateFileCreationDateInEXIF(
|
|||
}
|
||||
}
|
||||
|
||||
export async function convertImageToDataURL(url: string) {
|
||||
export async function convertImageToDataURL(reader: FileReader, url: string) {
|
||||
const blob = await fetch(url).then((r) => r.blob());
|
||||
const dataUrl = await new Promise<string>((resolve) => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => resolve(reader.result as string);
|
||||
reader.readAsDataURL(blob);
|
||||
});
|
||||
|
|
|
@ -44,10 +44,12 @@ export async function extractMetadata(
|
|||
export const getMetadataMapKey = (collectionID: number, title: string) =>
|
||||
`${collectionID}_${title}`;
|
||||
|
||||
export async function parseMetadataJSON(receivedFile: File) {
|
||||
export async function parseMetadataJSON(
|
||||
reader: FileReader,
|
||||
receivedFile: File
|
||||
) {
|
||||
try {
|
||||
const metadataJSON: object = await new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onabort = () => reject(Error('file reading was aborted'));
|
||||
reader.onerror = () => reject(Error('file reading has failed'));
|
||||
reader.onload = () => {
|
||||
|
|
|
@ -15,21 +15,21 @@ const TYPE_IMAGE = 'image';
|
|||
const EDITED_FILE_SUFFIX = '-edited';
|
||||
const CHUNK_SIZE_FOR_TYPE_DETECTION = 4100;
|
||||
|
||||
export async function getFileData(worker, file: File) {
|
||||
export async function getFileData(reader: FileReader, file: File) {
|
||||
if (file.size > MULTIPART_PART_SIZE) {
|
||||
return getFileStream(worker, file, FILE_READER_CHUNK_SIZE);
|
||||
return getFileStream(reader, file, FILE_READER_CHUNK_SIZE);
|
||||
} else {
|
||||
return await worker.getUint8ArrayView(file);
|
||||
return await getUint8ArrayView(reader, file);
|
||||
}
|
||||
}
|
||||
|
||||
export async function getFileType(
|
||||
worker,
|
||||
reader: FileReader,
|
||||
receivedFile: File
|
||||
): Promise<FileTypeInfo> {
|
||||
try {
|
||||
let fileType: FILE_TYPE;
|
||||
const mimeType = await getMimeType(worker, receivedFile);
|
||||
const mimeType = await getMimeType(reader, receivedFile);
|
||||
const typeParts = mimeType?.split('/');
|
||||
if (typeParts?.length !== 2) {
|
||||
throw Error(CustomError.TYPE_DETECTION_FAILED);
|
||||
|
@ -85,14 +85,14 @@ export function getFileOriginalName(file: File) {
|
|||
return originalName;
|
||||
}
|
||||
|
||||
async function getMimeType(worker, file: File) {
|
||||
async function getMimeType(reader: FileReader, file: File) {
|
||||
const fileChunkBlob = file.slice(0, CHUNK_SIZE_FOR_TYPE_DETECTION);
|
||||
return getMimeTypeFromBlob(worker, fileChunkBlob);
|
||||
return getMimeTypeFromBlob(reader, fileChunkBlob);
|
||||
}
|
||||
|
||||
export async function getMimeTypeFromBlob(worker, fileBlob: Blob) {
|
||||
export async function getMimeTypeFromBlob(reader: FileReader, fileBlob: Blob) {
|
||||
try {
|
||||
const initialFiledata = await worker.getUint8ArrayView(fileBlob);
|
||||
const initialFiledata = await getUint8ArrayView(reader, fileBlob);
|
||||
const result = await FileType.fromBuffer(initialFiledata);
|
||||
return result.mime;
|
||||
} catch (e) {
|
||||
|
@ -100,8 +100,8 @@ export async function getMimeTypeFromBlob(worker, fileBlob: Blob) {
|
|||
}
|
||||
}
|
||||
|
||||
function getFileStream(worker, file: File, chunkSize: number) {
|
||||
const fileChunkReader = fileChunkReaderMaker(worker, file, chunkSize);
|
||||
function getFileStream(reader: FileReader, file: File, chunkSize: number) {
|
||||
const fileChunkReader = fileChunkReaderMaker(reader, file, chunkSize);
|
||||
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
async pull(controller: ReadableStreamDefaultController) {
|
||||
|
@ -120,11 +120,15 @@ function getFileStream(worker, file: File, chunkSize: number) {
|
|||
};
|
||||
}
|
||||
|
||||
async function* fileChunkReaderMaker(worker, file: File, chunkSize: number) {
|
||||
async function* fileChunkReaderMaker(
|
||||
reader: FileReader,
|
||||
file: File,
|
||||
chunkSize: number
|
||||
) {
|
||||
let offset = 0;
|
||||
while (offset < file.size) {
|
||||
const blob = file.slice(offset, chunkSize + offset);
|
||||
const fileChunk = await worker.getUint8ArrayView(blob);
|
||||
const fileChunk = await getUint8ArrayView(reader, blob);
|
||||
yield fileChunk;
|
||||
offset += chunkSize;
|
||||
}
|
||||
|
|
|
@ -6,6 +6,7 @@ import FFmpegService from 'services/ffmpegService';
|
|||
import { convertToHumanReadable } from 'utils/billing';
|
||||
import { isFileHEIC } from 'utils/file';
|
||||
import { FileTypeInfo } from 'types/upload';
|
||||
import { getUint8ArrayView } from './readFileService';
|
||||
|
||||
const MAX_THUMBNAIL_DIMENSION = 720;
|
||||
const MIN_COMPRESSION_PERCENTAGE_SIZE_DIFF = 10;
|
||||
|
@ -22,6 +23,7 @@ interface Dimension {
|
|||
|
||||
export async function generateThumbnail(
|
||||
worker,
|
||||
reader: FileReader,
|
||||
file: File,
|
||||
fileTypeInfo: FileTypeInfo
|
||||
): Promise<{ thumbnail: Uint8Array; hasStaticThumbnail: boolean }> {
|
||||
|
@ -50,7 +52,7 @@ export async function generateThumbnail(
|
|||
}
|
||||
}
|
||||
const thumbnailBlob = await thumbnailCanvasToBlob(canvas);
|
||||
thumbnail = await worker.getUint8ArrayView(thumbnailBlob);
|
||||
thumbnail = await getUint8ArrayView(reader, thumbnailBlob);
|
||||
if (thumbnail.length === 0) {
|
||||
throw Error('EMPTY THUMBNAIL');
|
||||
}
|
||||
|
|
|
@ -92,9 +92,10 @@ class UploadManager {
|
|||
private async seedMetadataMap(metadataFiles: FileWithCollection[]) {
|
||||
try {
|
||||
UIService.reset(metadataFiles.length);
|
||||
|
||||
const reader = new FileReader();
|
||||
for (const fileWithCollection of metadataFiles) {
|
||||
const parsedMetaDataJSONWithTitle = await parseMetadataJSON(
|
||||
reader,
|
||||
fileWithCollection.file
|
||||
);
|
||||
if (parsedMetaDataJSONWithTitle) {
|
||||
|
@ -137,14 +138,15 @@ class UploadManager {
|
|||
this.cryptoWorkers[i] = cryptoWorker;
|
||||
uploadProcesses.push(
|
||||
this.uploadNextFileInQueue(
|
||||
await new this.cryptoWorkers[i].comlink()
|
||||
await new this.cryptoWorkers[i].comlink(),
|
||||
new FileReader()
|
||||
)
|
||||
);
|
||||
}
|
||||
await Promise.all(uploadProcesses);
|
||||
}
|
||||
|
||||
private async uploadNextFileInQueue(worker: any) {
|
||||
private async uploadNextFileInQueue(worker: any, reader: FileReader) {
|
||||
while (this.filesToBeUploaded.length > 0) {
|
||||
const fileWithCollection = this.filesToBeUploaded.pop();
|
||||
const existingFilesInCollection =
|
||||
|
@ -157,6 +159,7 @@ class UploadManager {
|
|||
fileWithCollection.collection = collection;
|
||||
const { fileUploadResult, file } = await uploader(
|
||||
worker,
|
||||
reader,
|
||||
existingFilesInCollection,
|
||||
fileWithCollection
|
||||
);
|
||||
|
|
|
@ -38,16 +38,18 @@ class UploadService {
|
|||
|
||||
async readFile(
|
||||
worker: any,
|
||||
reader: FileReader,
|
||||
rawFile: File,
|
||||
fileTypeInfo: FileTypeInfo
|
||||
): Promise<FileInMemory> {
|
||||
const { thumbnail, hasStaticThumbnail } = await generateThumbnail(
|
||||
worker,
|
||||
reader,
|
||||
rawFile,
|
||||
fileTypeInfo
|
||||
);
|
||||
|
||||
const filedata = await getFileData(worker, rawFile);
|
||||
const filedata = await getFileData(reader, rawFile);
|
||||
|
||||
return {
|
||||
filedata,
|
||||
|
|
|
@ -30,6 +30,7 @@ interface UploadResponse {
|
|||
}
|
||||
export default async function uploader(
|
||||
worker: any,
|
||||
reader: FileReader,
|
||||
existingFilesInCollection: EnteFile[],
|
||||
fileWithCollection: FileWithCollection
|
||||
): Promise<UploadResponse> {
|
||||
|
@ -53,7 +54,7 @@ export default async function uploader(
|
|||
await sleep(TwoSecondInMillSeconds);
|
||||
return { fileUploadResult: FileUploadResults.TOO_LARGE };
|
||||
}
|
||||
fileTypeInfo = await getFileType(worker, rawFile);
|
||||
fileTypeInfo = await getFileType(reader, rawFile);
|
||||
if (fileTypeInfo.fileType === FILE_TYPE.OTHERS) {
|
||||
throw Error(CustomError.UNSUPPORTED_FILE_FORMAT);
|
||||
}
|
||||
|
@ -70,7 +71,12 @@ export default async function uploader(
|
|||
return { fileUploadResult: FileUploadResults.SKIPPED };
|
||||
}
|
||||
|
||||
file = await UploadService.readFile(worker, rawFile, fileTypeInfo);
|
||||
file = await UploadService.readFile(
|
||||
worker,
|
||||
reader,
|
||||
rawFile,
|
||||
fileTypeInfo
|
||||
);
|
||||
if (file.hasStaticThumbnail) {
|
||||
metadata.hasStaticThumbnail = true;
|
||||
}
|
||||
|
|
|
@ -60,6 +60,7 @@ export async function downloadFile(file: EnteFile) {
|
|||
let fileBlob = await (await fetch(fileURL)).blob();
|
||||
|
||||
fileBlob = await updateFileCreationDateInEXIF(
|
||||
new FileReader(),
|
||||
fileBlob,
|
||||
new Date(file.pubMagicMetadata.data.editedTime / 1000)
|
||||
);
|
||||
|
@ -295,9 +296,10 @@ export async function convertForPreview(file: EnteFile, fileBlob: Blob) {
|
|||
|
||||
const typeFromExtension = getFileExtension(file.metadata.title);
|
||||
const worker = await new CryptoWorker();
|
||||
const reader = new FileReader();
|
||||
|
||||
const mimeType =
|
||||
(await getMimeTypeFromBlob(worker, fileBlob)) ?? typeFromExtension;
|
||||
(await getMimeTypeFromBlob(reader, fileBlob)) ?? typeFromExtension;
|
||||
if (isFileHEIC(mimeType)) {
|
||||
fileBlob = await worker.convertHEIC2JPEG(fileBlob);
|
||||
}
|
||||
|
|
|
@ -149,30 +149,6 @@ export class Crypto {
|
|||
return libsodium.fromHex(string);
|
||||
}
|
||||
|
||||
// temporary fix for https://github.com/vercel/next.js/issues/25484
|
||||
async getUint8ArrayView(file) {
|
||||
try {
|
||||
return await new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onabort = () =>
|
||||
reject(Error('file reading was aborted'));
|
||||
reader.onerror = () => reject(Error('file reading has failed'));
|
||||
reader.onload = () => {
|
||||
// Do whatever you want with the file contents
|
||||
const result =
|
||||
typeof reader.result === 'string'
|
||||
? new TextEncoder().encode(reader.result)
|
||||
: new Uint8Array(reader.result);
|
||||
resolve(result);
|
||||
};
|
||||
reader.readAsArrayBuffer(file);
|
||||
});
|
||||
} catch (e) {
|
||||
console.log(e, 'error reading file to byte-array');
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async convertHEIC2JPEG(file) {
|
||||
return convertHEIC2JPEG(file);
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue