diff --git a/src/services/exportService.ts b/src/services/exportService.ts index d0c08b753..c4ccbf2b1 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -62,6 +62,7 @@ class ExportService { private stopExport: boolean = false; private pauseExport: boolean = false; private allElectronAPIsExist: boolean = false; + private fileReader: FileReader = null; constructor() { this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs']; @@ -438,7 +439,11 @@ class ExportService { (fileType === TYPE_JPEG || fileType === TYPE_JPG) ) { const fileBlob = await new Response(fileStream).blob(); + if (!this.fileReader) { + this.fileReader = new FileReader(); + } const updatedFileBlob = await updateFileCreationDateInEXIF( + this.fileReader, fileBlob, new Date(file.pubMagicMetadata.data.editedTime / 1000) ); diff --git a/src/services/ffmpegService.ts b/src/services/ffmpegService.ts index b204504df..3914cfea2 100644 --- a/src/services/ffmpegService.ts +++ b/src/services/ffmpegService.ts @@ -7,6 +7,7 @@ import { getUint8ArrayView } from './upload/readFileService'; class FFmpegService { private ffmpeg: FFmpeg = null; private isLoading = null; + private fileReader: FileReader = null; private generateThumbnailProcessor = new QueueProcessor(1); async init() { @@ -29,11 +30,19 @@ class FFmpegService { if (!this.ffmpeg) { await this.init(); } + if (!this.fileReader) { + this.fileReader = new FileReader(); + } if (this.isLoading) { await this.isLoading; } const response = this.generateThumbnailProcessor.queueUpRequest( - generateThumbnailHelper.bind(null, this.ffmpeg, file) + generateThumbnailHelper.bind( + null, + this.ffmpeg, + this.fileReader, + file + ) ); try { return await response.promise; @@ -49,14 +58,18 @@ class FFmpegService { } } -async function generateThumbnailHelper(ffmpeg: FFmpeg, file: File) { +async function generateThumbnailHelper( + ffmpeg: FFmpeg, + reader: FileReader, + file: File +) { try { const inputFileName = `${Date.now().toString()}-${file.name}`; const thumbFileName = `${Date.now().toString()}-thumb.jpeg`; ffmpeg.FS( 'writeFile', inputFileName, - await getUint8ArrayView(new FileReader(), file) + await getUint8ArrayView(reader, file) ); let seekTime = 1.0; let thumb = null; diff --git a/src/services/migrateThumbnailService.ts b/src/services/migrateThumbnailService.ts index 03d26d940..a5c693e97 100644 --- a/src/services/migrateThumbnailService.ts +++ b/src/services/migrateThumbnailService.ts @@ -44,6 +44,7 @@ export async function replaceThumbnail( try { const token = getToken(); const worker = await new CryptoWorker(); + const reader = new FileReader(); const files = await getLocalFiles(); const trash = await getLocalTrash(); const trashFiles = getTrashedFiles(trash); @@ -76,9 +77,10 @@ export async function replaceThumbnail( [originalThumbnail], file.metadata.title ); - const fileTypeInfo = await getFileType(worker, dummyImageFile); + const fileTypeInfo = await getFileType(reader, dummyImageFile); const { thumbnail: newThumbnail } = await generateThumbnail( worker, + reader, dummyImageFile, fileTypeInfo ); diff --git a/src/services/updateCreationTimeWithExif.ts b/src/services/updateCreationTimeWithExif.ts index edc58327d..e34f889c0 100644 --- a/src/services/updateCreationTimeWithExif.ts +++ b/src/services/updateCreationTimeWithExif.ts @@ -1,6 +1,5 @@ import { FIX_OPTIONS } from 'components/FixCreationTime'; import { SetProgressTracker } from 'components/FixLargeThumbnail'; -import CryptoWorker from 'utils/crypto'; import { changeFileCreationTime, getFileFromURL, @@ -38,8 +37,8 @@ export async function updateCreationTimeWithExif( } else { const fileURL = await downloadManager.getFile(file); const fileObject = await getFileFromURL(fileURL); - const worker = await new CryptoWorker(); - const fileTypeInfo = await getFileType(worker, fileObject); + const reader = new FileReader(); + const fileTypeInfo = await getFileType(reader, fileObject); const exifData = await getRawExif(fileObject, fileTypeInfo); if (fixOption === FIX_OPTIONS.DATE_TIME_ORIGINAL) { correctCreationTime = getUNIXTime( diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts index 6a97e78e1..e463598ed 100644 --- a/src/services/upload/exifService.ts +++ b/src/services/upload/exifService.ts @@ -57,12 +57,13 @@ export async function getExifData( } export async function updateFileCreationDateInEXIF( + reader: FileReader, fileBlob: Blob, updatedDate: Date ) { try { const fileURL = URL.createObjectURL(fileBlob); - let imageDataURL = await convertImageToDataURL(fileURL); + let imageDataURL = await convertImageToDataURL(reader, fileURL); imageDataURL = 'data:image/jpeg;base64' + imageDataURL.slice(imageDataURL.indexOf(',')); @@ -82,10 +83,9 @@ export async function updateFileCreationDateInEXIF( } } -export async function convertImageToDataURL(url: string) { +export async function convertImageToDataURL(reader: FileReader, url: string) { const blob = await fetch(url).then((r) => r.blob()); const dataUrl = await new Promise((resolve) => { - const reader = new FileReader(); reader.onload = () => resolve(reader.result as string); reader.readAsDataURL(blob); }); diff --git a/src/services/upload/metadataService.ts b/src/services/upload/metadataService.ts index 2c114017a..a7233f37d 100644 --- a/src/services/upload/metadataService.ts +++ b/src/services/upload/metadataService.ts @@ -44,10 +44,12 @@ export async function extractMetadata( export const getMetadataMapKey = (collectionID: number, title: string) => `${collectionID}_${title}`; -export async function parseMetadataJSON(receivedFile: File) { +export async function parseMetadataJSON( + reader: FileReader, + receivedFile: File +) { try { const metadataJSON: object = await new Promise((resolve, reject) => { - const reader = new FileReader(); reader.onabort = () => reject(Error('file reading was aborted')); reader.onerror = () => reject(Error('file reading has failed')); reader.onload = () => { diff --git a/src/services/upload/readFileService.ts b/src/services/upload/readFileService.ts index 283d0069c..d4e89ad13 100644 --- a/src/services/upload/readFileService.ts +++ b/src/services/upload/readFileService.ts @@ -15,21 +15,21 @@ const TYPE_IMAGE = 'image'; const EDITED_FILE_SUFFIX = '-edited'; const CHUNK_SIZE_FOR_TYPE_DETECTION = 4100; -export async function getFileData(worker, file: File) { +export async function getFileData(reader: FileReader, file: File) { if (file.size > MULTIPART_PART_SIZE) { - return getFileStream(worker, file, FILE_READER_CHUNK_SIZE); + return getFileStream(reader, file, FILE_READER_CHUNK_SIZE); } else { - return await worker.getUint8ArrayView(file); + return await getUint8ArrayView(reader, file); } } export async function getFileType( - worker, + reader: FileReader, receivedFile: File ): Promise { try { let fileType: FILE_TYPE; - const mimeType = await getMimeType(worker, receivedFile); + const mimeType = await getMimeType(reader, receivedFile); const typeParts = mimeType?.split('/'); if (typeParts?.length !== 2) { throw Error(CustomError.TYPE_DETECTION_FAILED); @@ -85,14 +85,14 @@ export function getFileOriginalName(file: File) { return originalName; } -async function getMimeType(worker, file: File) { +async function getMimeType(reader: FileReader, file: File) { const fileChunkBlob = file.slice(0, CHUNK_SIZE_FOR_TYPE_DETECTION); - return getMimeTypeFromBlob(worker, fileChunkBlob); + return getMimeTypeFromBlob(reader, fileChunkBlob); } -export async function getMimeTypeFromBlob(worker, fileBlob: Blob) { +export async function getMimeTypeFromBlob(reader: FileReader, fileBlob: Blob) { try { - const initialFiledata = await worker.getUint8ArrayView(fileBlob); + const initialFiledata = await getUint8ArrayView(reader, fileBlob); const result = await FileType.fromBuffer(initialFiledata); return result.mime; } catch (e) { @@ -100,8 +100,8 @@ export async function getMimeTypeFromBlob(worker, fileBlob: Blob) { } } -function getFileStream(worker, file: File, chunkSize: number) { - const fileChunkReader = fileChunkReaderMaker(worker, file, chunkSize); +function getFileStream(reader: FileReader, file: File, chunkSize: number) { + const fileChunkReader = fileChunkReaderMaker(reader, file, chunkSize); const stream = new ReadableStream({ async pull(controller: ReadableStreamDefaultController) { @@ -120,11 +120,15 @@ function getFileStream(worker, file: File, chunkSize: number) { }; } -async function* fileChunkReaderMaker(worker, file: File, chunkSize: number) { +async function* fileChunkReaderMaker( + reader: FileReader, + file: File, + chunkSize: number +) { let offset = 0; while (offset < file.size) { const blob = file.slice(offset, chunkSize + offset); - const fileChunk = await worker.getUint8ArrayView(blob); + const fileChunk = await getUint8ArrayView(reader, blob); yield fileChunk; offset += chunkSize; } diff --git a/src/services/upload/thumbnailService.ts b/src/services/upload/thumbnailService.ts index 7cd3ebd85..7d908899d 100644 --- a/src/services/upload/thumbnailService.ts +++ b/src/services/upload/thumbnailService.ts @@ -6,6 +6,7 @@ import FFmpegService from 'services/ffmpegService'; import { convertToHumanReadable } from 'utils/billing'; import { isFileHEIC } from 'utils/file'; import { FileTypeInfo } from 'types/upload'; +import { getUint8ArrayView } from './readFileService'; const MAX_THUMBNAIL_DIMENSION = 720; const MIN_COMPRESSION_PERCENTAGE_SIZE_DIFF = 10; @@ -22,6 +23,7 @@ interface Dimension { export async function generateThumbnail( worker, + reader: FileReader, file: File, fileTypeInfo: FileTypeInfo ): Promise<{ thumbnail: Uint8Array; hasStaticThumbnail: boolean }> { @@ -50,7 +52,7 @@ export async function generateThumbnail( } } const thumbnailBlob = await thumbnailCanvasToBlob(canvas); - thumbnail = await worker.getUint8ArrayView(thumbnailBlob); + thumbnail = await getUint8ArrayView(reader, thumbnailBlob); if (thumbnail.length === 0) { throw Error('EMPTY THUMBNAIL'); } diff --git a/src/services/upload/uploadManager.ts b/src/services/upload/uploadManager.ts index 9b1ec8851..4c8aad969 100644 --- a/src/services/upload/uploadManager.ts +++ b/src/services/upload/uploadManager.ts @@ -92,9 +92,10 @@ class UploadManager { private async seedMetadataMap(metadataFiles: FileWithCollection[]) { try { UIService.reset(metadataFiles.length); - + const reader = new FileReader(); for (const fileWithCollection of metadataFiles) { const parsedMetaDataJSONWithTitle = await parseMetadataJSON( + reader, fileWithCollection.file ); if (parsedMetaDataJSONWithTitle) { @@ -137,14 +138,15 @@ class UploadManager { this.cryptoWorkers[i] = cryptoWorker; uploadProcesses.push( this.uploadNextFileInQueue( - await new this.cryptoWorkers[i].comlink() + await new this.cryptoWorkers[i].comlink(), + new FileReader() ) ); } await Promise.all(uploadProcesses); } - private async uploadNextFileInQueue(worker: any) { + private async uploadNextFileInQueue(worker: any, reader: FileReader) { while (this.filesToBeUploaded.length > 0) { const fileWithCollection = this.filesToBeUploaded.pop(); const existingFilesInCollection = @@ -157,6 +159,7 @@ class UploadManager { fileWithCollection.collection = collection; const { fileUploadResult, file } = await uploader( worker, + reader, existingFilesInCollection, fileWithCollection ); diff --git a/src/services/upload/uploadService.ts b/src/services/upload/uploadService.ts index 87535419b..6f33b06dd 100644 --- a/src/services/upload/uploadService.ts +++ b/src/services/upload/uploadService.ts @@ -38,16 +38,18 @@ class UploadService { async readFile( worker: any, + reader: FileReader, rawFile: File, fileTypeInfo: FileTypeInfo ): Promise { const { thumbnail, hasStaticThumbnail } = await generateThumbnail( worker, + reader, rawFile, fileTypeInfo ); - const filedata = await getFileData(worker, rawFile); + const filedata = await getFileData(reader, rawFile); return { filedata, diff --git a/src/services/upload/uploader.ts b/src/services/upload/uploader.ts index 62ea32b23..ebb86a284 100644 --- a/src/services/upload/uploader.ts +++ b/src/services/upload/uploader.ts @@ -30,6 +30,7 @@ interface UploadResponse { } export default async function uploader( worker: any, + reader: FileReader, existingFilesInCollection: EnteFile[], fileWithCollection: FileWithCollection ): Promise { @@ -53,7 +54,7 @@ export default async function uploader( await sleep(TwoSecondInMillSeconds); return { fileUploadResult: FileUploadResults.TOO_LARGE }; } - fileTypeInfo = await getFileType(worker, rawFile); + fileTypeInfo = await getFileType(reader, rawFile); if (fileTypeInfo.fileType === FILE_TYPE.OTHERS) { throw Error(CustomError.UNSUPPORTED_FILE_FORMAT); } @@ -70,7 +71,12 @@ export default async function uploader( return { fileUploadResult: FileUploadResults.SKIPPED }; } - file = await UploadService.readFile(worker, rawFile, fileTypeInfo); + file = await UploadService.readFile( + worker, + reader, + rawFile, + fileTypeInfo + ); if (file.hasStaticThumbnail) { metadata.hasStaticThumbnail = true; } diff --git a/src/utils/file/index.ts b/src/utils/file/index.ts index 507f6638a..929c319ea 100644 --- a/src/utils/file/index.ts +++ b/src/utils/file/index.ts @@ -60,6 +60,7 @@ export async function downloadFile(file: EnteFile) { let fileBlob = await (await fetch(fileURL)).blob(); fileBlob = await updateFileCreationDateInEXIF( + new FileReader(), fileBlob, new Date(file.pubMagicMetadata.data.editedTime / 1000) ); @@ -295,9 +296,10 @@ export async function convertForPreview(file: EnteFile, fileBlob: Blob) { const typeFromExtension = getFileExtension(file.metadata.title); const worker = await new CryptoWorker(); + const reader = new FileReader(); const mimeType = - (await getMimeTypeFromBlob(worker, fileBlob)) ?? typeFromExtension; + (await getMimeTypeFromBlob(reader, fileBlob)) ?? typeFromExtension; if (isFileHEIC(mimeType)) { fileBlob = await worker.convertHEIC2JPEG(fileBlob); } diff --git a/src/worker/crypto.worker.js b/src/worker/crypto.worker.js index e4da90107..cf0137831 100644 --- a/src/worker/crypto.worker.js +++ b/src/worker/crypto.worker.js @@ -149,30 +149,6 @@ export class Crypto { return libsodium.fromHex(string); } - // temporary fix for https://github.com/vercel/next.js/issues/25484 - async getUint8ArrayView(file) { - try { - return await new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onabort = () => - reject(Error('file reading was aborted')); - reader.onerror = () => reject(Error('file reading has failed')); - reader.onload = () => { - // Do whatever you want with the file contents - const result = - typeof reader.result === 'string' - ? new TextEncoder().encode(reader.result) - : new Uint8Array(reader.result); - resolve(result); - }; - reader.readAsArrayBuffer(file); - }); - } catch (e) { - console.log(e, 'error reading file to byte-array'); - throw e; - } - } - async convertHEIC2JPEG(file) { return convertHEIC2JPEG(file); }