From 47737d2dea64928fd7c2a5d9dbbb3fd86e058095 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 7 Mar 2022 17:25:07 +0530 Subject: [PATCH] refactor and create typeDetectionService and readerService --- src/services/migrateThumbnailService.ts | 2 +- src/services/readerService.ts | 57 ++++++++ src/services/typeDetectionService.ts | 63 ++++++++ src/services/updateCreationTimeWithExif.ts | 2 +- src/services/upload/fileService.ts | 40 +++++- src/services/upload/livePhotoService.ts | 2 +- src/services/upload/readFileService.ts | 159 --------------------- src/services/upload/thumbnailService.ts | 2 +- src/services/upload/uploadService.ts | 2 +- src/utils/file/index.ts | 2 +- 10 files changed, 163 insertions(+), 168 deletions(-) create mode 100644 src/services/readerService.ts create mode 100644 src/services/typeDetectionService.ts delete mode 100644 src/services/upload/readFileService.ts diff --git a/src/services/migrateThumbnailService.ts b/src/services/migrateThumbnailService.ts index f1ded2766..72e44ae17 100644 --- a/src/services/migrateThumbnailService.ts +++ b/src/services/migrateThumbnailService.ts @@ -8,7 +8,7 @@ import HTTPService from 'services/HTTPService'; import CryptoWorker from 'utils/crypto'; import uploadHttpClient from 'services/upload/uploadHttpClient'; import { SetProgressTracker } from 'components/FixLargeThumbnail'; -import { getFileType } from './upload/readFileService'; +import { getFileType } from 'services/typeDetectionService'; import { getLocalTrash, getTrashedFiles } from './trashService'; import { EncryptionResult, UploadURL } from 'types/upload'; import { fileAttribute } from 'types/file'; diff --git a/src/services/readerService.ts b/src/services/readerService.ts new file mode 100644 index 000000000..93e3240b7 --- /dev/null +++ b/src/services/readerService.ts @@ -0,0 +1,57 @@ +export async function getUint8ArrayView( + reader: FileReader, + file: Blob +): Promise { + return await new Promise((resolve, reject) => { + reader.onabort = () => reject(Error('file reading was aborted')); + reader.onerror = () => reject(Error('file reading has failed')); + reader.onload = () => { + // Do whatever you want with the file contents + const result = + typeof reader.result === 'string' + ? new TextEncoder().encode(reader.result) + : new Uint8Array(reader.result); + resolve(result); + }; + reader.readAsArrayBuffer(file); + }); +} + +export function getFileStream( + reader: FileReader, + file: File, + chunkSize: number +) { + const fileChunkReader = fileChunkReaderMaker(reader, file, chunkSize); + + const stream = new ReadableStream({ + async pull(controller: ReadableStreamDefaultController) { + const chunk = await fileChunkReader.next(); + if (chunk.done) { + controller.close(); + } else { + controller.enqueue(chunk.value); + } + }, + }); + const chunkCount = Math.ceil(file.size / chunkSize); + return { + stream, + chunkCount, + }; +} + +async function* fileChunkReaderMaker( + reader: FileReader, + file: File, + chunkSize: number +) { + let offset = 0; + while (offset < file.size) { + const blob = file.slice(offset, chunkSize + offset); + const fileChunk = await getUint8ArrayView(reader, blob); + yield fileChunk; + offset += chunkSize; + } + return null; +} diff --git a/src/services/typeDetectionService.ts b/src/services/typeDetectionService.ts new file mode 100644 index 000000000..a9fc46f62 --- /dev/null +++ b/src/services/typeDetectionService.ts @@ -0,0 +1,63 @@ +import { FILE_TYPE } from 'constants/file'; +import { FORMAT_MISSED_BY_FILE_TYPE_LIB } from 'constants/upload'; +import { FileTypeInfo } from 'types/upload'; +import { CustomError } from 'utils/error'; +import { getFileExtension } from 'utils/file'; +import { logError } from 'utils/sentry'; +import { getUint8ArrayView } from './readerService'; +import FileType from 'file-type/browser'; + +const TYPE_VIDEO = 'video'; +const TYPE_IMAGE = 'image'; +const CHUNK_SIZE_FOR_TYPE_DETECTION = 4100; + +export async function getFileType( + reader: FileReader, + receivedFile: File +): Promise { + try { + let fileType: FILE_TYPE; + const typeResult = await extractFileType(reader, receivedFile); + const mimTypeParts = typeResult.mime?.split('/'); + if (mimTypeParts?.length !== 2) { + throw Error(CustomError.TYPE_DETECTION_FAILED); + } + switch (mimTypeParts[0]) { + case TYPE_IMAGE: + fileType = FILE_TYPE.IMAGE; + break; + case TYPE_VIDEO: + fileType = FILE_TYPE.VIDEO; + break; + default: + fileType = FILE_TYPE.OTHERS; + } + return { fileType, exactType: typeResult.ext }; + } catch (e) { + const fileFormat = getFileExtension(receivedFile.name); + const formatMissedByTypeDetection = FORMAT_MISSED_BY_FILE_TYPE_LIB.find( + (a) => a.exactType === fileFormat + ); + if (formatMissedByTypeDetection) { + return formatMissedByTypeDetection; + } + logError(e, CustomError.TYPE_DETECTION_FAILED, { + fileFormat, + }); + return { fileType: FILE_TYPE.OTHERS, exactType: fileFormat }; + } +} + +async function extractFileType(reader: FileReader, file: File) { + const fileChunkBlob = file.slice(0, CHUNK_SIZE_FOR_TYPE_DETECTION); + return getFileTypeFromBlob(reader, fileChunkBlob); +} + +export async function getFileTypeFromBlob(reader: FileReader, fileBlob: Blob) { + try { + const initialFiledata = await getUint8ArrayView(reader, fileBlob); + return await FileType.fromBuffer(initialFiledata); + } catch (e) { + throw Error(CustomError.TYPE_DETECTION_FAILED); + } +} diff --git a/src/services/updateCreationTimeWithExif.ts b/src/services/updateCreationTimeWithExif.ts index 64b989fdc..ab83c8b63 100644 --- a/src/services/updateCreationTimeWithExif.ts +++ b/src/services/updateCreationTimeWithExif.ts @@ -11,7 +11,7 @@ import { updatePublicMagicMetadata } from './fileService'; import { EnteFile } from 'types/file'; import { getRawExif } from './upload/exifService'; -import { getFileType } from './upload/readFileService'; +import { getFileType } from 'services/typeDetectionService'; import { FILE_TYPE } from 'constants/file'; import { getUnixTimeInMicroSeconds } from 'utils/time'; diff --git a/src/services/upload/fileService.ts b/src/services/upload/fileService.ts index 4b69abc1b..5601f6068 100644 --- a/src/services/upload/fileService.ts +++ b/src/services/upload/fileService.ts @@ -1,3 +1,4 @@ +import { MULTIPART_PART_SIZE, FILE_READER_CHUNK_SIZE } from 'constants/upload'; import { FileTypeInfo, FileInMemory, @@ -7,14 +8,18 @@ import { EncryptionResult, FileWithMetadata, ParsedMetadataJSONMap, + DataStream, } from 'types/upload'; +import { splitFilenameAndExtension } from 'utils/file'; import { logError } from 'utils/sentry'; import { getFileNameSize, logUploadInfo } from 'utils/upload'; import { encryptFiledata } from './encryptionService'; import { extractMetadata, getMetadataJSONMapKey } from './metadataService'; -import { getFileData, getFileOriginalName } from './readFileService'; +import { getFileStream, getUint8ArrayView } from '../readerService'; import { generateThumbnail } from './thumbnailService'; +const EDITED_FILE_SUFFIX = '-edited'; + export function getFileSize(file: File) { return file.size; } @@ -34,8 +39,12 @@ export async function readFile( fileTypeInfo ); logUploadInfo(`reading file datal${getFileNameSize(rawFile)} `); - - const filedata = await getFileData(reader, rawFile); + let filedata: Uint8Array | DataStream; + if (rawFile.size > MULTIPART_PART_SIZE) { + filedata = getFileStream(reader, rawFile, FILE_READER_CHUNK_SIZE); + } else { + filedata = await getUint8ArrayView(reader, rawFile); + } logUploadInfo(`read file data successfully ${getFileNameSize(rawFile)} `); @@ -107,3 +116,28 @@ export async function encryptFile( throw e; } } + +/* + Get the original file name for edited file to associate it to original file's metadataJSON file + as edited file doesn't have their own metadata file +*/ +function getFileOriginalName(file: File) { + let originalName: string = null; + const [nameWithoutExtension, extension] = splitFilenameAndExtension( + file.name + ); + + const isEditedFile = nameWithoutExtension.endsWith(EDITED_FILE_SUFFIX); + if (isEditedFile) { + originalName = nameWithoutExtension.slice( + 0, + -1 * EDITED_FILE_SUFFIX.length + ); + } else { + originalName = nameWithoutExtension; + } + if (extension) { + originalName += '.' + extension; + } + return originalName; +} diff --git a/src/services/upload/livePhotoService.ts b/src/services/upload/livePhotoService.ts index 5ad9a2647..def17ad45 100644 --- a/src/services/upload/livePhotoService.ts +++ b/src/services/upload/livePhotoService.ts @@ -10,7 +10,7 @@ import { import { CustomError } from 'utils/error'; import { isImageOrVideo, splitFilenameAndExtension } from 'utils/file'; import { logError } from 'utils/sentry'; -import { getUint8ArrayView } from './readFileService'; +import { getUint8ArrayView } from '../readerService'; import { generateThumbnail } from './thumbnailService'; import uploadService from './uploadService'; import UploadService from './uploadService'; diff --git a/src/services/upload/readFileService.ts b/src/services/upload/readFileService.ts deleted file mode 100644 index 65508afbc..000000000 --- a/src/services/upload/readFileService.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { FILE_TYPE } from 'constants/file'; -import { logError } from 'utils/sentry'; -import { - FILE_READER_CHUNK_SIZE, - FORMAT_MISSED_BY_FILE_TYPE_LIB, - MULTIPART_PART_SIZE, -} from 'constants/upload'; -import FileType from 'file-type/browser'; -import { CustomError } from 'utils/error'; -import { getFileExtension, splitFilenameAndExtension } from 'utils/file'; -import { FileTypeInfo } from 'types/upload'; - -const TYPE_VIDEO = 'video'; -const TYPE_IMAGE = 'image'; -const EDITED_FILE_SUFFIX = '-edited'; -const CHUNK_SIZE_FOR_TYPE_DETECTION = 4100; - -export async function getFileData(reader: FileReader, file: File) { - if (file.size > MULTIPART_PART_SIZE) { - return getFileStream(reader, file, FILE_READER_CHUNK_SIZE); - } else { - return await getUint8ArrayView(reader, file); - } -} - -export async function getFileType( - reader: FileReader, - receivedFile: File -): Promise { - try { - let fileType: FILE_TYPE; - const typeResult = await extractFileType(reader, receivedFile); - const mimTypeParts = typeResult.mime?.split('/'); - if (mimTypeParts?.length !== 2) { - throw Error(CustomError.TYPE_DETECTION_FAILED); - } - switch (mimTypeParts[0]) { - case TYPE_IMAGE: - fileType = FILE_TYPE.IMAGE; - break; - case TYPE_VIDEO: - fileType = FILE_TYPE.VIDEO; - break; - default: - fileType = FILE_TYPE.OTHERS; - } - return { fileType, exactType: typeResult.ext }; - } catch (e) { - const fileFormat = getFileExtension(receivedFile.name); - const formatMissedByTypeDetection = FORMAT_MISSED_BY_FILE_TYPE_LIB.find( - (a) => a.exactType === fileFormat - ); - if (formatMissedByTypeDetection) { - return formatMissedByTypeDetection; - } - logError(e, CustomError.TYPE_DETECTION_FAILED, { - fileFormat, - }); - return { fileType: FILE_TYPE.OTHERS, exactType: fileFormat }; - } -} - -/* - Get the original file name for edited file to associate it to original file's metadataJSON file - as edited file doesn't have their own metadata file -*/ -export function getFileOriginalName(file: File) { - let originalName: string = null; - const [nameWithoutExtension, extension] = splitFilenameAndExtension( - file.name - ); - - const isEditedFile = nameWithoutExtension.endsWith(EDITED_FILE_SUFFIX); - if (isEditedFile) { - originalName = nameWithoutExtension.slice( - 0, - -1 * EDITED_FILE_SUFFIX.length - ); - } else { - originalName = nameWithoutExtension; - } - if (extension) { - originalName += '.' + extension; - } - return originalName; -} - -async function extractFileType(reader: FileReader, file: File) { - const fileChunkBlob = file.slice(0, CHUNK_SIZE_FOR_TYPE_DETECTION); - return getFileTypeFromBlob(reader, fileChunkBlob); -} - -export async function getFileTypeFromBlob(reader: FileReader, fileBlob: Blob) { - try { - const initialFiledata = await getUint8ArrayView(reader, fileBlob); - return await FileType.fromBuffer(initialFiledata); - } catch (e) { - throw Error(CustomError.TYPE_DETECTION_FAILED); - } -} - -function getFileStream(reader: FileReader, file: File, chunkSize: number) { - const fileChunkReader = fileChunkReaderMaker(reader, file, chunkSize); - - const stream = new ReadableStream({ - async pull(controller: ReadableStreamDefaultController) { - const chunk = await fileChunkReader.next(); - if (chunk.done) { - controller.close(); - } else { - controller.enqueue(chunk.value); - } - }, - }); - const chunkCount = Math.ceil(file.size / chunkSize); - return { - stream, - chunkCount, - }; -} - -async function* fileChunkReaderMaker( - reader: FileReader, - file: File, - chunkSize: number -) { - let offset = 0; - while (offset < file.size) { - const blob = file.slice(offset, chunkSize + offset); - const fileChunk = await getUint8ArrayView(reader, blob); - yield fileChunk; - offset += chunkSize; - } - return null; -} - -export async function getUint8ArrayView( - reader: FileReader, - file: Blob -): Promise { - try { - return await new Promise((resolve, reject) => { - reader.onabort = () => reject(Error('file reading was aborted')); - reader.onerror = () => reject(Error('file reading has failed')); - reader.onload = () => { - // Do whatever you want with the file contents - const result = - typeof reader.result === 'string' - ? new TextEncoder().encode(reader.result) - : new Uint8Array(reader.result); - resolve(result); - }; - reader.readAsArrayBuffer(file); - }); - } catch (e) { - logError(e, 'error reading file to byte-array'); - throw e; - } -} diff --git a/src/services/upload/thumbnailService.ts b/src/services/upload/thumbnailService.ts index 29407425d..78c71b77e 100644 --- a/src/services/upload/thumbnailService.ts +++ b/src/services/upload/thumbnailService.ts @@ -6,7 +6,7 @@ import FFmpegService from 'services/ffmpegService'; import { convertToHumanReadable } from 'utils/billing'; import { isFileHEIC } from 'utils/file'; import { FileTypeInfo } from 'types/upload'; -import { getUint8ArrayView } from './readFileService'; +import { getUint8ArrayView } from '../readerService'; import HEICConverter from 'services/HEICConverter'; import { getFileNameSize, logUploadInfo } from 'utils/upload'; diff --git a/src/services/upload/uploadService.ts b/src/services/upload/uploadService.ts index 988a02fd4..1d6ec8bef 100644 --- a/src/services/upload/uploadService.ts +++ b/src/services/upload/uploadService.ts @@ -2,7 +2,7 @@ import { Collection } from 'types/collection'; import { logError } from 'utils/sentry'; import UploadHttpClient from './uploadHttpClient'; import { extractFileMetadata, getFilename } from './fileService'; -import { getFileType } from './readFileService'; +import { getFileType } from '../typeDetectionService'; import { handleUploadError } from 'utils/error'; import { B64EncryptionResult, diff --git a/src/utils/file/index.ts b/src/utils/file/index.ts index 5ad062ed8..cb3cd284e 100644 --- a/src/utils/file/index.ts +++ b/src/utils/file/index.ts @@ -7,7 +7,7 @@ import { PublicMagicMetadataProps, } from 'types/file'; import { decodeMotionPhoto } from 'services/motionPhotoService'; -import { getFileTypeFromBlob } from 'services/upload/readFileService'; +import { getFileTypeFromBlob } from 'services/typeDetectionService'; import DownloadManager from 'services/downloadManager'; import { logError } from 'utils/sentry'; import { User } from 'types/user';