diff --git a/apps/photos/src/constants/upload.ts b/apps/photos/src/constants/upload.ts index d12bb2255..6ec37fb73 100644 --- a/apps/photos/src/constants/upload.ts +++ b/apps/photos/src/constants/upload.ts @@ -85,6 +85,8 @@ export const LIVE_PHOTO_ASSET_SIZE_LIMIT = 20 * 1024 * 1024; // 20MB export const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = { location: NULL_LOCATION, creationTime: null, + width: null, + height: null, }; export const A_SEC_IN_MICROSECONDS = 1e6; diff --git a/apps/photos/src/services/upload/exifService.ts b/apps/photos/src/services/upload/exifService.ts index c05e5c1d3..3d2385433 100644 --- a/apps/photos/src/services/upload/exifService.ts +++ b/apps/photos/src/services/upload/exifService.ts @@ -18,6 +18,8 @@ type ParsedEXIFData = Record & MetadataDate: Date; latitude: number; longitude: number; + imageWidth: number; + imageHeight: number; }>; type RawEXIFData = Record & @@ -31,6 +33,8 @@ type RawEXIFData = Record & GPSLongitude: number[]; GPSLatitudeRef: string; GPSLongitudeRef: string; + ImageWidth: number; + ImageHeight: number; }>; export async function getParsedExifData( @@ -83,6 +87,12 @@ function parseExifData(exifData: RawEXIFData): ParsedEXIFData { CreateDate, ModifyDate, DateCreated, + ImageHeight, + ImageWidth, + ExifImageHeight, + ExifImageWidth, + PixelXDimension, + PixelYDimension, MetadataDate, ...rest } = exifData; @@ -99,6 +109,9 @@ function parseExifData(exifData: RawEXIFData): ParsedEXIFData { if (DateCreated) { parsedExif.DateCreated = parseEXIFDate(exifData.DateCreated); } + if (MetadataDate) { + parsedExif.MetadataDate = parseEXIFDate(exifData.MetadataDate); + } if (exifData.GPSLatitude && exifData.GPSLongitude) { const parsedLocation = parseEXIFLocation( exifData.GPSLatitude, @@ -109,8 +122,54 @@ function parseExifData(exifData: RawEXIFData): ParsedEXIFData { parsedExif.latitude = parsedLocation.latitude; parsedExif.longitude = parsedLocation.longitude; } - if (MetadataDate) { - parsedExif.MetadataDate = parseEXIFDate(exifData.MetadataDate); + if (ImageWidth && ImageHeight) { + if (typeof ImageWidth === 'number' && typeof ImageHeight === 'number') { + parsedExif.imageWidth = ImageWidth; + parsedExif.imageHeight = ImageHeight; + } else { + logError( + new Error('ImageWidth or ImageHeight is not a number'), + 'Image dimension parsing failed', + { + ImageWidth, + ImageHeight, + } + ); + } + } else if (ExifImageWidth && ExifImageHeight) { + if ( + typeof ExifImageWidth === 'number' && + typeof ExifImageHeight === 'number' + ) { + parsedExif.imageWidth = ExifImageWidth; + parsedExif.imageHeight = ExifImageHeight; + } else { + logError( + new Error('ExifImageWidth or ExifImageHeight is not a number'), + 'Image dimension parsing failed', + { + ExifImageWidth, + ExifImageHeight, + } + ); + } + } else if (PixelXDimension && PixelYDimension) { + if ( + typeof PixelXDimension === 'number' && + typeof PixelYDimension === 'number' + ) { + parsedExif.imageWidth = PixelXDimension; + parsedExif.imageHeight = PixelYDimension; + } else { + logError( + new Error('PixelXDimension or PixelYDimension is not a number'), + 'Image dimension parsing failed', + { + PixelXDimension, + PixelYDimension, + } + ); + } } return parsedExif; } diff --git a/apps/photos/src/services/upload/fileService.ts b/apps/photos/src/services/upload/fileService.ts index 38a32455e..ca3c5dc55 100644 --- a/apps/photos/src/services/upload/fileService.ts +++ b/apps/photos/src/services/upload/fileService.ts @@ -2,12 +2,12 @@ import { MULTIPART_PART_SIZE, FILE_READER_CHUNK_SIZE } from 'constants/upload'; import { FileTypeInfo, FileInMemory, - Metadata, EncryptedFile, FileWithMetadata, ParsedMetadataJSONMap, DataStream, ElectronFile, + ExtractMetadataResult, } from 'types/upload'; import { splitFilenameAndExtension } from 'utils/file'; import { logError } from 'utils/sentry'; @@ -74,13 +74,14 @@ export async function extractFileMetadata( collectionID: number, fileTypeInfo: FileTypeInfo, rawFile: File | ElectronFile -) { +): Promise { const originalName = getFileOriginalName(rawFile); const googleMetadata = parsedMetadataJSONMap.get( getMetadataJSONMapKey(collectionID, originalName) ) ?? {}; - const extractedMetadata: Metadata = await extractMetadata( + + const { metadata, publicMagicMetadata } = await extractMetadata( worker, rawFile, fileTypeInfo @@ -90,9 +91,9 @@ export async function extractFileMetadata( if (!value) { continue; } - extractedMetadata[key] = value; + metadata[key] = value; } - return extractedMetadata; + return { metadata, publicMagicMetadata }; } export async function encryptFile( diff --git a/apps/photos/src/services/upload/livePhotoService.ts b/apps/photos/src/services/upload/livePhotoService.ts index 98a21a803..fd345a345 100644 --- a/apps/photos/src/services/upload/livePhotoService.ts +++ b/apps/photos/src/services/upload/livePhotoService.ts @@ -8,6 +8,7 @@ import { FileWithCollection, LivePhotoAssets, ParsedMetadataJSONMap, + ExtractMetadataResult, } from 'types/upload'; import { CustomError } from 'utils/error'; import { getFileTypeFromExtensionForLivePhotoClustering } from 'utils/file/livePhoto'; @@ -51,12 +52,15 @@ export async function extractLivePhotoMetadata( collectionID: number, fileTypeInfo: FileTypeInfo, livePhotoAssets: LivePhotoAssets -) { +): Promise { const imageFileTypeInfo: FileTypeInfo = { fileType: FILE_TYPE.IMAGE, exactType: fileTypeInfo.imageType, }; - const imageMetadata = await extractFileMetadata( + const { + metadata: imageMetadata, + publicMagicMetadata: imagePublicMagicMetadata, + } = await extractFileMetadata( worker, parsedMetadataJSONMap, collectionID, @@ -65,12 +69,15 @@ export async function extractLivePhotoMetadata( ); const videoHash = await getFileHash(worker, livePhotoAssets.video); return { - ...imageMetadata, - title: getLivePhotoName(livePhotoAssets), - fileType: FILE_TYPE.LIVE_PHOTO, - imageHash: imageMetadata.hash, - videoHash: videoHash, - hash: undefined, + metadata: { + ...imageMetadata, + title: getLivePhotoName(livePhotoAssets), + fileType: FILE_TYPE.LIVE_PHOTO, + imageHash: imageMetadata.hash, + videoHash: videoHash, + hash: undefined, + }, + publicMagicMetadata: imagePublicMagicMetadata, }; } diff --git a/apps/photos/src/services/upload/magicMetadataService.ts b/apps/photos/src/services/upload/magicMetadataService.ts index ad19072cc..1fff0ffcc 100644 --- a/apps/photos/src/services/upload/magicMetadataService.ts +++ b/apps/photos/src/services/upload/magicMetadataService.ts @@ -2,13 +2,20 @@ import { FilePublicMagicMetadataProps, FilePublicMagicMetadata, } from 'types/file'; -import { updateMagicMetadata } from 'utils/magicMetadata'; +import { + getNonEmptyMagicMetadataProps, + updateMagicMetadata, +} from 'utils/magicMetadata'; export async function constructPublicMagicMetadata( publicMagicMetadataProps: FilePublicMagicMetadataProps ): Promise { - const pubMagicMetadata = await updateMagicMetadata( + const nonEmptyPublicMagicMetadataProps = getNonEmptyMagicMetadataProps( publicMagicMetadataProps ); - return pubMagicMetadata; + + if (Object.values(nonEmptyPublicMagicMetadataProps)?.length === 0) { + return null; + } + return await updateMagicMetadata(publicMagicMetadataProps); } diff --git a/apps/photos/src/services/upload/metadataService.ts b/apps/photos/src/services/upload/metadataService.ts index 8ec04e1e0..19c351a6b 100644 --- a/apps/photos/src/services/upload/metadataService.ts +++ b/apps/photos/src/services/upload/metadataService.ts @@ -8,6 +8,7 @@ import { FileTypeInfo, ParsedExtractedMetadata, ElectronFile, + ExtractMetadataResult as ExtractMetadataResult, } from 'types/upload'; import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload'; import { getVideoMetadata } from './videoMetadataService'; @@ -19,6 +20,7 @@ import { import { getFileHash } from './hashService'; import { Remote } from 'comlink'; import { DedicatedCryptoWorker } from 'worker/crypto.worker'; +import { FilePublicMagicMetadataProps } from 'types/file'; interface ParsedMetadataJSONWithTitle { title: string; @@ -40,6 +42,12 @@ const EXIF_TAGS_NEEDED = [ 'GPSLatitudeRef', 'GPSLongitudeRef', 'DateCreated', + 'ExifImageWidth', + 'ExifImageHeight', + 'ImageWidth', + 'ImageHeight', + 'PixelXDimension', + 'PixelYDimension', 'MetadataDate', ]; @@ -47,7 +55,7 @@ export async function extractMetadata( worker: Remote, receivedFile: File | ElectronFile, fileTypeInfo: FileTypeInfo -) { +): Promise { let extractedMetadata: ParsedExtractedMetadata = NULL_EXTRACTED_METADATA; if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) { extractedMetadata = await getImageMetadata(receivedFile, fileTypeInfo); @@ -68,7 +76,11 @@ export async function extractMetadata( fileType: fileTypeInfo.fileType, hash: fileHash, }; - return metadata; + const publicMagicMetadata: FilePublicMagicMetadataProps = { + w: extractedMetadata.width, + h: extractedMetadata.height, + }; + return { metadata, publicMagicMetadata }; } export async function getImageMetadata( @@ -91,9 +103,12 @@ export async function getImageMetadata( fileTypeInfo, EXIF_TAGS_NEEDED ); + imageMetadata = { location: getEXIFLocation(exifData), creationTime: getEXIFTime(exifData), + width: exifData?.imageWidth ?? null, + height: exifData?.imageHeight ?? null, }; } catch (e) { logError(e, 'getExifData failed'); diff --git a/apps/photos/src/services/upload/uploadService.ts b/apps/photos/src/services/upload/uploadService.ts index 470c982f1..f345cee47 100644 --- a/apps/photos/src/services/upload/uploadService.ts +++ b/apps/photos/src/services/upload/uploadService.ts @@ -7,11 +7,11 @@ import { CustomError, handleUploadError } from 'utils/error'; import { BackupedFile, EncryptedFile, + ExtractMetadataResult, FileTypeInfo, FileWithCollection, FileWithMetadata, isDataStream, - Metadata, ParsedMetadataJSON, ParsedMetadataJSONMap, ProcessedFile, @@ -109,7 +109,7 @@ class UploadService { { isLivePhoto, file, livePhotoAssets }: UploadAsset, collectionID: number, fileTypeInfo: FileTypeInfo - ): Promise { + ): Promise { return isLivePhoto ? extractLivePhotoMetadata( worker, diff --git a/apps/photos/src/services/upload/uploader.ts b/apps/photos/src/services/upload/uploader.ts index 6c5c44c57..1ae675760 100644 --- a/apps/photos/src/services/upload/uploader.ts +++ b/apps/photos/src/services/upload/uploader.ts @@ -1,4 +1,4 @@ -import { EnteFile, FilePublicMagicMetadata } from 'types/file'; +import { EnteFile } from 'types/file'; import { handleUploadError, CustomError } from 'utils/error'; import { logError } from 'utils/sentry'; import { findMatchingExistingFiles } from 'utils/upload'; @@ -54,12 +54,13 @@ export default async function uploader( ); addLogLine(`extracting metadata ${fileNameSize}`); - const metadata = await UploadService.extractAssetMetadata( - worker, - uploadAsset, - collection.id, - fileTypeInfo - ); + const { metadata, publicMagicMetadata } = + await UploadService.extractAssetMetadata( + worker, + uploadAsset, + collection.id, + fileTypeInfo + ); const matchingExistingFiles = findMatchingExistingFiles( existingFiles, @@ -115,12 +116,13 @@ export default async function uploader( if (file.hasStaticThumbnail) { metadata.hasStaticThumbnail = true; } - let pubMagicMetadata: FilePublicMagicMetadata; - if (uploaderName) { - pubMagicMetadata = await uploadService.constructPublicMagicMetadata( - { uploaderName } - ); - } + + const pubMagicMetadata = + await uploadService.constructPublicMagicMetadata({ + ...publicMagicMetadata, + uploaderName, + }); + const fileWithMetadata: FileWithMetadata = { localID, filedata: file.filedata, diff --git a/apps/photos/src/types/file/index.ts b/apps/photos/src/types/file/index.ts index 7514ce628..62455f34c 100644 --- a/apps/photos/src/types/file/index.ts +++ b/apps/photos/src/types/file/index.ts @@ -93,6 +93,8 @@ export interface FilePublicMagicMetadataProps { editedName?: string; caption?: string; uploaderName?: string; + w?: number; + h?: number; } export type FilePublicMagicMetadata = diff --git a/apps/photos/src/types/upload/index.ts b/apps/photos/src/types/upload/index.ts index aecb2d64d..085e7f645 100644 --- a/apps/photos/src/types/upload/index.ts +++ b/apps/photos/src/types/upload/index.ts @@ -5,6 +5,7 @@ import { MetadataFileAttributes, S3FileAttributes, FilePublicMagicMetadata, + FilePublicMagicMetadataProps, } from 'types/file'; import { EncryptedMagicMetadata } from 'types/magicMetadata'; @@ -138,6 +139,8 @@ export interface UploadFile extends BackupedFile { export interface ParsedExtractedMetadata { location: Location; creationTime: number; + width: number; + height: number; } // This is used to prompt the user the make upload strategy choice @@ -152,3 +155,8 @@ export interface PublicUploadProps { passwordToken: string; accessedThroughSharedURL: boolean; } + +export interface ExtractMetadataResult { + metadata: Metadata; + publicMagicMetadata: FilePublicMagicMetadataProps; +} diff --git a/apps/photos/src/utils/ffmpeg/index.ts b/apps/photos/src/utils/ffmpeg/index.ts index 916806c3e..80673c907 100644 --- a/apps/photos/src/utils/ffmpeg/index.ts +++ b/apps/photos/src/utils/ffmpeg/index.ts @@ -38,6 +38,8 @@ export function parseFFmpegExtractedMetadata(encodedMetadata: Uint8Array) { latitude: location.latitude, longitude: location.longitude, }, + width: null, + height: null, }; return parsedMetadata; } diff --git a/apps/photos/src/utils/magicMetadata/index.ts b/apps/photos/src/utils/magicMetadata/index.ts index 5af73e2ce..457a451db 100644 --- a/apps/photos/src/utils/magicMetadata/index.ts +++ b/apps/photos/src/utils/magicMetadata/index.ts @@ -28,29 +28,29 @@ export async function updateMagicMetadata( } if (typeof originalMagicMetadata?.data === 'string') { - originalMagicMetadata.data = (await cryptoWorker.decryptMetadata( + originalMagicMetadata.data = await cryptoWorker.decryptMetadata( originalMagicMetadata.data, originalMagicMetadata.header, decryptionKey - )) as T; + ); } + // copies the existing magic metadata properties of the files and updates the visibility value + // The expected behavior while updating magic metadata is to let the existing property as it is and update/add the property you want + const magicMetadataProps: T = { + ...originalMagicMetadata.data, + ...magicMetadataUpdates, + }; - if (magicMetadataUpdates) { - // copies the existing magic metadata properties of the files and updates the visibility value - // The expected behavior while updating magic metadata is to let the existing property as it is and update/add the property you want - const magicMetadataProps: T = { - ...originalMagicMetadata.data, - ...magicMetadataUpdates, - }; + const nonEmptyMagicMetadataProps = + getNonEmptyMagicMetadataProps(magicMetadataProps); - return { - ...originalMagicMetadata, - data: magicMetadataProps, - count: Object.keys(magicMetadataProps).length, - }; - } else { - return originalMagicMetadata; - } + const magicMetadata = { + ...originalMagicMetadata, + data: nonEmptyMagicMetadataProps, + count: Object.keys(nonEmptyMagicMetadataProps).length, + }; + + return magicMetadata; } export const getNewMagicMetadata = (): MagicMetadataCore => { @@ -61,3 +61,12 @@ export const getNewMagicMetadata = (): MagicMetadataCore => { count: 0, }; }; + +export const getNonEmptyMagicMetadataProps = (magicMetadataProps: T): T => { + return Object.fromEntries( + Object.entries(magicMetadataProps).filter( + // eslint-disable-next-line @typescript-eslint/no-unused-vars + ([_, v]) => v !== null && v !== undefined + ) + ) as T; +}; diff --git a/apps/photos/tests/upload.test.ts b/apps/photos/tests/upload.test.ts index 53a0fc6ae..d7f26fda9 100644 --- a/apps/photos/tests/upload.test.ts +++ b/apps/photos/tests/upload.test.ts @@ -23,6 +23,7 @@ export async function testUpload() { await thumbnailGenerationFailedFilesCheck(expectedState); await livePhotoClubbingCheck(expectedState); await exifDataParsingCheck(expectedState); + await fileDimensionExtractionCheck(expectedState); await googleMetadataReadingCheck(expectedState); await totalFileCountCheck(expectedState); } catch (e) { @@ -210,6 +211,33 @@ async function exifDataParsingCheck(expectedState) { console.log('exif data parsing check passed ✅'); } +async function fileDimensionExtractionCheck(expectedState) { + const files = await getLocalFiles(); + Object.entries(expectedState['file_dimensions']).map( + ([fileName, dimensions]) => { + const matchingFile = files.find( + (file) => file.metadata.title === fileName + ); + if (!matchingFile) { + throw Error( + `fileDimensionExtractionCheck failed , ${fileName} missing` + ); + } + if ( + dimensions['width'] && + dimensions['width'] !== matchingFile.pubMagicMetadata.data.w && + dimensions['height'] && + dimensions['height'] !== matchingFile.pubMagicMetadata.data.h + ) { + throw Error(`fileDimensionExtractionCheck failed ❌ , + for ${fileName} + expected: ${dimensions['width']} x ${dimensions['height']} got: ${matchingFile.pubMagicMetadata.data.w} x ${matchingFile.pubMagicMetadata.data.h}`); + } + } + ); + console.log('file dimension extraction check passed ✅'); +} + async function googleMetadataReadingCheck(expectedState) { const files = await getLocalFiles(); Object.entries(expectedState['google_import']).map(