beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Image exif aspect ratio (#1162)

Browse source
This commit is contained in:
Abhinav Kumar 2023-06-06 13:58:14 +05:30 committed by GitHub
parent aff199f37b 5255152883
commit 8a9a46deec
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 194 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
apps/photos/src/constants/upload.ts
View file

@ -85,6 +85,8 @@ export const LIVE_PHOTO_ASSET_SIZE_LIMIT = 20 * 1024 * 1024; // 20MB
export const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = {
location: NULL_LOCATION,
creationTime: null,
width: null,
height: null,
};
export const A_SEC_IN_MICROSECONDS = 1e6;

63
apps/photos/src/services/upload/exifService.ts
View file

@ -18,6 +18,8 @@ type ParsedEXIFData = Record<string, any> &
MetadataDate: Date;
latitude: number;
longitude: number;
imageWidth: number;
imageHeight: number;
}>;
type RawEXIFData = Record<string, any> &
@ -31,6 +33,8 @@ type RawEXIFData = Record<string, any> &
GPSLongitude: number[];
GPSLatitudeRef: string;
GPSLongitudeRef: string;
ImageWidth: number;
ImageHeight: number;
}>;
export async function getParsedExifData(
@ -83,6 +87,12 @@ function parseExifData(exifData: RawEXIFData): ParsedEXIFData {
CreateDate,
ModifyDate,
DateCreated,
ImageHeight,
ImageWidth,
ExifImageHeight,
ExifImageWidth,
PixelXDimension,
PixelYDimension,
MetadataDate,
...rest
} = exifData;
@ -99,6 +109,9 @@ function parseExifData(exifData: RawEXIFData): ParsedEXIFData {
if (DateCreated) {
parsedExif.DateCreated = parseEXIFDate(exifData.DateCreated);
}
if (MetadataDate) {
parsedExif.MetadataDate = parseEXIFDate(exifData.MetadataDate);
}
if (exifData.GPSLatitude && exifData.GPSLongitude) {
const parsedLocation = parseEXIFLocation(
exifData.GPSLatitude,
@ -109,8 +122,54 @@ function parseExifData(exifData: RawEXIFData): ParsedEXIFData {
parsedExif.latitude = parsedLocation.latitude;
parsedExif.longitude = parsedLocation.longitude;
}
if (MetadataDate) {
parsedExif.MetadataDate = parseEXIFDate(exifData.MetadataDate);
if (ImageWidth && ImageHeight) {
if (typeof ImageWidth === 'number' && typeof ImageHeight === 'number') {
parsedExif.imageWidth = ImageWidth;
parsedExif.imageHeight = ImageHeight;
} else {
logError(
new Error('ImageWidth or ImageHeight is not a number'),
'Image dimension parsing failed',
{
ImageWidth,
ImageHeight,
}
);
}
} else if (ExifImageWidth && ExifImageHeight) {
if (
typeof ExifImageWidth === 'number' &&
typeof ExifImageHeight === 'number'
) {
parsedExif.imageWidth = ExifImageWidth;
parsedExif.imageHeight = ExifImageHeight;
} else {
logError(
new Error('ExifImageWidth or ExifImageHeight is not a number'),
'Image dimension parsing failed',
{
ExifImageWidth,
ExifImageHeight,
}
);
}
} else if (PixelXDimension && PixelYDimension) {
if (
typeof PixelXDimension === 'number' &&
typeof PixelYDimension === 'number'
) {
parsedExif.imageWidth = PixelXDimension;
parsedExif.imageHeight = PixelYDimension;
} else {
logError(
new Error('PixelXDimension or PixelYDimension is not a number'),
'Image dimension parsing failed',
{
PixelXDimension,
PixelYDimension,
}
);
}
}
return parsedExif;
}

11
apps/photos/src/services/upload/fileService.ts
View file

@ -2,12 +2,12 @@ import { MULTIPART_PART_SIZE, FILE_READER_CHUNK_SIZE } from 'constants/upload';
import {
FileTypeInfo,
FileInMemory,
Metadata,
EncryptedFile,
FileWithMetadata,
ParsedMetadataJSONMap,
DataStream,
ElectronFile,
ExtractMetadataResult,
} from 'types/upload';
import { splitFilenameAndExtension } from 'utils/file';
import { logError } from 'utils/sentry';
@ -74,13 +74,14 @@ export async function extractFileMetadata(
collectionID: number,
fileTypeInfo: FileTypeInfo,
rawFile: File | ElectronFile
) {
): Promise<ExtractMetadataResult> {
const originalName = getFileOriginalName(rawFile);
const googleMetadata =
parsedMetadataJSONMap.get(
getMetadataJSONMapKey(collectionID, originalName)
) ?? {};
const extractedMetadata: Metadata = await extractMetadata(
const { metadata, publicMagicMetadata } = await extractMetadata(
worker,
rawFile,
fileTypeInfo
@ -90,9 +91,9 @@ export async function extractFileMetadata(
if (!value) {
continue;
}
extractedMetadata[key] = value;
metadata[key] = value;
}
return extractedMetadata;
return { metadata, publicMagicMetadata };
}
export async function encryptFile(

11
apps/photos/src/services/upload/livePhotoService.ts
View file

@ -8,6 +8,7 @@ import {
FileWithCollection,
LivePhotoAssets,
ParsedMetadataJSONMap,
ExtractMetadataResult,
} from 'types/upload';
import { CustomError } from 'utils/error';
import { getFileTypeFromExtensionForLivePhotoClustering } from 'utils/file/livePhoto';
@ -51,12 +52,15 @@ export async function extractLivePhotoMetadata(
collectionID: number,
fileTypeInfo: FileTypeInfo,
livePhotoAssets: LivePhotoAssets
) {
): Promise<ExtractMetadataResult> {
const imageFileTypeInfo: FileTypeInfo = {
fileType: FILE_TYPE.IMAGE,
exactType: fileTypeInfo.imageType,
};
const imageMetadata = await extractFileMetadata(
const {
metadata: imageMetadata,
publicMagicMetadata: imagePublicMagicMetadata,
} = await extractFileMetadata(
worker,
parsedMetadataJSONMap,
collectionID,
@ -65,12 +69,15 @@ export async function extractLivePhotoMetadata(
);
const videoHash = await getFileHash(worker, livePhotoAssets.video);
return {
metadata: {
...imageMetadata,
title: getLivePhotoName(livePhotoAssets),
fileType: FILE_TYPE.LIVE_PHOTO,
imageHash: imageMetadata.hash,
videoHash: videoHash,
hash: undefined,
},
publicMagicMetadata: imagePublicMagicMetadata,
};
}

13
apps/photos/src/services/upload/magicMetadataService.ts
View file

@ -2,13 +2,20 @@ import {
FilePublicMagicMetadataProps,
FilePublicMagicMetadata,
} from 'types/file';
import { updateMagicMetadata } from 'utils/magicMetadata';
import {
getNonEmptyMagicMetadataProps,
updateMagicMetadata,
} from 'utils/magicMetadata';
export async function constructPublicMagicMetadata(
publicMagicMetadataProps: FilePublicMagicMetadataProps
): Promise<FilePublicMagicMetadata> {
const pubMagicMetadata = await updateMagicMetadata(
const nonEmptyPublicMagicMetadataProps = getNonEmptyMagicMetadataProps(
publicMagicMetadataProps
);
return pubMagicMetadata;
if (Object.values(nonEmptyPublicMagicMetadataProps)?.length === 0) {
return null;
}
return await updateMagicMetadata(publicMagicMetadataProps);
}

19
apps/photos/src/services/upload/metadataService.ts
View file

@ -8,6 +8,7 @@ import {
FileTypeInfo,
ParsedExtractedMetadata,
ElectronFile,
ExtractMetadataResult as ExtractMetadataResult,
} from 'types/upload';
import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload';
import { getVideoMetadata } from './videoMetadataService';
@ -19,6 +20,7 @@ import {
import { getFileHash } from './hashService';
import { Remote } from 'comlink';
import { DedicatedCryptoWorker } from 'worker/crypto.worker';
import { FilePublicMagicMetadataProps } from 'types/file';
interface ParsedMetadataJSONWithTitle {
title: string;
@ -40,6 +42,12 @@ const EXIF_TAGS_NEEDED = [
'GPSLatitudeRef',
'GPSLongitudeRef',
'DateCreated',
'ExifImageWidth',
'ExifImageHeight',
'ImageWidth',
'ImageHeight',
'PixelXDimension',
'PixelYDimension',
'MetadataDate',
];
@ -47,7 +55,7 @@ export async function extractMetadata(
worker: Remote<DedicatedCryptoWorker>,
receivedFile: File | ElectronFile,
fileTypeInfo: FileTypeInfo
) {
): Promise<ExtractMetadataResult> {
let extractedMetadata: ParsedExtractedMetadata = NULL_EXTRACTED_METADATA;
if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) {
extractedMetadata = await getImageMetadata(receivedFile, fileTypeInfo);
@ -68,7 +76,11 @@ export async function extractMetadata(
fileType: fileTypeInfo.fileType,
hash: fileHash,
};
return metadata;
const publicMagicMetadata: FilePublicMagicMetadataProps = {
w: extractedMetadata.width,
h: extractedMetadata.height,
};
return { metadata, publicMagicMetadata };
}
export async function getImageMetadata(
@ -91,9 +103,12 @@ export async function getImageMetadata(
fileTypeInfo,
EXIF_TAGS_NEEDED
);
imageMetadata = {
location: getEXIFLocation(exifData),
creationTime: getEXIFTime(exifData),
width: exifData?.imageWidth ?? null,
height: exifData?.imageHeight ?? null,
};
} catch (e) {
logError(e, 'getExifData failed');

4
apps/photos/src/services/upload/uploadService.ts
View file

@ -7,11 +7,11 @@ import { CustomError, handleUploadError } from 'utils/error';
import {
BackupedFile,
EncryptedFile,
ExtractMetadataResult,
FileTypeInfo,
FileWithCollection,
FileWithMetadata,
isDataStream,
Metadata,
ParsedMetadataJSON,
ParsedMetadataJSONMap,
ProcessedFile,
@ -109,7 +109,7 @@ class UploadService {
{ isLivePhoto, file, livePhotoAssets }: UploadAsset,
collectionID: number,
fileTypeInfo: FileTypeInfo
): Promise<Metadata> {
): Promise<ExtractMetadataResult> {
return isLivePhoto
? extractLivePhotoMetadata(
worker,

18
apps/photos/src/services/upload/uploader.ts
View file

@ -1,4 +1,4 @@
import { EnteFile, FilePublicMagicMetadata } from 'types/file';
import { EnteFile } from 'types/file';
import { handleUploadError, CustomError } from 'utils/error';
import { logError } from 'utils/sentry';
import { findMatchingExistingFiles } from 'utils/upload';
@ -54,7 +54,8 @@ export default async function uploader(
);
addLogLine(`extracting metadata ${fileNameSize}`);
const metadata = await UploadService.extractAssetMetadata(
const { metadata, publicMagicMetadata } =
await UploadService.extractAssetMetadata(
worker,
uploadAsset,
collection.id,
@ -115,12 +116,13 @@ export default async function uploader(
if (file.hasStaticThumbnail) {
metadata.hasStaticThumbnail = true;
}
let pubMagicMetadata: FilePublicMagicMetadata;
if (uploaderName) {
pubMagicMetadata = await uploadService.constructPublicMagicMetadata(
{ uploaderName }
);
}
const pubMagicMetadata =
await uploadService.constructPublicMagicMetadata({
...publicMagicMetadata,
uploaderName,
});
const fileWithMetadata: FileWithMetadata = {
localID,
filedata: file.filedata,

2
apps/photos/src/types/file/index.ts
View file

@ -93,6 +93,8 @@ export interface FilePublicMagicMetadataProps {
editedName?: string;
caption?: string;
uploaderName?: string;
w?: number;
h?: number;
}
export type FilePublicMagicMetadata =

8
apps/photos/src/types/upload/index.ts
View file

@ -5,6 +5,7 @@ import {
MetadataFileAttributes,
S3FileAttributes,
FilePublicMagicMetadata,
FilePublicMagicMetadataProps,
} from 'types/file';
import { EncryptedMagicMetadata } from 'types/magicMetadata';
@ -138,6 +139,8 @@ export interface UploadFile extends BackupedFile {
export interface ParsedExtractedMetadata {
location: Location;
creationTime: number;
width: number;
height: number;
}
// This is used to prompt the user the make upload strategy choice
@ -152,3 +155,8 @@ export interface PublicUploadProps {
passwordToken: string;
accessedThroughSharedURL: boolean;
}
export interface ExtractMetadataResult {
metadata: Metadata;
publicMagicMetadata: FilePublicMagicMetadataProps;
}

2
apps/photos/src/utils/ffmpeg/index.ts
View file

@ -38,6 +38,8 @@ export function parseFFmpegExtractedMetadata(encodedMetadata: Uint8Array) {
latitude: location.latitude,
longitude: location.longitude,
},
width: null,
height: null,
};
return parsedMetadata;
}

29
apps/photos/src/utils/magicMetadata/index.ts
View file

@ -28,14 +28,12 @@ export async function updateMagicMetadata<T>(
}
if (typeof originalMagicMetadata?.data === 'string') {
originalMagicMetadata.data = (await cryptoWorker.decryptMetadata(
originalMagicMetadata.data = await cryptoWorker.decryptMetadata(
originalMagicMetadata.data,
originalMagicMetadata.header,
decryptionKey
)) as T;
);
}
if (magicMetadataUpdates) {
// copies the existing magic metadata properties of the files and updates the visibility value
// The expected behavior while updating magic metadata is to let the existing property as it is and update/add the property you want
const magicMetadataProps: T = {
@ -43,14 +41,16 @@ export async function updateMagicMetadata<T>(
...magicMetadataUpdates,
};
return {
const nonEmptyMagicMetadataProps =
getNonEmptyMagicMetadataProps(magicMetadataProps);
const magicMetadata = {
...originalMagicMetadata,
data: magicMetadataProps,
count: Object.keys(magicMetadataProps).length,
data: nonEmptyMagicMetadataProps,
count: Object.keys(nonEmptyMagicMetadataProps).length,
};
} else {
return originalMagicMetadata;
}
return magicMetadata;
}
export const getNewMagicMetadata = <T>(): MagicMetadataCore<T> => {
@ -61,3 +61,12 @@ export const getNewMagicMetadata = <T>(): MagicMetadataCore<T> => {
count: 0,
};
};
export const getNonEmptyMagicMetadataProps = <T>(magicMetadataProps: T): T => {
return Object.fromEntries(
Object.entries(magicMetadataProps).filter(
// eslint-disable-next-line @typescript-eslint/no-unused-vars
([_, v]) => v !== null && v !== undefined
)
) as T;
};

28
apps/photos/tests/upload.test.ts
View file

@ -23,6 +23,7 @@ export async function testUpload() {
await thumbnailGenerationFailedFilesCheck(expectedState);
await livePhotoClubbingCheck(expectedState);
await exifDataParsingCheck(expectedState);
await fileDimensionExtractionCheck(expectedState);
await googleMetadataReadingCheck(expectedState);
await totalFileCountCheck(expectedState);
} catch (e) {
@ -210,6 +211,33 @@ async function exifDataParsingCheck(expectedState) {
console.log('exif data parsing check passed ✅');
}
async function fileDimensionExtractionCheck(expectedState) {
const files = await getLocalFiles();
Object.entries(expectedState['file_dimensions']).map(
([fileName, dimensions]) => {
const matchingFile = files.find(
(file) => file.metadata.title === fileName
);
if (!matchingFile) {
throw Error(
`fileDimensionExtractionCheck failed , ${fileName} missing`
);
}
if (
dimensions['width'] &&
dimensions['width'] !== matchingFile.pubMagicMetadata.data.w &&
dimensions['height'] &&
dimensions['height'] !== matchingFile.pubMagicMetadata.data.h
) {
throw Error(`fileDimensionExtractionCheck failed ❌ ,
for ${fileName}
expected: ${dimensions['width']} x ${dimensions['height']} got: ${matchingFile.pubMagicMetadata.data.w} x ${matchingFile.pubMagicMetadata.data.h}`);
}
}
);
console.log('file dimension extraction check passed ✅');
}
async function googleMetadataReadingCheck(expectedState) {
const files = await getLocalFiles();
Object.entries(expectedState['google_import']).map(