diff --git a/src/constants/upload/index.ts b/src/constants/upload/index.ts index 2109e566b..fb9f0a15d 100644 --- a/src/constants/upload/index.ts +++ b/src/constants/upload/index.ts @@ -1,6 +1,6 @@ import { ENCRYPTION_CHUNK_SIZE } from 'constants/crypto'; import { FILE_TYPE } from 'constants/file'; -import { Location } from 'types/upload'; +import { Location, ParsedExtractedMetadata } from 'types/upload'; // list of format that were missed by type-detection for some files. export const FORMAT_MISSED_BY_FILE_TYPE_LIB = [ @@ -43,3 +43,8 @@ export enum FileUploadResults { export const MAX_FILE_SIZE_SUPPORTED = 5 * 1024 * 1024 * 1024; // 5 GB export const LIVE_PHOTO_ASSET_SIZE_LIMIT = 20 * 1024 * 1024; // 20MB + +export const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = { + location: NULL_LOCATION, + creationTime: null, +}; diff --git a/src/services/ffmpegService.ts b/src/services/ffmpegService.ts index 3914cfea2..30e1e5fcd 100644 --- a/src/services/ffmpegService.ts +++ b/src/services/ffmpegService.ts @@ -2,14 +2,17 @@ import { createFFmpeg, FFmpeg } from '@ffmpeg/ffmpeg'; import { CustomError } from 'utils/error'; import { logError } from 'utils/sentry'; import QueueProcessor from './queueProcessor'; +import { ParsedExtractedMetadata } from 'types/upload'; + import { getUint8ArrayView } from './upload/readFileService'; +import { parseFFmpegExtractedMetadata } from './upload/videoMetadataService'; class FFmpegService { private ffmpeg: FFmpeg = null; private isLoading = null; private fileReader: FileReader = null; - private generateThumbnailProcessor = new QueueProcessor(1); + private ffmpegTaskQueue = new QueueProcessor(1); async init() { try { this.ffmpeg = createFFmpeg({ @@ -26,7 +29,7 @@ class FFmpegService { } } - async generateThumbnail(file: File) { + async generateThumbnail(file: File): Promise { if (!this.ffmpeg) { await this.init(); } @@ -36,7 +39,7 @@ class FFmpegService { if (this.isLoading) { await this.isLoading; } - const response = this.generateThumbnailProcessor.queueUpRequest( + const response = this.ffmpegTaskQueue.queueUpRequest( generateThumbnailHelper.bind( null, this.ffmpeg, @@ -56,6 +59,37 @@ class FFmpegService { } } } + + async extractMetadata(file: File): Promise { + if (!this.ffmpeg) { + await this.init(); + } + if (!this.fileReader) { + this.fileReader = new FileReader(); + } + if (this.isLoading) { + await this.isLoading; + } + const response = this.ffmpegTaskQueue.queueUpRequest( + extractVideoMetadataHelper.bind( + null, + this.ffmpeg, + this.fileReader, + file + ) + ); + try { + return await response.promise; + } catch (e) { + if (e.message === CustomError.REQUEST_CANCELLED) { + // ignore + return null; + } else { + logError(e, 'ffmpeg metadata extraction failed'); + throw e; + } + } + } } async function generateThumbnailHelper( @@ -101,4 +135,44 @@ async function generateThumbnailHelper( } } +async function extractVideoMetadataHelper( + ffmpeg: FFmpeg, + reader: FileReader, + file: File +) { + try { + const inputFileName = `${Date.now().toString()}-${file.name}`; + const outFileName = `${Date.now().toString()}-metadata.txt`; + ffmpeg.FS( + 'writeFile', + inputFileName, + await getUint8ArrayView(reader, file) + ); + let metadata = null; + + // https://stackoverflow.com/questions/9464617/retrieving-and-saving-media-metadata-using-ffmpeg + // -c [short for codex] copy[(stream_specifier)[ffmpeg.org/ffmpeg.html#Stream-specifiers]] => copies all the stream without re-encoding + // -map_metadata [http://ffmpeg.org/ffmpeg.html#Advanced-options search for map_metadata] => copies all stream metadata to the out + // -f ffmetadata [https://ffmpeg.org/ffmpeg-formats.html#Metadata-1] => dump metadata from media files into a simple UTF-8-encoded INI-like text file + await ffmpeg.run( + '-i', + inputFileName, + '-c', + 'copy', + '-map_metadata', + '0', + '-f', + 'ffmetadata', + outFileName + ); + metadata = ffmpeg.FS('readFile', outFileName); + ffmpeg.FS('unlink', outFileName); + ffmpeg.FS('unlink', inputFileName); + return parseFFmpegExtractedMetadata(metadata); + } catch (e) { + logError(e, 'ffmpeg metadata extraction failed'); + throw e; + } +} + export default new FFmpegService(); diff --git a/src/services/updateCreationTimeWithExif.ts b/src/services/updateCreationTimeWithExif.ts index e34f889c0..e20bfd16c 100644 --- a/src/services/updateCreationTimeWithExif.ts +++ b/src/services/updateCreationTimeWithExif.ts @@ -10,9 +10,10 @@ import downloadManager from './downloadManager'; import { updatePublicMagicMetadata } from './fileService'; import { EnteFile } from 'types/file'; -import { getRawExif, getUNIXTime } from './upload/exifService'; +import { getRawExif } from './upload/exifService'; import { getFileType } from './upload/readFileService'; import { FILE_TYPE } from 'constants/file'; +import { getUnixTimeInMicroSeconds } from 'utils/time'; export async function updateCreationTimeWithExif( filesToBeUpdated: EnteFile[], @@ -33,7 +34,7 @@ export async function updateCreationTimeWithExif( } let correctCreationTime: number; if (fixOption === FIX_OPTIONS.CUSTOM_TIME) { - correctCreationTime = getUNIXTime(customTime); + correctCreationTime = getUnixTimeInMicroSeconds(customTime); } else { const fileURL = await downloadManager.getFile(file); const fileObject = await getFileFromURL(fileURL); @@ -41,11 +42,13 @@ export async function updateCreationTimeWithExif( const fileTypeInfo = await getFileType(reader, fileObject); const exifData = await getRawExif(fileObject, fileTypeInfo); if (fixOption === FIX_OPTIONS.DATE_TIME_ORIGINAL) { - correctCreationTime = getUNIXTime( + correctCreationTime = getUnixTimeInMicroSeconds( exifData?.DateTimeOriginal ); } else { - correctCreationTime = getUNIXTime(exifData?.CreateDate); + correctCreationTime = getUnixTimeInMicroSeconds( + exifData?.CreateDate + ); } } if ( diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts index 685994235..ee81da515 100644 --- a/src/services/upload/exifService.ts +++ b/src/services/upload/exifService.ts @@ -1,9 +1,11 @@ -import { NULL_LOCATION } from 'constants/upload'; +import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload'; import { Location } from 'types/upload'; import exifr from 'exifr'; import piexif from 'piexifjs'; import { FileTypeInfo } from 'types/upload'; import { logError } from 'utils/sentry'; +import { ParsedExtractedMetadata } from 'types/upload'; +import { getUnixTimeInMicroSeconds } from 'utils/time'; const EXIF_TAGS_NEEDED = [ 'DateTimeOriginal', @@ -23,37 +25,29 @@ interface Exif { GPSLatitudeRef?: number; GPSLongitudeRef?: number; } -interface ParsedEXIFData { - location: Location; - creationTime: number; -} export async function getExifData( receivedFile: File, fileTypeInfo: FileTypeInfo -): Promise { - const nullExifData: ParsedEXIFData = { - location: NULL_LOCATION, - creationTime: null, - }; +): Promise { + let parsedEXIFData = NULL_EXTRACTED_METADATA; try { const exifData = await getRawExif(receivedFile, fileTypeInfo); if (!exifData) { - return nullExifData; + return parsedEXIFData; } - const parsedEXIFData = { + parsedEXIFData = { location: getEXIFLocation(exifData), - creationTime: getUNIXTime( + creationTime: getUnixTimeInMicroSeconds( exifData.DateTimeOriginal ?? exifData.CreateDate ?? exifData.ModifyDate ), }; - return parsedEXIFData; } catch (e) { logError(e, 'getExifData failed'); - return nullExifData; } + return parsedEXIFData; } export async function updateFileCreationDateInEXIF( @@ -131,22 +125,6 @@ export async function getRawExif( return exifData; } -export function getUNIXTime(dateTime: Date) { - try { - if (!dateTime) { - return null; - } - const unixTime = dateTime.getTime() * 1000; - if (unixTime <= 0) { - return null; - } else { - return unixTime; - } - } catch (e) { - logError(e, 'getUNIXTime failed', { dateTime }); - } -} - function getEXIFLocation(exifData): Location { if (!exifData.latitude || !exifData.longitude) { return NULL_LOCATION; diff --git a/src/services/upload/metadataService.ts b/src/services/upload/metadataService.ts index 89564aee5..51985e460 100644 --- a/src/services/upload/metadataService.ts +++ b/src/services/upload/metadataService.ts @@ -6,9 +6,11 @@ import { ParsedMetadataJSON, Location, FileTypeInfo, + ParsedExtractedMetadata, } from 'types/upload'; -import { NULL_LOCATION } from 'constants/upload'; +import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload'; import { splitFilenameAndExtension } from 'utils/file'; +import { getVideoMetadata } from './videoMetadataService'; interface ParsedMetadataJSONWithTitle { title: string; @@ -25,23 +27,25 @@ export async function extractMetadata( receivedFile: File, fileTypeInfo: FileTypeInfo ) { - let exifData = null; + let extractedMetadata: ParsedExtractedMetadata = NULL_EXTRACTED_METADATA; if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) { - exifData = await getExifData(receivedFile, fileTypeInfo); + extractedMetadata = await getExifData(receivedFile, fileTypeInfo); + } else if (fileTypeInfo.fileType === FILE_TYPE.VIDEO) { + extractedMetadata = await getVideoMetadata(receivedFile); } - const extractedMetadata: Metadata = { + const metadata: Metadata = { title: `${splitFilenameAndExtension(receivedFile.name)[0]}.${ fileTypeInfo.exactType }`, creationTime: - exifData?.creationTime ?? receivedFile.lastModified * 1000, + extractedMetadata.creationTime ?? receivedFile.lastModified * 1000, modificationTime: receivedFile.lastModified * 1000, - latitude: exifData?.location?.latitude, - longitude: exifData?.location?.longitude, + latitude: extractedMetadata.location.latitude, + longitude: extractedMetadata.location.longitude, fileType: fileTypeInfo.fileType, }; - return extractedMetadata; + return metadata; } export const getMetadataJSONMapKey = ( diff --git a/src/services/upload/videoMetadataService.ts b/src/services/upload/videoMetadataService.ts new file mode 100644 index 000000000..0a13a5dc9 --- /dev/null +++ b/src/services/upload/videoMetadataService.ts @@ -0,0 +1,76 @@ +import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload'; +import ffmpegService from 'services/ffmpegService'; +import { getUnixTimeInMicroSeconds } from 'utils/time'; +import { ParsedExtractedMetadata } from 'types/upload'; +import { logError } from 'utils/sentry'; + +enum VideoMetadata { + CREATION_TIME = 'creation_time', + APPLE_CONTENT_IDENTIFIER = 'com.apple.quicktime.content.identifier', + APPLE_LIVE_PHOTO_IDENTIFIER = 'com.apple.quicktime.live-photo.auto', + APPLE_CREATION_DATE = 'com.apple.quicktime.creationdate', + APPLE_LOCATION_ISO = 'com.apple.quicktime.location.ISO6709', + LOCATION = 'location', +} + +export async function getVideoMetadata(file: File) { + let videoMetadata = NULL_EXTRACTED_METADATA; + try { + videoMetadata = await ffmpegService.extractMetadata(file); + } catch (e) { + logError(e, 'failed to get video metadata'); + } + + return videoMetadata; +} + +export function parseFFmpegExtractedMetadata(encodedMetadata: Uint8Array) { + const metadataString = new TextDecoder().decode(encodedMetadata); + const metadataPropertyArray = metadataString.split('\n'); + const metadataKeyValueArray = metadataPropertyArray.map((property) => + property.split('=') + ); + const validKeyValuePairs = metadataKeyValueArray.filter( + (keyValueArray) => keyValueArray.length === 2 + ) as Array<[string, string]>; + + const metadataMap = Object.fromEntries(validKeyValuePairs); + + const location = parseAppleISOLocation( + metadataMap[VideoMetadata.APPLE_LOCATION_ISO] ?? + metadataMap[VideoMetadata.LOCATION] + ); + + const creationTime = parseCreationTime( + metadataMap[VideoMetadata.APPLE_CREATION_DATE] ?? + metadataMap[VideoMetadata.CREATION_TIME] + ); + const parsedMetadata: ParsedExtractedMetadata = { + creationTime, + location: { + latitude: location.latitude, + longitude: location.longitude, + }, + }; + return parsedMetadata; +} + +function parseAppleISOLocation(isoLocation: string) { + let location = NULL_LOCATION; + if (isoLocation) { + const [latitude, longitude] = isoLocation + .match(/(\+|-)\d+\.*\d+/g) + .map((x) => parseFloat(x)); + + location = { latitude, longitude }; + } + return location; +} + +function parseCreationTime(creationTime: string) { + let dateTime = null; + if (creationTime) { + dateTime = getUnixTimeInMicroSeconds(new Date(creationTime)); + } + return dateTime; +} diff --git a/src/types/upload/index.ts b/src/types/upload/index.ts index a49facddf..35a4692a6 100644 --- a/src/types/upload/index.ts +++ b/src/types/upload/index.ts @@ -130,3 +130,8 @@ export interface UploadFile extends BackupedFile { encryptedKey: string; keyDecryptionNonce: string; } + +export interface ParsedExtractedMetadata { + location: Location; + creationTime: number; +} diff --git a/src/utils/time/index.ts b/src/utils/time/index.ts new file mode 100644 index 000000000..65e15fbce --- /dev/null +++ b/src/utils/time/index.ts @@ -0,0 +1,11 @@ +export function getUnixTimeInMicroSeconds(dateTime: Date) { + if (!dateTime || isNaN(dateTime.getTime())) { + return null; + } + const unixTime = dateTime.getTime() * 1000; + if (unixTime <= 0) { + return null; + } else { + return unixTime; + } +}