From 1af2dc24ca22401de677e7c6d9f7e9069f989712 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Thu, 24 Feb 2022 10:09:53 +0530 Subject: [PATCH 01/11] add metadata extraction logic --- src/services/ffmpegService.ts | 81 +++++++++++++++++++++++++- src/services/upload/metadataService.ts | 69 ++++++++++++++++++++-- 2 files changed, 142 insertions(+), 8 deletions(-) diff --git a/src/services/ffmpegService.ts b/src/services/ffmpegService.ts index 3914cfea2..72c835b9a 100644 --- a/src/services/ffmpegService.ts +++ b/src/services/ffmpegService.ts @@ -2,6 +2,10 @@ import { createFFmpeg, FFmpeg } from '@ffmpeg/ffmpeg'; import { CustomError } from 'utils/error'; import { logError } from 'utils/sentry'; import QueueProcessor from './queueProcessor'; +import { + ParsedVideoMetadata, + parseFFmpegExtractedMetadata, +} from './upload/metadataService'; import { getUint8ArrayView } from './upload/readFileService'; class FFmpegService { @@ -9,7 +13,7 @@ class FFmpegService { private isLoading = null; private fileReader: FileReader = null; - private generateThumbnailProcessor = new QueueProcessor(1); + private ffmpegTaskQueue = new QueueProcessor(1); async init() { try { this.ffmpeg = createFFmpeg({ @@ -26,7 +30,7 @@ class FFmpegService { } } - async generateThumbnail(file: File) { + async generateThumbnail(file: File): Promise { if (!this.ffmpeg) { await this.init(); } @@ -36,7 +40,7 @@ class FFmpegService { if (this.isLoading) { await this.isLoading; } - const response = this.generateThumbnailProcessor.queueUpRequest( + const response = this.ffmpegTaskQueue.queueUpRequest( generateThumbnailHelper.bind( null, this.ffmpeg, @@ -56,6 +60,37 @@ class FFmpegService { } } } + + async extractMetadata(file: File): Promise { + if (!this.ffmpeg) { + await this.init(); + } + if (!this.fileReader) { + this.fileReader = new FileReader(); + } + if (this.isLoading) { + await this.isLoading; + } + const response = this.ffmpegTaskQueue.queueUpRequest( + extractVideoMetadataHelper.bind( + null, + this.ffmpeg, + this.fileReader, + file + ) + ); + try { + return await response.promise; + } catch (e) { + if (e.message === CustomError.REQUEST_CANCELLED) { + // ignore + return null; + } else { + logError(e, 'ffmpeg metadata extraction failed'); + throw e; + } + } + } } async function generateThumbnailHelper( @@ -101,4 +136,44 @@ async function generateThumbnailHelper( } } +async function extractVideoMetadataHelper( + ffmpeg: FFmpeg, + reader: FileReader, + file: File +) { + try { + const inputFileName = `${Date.now().toString()}-${file.name}`; + const outFileName = `${Date.now().toString()}-metadata.txt`; + ffmpeg.FS( + 'writeFile', + inputFileName, + await getUint8ArrayView(reader, file) + ); + let metadata = null; + + await ffmpeg.run( + '-i', + inputFileName, + '-c', + 'copy', + '-map_metadata', + '0', + '-map_metadata:s:v', + '0:s:v', + '-map_metadata:s:a', + '0:s:a', + '-f', + 'ffmetadata', + outFileName + ); + metadata = ffmpeg.FS('readFile', outFileName); + ffmpeg.FS('unlink', outFileName); + ffmpeg.FS('unlink', inputFileName); + return parseFFmpegExtractedMetadata(metadata); + } catch (e) { + logError(e, 'ffmpeg metadata extraction failed'); + throw e; + } +} + export default new FFmpegService(); diff --git a/src/services/upload/metadataService.ts b/src/services/upload/metadataService.ts index 89564aee5..db4b4469c 100644 --- a/src/services/upload/metadataService.ts +++ b/src/services/upload/metadataService.ts @@ -1,6 +1,6 @@ import { FILE_TYPE } from 'constants/file'; import { logError } from 'utils/sentry'; -import { getExifData } from './exifService'; +import { getExifData, ParsedEXIFData } from './exifService'; import { Metadata, ParsedMetadataJSON, @@ -9,6 +9,15 @@ import { } from 'types/upload'; import { NULL_LOCATION } from 'constants/upload'; import { splitFilenameAndExtension } from 'utils/file'; +import ffmpegService from 'services/ffmpegService'; + +enum VideoMetadata { + CREATION_TIME = 'creation_time', + APPLE_CONTENT_IDENTIFIER = 'com.apple.quicktime.content.identifier', + APPLE_LIVE_PHOTO_IDENTIFIER = 'com.apple.quicktime.live-photo.auto', + APPLE_CREATION_DATE = 'com.apple.quicktime.creationdate', + APPLE_LOCATION_ISO = 'com.apple.quicktime.location.ISO6709', +} interface ParsedMetadataJSONWithTitle { title: string; @@ -21,13 +30,21 @@ const NULL_PARSED_METADATA_JSON: ParsedMetadataJSON = { ...NULL_LOCATION, }; +export interface ParsedVideoMetadata { + location: Location; + creationTime: number; +} + export async function extractMetadata( receivedFile: File, fileTypeInfo: FileTypeInfo ) { - let exifData = null; + let exifData: ParsedEXIFData = null; + let videoMetadata: ParsedVideoMetadata = null; if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) { exifData = await getExifData(receivedFile, fileTypeInfo); + } else if (fileTypeInfo.fileType === FILE_TYPE.VIDEO) { + videoMetadata = await ffmpegService.extractMetadata(receivedFile); } const extractedMetadata: Metadata = { @@ -35,10 +52,14 @@ export async function extractMetadata( fileTypeInfo.exactType }`, creationTime: - exifData?.creationTime ?? receivedFile.lastModified * 1000, + exifData?.creationTime ?? + videoMetadata.creationTime ?? + receivedFile.lastModified * 1000, modificationTime: receivedFile.lastModified * 1000, - latitude: exifData?.location?.latitude, - longitude: exifData?.location?.longitude, + latitude: + exifData?.location?.latitude ?? videoMetadata.location?.latitude, + longitude: + exifData?.location?.longitude ?? videoMetadata.location?.longitude, fileType: fileTypeInfo.fileType, }; return extractedMetadata; @@ -119,3 +140,41 @@ export async function parseMetadataJSON( // ignore } } + +export function parseFFmpegExtractedMetadata(metadata: Uint8Array) { + const metadataString = new TextDecoder().decode(metadata); + const metadataPropertyArray = metadataString.split('\n'); + const metadataKeyValueArray = metadataPropertyArray.map((property) => + property.split('=') + ); + const validKeyValuePairs = metadataKeyValueArray.filter( + (keyValueArray) => keyValueArray.length === 2 + ) as Array<[string, string]>; + + const metadataMap = new Map(validKeyValuePairs); + + const location = parseAppleISOLocation( + metadata[VideoMetadata.APPLE_LOCATION_ISO] + ); + + const parsedMetadata: ParsedVideoMetadata = { + creationTime: + metadataMap[VideoMetadata.APPLE_CREATION_DATE] || + metadataMap[VideoMetadata.CREATION_TIME], + location: { + latitude: location.latitude, + longitude: location.longitude, + }, + }; + return parsedMetadata; +} + +function parseAppleISOLocation(isoLocation: string) { + if (isoLocation) { + const [latitude, longitude, altitude] = isoLocation + .match(/(\+|-)\d+\.*\d+/g) + .map((x) => parseFloat(x)); + + return { latitude, longitude, altitude }; + } +} From 540620373b3fde6f79a1283b0ce6da0a602e76eb Mon Sep 17 00:00:00 2001 From: Abhinav Date: Thu, 24 Feb 2022 17:10:41 +0530 Subject: [PATCH 02/11] fix minor bugs --- src/services/upload/exifService.ts | 2 +- src/services/upload/metadataService.ts | 38 +++++++++++++++++++------- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts index 685994235..81dee1aec 100644 --- a/src/services/upload/exifService.ts +++ b/src/services/upload/exifService.ts @@ -23,7 +23,7 @@ interface Exif { GPSLatitudeRef?: number; GPSLongitudeRef?: number; } -interface ParsedEXIFData { +export interface ParsedEXIFData { location: Location; creationTime: number; } diff --git a/src/services/upload/metadataService.ts b/src/services/upload/metadataService.ts index db4b4469c..06ec15e92 100644 --- a/src/services/upload/metadataService.ts +++ b/src/services/upload/metadataService.ts @@ -1,6 +1,6 @@ import { FILE_TYPE } from 'constants/file'; import { logError } from 'utils/sentry'; -import { getExifData, ParsedEXIFData } from './exifService'; +import { getExifData, getUNIXTime, ParsedEXIFData } from './exifService'; import { Metadata, ParsedMetadataJSON, @@ -141,8 +141,8 @@ export async function parseMetadataJSON( } } -export function parseFFmpegExtractedMetadata(metadata: Uint8Array) { - const metadataString = new TextDecoder().decode(metadata); +export function parseFFmpegExtractedMetadata(encodedMetadata: Uint8Array) { + const metadataString = new TextDecoder().decode(encodedMetadata); const metadataPropertyArray = metadataString.split('\n'); const metadataKeyValueArray = metadataPropertyArray.map((property) => property.split('=') @@ -151,16 +151,18 @@ export function parseFFmpegExtractedMetadata(metadata: Uint8Array) { (keyValueArray) => keyValueArray.length === 2 ) as Array<[string, string]>; - const metadataMap = new Map(validKeyValuePairs); + const metadataMap = Object.fromEntries(validKeyValuePairs); const location = parseAppleISOLocation( - metadata[VideoMetadata.APPLE_LOCATION_ISO] + metadataMap[VideoMetadata.APPLE_LOCATION_ISO] ); + const creationTime = parseCreationTime( + metadataMap[VideoMetadata.APPLE_CREATION_DATE] ?? + metadataMap[VideoMetadata.CREATION_TIME] + ); const parsedMetadata: ParsedVideoMetadata = { - creationTime: - metadataMap[VideoMetadata.APPLE_CREATION_DATE] || - metadataMap[VideoMetadata.CREATION_TIME], + creationTime, location: { latitude: location.latitude, longitude: location.longitude, @@ -170,11 +172,27 @@ export function parseFFmpegExtractedMetadata(metadata: Uint8Array) { } function parseAppleISOLocation(isoLocation: string) { + let location = NULL_LOCATION; if (isoLocation) { - const [latitude, longitude, altitude] = isoLocation + const [latitude, longitude] = isoLocation .match(/(\+|-)\d+\.*\d+/g) .map((x) => parseFloat(x)); - return { latitude, longitude, altitude }; + location = { latitude, longitude }; } + return location; +} + +function parseCreationTime(creationTime: string) { + let dateTime = null; + if (creationTime) { + dateTime = new Date(creationTime); + if (isNaN(dateTime.getTime())) { + dateTime = null; + } + } + if (dateTime) { + dateTime = getUNIXTime(dateTime); + } + return dateTime; } From 5887aec373202e83ce9685abccab80ef70ab0163 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 28 Feb 2022 13:05:14 +0530 Subject: [PATCH 03/11] read all available stream metadata dont need to specifically read video and audio streams --- src/services/ffmpegService.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/services/ffmpegService.ts b/src/services/ffmpegService.ts index 72c835b9a..2a200f4e3 100644 --- a/src/services/ffmpegService.ts +++ b/src/services/ffmpegService.ts @@ -158,10 +158,6 @@ async function extractVideoMetadataHelper( 'copy', '-map_metadata', '0', - '-map_metadata:s:v', - '0:s:v', - '-map_metadata:s:a', - '0:s:a', '-f', 'ffmetadata', outFileName @@ -177,3 +173,12 @@ async function extractVideoMetadataHelper( } export default new FFmpegService(); + +/* + +ffmpeg -i largeVideo.mp4 -c copy -map_metadata 0 -map_metadata:s:v 0:s:v -map_metadata:s:a 0:s:a -f ffmetadata out.txt + +explanation +"-c copy" => will copy all the stream without re-encoding +-map_metadata[:metadata_spec_out] infile[:metadata_spec_in] (output,per-metadata) => +*/ From c488d76609e7ea59805dc10e8a0f0869b87dd53f Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 28 Feb 2022 13:23:34 +0530 Subject: [PATCH 04/11] update comment for extractVideoMetadata command --- src/services/ffmpegService.ts | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/services/ffmpegService.ts b/src/services/ffmpegService.ts index 2a200f4e3..3428c3297 100644 --- a/src/services/ffmpegService.ts +++ b/src/services/ffmpegService.ts @@ -151,6 +151,10 @@ async function extractVideoMetadataHelper( ); let metadata = null; + // https://stackoverflow.com/questions/9464617/retrieving-and-saving-media-metadata-using-ffmpeg + // -c [short for codex] copy[(stream_specifier)[ffmpeg.org/ffmpeg.html#Stream-specifiers]] => copies all the stream without re-encoding + // -map_metadata [http://ffmpeg.org/ffmpeg.html#Advanced-options search for map_metadata] => copies all stream metadata to the out + // -f ffmetadata [https://ffmpeg.org/ffmpeg-formats.html#Metadata-1] => dump metadata from media files into a simple UTF-8-encoded INI-like text file await ffmpeg.run( '-i', inputFileName, @@ -173,12 +177,3 @@ async function extractVideoMetadataHelper( } export default new FFmpegService(); - -/* - -ffmpeg -i largeVideo.mp4 -c copy -map_metadata 0 -map_metadata:s:v 0:s:v -map_metadata:s:a 0:s:a -f ffmetadata out.txt - -explanation -"-c copy" => will copy all the stream without re-encoding --map_metadata[:metadata_spec_out] infile[:metadata_spec_in] (output,per-metadata) => -*/ From e49a7f9ec9a1aa40f27c5dfca09520ac54ae8164 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 28 Feb 2022 13:29:30 +0530 Subject: [PATCH 05/11] better types --- src/services/ffmpegService.ts | 4 ++-- src/services/upload/exifService.ts | 9 +++------ src/services/upload/metadataService.ts | 27 +++++++++++--------------- 3 files changed, 16 insertions(+), 24 deletions(-) diff --git a/src/services/ffmpegService.ts b/src/services/ffmpegService.ts index 3428c3297..a863e887f 100644 --- a/src/services/ffmpegService.ts +++ b/src/services/ffmpegService.ts @@ -3,7 +3,7 @@ import { CustomError } from 'utils/error'; import { logError } from 'utils/sentry'; import QueueProcessor from './queueProcessor'; import { - ParsedVideoMetadata, + ParsedExtractedMetadata, parseFFmpegExtractedMetadata, } from './upload/metadataService'; import { getUint8ArrayView } from './upload/readFileService'; @@ -61,7 +61,7 @@ class FFmpegService { } } - async extractMetadata(file: File): Promise { + async extractMetadata(file: File): Promise { if (!this.ffmpeg) { await this.init(); } diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts index 81dee1aec..17e438f19 100644 --- a/src/services/upload/exifService.ts +++ b/src/services/upload/exifService.ts @@ -4,6 +4,7 @@ import exifr from 'exifr'; import piexif from 'piexifjs'; import { FileTypeInfo } from 'types/upload'; import { logError } from 'utils/sentry'; +import { ParsedExtractedMetadata } from './metadataService'; const EXIF_TAGS_NEEDED = [ 'DateTimeOriginal', @@ -23,16 +24,12 @@ interface Exif { GPSLatitudeRef?: number; GPSLongitudeRef?: number; } -export interface ParsedEXIFData { - location: Location; - creationTime: number; -} export async function getExifData( receivedFile: File, fileTypeInfo: FileTypeInfo -): Promise { - const nullExifData: ParsedEXIFData = { +): Promise { + const nullExifData: ParsedExtractedMetadata = { location: NULL_LOCATION, creationTime: null, }; diff --git a/src/services/upload/metadataService.ts b/src/services/upload/metadataService.ts index 06ec15e92..d61551fae 100644 --- a/src/services/upload/metadataService.ts +++ b/src/services/upload/metadataService.ts @@ -1,6 +1,6 @@ import { FILE_TYPE } from 'constants/file'; import { logError } from 'utils/sentry'; -import { getExifData, getUNIXTime, ParsedEXIFData } from './exifService'; +import { getExifData, getUNIXTime } from './exifService'; import { Metadata, ParsedMetadataJSON, @@ -30,7 +30,7 @@ const NULL_PARSED_METADATA_JSON: ParsedMetadataJSON = { ...NULL_LOCATION, }; -export interface ParsedVideoMetadata { +export interface ParsedExtractedMetadata { location: Location; creationTime: number; } @@ -39,30 +39,25 @@ export async function extractMetadata( receivedFile: File, fileTypeInfo: FileTypeInfo ) { - let exifData: ParsedEXIFData = null; - let videoMetadata: ParsedVideoMetadata = null; + let extractedMetadata: ParsedExtractedMetadata = null; if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) { - exifData = await getExifData(receivedFile, fileTypeInfo); + extractedMetadata = await getExifData(receivedFile, fileTypeInfo); } else if (fileTypeInfo.fileType === FILE_TYPE.VIDEO) { - videoMetadata = await ffmpegService.extractMetadata(receivedFile); + extractedMetadata = await ffmpegService.extractMetadata(receivedFile); } - const extractedMetadata: Metadata = { + const metadata: Metadata = { title: `${splitFilenameAndExtension(receivedFile.name)[0]}.${ fileTypeInfo.exactType }`, creationTime: - exifData?.creationTime ?? - videoMetadata.creationTime ?? - receivedFile.lastModified * 1000, + extractedMetadata?.creationTime ?? receivedFile.lastModified * 1000, modificationTime: receivedFile.lastModified * 1000, - latitude: - exifData?.location?.latitude ?? videoMetadata.location?.latitude, - longitude: - exifData?.location?.longitude ?? videoMetadata.location?.longitude, + latitude: extractedMetadata.location?.latitude, + longitude: extractedMetadata.location?.longitude, fileType: fileTypeInfo.fileType, }; - return extractedMetadata; + return metadata; } export const getMetadataJSONMapKey = ( @@ -161,7 +156,7 @@ export function parseFFmpegExtractedMetadata(encodedMetadata: Uint8Array) { metadataMap[VideoMetadata.APPLE_CREATION_DATE] ?? metadataMap[VideoMetadata.CREATION_TIME] ); - const parsedMetadata: ParsedVideoMetadata = { + const parsedMetadata: ParsedExtractedMetadata = { creationTime, location: { latitude: location.latitude, From a98d35ad66a976bc8bdc381ee8130b7da0fecf3b Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 28 Feb 2022 13:42:21 +0530 Subject: [PATCH 06/11] refactoring moved videoMetadata extraction to seperate service --- src/services/upload/exifService.ts | 17 +---- src/services/upload/metadataService.ts | 70 +-------------------- src/services/upload/videoMetadataService.ts | 66 +++++++++++++++++++ src/utils/upload/index.ts | 12 ++++ 4 files changed, 82 insertions(+), 83 deletions(-) create mode 100644 src/services/upload/videoMetadataService.ts diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts index 17e438f19..343736854 100644 --- a/src/services/upload/exifService.ts +++ b/src/services/upload/exifService.ts @@ -5,6 +5,7 @@ import piexif from 'piexifjs'; import { FileTypeInfo } from 'types/upload'; import { logError } from 'utils/sentry'; import { ParsedExtractedMetadata } from './metadataService'; +import { getUNIXTime } from 'utils/upload'; const EXIF_TAGS_NEEDED = [ 'DateTimeOriginal', @@ -128,22 +129,6 @@ export async function getRawExif( return exifData; } -export function getUNIXTime(dateTime: Date) { - try { - if (!dateTime) { - return null; - } - const unixTime = dateTime.getTime() * 1000; - if (unixTime <= 0) { - return null; - } else { - return unixTime; - } - } catch (e) { - logError(e, 'getUNIXTime failed', { dateTime }); - } -} - function getEXIFLocation(exifData): Location { if (!exifData.latitude || !exifData.longitude) { return NULL_LOCATION; diff --git a/src/services/upload/metadataService.ts b/src/services/upload/metadataService.ts index d61551fae..1777e8f6f 100644 --- a/src/services/upload/metadataService.ts +++ b/src/services/upload/metadataService.ts @@ -1,6 +1,6 @@ import { FILE_TYPE } from 'constants/file'; import { logError } from 'utils/sentry'; -import { getExifData, getUNIXTime } from './exifService'; +import { getExifData } from './exifService'; import { Metadata, ParsedMetadataJSON, @@ -9,15 +9,7 @@ import { } from 'types/upload'; import { NULL_LOCATION } from 'constants/upload'; import { splitFilenameAndExtension } from 'utils/file'; -import ffmpegService from 'services/ffmpegService'; - -enum VideoMetadata { - CREATION_TIME = 'creation_time', - APPLE_CONTENT_IDENTIFIER = 'com.apple.quicktime.content.identifier', - APPLE_LIVE_PHOTO_IDENTIFIER = 'com.apple.quicktime.live-photo.auto', - APPLE_CREATION_DATE = 'com.apple.quicktime.creationdate', - APPLE_LOCATION_ISO = 'com.apple.quicktime.location.ISO6709', -} +import { getVideoMetadata } from './videoMetadataService'; interface ParsedMetadataJSONWithTitle { title: string; @@ -43,7 +35,7 @@ export async function extractMetadata( if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) { extractedMetadata = await getExifData(receivedFile, fileTypeInfo); } else if (fileTypeInfo.fileType === FILE_TYPE.VIDEO) { - extractedMetadata = await ffmpegService.extractMetadata(receivedFile); + extractedMetadata = await getVideoMetadata(receivedFile); } const metadata: Metadata = { @@ -135,59 +127,3 @@ export async function parseMetadataJSON( // ignore } } - -export function parseFFmpegExtractedMetadata(encodedMetadata: Uint8Array) { - const metadataString = new TextDecoder().decode(encodedMetadata); - const metadataPropertyArray = metadataString.split('\n'); - const metadataKeyValueArray = metadataPropertyArray.map((property) => - property.split('=') - ); - const validKeyValuePairs = metadataKeyValueArray.filter( - (keyValueArray) => keyValueArray.length === 2 - ) as Array<[string, string]>; - - const metadataMap = Object.fromEntries(validKeyValuePairs); - - const location = parseAppleISOLocation( - metadataMap[VideoMetadata.APPLE_LOCATION_ISO] - ); - - const creationTime = parseCreationTime( - metadataMap[VideoMetadata.APPLE_CREATION_DATE] ?? - metadataMap[VideoMetadata.CREATION_TIME] - ); - const parsedMetadata: ParsedExtractedMetadata = { - creationTime, - location: { - latitude: location.latitude, - longitude: location.longitude, - }, - }; - return parsedMetadata; -} - -function parseAppleISOLocation(isoLocation: string) { - let location = NULL_LOCATION; - if (isoLocation) { - const [latitude, longitude] = isoLocation - .match(/(\+|-)\d+\.*\d+/g) - .map((x) => parseFloat(x)); - - location = { latitude, longitude }; - } - return location; -} - -function parseCreationTime(creationTime: string) { - let dateTime = null; - if (creationTime) { - dateTime = new Date(creationTime); - if (isNaN(dateTime.getTime())) { - dateTime = null; - } - } - if (dateTime) { - dateTime = getUNIXTime(dateTime); - } - return dateTime; -} diff --git a/src/services/upload/videoMetadataService.ts b/src/services/upload/videoMetadataService.ts new file mode 100644 index 000000000..617819765 --- /dev/null +++ b/src/services/upload/videoMetadataService.ts @@ -0,0 +1,66 @@ +import { NULL_LOCATION } from 'constants/upload'; +import ffmpegService from 'services/ffmpegService'; +import { getUNIXTime } from 'utils/upload'; +import { ParsedExtractedMetadata } from './metadataService'; + +enum VideoMetadata { + CREATION_TIME = 'creation_time', + APPLE_CONTENT_IDENTIFIER = 'com.apple.quicktime.content.identifier', + APPLE_LIVE_PHOTO_IDENTIFIER = 'com.apple.quicktime.live-photo.auto', + APPLE_CREATION_DATE = 'com.apple.quicktime.creationdate', + APPLE_LOCATION_ISO = 'com.apple.quicktime.location.ISO6709', +} + +export function getVideoMetadata(file: File) { + return ffmpegService.extractMetadata(file); +} + +export function parseFFmpegExtractedMetadata(encodedMetadata: Uint8Array) { + const metadataString = new TextDecoder().decode(encodedMetadata); + const metadataPropertyArray = metadataString.split('\n'); + const metadataKeyValueArray = metadataPropertyArray.map((property) => + property.split('=') + ); + const validKeyValuePairs = metadataKeyValueArray.filter( + (keyValueArray) => keyValueArray.length === 2 + ) as Array<[string, string]>; + + const metadataMap = Object.fromEntries(validKeyValuePairs); + + const location = parseAppleISOLocation( + metadataMap[VideoMetadata.APPLE_LOCATION_ISO] + ); + + const creationTime = parseCreationTime( + metadataMap[VideoMetadata.APPLE_CREATION_DATE] ?? + metadataMap[VideoMetadata.CREATION_TIME] + ); + const parsedMetadata: ParsedExtractedMetadata = { + creationTime, + location: { + latitude: location.latitude, + longitude: location.longitude, + }, + }; + return parsedMetadata; +} + +function parseAppleISOLocation(isoLocation: string) { + let location = NULL_LOCATION; + if (isoLocation) { + const [latitude, longitude] = isoLocation + .match(/(\+|-)\d+\.*\d+/g) + .map((x) => parseFloat(x)); + + location = { latitude, longitude }; + } + return location; +} + +function parseCreationTime(creationTime: string) { + let dateTime = null; + if (creationTime) { + dateTime = getUNIXTime(new Date(creationTime)); + } + return dateTime; +} diff --git a/src/utils/upload/index.ts b/src/utils/upload/index.ts index 29ed9ecf2..f0ea3474b 100644 --- a/src/utils/upload/index.ts +++ b/src/utils/upload/index.ts @@ -49,3 +49,15 @@ export function segregateMetadataAndMediaFiles( }); return { mediaFiles, metadataJSONFiles }; } + +export function getUNIXTime(dateTime: Date) { + if (!dateTime || isNaN(dateTime.getTime())) { + return null; + } + const unixTime = dateTime.getTime() * 1000; + if (unixTime <= 0) { + return null; + } else { + return unixTime; + } +} From cf1f55e68af4457e80d43a4555e8455ccc0a4b9c Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 28 Feb 2022 13:48:52 +0530 Subject: [PATCH 07/11] moved types and getUnixTime to time util --- src/services/ffmpegService.ts | 7 +++---- src/services/updateCreationTimeWithExif.ts | 3 ++- src/services/upload/exifService.ts | 4 ++-- src/services/upload/metadataService.ts | 6 +----- src/services/upload/videoMetadataService.ts | 4 ++-- src/types/upload/index.ts | 5 +++++ src/utils/time/index.ts | 11 +++++++++++ src/utils/upload/index.ts | 12 ------------ 8 files changed, 26 insertions(+), 26 deletions(-) create mode 100644 src/utils/time/index.ts diff --git a/src/services/ffmpegService.ts b/src/services/ffmpegService.ts index a863e887f..30e1e5fcd 100644 --- a/src/services/ffmpegService.ts +++ b/src/services/ffmpegService.ts @@ -2,11 +2,10 @@ import { createFFmpeg, FFmpeg } from '@ffmpeg/ffmpeg'; import { CustomError } from 'utils/error'; import { logError } from 'utils/sentry'; import QueueProcessor from './queueProcessor'; -import { - ParsedExtractedMetadata, - parseFFmpegExtractedMetadata, -} from './upload/metadataService'; +import { ParsedExtractedMetadata } from 'types/upload'; + import { getUint8ArrayView } from './upload/readFileService'; +import { parseFFmpegExtractedMetadata } from './upload/videoMetadataService'; class FFmpegService { private ffmpeg: FFmpeg = null; diff --git a/src/services/updateCreationTimeWithExif.ts b/src/services/updateCreationTimeWithExif.ts index e34f889c0..4d4b6bc27 100644 --- a/src/services/updateCreationTimeWithExif.ts +++ b/src/services/updateCreationTimeWithExif.ts @@ -10,9 +10,10 @@ import downloadManager from './downloadManager'; import { updatePublicMagicMetadata } from './fileService'; import { EnteFile } from 'types/file'; -import { getRawExif, getUNIXTime } from './upload/exifService'; +import { getRawExif } from './upload/exifService'; import { getFileType } from './upload/readFileService'; import { FILE_TYPE } from 'constants/file'; +import { getUNIXTime } from 'utils/time'; export async function updateCreationTimeWithExif( filesToBeUpdated: EnteFile[], diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts index 343736854..c8ba7b8d0 100644 --- a/src/services/upload/exifService.ts +++ b/src/services/upload/exifService.ts @@ -4,8 +4,8 @@ import exifr from 'exifr'; import piexif from 'piexifjs'; import { FileTypeInfo } from 'types/upload'; import { logError } from 'utils/sentry'; -import { ParsedExtractedMetadata } from './metadataService'; -import { getUNIXTime } from 'utils/upload'; +import { ParsedExtractedMetadata } from 'types/upload'; +import { getUNIXTime } from 'utils/time'; const EXIF_TAGS_NEEDED = [ 'DateTimeOriginal', diff --git a/src/services/upload/metadataService.ts b/src/services/upload/metadataService.ts index 1777e8f6f..2b55a441a 100644 --- a/src/services/upload/metadataService.ts +++ b/src/services/upload/metadataService.ts @@ -6,6 +6,7 @@ import { ParsedMetadataJSON, Location, FileTypeInfo, + ParsedExtractedMetadata, } from 'types/upload'; import { NULL_LOCATION } from 'constants/upload'; import { splitFilenameAndExtension } from 'utils/file'; @@ -22,11 +23,6 @@ const NULL_PARSED_METADATA_JSON: ParsedMetadataJSON = { ...NULL_LOCATION, }; -export interface ParsedExtractedMetadata { - location: Location; - creationTime: number; -} - export async function extractMetadata( receivedFile: File, fileTypeInfo: FileTypeInfo diff --git a/src/services/upload/videoMetadataService.ts b/src/services/upload/videoMetadataService.ts index 617819765..b7fa0d85e 100644 --- a/src/services/upload/videoMetadataService.ts +++ b/src/services/upload/videoMetadataService.ts @@ -1,7 +1,7 @@ import { NULL_LOCATION } from 'constants/upload'; import ffmpegService from 'services/ffmpegService'; -import { getUNIXTime } from 'utils/upload'; -import { ParsedExtractedMetadata } from './metadataService'; +import { getUNIXTime } from 'utils/time'; +import { ParsedExtractedMetadata } from 'types/upload'; enum VideoMetadata { CREATION_TIME = 'creation_time', diff --git a/src/types/upload/index.ts b/src/types/upload/index.ts index a49facddf..35a4692a6 100644 --- a/src/types/upload/index.ts +++ b/src/types/upload/index.ts @@ -130,3 +130,8 @@ export interface UploadFile extends BackupedFile { encryptedKey: string; keyDecryptionNonce: string; } + +export interface ParsedExtractedMetadata { + location: Location; + creationTime: number; +} diff --git a/src/utils/time/index.ts b/src/utils/time/index.ts new file mode 100644 index 000000000..cb04e5b65 --- /dev/null +++ b/src/utils/time/index.ts @@ -0,0 +1,11 @@ +export function getUNIXTime(dateTime: Date) { + if (!dateTime || isNaN(dateTime.getTime())) { + return null; + } + const unixTime = dateTime.getTime() * 1000; + if (unixTime <= 0) { + return null; + } else { + return unixTime; + } +} diff --git a/src/utils/upload/index.ts b/src/utils/upload/index.ts index f0ea3474b..29ed9ecf2 100644 --- a/src/utils/upload/index.ts +++ b/src/utils/upload/index.ts @@ -49,15 +49,3 @@ export function segregateMetadataAndMediaFiles( }); return { mediaFiles, metadataJSONFiles }; } - -export function getUNIXTime(dateTime: Date) { - if (!dateTime || isNaN(dateTime.getTime())) { - return null; - } - const unixTime = dateTime.getTime() * 1000; - if (unixTime <= 0) { - return null; - } else { - return unixTime; - } -} From 222167878a82973d292035d564750e51a776ea05 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 28 Feb 2022 14:03:47 +0530 Subject: [PATCH 08/11] better handle metadata extraction failure --- src/constants/upload/index.ts | 7 ++++++- src/services/upload/exifService.ts | 14 +++++--------- src/services/upload/videoMetadataService.ts | 14 +++++++++++--- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/constants/upload/index.ts b/src/constants/upload/index.ts index 2109e566b..fb9f0a15d 100644 --- a/src/constants/upload/index.ts +++ b/src/constants/upload/index.ts @@ -1,6 +1,6 @@ import { ENCRYPTION_CHUNK_SIZE } from 'constants/crypto'; import { FILE_TYPE } from 'constants/file'; -import { Location } from 'types/upload'; +import { Location, ParsedExtractedMetadata } from 'types/upload'; // list of format that were missed by type-detection for some files. export const FORMAT_MISSED_BY_FILE_TYPE_LIB = [ @@ -43,3 +43,8 @@ export enum FileUploadResults { export const MAX_FILE_SIZE_SUPPORTED = 5 * 1024 * 1024 * 1024; // 5 GB export const LIVE_PHOTO_ASSET_SIZE_LIMIT = 20 * 1024 * 1024; // 20MB + +export const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = { + location: NULL_LOCATION, + creationTime: null, +}; diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts index c8ba7b8d0..5144c1307 100644 --- a/src/services/upload/exifService.ts +++ b/src/services/upload/exifService.ts @@ -1,4 +1,4 @@ -import { NULL_LOCATION } from 'constants/upload'; +import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload'; import { Location } from 'types/upload'; import exifr from 'exifr'; import piexif from 'piexifjs'; @@ -30,16 +30,13 @@ export async function getExifData( receivedFile: File, fileTypeInfo: FileTypeInfo ): Promise { - const nullExifData: ParsedExtractedMetadata = { - location: NULL_LOCATION, - creationTime: null, - }; + let parsedEXIFData = NULL_EXTRACTED_METADATA; try { const exifData = await getRawExif(receivedFile, fileTypeInfo); if (!exifData) { - return nullExifData; + return parsedEXIFData; } - const parsedEXIFData = { + parsedEXIFData = { location: getEXIFLocation(exifData), creationTime: getUNIXTime( exifData.DateTimeOriginal ?? @@ -47,11 +44,10 @@ export async function getExifData( exifData.ModifyDate ), }; - return parsedEXIFData; } catch (e) { logError(e, 'getExifData failed'); - return nullExifData; } + return parsedEXIFData; } export async function updateFileCreationDateInEXIF( diff --git a/src/services/upload/videoMetadataService.ts b/src/services/upload/videoMetadataService.ts index b7fa0d85e..1b5e2e791 100644 --- a/src/services/upload/videoMetadataService.ts +++ b/src/services/upload/videoMetadataService.ts @@ -1,7 +1,8 @@ -import { NULL_LOCATION } from 'constants/upload'; +import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload'; import ffmpegService from 'services/ffmpegService'; import { getUNIXTime } from 'utils/time'; import { ParsedExtractedMetadata } from 'types/upload'; +import { logError } from 'utils/sentry'; enum VideoMetadata { CREATION_TIME = 'creation_time', @@ -11,8 +12,15 @@ enum VideoMetadata { APPLE_LOCATION_ISO = 'com.apple.quicktime.location.ISO6709', } -export function getVideoMetadata(file: File) { - return ffmpegService.extractMetadata(file); +export async function getVideoMetadata(file: File) { + let videoMetadata = NULL_EXTRACTED_METADATA; + try { + videoMetadata = await ffmpegService.extractMetadata(file); + } catch (e) { + logError(e, 'failed to get video metadata'); + } + + return videoMetadata; } export function parseFFmpegExtractedMetadata(encodedMetadata: Uint8Array) { From a866d109e5bc25ce5ca160997062660ba29f5e15 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 28 Feb 2022 17:44:23 +0530 Subject: [PATCH 09/11] rename getUNIXTime to getUnixTimeInMicroSeconds --- src/services/updateCreationTimeWithExif.ts | 10 ++++++---- src/services/upload/exifService.ts | 4 ++-- src/services/upload/videoMetadataService.ts | 4 ++-- src/utils/time/index.ts | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/services/updateCreationTimeWithExif.ts b/src/services/updateCreationTimeWithExif.ts index 4d4b6bc27..e20bfd16c 100644 --- a/src/services/updateCreationTimeWithExif.ts +++ b/src/services/updateCreationTimeWithExif.ts @@ -13,7 +13,7 @@ import { EnteFile } from 'types/file'; import { getRawExif } from './upload/exifService'; import { getFileType } from './upload/readFileService'; import { FILE_TYPE } from 'constants/file'; -import { getUNIXTime } from 'utils/time'; +import { getUnixTimeInMicroSeconds } from 'utils/time'; export async function updateCreationTimeWithExif( filesToBeUpdated: EnteFile[], @@ -34,7 +34,7 @@ export async function updateCreationTimeWithExif( } let correctCreationTime: number; if (fixOption === FIX_OPTIONS.CUSTOM_TIME) { - correctCreationTime = getUNIXTime(customTime); + correctCreationTime = getUnixTimeInMicroSeconds(customTime); } else { const fileURL = await downloadManager.getFile(file); const fileObject = await getFileFromURL(fileURL); @@ -42,11 +42,13 @@ export async function updateCreationTimeWithExif( const fileTypeInfo = await getFileType(reader, fileObject); const exifData = await getRawExif(fileObject, fileTypeInfo); if (fixOption === FIX_OPTIONS.DATE_TIME_ORIGINAL) { - correctCreationTime = getUNIXTime( + correctCreationTime = getUnixTimeInMicroSeconds( exifData?.DateTimeOriginal ); } else { - correctCreationTime = getUNIXTime(exifData?.CreateDate); + correctCreationTime = getUnixTimeInMicroSeconds( + exifData?.CreateDate + ); } } if ( diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts index 5144c1307..ee81da515 100644 --- a/src/services/upload/exifService.ts +++ b/src/services/upload/exifService.ts @@ -5,7 +5,7 @@ import piexif from 'piexifjs'; import { FileTypeInfo } from 'types/upload'; import { logError } from 'utils/sentry'; import { ParsedExtractedMetadata } from 'types/upload'; -import { getUNIXTime } from 'utils/time'; +import { getUnixTimeInMicroSeconds } from 'utils/time'; const EXIF_TAGS_NEEDED = [ 'DateTimeOriginal', @@ -38,7 +38,7 @@ export async function getExifData( } parsedEXIFData = { location: getEXIFLocation(exifData), - creationTime: getUNIXTime( + creationTime: getUnixTimeInMicroSeconds( exifData.DateTimeOriginal ?? exifData.CreateDate ?? exifData.ModifyDate diff --git a/src/services/upload/videoMetadataService.ts b/src/services/upload/videoMetadataService.ts index 1b5e2e791..9e124d51d 100644 --- a/src/services/upload/videoMetadataService.ts +++ b/src/services/upload/videoMetadataService.ts @@ -1,6 +1,6 @@ import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload'; import ffmpegService from 'services/ffmpegService'; -import { getUNIXTime } from 'utils/time'; +import { getUnixTimeInMicroSeconds } from 'utils/time'; import { ParsedExtractedMetadata } from 'types/upload'; import { logError } from 'utils/sentry'; @@ -68,7 +68,7 @@ function parseAppleISOLocation(isoLocation: string) { function parseCreationTime(creationTime: string) { let dateTime = null; if (creationTime) { - dateTime = getUNIXTime(new Date(creationTime)); + dateTime = getUnixTimeInMicroSeconds(new Date(creationTime)); } return dateTime; } diff --git a/src/utils/time/index.ts b/src/utils/time/index.ts index cb04e5b65..65e15fbce 100644 --- a/src/utils/time/index.ts +++ b/src/utils/time/index.ts @@ -1,4 +1,4 @@ -export function getUNIXTime(dateTime: Date) { +export function getUnixTimeInMicroSeconds(dateTime: Date) { if (!dateTime || isNaN(dateTime.getTime())) { return null; } From ca1369168840da82d6d136c1692073cef7d95ec6 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 28 Feb 2022 18:27:44 +0530 Subject: [PATCH 10/11] extractedMetadata will allows have the properties so no need of conditional access --- src/services/upload/metadataService.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/services/upload/metadataService.ts b/src/services/upload/metadataService.ts index 2b55a441a..51985e460 100644 --- a/src/services/upload/metadataService.ts +++ b/src/services/upload/metadataService.ts @@ -8,7 +8,7 @@ import { FileTypeInfo, ParsedExtractedMetadata, } from 'types/upload'; -import { NULL_LOCATION } from 'constants/upload'; +import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload'; import { splitFilenameAndExtension } from 'utils/file'; import { getVideoMetadata } from './videoMetadataService'; @@ -27,7 +27,7 @@ export async function extractMetadata( receivedFile: File, fileTypeInfo: FileTypeInfo ) { - let extractedMetadata: ParsedExtractedMetadata = null; + let extractedMetadata: ParsedExtractedMetadata = NULL_EXTRACTED_METADATA; if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) { extractedMetadata = await getExifData(receivedFile, fileTypeInfo); } else if (fileTypeInfo.fileType === FILE_TYPE.VIDEO) { @@ -39,10 +39,10 @@ export async function extractMetadata( fileTypeInfo.exactType }`, creationTime: - extractedMetadata?.creationTime ?? receivedFile.lastModified * 1000, + extractedMetadata.creationTime ?? receivedFile.lastModified * 1000, modificationTime: receivedFile.lastModified * 1000, - latitude: extractedMetadata.location?.latitude, - longitude: extractedMetadata.location?.longitude, + latitude: extractedMetadata.location.latitude, + longitude: extractedMetadata.location.longitude, fileType: fileTypeInfo.fileType, }; return metadata; From a86b9134d2808ce112e251e32ea49c0de2b9d77d Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 1 Mar 2022 12:50:45 +0530 Subject: [PATCH 11/11] also check location tag in the metadata as fallback --- src/services/upload/videoMetadataService.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/services/upload/videoMetadataService.ts b/src/services/upload/videoMetadataService.ts index 9e124d51d..0a13a5dc9 100644 --- a/src/services/upload/videoMetadataService.ts +++ b/src/services/upload/videoMetadataService.ts @@ -10,6 +10,7 @@ enum VideoMetadata { APPLE_LIVE_PHOTO_IDENTIFIER = 'com.apple.quicktime.live-photo.auto', APPLE_CREATION_DATE = 'com.apple.quicktime.creationdate', APPLE_LOCATION_ISO = 'com.apple.quicktime.location.ISO6709', + LOCATION = 'location', } export async function getVideoMetadata(file: File) { @@ -36,7 +37,8 @@ export function parseFFmpegExtractedMetadata(encodedMetadata: Uint8Array) { const metadataMap = Object.fromEntries(validKeyValuePairs); const location = parseAppleISOLocation( - metadataMap[VideoMetadata.APPLE_LOCATION_ISO] + metadataMap[VideoMetadata.APPLE_LOCATION_ISO] ?? + metadataMap[VideoMetadata.LOCATION] ); const creationTime = parseCreationTime(