Merge pull request #404 from ente-io/master

release video metadata extraction

src/constants/upload/index.ts

@@ -1,6 +1,6 @@
 import { ENCRYPTION_CHUNK_SIZE } from 'constants/crypto';
 import { FILE_TYPE } from 'constants/file';
-import { Location } from 'types/upload';
+import { Location, ParsedExtractedMetadata } from 'types/upload';
 
 // list of format that were missed by type-detection for some files.
 export const FORMAT_MISSED_BY_FILE_TYPE_LIB = [
@@ -43,3 +43,8 @@ export enum FileUploadResults {
 export const MAX_FILE_SIZE_SUPPORTED = 5 * 1024 * 1024 * 1024; // 5 GB
 
 export const LIVE_PHOTO_ASSET_SIZE_LIMIT = 20 * 1024 * 1024; // 20MB
+
+export const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = {
+    location: NULL_LOCATION,
+    creationTime: null,
+};

src/services/ffmpegService.ts

@@ -2,14 +2,17 @@ import { createFFmpeg, FFmpeg } from '@ffmpeg/ffmpeg';
 import { CustomError } from 'utils/error';
 import { logError } from 'utils/sentry';
 import QueueProcessor from './queueProcessor';
+import { ParsedExtractedMetadata } from 'types/upload';
+
 import { getUint8ArrayView } from './upload/readFileService';
+import { parseFFmpegExtractedMetadata } from './upload/videoMetadataService';
 
 class FFmpegService {
     private ffmpeg: FFmpeg = null;
     private isLoading = null;
     private fileReader: FileReader = null;
 
-    private generateThumbnailProcessor = new QueueProcessor<Uint8Array>(1);
+    private ffmpegTaskQueue = new QueueProcessor<any>(1);
     async init() {
         try {
             this.ffmpeg = createFFmpeg({
@@ -26,7 +29,7 @@ class FFmpegService {
         }
     }
 
-    async generateThumbnail(file: File) {
+    async generateThumbnail(file: File): Promise<Uint8Array> {
         if (!this.ffmpeg) {
             await this.init();
         }
@@ -36,7 +39,7 @@ class FFmpegService {
         if (this.isLoading) {
             await this.isLoading;
         }
-        const response = this.generateThumbnailProcessor.queueUpRequest(
+        const response = this.ffmpegTaskQueue.queueUpRequest(
             generateThumbnailHelper.bind(
                 null,
                 this.ffmpeg,
@@ -56,6 +59,37 @@ class FFmpegService {
             }
         }
     }
+
+    async extractMetadata(file: File): Promise<ParsedExtractedMetadata> {
+        if (!this.ffmpeg) {
+            await this.init();
+        }
+        if (!this.fileReader) {
+            this.fileReader = new FileReader();
+        }
+        if (this.isLoading) {
+            await this.isLoading;
+        }
+        const response = this.ffmpegTaskQueue.queueUpRequest(
+            extractVideoMetadataHelper.bind(
+                null,
+                this.ffmpeg,
+                this.fileReader,
+                file
+            )
+        );
+        try {
+            return await response.promise;
+        } catch (e) {
+            if (e.message === CustomError.REQUEST_CANCELLED) {
+                // ignore
+                return null;
+            } else {
+                logError(e, 'ffmpeg metadata extraction failed');
+                throw e;
+            }
+        }
+    }
 }
 
 async function generateThumbnailHelper(
@@ -101,4 +135,44 @@ async function generateThumbnailHelper(
     }
 }
 
+async function extractVideoMetadataHelper(
+    ffmpeg: FFmpeg,
+    reader: FileReader,
+    file: File
+) {
+    try {
+        const inputFileName = `${Date.now().toString()}-${file.name}`;
+        const outFileName = `${Date.now().toString()}-metadata.txt`;
+        ffmpeg.FS(
+            'writeFile',
+            inputFileName,
+            await getUint8ArrayView(reader, file)
+        );
+        let metadata = null;
+
+        // https://stackoverflow.com/questions/9464617/retrieving-and-saving-media-metadata-using-ffmpeg
+        // -c [short for codex] copy[(stream_specifier)[ffmpeg.org/ffmpeg.html#Stream-specifiers]] => copies all the stream without re-encoding
+        // -map_metadata [http://ffmpeg.org/ffmpeg.html#Advanced-options search for map_metadata] => copies all stream metadata to the out
+        // -f ffmetadata [https://ffmpeg.org/ffmpeg-formats.html#Metadata-1] => dump metadata from media files into a simple UTF-8-encoded INI-like text file
+        await ffmpeg.run(
+            '-i',
+            inputFileName,
+            '-c',
+            'copy',
+            '-map_metadata',
+            '0',
+            '-f',
+            'ffmetadata',
+            outFileName
+        );
+        metadata = ffmpeg.FS('readFile', outFileName);
+        ffmpeg.FS('unlink', outFileName);
+        ffmpeg.FS('unlink', inputFileName);
+        return parseFFmpegExtractedMetadata(metadata);
+    } catch (e) {
+        logError(e, 'ffmpeg metadata extraction failed');
+        throw e;
+    }
+}
+
 export default new FFmpegService();

src/services/updateCreationTimeWithExif.ts

@@ -10,9 +10,10 @@ import downloadManager from './downloadManager';
 import { updatePublicMagicMetadata } from './fileService';
 import { EnteFile } from 'types/file';
 
-import { getRawExif, getUNIXTime } from './upload/exifService';
+import { getRawExif } from './upload/exifService';
 import { getFileType } from './upload/readFileService';
 import { FILE_TYPE } from 'constants/file';
+import { getUnixTimeInMicroSeconds } from 'utils/time';
 
 export async function updateCreationTimeWithExif(
     filesToBeUpdated: EnteFile[],
@@ -33,7 +34,7 @@ export async function updateCreationTimeWithExif(
                 }
                 let correctCreationTime: number;
                 if (fixOption === FIX_OPTIONS.CUSTOM_TIME) {
-                    correctCreationTime = getUNIXTime(customTime);
+                    correctCreationTime = getUnixTimeInMicroSeconds(customTime);
                 } else {
                     const fileURL = await downloadManager.getFile(file);
                     const fileObject = await getFileFromURL(fileURL);
@@ -41,11 +42,13 @@ export async function updateCreationTimeWithExif(
                     const fileTypeInfo = await getFileType(reader, fileObject);
                     const exifData = await getRawExif(fileObject, fileTypeInfo);
                     if (fixOption === FIX_OPTIONS.DATE_TIME_ORIGINAL) {
-                        correctCreationTime = getUNIXTime(
+                        correctCreationTime = getUnixTimeInMicroSeconds(
                             exifData?.DateTimeOriginal
                         );
                     } else {
-                        correctCreationTime = getUNIXTime(exifData?.CreateDate);
+                        correctCreationTime = getUnixTimeInMicroSeconds(
+                            exifData?.CreateDate
+                        );
                     }
                 }
                 if (

src/services/upload/exifService.ts

@@ -1,9 +1,11 @@
-import { NULL_LOCATION } from 'constants/upload';
+import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload';
 import { Location } from 'types/upload';
 import exifr from 'exifr';
 import piexif from 'piexifjs';
 import { FileTypeInfo } from 'types/upload';
 import { logError } from 'utils/sentry';
+import { ParsedExtractedMetadata } from 'types/upload';
+import { getUnixTimeInMicroSeconds } from 'utils/time';
 
 const EXIF_TAGS_NEEDED = [
     'DateTimeOriginal',
@@ -23,37 +25,29 @@ interface Exif {
     GPSLatitudeRef?: number;
     GPSLongitudeRef?: number;
 }
-interface ParsedEXIFData {
-    location: Location;
-    creationTime: number;
-}
 
 export async function getExifData(
     receivedFile: File,
     fileTypeInfo: FileTypeInfo
-): Promise<ParsedEXIFData> {
+): Promise<ParsedExtractedMetadata> {
-    const nullExifData: ParsedEXIFData = {
+    let parsedEXIFData = NULL_EXTRACTED_METADATA;
-        location: NULL_LOCATION,
-        creationTime: null,
-    };
     try {
         const exifData = await getRawExif(receivedFile, fileTypeInfo);
         if (!exifData) {
-            return nullExifData;
+            return parsedEXIFData;
         }
-        const parsedEXIFData = {
+        parsedEXIFData = {
             location: getEXIFLocation(exifData),
-            creationTime: getUNIXTime(
+            creationTime: getUnixTimeInMicroSeconds(
                 exifData.DateTimeOriginal ??
                     exifData.CreateDate ??
                     exifData.ModifyDate
             ),
         };
-        return parsedEXIFData;
     } catch (e) {
         logError(e, 'getExifData failed');
-        return nullExifData;
     }
+    return parsedEXIFData;
 }
 
 export async function updateFileCreationDateInEXIF(
@@ -131,22 +125,6 @@ export async function getRawExif(
     return exifData;
 }
 
-export function getUNIXTime(dateTime: Date) {
-    try {
-        if (!dateTime) {
-            return null;
-        }
-        const unixTime = dateTime.getTime() * 1000;
-        if (unixTime <= 0) {
-            return null;
-        } else {
-            return unixTime;
-        }
-    } catch (e) {
-        logError(e, 'getUNIXTime failed', { dateTime });
-    }
-}
-
 function getEXIFLocation(exifData): Location {
     if (!exifData.latitude || !exifData.longitude) {
         return NULL_LOCATION;

src/services/upload/metadataService.ts

@@ -6,9 +6,11 @@ import {
     ParsedMetadataJSON,
     Location,
     FileTypeInfo,
+    ParsedExtractedMetadata,
 } from 'types/upload';
-import { NULL_LOCATION } from 'constants/upload';
+import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload';
 import { splitFilenameAndExtension } from 'utils/file';
+import { getVideoMetadata } from './videoMetadataService';
 
 interface ParsedMetadataJSONWithTitle {
     title: string;
@@ -25,23 +27,25 @@ export async function extractMetadata(
     receivedFile: File,
     fileTypeInfo: FileTypeInfo
 ) {
-    let exifData = null;
+    let extractedMetadata: ParsedExtractedMetadata = NULL_EXTRACTED_METADATA;
     if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) {
-        exifData = await getExifData(receivedFile, fileTypeInfo);
+        extractedMetadata = await getExifData(receivedFile, fileTypeInfo);
+    } else if (fileTypeInfo.fileType === FILE_TYPE.VIDEO) {
+        extractedMetadata = await getVideoMetadata(receivedFile);
     }
 
-    const extractedMetadata: Metadata = {
+    const metadata: Metadata = {
         title: `${splitFilenameAndExtension(receivedFile.name)[0]}.${
             fileTypeInfo.exactType
         }`,
         creationTime:
-            exifData?.creationTime ?? receivedFile.lastModified * 1000,
+            extractedMetadata.creationTime ?? receivedFile.lastModified * 1000,
         modificationTime: receivedFile.lastModified * 1000,
-        latitude: exifData?.location?.latitude,
+        latitude: extractedMetadata.location.latitude,
-        longitude: exifData?.location?.longitude,
+        longitude: extractedMetadata.location.longitude,
         fileType: fileTypeInfo.fileType,
     };
-    return extractedMetadata;
+    return metadata;
 }
 
 export const getMetadataJSONMapKey = (

src/services/upload/videoMetadataService.ts

@@ -0,0 +1,76 @@
+import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload';
+import ffmpegService from 'services/ffmpegService';
+import { getUnixTimeInMicroSeconds } from 'utils/time';
+import { ParsedExtractedMetadata } from 'types/upload';
+import { logError } from 'utils/sentry';
+
+enum VideoMetadata {
+    CREATION_TIME = 'creation_time',
+    APPLE_CONTENT_IDENTIFIER = 'com.apple.quicktime.content.identifier',
+    APPLE_LIVE_PHOTO_IDENTIFIER = 'com.apple.quicktime.live-photo.auto',
+    APPLE_CREATION_DATE = 'com.apple.quicktime.creationdate',
+    APPLE_LOCATION_ISO = 'com.apple.quicktime.location.ISO6709',
+    LOCATION = 'location',
+}
+
+export async function getVideoMetadata(file: File) {
+    let videoMetadata = NULL_EXTRACTED_METADATA;
+    try {
+        videoMetadata = await ffmpegService.extractMetadata(file);
+    } catch (e) {
+        logError(e, 'failed to get video metadata');
+    }
+
+    return videoMetadata;
+}
+
+export function parseFFmpegExtractedMetadata(encodedMetadata: Uint8Array) {
+    const metadataString = new TextDecoder().decode(encodedMetadata);
+    const metadataPropertyArray = metadataString.split('\n');
+    const metadataKeyValueArray = metadataPropertyArray.map((property) =>
+        property.split('=')
+    );
+    const validKeyValuePairs = metadataKeyValueArray.filter(
+        (keyValueArray) => keyValueArray.length === 2
+    ) as Array<[string, string]>;
+
+    const metadataMap = Object.fromEntries(validKeyValuePairs);
+
+    const location = parseAppleISOLocation(
+        metadataMap[VideoMetadata.APPLE_LOCATION_ISO] ??
+            metadataMap[VideoMetadata.LOCATION]
+    );
+
+    const creationTime = parseCreationTime(
+        metadataMap[VideoMetadata.APPLE_CREATION_DATE] ??
+            metadataMap[VideoMetadata.CREATION_TIME]
+    );
+    const parsedMetadata: ParsedExtractedMetadata = {
+        creationTime,
+        location: {
+            latitude: location.latitude,
+            longitude: location.longitude,
+        },
+    };
+    return parsedMetadata;
+}
+
+function parseAppleISOLocation(isoLocation: string) {
+    let location = NULL_LOCATION;
+    if (isoLocation) {
+        const [latitude, longitude] = isoLocation
+            .match(/(\+|-)\d+\.*\d+/g)
+            .map((x) => parseFloat(x));
+
+        location = { latitude, longitude };
+    }
+    return location;
+}
+
+function parseCreationTime(creationTime: string) {
+    let dateTime = null;
+    if (creationTime) {
+        dateTime = getUnixTimeInMicroSeconds(new Date(creationTime));
+    }
+    return dateTime;
+}

src/types/upload/index.ts

@@ -130,3 +130,8 @@ export interface UploadFile extends BackupedFile {
     encryptedKey: string;
     keyDecryptionNonce: string;
 }
+
+export interface ParsedExtractedMetadata {
+    location: Location;
+    creationTime: number;
+}

src/utils/time/index.ts

@@ -0,0 +1,11 @@
+export function getUnixTimeInMicroSeconds(dateTime: Date) {
+    if (!dateTime || isNaN(dateTime.getTime())) {
+        return null;
+    }
+    const unixTime = dateTime.getTime() * 1000;
+    if (unixTime <= 0) {
+        return null;
+    } else {
+        return unixTime;
+    }
+}