beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #391 from ente-io/ffmpeg-video-metadata

Browse source 
Ffmpeg video metadata
This commit is contained in:
Abhinav Kumar 2022-03-01 13:18:24 +05:30 committed by GitHub
parent 7a1b2e46ba a86b9134d2
commit ba2cdacec5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 203 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/constants/upload/index.ts
View file

@ -1,6 +1,6 @@
import { ENCRYPTION_CHUNK_SIZE } from 'constants/crypto';
import { FILE_TYPE } from 'constants/file';
import { Location } from 'types/upload';
import { Location, ParsedExtractedMetadata } from 'types/upload';
// list of format that were missed by type-detection for some files.
export const FORMAT_MISSED_BY_FILE_TYPE_LIB = [
@ -43,3 +43,8 @@ export enum FileUploadResults {
export const MAX_FILE_SIZE_SUPPORTED = 5 * 1024 * 1024 * 1024; // 5 GB
export const LIVE_PHOTO_ASSET_SIZE_LIMIT = 20 * 1024 * 1024; // 20MB
export const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = {
location: NULL_LOCATION,
creationTime: null,
};

80
src/services/ffmpegService.ts
View file

@ -2,14 +2,17 @@ import { createFFmpeg, FFmpeg } from '@ffmpeg/ffmpeg';
import { CustomError } from 'utils/error';
import { logError } from 'utils/sentry';
import QueueProcessor from './queueProcessor';
import { ParsedExtractedMetadata } from 'types/upload';
import { getUint8ArrayView } from './upload/readFileService';
import { parseFFmpegExtractedMetadata } from './upload/videoMetadataService';
class FFmpegService {
private ffmpeg: FFmpeg = null;
private isLoading = null;
private fileReader: FileReader = null;
private generateThumbnailProcessor = new QueueProcessor<Uint8Array>(1);
private ffmpegTaskQueue = new QueueProcessor<any>(1);
async init() {
try {
this.ffmpeg = createFFmpeg({
@ -26,7 +29,7 @@ class FFmpegService {
}
}
async generateThumbnail(file: File) {
async generateThumbnail(file: File): Promise<Uint8Array> {
if (!this.ffmpeg) {
await this.init();
}
@ -36,7 +39,7 @@ class FFmpegService {
if (this.isLoading) {
await this.isLoading;
}
const response = this.generateThumbnailProcessor.queueUpRequest(
const response = this.ffmpegTaskQueue.queueUpRequest(
generateThumbnailHelper.bind(
null,
this.ffmpeg,
@ -56,6 +59,37 @@ class FFmpegService {
}
}
}
async extractMetadata(file: File): Promise<ParsedExtractedMetadata> {
if (!this.ffmpeg) {
await this.init();
}
if (!this.fileReader) {
this.fileReader = new FileReader();
}
if (this.isLoading) {
await this.isLoading;
}
const response = this.ffmpegTaskQueue.queueUpRequest(
extractVideoMetadataHelper.bind(
null,
this.ffmpeg,
this.fileReader,
file
)
);
try {
return await response.promise;
} catch (e) {
if (e.message === CustomError.REQUEST_CANCELLED) {
// ignore
return null;
} else {
logError(e, 'ffmpeg metadata extraction failed');
throw e;
}
}
}
}
async function generateThumbnailHelper(
@ -101,4 +135,44 @@ async function generateThumbnailHelper(
}
}
async function extractVideoMetadataHelper(
ffmpeg: FFmpeg,
reader: FileReader,
file: File
) {
try {
const inputFileName = `${Date.now().toString()}-${file.name}`;
const outFileName = `${Date.now().toString()}-metadata.txt`;
ffmpeg.FS(
'writeFile',
inputFileName,
await getUint8ArrayView(reader, file)
);
let metadata = null;
// https://stackoverflow.com/questions/9464617/retrieving-and-saving-media-metadata-using-ffmpeg
// -c [short for codex] copy[(stream_specifier)[ffmpeg.org/ffmpeg.html#Stream-specifiers]] => copies all the stream without re-encoding
// -map_metadata [http://ffmpeg.org/ffmpeg.html#Advanced-options search for map_metadata] => copies all stream metadata to the out
// -f ffmetadata [https://ffmpeg.org/ffmpeg-formats.html#Metadata-1] => dump metadata from media files into a simple UTF-8-encoded INI-like text file
await ffmpeg.run(
'-i',
inputFileName,
'-c',
'copy',
'-map_metadata',
'0',
'-f',
'ffmetadata',
outFileName
);
metadata = ffmpeg.FS('readFile', outFileName);
ffmpeg.FS('unlink', outFileName);
ffmpeg.FS('unlink', inputFileName);
return parseFFmpegExtractedMetadata(metadata);
} catch (e) {
logError(e, 'ffmpeg metadata extraction failed');
throw e;
}
}
export default new FFmpegService();

11
src/services/updateCreationTimeWithExif.ts
View file

@ -10,9 +10,10 @@ import downloadManager from './downloadManager';
import { updatePublicMagicMetadata } from './fileService';
import { EnteFile } from 'types/file';
import { getRawExif, getUNIXTime } from './upload/exifService';
import { getRawExif } from './upload/exifService';
import { getFileType } from './upload/readFileService';
import { FILE_TYPE } from 'constants/file';
import { getUnixTimeInMicroSeconds } from 'utils/time';
export async function updateCreationTimeWithExif(
filesToBeUpdated: EnteFile[],
@ -33,7 +34,7 @@ export async function updateCreationTimeWithExif(
}
let correctCreationTime: number;
if (fixOption === FIX_OPTIONS.CUSTOM_TIME) {
correctCreationTime = getUNIXTime(customTime);
correctCreationTime = getUnixTimeInMicroSeconds(customTime);
} else {
const fileURL = await downloadManager.getFile(file);
const fileObject = await getFileFromURL(fileURL);
@ -41,11 +42,13 @@ export async function updateCreationTimeWithExif(
const fileTypeInfo = await getFileType(reader, fileObject);
const exifData = await getRawExif(fileObject, fileTypeInfo);
if (fixOption === FIX_OPTIONS.DATE_TIME_ORIGINAL) {
correctCreationTime = getUNIXTime(
correctCreationTime = getUnixTimeInMicroSeconds(
exifData?.DateTimeOriginal
);
} else {
correctCreationTime = getUNIXTime(exifData?.CreateDate);
correctCreationTime = getUnixTimeInMicroSeconds(
exifData?.CreateDate
);
}
}
if (

40
src/services/upload/exifService.ts
View file

@ -1,9 +1,11 @@
import { NULL_LOCATION } from 'constants/upload';
import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload';
import { Location } from 'types/upload';
import exifr from 'exifr';
import piexif from 'piexifjs';
import { FileTypeInfo } from 'types/upload';
import { logError } from 'utils/sentry';
import { ParsedExtractedMetadata } from 'types/upload';
import { getUnixTimeInMicroSeconds } from 'utils/time';
const EXIF_TAGS_NEEDED = [
'DateTimeOriginal',
@ -23,37 +25,29 @@ interface Exif {
GPSLatitudeRef?: number;
GPSLongitudeRef?: number;
}
interface ParsedEXIFData {
location: Location;
creationTime: number;
}
export async function getExifData(
receivedFile: File,
fileTypeInfo: FileTypeInfo
): Promise<ParsedEXIFData> {
const nullExifData: ParsedEXIFData = {
location: NULL_LOCATION,
creationTime: null,
};
): Promise<ParsedExtractedMetadata> {
let parsedEXIFData = NULL_EXTRACTED_METADATA;
try {
const exifData = await getRawExif(receivedFile, fileTypeInfo);
if (!exifData) {
return nullExifData;
return parsedEXIFData;
}
const parsedEXIFData = {
parsedEXIFData = {
location: getEXIFLocation(exifData),
creationTime: getUNIXTime(
creationTime: getUnixTimeInMicroSeconds(
exifData.DateTimeOriginal ??
exifData.CreateDate ??
exifData.ModifyDate
),
};
return parsedEXIFData;
} catch (e) {
logError(e, 'getExifData failed');
return nullExifData;
}
return parsedEXIFData;
}
export async function updateFileCreationDateInEXIF(
@ -131,22 +125,6 @@ export async function getRawExif(
return exifData;
}
export function getUNIXTime(dateTime: Date) {
try {
if (!dateTime) {
return null;
}
const unixTime = dateTime.getTime() * 1000;
if (unixTime <= 0) {
return null;
} else {
return unixTime;
}
} catch (e) {
logError(e, 'getUNIXTime failed', { dateTime });
}
}
function getEXIFLocation(exifData): Location {
if (!exifData.latitude || !exifData.longitude) {
return NULL_LOCATION;

20
src/services/upload/metadataService.ts
View file

@ -6,9 +6,11 @@ import {
ParsedMetadataJSON,
Location,
FileTypeInfo,
ParsedExtractedMetadata,
} from 'types/upload';
import { NULL_LOCATION } from 'constants/upload';
import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload';
import { splitFilenameAndExtension } from 'utils/file';
import { getVideoMetadata } from './videoMetadataService';
interface ParsedMetadataJSONWithTitle {
title: string;
@ -25,23 +27,25 @@ export async function extractMetadata(
receivedFile: File,
fileTypeInfo: FileTypeInfo
) {
let exifData = null;
let extractedMetadata: ParsedExtractedMetadata = NULL_EXTRACTED_METADATA;
if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) {
exifData = await getExifData(receivedFile, fileTypeInfo);
extractedMetadata = await getExifData(receivedFile, fileTypeInfo);
} else if (fileTypeInfo.fileType === FILE_TYPE.VIDEO) {
extractedMetadata = await getVideoMetadata(receivedFile);
}
const extractedMetadata: Metadata = {
const metadata: Metadata = {
title: `${splitFilenameAndExtension(receivedFile.name)[0]}.${
fileTypeInfo.exactType
}`,
creationTime:
exifData?.creationTime ?? receivedFile.lastModified * 1000,
extractedMetadata.creationTime ?? receivedFile.lastModified * 1000,
modificationTime: receivedFile.lastModified * 1000,
latitude: exifData?.location?.latitude,
longitude: exifData?.location?.longitude,
latitude: extractedMetadata.location.latitude,
longitude: extractedMetadata.location.longitude,
fileType: fileTypeInfo.fileType,
};
return extractedMetadata;
return metadata;
}
export const getMetadataJSONMapKey = (

76
src/services/upload/videoMetadataService.ts Normal file
View file

@ -0,0 +1,76 @@
import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload';
import ffmpegService from 'services/ffmpegService';
import { getUnixTimeInMicroSeconds } from 'utils/time';
import { ParsedExtractedMetadata } from 'types/upload';
import { logError } from 'utils/sentry';
enum VideoMetadata {
CREATION_TIME = 'creation_time',
APPLE_CONTENT_IDENTIFIER = 'com.apple.quicktime.content.identifier',
APPLE_LIVE_PHOTO_IDENTIFIER = 'com.apple.quicktime.live-photo.auto',
APPLE_CREATION_DATE = 'com.apple.quicktime.creationdate',
APPLE_LOCATION_ISO = 'com.apple.quicktime.location.ISO6709',
LOCATION = 'location',
}
export async function getVideoMetadata(file: File) {
let videoMetadata = NULL_EXTRACTED_METADATA;
try {
videoMetadata = await ffmpegService.extractMetadata(file);
} catch (e) {
logError(e, 'failed to get video metadata');
}
return videoMetadata;
}
export function parseFFmpegExtractedMetadata(encodedMetadata: Uint8Array) {
const metadataString = new TextDecoder().decode(encodedMetadata);
const metadataPropertyArray = metadataString.split('\n');
const metadataKeyValueArray = metadataPropertyArray.map((property) =>
property.split('=')
);
const validKeyValuePairs = metadataKeyValueArray.filter(
(keyValueArray) => keyValueArray.length === 2
) as Array<[string, string]>;
const metadataMap = Object.fromEntries(validKeyValuePairs);
const location = parseAppleISOLocation(
metadataMap[VideoMetadata.APPLE_LOCATION_ISO] ??
metadataMap[VideoMetadata.LOCATION]
);
const creationTime = parseCreationTime(
metadataMap[VideoMetadata.APPLE_CREATION_DATE] ??
metadataMap[VideoMetadata.CREATION_TIME]
);
const parsedMetadata: ParsedExtractedMetadata = {
creationTime,
location: {
latitude: location.latitude,
longitude: location.longitude,
},
};
return parsedMetadata;
}
function parseAppleISOLocation(isoLocation: string) {
let location = NULL_LOCATION;
if (isoLocation) {
const [latitude, longitude] = isoLocation
.match(/(\+|-)\d+\.*\d+/g)
.map((x) => parseFloat(x));
location = { latitude, longitude };
}
return location;
}
function parseCreationTime(creationTime: string) {
let dateTime = null;
if (creationTime) {
dateTime = getUnixTimeInMicroSeconds(new Date(creationTime));
}
return dateTime;
}

5
src/types/upload/index.ts
View file

@ -130,3 +130,8 @@ export interface UploadFile extends BackupedFile {
encryptedKey: string;
keyDecryptionNonce: string;
}
export interface ParsedExtractedMetadata {
location: Location;
creationTime: number;
}

11
src/utils/time/index.ts Normal file
View file

@ -0,0 +1,11 @@
export function getUnixTimeInMicroSeconds(dateTime: Date) {
if (!dateTime || isNaN(dateTime.getTime())) {
return null;
}
const unixTime = dateTime.getTime() * 1000;
if (unixTime <= 0) {
return null;
} else {
return unixTime;
}
}