Merge pull request #404 from ente-io/master
release video metadata extraction
This commit is contained in:
commit
91e71257cb
|
@ -1,6 +1,6 @@
|
|||
import { ENCRYPTION_CHUNK_SIZE } from 'constants/crypto';
|
||||
import { FILE_TYPE } from 'constants/file';
|
||||
import { Location } from 'types/upload';
|
||||
import { Location, ParsedExtractedMetadata } from 'types/upload';
|
||||
|
||||
// list of format that were missed by type-detection for some files.
|
||||
export const FORMAT_MISSED_BY_FILE_TYPE_LIB = [
|
||||
|
@ -43,3 +43,8 @@ export enum FileUploadResults {
|
|||
export const MAX_FILE_SIZE_SUPPORTED = 5 * 1024 * 1024 * 1024; // 5 GB
|
||||
|
||||
export const LIVE_PHOTO_ASSET_SIZE_LIMIT = 20 * 1024 * 1024; // 20MB
|
||||
|
||||
export const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = {
|
||||
location: NULL_LOCATION,
|
||||
creationTime: null,
|
||||
};
|
||||
|
|
|
@ -2,14 +2,17 @@ import { createFFmpeg, FFmpeg } from '@ffmpeg/ffmpeg';
|
|||
import { CustomError } from 'utils/error';
|
||||
import { logError } from 'utils/sentry';
|
||||
import QueueProcessor from './queueProcessor';
|
||||
import { ParsedExtractedMetadata } from 'types/upload';
|
||||
|
||||
import { getUint8ArrayView } from './upload/readFileService';
|
||||
import { parseFFmpegExtractedMetadata } from './upload/videoMetadataService';
|
||||
|
||||
class FFmpegService {
|
||||
private ffmpeg: FFmpeg = null;
|
||||
private isLoading = null;
|
||||
private fileReader: FileReader = null;
|
||||
|
||||
private generateThumbnailProcessor = new QueueProcessor<Uint8Array>(1);
|
||||
private ffmpegTaskQueue = new QueueProcessor<any>(1);
|
||||
async init() {
|
||||
try {
|
||||
this.ffmpeg = createFFmpeg({
|
||||
|
@ -26,7 +29,7 @@ class FFmpegService {
|
|||
}
|
||||
}
|
||||
|
||||
async generateThumbnail(file: File) {
|
||||
async generateThumbnail(file: File): Promise<Uint8Array> {
|
||||
if (!this.ffmpeg) {
|
||||
await this.init();
|
||||
}
|
||||
|
@ -36,7 +39,7 @@ class FFmpegService {
|
|||
if (this.isLoading) {
|
||||
await this.isLoading;
|
||||
}
|
||||
const response = this.generateThumbnailProcessor.queueUpRequest(
|
||||
const response = this.ffmpegTaskQueue.queueUpRequest(
|
||||
generateThumbnailHelper.bind(
|
||||
null,
|
||||
this.ffmpeg,
|
||||
|
@ -56,6 +59,37 @@ class FFmpegService {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
async extractMetadata(file: File): Promise<ParsedExtractedMetadata> {
|
||||
if (!this.ffmpeg) {
|
||||
await this.init();
|
||||
}
|
||||
if (!this.fileReader) {
|
||||
this.fileReader = new FileReader();
|
||||
}
|
||||
if (this.isLoading) {
|
||||
await this.isLoading;
|
||||
}
|
||||
const response = this.ffmpegTaskQueue.queueUpRequest(
|
||||
extractVideoMetadataHelper.bind(
|
||||
null,
|
||||
this.ffmpeg,
|
||||
this.fileReader,
|
||||
file
|
||||
)
|
||||
);
|
||||
try {
|
||||
return await response.promise;
|
||||
} catch (e) {
|
||||
if (e.message === CustomError.REQUEST_CANCELLED) {
|
||||
// ignore
|
||||
return null;
|
||||
} else {
|
||||
logError(e, 'ffmpeg metadata extraction failed');
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function generateThumbnailHelper(
|
||||
|
@ -101,4 +135,44 @@ async function generateThumbnailHelper(
|
|||
}
|
||||
}
|
||||
|
||||
async function extractVideoMetadataHelper(
|
||||
ffmpeg: FFmpeg,
|
||||
reader: FileReader,
|
||||
file: File
|
||||
) {
|
||||
try {
|
||||
const inputFileName = `${Date.now().toString()}-${file.name}`;
|
||||
const outFileName = `${Date.now().toString()}-metadata.txt`;
|
||||
ffmpeg.FS(
|
||||
'writeFile',
|
||||
inputFileName,
|
||||
await getUint8ArrayView(reader, file)
|
||||
);
|
||||
let metadata = null;
|
||||
|
||||
// https://stackoverflow.com/questions/9464617/retrieving-and-saving-media-metadata-using-ffmpeg
|
||||
// -c [short for codex] copy[(stream_specifier)[ffmpeg.org/ffmpeg.html#Stream-specifiers]] => copies all the stream without re-encoding
|
||||
// -map_metadata [http://ffmpeg.org/ffmpeg.html#Advanced-options search for map_metadata] => copies all stream metadata to the out
|
||||
// -f ffmetadata [https://ffmpeg.org/ffmpeg-formats.html#Metadata-1] => dump metadata from media files into a simple UTF-8-encoded INI-like text file
|
||||
await ffmpeg.run(
|
||||
'-i',
|
||||
inputFileName,
|
||||
'-c',
|
||||
'copy',
|
||||
'-map_metadata',
|
||||
'0',
|
||||
'-f',
|
||||
'ffmetadata',
|
||||
outFileName
|
||||
);
|
||||
metadata = ffmpeg.FS('readFile', outFileName);
|
||||
ffmpeg.FS('unlink', outFileName);
|
||||
ffmpeg.FS('unlink', inputFileName);
|
||||
return parseFFmpegExtractedMetadata(metadata);
|
||||
} catch (e) {
|
||||
logError(e, 'ffmpeg metadata extraction failed');
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
export default new FFmpegService();
|
||||
|
|
|
@ -10,9 +10,10 @@ import downloadManager from './downloadManager';
|
|||
import { updatePublicMagicMetadata } from './fileService';
|
||||
import { EnteFile } from 'types/file';
|
||||
|
||||
import { getRawExif, getUNIXTime } from './upload/exifService';
|
||||
import { getRawExif } from './upload/exifService';
|
||||
import { getFileType } from './upload/readFileService';
|
||||
import { FILE_TYPE } from 'constants/file';
|
||||
import { getUnixTimeInMicroSeconds } from 'utils/time';
|
||||
|
||||
export async function updateCreationTimeWithExif(
|
||||
filesToBeUpdated: EnteFile[],
|
||||
|
@ -33,7 +34,7 @@ export async function updateCreationTimeWithExif(
|
|||
}
|
||||
let correctCreationTime: number;
|
||||
if (fixOption === FIX_OPTIONS.CUSTOM_TIME) {
|
||||
correctCreationTime = getUNIXTime(customTime);
|
||||
correctCreationTime = getUnixTimeInMicroSeconds(customTime);
|
||||
} else {
|
||||
const fileURL = await downloadManager.getFile(file);
|
||||
const fileObject = await getFileFromURL(fileURL);
|
||||
|
@ -41,11 +42,13 @@ export async function updateCreationTimeWithExif(
|
|||
const fileTypeInfo = await getFileType(reader, fileObject);
|
||||
const exifData = await getRawExif(fileObject, fileTypeInfo);
|
||||
if (fixOption === FIX_OPTIONS.DATE_TIME_ORIGINAL) {
|
||||
correctCreationTime = getUNIXTime(
|
||||
correctCreationTime = getUnixTimeInMicroSeconds(
|
||||
exifData?.DateTimeOriginal
|
||||
);
|
||||
} else {
|
||||
correctCreationTime = getUNIXTime(exifData?.CreateDate);
|
||||
correctCreationTime = getUnixTimeInMicroSeconds(
|
||||
exifData?.CreateDate
|
||||
);
|
||||
}
|
||||
}
|
||||
if (
|
||||
|
|
|
@ -1,9 +1,11 @@
|
|||
import { NULL_LOCATION } from 'constants/upload';
|
||||
import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload';
|
||||
import { Location } from 'types/upload';
|
||||
import exifr from 'exifr';
|
||||
import piexif from 'piexifjs';
|
||||
import { FileTypeInfo } from 'types/upload';
|
||||
import { logError } from 'utils/sentry';
|
||||
import { ParsedExtractedMetadata } from 'types/upload';
|
||||
import { getUnixTimeInMicroSeconds } from 'utils/time';
|
||||
|
||||
const EXIF_TAGS_NEEDED = [
|
||||
'DateTimeOriginal',
|
||||
|
@ -23,37 +25,29 @@ interface Exif {
|
|||
GPSLatitudeRef?: number;
|
||||
GPSLongitudeRef?: number;
|
||||
}
|
||||
interface ParsedEXIFData {
|
||||
location: Location;
|
||||
creationTime: number;
|
||||
}
|
||||
|
||||
export async function getExifData(
|
||||
receivedFile: File,
|
||||
fileTypeInfo: FileTypeInfo
|
||||
): Promise<ParsedEXIFData> {
|
||||
const nullExifData: ParsedEXIFData = {
|
||||
location: NULL_LOCATION,
|
||||
creationTime: null,
|
||||
};
|
||||
): Promise<ParsedExtractedMetadata> {
|
||||
let parsedEXIFData = NULL_EXTRACTED_METADATA;
|
||||
try {
|
||||
const exifData = await getRawExif(receivedFile, fileTypeInfo);
|
||||
if (!exifData) {
|
||||
return nullExifData;
|
||||
return parsedEXIFData;
|
||||
}
|
||||
const parsedEXIFData = {
|
||||
parsedEXIFData = {
|
||||
location: getEXIFLocation(exifData),
|
||||
creationTime: getUNIXTime(
|
||||
creationTime: getUnixTimeInMicroSeconds(
|
||||
exifData.DateTimeOriginal ??
|
||||
exifData.CreateDate ??
|
||||
exifData.ModifyDate
|
||||
),
|
||||
};
|
||||
return parsedEXIFData;
|
||||
} catch (e) {
|
||||
logError(e, 'getExifData failed');
|
||||
return nullExifData;
|
||||
}
|
||||
return parsedEXIFData;
|
||||
}
|
||||
|
||||
export async function updateFileCreationDateInEXIF(
|
||||
|
@ -131,22 +125,6 @@ export async function getRawExif(
|
|||
return exifData;
|
||||
}
|
||||
|
||||
export function getUNIXTime(dateTime: Date) {
|
||||
try {
|
||||
if (!dateTime) {
|
||||
return null;
|
||||
}
|
||||
const unixTime = dateTime.getTime() * 1000;
|
||||
if (unixTime <= 0) {
|
||||
return null;
|
||||
} else {
|
||||
return unixTime;
|
||||
}
|
||||
} catch (e) {
|
||||
logError(e, 'getUNIXTime failed', { dateTime });
|
||||
}
|
||||
}
|
||||
|
||||
function getEXIFLocation(exifData): Location {
|
||||
if (!exifData.latitude || !exifData.longitude) {
|
||||
return NULL_LOCATION;
|
||||
|
|
|
@ -6,9 +6,11 @@ import {
|
|||
ParsedMetadataJSON,
|
||||
Location,
|
||||
FileTypeInfo,
|
||||
ParsedExtractedMetadata,
|
||||
} from 'types/upload';
|
||||
import { NULL_LOCATION } from 'constants/upload';
|
||||
import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload';
|
||||
import { splitFilenameAndExtension } from 'utils/file';
|
||||
import { getVideoMetadata } from './videoMetadataService';
|
||||
|
||||
interface ParsedMetadataJSONWithTitle {
|
||||
title: string;
|
||||
|
@ -25,23 +27,25 @@ export async function extractMetadata(
|
|||
receivedFile: File,
|
||||
fileTypeInfo: FileTypeInfo
|
||||
) {
|
||||
let exifData = null;
|
||||
let extractedMetadata: ParsedExtractedMetadata = NULL_EXTRACTED_METADATA;
|
||||
if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) {
|
||||
exifData = await getExifData(receivedFile, fileTypeInfo);
|
||||
extractedMetadata = await getExifData(receivedFile, fileTypeInfo);
|
||||
} else if (fileTypeInfo.fileType === FILE_TYPE.VIDEO) {
|
||||
extractedMetadata = await getVideoMetadata(receivedFile);
|
||||
}
|
||||
|
||||
const extractedMetadata: Metadata = {
|
||||
const metadata: Metadata = {
|
||||
title: `${splitFilenameAndExtension(receivedFile.name)[0]}.${
|
||||
fileTypeInfo.exactType
|
||||
}`,
|
||||
creationTime:
|
||||
exifData?.creationTime ?? receivedFile.lastModified * 1000,
|
||||
extractedMetadata.creationTime ?? receivedFile.lastModified * 1000,
|
||||
modificationTime: receivedFile.lastModified * 1000,
|
||||
latitude: exifData?.location?.latitude,
|
||||
longitude: exifData?.location?.longitude,
|
||||
latitude: extractedMetadata.location.latitude,
|
||||
longitude: extractedMetadata.location.longitude,
|
||||
fileType: fileTypeInfo.fileType,
|
||||
};
|
||||
return extractedMetadata;
|
||||
return metadata;
|
||||
}
|
||||
|
||||
export const getMetadataJSONMapKey = (
|
||||
|
|
76
src/services/upload/videoMetadataService.ts
Normal file
76
src/services/upload/videoMetadataService.ts
Normal file
|
@ -0,0 +1,76 @@
|
|||
import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload';
|
||||
import ffmpegService from 'services/ffmpegService';
|
||||
import { getUnixTimeInMicroSeconds } from 'utils/time';
|
||||
import { ParsedExtractedMetadata } from 'types/upload';
|
||||
import { logError } from 'utils/sentry';
|
||||
|
||||
enum VideoMetadata {
|
||||
CREATION_TIME = 'creation_time',
|
||||
APPLE_CONTENT_IDENTIFIER = 'com.apple.quicktime.content.identifier',
|
||||
APPLE_LIVE_PHOTO_IDENTIFIER = 'com.apple.quicktime.live-photo.auto',
|
||||
APPLE_CREATION_DATE = 'com.apple.quicktime.creationdate',
|
||||
APPLE_LOCATION_ISO = 'com.apple.quicktime.location.ISO6709',
|
||||
LOCATION = 'location',
|
||||
}
|
||||
|
||||
export async function getVideoMetadata(file: File) {
|
||||
let videoMetadata = NULL_EXTRACTED_METADATA;
|
||||
try {
|
||||
videoMetadata = await ffmpegService.extractMetadata(file);
|
||||
} catch (e) {
|
||||
logError(e, 'failed to get video metadata');
|
||||
}
|
||||
|
||||
return videoMetadata;
|
||||
}
|
||||
|
||||
export function parseFFmpegExtractedMetadata(encodedMetadata: Uint8Array) {
|
||||
const metadataString = new TextDecoder().decode(encodedMetadata);
|
||||
const metadataPropertyArray = metadataString.split('\n');
|
||||
const metadataKeyValueArray = metadataPropertyArray.map((property) =>
|
||||
property.split('=')
|
||||
);
|
||||
const validKeyValuePairs = metadataKeyValueArray.filter(
|
||||
(keyValueArray) => keyValueArray.length === 2
|
||||
) as Array<[string, string]>;
|
||||
|
||||
const metadataMap = Object.fromEntries(validKeyValuePairs);
|
||||
|
||||
const location = parseAppleISOLocation(
|
||||
metadataMap[VideoMetadata.APPLE_LOCATION_ISO] ??
|
||||
metadataMap[VideoMetadata.LOCATION]
|
||||
);
|
||||
|
||||
const creationTime = parseCreationTime(
|
||||
metadataMap[VideoMetadata.APPLE_CREATION_DATE] ??
|
||||
metadataMap[VideoMetadata.CREATION_TIME]
|
||||
);
|
||||
const parsedMetadata: ParsedExtractedMetadata = {
|
||||
creationTime,
|
||||
location: {
|
||||
latitude: location.latitude,
|
||||
longitude: location.longitude,
|
||||
},
|
||||
};
|
||||
return parsedMetadata;
|
||||
}
|
||||
|
||||
function parseAppleISOLocation(isoLocation: string) {
|
||||
let location = NULL_LOCATION;
|
||||
if (isoLocation) {
|
||||
const [latitude, longitude] = isoLocation
|
||||
.match(/(\+|-)\d+\.*\d+/g)
|
||||
.map((x) => parseFloat(x));
|
||||
|
||||
location = { latitude, longitude };
|
||||
}
|
||||
return location;
|
||||
}
|
||||
|
||||
function parseCreationTime(creationTime: string) {
|
||||
let dateTime = null;
|
||||
if (creationTime) {
|
||||
dateTime = getUnixTimeInMicroSeconds(new Date(creationTime));
|
||||
}
|
||||
return dateTime;
|
||||
}
|
|
@ -130,3 +130,8 @@ export interface UploadFile extends BackupedFile {
|
|||
encryptedKey: string;
|
||||
keyDecryptionNonce: string;
|
||||
}
|
||||
|
||||
export interface ParsedExtractedMetadata {
|
||||
location: Location;
|
||||
creationTime: number;
|
||||
}
|
||||
|
|
11
src/utils/time/index.ts
Normal file
11
src/utils/time/index.ts
Normal file
|
@ -0,0 +1,11 @@
|
|||
export function getUnixTimeInMicroSeconds(dateTime: Date) {
|
||||
if (!dateTime || isNaN(dateTime.getTime())) {
|
||||
return null;
|
||||
}
|
||||
const unixTime = dateTime.getTime() * 1000;
|
||||
if (unixTime <= 0) {
|
||||
return null;
|
||||
} else {
|
||||
return unixTime;
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue