Merge pull request #404 from ente-io/master
release video metadata extraction
This commit is contained in:
commit
91e71257cb
|
@ -1,6 +1,6 @@
|
||||||
import { ENCRYPTION_CHUNK_SIZE } from 'constants/crypto';
|
import { ENCRYPTION_CHUNK_SIZE } from 'constants/crypto';
|
||||||
import { FILE_TYPE } from 'constants/file';
|
import { FILE_TYPE } from 'constants/file';
|
||||||
import { Location } from 'types/upload';
|
import { Location, ParsedExtractedMetadata } from 'types/upload';
|
||||||
|
|
||||||
// list of format that were missed by type-detection for some files.
|
// list of format that were missed by type-detection for some files.
|
||||||
export const FORMAT_MISSED_BY_FILE_TYPE_LIB = [
|
export const FORMAT_MISSED_BY_FILE_TYPE_LIB = [
|
||||||
|
@ -43,3 +43,8 @@ export enum FileUploadResults {
|
||||||
export const MAX_FILE_SIZE_SUPPORTED = 5 * 1024 * 1024 * 1024; // 5 GB
|
export const MAX_FILE_SIZE_SUPPORTED = 5 * 1024 * 1024 * 1024; // 5 GB
|
||||||
|
|
||||||
export const LIVE_PHOTO_ASSET_SIZE_LIMIT = 20 * 1024 * 1024; // 20MB
|
export const LIVE_PHOTO_ASSET_SIZE_LIMIT = 20 * 1024 * 1024; // 20MB
|
||||||
|
|
||||||
|
export const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = {
|
||||||
|
location: NULL_LOCATION,
|
||||||
|
creationTime: null,
|
||||||
|
};
|
||||||
|
|
|
@ -2,14 +2,17 @@ import { createFFmpeg, FFmpeg } from '@ffmpeg/ffmpeg';
|
||||||
import { CustomError } from 'utils/error';
|
import { CustomError } from 'utils/error';
|
||||||
import { logError } from 'utils/sentry';
|
import { logError } from 'utils/sentry';
|
||||||
import QueueProcessor from './queueProcessor';
|
import QueueProcessor from './queueProcessor';
|
||||||
|
import { ParsedExtractedMetadata } from 'types/upload';
|
||||||
|
|
||||||
import { getUint8ArrayView } from './upload/readFileService';
|
import { getUint8ArrayView } from './upload/readFileService';
|
||||||
|
import { parseFFmpegExtractedMetadata } from './upload/videoMetadataService';
|
||||||
|
|
||||||
class FFmpegService {
|
class FFmpegService {
|
||||||
private ffmpeg: FFmpeg = null;
|
private ffmpeg: FFmpeg = null;
|
||||||
private isLoading = null;
|
private isLoading = null;
|
||||||
private fileReader: FileReader = null;
|
private fileReader: FileReader = null;
|
||||||
|
|
||||||
private generateThumbnailProcessor = new QueueProcessor<Uint8Array>(1);
|
private ffmpegTaskQueue = new QueueProcessor<any>(1);
|
||||||
async init() {
|
async init() {
|
||||||
try {
|
try {
|
||||||
this.ffmpeg = createFFmpeg({
|
this.ffmpeg = createFFmpeg({
|
||||||
|
@ -26,7 +29,7 @@ class FFmpegService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async generateThumbnail(file: File) {
|
async generateThumbnail(file: File): Promise<Uint8Array> {
|
||||||
if (!this.ffmpeg) {
|
if (!this.ffmpeg) {
|
||||||
await this.init();
|
await this.init();
|
||||||
}
|
}
|
||||||
|
@ -36,7 +39,7 @@ class FFmpegService {
|
||||||
if (this.isLoading) {
|
if (this.isLoading) {
|
||||||
await this.isLoading;
|
await this.isLoading;
|
||||||
}
|
}
|
||||||
const response = this.generateThumbnailProcessor.queueUpRequest(
|
const response = this.ffmpegTaskQueue.queueUpRequest(
|
||||||
generateThumbnailHelper.bind(
|
generateThumbnailHelper.bind(
|
||||||
null,
|
null,
|
||||||
this.ffmpeg,
|
this.ffmpeg,
|
||||||
|
@ -56,6 +59,37 @@ class FFmpegService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async extractMetadata(file: File): Promise<ParsedExtractedMetadata> {
|
||||||
|
if (!this.ffmpeg) {
|
||||||
|
await this.init();
|
||||||
|
}
|
||||||
|
if (!this.fileReader) {
|
||||||
|
this.fileReader = new FileReader();
|
||||||
|
}
|
||||||
|
if (this.isLoading) {
|
||||||
|
await this.isLoading;
|
||||||
|
}
|
||||||
|
const response = this.ffmpegTaskQueue.queueUpRequest(
|
||||||
|
extractVideoMetadataHelper.bind(
|
||||||
|
null,
|
||||||
|
this.ffmpeg,
|
||||||
|
this.fileReader,
|
||||||
|
file
|
||||||
|
)
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
return await response.promise;
|
||||||
|
} catch (e) {
|
||||||
|
if (e.message === CustomError.REQUEST_CANCELLED) {
|
||||||
|
// ignore
|
||||||
|
return null;
|
||||||
|
} else {
|
||||||
|
logError(e, 'ffmpeg metadata extraction failed');
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function generateThumbnailHelper(
|
async function generateThumbnailHelper(
|
||||||
|
@ -101,4 +135,44 @@ async function generateThumbnailHelper(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function extractVideoMetadataHelper(
|
||||||
|
ffmpeg: FFmpeg,
|
||||||
|
reader: FileReader,
|
||||||
|
file: File
|
||||||
|
) {
|
||||||
|
try {
|
||||||
|
const inputFileName = `${Date.now().toString()}-${file.name}`;
|
||||||
|
const outFileName = `${Date.now().toString()}-metadata.txt`;
|
||||||
|
ffmpeg.FS(
|
||||||
|
'writeFile',
|
||||||
|
inputFileName,
|
||||||
|
await getUint8ArrayView(reader, file)
|
||||||
|
);
|
||||||
|
let metadata = null;
|
||||||
|
|
||||||
|
// https://stackoverflow.com/questions/9464617/retrieving-and-saving-media-metadata-using-ffmpeg
|
||||||
|
// -c [short for codex] copy[(stream_specifier)[ffmpeg.org/ffmpeg.html#Stream-specifiers]] => copies all the stream without re-encoding
|
||||||
|
// -map_metadata [http://ffmpeg.org/ffmpeg.html#Advanced-options search for map_metadata] => copies all stream metadata to the out
|
||||||
|
// -f ffmetadata [https://ffmpeg.org/ffmpeg-formats.html#Metadata-1] => dump metadata from media files into a simple UTF-8-encoded INI-like text file
|
||||||
|
await ffmpeg.run(
|
||||||
|
'-i',
|
||||||
|
inputFileName,
|
||||||
|
'-c',
|
||||||
|
'copy',
|
||||||
|
'-map_metadata',
|
||||||
|
'0',
|
||||||
|
'-f',
|
||||||
|
'ffmetadata',
|
||||||
|
outFileName
|
||||||
|
);
|
||||||
|
metadata = ffmpeg.FS('readFile', outFileName);
|
||||||
|
ffmpeg.FS('unlink', outFileName);
|
||||||
|
ffmpeg.FS('unlink', inputFileName);
|
||||||
|
return parseFFmpegExtractedMetadata(metadata);
|
||||||
|
} catch (e) {
|
||||||
|
logError(e, 'ffmpeg metadata extraction failed');
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export default new FFmpegService();
|
export default new FFmpegService();
|
||||||
|
|
|
@ -10,9 +10,10 @@ import downloadManager from './downloadManager';
|
||||||
import { updatePublicMagicMetadata } from './fileService';
|
import { updatePublicMagicMetadata } from './fileService';
|
||||||
import { EnteFile } from 'types/file';
|
import { EnteFile } from 'types/file';
|
||||||
|
|
||||||
import { getRawExif, getUNIXTime } from './upload/exifService';
|
import { getRawExif } from './upload/exifService';
|
||||||
import { getFileType } from './upload/readFileService';
|
import { getFileType } from './upload/readFileService';
|
||||||
import { FILE_TYPE } from 'constants/file';
|
import { FILE_TYPE } from 'constants/file';
|
||||||
|
import { getUnixTimeInMicroSeconds } from 'utils/time';
|
||||||
|
|
||||||
export async function updateCreationTimeWithExif(
|
export async function updateCreationTimeWithExif(
|
||||||
filesToBeUpdated: EnteFile[],
|
filesToBeUpdated: EnteFile[],
|
||||||
|
@ -33,7 +34,7 @@ export async function updateCreationTimeWithExif(
|
||||||
}
|
}
|
||||||
let correctCreationTime: number;
|
let correctCreationTime: number;
|
||||||
if (fixOption === FIX_OPTIONS.CUSTOM_TIME) {
|
if (fixOption === FIX_OPTIONS.CUSTOM_TIME) {
|
||||||
correctCreationTime = getUNIXTime(customTime);
|
correctCreationTime = getUnixTimeInMicroSeconds(customTime);
|
||||||
} else {
|
} else {
|
||||||
const fileURL = await downloadManager.getFile(file);
|
const fileURL = await downloadManager.getFile(file);
|
||||||
const fileObject = await getFileFromURL(fileURL);
|
const fileObject = await getFileFromURL(fileURL);
|
||||||
|
@ -41,11 +42,13 @@ export async function updateCreationTimeWithExif(
|
||||||
const fileTypeInfo = await getFileType(reader, fileObject);
|
const fileTypeInfo = await getFileType(reader, fileObject);
|
||||||
const exifData = await getRawExif(fileObject, fileTypeInfo);
|
const exifData = await getRawExif(fileObject, fileTypeInfo);
|
||||||
if (fixOption === FIX_OPTIONS.DATE_TIME_ORIGINAL) {
|
if (fixOption === FIX_OPTIONS.DATE_TIME_ORIGINAL) {
|
||||||
correctCreationTime = getUNIXTime(
|
correctCreationTime = getUnixTimeInMicroSeconds(
|
||||||
exifData?.DateTimeOriginal
|
exifData?.DateTimeOriginal
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
correctCreationTime = getUNIXTime(exifData?.CreateDate);
|
correctCreationTime = getUnixTimeInMicroSeconds(
|
||||||
|
exifData?.CreateDate
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (
|
if (
|
||||||
|
|
|
@ -1,9 +1,11 @@
|
||||||
import { NULL_LOCATION } from 'constants/upload';
|
import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload';
|
||||||
import { Location } from 'types/upload';
|
import { Location } from 'types/upload';
|
||||||
import exifr from 'exifr';
|
import exifr from 'exifr';
|
||||||
import piexif from 'piexifjs';
|
import piexif from 'piexifjs';
|
||||||
import { FileTypeInfo } from 'types/upload';
|
import { FileTypeInfo } from 'types/upload';
|
||||||
import { logError } from 'utils/sentry';
|
import { logError } from 'utils/sentry';
|
||||||
|
import { ParsedExtractedMetadata } from 'types/upload';
|
||||||
|
import { getUnixTimeInMicroSeconds } from 'utils/time';
|
||||||
|
|
||||||
const EXIF_TAGS_NEEDED = [
|
const EXIF_TAGS_NEEDED = [
|
||||||
'DateTimeOriginal',
|
'DateTimeOriginal',
|
||||||
|
@ -23,37 +25,29 @@ interface Exif {
|
||||||
GPSLatitudeRef?: number;
|
GPSLatitudeRef?: number;
|
||||||
GPSLongitudeRef?: number;
|
GPSLongitudeRef?: number;
|
||||||
}
|
}
|
||||||
interface ParsedEXIFData {
|
|
||||||
location: Location;
|
|
||||||
creationTime: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function getExifData(
|
export async function getExifData(
|
||||||
receivedFile: File,
|
receivedFile: File,
|
||||||
fileTypeInfo: FileTypeInfo
|
fileTypeInfo: FileTypeInfo
|
||||||
): Promise<ParsedEXIFData> {
|
): Promise<ParsedExtractedMetadata> {
|
||||||
const nullExifData: ParsedEXIFData = {
|
let parsedEXIFData = NULL_EXTRACTED_METADATA;
|
||||||
location: NULL_LOCATION,
|
|
||||||
creationTime: null,
|
|
||||||
};
|
|
||||||
try {
|
try {
|
||||||
const exifData = await getRawExif(receivedFile, fileTypeInfo);
|
const exifData = await getRawExif(receivedFile, fileTypeInfo);
|
||||||
if (!exifData) {
|
if (!exifData) {
|
||||||
return nullExifData;
|
return parsedEXIFData;
|
||||||
}
|
}
|
||||||
const parsedEXIFData = {
|
parsedEXIFData = {
|
||||||
location: getEXIFLocation(exifData),
|
location: getEXIFLocation(exifData),
|
||||||
creationTime: getUNIXTime(
|
creationTime: getUnixTimeInMicroSeconds(
|
||||||
exifData.DateTimeOriginal ??
|
exifData.DateTimeOriginal ??
|
||||||
exifData.CreateDate ??
|
exifData.CreateDate ??
|
||||||
exifData.ModifyDate
|
exifData.ModifyDate
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
return parsedEXIFData;
|
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
logError(e, 'getExifData failed');
|
logError(e, 'getExifData failed');
|
||||||
return nullExifData;
|
|
||||||
}
|
}
|
||||||
|
return parsedEXIFData;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function updateFileCreationDateInEXIF(
|
export async function updateFileCreationDateInEXIF(
|
||||||
|
@ -131,22 +125,6 @@ export async function getRawExif(
|
||||||
return exifData;
|
return exifData;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getUNIXTime(dateTime: Date) {
|
|
||||||
try {
|
|
||||||
if (!dateTime) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
const unixTime = dateTime.getTime() * 1000;
|
|
||||||
if (unixTime <= 0) {
|
|
||||||
return null;
|
|
||||||
} else {
|
|
||||||
return unixTime;
|
|
||||||
}
|
|
||||||
} catch (e) {
|
|
||||||
logError(e, 'getUNIXTime failed', { dateTime });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function getEXIFLocation(exifData): Location {
|
function getEXIFLocation(exifData): Location {
|
||||||
if (!exifData.latitude || !exifData.longitude) {
|
if (!exifData.latitude || !exifData.longitude) {
|
||||||
return NULL_LOCATION;
|
return NULL_LOCATION;
|
||||||
|
|
|
@ -6,9 +6,11 @@ import {
|
||||||
ParsedMetadataJSON,
|
ParsedMetadataJSON,
|
||||||
Location,
|
Location,
|
||||||
FileTypeInfo,
|
FileTypeInfo,
|
||||||
|
ParsedExtractedMetadata,
|
||||||
} from 'types/upload';
|
} from 'types/upload';
|
||||||
import { NULL_LOCATION } from 'constants/upload';
|
import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload';
|
||||||
import { splitFilenameAndExtension } from 'utils/file';
|
import { splitFilenameAndExtension } from 'utils/file';
|
||||||
|
import { getVideoMetadata } from './videoMetadataService';
|
||||||
|
|
||||||
interface ParsedMetadataJSONWithTitle {
|
interface ParsedMetadataJSONWithTitle {
|
||||||
title: string;
|
title: string;
|
||||||
|
@ -25,23 +27,25 @@ export async function extractMetadata(
|
||||||
receivedFile: File,
|
receivedFile: File,
|
||||||
fileTypeInfo: FileTypeInfo
|
fileTypeInfo: FileTypeInfo
|
||||||
) {
|
) {
|
||||||
let exifData = null;
|
let extractedMetadata: ParsedExtractedMetadata = NULL_EXTRACTED_METADATA;
|
||||||
if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) {
|
if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) {
|
||||||
exifData = await getExifData(receivedFile, fileTypeInfo);
|
extractedMetadata = await getExifData(receivedFile, fileTypeInfo);
|
||||||
|
} else if (fileTypeInfo.fileType === FILE_TYPE.VIDEO) {
|
||||||
|
extractedMetadata = await getVideoMetadata(receivedFile);
|
||||||
}
|
}
|
||||||
|
|
||||||
const extractedMetadata: Metadata = {
|
const metadata: Metadata = {
|
||||||
title: `${splitFilenameAndExtension(receivedFile.name)[0]}.${
|
title: `${splitFilenameAndExtension(receivedFile.name)[0]}.${
|
||||||
fileTypeInfo.exactType
|
fileTypeInfo.exactType
|
||||||
}`,
|
}`,
|
||||||
creationTime:
|
creationTime:
|
||||||
exifData?.creationTime ?? receivedFile.lastModified * 1000,
|
extractedMetadata.creationTime ?? receivedFile.lastModified * 1000,
|
||||||
modificationTime: receivedFile.lastModified * 1000,
|
modificationTime: receivedFile.lastModified * 1000,
|
||||||
latitude: exifData?.location?.latitude,
|
latitude: extractedMetadata.location.latitude,
|
||||||
longitude: exifData?.location?.longitude,
|
longitude: extractedMetadata.location.longitude,
|
||||||
fileType: fileTypeInfo.fileType,
|
fileType: fileTypeInfo.fileType,
|
||||||
};
|
};
|
||||||
return extractedMetadata;
|
return metadata;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const getMetadataJSONMapKey = (
|
export const getMetadataJSONMapKey = (
|
||||||
|
|
76
src/services/upload/videoMetadataService.ts
Normal file
76
src/services/upload/videoMetadataService.ts
Normal file
|
@ -0,0 +1,76 @@
|
||||||
|
import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload';
|
||||||
|
import ffmpegService from 'services/ffmpegService';
|
||||||
|
import { getUnixTimeInMicroSeconds } from 'utils/time';
|
||||||
|
import { ParsedExtractedMetadata } from 'types/upload';
|
||||||
|
import { logError } from 'utils/sentry';
|
||||||
|
|
||||||
|
enum VideoMetadata {
|
||||||
|
CREATION_TIME = 'creation_time',
|
||||||
|
APPLE_CONTENT_IDENTIFIER = 'com.apple.quicktime.content.identifier',
|
||||||
|
APPLE_LIVE_PHOTO_IDENTIFIER = 'com.apple.quicktime.live-photo.auto',
|
||||||
|
APPLE_CREATION_DATE = 'com.apple.quicktime.creationdate',
|
||||||
|
APPLE_LOCATION_ISO = 'com.apple.quicktime.location.ISO6709',
|
||||||
|
LOCATION = 'location',
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getVideoMetadata(file: File) {
|
||||||
|
let videoMetadata = NULL_EXTRACTED_METADATA;
|
||||||
|
try {
|
||||||
|
videoMetadata = await ffmpegService.extractMetadata(file);
|
||||||
|
} catch (e) {
|
||||||
|
logError(e, 'failed to get video metadata');
|
||||||
|
}
|
||||||
|
|
||||||
|
return videoMetadata;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseFFmpegExtractedMetadata(encodedMetadata: Uint8Array) {
|
||||||
|
const metadataString = new TextDecoder().decode(encodedMetadata);
|
||||||
|
const metadataPropertyArray = metadataString.split('\n');
|
||||||
|
const metadataKeyValueArray = metadataPropertyArray.map((property) =>
|
||||||
|
property.split('=')
|
||||||
|
);
|
||||||
|
const validKeyValuePairs = metadataKeyValueArray.filter(
|
||||||
|
(keyValueArray) => keyValueArray.length === 2
|
||||||
|
) as Array<[string, string]>;
|
||||||
|
|
||||||
|
const metadataMap = Object.fromEntries(validKeyValuePairs);
|
||||||
|
|
||||||
|
const location = parseAppleISOLocation(
|
||||||
|
metadataMap[VideoMetadata.APPLE_LOCATION_ISO] ??
|
||||||
|
metadataMap[VideoMetadata.LOCATION]
|
||||||
|
);
|
||||||
|
|
||||||
|
const creationTime = parseCreationTime(
|
||||||
|
metadataMap[VideoMetadata.APPLE_CREATION_DATE] ??
|
||||||
|
metadataMap[VideoMetadata.CREATION_TIME]
|
||||||
|
);
|
||||||
|
const parsedMetadata: ParsedExtractedMetadata = {
|
||||||
|
creationTime,
|
||||||
|
location: {
|
||||||
|
latitude: location.latitude,
|
||||||
|
longitude: location.longitude,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
return parsedMetadata;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseAppleISOLocation(isoLocation: string) {
|
||||||
|
let location = NULL_LOCATION;
|
||||||
|
if (isoLocation) {
|
||||||
|
const [latitude, longitude] = isoLocation
|
||||||
|
.match(/(\+|-)\d+\.*\d+/g)
|
||||||
|
.map((x) => parseFloat(x));
|
||||||
|
|
||||||
|
location = { latitude, longitude };
|
||||||
|
}
|
||||||
|
return location;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseCreationTime(creationTime: string) {
|
||||||
|
let dateTime = null;
|
||||||
|
if (creationTime) {
|
||||||
|
dateTime = getUnixTimeInMicroSeconds(new Date(creationTime));
|
||||||
|
}
|
||||||
|
return dateTime;
|
||||||
|
}
|
|
@ -130,3 +130,8 @@ export interface UploadFile extends BackupedFile {
|
||||||
encryptedKey: string;
|
encryptedKey: string;
|
||||||
keyDecryptionNonce: string;
|
keyDecryptionNonce: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ParsedExtractedMetadata {
|
||||||
|
location: Location;
|
||||||
|
creationTime: number;
|
||||||
|
}
|
||||||
|
|
11
src/utils/time/index.ts
Normal file
11
src/utils/time/index.ts
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
export function getUnixTimeInMicroSeconds(dateTime: Date) {
|
||||||
|
if (!dateTime || isNaN(dateTime.getTime())) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const unixTime = dateTime.getTime() * 1000;
|
||||||
|
if (unixTime <= 0) {
|
||||||
|
return null;
|
||||||
|
} else {
|
||||||
|
return unixTime;
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in a new issue