From 48bace50df5bd9ee0eedf50d1115fb33827457db Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 10:04:29 +0530 Subject: [PATCH] Extract --- .../src/services/upload/metadataService.ts | 159 +----------------- .../photos/src/services/upload/takeout.ts | 155 +++++++++++++++++ .../src/services/upload/uploadManager.ts | 5 +- .../src/services/upload/uploadService.ts | 4 +- web/apps/photos/src/types/upload/index.ts | 9 - web/apps/photos/tests/upload.test.ts | 2 +- 6 files changed, 169 insertions(+), 165 deletions(-) create mode 100644 web/apps/photos/src/services/upload/takeout.ts diff --git a/web/apps/photos/src/services/upload/metadataService.ts b/web/apps/photos/src/services/upload/metadataService.ts index f590e50a3..cb17ba4c1 100644 --- a/web/apps/photos/src/services/upload/metadataService.ts +++ b/web/apps/photos/src/services/upload/metadataService.ts @@ -1,4 +1,3 @@ -import { ensureElectron } from "@/next/electron"; import { getFileNameSize } from "@/next/file"; import log from "@/next/log"; import { ElectronFile } from "@/next/types/file"; @@ -19,11 +18,8 @@ import { FilePublicMagicMetadataProps } from "types/file"; import { FileTypeInfo, LivePhotoAssets, - Location, Metadata, ParsedExtractedMetadata, - ParsedMetadataJSON, - ParsedMetadataJSONMap, type DataStream, type FileWithCollection, type FileWithCollection2, @@ -32,15 +28,15 @@ import { } from "types/upload"; import { getFileTypeFromExtensionForLivePhotoClustering } from "utils/file/livePhoto"; import { getEXIFLocation, getEXIFTime, getParsedExifData } from "./exifService"; +import { + MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT, + getClippedMetadataJSONMapKeyForFile, + getMetadataJSONMapKeyForFile, + type ParsedMetadataJSON, +} from "./takeout"; import uploadCancelService from "./uploadCancelService"; import { getFileName } from "./uploadService"; -const NULL_PARSED_METADATA_JSON: ParsedMetadataJSON = { - creationTime: null, - modificationTime: null, - ...NULL_LOCATION, -}; - const EXIF_TAGS_NEEDED = [ "DateTimeOriginal", "CreateDate", @@ -59,8 +55,6 @@ const EXIF_TAGS_NEEDED = [ "MetadataDate", ]; -export const MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT = 46; - export const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = { location: NULL_LOCATION, creationTime: null, @@ -138,115 +132,6 @@ export async function getImageMetadata( return imageMetadata; } -export const getMetadataJSONMapKeyForJSON = ( - collectionID: number, - jsonFileName: string, -) => { - let title = jsonFileName.slice(0, -1 * ".json".length); - const endsWithNumberedSuffixWithBrackets = title.match(/\(\d+\)$/); - if (endsWithNumberedSuffixWithBrackets) { - title = title.slice( - 0, - -1 * endsWithNumberedSuffixWithBrackets[0].length, - ); - const [name, extension] = splitFilenameAndExtension(title); - return `${collectionID}-${name}${endsWithNumberedSuffixWithBrackets[0]}.${extension}`; - } - return `${collectionID}-${title}`; -}; - -// if the file name is greater than MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT(46) , then google photos clips the file name -// so we need to use the clipped file name to get the metadataJSON file -export const getClippedMetadataJSONMapKeyForFile = ( - collectionID: number, - fileName: string, -) => { - return `${collectionID}-${fileName.slice( - 0, - MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT, - )}`; -}; - -export const getMetadataJSONMapKeyForFile = ( - collectionID: number, - fileName: string, -) => { - return `${collectionID}-${getFileOriginalName(fileName)}`; -}; - -export async function parseMetadataJSON( - receivedFile: File | ElectronFile | string, -) { - try { - let text: string; - if (typeof receivedFile == "string") { - text = await ensureElectron().fs.readTextFile(receivedFile); - } else { - if (!(receivedFile instanceof File)) { - receivedFile = new File( - [await receivedFile.blob()], - receivedFile.name, - ); - } - text = await receivedFile.text(); - } - - return parseMetadataJSONText(text); - } catch (e) { - log.error("parseMetadataJSON failed", e); - // ignore - } -} - -export async function parseMetadataJSONText(text: string) { - const metadataJSON: object = JSON.parse(text); - - const parsedMetadataJSON: ParsedMetadataJSON = NULL_PARSED_METADATA_JSON; - if (!metadataJSON) { - return; - } - - if ( - metadataJSON["photoTakenTime"] && - metadataJSON["photoTakenTime"]["timestamp"] - ) { - parsedMetadataJSON.creationTime = - metadataJSON["photoTakenTime"]["timestamp"] * 1000000; - } else if ( - metadataJSON["creationTime"] && - metadataJSON["creationTime"]["timestamp"] - ) { - parsedMetadataJSON.creationTime = - metadataJSON["creationTime"]["timestamp"] * 1000000; - } - if ( - metadataJSON["modificationTime"] && - metadataJSON["modificationTime"]["timestamp"] - ) { - parsedMetadataJSON.modificationTime = - metadataJSON["modificationTime"]["timestamp"] * 1000000; - } - let locationData: Location = NULL_LOCATION; - if ( - metadataJSON["geoData"] && - (metadataJSON["geoData"]["latitude"] !== 0.0 || - metadataJSON["geoData"]["longitude"] !== 0.0) - ) { - locationData = metadataJSON["geoData"]; - } else if ( - metadataJSON["geoDataExif"] && - (metadataJSON["geoDataExif"]["latitude"] !== 0.0 || - metadataJSON["geoDataExif"]["longitude"] !== 0.0) - ) { - locationData = metadataJSON["geoDataExif"]; - } - if (locationData !== null) { - parsedMetadataJSON.latitude = locationData.latitude; - parsedMetadataJSON.longitude = locationData.longitude; - } - return parsedMetadataJSON; -} - // tries to extract date from file name if available else returns null export function extractDateFromFileName(filename: string): number { try { @@ -283,32 +168,6 @@ function convertSignalNameToFusedDateString(filename: string) { return `${dateStringParts[1]}${dateStringParts[2]}${dateStringParts[3]}-${dateStringParts[4]}`; } -const EDITED_FILE_SUFFIX = "-edited"; - -/* - Get the original file name for edited file to associate it to original file's metadataJSON file - as edited file doesn't have their own metadata file -*/ -function getFileOriginalName(fileName: string) { - let originalName: string = null; - const [nameWithoutExtension, extension] = - splitFilenameAndExtension(fileName); - - const isEditedFile = nameWithoutExtension.endsWith(EDITED_FILE_SUFFIX); - if (isEditedFile) { - originalName = nameWithoutExtension.slice( - 0, - -1 * EDITED_FILE_SUFFIX.length, - ); - } else { - originalName = nameWithoutExtension; - } - if (extension) { - originalName += "." + extension; - } - return originalName; -} - async function getVideoMetadata(file: File | ElectronFile) { let videoMetadata = NULL_EXTRACTED_METADATA; try { @@ -356,7 +215,7 @@ export async function getLivePhotoFileType( export const extractAssetMetadata = async ( worker: Remote, - parsedMetadataJSONMap: ParsedMetadataJSONMap, + parsedMetadataJSONMap: Map, { isLivePhoto, file, livePhotoAssets }: UploadAsset2, collectionID: number, fileTypeInfo: FileTypeInfo, @@ -380,7 +239,7 @@ export const extractAssetMetadata = async ( async function extractFileMetadata( worker: Remote, - parsedMetadataJSONMap: ParsedMetadataJSONMap, + parsedMetadataJSONMap: Map, collectionID: number, fileTypeInfo: FileTypeInfo, rawFile: File | ElectronFile | string, @@ -412,7 +271,7 @@ async function extractFileMetadata( async function extractLivePhotoMetadata( worker: Remote, - parsedMetadataJSONMap: ParsedMetadataJSONMap, + parsedMetadataJSONMap: Map, collectionID: number, fileTypeInfo: FileTypeInfo, livePhotoAssets: LivePhotoAssets2, diff --git a/web/apps/photos/src/services/upload/takeout.ts b/web/apps/photos/src/services/upload/takeout.ts new file mode 100644 index 000000000..92849cac2 --- /dev/null +++ b/web/apps/photos/src/services/upload/takeout.ts @@ -0,0 +1,155 @@ +/** @file Dealing with the JSON metadata in Google Takeouts */ + +import { ensureElectron } from "@/next/electron"; +import { nameAndExtension } from "@/next/file"; +import log from "@/next/log"; +import type { ElectronFile } from "@/next/types/file"; +import { NULL_LOCATION } from "constants/upload"; +import { type Location } from "types/upload"; + +export interface ParsedMetadataJSON { + creationTime: number; + modificationTime: number; + latitude: number; + longitude: number; +} + +export const MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT = 46; + +export const getMetadataJSONMapKeyForJSON = ( + collectionID: number, + jsonFileName: string, +) => { + let title = jsonFileName.slice(0, -1 * ".json".length); + const endsWithNumberedSuffixWithBrackets = title.match(/\(\d+\)$/); + if (endsWithNumberedSuffixWithBrackets) { + title = title.slice( + 0, + -1 * endsWithNumberedSuffixWithBrackets[0].length, + ); + const [name, extension] = nameAndExtension(title); + return `${collectionID}-${name}${endsWithNumberedSuffixWithBrackets[0]}.${extension}`; + } + return `${collectionID}-${title}`; +}; + +// if the file name is greater than MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT(46) , then google photos clips the file name +// so we need to use the clipped file name to get the metadataJSON file +export const getClippedMetadataJSONMapKeyForFile = ( + collectionID: number, + fileName: string, +) => { + return `${collectionID}-${fileName.slice( + 0, + MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT, + )}`; +}; + +export const getMetadataJSONMapKeyForFile = ( + collectionID: number, + fileName: string, +) => { + return `${collectionID}-${getFileOriginalName(fileName)}`; +}; + +const EDITED_FILE_SUFFIX = "-edited"; + +/* + Get the original file name for edited file to associate it to original file's metadataJSON file + as edited file doesn't have their own metadata file +*/ +function getFileOriginalName(fileName: string) { + let originalName: string = null; + const [name, extension] = nameAndExtension(fileName); + + const isEditedFile = name.endsWith(EDITED_FILE_SUFFIX); + if (isEditedFile) { + originalName = name.slice(0, -1 * EDITED_FILE_SUFFIX.length); + } else { + originalName = name; + } + if (extension) { + originalName += "." + extension; + } + return originalName; +} + +export async function parseMetadataJSON( + receivedFile: File | ElectronFile | string, +) { + try { + let text: string; + if (typeof receivedFile == "string") { + text = await ensureElectron().fs.readTextFile(receivedFile); + } else { + if (!(receivedFile instanceof File)) { + receivedFile = new File( + [await receivedFile.blob()], + receivedFile.name, + ); + } + text = await receivedFile.text(); + } + + return parseMetadataJSONText(text); + } catch (e) { + log.error("parseMetadataJSON failed", e); + // ignore + } +} + +const NULL_PARSED_METADATA_JSON: ParsedMetadataJSON = { + creationTime: null, + modificationTime: null, + latitude: null, longitude: null + ...NULL_LOCATION, +}; + +export async function parseMetadataJSONText(text: string) { + const metadataJSON: object = JSON.parse(text); + + const parsedMetadataJSON: ParsedMetadataJSON = NULL_PARSED_METADATA_JSON; + if (!metadataJSON) { + return; + } + + if ( + metadataJSON["photoTakenTime"] && + metadataJSON["photoTakenTime"]["timestamp"] + ) { + parsedMetadataJSON.creationTime = + metadataJSON["photoTakenTime"]["timestamp"] * 1000000; + } else if ( + metadataJSON["creationTime"] && + metadataJSON["creationTime"]["timestamp"] + ) { + parsedMetadataJSON.creationTime = + metadataJSON["creationTime"]["timestamp"] * 1000000; + } + if ( + metadataJSON["modificationTime"] && + metadataJSON["modificationTime"]["timestamp"] + ) { + parsedMetadataJSON.modificationTime = + metadataJSON["modificationTime"]["timestamp"] * 1000000; + } + let locationData: Location = NULL_LOCATION; + if ( + metadataJSON["geoData"] && + (metadataJSON["geoData"]["latitude"] !== 0.0 || + metadataJSON["geoData"]["longitude"] !== 0.0) + ) { + locationData = metadataJSON["geoData"]; + } else if ( + metadataJSON["geoDataExif"] && + (metadataJSON["geoDataExif"]["latitude"] !== 0.0 || + metadataJSON["geoDataExif"]["longitude"] !== 0.0) + ) { + locationData = metadataJSON["geoDataExif"]; + } + if (locationData !== null) { + parsedMetadataJSON.latitude = locationData.latitude; + parsedMetadataJSON.longitude = locationData.longitude; + } + return parsedMetadataJSON; +} diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index f2a386192..bb6c5aba8 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -26,8 +26,6 @@ import { EncryptedEnteFile, EnteFile } from "types/file"; import { SetFiles } from "types/gallery"; import { FileWithCollection, - ParsedMetadataJSON, - ParsedMetadataJSONMap, PublicUploadProps, type FileWithCollection2, } from "types/upload"; @@ -50,6 +48,7 @@ import { getMetadataJSONMapKeyForJSON, parseMetadataJSON, } from "./metadataService"; +import type { ParsedMetadataJSON } from "./takeout"; import uploadCancelService from "./uploadCancelService"; import UploadService, { assetName, @@ -264,7 +263,7 @@ class UploadManager { private cryptoWorkers = new Array< ComlinkWorker >(MAX_CONCURRENT_UPLOADS); - private parsedMetadataJSONMap: ParsedMetadataJSONMap; + private parsedMetadataJSONMap: Map; private filesToBeUploaded: FileWithCollection2[]; private remainingFiles: FileWithCollection2[] = []; private failedFiles: FileWithCollection2[]; diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 442f6e6cd..f1583dabf 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -29,7 +29,6 @@ import { FileInMemory, FileTypeInfo, FileWithMetadata, - ParsedMetadataJSONMap, ProcessedFile, PublicUploadProps, UploadAsset, @@ -64,6 +63,7 @@ import { } from "./thumbnail"; import uploadCancelService from "./uploadCancelService"; import UploadHttpClient from "./uploadHttpClient"; +import type { ParsedMetadataJSON } from "./takeout"; /** Upload files to cloud storage */ class UploadService { @@ -169,7 +169,7 @@ export const uploader = async ( worker: Remote, existingFiles: EnteFile[], fileWithCollection: FileWithCollection2, - parsedMetadataJSONMap: ParsedMetadataJSONMap, + parsedMetadataJSONMap: Map, uploaderName: string, isCFUploadProxyDisabled: boolean, makeProgessTracker: MakeProgressTracker, diff --git a/web/apps/photos/src/types/upload/index.ts b/web/apps/photos/src/types/upload/index.ts index 98e129bf4..95913531f 100644 --- a/web/apps/photos/src/types/upload/index.ts +++ b/web/apps/photos/src/types/upload/index.ts @@ -38,13 +38,6 @@ export interface Location { longitude: number; } -export interface ParsedMetadataJSON { - creationTime: number; - modificationTime: number; - latitude: number; - longitude: number; -} - export interface MultipartUploadURLs { objectKey: string; partURLs: string[]; @@ -93,8 +86,6 @@ export interface FileWithCollection2 extends UploadAsset2 { collectionID?: number; } -export type ParsedMetadataJSONMap = Map; - export interface UploadURL { url: string; objectKey: string; diff --git a/web/apps/photos/tests/upload.test.ts b/web/apps/photos/tests/upload.test.ts index 6e58cf0c2..5a05bb991 100644 --- a/web/apps/photos/tests/upload.test.ts +++ b/web/apps/photos/tests/upload.test.ts @@ -7,7 +7,7 @@ import { getClippedMetadataJSONMapKeyForFile, getMetadataJSONMapKeyForFile, getMetadataJSONMapKeyForJSON, -} from "services/upload/metadataService"; +} from "services/upload/takeout"; import { getUserDetailsV2 } from "services/userService"; import { groupFilesBasedOnCollectionID } from "utils/file";