import { FILE_TYPE } from "@/media/file-type"; import { getLocalCollections } from "services/collectionService"; import { getLocalFiles } from "services/fileService"; import { tryToParseDateTime } from "services/upload/date"; import { MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT, getClippedMetadataJSONMapKeyForFile, getMetadataJSONMapKeyForFile, getMetadataJSONMapKeyForJSON, } from "services/upload/takeout"; import { getUserDetailsV2 } from "services/userService"; import { groupFilesBasedOnCollectionID } from "utils/file"; const DATE_TIME_PARSING_TEST_FILE_NAMES = [ { fileName: "Screenshot_20220807-195908_Firefox", expectedDateTime: "2022-08-07 19:59:08", }, { fileName: "Screenshot_20220507-195908", expectedDateTime: "2022-05-07 19:59:08", }, { fileName: "2022-02-18 16.00.12-DCMX.png", expectedDateTime: "2022-02-18 16:00:12", }, { fileName: "20221107_231730", expectedDateTime: "2022-11-07 23:17:30", }, { fileName: "2020-11-01 02.31.02", expectedDateTime: "2020-11-01 02:31:02", }, { fileName: "IMG_20210921_144423", expectedDateTime: "2021-09-21 14:44:23", }, { // we don't parse the time from this format, will improve later fileName: "2019-10-31 155703", expectedDateTime: "2019-10-31 00:00:00", correctExpectedDateTime: "2019-10-31 15:57:03", }, { fileName: "IMG_20210921_144423_783", expectedDateTime: "2021-09-21 14:44:23", }, { fileName: "Screenshot_2022-06-21-16-51-29-164_newFormat.heic", expectedDateTime: "2022-06-21 16:51:29", }, { fileName: "Screenshot 20221106 211633.com.google.android.apps.nbu.paisa.user.jpg", expectedDateTime: "2022-11-06 21:16:33", }, { fileName: "signal-2022-12-17-15-16-04-718.jpg", expectedDateTime: "2022-12-17 15:16:04", }, ]; const DATE_TIME_PARSING_TEST_FILE_NAMES_MUST_FAIL = [ "Snapchat-431959199.mp4.", "Snapchat-400000000.mp4", "Snapchat-900000000.mp4", "Snapchat-100-10-20-19-15-12", ]; const FILE_NAME_TO_JSON_NAME = [ { filename: "IMG20210211125718-edited.jpg", jsonFilename: "IMG20210211125718.jpg.json", }, { filename: "IMG20210211174722.jpg", jsonFilename: "IMG20210211174722.jpg.json", }, { filename: "21345678901234567890123456789012345678901234567.png", jsonFilename: "2134567890123456789012345678901234567890123456.json", }, { filename: "IMG20210211174722(1).jpg", jsonFilename: "IMG20210211174722.jpg(1).json", }, { filename: "IMG2021021(4455)74722(1).jpg", jsonFilename: "IMG2021021(4455)74722.jpg(1).json", }, { filename: "IMG2021021.json74722(1).jpg", jsonFilename: "IMG2021021.json74722.jpg(1).json", }, { filename: "IMG2021021(1)74722(1).jpg", jsonFilename: "IMG2021021(1)74722.jpg(1).json", }, ]; export async function testUpload() { const jsonString = process.env.NEXT_PUBLIC_ENTE_TEST_EXPECTED_JSON; if (!jsonString) { throw Error( "Please specify the NEXT_PUBLIC_ENTE_TEST_EXPECTED_JSON to run the upload tests", ); } const expectedState = JSON.parse(jsonString); if (!expectedState) { throw Error("upload test failed expectedState missing"); } try { await totalCollectionCountCheck(expectedState); await collectionWiseFileCount(expectedState); await thumbnailGenerationFailedFilesCheck(expectedState); await livePhotoClubbingCheck(expectedState); await exifDataParsingCheck(expectedState); await fileDimensionExtractionCheck(expectedState); await googleMetadataReadingCheck(expectedState); await totalFileCountCheck(expectedState); parseDateTimeFromFileNameTest(); mappingFileAndJSONFileCheck(); } catch (e) { console.log(e); } } async function totalFileCountCheck(expectedState) { const userDetails = await getUserDetailsV2(); if (expectedState["total_file_count"] === userDetails.fileCount) { console.log("file count check passed ✅"); } else { throw Error( `total file count check failed ❌, expected: ${expectedState["total_file_count"]}, got: ${userDetails.fileCount}`, ); } } async function totalCollectionCountCheck(expectedState) { const collections = await getLocalCollections(); const files = await getLocalFiles(); const nonEmptyCollectionIds = new Set( files.map((file) => file.collectionID), ); const nonEmptyCollections = collections.filter((collection) => nonEmptyCollectionIds.has(collection.id), ); if (expectedState["collection_count"] === nonEmptyCollections.length) { console.log("collection count check passed ✅"); } else { throw Error( `total Collection count check failed ❌ expected : ${expectedState["collection_count"]}, got: ${nonEmptyCollections.length}`, ); } } async function collectionWiseFileCount(expectedState) { const files = await getLocalFiles(); const collections = await getLocalCollections(); const collectionToFilesMap = groupFilesBasedOnCollectionID(files); const collectionIDToNameMap = new Map( collections.map((collection) => [collection.id, collection.name]), ); const collectionNameToFileCount = new Map( [...collectionToFilesMap.entries()].map(([collectionID, files]) => [ collectionIDToNameMap.get(collectionID), files.length, ]), ); Object.entries(expectedState["collection_files_count"]).forEach( ([collectionName, fileCount]) => { if (fileCount !== collectionNameToFileCount.get(collectionName)) { throw Error( `collectionWiseFileCount check failed ❌ for collection ${collectionName} expected File count : ${fileCount} , got: ${collectionNameToFileCount.get( collectionName, )}`, ); } }, ); console.log("collection wise file count check passed ✅"); } async function thumbnailGenerationFailedFilesCheck(expectedState) { const files = await getLocalFiles(); const filesWithStaticThumbnail = files.filter( (file) => file.metadata.hasStaticThumbnail, ); const fileIDSet = new Set(); const uniqueFilesWithStaticThumbnail = filesWithStaticThumbnail.filter( (file) => { if (fileIDSet.has(file.id)) { return false; } else { fileIDSet.add(file.id); return true; } }, ); const fileNamesWithStaticThumbnail = uniqueFilesWithStaticThumbnail.map( (file) => file.metadata.title, ); if ( expectedState["thumbnail_generation_failure"]["count"] < uniqueFilesWithStaticThumbnail.length ) { throw Error( `thumbnailGenerationFailedFiles Count Check failed ❌ expected: ${expectedState["thumbnail_generation_failure"]["count"]}, got: ${uniqueFilesWithStaticThumbnail.length}`, ); } fileNamesWithStaticThumbnail.forEach((fileName) => { if ( !expectedState["thumbnail_generation_failure"]["files"].includes( fileName, ) ) { throw Error( `thumbnailGenerationFailedFiles Check failed ❌ expected: ${expectedState["thumbnail_generation_failure"]["files"]}, got: ${fileNamesWithStaticThumbnail}`, ); } }); console.log("thumbnail generation failure check passed ✅"); } async function livePhotoClubbingCheck(expectedState) { const files = await getLocalFiles(); const livePhotos = files.filter( (file) => file.metadata.fileType === FILE_TYPE.LIVE_PHOTO, ); const fileIDSet = new Set(); const uniqueLivePhotos = livePhotos.filter((file) => { if (fileIDSet.has(file.id)) { return false; } else { fileIDSet.add(file.id); return true; } }); const livePhotoFileNames = uniqueLivePhotos.map( (file) => file.metadata.title, ); if (expectedState["live_photo"]["count"] !== livePhotoFileNames.length) { throw Error( `livePhotoClubbing Check failed ❌ expected: ${expectedState["live_photo"]["count"]}, got: ${livePhotoFileNames.length}`, ); } expectedState["live_photo"]["files"].forEach((fileName) => { if (!livePhotoFileNames.includes(fileName)) { throw Error( `livePhotoClubbing Check failed ❌ expected: ${expectedState["live_photo"]["files"]}, got: ${livePhotoFileNames}`, ); } }); console.log("live-photo clubbing check passed ✅"); } async function exifDataParsingCheck(expectedState) { const files = await getLocalFiles(); Object.entries(expectedState["exif"]).map(([fileName, exifValues]) => { const matchingFile = files.find( (file) => file.metadata.title === fileName, ); if (!matchingFile) { throw Error(`exifDataParsingCheck failed , ${fileName} missing`); } if ( exifValues["creation_time"] && exifValues["creation_time"] !== matchingFile.metadata.creationTime ) { throw Error(`exifDataParsingCheck failed ❌ , for ${fileName} expected: ${exifValues["creation_time"]} got: ${matchingFile.metadata.creationTime}`); } if ( exifValues["location"] && (Math.abs( exifValues["location"]["latitude"] - matchingFile.metadata.latitude, ) > 1 || Math.abs( exifValues["location"]["longitude"] - matchingFile.metadata.longitude, ) > 1) ) { throw Error(`exifDataParsingCheck failed ❌ , for ${fileName} expected: ${JSON.stringify(exifValues["location"])} got: [${matchingFile.metadata.latitude},${ matchingFile.metadata.longitude }]`); } }); console.log("exif data parsing check passed ✅"); } async function fileDimensionExtractionCheck(expectedState) { const files = await getLocalFiles(); Object.entries(expectedState["file_dimensions"]).map( ([fileName, dimensions]) => { const matchingFile = files.find( (file) => file.metadata.title === fileName, ); if (!matchingFile) { throw Error( `fileDimensionExtractionCheck failed , ${fileName} missing`, ); } if ( dimensions["width"] && dimensions["width"] !== matchingFile.pubMagicMetadata.data.w && dimensions["height"] && dimensions["height"] !== matchingFile.pubMagicMetadata.data.h ) { throw Error(`fileDimensionExtractionCheck failed ❌ , for ${fileName} expected: ${dimensions["width"]} x ${dimensions["height"]} got: ${matchingFile.pubMagicMetadata.data.w} x ${matchingFile.pubMagicMetadata.data.h}`); } }, ); console.log("file dimension extraction check passed ✅"); } async function googleMetadataReadingCheck(expectedState) { const files = await getLocalFiles(); Object.entries(expectedState["google_import"]).map( ([fileName, metadata]) => { const matchingFile = files.find( (file) => file.metadata.title === fileName, ); if (!matchingFile) { throw Error( `exifDataParsingCheck failed , ${fileName} missing`, ); } if ( metadata["creation_time"] && metadata["creation_time"] !== matchingFile.metadata.creationTime ) { throw Error(`googleMetadataJSON reading check failed ❌ , for ${fileName} expected: ${metadata["creation_time"]} got: ${matchingFile.metadata.creationTime}`); } if ( metadata["location"] && (Math.abs( metadata["location"]["latitude"] - matchingFile.metadata.latitude, ) > 1 || Math.abs( metadata["location"]["longitude"] - matchingFile.metadata.longitude, ) > 1) ) { throw Error(`googleMetadataJSON reading check failed ❌ , for ${fileName} expected: ${JSON.stringify( metadata["location"], )} got: [${matchingFile.metadata.latitude},${ matchingFile.metadata.longitude }]`); } }, ); console.log("googleMetadataJSON reading check passed ✅"); } function parseDateTimeFromFileNameTest() { DATE_TIME_PARSING_TEST_FILE_NAMES.forEach( ({ fileName, expectedDateTime }) => { const dateTime = tryToParseDateTime(fileName); const formattedDateTime = getFormattedDateTime(dateTime); if (formattedDateTime !== expectedDateTime) { throw Error( `parseDateTimeFromFileNameTest failed ❌ , for ${fileName} expected: ${expectedDateTime} got: ${formattedDateTime}`, ); } }, ); DATE_TIME_PARSING_TEST_FILE_NAMES_MUST_FAIL.forEach((fileName) => { const dateTime = tryToParseDateTime(fileName); if (dateTime) { throw Error( `parseDateTimeFromFileNameTest failed ❌ , for ${fileName} expected: null got: ${dateTime}`, ); } }); console.log("parseDateTimeFromFileNameTest passed ✅"); } function mappingFileAndJSONFileCheck() { FILE_NAME_TO_JSON_NAME.forEach(({ filename, jsonFilename }) => { const jsonFileNameGeneratedKey = getMetadataJSONMapKeyForJSON( 0, jsonFilename, ); let fileNameGeneratedKey = getMetadataJSONMapKeyForFile(0, filename); if ( fileNameGeneratedKey !== jsonFileNameGeneratedKey && filename.length > MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT ) { fileNameGeneratedKey = getClippedMetadataJSONMapKeyForFile( 0, filename, ); } if (fileNameGeneratedKey !== jsonFileNameGeneratedKey) { throw Error( `mappingFileAndJSONFileCheck failed ❌ , for ${filename} expected: ${jsonFileNameGeneratedKey} got: ${fileNameGeneratedKey}`, ); } }); console.log("mappingFileAndJSONFileCheck passed ✅"); } // format: YYYY-MM-DD HH:MM:SS function getFormattedDateTime(date: Date) { const year = date.getFullYear(); const month = padZero(date.getMonth() + 1); const day = padZero(date.getDate()); const hour = padZero(date.getHours()); const minute = padZero(date.getMinutes()); const second = padZero(date.getSeconds()); return `${year}-${month}-${day} ${hour}:${minute}:${second}`; } function padZero(number: number) { return number < 10 ? `0${number}` : number; }