Merge branch 'main' into watch

sentry.client.config.js

@@ -1,6 +1,6 @@
 import * as Sentry from '@sentry/nextjs';
 import { getSentryTunnelURL } from 'utils/common/apiUtil';
-import { getUserAnonymizedID } from 'utils/user';
+import { getSentryUserID } from 'utils/user';
 import {
     getSentryDSN,
     getSentryENV,
@@ -13,7 +13,7 @@ const SENTRY_ENV = getSentryENV();
 const SENTRY_RELEASE = getSentryRelease();
 const IS_ENABLED = getIsSentryEnabled();
 
-Sentry.setUser({ id: getUserAnonymizedID() });
+Sentry.setUser({ id: getSentryUserID() });
 Sentry.init({
     dsn: SENTRY_DSN,
     enabled: IS_ENABLED,

src/components/ExportFinished.tsx

@@ -1,7 +1,7 @@
 import { Button, DialogActions, DialogContent, Stack } from '@mui/material';
 import React from 'react';
 import { ExportStats } from 'types/export';
-import { formatDateTime } from 'utils/file';
+import { formatDateTime } from 'utils/time';
 import constants from 'utils/strings/constants';
 import { FlexWrapper, Label, Value } from './Container';
 import { ComfySpan } from './ExportInProgress';

src/components/ExportModal.tsx

@@ -29,7 +29,7 @@ import DialogTitleWithCloseButton from './DialogBox/TitleWithCloseButton';
 import MoreHoriz from '@mui/icons-material/MoreHoriz';
 import OverflowMenu from './OverflowMenu/menu';
 import { OverflowMenuOption } from './OverflowMenu/option';
-import { convertBytesToHumanReadable } from 'utils/billing';
+import { convertBytesToHumanReadable } from 'utils/file/size';
 import { CustomError } from 'utils/error';
 import { getLocalUserDetails } from 'utils/user';
 

src/components/PhotoList.tsx

@@ -16,7 +16,7 @@ import constants from 'utils/strings/constants';
 import { PublicCollectionGalleryContext } from 'utils/publicCollectionGallery';
 import { ENTE_WEBSITE_LINK } from 'constants/urls';
 import { getVariantColor, ButtonVariant } from './pages/gallery/LinkButton';
-import { convertBytesToHumanReadable } from 'utils/billing';
+import { convertBytesToHumanReadable } from 'utils/file/size';
 import { DeduplicateContext } from 'pages/deduplicate';
 import { FlexWrapper } from './Container';
 import { Typography } from '@mui/material';

src/components/PhotoSwipe/InfoDialog/RenderCreationTime.tsx

@@ -4,9 +4,9 @@ import { EnteFile } from 'types/file';
 import constants from 'utils/strings/constants';
 import {
     changeFileCreationTime,
-    formatDateTime,
     updateExistingFilePubMetadata,
 } from 'utils/file';
+import { formatDateTime } from 'utils/time';
 import EditIcon from '@mui/icons-material/Edit';
 import { Label, Row, Value } from 'components/Container';
 import { logError } from 'utils/sentry';

src/components/PhotoSwipe/InfoDialog/index.tsx

@@ -1,6 +1,6 @@
-import React, { useContext } from 'react';
+import React, { useContext, useEffect, useState } from 'react';
 import constants from 'utils/strings/constants';
-import { formatDateTime } from 'utils/file';
+import { formatDateTime } from 'utils/time';
 import { RenderFileName } from './RenderFileName';
 import { ExifData } from './ExifData';
 import { RenderCreationTime } from './RenderCreationTime';
@@ -8,6 +8,9 @@ import { RenderInfoItem } from './RenderInfoItem';
 import DialogTitleWithCloseButton from 'components/DialogBox/TitleWithCloseButton';
 import { Dialog, DialogContent, Link, styled, Typography } from '@mui/material';
 import { AppContext } from 'pages/_app';
+import { Location, Metadata } from 'types/upload';
+import Photoswipe from 'photoswipe';
+import { getEXIFLocation } from 'services/upload/exifService';
 
 const FileInfoDialog = styled(Dialog)(({ theme }) => ({
     zIndex: 1501,
@@ -19,6 +22,17 @@ const FileInfoDialog = styled(Dialog)(({ theme }) => ({
     },
 }));
 
+interface Iprops {
+    shouldDisableEdits: boolean;
+    showInfo: boolean;
+    handleCloseInfo: () => void;
+    items: any[];
+    photoSwipe: Photoswipe<Photoswipe.Options>;
+    metadata: Metadata;
+    exif: any;
+    scheduleUpdate: () => void;
+}
+
 export function FileInfo({
     shouldDisableEdits,
     showInfo,
@@ -28,8 +42,30 @@ export function FileInfo({
     metadata,
     exif,
     scheduleUpdate,
-}) {
+}: Iprops) {
     const appContext = useContext(AppContext);
+    const [location, setLocation] = useState<Location>(null);
+
+    useEffect(() => {
+        if (!location && metadata) {
+            if (metadata.longitude || metadata.longitude === 0) {
+                setLocation({
+                    latitude: metadata.latitude,
+                    longitude: metadata.longitude,
+                });
+            }
+        }
+    }, [metadata]);
+
+    useEffect(() => {
+        if (!location && exif) {
+            const exifLocation = getEXIFLocation(exif);
+            if (exifLocation.latitude || exifLocation.latitude === 0) {
+                setLocation(exifLocation);
+            }
+        }
+    }, [exif]);
+
     return (
         <FileInfoDialog
             open={showInfo}
@@ -66,8 +102,7 @@ export function FileInfo({
                         constants.UPDATED_ON,
                         formatDateTime(metadata.modificationTime / 1000)
                     )}
-                {metadata?.longitude > 0 &&
+                {location &&
-                    metadata?.longitude > 0 &&
                     RenderInfoItem(
                         constants.LOCATION,
                         <Link

src/components/Sidebar/DebugLogs.tsx

@@ -2,7 +2,7 @@ import { AppContext } from 'pages/_app';
 import React, { useContext } from 'react';
 import { downloadAsFile } from 'utils/file';
 import constants from 'utils/strings/constants';
-import { logUploadInfo, getUploadLogs } from 'utils/upload';
+import { addLogLine, getDebugLogs } from 'utils/logging';
 import SidebarButton from './Button';
 
 export default function DebugLogs() {
@@ -14,18 +14,18 @@ export default function DebugLogs() {
             proceed: {
                 text: constants.DOWNLOAD,
                 variant: 'accent',
-                action: downloadUploadLogs,
+                action: downloadDebugLogs,
             },
             close: {
                 text: constants.CANCEL,
             },
         });
 
-    const downloadUploadLogs = () => {
+    const downloadDebugLogs = () => {
-        logUploadInfo('exporting logs');
+        addLogLine('exporting logs');
-        const logs = getUploadLogs();
+        const logs = getDebugLogs();
         const logString = logs.join('\n');
-        downloadAsFile(`upload_logs_${Date.now()}.txt`, logString);
+        downloadAsFile(`debug_logs_${Date.now()}.txt`, logString);
     };
 
     return (

src/components/pages/gallery/PreviewCard.tsx

@@ -3,17 +3,18 @@ import { EnteFile } from 'types/file';
 import { styled } from '@mui/material';
 import PlayCircleOutlineOutlinedIcon from '@mui/icons-material/PlayCircleOutlineOutlined';
 import DownloadManager from 'services/downloadManager';
-import useLongPress from 'utils/common/useLongPress';
+import useLongPress from 'hooks/useLongPress';
 import { GalleryContext } from 'pages/gallery';
 import { GAP_BTW_TILES, IMAGE_CONTAINER_MAX_WIDTH } from 'constants/gallery';
 import { PublicCollectionGalleryContext } from 'utils/publicCollectionGallery';
 import PublicCollectionDownloadManager from 'services/publicCollectionDownloadManager';
 import LivePhotoIcon from '@mui/icons-material/LightMode';
-import { formatDateRelative, isLivePhoto } from 'utils/file';
+import { isLivePhoto } from 'utils/file';
 import { DeduplicateContext } from 'pages/deduplicate';
 import { logError } from 'utils/sentry';
 import { Overlay } from 'components/Container';
 import { TRASH_SECTION } from 'constants/collection';
+import { formatDateRelative } from 'utils/time';
 
 interface IProps {
     file: EnteFile;

src/pages/_app.tsx

@@ -13,7 +13,7 @@ import { getData, LS_KEYS } from 'utils/storage/localStorage';
 import HTTPService from 'services/HTTPService';
 import FlashMessageBar from 'components/FlashMessageBar';
 import Head from 'next/head';
-import { logUploadInfo } from 'utils/upload';
+import { addLogLine } from 'utils/logging';
 import LoadingBar from 'react-top-loading-bar';
 import DialogBox from 'components/DialogBox';
 import { styled, ThemeProvider } from '@mui/material/styles';
@@ -27,6 +27,7 @@ import {
     getRoadmapRedirectURL,
 } from 'services/userService';
 import { CustomError } from 'utils/error';
+import { getSentryUserID } from 'utils/user';
 
 export const MessageContainer = styled('div')`
     background-color: #111;
@@ -206,10 +207,11 @@ export default function App({ Component, err }) {
     }, [redirectName]);
 
     useEffect(() => {
-        logUploadInfo(`app started`);
+        addLogLine(`app started`);
-        logUploadInfo(
+        addLogLine(
             `latest commit id :${process.env.NEXT_PUBLIC_LATEST_COMMIT_HASH}`
         );
+        addLogLine(`user sentry id ${getSentryUserID()}`);
     }, []);
 
     useEffect(() => setMessageDialogView(true), [dialogMessage]);

src/pages/gallery/index.tsx

@@ -99,6 +99,8 @@ import { NotificationAttributes } from 'types/Notification';
 import { ITEM_TYPE, TimeStampListItem } from 'components/PhotoList';
 import UploadInputs from 'components/UploadSelectorInputs';
 import useFileInput from 'hooks/useFileInput';
+import { User } from 'types/user';
+import { getData, LS_KEYS } from 'utils/storage/localStorage';
 
 export const DeadCenter = styled('div')`
     flex: 1;
@@ -132,6 +134,7 @@ export const GalleryContext = createContext<GalleryContextType>(
 
 export default function Gallery() {
     const router = useRouter();
+    const [user, setUser] = useState(null);
     const [collections, setCollections] = useState<Collection[]>(null);
 
     const [files, setFiles] = useState<EnteFile[]>(null);
@@ -245,10 +248,12 @@ export default function Gallery() {
                 setPlanModalView(true);
             }
             setIsFirstLogin(false);
+            const user = getData(LS_KEYS.USER);
             const files = mergeMetadata(await getLocalFiles());
             const collections = await getLocalCollections();
             const trash = await getLocalTrash();
             files.push(...getTrashedFiles(trash));
+            setUser(user);
             setFiles(sortFiles(files));
             setCollections(collections);
             await syncWithRemote(true);
@@ -261,7 +266,10 @@ export default function Gallery() {
     }, []);
 
     useEffect(() => {
-        setDerivativeState(collections, files);
+        if (!user || !files || !collections) {
+            return;
+        }
+        setDerivativeState(user, collections, files);
     }, [collections, files]);
 
     useEffect(
@@ -378,18 +386,17 @@ export default function Gallery() {
     };
 
     const setDerivativeState = async (
+        user: User,
         collections: Collection[],
         files: EnteFile[]
     ) => {
-        if (!collections || !files) {
-            return;
-        }
         const favItemIds = await getFavItemIds(files);
         setFavItemIds(favItemIds);
         const archivedCollections = getArchivedCollections(collections);
         setArchivedCollections(archivedCollections);
 
         const collectionSummaries = getCollectionSummaries(
+            user,
             collections,
             files,
             archivedCollections

src/services/collectionService.ts

@@ -770,6 +770,7 @@ function compareCollectionsLatestFile(first: EnteFile, second: EnteFile) {
 }
 
 export function getCollectionSummaries(
+    user: User,
     collections: Collection[],
     files: EnteFile[],
     archivedCollections: Set<number>
@@ -781,7 +782,6 @@ export function getCollectionSummaries(
     );
     const collectionFilesCount = getCollectionsFileCount(files);
     const uniqueFileCount = new Set(files.map((file) => file.id)).size;
-    const user: User = getData(LS_KEYS.USER);
 
     for (const collection of collections) {
         if (collectionFilesCount.get(collection.id)) {

src/services/fileService.ts

@@ -17,7 +17,7 @@ import { EnteFile, TrashRequest } from 'types/file';
 import { SetFiles } from 'types/gallery';
 import { MAX_TRASH_BATCH_SIZE } from 'constants/file';
 import { BulkUpdateMagicMetadataRequest } from 'types/magicMetadata';
-import { logUploadInfo } from 'utils/upload';
+import { addLogLine } from 'utils/logging';
 
 const ENDPOINT = getEndpoint();
 const FILES_TABLE = 'files';
@@ -37,9 +37,7 @@ export const setLocalFiles = async (files: EnteFile[]) => {
             logError(e1, 'failed to save files to indexedDB', {
                 storageEstimate,
             });
-            logUploadInfo(
+            addLogLine(`storage estimate ${JSON.stringify(storageEstimate)}`);
-                `storage estimate ${JSON.stringify(storageEstimate)}`
-            );
         } catch (e2) {
             logError(e1, 'failed to save files to indexedDB');
             logError(e2, 'failed to get storage stats');

src/services/heicConverter/heicConverterService.ts

@@ -3,7 +3,7 @@ import { CustomError } from 'utils/error';
 import { createNewConvertWorker } from 'utils/heicConverter';
 import { retryAsyncFunction } from 'utils/network';
 import { logError } from 'utils/sentry';
-import { logUploadInfo } from 'utils/upload';
+import { addLogLine } from 'utils/logging';
 
 const WORKER_POOL_SIZE = 2;
 const MAX_CONVERSION_IN_PARALLEL = 1;
@@ -63,7 +63,7 @@ class HEICConverter {
                     this.workerPool.push({ comlink, worker });
                     return convertedHEIC;
                 } catch (e) {
-                    logUploadInfo('heic conversion failed-' + e.message);
+                    addLogLine('heic conversion failed-' + e.message);
                     logError(e, 'heic conversion failed');
                     worker.terminate();
                     this.workerPool.push(await createNewConvertWorker());

src/services/typeDetectionService.ts

@@ -47,7 +47,7 @@ export async function getFileType(
     } catch (e) {
         const fileFormat = getFileExtension(receivedFile.name);
         const formatMissedByTypeDetection = FORMAT_MISSED_BY_FILE_TYPE_LIB.find(
-            (a) => a.exactType === fileFormat.toLocaleLowerCase()
+            (a) => a.exactType === fileFormat
         );
         if (formatMissedByTypeDetection) {
             return formatMissedByTypeDetection;
@@ -64,22 +64,30 @@ export async function getFileType(
 }
 
 async function extractFileType(file: File) {
-    const fileChunkBlob = file.slice(0, CHUNK_SIZE_FOR_TYPE_DETECTION);
+    const fileBlobChunk = file.slice(0, CHUNK_SIZE_FOR_TYPE_DETECTION);
-    return getFileTypeFromBlob(fileChunkBlob);
+    const fileDataChunk = await getUint8ArrayView(fileBlobChunk);
+    return getFileTypeFromBuffer(fileDataChunk);
 }
 
 async function extractElectronFileType(file: ElectronFile) {
     const stream = await file.stream();
     const reader = stream.getReader();
-    const { value } = await reader.read();
+    const { value: fileDataChunk } = await reader.read();
-    const fileTypeResult = await FileType.fromBuffer(value);
+    return getFileTypeFromBuffer(fileDataChunk);
-    return fileTypeResult;
 }
 
-async function getFileTypeFromBlob(fileBlob: Blob) {
+async function getFileTypeFromBuffer(buffer: Uint8Array) {
     try {
-        const initialFiledata = await getUint8ArrayView(fileBlob);
+        const result = await FileType.fromBuffer(buffer);
-        return await FileType.fromBuffer(initialFiledata);
+        if (!result.mime) {
+            logError(
+                Error('mimetype missing from file type result'),
+                CustomError.TYPE_DETECTION_FAILED,
+                { result }
+            );
+            throw Error(CustomError.TYPE_DETECTION_FAILED);
+        }
+        return result;
     } catch (e) {
         throw Error(CustomError.TYPE_DETECTION_FAILED);
     }

src/services/upload/exifService.ts

@@ -131,7 +131,7 @@ export async function getRawExif(
     return exifData;
 }
 
-function getEXIFLocation(exifData): Location {
+export function getEXIFLocation(exifData): Location {
     if (!exifData.latitude || !exifData.longitude) {
         return NULL_LOCATION;
     }

src/services/upload/fileService.ts

@@ -13,7 +13,7 @@ import {
 } from 'types/upload';
 import { splitFilenameAndExtension } from 'utils/file';
 import { logError } from 'utils/sentry';
-import { getFileNameSize, logUploadInfo } from 'utils/upload';
+import { getFileNameSize, addLogLine } from 'utils/logging';
 import { encryptFiledata } from './encryptionService';
 import { extractMetadata, getMetadataJSONMapKey } from './metadataService';
 import {
@@ -41,7 +41,7 @@ export async function readFile(
         rawFile,
         fileTypeInfo
     );
-    logUploadInfo(`reading file data ${getFileNameSize(rawFile)} `);
+    addLogLine(`reading file data ${getFileNameSize(rawFile)} `);
     let filedata: Uint8Array | DataStream;
     if (!(rawFile instanceof File)) {
         if (rawFile.size > MULTIPART_PART_SIZE) {
@@ -58,7 +58,7 @@ export async function readFile(
         filedata = await getUint8ArrayView(rawFile);
     }
 
-    logUploadInfo(`read file data successfully ${getFileNameSize(rawFile)} `);
+    addLogLine(`read file data successfully ${getFileNameSize(rawFile)} `);
 
     return {
         filedata,

src/services/upload/thumbnailService.ts

@@ -3,12 +3,12 @@ import { CustomError, errorWithContext } from 'utils/error';
 import { logError } from 'utils/sentry';
 import { BLACK_THUMBNAIL_BASE64 } from 'constants/upload';
 import FFmpegService from 'services/ffmpeg/ffmpegService';
-import { convertBytesToHumanReadable } from 'utils/billing';
+import { convertBytesToHumanReadable } from 'utils/file/size';
 import { isFileHEIC } from 'utils/file';
 import { ElectronFile, FileTypeInfo } from 'types/upload';
 import { getUint8ArrayView } from '../readerService';
 import HEICConverter from 'services/heicConverter/heicConverterService';
-import { getFileNameSize, logUploadInfo } from 'utils/upload';
+import { getFileNameSize, addLogLine } from 'utils/logging';
 
 const MAX_THUMBNAIL_DIMENSION = 720;
 const MIN_COMPRESSION_PERCENTAGE_SIZE_DIFF = 10;
@@ -28,7 +28,7 @@ export async function generateThumbnail(
     fileTypeInfo: FileTypeInfo
 ): Promise<{ thumbnail: Uint8Array; hasStaticThumbnail: boolean }> {
     try {
-        logUploadInfo(`generating thumbnail for ${getFileNameSize(file)}`);
+        addLogLine(`generating thumbnail for ${getFileNameSize(file)}`);
         let hasStaticThumbnail = false;
         let canvas = document.createElement('canvas');
         let thumbnail: Uint8Array;
@@ -41,14 +41,14 @@ export async function generateThumbnail(
                 canvas = await generateImageThumbnail(file, isHEIC);
             } else {
                 try {
-                    logUploadInfo(
+                    addLogLine(
                         `ffmpeg generateThumbnail called for ${getFileNameSize(
                             file
                         )}`
                     );
 
                     const thumb = await FFmpegService.generateThumbnail(file);
-                    logUploadInfo(
+                    addLogLine(
                         `ffmpeg thumbnail successfully generated ${getFileNameSize(
                             file
                         )}`
@@ -59,7 +59,7 @@ export async function generateThumbnail(
                         false
                     );
                 } catch (e) {
-                    logUploadInfo(
+                    addLogLine(
                         `ffmpeg thumbnail generated failed  ${getFileNameSize(
                             file
                         )} error: ${e.message}`
@@ -75,14 +75,14 @@ export async function generateThumbnail(
             if (thumbnail.length === 0) {
                 throw Error('EMPTY THUMBNAIL');
             }
-            logUploadInfo(
+            addLogLine(
                 `thumbnail successfully generated ${getFileNameSize(file)}`
             );
         } catch (e) {
             logError(e, 'uploading static thumbnail', {
                 fileFormat: fileTypeInfo.exactType,
             });
-            logUploadInfo(
+            addLogLine(
                 `thumbnail generation failed ${getFileNameSize(file)} error: ${
                     e.message
                 }`
@@ -107,9 +107,9 @@ export async function generateImageThumbnail(file: File, isHEIC: boolean) {
     let timeout = null;
 
     if (isHEIC) {
-        logUploadInfo(`HEICConverter called for ${getFileNameSize(file)}`);
+        addLogLine(`HEICConverter called for ${getFileNameSize(file)}`);
         file = new File([await HEICConverter.convert(file)], file.name);
-        logUploadInfo(`${getFileNameSize(file)} successfully converted`);
+        addLogLine(`${getFileNameSize(file)} successfully converted`);
     }
     let image = new Image();
     imageURL = URL.createObjectURL(file);

src/services/upload/uploadManager.ts

@@ -16,7 +16,6 @@ import { logError } from 'utils/sentry';
 import { getMetadataJSONMapKey, parseMetadataJSON } from './metadataService';
 import {
     areFileWithCollectionsSame,
-    getFileNameSize,
     segregateMetadataAndMediaFiles,
 } from 'utils/upload';
 import uploader from './uploader';
@@ -28,6 +27,7 @@ import { EnteFile } from 'types/file';
 import {
     ElectronFile,
     FileWithCollection,
+    Metadata,
     MetadataAndFileTypeInfo,
     MetadataAndFileTypeInfoMap,
     ParsedMetadataJSON,
@@ -41,7 +41,7 @@ import {
 import { ComlinkWorker } from 'utils/comlink';
 import { FILE_TYPE } from 'constants/file';
 import uiService from './uiService';
-import { logUploadInfo } from 'utils/upload';
+import { addLogLine, getFileNameSize } from 'utils/logging';
 import isElectron from 'is-electron';
 import ImportService from 'services/importService';
 import watchFolderService from 'services/watchFolder/watchFolderService';
@@ -95,15 +95,13 @@ class UploadManager {
     ) {
         try {
             await this.init(collections);
-            logUploadInfo(
+            addLogLine(
                 `received ${fileWithCollectionToBeUploaded.length} files to upload`
             );
             const { metadataJSONFiles, mediaFiles } =
                 segregateMetadataAndMediaFiles(fileWithCollectionToBeUploaded);
-            logUploadInfo(
+            addLogLine(`has ${metadataJSONFiles.length} metadata json files`);
-                `has ${metadataJSONFiles.length} metadata json files`
+            addLogLine(`has ${mediaFiles.length} media files`);
-            );
-            logUploadInfo(`has ${mediaFiles.length} media files`);
             if (metadataJSONFiles.length) {
                 UIService.setUploadStage(
                     UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES
@@ -129,7 +127,7 @@ class UploadManager {
                 );
 
                 UIService.setUploadStage(UPLOAD_STAGES.START);
-                logUploadInfo(`clusterLivePhotoFiles called`);
+                addLogLine(`clusterLivePhotoFiles called`);
 
                 // filter out files whose metadata detection failed or those that have been skipped because the files are too large,
                 // as they will be rejected during upload and are not valid upload files which we need to clustering
@@ -172,7 +170,7 @@ class UploadManager {
                 UIService.setHasLivePhoto(
                     mediaFiles.length !== allFiles.length
                 );
-                logUploadInfo(
+                addLogLine(
                     `got live photos: ${mediaFiles.length !== allFiles.length}`
                 );
 
@@ -186,7 +184,7 @@ class UploadManager {
             UIService.setPercentComplete(FILE_UPLOAD_COMPLETED);
         } catch (e) {
             logError(e, 'uploading failed with error');
-            logUploadInfo(
+            addLogLine(
                 `uploading failed with error -> ${e.message}
                 ${(e as Error).stack}`
             );
@@ -200,7 +198,7 @@ class UploadManager {
 
     private async parseMetadataJSONFiles(metadataFiles: FileWithCollection[]) {
         try {
-            logUploadInfo(`parseMetadataJSONFiles function executed `);
+            addLogLine(`parseMetadataJSONFiles function executed `);
 
             UIService.reset(metadataFiles.length);
 
@@ -210,7 +208,7 @@ class UploadManager {
                         return;
                     }
 
-                    logUploadInfo(
+                    addLogLine(
                         `parsing metadata json file ${getFileNameSize(file)}`
                     );
 
@@ -226,14 +224,14 @@ class UploadManager {
                         );
                         UIService.increaseFileUploaded();
                     }
-                    logUploadInfo(
+                    addLogLine(
                         `successfully parsed metadata json file ${getFileNameSize(
                             file
                         )}`
                     );
                 } catch (e) {
                     logError(e, 'parsing failed for a file');
-                    logUploadInfo(
+                    addLogLine(
                         `failed to parse metadata json file ${getFileNameSize(
                             file
                         )} error: ${e.message}`
@@ -248,7 +246,7 @@ class UploadManager {
 
     private async extractMetadataFromFiles(mediaFiles: FileWithCollection[]) {
         try {
-            logUploadInfo(`extractMetadataFromFiles executed`);
+            addLogLine(`extractMetadataFromFiles executed`);
             UIService.reset(mediaFiles.length);
             for (const { file, localID, collectionID } of mediaFiles) {
                 if (this.isUploadPausing) {
@@ -258,7 +256,7 @@ class UploadManager {
                 let metadata = null;
                 let filePath = null;
                 try {
-                    logUploadInfo(
+                    addLogLine(
                         `metadata extraction started ${getFileNameSize(file)} `
                     );
                     const result = await this.extractFileTypeAndMetadata(
@@ -268,14 +266,14 @@ class UploadManager {
                     fileTypeInfo = result.fileTypeInfo;
                     metadata = result.metadata;
                     filePath = result.filePath;
-                    logUploadInfo(
+                    addLogLine(
                         `metadata extraction successful${getFileNameSize(
                             file
                         )} `
                     );
                 } catch (e) {
-                    logError(e, 'metadata extraction failed for a file');
+                    logError(e, 'extractFileTypeAndMetadata failed');
-                    logUploadInfo(
+                    addLogLine(
                         `metadata extraction failed ${getFileNameSize(
                             file
                         )} error: ${e.message}`
@@ -299,7 +297,7 @@ class UploadManager {
         collectionID: number
     ) {
         if (file.size >= MAX_FILE_SIZE_SUPPORTED) {
-            logUploadInfo(
+            addLogLine(
                 `${getFileNameSize(file)} rejected  because of large size`
             );
 
@@ -307,15 +305,17 @@ class UploadManager {
         }
         const fileTypeInfo = await UploadService.getFileType(file);
         if (fileTypeInfo.fileType === FILE_TYPE.OTHERS) {
-            logUploadInfo(
+            addLogLine(
                 `${getFileNameSize(
                     file
                 )} rejected  because of unknown file format`
             );
             return { fileTypeInfo, metadata: null };
         }
-        logUploadInfo(` extracting ${getFileNameSize(file)} metadata`);
+        addLogLine(` extracting ${getFileNameSize(file)} metadata`);
-        const metadata =
+        let metadata: Metadata;
+        try {
+            metadata =
                 (await UploadService.extractFileMetadata(
                     file,
                     collectionID,
@@ -323,10 +323,15 @@ class UploadManager {
                 )) || null;
             const filePath = (file as any).path as string;
             return { fileTypeInfo, metadata, filePath };
+        } catch (e) {
+            logError(e, 'failed to extract file metadata');
+            return { fileTypeInfo, metadata: null };
+        }
+        return { fileTypeInfo, metadata };
     }
 
     private async uploadMediaFiles(mediaFiles: FileWithCollection[]) {
-        logUploadInfo(`uploadMediaFiles called`);
+        addLogLine(`uploadMediaFiles called`);
         this.filesToBeUploaded.push(...mediaFiles);
 
         if (isElectron()) {
@@ -398,7 +403,7 @@ class UploadManager {
     ) {
         try {
             let decryptedFile: EnteFile;
-            logUploadInfo(`uploadedFile ${JSON.stringify(uploadedFile)}`);
+            addLogLine(`uploadedFile ${JSON.stringify(uploadedFile)}`);
             this.updateElectronRemainingFiles(fileWithCollection);
             switch (fileUploadResult) {
                 case UPLOAD_RESULT.FAILED:
@@ -442,7 +447,7 @@ class UploadManager {
             return fileUploadResult;
         } catch (e) {
             logError(e, 'failed to do post file upload action');
-            logUploadInfo(
+            addLogLine(
                 `failed to do post file upload action -> ${e.message}
                 ${(e as Error).stack}`
             );

src/services/upload/uploader.ts

@@ -12,8 +12,8 @@ import UploadService from './uploadService';
 import { FILE_TYPE } from 'constants/file';
 import { UPLOAD_RESULT, MAX_FILE_SIZE_SUPPORTED } from 'constants/upload';
 import { FileWithCollection, BackupedFile, UploadFile } from 'types/upload';
-import { logUploadInfo } from 'utils/upload';
+import { addLogLine } from 'utils/logging';
-import { convertBytesToHumanReadable } from 'utils/billing';
+import { convertBytesToHumanReadable } from 'utils/file/size';
 import { sleep } from 'utils/common';
 import { addToCollection } from 'services/collectionService';
 
@@ -33,7 +33,7 @@ export default async function uploader(
         fileWithCollection
     )}_${convertBytesToHumanReadable(UploadService.getAssetSize(uploadAsset))}`;
 
-    logUploadInfo(`uploader called for  ${fileNameSize}`);
+    addLogLine(`uploader called for  ${fileNameSize}`);
     UIService.setFileProgress(localID, 0);
     await sleep(0);
     const { fileTypeInfo, metadata } =
@@ -55,7 +55,7 @@ export default async function uploader(
             metadata
         );
         if (sameFileInSameCollection) {
-            logUploadInfo(`skipped upload for  ${fileNameSize}`);
+            addLogLine(`skipped upload for  ${fileNameSize}`);
             return {
                 fileUploadResult: UPLOAD_RESULT.ALREADY_UPLOADED,
                 uploadedFile: sameFileInSameCollection,
@@ -68,7 +68,7 @@ export default async function uploader(
         );
 
         if (sameFileInOtherCollection) {
-            logUploadInfo(
+            addLogLine(
                 `same file in other collection found for  ${fileNameSize}`
             );
             const resultFile = Object.assign({}, sameFileInOtherCollection);
@@ -84,7 +84,7 @@ export default async function uploader(
         // This change allow users to export by albums, upload to ente. And export all photos -> upload files which are not already uploaded
         // as part of the albums
         if (shouldDedupeAcrossCollection(fileWithCollection.collection.name)) {
-            logUploadInfo(`deduped upload for  ${fileNameSize}`);
+            addLogLine(`deduped upload for  ${fileNameSize}`);
             const sameFileInOtherCollection = findSameFileInCollection(
                 existingFiles,
                 metadata
@@ -96,7 +96,7 @@ export default async function uploader(
                 };
             }
         }
-        logUploadInfo(`reading asset ${fileNameSize}`);
+        addLogLine(`reading asset ${fileNameSize}`);
 
         const file = await UploadService.readAsset(fileTypeInfo, uploadAsset);
 
@@ -110,14 +110,14 @@ export default async function uploader(
             metadata,
         };
 
-        logUploadInfo(`encryptAsset ${fileNameSize}`);
+        addLogLine(`encryptAsset ${fileNameSize}`);
         const encryptedFile = await UploadService.encryptAsset(
             worker,
             fileWithMetadata,
             collection.key
         );
 
-        logUploadInfo(`uploadToBucket ${fileNameSize}`);
+        addLogLine(`uploadToBucket ${fileNameSize}`);
 
         const backupedFile: BackupedFile = await UploadService.uploadToBucket(
             encryptedFile.file
@@ -128,12 +128,12 @@ export default async function uploader(
             backupedFile,
             encryptedFile.fileKey
         );
-        logUploadInfo(`uploadFile ${fileNameSize}`);
+        addLogLine(`uploadFile ${fileNameSize}`);
 
         const uploadedFile = await UploadHttpClient.uploadFile(uploadFile);
 
         UIService.increaseFileUploaded();
-        logUploadInfo(`${fileNameSize} successfully uploaded`);
+        addLogLine(`${fileNameSize} successfully uploaded`);
 
         return {
             fileUploadResult: metadata.hasStaticThumbnail
@@ -142,9 +142,7 @@ export default async function uploader(
             uploadedFile: uploadedFile,
         };
     } catch (e) {
-        logUploadInfo(
+        addLogLine(`upload failed for  ${fileNameSize} ,error: ${e.message}`);
-            `upload failed for  ${fileNameSize} ,error: ${e.message}`
-        );
 
         logError(e, 'file upload failed', {
             fileFormat: fileTypeInfo?.exactType,

src/services/upload/videoMetadataService.ts

@@ -2,28 +2,28 @@ import { NULL_EXTRACTED_METADATA } from 'constants/upload';
 import ffmpegService from 'services/ffmpeg/ffmpegService';
 import { ElectronFile } from 'types/upload';
 import { logError } from 'utils/sentry';
-import { getFileNameSize, logUploadInfo } from 'utils/upload';
+import { getFileNameSize, addLogLine } from 'utils/logging';
 
 export async function getVideoMetadata(file: File | ElectronFile) {
     let videoMetadata = NULL_EXTRACTED_METADATA;
     try {
-        logUploadInfo(`getVideoMetadata called for ${getFileNameSize(file)}`);
+        addLogLine(`getVideoMetadata called for ${getFileNameSize(file)}`);
         if (!(file instanceof File)) {
-            logUploadInfo('get file blob for video metadata extraction');
+            addLogLine('get file blob for video metadata extraction');
             file = new File([await file.blob()], file.name, {
                 lastModified: file.lastModified,
             });
-            logUploadInfo(
+            addLogLine(
                 'get file blob for video metadata extraction successfully'
             );
         }
         videoMetadata = await ffmpegService.extractMetadata(file);
-        logUploadInfo(
+        addLogLine(
             `videoMetadata successfully extracted ${getFileNameSize(file)}`
         );
     } catch (e) {
         logError(e, 'failed to get video metadata');
-        logUploadInfo(
+        addLogLine(
             `videoMetadata extracted failed ${getFileNameSize(file)} ,${
                 e.message
             } `

src/utils/billing/index.ts

@@ -38,19 +38,6 @@ export function convertBytesToGBs(bytes: number, precision = 0): string {
     return (bytes / (1024 * 1024 * 1024)).toFixed(precision);
 }
 
-export function convertBytesToHumanReadable(
-    bytes: number,
-    precision = 2
-): string {
-    if (bytes === 0) {
-        return '0 MB';
-    }
-    const i = Math.floor(Math.log(bytes) / Math.log(1024));
-    const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
-
-    return (bytes / Math.pow(1024, i)).toFixed(precision) + ' ' + sizes[i];
-}
-
 export function makeHumanReadableStorage(
     bytes: number,
     round: 'round-up' | 'round-down' = 'round-down'

src/utils/file/index.ts

@@ -195,44 +195,6 @@ export function formatDate(date: number | Date) {
     return dateTimeFormat.format(date);
 }
 
-export function formatDateTime(date: number | Date) {
-    const dateTimeFormat = new Intl.DateTimeFormat('en-IN', {
-        weekday: 'short',
-        year: 'numeric',
-        month: 'long',
-        day: 'numeric',
-    });
-    const timeFormat = new Intl.DateTimeFormat('en-IN', {
-        timeStyle: 'short',
-    });
-    return `${dateTimeFormat.format(date)} ${timeFormat.format(date)}`;
-}
-
-export function formatDateRelative(date: number) {
-    const units = {
-        year: 24 * 60 * 60 * 1000 * 365,
-        month: (24 * 60 * 60 * 1000 * 365) / 12,
-        day: 24 * 60 * 60 * 1000,
-        hour: 60 * 60 * 1000,
-        minute: 60 * 1000,
-        second: 1000,
-    };
-    const relativeDateFormat = new Intl.RelativeTimeFormat('en-IN', {
-        localeMatcher: 'best fit',
-        numeric: 'always',
-        style: 'long',
-    });
-    const elapsed = date - Date.now();
-
-    // "Math.abs" accounts for both "past" & "future" scenarios
-    for (const u in units)
-        if (Math.abs(elapsed) > units[u] || u === 'second')
-            return relativeDateFormat.format(
-                Math.round(elapsed / units[u]),
-                u as Intl.RelativeTimeFormatUnit
-            );
-}
-
 export function sortFiles(files: EnteFile[]) {
     // sort according to modification time first
     files = files.sort((a, b) => {
@@ -352,8 +314,8 @@ export function splitFilenameAndExtension(filename: string): [string, string] {
         ];
 }
 
-export function getFileExtension(filename) {
+export function getFileExtension(filename: string) {
-    return splitFilenameAndExtension(filename)[1];
+    return splitFilenameAndExtension(filename)[1]?.toLocaleLowerCase();
 }
 
 export function generateStreamFromArrayBuffer(data: Uint8Array) {

src/utils/file/size.ts

@@ -0,0 +1,12 @@
+export function convertBytesToHumanReadable(
+    bytes: number,
+    precision = 2
+): string {
+    if (bytes === 0) {
+        return '0 MB';
+    }
+
+    const i = Math.floor(Math.log(bytes) / Math.log(1024));
+    const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+    return (bytes / Math.pow(1024, i)).toFixed(precision) + ' ' + sizes[i];
+}

src/utils/logging/index.ts

@@ -0,0 +1,21 @@
+import { ElectronFile } from 'types/upload';
+import { convertBytesToHumanReadable } from 'utils/file/size';
+import { formatDateTime } from 'utils/time';
+import { saveLogLine, getLogs } from 'utils/storage';
+
+export function addLogLine(log: string) {
+    saveLogLine({
+        timestamp: Date.now(),
+        logLine: log,
+    });
+}
+
+export function getDebugLogs() {
+    return getLogs().map(
+        (log) => `[${formatDateTime(log.timestamp)}] ${log.logLine}`
+    );
+}
+
+export function getFileNameSize(file: File | ElectronFile) {
+    return `${file.name}_${convertBytesToHumanReadable(file.size)}`;
+}

src/utils/sentry/index.ts

@@ -1,5 +1,6 @@
 import * as Sentry from '@sentry/nextjs';
-import { getUserAnonymizedID } from 'utils/user';
+import { addLogLine } from 'utils/logging';
+import { getSentryUserID } from 'utils/user';
 
 export const logError = (
     error: any,
@@ -10,12 +11,17 @@ export const logError = (
         return;
     }
     const err = errorWithContext(error, msg);
+    addLogLine(
+        `error: ${error?.name} ${error?.message} ${
+            error?.stack
+        } msg: ${msg} info: ${JSON.stringify(info)}`
+    );
     if (!process.env.NEXT_PUBLIC_SENTRY_ENV) {
         console.log(error, { msg, info });
     }
     Sentry.captureException(err, {
         level: Sentry.Severity.Info,
-        user: { id: getUserAnonymizedID() },
+        user: { id: getSentryUserID() },
         contexts: {
             ...(info && {
                 info: info,

src/utils/storage/index.ts

@@ -1,7 +1,6 @@
 import { getData, LS_KEYS, setData } from './localStorage';
 
 export interface Log {
-    type: string;
     timestamp: number;
     logLine: string;
 }

src/utils/storage/localStorage.ts

@@ -12,7 +12,7 @@ export enum LS_KEYS {
     JUST_SIGNED_UP = 'justSignedUp',
     SHOW_BACK_BUTTON = 'showBackButton',
     EXPORT = 'export',
-    AnonymizeUserID = 'anonymizedUserID',
+    AnonymizedUserID = 'anonymizedUserID',
     THUMBNAIL_FIX_STATE = 'thumbnailFixState',
     LIVE_PHOTO_INFO_SHOWN_COUNT = 'livePhotoInfoShownCount',
     LOGS = 'logs',

src/utils/time/index.ts

@@ -163,3 +163,39 @@ function getDateFromComponents(dateComponent: DateComponent<string>) {
         ? new Date(year, month, day, hour, minute, second)
         : new Date(year, month, day);
 }
+
+export function formatDateTime(date: number | Date) {
+    const dateTimeFormat = new Intl.DateTimeFormat('en-IN', {
+        weekday: 'short',
+        year: 'numeric',
+        month: 'long',
+        day: 'numeric',
+    });
+    const timeFormat = new Intl.DateTimeFormat('en-IN', {
+        timeStyle: 'short',
+    });
+    return `${dateTimeFormat.format(date)} ${timeFormat.format(date)}`;
+}
+export function formatDateRelative(date: number) {
+    const units = {
+        year: 24 * 60 * 60 * 1000 * 365,
+        month: (24 * 60 * 60 * 1000 * 365) / 12,
+        day: 24 * 60 * 60 * 1000,
+        hour: 60 * 60 * 1000,
+        minute: 60 * 1000,
+        second: 1000,
+    };
+    const relativeDateFormat = new Intl.RelativeTimeFormat('en-IN', {
+        localeMatcher: 'best fit',
+        numeric: 'always',
+        style: 'long',
+    });
+    const elapsed = date - Date.now(); // "Math.abs" accounts for both "past" & "future" scenarios
+
+    for (const u in units)
+        if (Math.abs(elapsed) > units[u] || u === 'second')
+            return relativeDateFormat.format(
+                Math.round(elapsed / units[u]),
+                u as Intl.RelativeTimeFormatUnit
+            );
+}

src/utils/upload/index.ts

@@ -1,8 +1,5 @@
-import { ElectronFile, FileWithCollection, Metadata } from 'types/upload';
+import { FileWithCollection, Metadata } from 'types/upload';
 import { EnteFile } from 'types/file';
-import { convertBytesToHumanReadable } from 'utils/billing';
-import { formatDateTime } from 'utils/file';
-import { getLogs, saveLogLine } from 'utils/storage';
 import { A_SEC_IN_MICROSECONDS } from 'constants/upload';
 import { FILE_TYPE } from 'constants/file';
 
@@ -117,24 +114,6 @@ export function segregateMetadataAndMediaFiles(
     return { mediaFiles, metadataJSONFiles };
 }
 
-export function logUploadInfo(log: string) {
-    saveLogLine({
-        type: 'upload',
-        timestamp: Date.now(),
-        logLine: log,
-    });
-}
-
-export function getUploadLogs() {
-    return getLogs()
-        .filter((log) => log.type === 'upload')
-        .map((log) => `[${formatDateTime(log.timestamp)}] ${log.logLine}`);
-}
-
-export function getFileNameSize(file: File | ElectronFile) {
-    return `${file.name}_${convertBytesToHumanReadable(file.size)}`;
-}
-
 export function areFileWithCollectionsSame(
     firstFile: FileWithCollection,
     secondFile: FileWithCollection

src/utils/user/index.ts

@@ -14,11 +14,11 @@ export function makeID(length) {
     return result;
 }
 
-export function getUserAnonymizedID() {
+export function getSentryUserID() {
-    let anonymizeUserID = getData(LS_KEYS.AnonymizeUserID)?.id;
+    let anonymizeUserID = getData(LS_KEYS.AnonymizedUserID)?.id;
     if (!anonymizeUserID) {
         anonymizeUserID = makeID(6);
-        setData(LS_KEYS.AnonymizeUserID, { id: anonymizeUserID });
+        setData(LS_KEYS.AnonymizedUserID, { id: anonymizeUserID });
     }
     return anonymizeUserID;
 }