Merge branch 'add-segmented-live-photo-loading' into add-file-disk-caching

2023-12-08 19:08:12 +05:30 · 2023-12-08 19:08:12 +05:30 · d4776ce40b
parent df574d9b39 69c817b2b4
commit d4776ce40b
6 changed files with 185 additions and 84 deletions
--- a/apps/photos/src/components/PhotoFrame.tsx
+++ b/apps/photos/src/components/PhotoFrame.tsx
@ -3,7 +3,10 @@ import PreviewCard from './pages/gallery/PreviewCard';
 import { useContext, useEffect, useState } from 'react';
 import { EnteFile } from 'types/file';
 import { styled } from '@mui/material';
-import DownloadManager, { SourceURLs } from 'services/download';
+import DownloadManager, {
+    LivePhotoSourceURL,
+    SourceURLs,
+} from 'services/download';
 import AutoSizer from 'react-virtualized-auto-sizer';
 import PhotoViewer from 'components/PhotoViewer';
 import { TRASH_SECTION } from 'constants/collection';
@ -197,16 +200,14 @@ const PhotoFrame = ({
        // this is to prevent outdate updateSrcURL call from updating the wrong file
        if (file.id !== id) {
            addLogLine(
-                `[${id}]PhotoSwipe: updateSrcURL: file id mismatch: ${file.id} !== ${id}`
+                `[${id}]PhotoSwipe: updateSrcURL: file id mismatch: ${file.id}`
            );
            throw Error(CustomError.UPDATE_URL_FILE_ID_MISMATCH);
        }
        if (file.isSourceLoaded && !forceUpdate) {
            throw Error(CustomError.URL_ALREADY_SET);
        } else if (file.conversionFailed) {
-            addLogLine(
-                `[${id}]PhotoSwipe: updateSrcURL: conversion failed: ${file.id}`
-            );
+            addLogLine(`[${id}]PhotoSwipe: updateSrcURL: conversion failed`);
            throw Error(CustomError.FILE_CONVERSION_FAILED);
        }

@ -407,23 +408,85 @@ const PhotoFrame = ({
            addLogLine(`[${item.id}] new file src request`);
            fetching[item.id] = true;
            const srcURLs = await DownloadManager.getFileForPreview(item);
-            try {
-                await updateSrcURL(index, item.id, srcURLs);
-                addLogLine(
-                    `[${item.id}] calling invalidateCurrItems for src, source loaded :${item.isSourceLoaded}`
-                );
-                instance.invalidateCurrItems();
-                if ((instance as any).isOpen()) {
-                    instance.updateSize(true);
-                }
-            } catch (e) {
-                if (e.message !== CustomError.URL_ALREADY_SET) {
-                    logError(
-                        e,
-                        'updating photoswipe after src url update failed'
+            if (item.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
+                const srcImgURL = srcURLs.url as LivePhotoSourceURL;
+                const imageURL = await srcImgURL.image();
+
+                const dummyImgSrcUrl: SourceURLs = {
+                    url: imageURL,
+                    isOriginal: false,
+                    isRenderable: !!imageURL,
+                    type: 'normal',
+                };
+                try {
+                    await updateSrcURL(index, item.id, dummyImgSrcUrl);
+                    addLogLine(
+                        `[${item.id}] calling invalidateCurrItems for live photo imgSrc, source loaded :${item.isSourceLoaded}`
                    );
+                    instance.invalidateCurrItems();
+                    if ((instance as any).isOpen()) {
+                        instance.updateSize(true);
+                    }
+                } catch (e) {
+                    if (e.message !== CustomError.URL_ALREADY_SET) {
+                        logError(
+                            e,
+                            'updating photoswipe after for live photo imgSrc update failed'
+                        );
+                    }
+                }
+                if (!imageURL) {
+                    // no image url, no need to load video
+                    return;
+                }
+
+                const videoURL = await srcImgURL.video();
+                const loadedLivePhotoSrcURL: SourceURLs = {
+                    url: { video: videoURL, image: imageURL },
+                    isOriginal: false,
+                    isRenderable: !!videoURL,
+                    type: 'livePhoto',
+                };
+                try {
+                    await updateSrcURL(
+                        index,
+                        item.id,
+                        loadedLivePhotoSrcURL,
+                        true
+                    );
+                    addLogLine(
+                        `[${item.id}] calling invalidateCurrItems for live photo complete, source loaded :${item.isSourceLoaded}`
+                    );
+                    instance.invalidateCurrItems();
+                    if ((instance as any).isOpen()) {
+                        instance.updateSize(true);
+                    }
+                } catch (e) {
+                    if (e.message !== CustomError.URL_ALREADY_SET) {
+                        logError(
+                            e,
+                            'updating photoswipe for live photo complete update failed'
+                        );
+                    }
+                }
+            } else {
+                try {
+                    await updateSrcURL(index, item.id, srcURLs);
+                    addLogLine(
+                        `[${item.id}] calling invalidateCurrItems for src, source loaded :${item.isSourceLoaded}`
+                    );
+                    instance.invalidateCurrItems();
+                    if ((instance as any).isOpen()) {
+                        instance.updateSize(true);
+                    }
+                } catch (e) {
+                    if (e.message !== CustomError.URL_ALREADY_SET) {
+                        logError(
+                            e,
+                            'updating photoswipe after src url update failed'
+                        );
+                    }
                }
-                throw e;
            }
        } catch (e) {
            logError(e, 'getSlideData failed get src url failed');
--- a/apps/photos/src/components/PhotoViewer/index.tsx
+++ b/apps/photos/src/components/PhotoViewer/index.tsx
@ -49,7 +49,7 @@ import { getParsedExifData } from 'services/upload/exifService';
 import { getFileType } from 'services/typeDetectionService';
 import { ConversionFailedNotification } from './styledComponents/ConversionFailedNotification';
 import { GalleryContext } from 'pages/gallery';
-import downloadManager from 'services/download';
+import downloadManager, { LoadedLivePhotoSourceURL } from 'services/download';
 import CircularProgressWithLabel from './styledComponents/CircularProgressWithLabel';
 import EnteSpinner from '@ente/shared/components/EnteSpinner';
 import AlbumOutlined from '@mui/icons-material/AlbumOutlined';
@ -294,18 +294,19 @@ function PhotoViewer(props: Iprops) {
            setExif({ key: file.src, value: null });
            return;
        }
-        if (!file.isSourceLoaded) {
+        if (!file.isSourceLoaded || file.conversionFailed) {
+            return;
+        }
+
+        if (!file || !exifCopy?.current?.value === null) {
            return;
        }
        const key =
            file.metadata.fileType === FILE_TYPE.IMAGE
                ? file.src
-                : (file.srcURLs.url as any).image;
-        if (
-            !file ||
-            !exifCopy?.current?.value === null ||
-            exifCopy?.current?.key === key
-        ) {
+                : (file.srcURLs.url as LoadedLivePhotoSourceURL).image;
+
+        if (exifCopy?.current?.key === key) {
            return;
        }
        setExif({ key, value: undefined });
@ -555,9 +556,8 @@ function PhotoViewer(props: Iprops) {
                        file.metadata.title
                    );
                } else {
-                    const url = (
-                        file.srcURLs.url as { image: string; video: string }
-                    ).image;
+                    const url = (file.srcURLs.url as LoadedLivePhotoSourceURL)
+                        .image;
                    fileObject = await getFileFromURL(url, file.metadata.title);
                }
                const fileTypeInfo = await getFileType(fileObject);
--- a/apps/photos/src/pages/gallery/index.tsx
+++ b/apps/photos/src/pages/gallery/index.tsx
@ -410,7 +410,7 @@ export default function Gallery() {
    }, [fixCreationTimeAttributes]);

    useEffect(() => {
-        if (typeof activeCollectionID === 'undefined') {
+        if (typeof activeCollectionID === 'undefined' || !router.isReady) {
            return;
        }
        let collectionURL = '';
@ -429,14 +429,8 @@ export default function Gallery() {
            }
        }
        const href = `/gallery${collectionURL}`;
-        const delayRouteChange = () => {
-            setTimeout(() => {
-                router.push(href, undefined, { shallow: true });
-            }, 1000);
-        };
-
-        delayRouteChange();
-    }, [activeCollectionID]);
+        router.push(href, undefined, { shallow: true });
+    }, [activeCollectionID, router.isReady]);

    useEffect(() => {
        const key = getKey(SESSION_KEYS.ENCRYPTION_KEY);
--- a/apps/photos/src/services/download/index.ts
+++ b/apps/photos/src/services/download/index.ts
@ -18,15 +18,21 @@ import { APPS } from '@ente/shared/apps/constants';
 import { PhotosDownloadClient } from './clients/photos';
 import { PublicAlbumsDownloadClient } from './clients/publicAlbums';

+export type LivePhotoSourceURL = {
+    image: () => Promise<string>;
+    video: () => Promise<string>;
+};
+
+export type LoadedLivePhotoSourceURL = {
+    image: string;
+    video: string;
+};
+
 export type SourceURLs = {
-    url:
-        | {
-              image: string;
-              video: string;
-          }
-        | string;
+    url: string | LivePhotoSourceURL | LoadedLivePhotoSourceURL;
    isOriginal: boolean;
    isRenderable: boolean;
+    type: 'normal' | 'livePhoto';
 };

 export type OnDownloadProgress = (event: {
@ -218,15 +224,13 @@ class DownloadManager {
                );
                return converted;
            };
-            if (!this.fileConversionPromises.has(file.id)) {
+            if (forceConvert || !this.fileConversionPromises.has(file.id)) {
                this.fileConversionPromises.set(
                    file.id,
                    getFileForPreviewPromise()
                );
            }
            const fileURLs = await this.fileConversionPromises.get(file.id);
-            this.fileConversionPromises.delete(file.id);
-            this.fileObjectURLPromises.set(file.id, Promise.resolve(fileURLs));
            return fileURLs;
        } catch (e) {
            this.fileConversionPromises.delete(file.id);
@ -250,6 +254,7 @@ class DownloadManager {
                    url: URL.createObjectURL(fileBlob),
                    isOriginal: true,
                    isRenderable: false,
+                    type: 'normal',
                };
            };
            if (!this.fileObjectURLPromises.has(file.id)) {
--- a/apps/photos/src/utils/file/index.ts
+++ b/apps/photos/src/utils/file/index.ts
@ -10,7 +10,10 @@ import {
 } from 'types/file';
 import { decodeLivePhoto } from 'services/livePhotoService';
 import { getFileType } from 'services/typeDetectionService';
-import DownloadManager, { SourceURLs } from 'services/download';
+import DownloadManager, {
+    LivePhotoSourceURL,
+    SourceURLs,
+} from 'services/download';
 import { logError } from '@ente/shared/sentry';
 import { User } from '@ente/shared/user/types';
 import { getData, LS_KEYS } from '@ente/shared/storage/localStorage';
@ -336,7 +339,12 @@ export async function getRenderableFileURL(
    return {
        url: srcURLs,
        isOriginal,
-        isRenderable: !!srcURLs,
+        isRenderable:
+            file.metadata.fileType !== FILE_TYPE.LIVE_PHOTO && !!srcURLs,
+        type:
+            file.metadata.fileType === FILE_TYPE.LIVE_PHOTO
+                ? 'livePhoto'
+                : 'normal',
    };
 }

@ -344,32 +352,52 @@ async function getRenderableLivePhotoURL(
    file: EnteFile,
    fileBlob: Blob,
    forceConvert: boolean
-): Promise<{ image: string; video: string }> {
+): Promise<LivePhotoSourceURL> {
    const livePhoto = await decodeLivePhoto(file, fileBlob);
-    const imageBlob = new Blob([livePhoto.image]);
-    const videoBlob = new Blob([livePhoto.video]);
-    const convertedImageBlob = await getRenderableImage(
-        livePhoto.imageNameTitle,
-        imageBlob
-    );
-    const convertedVideoBlob = await getPlayableVideo(
-        livePhoto.videoNameTitle,
-        videoBlob,
-        forceConvert
-    );
-    const convertedImageURL = URL.createObjectURL(convertedImageBlob);
-    const convertedVideoURL = URL.createObjectURL(convertedVideoBlob);
+
+    const getRenderableLivePhotoImageURL = async () => {
+        try {
+            const imageBlob = new Blob([livePhoto.image]);
+            const convertedImageBlob = await getRenderableImage(
+                livePhoto.imageNameTitle,
+                imageBlob
+            );
+
+            return URL.createObjectURL(convertedImageBlob);
+        } catch (e) {
+            //ignore and return null
+            return null;
+        }
+    };
+
+    const getRenderableLivePhotoVideoURL = async () => {
+        try {
+            const videoBlob = new Blob([livePhoto.video]);
+
+            const convertedVideoBlob = await getPlayableVideo(
+                livePhoto.videoNameTitle,
+                videoBlob,
+                forceConvert,
+                true
+            );
+            return URL.createObjectURL(convertedVideoBlob);
+        } catch (e) {
+            //ignore and return null
+            return null;
+        }
+    };

    return {
-        image: convertedImageURL,
-        video: convertedVideoURL,
+        image: getRenderableLivePhotoImageURL,
+        video: getRenderableLivePhotoVideoURL,
    };
 }

 export async function getPlayableVideo(
    videoNameTitle: string,
    videoBlob: Blob,
-    forceConvert = false
+    forceConvert = false,
+    runOnWeb = false
 ) {
    try {
        const isPlayable = await isPlaybackPossible(
@ -378,7 +406,7 @@ export async function getPlayableVideo(
        if (isPlayable && !forceConvert) {
            return videoBlob;
        } else {
-            if (!forceConvert && !isElectron()) {
+            if (!forceConvert && !runOnWeb && !isElectron()) {
                return null;
            }
            addLogLine(
--- a/apps/photos/src/utils/photoFrame/index.ts
+++ b/apps/photos/src/utils/photoFrame/index.ts
@ -1,7 +1,7 @@
 import { FILE_TYPE } from 'constants/file';
 import { EnteFile } from 'types/file';
 import { logError } from '@ente/shared/sentry';
-import { SourceURLs } from 'services/download';
+import { LivePhotoSourceURL, SourceURLs } from 'services/download';

 const WAIT_FOR_VIDEO_PLAYBACK = 1 * 1000;

@ -70,14 +70,18 @@ export function updateFileMsrcProps(file: EnteFile, url: string) {
 }

 export async function updateFileSrcProps(file: EnteFile, srcURLs: SourceURLs) {
-    const { url, isRenderable } = srcURLs;
+    const { url, isRenderable, isOriginal } = srcURLs;
    file.w = window.innerWidth;
    file.h = window.innerHeight;
-    file.isSourceLoaded = true;
-    file.isConverted = !srcURLs.isOriginal;
-    file.conversionFailed = !srcURLs.isRenderable;
+    file.isSourceLoaded =
+        file.metadata.fileType === FILE_TYPE.LIVE_PHOTO
+            ? srcURLs.type === 'livePhoto'
+            : true;
+    file.isConverted = !isOriginal;
+    file.conversionFailed = !isRenderable;
    file.srcURLs = srcURLs;
    if (!isRenderable) {
+        file.isSourceLoaded = true;
        return;
    }

@ -89,19 +93,26 @@ export async function updateFileSrcProps(file: EnteFile, srcURLs: SourceURLs) {
                </video>
                `;
    } else if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
-        const { image: imageURL, video: videoURL } = url as {
-            image: string;
-            video: string;
-        };
-        file.html = `
+        if (srcURLs.type === 'normal') {
+            file.html = `
                <div class = 'pswp-item-container'>
-                    <img id = "live-photo-image-${file.id}" src="${imageURL}" onContextMenu="return false;"/>
-                    <video id = "live-photo-video-${file.id}" loop muted onContextMenu="return false;">
-                        <source src="${videoURL}" />
-                        Your browser does not support the video tag.
-                    </video>
+                    <img id = "live-photo-image-${file.id}" src="${url}" onContextMenu="return false;"/>
                </div>
                `;
+        } else {
+            const { image: imageURL, video: videoURL } =
+                url as LivePhotoSourceURL;
+
+            file.html = `
+            <div class = 'pswp-item-container'>
+                <img id = "live-photo-image-${file.id}" src="${imageURL}" onContextMenu="return false;"/>
+                <video id = "live-photo-video-${file.id}" loop muted onContextMenu="return false;">
+                    <source src="${videoURL}" />
+                    Your browser does not support the video tag.
+                </video>
+            </div>
+            `;
+        }
    } else if (file.metadata.fileType === FILE_TYPE.IMAGE) {
        file.src = url as string;
    } else {