Merge pull request #392 from ente-io/add-livephoto-support

Live photo playback support
This commit is contained in:
Rushikesh Tote 2022-03-03 13:08:42 +05:30 committed by GitHub
commit def22493b9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 350 additions and 61 deletions

View file

@ -0,0 +1,12 @@
import React from 'react';
export const livePhotoBtnHTML = (
<svg
xmlns="http://www.w3.org/2000/svg"
height="25px"
width="25px"
fill="currentColor">
<path d="M0 0h24v24H0V0z" fill="none" />
<path d="M6.76 4.84l-1.8-1.79-1.41 1.41 1.79 1.79zM1 10.5h3v2H1zM11 .55h2V3.5h-2zm8.04 2.495l1.408 1.407-1.79 1.79-1.407-1.408zm-1.8 15.115l1.79 1.8 1.41-1.41-1.8-1.79zM20 10.5h3v2h-3zm-8-5c-3.31 0-6 2.69-6 6s2.69 6 6 6 6-2.69 6-6-2.69-6-6-6zm0 10c-2.21 0-4-1.79-4-4s1.79-4 4-4 4 1.79 4 4-1.79 4-4 4zm-1 4h2v2.95h-2zm-7.45-.96l1.41 1.41 1.79-1.8-1.41-1.41z" />
</svg>
);

View file

@ -72,6 +72,11 @@ interface Props {
enableDownload: boolean;
}
type SourceURL = {
imageURL?: string;
videoURL?: string;
};
const PhotoFrame = ({
files,
setFiles,
@ -103,6 +108,7 @@ const PhotoFrame = ({
const filteredDataRef = useRef([]);
const filteredData = filteredDataRef?.current ?? [];
const router = useRouter();
const [isSourceLoaded, setIsSourceLoaded] = useState(false);
useEffect(() => {
const handleKeyDown = (e: KeyboardEvent) => {
if (e.key === 'Shift') {
@ -261,8 +267,19 @@ const PhotoFrame = ({
</div>
`;
} else if (
(file.metadata.fileType === FILE_TYPE.IMAGE ||
file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) &&
file.metadata.fileType === FILE_TYPE.LIVE_PHOTO &&
!file.html
) {
file.html = `
<div class="video-loading">
<img src="${url}" />
<div class="spinner-border text-light" role="status">
<span class="sr-only">Loading...</span>
</div>
</div>
`;
} else if (
file.metadata.fileType === FILE_TYPE.IMAGE &&
!file.src
) {
file.src = url;
@ -276,8 +293,9 @@ const PhotoFrame = ({
return updateFile(files[index]);
};
const updateSrcURL = async (index: number, url: string) => {
const isPlayable = await isPlaybackPossible(url);
const updateSrcURL = async (index: number, srcURL: SourceURL) => {
const { videoURL, imageURL } = srcURL;
const isPlayable = videoURL && (await isPlaybackPossible(videoURL));
const updateFile = (file: EnteFile) => {
file = {
...file,
@ -288,7 +306,7 @@ const PhotoFrame = ({
if (isPlayable) {
file.html = `
<video controls>
<source src="${url}" />
<source src="${videoURL}" />
Your browser does not support the video tag.
</video>
`;
@ -298,13 +316,35 @@ const PhotoFrame = ({
<img src="${file.msrc}" />
<div class="download-message" >
${constants.VIDEO_PLAYBACK_FAILED_DOWNLOAD_INSTEAD}
<a class="btn btn-outline-success" href=${url} download="${file.metadata.title}"">Download</button>
<a class="btn btn-outline-success" href=${videoURL} download="${file.metadata.title}"">Download</button>
</div>
</div>
`;
}
} else if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
if (isPlayable) {
file.html = `
<div class = 'live-photo-container'>
<img id = "live-photo-image-${file.id}" src="${imageURL}" />
<video id = "live-photo-video-${file.id}" loop muted>
<source src="${videoURL}" />
Your browser does not support the video tag.
</video>
</div>
`;
} else {
file.html = `
<div class="video-loading">
<img src="${file.msrc}" />
<div class="download-message">
${constants.VIDEO_PLAYBACK_FAILED_DOWNLOAD_INSTEAD}
<button class = "btn btn-outline-success" id = "download-btn-${file.id}">Download</button>
</div>
</div>
`;
}
} else {
file.src = url;
file.src = imageURL;
}
return file;
};
@ -312,6 +352,7 @@ const PhotoFrame = ({
files[index] = updateFile(files[index]);
return [...files];
});
setIsSourceLoaded(true);
return updateFile(files[index]);
};
@ -448,27 +489,42 @@ const PhotoFrame = ({
if (!fetching[item.dataIndex]) {
try {
fetching[item.dataIndex] = true;
let url: string;
let urls: string[];
if (galleryContext.files.has(item.id)) {
url = galleryContext.files.get(item.id);
const mergedURL = galleryContext.files.get(item.id);
urls = mergedURL.split(',');
} else {
galleryContext.startLoading();
if (
publicCollectionGalleryContext.accessedThroughSharedURL
) {
url = await PublicCollectionDownloadManager.getFile(
urls = await PublicCollectionDownloadManager.getFile(
item,
publicCollectionGalleryContext.token,
publicCollectionGalleryContext.passwordToken,
true
);
} else {
url = await DownloadManager.getFile(item, true);
urls = await DownloadManager.getFile(item, true);
}
galleryContext.finishLoading();
galleryContext.files.set(item.id, url);
const mergedURL = urls.join(',');
galleryContext.files.set(item.id, mergedURL);
}
const newFile = await updateSrcURL(item.dataIndex, url);
let imageURL;
let videoURL;
if (item.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
[imageURL, videoURL] = urls;
} else if (item.metadata.fileType === FILE_TYPE.VIDEO) {
[videoURL] = urls;
} else {
[imageURL] = urls;
}
setIsSourceLoaded(false);
const newFile = await updateSrcURL(item.dataIndex, {
imageURL,
videoURL,
});
item.msrc = newFile.msrc;
item.html = newFile.html;
item.src = newFile.src;
@ -537,6 +593,7 @@ const PhotoFrame = ({
isSharedCollection={isSharedCollection}
isTrashCollection={activeCollection === TRASH_SECTION}
enableDownload={enableDownload}
isSourceLoaded={isSourceLoaded}
/>
</Container>
)}

View file

@ -20,7 +20,6 @@ import {
changeFileName,
downloadFile,
formatDateTime,
isLivePhoto,
splitFilenameAndExtension,
updateExistingFilePubMetadata,
} from 'utils/file';
@ -35,6 +34,7 @@ import {
Row,
Value,
} from 'components/Container';
import { livePhotoBtnHTML } from 'components/LivePhotoBtn';
import { logError } from 'utils/sentry';
import CloseIcon from 'components/icons/CloseIcon';
@ -43,14 +43,11 @@ import { Formik } from 'formik';
import * as Yup from 'yup';
import EnteSpinner from 'components/EnteSpinner';
import EnteDateTimePicker from 'components/EnteDateTimePicker';
import { MAX_EDITED_FILE_NAME_LENGTH } from 'constants/file';
import { MAX_EDITED_FILE_NAME_LENGTH, FILE_TYPE } from 'constants/file';
import { sleep } from 'utils/common';
import { playVideo, pauseVideo } from 'utils/photoFrame';
import { PublicCollectionGalleryContext } from 'utils/publicCollectionGallery';
import { GalleryContext } from 'pages/gallery';
import {
getLivePhotoInfoShownCount,
setLivePhotoInfoShownCount,
} from 'utils/storage';
const SmallLoadingSpinner = () => (
<EnteSpinner
@ -72,6 +69,7 @@ interface Iprops {
isSharedCollection: boolean;
isTrashCollection: boolean;
enableDownload: boolean;
isSourceLoaded: boolean;
}
const LegendContainer = styled.div`
@ -90,6 +88,29 @@ const Pre = styled.pre`
padding: 7px 15px;
`;
const LivePhotoBtn = styled.button`
position: absolute;
bottom: 6vh;
right: 6vh;
height: 40px;
width: 80px;
background: #d7d7d7;
outline: none;
border: none;
border-radius: 10%;
z-index: 10;
cursor: ${(props) => (props.disabled ? 'not-allowed' : 'pointer')};
}
`;
const livePhotoDefaultOptions = {
click: () => {},
hide: () => {},
show: () => {},
loading: false,
visible: false,
};
const renderInfoItem = (label: string, value: string | JSX.Element) => (
<Row>
<Label width="30%">{label}</Label>
@ -493,11 +514,14 @@ function PhotoSwipe(props: Iprops) {
const [photoSwipe, setPhotoSwipe] =
useState<Photoswipe<Photoswipe.Options>>();
const { isOpen, items } = props;
const { isOpen, items, isSourceLoaded } = props;
const [isFav, setIsFav] = useState(false);
const [showInfo, setShowInfo] = useState(false);
const [metadata, setMetaData] = useState<EnteFile['metadata']>(null);
const [exif, setExif] = useState<any>(null);
const [livePhotoBtnOptions, setLivePhotoBtnOptions] = useState(
livePhotoDefaultOptions
);
const needUpdate = useRef(false);
const publicCollectionGalleryContext = useContext(
PublicCollectionGalleryContext
@ -528,21 +552,73 @@ function PhotoSwipe(props: Iprops) {
}
}, [showInfo]);
function updateFavButton() {
setIsFav(isInFav(this?.currItem));
useEffect(() => {
if (!isOpen) return;
const item = items[photoSwipe?.getCurrentIndex()];
if (item && item.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
const getVideoAndImage = () => {
const video = document.getElementById(
`live-photo-video-${item.id}`
);
const image = document.getElementById(
`live-photo-image-${item.id}`
);
return { video, image };
};
const { video, image } = getVideoAndImage();
if (video && image) {
setLivePhotoBtnOptions({
click: async () => {
await playVideo(video, image);
},
hide: async () => {
await pauseVideo(video, image);
},
show: async () => {
await playVideo(video, image);
},
visible: true,
loading: false,
});
} else {
setLivePhotoBtnOptions({
...livePhotoDefaultOptions,
visible: true,
loading: true,
});
}
function handleLivePhotoNotification() {
if (isLivePhoto(this?.currItem)) {
const infoShownCount = getLivePhotoInfoShownCount();
if (infoShownCount < 3) {
galleryContext.setNotificationAttributes({
message: constants.PLAYBACK_SUPPORT_COMING,
title: constants.LIVE_PHOTO,
});
setLivePhotoInfoShownCount(infoShownCount + 1);
const downloadLivePhotoBtn = document.getElementById(
`download-btn-${item.id}`
) as HTMLButtonElement;
if (downloadLivePhotoBtn) {
const downloadLivePhoto = () => {
downloadFileHelper(photoSwipe.currItem);
};
downloadLivePhotoBtn.addEventListener(
'click',
downloadLivePhoto
);
return () => {
downloadLivePhotoBtn.removeEventListener(
'click',
downloadLivePhoto
);
setLivePhotoBtnOptions(livePhotoDefaultOptions);
};
}
return () => {
setLivePhotoBtnOptions(livePhotoDefaultOptions);
};
}
}, [photoSwipe?.currItem, isOpen, isSourceLoaded]);
function updateFavButton() {
setIsFav(isInFav(this?.currItem));
}
const openPhotoSwipe = () => {
@ -607,7 +683,6 @@ function PhotoSwipe(props: Iprops) {
photoSwipe.listen('beforeChange', function () {
updateInfo.call(this);
updateFavButton.call(this);
handleLivePhotoNotification.call(this);
});
photoSwipe.listen('resize', checkExifAvailable);
photoSwipe.init();
@ -727,6 +802,18 @@ function PhotoSwipe(props: Iprops) {
ref={pswpElement}>
<div className="pswp__bg" />
<div className="pswp__scroll-wrap">
<LivePhotoBtn
onClick={livePhotoBtnOptions.click}
onMouseEnter={livePhotoBtnOptions.show}
onMouseLeave={livePhotoBtnOptions.hide}
disabled={livePhotoBtnOptions.loading}
style={{
display: livePhotoBtnOptions.visible
? 'block'
: 'none',
}}>
{livePhotoBtnHTML} {constants.LIVE}
</LivePhotoBtn>
<div className="pswp__container">
<div className="pswp__item" />
<div className="pswp__item" />

View file

@ -69,6 +69,35 @@ const GlobalStyles = createGlobalStyle`
height: 100%;
}
.live-photo-container{
width: 100%;
height: 100%;
position: relative;
object-fit: contain;
}
.live-photo-container > img{
opacity: 1;
max-width: 100%;
max-height: 100%;
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
transition: opacity 1s ease;
}
.live-photo-container > video{
opacity: 0;
max-width: 100%;
max-height: 100%;
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
transition: opacity 1s ease;
}
.video-loading {
width: 100%;
height: 100%;

View file

@ -14,7 +14,7 @@ import { FILE_TYPE } from 'constants/file';
import { CustomError } from 'utils/error';
class DownloadManager {
private fileObjectURLPromise = new Map<string, Promise<string>>();
private fileObjectURLPromise = new Map<string, Promise<string[]>>();
private thumbnailObjectURLPromise = new Map<number, Promise<string>>();
public async getThumbnail(file: EnteFile) {
@ -90,11 +90,17 @@ class DownloadManager {
try {
const getFilePromise = async (convert: boolean) => {
const fileStream = await this.downloadFile(file);
let fileBlob = await new Response(fileStream).blob();
const fileBlob = await new Response(fileStream).blob();
if (convert) {
fileBlob = await convertForPreview(file, fileBlob);
const convertedBlobs = await convertForPreview(
file,
fileBlob
);
return convertedBlobs.map((blob) =>
URL.createObjectURL(blob)
);
}
return URL.createObjectURL(fileBlob);
return [URL.createObjectURL(fileBlob)];
};
if (!this.fileObjectURLPromise.get(fileKey)) {
this.fileObjectURLPromise.set(
@ -102,8 +108,8 @@ class DownloadManager {
getFilePromise(shouldBeConverted)
);
}
const fileURL = await this.fileObjectURLPromise.get(fileKey);
return fileURL;
const fileURLs = await this.fileObjectURLPromise.get(fileKey);
return fileURLs;
} catch (e) {
this.fileObjectURLPromise.delete(fileKey);
logError(e, 'Failed to get File');

View file

@ -90,6 +90,34 @@ class FFmpegService {
}
}
}
async convertToMP4(
file: Uint8Array,
fileName: string
): Promise<Uint8Array> {
if (!this.ffmpeg) {
await this.init();
}
if (this.isLoading) {
await this.isLoading;
}
const response = this.ffmpegTaskQueue.queueUpRequest(
convertToMP4Helper.bind(null, this.ffmpeg, file, fileName)
);
try {
return await response.promise;
} catch (e) {
if (e.message === CustomError.REQUEST_CANCELLED) {
// ignore
return null;
} else {
logError(e, 'ffmpeg MP4 conversion failed');
throw e;
}
}
}
}
async function generateThumbnailHelper(
@ -175,4 +203,28 @@ async function extractVideoMetadataHelper(
}
}
async function convertToMP4Helper(
ffmpeg: FFmpeg,
file: Uint8Array,
inputFileName: string
) {
try {
ffmpeg.FS('writeFile', inputFileName, file);
await ffmpeg.run(
'-i',
inputFileName,
'-preset',
'ultrafast',
'output.mp4'
);
const convertedFile = ffmpeg.FS('readFile', 'output.mp4');
ffmpeg.FS('unlink', inputFileName);
ffmpeg.FS('unlink', 'output.mp4');
return convertedFile;
} catch (e) {
logError(e, 'ffmpeg MP4 conversion failed');
throw e;
}
}
export default new FFmpegService();

View file

@ -16,7 +16,7 @@ import { FILE_TYPE } from 'constants/file';
import { CustomError } from 'utils/error';
class PublicCollectionDownloadManager {
private fileObjectURLPromise = new Map<string, Promise<string>>();
private fileObjectURLPromise = new Map<string, Promise<string[]>>();
private thumbnailObjectURLPromise = new Map<number, Promise<string>>();
public async getThumbnail(
@ -116,11 +116,17 @@ class PublicCollectionDownloadManager {
passwordToken,
file
);
let fileBlob = await new Response(fileStream).blob();
const fileBlob = await new Response(fileStream).blob();
if (convert) {
fileBlob = await convertForPreview(file, fileBlob);
const convertedBlobs = await convertForPreview(
file,
fileBlob
);
return convertedBlobs.map((blob) =>
URL.createObjectURL(blob)
);
}
return URL.createObjectURL(fileBlob);
return [URL.createObjectURL(fileBlob)];
};
if (!this.fileObjectURLPromise.get(fileKey)) {
this.fileObjectURLPromise.set(
@ -128,8 +134,8 @@ class PublicCollectionDownloadManager {
getFilePromise(shouldBeConverted)
);
}
const fileURL = await this.fileObjectURLPromise.get(fileKey);
return fileURL;
const fileURLs = await this.fileObjectURLPromise.get(fileKey);
return fileURLs;
} catch (e) {
this.fileObjectURLPromise.delete(fileKey);
logError(e, 'Failed to get File');

View file

@ -36,7 +36,7 @@ export async function updateCreationTimeWithExif(
if (fixOption === FIX_OPTIONS.CUSTOM_TIME) {
correctCreationTime = getUnixTimeInMicroSeconds(customTime);
} else {
const fileURL = await downloadManager.getFile(file);
const fileURL = await downloadManager.getFile(file)[0];
const fileObject = await getFileFromURL(fileURL);
const reader = new FileReader();
const fileTypeInfo = await getFileType(reader, fileObject);

View file

@ -24,6 +24,7 @@ import {
} from 'constants/file';
import PublicCollectionDownloadManager from 'services/publicCollectionDownloadManager';
import HEICConverter from 'services/HEICConverter';
import ffmpegService from 'services/ffmpegService';
export function downloadAsFile(filename: string, content: string) {
const file = new Blob([content], {
@ -52,7 +53,7 @@ export async function downloadFile(
if (accessedThroughSharedURL) {
fileURL = await PublicCollectionDownloadManager.getCachedOriginalFile(
file
);
)[0];
tempURL;
if (!fileURL) {
tempURL = URL.createObjectURL(
@ -68,7 +69,7 @@ export async function downloadFile(
fileURL = tempURL;
}
} else {
fileURL = await DownloadManager.getCachedOriginalFile(file);
fileURL = await DownloadManager.getCachedOriginalFile(file)[0];
if (!fileURL) {
tempURL = URL.createObjectURL(
await new Response(
@ -329,16 +330,13 @@ export function generateStreamFromArrayBuffer(data: Uint8Array) {
});
}
export async function convertForPreview(file: EnteFile, fileBlob: Blob) {
if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
const originalName = fileNameWithoutExtension(file.metadata.title);
const motionPhoto = await decodeMotionPhoto(fileBlob, originalName);
fileBlob = new Blob([motionPhoto.image]);
}
const typeFromExtension = getFileExtension(file.metadata.title);
export async function convertForPreview(
file: EnteFile,
fileBlob: Blob
): Promise<Blob[]> {
const convertIfHEIC = async (fileName: string, fileBlob: Blob) => {
const typeFromExtension = getFileExtension(fileName);
const reader = new FileReader();
const mimeType =
(await getFileTypeFromBlob(reader, fileBlob))?.mime ??
typeFromExtension;
@ -346,6 +344,28 @@ export async function convertForPreview(file: EnteFile, fileBlob: Blob) {
fileBlob = await HEICConverter.convert(fileBlob);
}
return fileBlob;
};
if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
const originalName = fileNameWithoutExtension(file.metadata.title);
const motionPhoto = await decodeMotionPhoto(fileBlob, originalName);
let image = new Blob([motionPhoto.image]);
// can run conversion in parellel as video and image
// have different processes
const convertedVideo = ffmpegService.convertToMP4(
motionPhoto.video,
motionPhoto.videoNameTitle
);
image = await convertIfHEIC(motionPhoto.imageNameTitle, image);
const video = new Blob([await convertedVideo]);
return [image, video];
}
fileBlob = await convertIfHEIC(file.metadata.title, fileBlob);
return [fileBlob];
}
export function fileIsArchived(file: EnteFile) {

View file

@ -13,3 +13,22 @@ export async function isPlaybackPossible(url: string): Promise<boolean> {
video.src = url;
});
}
export async function playVideo(livePhotoVideo, livePhotoImage) {
const videoPlaying = !livePhotoVideo.paused;
if (videoPlaying) return;
livePhotoVideo.style.opacity = 1;
livePhotoImage.style.opacity = 0;
livePhotoVideo.load();
livePhotoVideo.play().catch(() => {
pauseVideo(livePhotoVideo, livePhotoImage);
});
}
export async function pauseVideo(livePhotoVideo, livePhotoImage) {
const videoPlaying = !livePhotoVideo.paused;
if (!videoPlaying) return;
livePhotoVideo.pause();
livePhotoVideo.style.opacity = 0;
livePhotoImage.style.opacity = 1;
}

View file

@ -673,6 +673,7 @@ const englishConstants = {
ENTE_IO: 'ente.io',
PLAYBACK_SUPPORT_COMING: 'playback support coming soon...',
LIVE_PHOTO: 'this is a live photo',
LIVE: 'LIVE',
};
export default englishConstants;