From 60fd87df8f67491806a5a4bb6f9451718905b48d Mon Sep 17 00:00:00 2001 From: Rushikesh Tote Date: Thu, 24 Feb 2022 19:26:57 +0530 Subject: [PATCH 01/45] added live photo playback support --- src/components/PhotoFrame.tsx | 12 ++++++++++- src/services/ffmpegService.ts | 32 ++++++++++++++++++++++++++++++ src/services/motionPhotoService.ts | 6 ++++-- src/utils/file/index.ts | 2 +- 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/components/PhotoFrame.tsx b/src/components/PhotoFrame.tsx index b5ab73aa3..c94e62857 100644 --- a/src/components/PhotoFrame.tsx +++ b/src/components/PhotoFrame.tsx @@ -250,7 +250,8 @@ const PhotoFrame = ({ h: window.innerHeight, }; if ( - files[index].metadata.fileType === FILE_TYPE.VIDEO && + (files[index].metadata.fileType === FILE_TYPE.VIDEO || + files[index].metadata.fileType === FILE_TYPE.LIVE_PHOTO) && !files[index].html ) { files[index].html = ` @@ -297,6 +298,15 @@ const PhotoFrame = ({ `; } + } else if (files[index].metadata.fileType === FILE_TYPE.LIVE_PHOTO) { + if (await isPlaybackPossible(url)) { + files[index].html = ` + + `; + } } else { files[index].src = url; } diff --git a/src/services/ffmpegService.ts b/src/services/ffmpegService.ts index 3914cfea2..f0ad2c17e 100644 --- a/src/services/ffmpegService.ts +++ b/src/services/ffmpegService.ts @@ -56,6 +56,38 @@ class FFmpegService { } } } + + async convertLivePhotoToMP4(file: Uint8Array): Promise { + if (!this.ffmpeg) { + await this.init(); + } + if (this.isLoading) { + await this.isLoading; + } + + try { + this.ffmpeg.FS('writeFile', 'input.mov', file); + console.log('starting encoding', new Date().toLocaleTimeString()); + await this.ffmpeg.run( + '-i', + 'input.mov', + '-preset', + 'ultrafast', + 'output.mp4' + ); + const convertedFile = await this.ffmpeg.FS( + 'readFile', + 'output.mp4' + ); + console.log('done encoding', new Date().toLocaleTimeString()); + await this.ffmpeg.FS('unlink', 'input.mov'); + await this.ffmpeg.FS('unlink', 'output.mp4'); + return convertedFile; + } catch (e) { + logError(e, 'ffmpeg live photo to MP4 conversion failed'); + throw e; + } + } } async function generateThumbnailHelper( diff --git a/src/services/motionPhotoService.ts b/src/services/motionPhotoService.ts index 9eacd2cc7..9abd2c3a4 100644 --- a/src/services/motionPhotoService.ts +++ b/src/services/motionPhotoService.ts @@ -1,5 +1,6 @@ import JSZip from 'jszip'; import { fileExtensionWithDot } from 'utils/file'; +import FFmpegService from 'services/ffmpegService'; class MotionPhoto { image: Uint8Array; @@ -25,8 +26,9 @@ export const decodeMotionPhoto = async ( } else if (zipFilename.startsWith('video')) { motionPhoto.videoNameTitle = originalName + fileExtensionWithDot(zipFilename); - motionPhoto.video = await zip.files[zipFilename].async( - 'uint8array' + const video = await zip.files[zipFilename].async('uint8array'); + motionPhoto.video = await FFmpegService.convertLivePhotoToMP4( + video ); } } diff --git a/src/utils/file/index.ts b/src/utils/file/index.ts index b56b519d8..2bf619f70 100644 --- a/src/utils/file/index.ts +++ b/src/utils/file/index.ts @@ -333,7 +333,7 @@ export async function convertForPreview(file: EnteFile, fileBlob: Blob) { if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { const originalName = fileNameWithoutExtension(file.metadata.title); const motionPhoto = await decodeMotionPhoto(fileBlob, originalName); - fileBlob = new Blob([motionPhoto.image]); + fileBlob = new Blob([motionPhoto.video]); } const typeFromExtension = getFileExtension(file.metadata.title); From 7ab43fb2f28b9b52dd2117388002f4471eab4767 Mon Sep 17 00:00:00 2001 From: Rushikesh Tote Date: Fri, 25 Feb 2022 16:49:55 +0530 Subject: [PATCH 02/45] added hover to play live photo --- src/components/PhotoFrame.tsx | 59 ++++++++++++++++++++++++++++++++--- src/pages/_app.tsx | 19 +++++++++++ 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/src/components/PhotoFrame.tsx b/src/components/PhotoFrame.tsx index c94e62857..662706020 100644 --- a/src/components/PhotoFrame.tsx +++ b/src/components/PhotoFrame.tsx @@ -101,6 +101,7 @@ const PhotoFrame = ({ const filteredDataRef = useRef([]); const filteredData = filteredDataRef?.current ?? []; const router = useRouter(); + const [livePhotoLoaded, setLivePhotoLoaded] = useState(false); useEffect(() => { const handleKeyDown = (e: KeyboardEvent) => { if (e.key === 'Shift') { @@ -241,6 +242,28 @@ const PhotoFrame = ({ } }, [open]); + useEffect(() => { + const livePhotoVideo: HTMLVideoElement = + document.querySelector('video'); + const livePhotoBtn: HTMLButtonElement = + document.querySelector('.live-photo-btn'); + if (livePhotoVideo && livePhotoBtn) { + livePhotoBtn.addEventListener('mouseout', function () { + livePhotoVideo.pause(); + }); + livePhotoBtn.addEventListener('mouseover', function () { + livePhotoVideo.play().catch(() => { + livePhotoVideo.pause(); + }); + }); + livePhotoBtn.addEventListener('click', function () { + livePhotoVideo.play().catch(() => { + livePhotoVideo.pause(); + }); + }); + } + }, [livePhotoLoaded]); + const updateURL = (index: number) => (url: string) => { files[index] = { ...files[index], @@ -301,11 +324,34 @@ const PhotoFrame = ({ } else if (files[index].metadata.fileType === FILE_TYPE.LIVE_PHOTO) { if (await isPlaybackPossible(url)) { files[index].html = ` - - `; +
+ + +
+ `; + setLivePhotoLoaded(true); + } else { + files[index].html = ` +
+ +
+ ${constants.VIDEO_PLAYBACK_FAILED_DOWNLOAD_INSTEAD} + Download +
+
+ `; } } else { files[index].src = url; @@ -462,6 +508,9 @@ const PhotoFrame = ({ } galleryContext.files.set(item.id, url); } + if (item.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { + setLivePhotoLoaded(false); + } await updateSrcURL(item.dataIndex, url); item.html = files[item.dataIndex].html; item.src = files[item.dataIndex].src; diff --git a/src/pages/_app.tsx b/src/pages/_app.tsx index 031791c39..b4ea3cd43 100644 --- a/src/pages/_app.tsx +++ b/src/pages/_app.tsx @@ -69,6 +69,25 @@ const GlobalStyles = createGlobalStyle` height: 100%; } + .live-photo-container{ + width: 100%; + height: 100%; + position: relative; + } + + .live-photo-container > button { + position: absolute; + bottom: 10vh; + left: calc(50% - 20px); + height: 40px; + width: 80px; + background: #d7d7d7; + outline: none; + border: none; + border-radius: 10%; + z-index: 10; + } + .video-loading { width: 100%; height: 100%; From b9a238c58326ccfefd6376b3e2c3e172caab45a7 Mon Sep 17 00:00:00 2001 From: Rushikesh Tote Date: Fri, 25 Feb 2022 17:45:56 +0530 Subject: [PATCH 03/45] shifted live photo button to top left --- src/pages/_app.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pages/_app.tsx b/src/pages/_app.tsx index b4ea3cd43..cba3b6387 100644 --- a/src/pages/_app.tsx +++ b/src/pages/_app.tsx @@ -77,8 +77,8 @@ const GlobalStyles = createGlobalStyle` .live-photo-container > button { position: absolute; - bottom: 10vh; - left: calc(50% - 20px); + top: 6vh; + left: 4vh; height: 40px; width: 80px; background: #d7d7d7; From 34371d4928138fd11ae6ef7a56c247fa17f7bd00 Mon Sep 17 00:00:00 2001 From: Rushikesh Tote Date: Fri, 25 Feb 2022 18:57:12 +0530 Subject: [PATCH 04/45] added looping --- src/components/PhotoFrame.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/PhotoFrame.tsx b/src/components/PhotoFrame.tsx index 662706020..259beb112 100644 --- a/src/components/PhotoFrame.tsx +++ b/src/components/PhotoFrame.tsx @@ -335,7 +335,7 @@ const PhotoFrame = ({ LIVE -
${constants.VIDEO_PLAYBACK_FAILED_DOWNLOAD_INSTEAD} - Download + Download
`; @@ -510,12 +516,32 @@ const PhotoFrame = ({ } else { url = await DownloadManager.getFile(item, true); } - galleryContext.files.set(item.id, url); + if (item.metadata.fileType !== FILE_TYPE.LIVE_PHOTO) { + galleryContext.files.set(item.id, url); + } } if (item.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { + let imageURL; + let videoURL; + if (galleryContext.files.has(item.id)) { + [imageURL, videoURL] = galleryContext.files + .get(item.id) + .split(','); + } else { + ({ imageURL, videoURL } = await splitLivePhoto( + item, + url + )); + galleryContext.files.set( + item.id, + imageURL + ',' + videoURL + ); + } setLivePhotoLoaded(false); + await updateSrcURL(item.dataIndex, { imageURL, videoURL }); + } else { + await updateSrcURL(item.dataIndex, { defaultURL: url }); } - await updateSrcURL(item.dataIndex, url); item.html = files[item.dataIndex].html; item.src = files[item.dataIndex].src; item.w = files[item.dataIndex].w; diff --git a/src/utils/file/index.ts b/src/utils/file/index.ts index 90f3fd7f6..7b35a0957 100644 --- a/src/utils/file/index.ts +++ b/src/utils/file/index.ts @@ -333,9 +333,7 @@ export function generateStreamFromArrayBuffer(data: Uint8Array) { export async function convertForPreview(file: EnteFile, fileBlob: Blob) { if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { - const originalName = fileNameWithoutExtension(file.metadata.title); - const motionPhoto = await decodeMotionPhoto(fileBlob, originalName); - fileBlob = new Blob([motionPhoto.video]); + return fileBlob; } const typeFromExtension = getFileExtension(file.metadata.title); @@ -350,6 +348,26 @@ export async function convertForPreview(file: EnteFile, fileBlob: Blob) { return fileBlob; } +export async function splitLivePhoto(file: EnteFile, url: string) { + const fileBlob = await fetch(url).then((res) => res.blob()); + const originalName = fileNameWithoutExtension(file.metadata.title); + const motionPhoto = await decodeMotionPhoto(fileBlob, originalName); + let image = new Blob([motionPhoto.image]); + const video = new Blob([motionPhoto.video]); + + const typeFromExtension = getFileExtension(motionPhoto.imageNameTitle); + const reader = new FileReader(); + + const mimeType = + (await getFileTypeFromBlob(reader, image))?.mime ?? typeFromExtension; + if (isFileHEIC(mimeType)) { + image = await HEICConverter.convert(image); + } + const imageURL = URL.createObjectURL(image); + const videoURL = URL.createObjectURL(video); + return { imageURL, videoURL }; +} + export function fileIsArchived(file: EnteFile) { if ( !file || diff --git a/src/utils/photoFrame/index.ts b/src/utils/photoFrame/index.ts index fdaeb8557..0ce17437f 100644 --- a/src/utils/photoFrame/index.ts +++ b/src/utils/photoFrame/index.ts @@ -13,3 +13,16 @@ export async function isPlaybackPossible(url: string): Promise { video.src = url; }); } + +export const livePhotoBtnHTML = ` + +`; From 6188c9eed4208b0187616e6beb4d1890f5c4dfa1 Mon Sep 17 00:00:00 2001 From: Rushikesh Tote Date: Sun, 27 Feb 2022 12:04:51 +0530 Subject: [PATCH 06/45] fixed issues with video conversion --- src/services/ffmpegService.ts | 61 ++++++++++++++++++++---------- src/services/motionPhotoService.ts | 6 +-- src/utils/file/index.ts | 12 +++++- 3 files changed, 54 insertions(+), 25 deletions(-) diff --git a/src/services/ffmpegService.ts b/src/services/ffmpegService.ts index f0ad2c17e..f7e08ae02 100644 --- a/src/services/ffmpegService.ts +++ b/src/services/ffmpegService.ts @@ -10,6 +10,7 @@ class FFmpegService { private fileReader: FileReader = null; private generateThumbnailProcessor = new QueueProcessor(1); + private generateMP4ConversionProcessor = new QueueProcessor(1); async init() { try { this.ffmpeg = createFFmpeg({ @@ -57,7 +58,10 @@ class FFmpegService { } } - async convertLivePhotoToMP4(file: Uint8Array): Promise { + async convertToMP4( + file: Uint8Array, + fileName: string + ): Promise { if (!this.ffmpeg) { await this.init(); } @@ -65,27 +69,20 @@ class FFmpegService { await this.isLoading; } + const response = this.generateMP4ConversionProcessor.queueUpRequest( + convertToMP4Helper.bind(null, this.ffmpeg, file, fileName) + ); + try { - this.ffmpeg.FS('writeFile', 'input.mov', file); - console.log('starting encoding', new Date().toLocaleTimeString()); - await this.ffmpeg.run( - '-i', - 'input.mov', - '-preset', - 'ultrafast', - 'output.mp4' - ); - const convertedFile = await this.ffmpeg.FS( - 'readFile', - 'output.mp4' - ); - console.log('done encoding', new Date().toLocaleTimeString()); - await this.ffmpeg.FS('unlink', 'input.mov'); - await this.ffmpeg.FS('unlink', 'output.mp4'); - return convertedFile; + return await response.promise; } catch (e) { - logError(e, 'ffmpeg live photo to MP4 conversion failed'); - throw e; + if (e.message === CustomError.REQUEST_CANCELLED) { + // ignore + return null; + } else { + logError(e, 'ffmpeg MP4 conversion failed'); + throw e; + } } } } @@ -133,4 +130,28 @@ async function generateThumbnailHelper( } } +async function convertToMP4Helper( + ffmpeg: FFmpeg, + file: Uint8Array, + inputFileName: string +) { + try { + ffmpeg.FS('writeFile', inputFileName, file); + await ffmpeg.run( + '-i', + inputFileName, + '-preset', + 'ultrafast', + 'output.mp4' + ); + const convertedFile = await ffmpeg.FS('readFile', 'output.mp4'); + ffmpeg.FS('unlink', inputFileName); + ffmpeg.FS('unlink', 'output.mp4'); + return convertedFile; + } catch (e) { + logError(e, 'ffmpeg MP4 conversion failed'); + throw e; + } +} + export default new FFmpegService(); diff --git a/src/services/motionPhotoService.ts b/src/services/motionPhotoService.ts index 9abd2c3a4..9eacd2cc7 100644 --- a/src/services/motionPhotoService.ts +++ b/src/services/motionPhotoService.ts @@ -1,6 +1,5 @@ import JSZip from 'jszip'; import { fileExtensionWithDot } from 'utils/file'; -import FFmpegService from 'services/ffmpegService'; class MotionPhoto { image: Uint8Array; @@ -26,9 +25,8 @@ export const decodeMotionPhoto = async ( } else if (zipFilename.startsWith('video')) { motionPhoto.videoNameTitle = originalName + fileExtensionWithDot(zipFilename); - const video = await zip.files[zipFilename].async('uint8array'); - motionPhoto.video = await FFmpegService.convertLivePhotoToMP4( - video + motionPhoto.video = await zip.files[zipFilename].async( + 'uint8array' ); } } diff --git a/src/utils/file/index.ts b/src/utils/file/index.ts index 7b35a0957..378e68a52 100644 --- a/src/utils/file/index.ts +++ b/src/utils/file/index.ts @@ -24,6 +24,7 @@ import { } from 'constants/file'; import PublicCollectionDownloadManager from 'services/publicCollectionDownloadManager'; import HEICConverter from 'services/HEICConverter'; +import ffmpegService from 'services/ffmpegService'; export function downloadAsFile(filename: string, content: string) { const file = new Blob([content], { @@ -353,7 +354,13 @@ export async function splitLivePhoto(file: EnteFile, url: string) { const originalName = fileNameWithoutExtension(file.metadata.title); const motionPhoto = await decodeMotionPhoto(fileBlob, originalName); let image = new Blob([motionPhoto.image]); - const video = new Blob([motionPhoto.video]); + + // can run conversion in parellel as video and image + // have different processes + const convertedVideo = ffmpegService.convertToMP4( + motionPhoto.video, + motionPhoto.videoNameTitle + ); const typeFromExtension = getFileExtension(motionPhoto.imageNameTitle); const reader = new FileReader(); @@ -363,6 +370,9 @@ export async function splitLivePhoto(file: EnteFile, url: string) { if (isFileHEIC(mimeType)) { image = await HEICConverter.convert(image); } + + const video = new Blob([await convertedVideo]); + const imageURL = URL.createObjectURL(image); const videoURL = URL.createObjectURL(video); return { imageURL, videoURL }; From 1eee5e8a74678971ee11018260fb8da3bd79b14d Mon Sep 17 00:00:00 2001 From: Rushikesh Tote Date: Mon, 28 Feb 2022 10:04:20 +0530 Subject: [PATCH 07/45] fixed: one ffmpeg process at any time --- src/services/ffmpegService.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/services/ffmpegService.ts b/src/services/ffmpegService.ts index f7e08ae02..7b3a25d64 100644 --- a/src/services/ffmpegService.ts +++ b/src/services/ffmpegService.ts @@ -9,8 +9,7 @@ class FFmpegService { private isLoading = null; private fileReader: FileReader = null; - private generateThumbnailProcessor = new QueueProcessor(1); - private generateMP4ConversionProcessor = new QueueProcessor(1); + private generateQueueProcessor = new QueueProcessor(1); async init() { try { this.ffmpeg = createFFmpeg({ @@ -37,7 +36,7 @@ class FFmpegService { if (this.isLoading) { await this.isLoading; } - const response = this.generateThumbnailProcessor.queueUpRequest( + const response = this.generateQueueProcessor.queueUpRequest( generateThumbnailHelper.bind( null, this.ffmpeg, @@ -69,7 +68,7 @@ class FFmpegService { await this.isLoading; } - const response = this.generateMP4ConversionProcessor.queueUpRequest( + const response = this.generateQueueProcessor.queueUpRequest( convertToMP4Helper.bind(null, this.ffmpeg, file, fileName) ); From 1af2dc24ca22401de677e7c6d9f7e9069f989712 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Thu, 24 Feb 2022 10:09:53 +0530 Subject: [PATCH 08/45] add metadata extraction logic --- src/services/ffmpegService.ts | 81 +++++++++++++++++++++++++- src/services/upload/metadataService.ts | 69 ++++++++++++++++++++-- 2 files changed, 142 insertions(+), 8 deletions(-) diff --git a/src/services/ffmpegService.ts b/src/services/ffmpegService.ts index 3914cfea2..72c835b9a 100644 --- a/src/services/ffmpegService.ts +++ b/src/services/ffmpegService.ts @@ -2,6 +2,10 @@ import { createFFmpeg, FFmpeg } from '@ffmpeg/ffmpeg'; import { CustomError } from 'utils/error'; import { logError } from 'utils/sentry'; import QueueProcessor from './queueProcessor'; +import { + ParsedVideoMetadata, + parseFFmpegExtractedMetadata, +} from './upload/metadataService'; import { getUint8ArrayView } from './upload/readFileService'; class FFmpegService { @@ -9,7 +13,7 @@ class FFmpegService { private isLoading = null; private fileReader: FileReader = null; - private generateThumbnailProcessor = new QueueProcessor(1); + private ffmpegTaskQueue = new QueueProcessor(1); async init() { try { this.ffmpeg = createFFmpeg({ @@ -26,7 +30,7 @@ class FFmpegService { } } - async generateThumbnail(file: File) { + async generateThumbnail(file: File): Promise { if (!this.ffmpeg) { await this.init(); } @@ -36,7 +40,7 @@ class FFmpegService { if (this.isLoading) { await this.isLoading; } - const response = this.generateThumbnailProcessor.queueUpRequest( + const response = this.ffmpegTaskQueue.queueUpRequest( generateThumbnailHelper.bind( null, this.ffmpeg, @@ -56,6 +60,37 @@ class FFmpegService { } } } + + async extractMetadata(file: File): Promise { + if (!this.ffmpeg) { + await this.init(); + } + if (!this.fileReader) { + this.fileReader = new FileReader(); + } + if (this.isLoading) { + await this.isLoading; + } + const response = this.ffmpegTaskQueue.queueUpRequest( + extractVideoMetadataHelper.bind( + null, + this.ffmpeg, + this.fileReader, + file + ) + ); + try { + return await response.promise; + } catch (e) { + if (e.message === CustomError.REQUEST_CANCELLED) { + // ignore + return null; + } else { + logError(e, 'ffmpeg metadata extraction failed'); + throw e; + } + } + } } async function generateThumbnailHelper( @@ -101,4 +136,44 @@ async function generateThumbnailHelper( } } +async function extractVideoMetadataHelper( + ffmpeg: FFmpeg, + reader: FileReader, + file: File +) { + try { + const inputFileName = `${Date.now().toString()}-${file.name}`; + const outFileName = `${Date.now().toString()}-metadata.txt`; + ffmpeg.FS( + 'writeFile', + inputFileName, + await getUint8ArrayView(reader, file) + ); + let metadata = null; + + await ffmpeg.run( + '-i', + inputFileName, + '-c', + 'copy', + '-map_metadata', + '0', + '-map_metadata:s:v', + '0:s:v', + '-map_metadata:s:a', + '0:s:a', + '-f', + 'ffmetadata', + outFileName + ); + metadata = ffmpeg.FS('readFile', outFileName); + ffmpeg.FS('unlink', outFileName); + ffmpeg.FS('unlink', inputFileName); + return parseFFmpegExtractedMetadata(metadata); + } catch (e) { + logError(e, 'ffmpeg metadata extraction failed'); + throw e; + } +} + export default new FFmpegService(); diff --git a/src/services/upload/metadataService.ts b/src/services/upload/metadataService.ts index 89564aee5..db4b4469c 100644 --- a/src/services/upload/metadataService.ts +++ b/src/services/upload/metadataService.ts @@ -1,6 +1,6 @@ import { FILE_TYPE } from 'constants/file'; import { logError } from 'utils/sentry'; -import { getExifData } from './exifService'; +import { getExifData, ParsedEXIFData } from './exifService'; import { Metadata, ParsedMetadataJSON, @@ -9,6 +9,15 @@ import { } from 'types/upload'; import { NULL_LOCATION } from 'constants/upload'; import { splitFilenameAndExtension } from 'utils/file'; +import ffmpegService from 'services/ffmpegService'; + +enum VideoMetadata { + CREATION_TIME = 'creation_time', + APPLE_CONTENT_IDENTIFIER = 'com.apple.quicktime.content.identifier', + APPLE_LIVE_PHOTO_IDENTIFIER = 'com.apple.quicktime.live-photo.auto', + APPLE_CREATION_DATE = 'com.apple.quicktime.creationdate', + APPLE_LOCATION_ISO = 'com.apple.quicktime.location.ISO6709', +} interface ParsedMetadataJSONWithTitle { title: string; @@ -21,13 +30,21 @@ const NULL_PARSED_METADATA_JSON: ParsedMetadataJSON = { ...NULL_LOCATION, }; +export interface ParsedVideoMetadata { + location: Location; + creationTime: number; +} + export async function extractMetadata( receivedFile: File, fileTypeInfo: FileTypeInfo ) { - let exifData = null; + let exifData: ParsedEXIFData = null; + let videoMetadata: ParsedVideoMetadata = null; if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) { exifData = await getExifData(receivedFile, fileTypeInfo); + } else if (fileTypeInfo.fileType === FILE_TYPE.VIDEO) { + videoMetadata = await ffmpegService.extractMetadata(receivedFile); } const extractedMetadata: Metadata = { @@ -35,10 +52,14 @@ export async function extractMetadata( fileTypeInfo.exactType }`, creationTime: - exifData?.creationTime ?? receivedFile.lastModified * 1000, + exifData?.creationTime ?? + videoMetadata.creationTime ?? + receivedFile.lastModified * 1000, modificationTime: receivedFile.lastModified * 1000, - latitude: exifData?.location?.latitude, - longitude: exifData?.location?.longitude, + latitude: + exifData?.location?.latitude ?? videoMetadata.location?.latitude, + longitude: + exifData?.location?.longitude ?? videoMetadata.location?.longitude, fileType: fileTypeInfo.fileType, }; return extractedMetadata; @@ -119,3 +140,41 @@ export async function parseMetadataJSON( // ignore } } + +export function parseFFmpegExtractedMetadata(metadata: Uint8Array) { + const metadataString = new TextDecoder().decode(metadata); + const metadataPropertyArray = metadataString.split('\n'); + const metadataKeyValueArray = metadataPropertyArray.map((property) => + property.split('=') + ); + const validKeyValuePairs = metadataKeyValueArray.filter( + (keyValueArray) => keyValueArray.length === 2 + ) as Array<[string, string]>; + + const metadataMap = new Map(validKeyValuePairs); + + const location = parseAppleISOLocation( + metadata[VideoMetadata.APPLE_LOCATION_ISO] + ); + + const parsedMetadata: ParsedVideoMetadata = { + creationTime: + metadataMap[VideoMetadata.APPLE_CREATION_DATE] || + metadataMap[VideoMetadata.CREATION_TIME], + location: { + latitude: location.latitude, + longitude: location.longitude, + }, + }; + return parsedMetadata; +} + +function parseAppleISOLocation(isoLocation: string) { + if (isoLocation) { + const [latitude, longitude, altitude] = isoLocation + .match(/(\+|-)\d+\.*\d+/g) + .map((x) => parseFloat(x)); + + return { latitude, longitude, altitude }; + } +} From 540620373b3fde6f79a1283b0ce6da0a602e76eb Mon Sep 17 00:00:00 2001 From: Abhinav Date: Thu, 24 Feb 2022 17:10:41 +0530 Subject: [PATCH 09/45] fix minor bugs --- src/services/upload/exifService.ts | 2 +- src/services/upload/metadataService.ts | 38 +++++++++++++++++++------- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts index 685994235..81dee1aec 100644 --- a/src/services/upload/exifService.ts +++ b/src/services/upload/exifService.ts @@ -23,7 +23,7 @@ interface Exif { GPSLatitudeRef?: number; GPSLongitudeRef?: number; } -interface ParsedEXIFData { +export interface ParsedEXIFData { location: Location; creationTime: number; } diff --git a/src/services/upload/metadataService.ts b/src/services/upload/metadataService.ts index db4b4469c..06ec15e92 100644 --- a/src/services/upload/metadataService.ts +++ b/src/services/upload/metadataService.ts @@ -1,6 +1,6 @@ import { FILE_TYPE } from 'constants/file'; import { logError } from 'utils/sentry'; -import { getExifData, ParsedEXIFData } from './exifService'; +import { getExifData, getUNIXTime, ParsedEXIFData } from './exifService'; import { Metadata, ParsedMetadataJSON, @@ -141,8 +141,8 @@ export async function parseMetadataJSON( } } -export function parseFFmpegExtractedMetadata(metadata: Uint8Array) { - const metadataString = new TextDecoder().decode(metadata); +export function parseFFmpegExtractedMetadata(encodedMetadata: Uint8Array) { + const metadataString = new TextDecoder().decode(encodedMetadata); const metadataPropertyArray = metadataString.split('\n'); const metadataKeyValueArray = metadataPropertyArray.map((property) => property.split('=') @@ -151,16 +151,18 @@ export function parseFFmpegExtractedMetadata(metadata: Uint8Array) { (keyValueArray) => keyValueArray.length === 2 ) as Array<[string, string]>; - const metadataMap = new Map(validKeyValuePairs); + const metadataMap = Object.fromEntries(validKeyValuePairs); const location = parseAppleISOLocation( - metadata[VideoMetadata.APPLE_LOCATION_ISO] + metadataMap[VideoMetadata.APPLE_LOCATION_ISO] ); + const creationTime = parseCreationTime( + metadataMap[VideoMetadata.APPLE_CREATION_DATE] ?? + metadataMap[VideoMetadata.CREATION_TIME] + ); const parsedMetadata: ParsedVideoMetadata = { - creationTime: - metadataMap[VideoMetadata.APPLE_CREATION_DATE] || - metadataMap[VideoMetadata.CREATION_TIME], + creationTime, location: { latitude: location.latitude, longitude: location.longitude, @@ -170,11 +172,27 @@ export function parseFFmpegExtractedMetadata(metadata: Uint8Array) { } function parseAppleISOLocation(isoLocation: string) { + let location = NULL_LOCATION; if (isoLocation) { - const [latitude, longitude, altitude] = isoLocation + const [latitude, longitude] = isoLocation .match(/(\+|-)\d+\.*\d+/g) .map((x) => parseFloat(x)); - return { latitude, longitude, altitude }; + location = { latitude, longitude }; } + return location; +} + +function parseCreationTime(creationTime: string) { + let dateTime = null; + if (creationTime) { + dateTime = new Date(creationTime); + if (isNaN(dateTime.getTime())) { + dateTime = null; + } + } + if (dateTime) { + dateTime = getUNIXTime(dateTime); + } + return dateTime; } From e08f416b8df941eb41e40447a8c055a2c91f419e Mon Sep 17 00:00:00 2001 From: Rushikesh Tote Date: Mon, 28 Feb 2022 10:51:55 +0530 Subject: [PATCH 10/45] added live photo button on loading --- src/components/PhotoFrame.tsx | 23 ++++++++++++++++++++--- src/utils/photoFrame/index.ts | 2 -- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/components/PhotoFrame.tsx b/src/components/PhotoFrame.tsx index 5884be86e..56755ab33 100644 --- a/src/components/PhotoFrame.tsx +++ b/src/components/PhotoFrame.tsx @@ -288,8 +288,7 @@ const PhotoFrame = ({ h: window.innerHeight, }; if ( - (files[index].metadata.fileType === FILE_TYPE.VIDEO || - files[index].metadata.fileType === FILE_TYPE.LIVE_PHOTO) && + files[index].metadata.fileType === FILE_TYPE.VIDEO && !files[index].html ) { files[index].html = ` @@ -301,6 +300,22 @@ const PhotoFrame = ({ `; delete files[index].src; + } else if ( + files[index].metadata.fileType === FILE_TYPE.LIVE_PHOTO && + !files[index].html + ) { + files[index].html = ` +
+ + + +
+ `; + delete files[index].src; } if ( files[index].metadata.fileType === FILE_TYPE.IMAGE && @@ -342,7 +357,9 @@ const PhotoFrame = ({ if (await isPlaybackPossible(videoURL)) { files[index].html = `
- ${livePhotoBtnHTML} + paypal@ente.io to manage your + paypal plan + + ), + PAYPAL_MANAGE_NOT_SUPPORTED: 'manage paypal plan', RENAME: 'rename', RENAME_COLLECTION: 'rename album', CONFIRM_DELETE_COLLECTION: 'confirm album deletion', From a866d109e5bc25ce5ca160997062660ba29f5e15 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 28 Feb 2022 17:44:23 +0530 Subject: [PATCH 19/45] rename getUNIXTime to getUnixTimeInMicroSeconds --- src/services/updateCreationTimeWithExif.ts | 10 ++++++---- src/services/upload/exifService.ts | 4 ++-- src/services/upload/videoMetadataService.ts | 4 ++-- src/utils/time/index.ts | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/services/updateCreationTimeWithExif.ts b/src/services/updateCreationTimeWithExif.ts index 4d4b6bc27..e20bfd16c 100644 --- a/src/services/updateCreationTimeWithExif.ts +++ b/src/services/updateCreationTimeWithExif.ts @@ -13,7 +13,7 @@ import { EnteFile } from 'types/file'; import { getRawExif } from './upload/exifService'; import { getFileType } from './upload/readFileService'; import { FILE_TYPE } from 'constants/file'; -import { getUNIXTime } from 'utils/time'; +import { getUnixTimeInMicroSeconds } from 'utils/time'; export async function updateCreationTimeWithExif( filesToBeUpdated: EnteFile[], @@ -34,7 +34,7 @@ export async function updateCreationTimeWithExif( } let correctCreationTime: number; if (fixOption === FIX_OPTIONS.CUSTOM_TIME) { - correctCreationTime = getUNIXTime(customTime); + correctCreationTime = getUnixTimeInMicroSeconds(customTime); } else { const fileURL = await downloadManager.getFile(file); const fileObject = await getFileFromURL(fileURL); @@ -42,11 +42,13 @@ export async function updateCreationTimeWithExif( const fileTypeInfo = await getFileType(reader, fileObject); const exifData = await getRawExif(fileObject, fileTypeInfo); if (fixOption === FIX_OPTIONS.DATE_TIME_ORIGINAL) { - correctCreationTime = getUNIXTime( + correctCreationTime = getUnixTimeInMicroSeconds( exifData?.DateTimeOriginal ); } else { - correctCreationTime = getUNIXTime(exifData?.CreateDate); + correctCreationTime = getUnixTimeInMicroSeconds( + exifData?.CreateDate + ); } } if ( diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts index 5144c1307..ee81da515 100644 --- a/src/services/upload/exifService.ts +++ b/src/services/upload/exifService.ts @@ -5,7 +5,7 @@ import piexif from 'piexifjs'; import { FileTypeInfo } from 'types/upload'; import { logError } from 'utils/sentry'; import { ParsedExtractedMetadata } from 'types/upload'; -import { getUNIXTime } from 'utils/time'; +import { getUnixTimeInMicroSeconds } from 'utils/time'; const EXIF_TAGS_NEEDED = [ 'DateTimeOriginal', @@ -38,7 +38,7 @@ export async function getExifData( } parsedEXIFData = { location: getEXIFLocation(exifData), - creationTime: getUNIXTime( + creationTime: getUnixTimeInMicroSeconds( exifData.DateTimeOriginal ?? exifData.CreateDate ?? exifData.ModifyDate diff --git a/src/services/upload/videoMetadataService.ts b/src/services/upload/videoMetadataService.ts index 1b5e2e791..9e124d51d 100644 --- a/src/services/upload/videoMetadataService.ts +++ b/src/services/upload/videoMetadataService.ts @@ -1,6 +1,6 @@ import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload'; import ffmpegService from 'services/ffmpegService'; -import { getUNIXTime } from 'utils/time'; +import { getUnixTimeInMicroSeconds } from 'utils/time'; import { ParsedExtractedMetadata } from 'types/upload'; import { logError } from 'utils/sentry'; @@ -68,7 +68,7 @@ function parseAppleISOLocation(isoLocation: string) { function parseCreationTime(creationTime: string) { let dateTime = null; if (creationTime) { - dateTime = getUNIXTime(new Date(creationTime)); + dateTime = getUnixTimeInMicroSeconds(new Date(creationTime)); } return dateTime; } diff --git a/src/utils/time/index.ts b/src/utils/time/index.ts index cb04e5b65..65e15fbce 100644 --- a/src/utils/time/index.ts +++ b/src/utils/time/index.ts @@ -1,4 +1,4 @@ -export function getUNIXTime(dateTime: Date) { +export function getUnixTimeInMicroSeconds(dateTime: Date) { if (!dateTime || isNaN(dateTime.getTime())) { return null; } From ca1369168840da82d6d136c1692073cef7d95ec6 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 28 Feb 2022 18:27:44 +0530 Subject: [PATCH 20/45] extractedMetadata will allows have the properties so no need of conditional access --- src/services/upload/metadataService.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/services/upload/metadataService.ts b/src/services/upload/metadataService.ts index 2b55a441a..51985e460 100644 --- a/src/services/upload/metadataService.ts +++ b/src/services/upload/metadataService.ts @@ -8,7 +8,7 @@ import { FileTypeInfo, ParsedExtractedMetadata, } from 'types/upload'; -import { NULL_LOCATION } from 'constants/upload'; +import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload'; import { splitFilenameAndExtension } from 'utils/file'; import { getVideoMetadata } from './videoMetadataService'; @@ -27,7 +27,7 @@ export async function extractMetadata( receivedFile: File, fileTypeInfo: FileTypeInfo ) { - let extractedMetadata: ParsedExtractedMetadata = null; + let extractedMetadata: ParsedExtractedMetadata = NULL_EXTRACTED_METADATA; if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) { extractedMetadata = await getExifData(receivedFile, fileTypeInfo); } else if (fileTypeInfo.fileType === FILE_TYPE.VIDEO) { @@ -39,10 +39,10 @@ export async function extractMetadata( fileTypeInfo.exactType }`, creationTime: - extractedMetadata?.creationTime ?? receivedFile.lastModified * 1000, + extractedMetadata.creationTime ?? receivedFile.lastModified * 1000, modificationTime: receivedFile.lastModified * 1000, - latitude: extractedMetadata.location?.latitude, - longitude: extractedMetadata.location?.longitude, + latitude: extractedMetadata.location.latitude, + longitude: extractedMetadata.location.longitude, fileType: fileTypeInfo.fileType, }; return metadata; From 7144618241c8af67e3aa1d2adc51979caa67a48b Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 28 Feb 2022 20:40:54 +0530 Subject: [PATCH 21/45] update string --- src/utils/strings/englishConstants.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/strings/englishConstants.tsx b/src/utils/strings/englishConstants.tsx index fc0e20204..2d73d4b0a 100644 --- a/src/utils/strings/englishConstants.tsx +++ b/src/utils/strings/englishConstants.tsx @@ -336,7 +336,7 @@ const englishConstants = { <> please contact us at{' '} paypal@ente.io to manage your - paypal plan + subscription ), PAYPAL_MANAGE_NOT_SUPPORTED: 'manage paypal plan', From b95223185d8bc00b2814c786f9b97ae1dd298ba8 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 1 Mar 2022 10:35:01 +0530 Subject: [PATCH 22/45] update csp --- configUtil.js | 12 ++++++++---- public/_headers | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/configUtil.js b/configUtil.js index 4bab3d5f0..54b44aff7 100644 --- a/configUtil.js +++ b/configUtil.js @@ -17,8 +17,10 @@ module.exports = { }, CSP_DIRECTIVES: { - 'default-src': "'none'", - 'img-src': "'self' blob:", + // self is safe enough + 'default-src': "'self'", + // data to allow two factor qr code + 'img-src': "'self' blob: data:", 'media-src': "'self' blob:", 'manifest-src': "'self'", 'style-src': "'self' 'unsafe-inline'", @@ -26,10 +28,12 @@ module.exports = { 'connect-src': "'self' https://*.ente.io http://localhost:8080 data: blob: https://ente-prod-eu.s3.eu-central-003.backblazeb2.com ", 'base-uri ': "'self'", + // to allow worker + 'child-src': "'self'", 'frame-ancestors': " 'none'", 'form-action': "'none'", - 'report-uri': ' https://csp-reporter.ente.io', - 'report-to': ' https://csp-reporter.ente.io', + 'report-uri': ' https://csp-reporter.ente.io/local', + 'report-to': ' https://csp-reporter.ente.io/local', }, WORKBOX_CONFIG: { diff --git a/public/_headers b/public/_headers index 18af99761..3a72736de 100644 --- a/public/_headers +++ b/public/_headers @@ -8,5 +8,5 @@ X-Frame-Options: deny X-XSS-Protection: 1; mode=block Referrer-Policy: same-origin - Content-Security-Policy-Report-Only: default-src 'none'; img-src 'self' blob:; media-src 'self' blob:; style-src 'self' 'unsafe-inline'; font-src 'self'; script-src 'self' 'unsafe-eval' blob:; manifest-src 'self'; connect-src 'self' https://*.ente.io data: blob: https://ente-prod-eu.s3.eu-central-003.backblazeb2.com ; base-uri 'self'; frame-ancestors 'none'; form-action 'none'; report-uri https://csp-reporter.ente.io; report-to https://csp-reporter.ente.io; + Content-Security-Policy-Report-Only: default-src 'self'; img-src 'self' blob: data:; media-src 'self' blob:; style-src 'self' 'unsafe-inline'; font-src 'self'; script-src 'self' 'unsafe-eval' blob:; manifest-src 'self'; child-src 'self'; connect-src 'self' https://*.ente.io data: blob: https://ente-prod-eu.s3.eu-central-003.backblazeb2.com ; base-uri 'self'; frame-ancestors 'none'; form-action 'none'; report-uri https://csp-reporter.ente.io; report-to https://csp-reporter.ente.io; From a86b9134d2808ce112e251e32ea49c0de2b9d77d Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 1 Mar 2022 12:50:45 +0530 Subject: [PATCH 23/45] also check location tag in the metadata as fallback --- src/services/upload/videoMetadataService.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/services/upload/videoMetadataService.ts b/src/services/upload/videoMetadataService.ts index 9e124d51d..0a13a5dc9 100644 --- a/src/services/upload/videoMetadataService.ts +++ b/src/services/upload/videoMetadataService.ts @@ -10,6 +10,7 @@ enum VideoMetadata { APPLE_LIVE_PHOTO_IDENTIFIER = 'com.apple.quicktime.live-photo.auto', APPLE_CREATION_DATE = 'com.apple.quicktime.creationdate', APPLE_LOCATION_ISO = 'com.apple.quicktime.location.ISO6709', + LOCATION = 'location', } export async function getVideoMetadata(file: File) { @@ -36,7 +37,8 @@ export function parseFFmpegExtractedMetadata(encodedMetadata: Uint8Array) { const metadataMap = Object.fromEntries(validKeyValuePairs); const location = parseAppleISOLocation( - metadataMap[VideoMetadata.APPLE_LOCATION_ISO] + metadataMap[VideoMetadata.APPLE_LOCATION_ISO] ?? + metadataMap[VideoMetadata.LOCATION] ); const creationTime = parseCreationTime( From 890c41da6934e2c04bbc0301032d6892780e052a Mon Sep 17 00:00:00 2001 From: Rushikesh Tote Date: Tue, 1 Mar 2022 18:14:19 +0530 Subject: [PATCH 24/45] fixed styling, live photo split --- src/components/LivePhotoBtn.tsx | 10 ++ src/components/PhotoFrame.tsx | 164 +++++++++++------- src/pages/_app.tsx | 21 +++ src/services/downloadManager.ts | 18 +- src/services/ffmpegService.ts | 8 +- .../publicCollectionDownloadManager.ts | 18 +- src/services/updateCreationTimeWithExif.ts | 2 +- src/utils/file/index.ts | 78 ++++----- src/utils/photoFrame/index.ts | 11 -- 9 files changed, 195 insertions(+), 135 deletions(-) create mode 100644 src/components/LivePhotoBtn.tsx diff --git a/src/components/LivePhotoBtn.tsx b/src/components/LivePhotoBtn.tsx new file mode 100644 index 000000000..b43609a00 --- /dev/null +++ b/src/components/LivePhotoBtn.tsx @@ -0,0 +1,10 @@ +export const livePhotoBtnHTML = ` + + + + LIVE +`; diff --git a/src/components/PhotoFrame.tsx b/src/components/PhotoFrame.tsx index 56755ab33..4f19d2bd7 100644 --- a/src/components/PhotoFrame.tsx +++ b/src/components/PhotoFrame.tsx @@ -15,8 +15,9 @@ import { ARCHIVE_SECTION, TRASH_SECTION, } from 'constants/collection'; -import { isSharedFile, splitLivePhoto } from 'utils/file'; -import { isPlaybackPossible, livePhotoBtnHTML } from 'utils/photoFrame'; +import { isSharedFile } from 'utils/file'; +import { isPlaybackPossible } from 'utils/photoFrame'; +import { livePhotoBtnHTML } from './LivePhotoBtn'; import { PhotoList } from './PhotoList'; import { SetFiles, SelectedState, Search, setSearchStats } from 'types/gallery'; import { FILE_TYPE } from 'constants/file'; @@ -73,7 +74,6 @@ interface Props { } type SourceURL = { - defaultURL?: string; imageURL?: string; videoURL?: string; }; @@ -255,18 +255,64 @@ const PhotoFrame = ({ document.querySelector('video'); const livePhotoBtn: HTMLButtonElement = document.querySelector('.live-photo-btn'); + const livePhotoImage: HTMLImageElement = + document.querySelector('.live-photo-image'); + if (livePhotoVideo && livePhotoBtn) { - const playVideo = () => { - livePhotoVideo.play().catch(() => { + const videoStyle = livePhotoVideo.style as React.CSSProperties; + const imageStyle = livePhotoImage.style as React.CSSProperties; + + videoStyle.opacity = 0; + imageStyle.opacity = 1; + + let videoPlaying = false; + + const timer = (ms) => new Promise((res) => setTimeout(res, ms)); + + const showVideoEffect = async (prevOpacity) => { + for (let i = prevOpacity * 100; i < 100; i++) { + videoStyle.opacity = i / 100; + imageStyle.opacity = (100 - i) / 100; + await timer(1); + if (!videoPlaying) { + return false; + } + } + return true; + }; + + const hideVideoEffect = async (prevOpacity) => { + for (let i = prevOpacity * 100; i >= 0; i--) { + videoStyle.opacity = i / 100; + imageStyle.opacity = (100 - i) / 100; + await timer(1); + if (videoPlaying) { + return false; + } + } + return true; + }; + + const playVideo = async () => { + if (videoPlaying) return; + videoPlaying = true; + if (!(await showVideoEffect(videoStyle.opacity))) { + return; + } + livePhotoVideo.play().catch(async () => { livePhotoVideo.pause(); + if (!videoPlaying) return; + videoPlaying = false; + await hideVideoEffect(videoStyle.opacity); }); }; - const pauseVideo = () => { - livePhotoVideo.pause(); + const pauseVideo = async () => { + if (!videoPlaying) return; + videoPlaying = false; + await hideVideoEffect(videoStyle.opacity); livePhotoVideo.load(); }; - livePhotoBtn.addEventListener('mouseout', pauseVideo); livePhotoBtn.addEventListener('mouseover', playVideo); livePhotoBtn.addEventListener('click', playVideo); @@ -326,47 +372,21 @@ const PhotoFrame = ({ setFiles(files); }; - const updateSrcURL = async (index: number, srcurl: SourceURL) => { + const updateSrcURL = async (index: number, srcURL: SourceURL) => { files[index] = { ...files[index], w: window.innerWidth, h: window.innerHeight, }; - const url = srcurl.defaultURL; + const { imageURL, videoURL } = srcURL; if (files[index].metadata.fileType === FILE_TYPE.VIDEO) { - if (await isPlaybackPossible(url)) { + if (await isPlaybackPossible(videoURL)) { files[index].html = ` `; - } else { - files[index].html = ` -
- -
- ${constants.VIDEO_PLAYBACK_FAILED_DOWNLOAD_INSTEAD} - Download -
-
- `; - } - } else if (files[index].metadata.fileType === FILE_TYPE.LIVE_PHOTO) { - const { imageURL, videoURL } = srcurl; - if (await isPlaybackPossible(videoURL)) { - files[index].html = ` -
- - -
- `; - setLivePhotoLoaded(true); } else { files[index].html = `
@@ -378,8 +398,35 @@ const PhotoFrame = ({
`; } + } else if (files[index].metadata.fileType === FILE_TYPE.LIVE_PHOTO) { + const { imageURL, videoURL } = srcURL; + if (await isPlaybackPossible(videoURL)) { + files[index].html = ` +
+ + + +
+ `; + } else { + files[index].html = ` +
+ +
+ ${constants.VIDEO_PLAYBACK_FAILED_DOWNLOAD_INSTEAD} + +
+
+ `; + } + setLivePhotoLoaded(true); } else { - files[index].src = url; + files[index].src = imageURL; } setFiles(files); }; @@ -517,48 +564,37 @@ const PhotoFrame = ({ if (!fetching[item.dataIndex]) { try { fetching[item.dataIndex] = true; - let url: string; + let URLs: string[]; if (galleryContext.files.has(item.id)) { - url = galleryContext.files.get(item.id); + const mergedURL = galleryContext.files.get(item.id); + URLs = mergedURL.split(','); } else { if ( publicCollectionGalleryContext.accessedThroughSharedURL ) { - url = await PublicCollectionDownloadManager.getFile( + URLs = await PublicCollectionDownloadManager.getFile( item, publicCollectionGalleryContext.token, publicCollectionGalleryContext.passwordToken, true ); } else { - url = await DownloadManager.getFile(item, true); - } - if (item.metadata.fileType !== FILE_TYPE.LIVE_PHOTO) { - galleryContext.files.set(item.id, url); + URLs = await DownloadManager.getFile(item, true); } + const mergedURL = URLs.join(','); + galleryContext.files.set(item.id, mergedURL); } + let imageURL; + let videoURL; if (item.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { - let imageURL; - let videoURL; - if (galleryContext.files.has(item.id)) { - [imageURL, videoURL] = galleryContext.files - .get(item.id) - .split(','); - } else { - ({ imageURL, videoURL } = await splitLivePhoto( - item, - url - )); - galleryContext.files.set( - item.id, - imageURL + ',' + videoURL - ); - } + [imageURL, videoURL] = URLs; setLivePhotoLoaded(false); - await updateSrcURL(item.dataIndex, { imageURL, videoURL }); + } else if (item.metadata.fileType === FILE_TYPE.VIDEO) { + [videoURL] = URLs; } else { - await updateSrcURL(item.dataIndex, { defaultURL: url }); + [imageURL] = URLs; } + await updateSrcURL(item.dataIndex, { imageURL, videoURL }); item.html = files[item.dataIndex].html; item.src = files[item.dataIndex].src; item.w = files[item.dataIndex].w; diff --git a/src/pages/_app.tsx b/src/pages/_app.tsx index cba3b6387..9ea6e600b 100644 --- a/src/pages/_app.tsx +++ b/src/pages/_app.tsx @@ -73,6 +73,27 @@ const GlobalStyles = createGlobalStyle` width: 100%; height: 100%; position: relative; + object-fit: contain; + } + + .live-photo-container > img{ + opacity: 1; + max-width: 100%; + max-height: 100%; + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + } + + .live-photo-container > video{ + opacity: 0; + max-width: 100%; + max-height: 100%; + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); } .live-photo-container > button { diff --git a/src/services/downloadManager.ts b/src/services/downloadManager.ts index 5752b1262..6f53f79ac 100644 --- a/src/services/downloadManager.ts +++ b/src/services/downloadManager.ts @@ -14,7 +14,7 @@ import { FILE_TYPE } from 'constants/file'; import { CustomError } from 'utils/error'; class DownloadManager { - private fileObjectURLPromise = new Map>(); + private fileObjectURLPromise = new Map>(); private thumbnailObjectURLPromise = new Map>(); public async getThumbnail(file: EnteFile) { @@ -90,11 +90,17 @@ class DownloadManager { try { const getFilePromise = async (convert: boolean) => { const fileStream = await this.downloadFile(file); - let fileBlob = await new Response(fileStream).blob(); + const fileBlob = await new Response(fileStream).blob(); if (convert) { - fileBlob = await convertForPreview(file, fileBlob); + const convertedBlobs = await convertForPreview( + file, + fileBlob + ); + return convertedBlobs.map((blob) => + URL.createObjectURL(blob) + ); } - return URL.createObjectURL(fileBlob); + return [URL.createObjectURL(fileBlob)]; }; if (!this.fileObjectURLPromise.get(fileKey)) { this.fileObjectURLPromise.set( @@ -102,8 +108,8 @@ class DownloadManager { getFilePromise(shouldBeConverted) ); } - const fileURL = await this.fileObjectURLPromise.get(fileKey); - return fileURL; + const fileURLs = await this.fileObjectURLPromise.get(fileKey); + return fileURLs; } catch (e) { this.fileObjectURLPromise.delete(fileKey); logError(e, 'Failed to get File'); diff --git a/src/services/ffmpegService.ts b/src/services/ffmpegService.ts index 7b3a25d64..0274a7e84 100644 --- a/src/services/ffmpegService.ts +++ b/src/services/ffmpegService.ts @@ -9,7 +9,7 @@ class FFmpegService { private isLoading = null; private fileReader: FileReader = null; - private generateQueueProcessor = new QueueProcessor(1); + private ffmpegTaskQueue = new QueueProcessor(1); async init() { try { this.ffmpeg = createFFmpeg({ @@ -36,7 +36,7 @@ class FFmpegService { if (this.isLoading) { await this.isLoading; } - const response = this.generateQueueProcessor.queueUpRequest( + const response = this.ffmpegTaskQueue.queueUpRequest( generateThumbnailHelper.bind( null, this.ffmpeg, @@ -68,7 +68,7 @@ class FFmpegService { await this.isLoading; } - const response = this.generateQueueProcessor.queueUpRequest( + const response = this.ffmpegTaskQueue.queueUpRequest( convertToMP4Helper.bind(null, this.ffmpeg, file, fileName) ); @@ -143,7 +143,7 @@ async function convertToMP4Helper( 'ultrafast', 'output.mp4' ); - const convertedFile = await ffmpeg.FS('readFile', 'output.mp4'); + const convertedFile = ffmpeg.FS('readFile', 'output.mp4'); ffmpeg.FS('unlink', inputFileName); ffmpeg.FS('unlink', 'output.mp4'); return convertedFile; diff --git a/src/services/publicCollectionDownloadManager.ts b/src/services/publicCollectionDownloadManager.ts index 0326c9e7c..70a4c7b38 100644 --- a/src/services/publicCollectionDownloadManager.ts +++ b/src/services/publicCollectionDownloadManager.ts @@ -16,7 +16,7 @@ import { FILE_TYPE } from 'constants/file'; import { CustomError } from 'utils/error'; class PublicCollectionDownloadManager { - private fileObjectURLPromise = new Map>(); + private fileObjectURLPromise = new Map>(); private thumbnailObjectURLPromise = new Map>(); public async getThumbnail( @@ -116,11 +116,17 @@ class PublicCollectionDownloadManager { passwordToken, file ); - let fileBlob = await new Response(fileStream).blob(); + const fileBlob = await new Response(fileStream).blob(); if (convert) { - fileBlob = await convertForPreview(file, fileBlob); + const convertedBlobs = await convertForPreview( + file, + fileBlob + ); + return convertedBlobs.map((blob) => + URL.createObjectURL(blob) + ); } - return URL.createObjectURL(fileBlob); + return [URL.createObjectURL(fileBlob)]; }; if (!this.fileObjectURLPromise.get(fileKey)) { this.fileObjectURLPromise.set( @@ -128,8 +134,8 @@ class PublicCollectionDownloadManager { getFilePromise(shouldBeConverted) ); } - const fileURL = await this.fileObjectURLPromise.get(fileKey); - return fileURL; + const fileURLs = await this.fileObjectURLPromise.get(fileKey); + return fileURLs; } catch (e) { this.fileObjectURLPromise.delete(fileKey); logError(e, 'Failed to get File'); diff --git a/src/services/updateCreationTimeWithExif.ts b/src/services/updateCreationTimeWithExif.ts index e34f889c0..482bfcbb3 100644 --- a/src/services/updateCreationTimeWithExif.ts +++ b/src/services/updateCreationTimeWithExif.ts @@ -35,7 +35,7 @@ export async function updateCreationTimeWithExif( if (fixOption === FIX_OPTIONS.CUSTOM_TIME) { correctCreationTime = getUNIXTime(customTime); } else { - const fileURL = await downloadManager.getFile(file); + const fileURL = await downloadManager.getFile(file)[0]; const fileObject = await getFileFromURL(fileURL); const reader = new FileReader(); const fileTypeInfo = await getFileType(reader, fileObject); diff --git a/src/utils/file/index.ts b/src/utils/file/index.ts index 378e68a52..42c2dc4ea 100644 --- a/src/utils/file/index.ts +++ b/src/utils/file/index.ts @@ -53,7 +53,7 @@ export async function downloadFile( if (accessedThroughSharedURL) { fileURL = await PublicCollectionDownloadManager.getCachedOriginalFile( file - ); + )[0]; tempURL; if (!fileURL) { tempURL = URL.createObjectURL( @@ -69,7 +69,7 @@ export async function downloadFile( fileURL = tempURL; } } else { - fileURL = await DownloadManager.getCachedOriginalFile(file); + fileURL = await DownloadManager.getCachedOriginalFile(file)[0]; if (!fileURL) { tempURL = URL.createObjectURL( await new Response( @@ -332,50 +332,42 @@ export function generateStreamFromArrayBuffer(data: Uint8Array) { }); } -export async function convertForPreview(file: EnteFile, fileBlob: Blob) { - if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { +export async function convertForPreview( + file: EnteFile, + fileBlob: Blob +): Promise { + const convertIfHEIC = async (fileName: string, fileBlob: Blob) => { + const typeFromExtension = getFileExtension(fileName); + const reader = new FileReader(); + const mimeType = + (await getFileTypeFromBlob(reader, fileBlob))?.mime ?? + typeFromExtension; + if (isFileHEIC(mimeType)) { + fileBlob = await HEICConverter.convert(fileBlob); + } return fileBlob; + }; + + if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { + const originalName = fileNameWithoutExtension(file.metadata.title); + const motionPhoto = await decodeMotionPhoto(fileBlob, originalName); + let image = new Blob([motionPhoto.image]); + + // can run conversion in parellel as video and image + // have different processes + const convertedVideo = ffmpegService.convertToMP4( + motionPhoto.video, + motionPhoto.videoNameTitle + ); + + image = await convertIfHEIC(motionPhoto.imageNameTitle, image); + const video = new Blob([await convertedVideo]); + + return [image, video]; } - const typeFromExtension = getFileExtension(file.metadata.title); - const reader = new FileReader(); - - const mimeType = - (await getFileTypeFromBlob(reader, fileBlob))?.mime ?? - typeFromExtension; - if (isFileHEIC(mimeType)) { - fileBlob = await HEICConverter.convert(fileBlob); - } - return fileBlob; -} - -export async function splitLivePhoto(file: EnteFile, url: string) { - const fileBlob = await fetch(url).then((res) => res.blob()); - const originalName = fileNameWithoutExtension(file.metadata.title); - const motionPhoto = await decodeMotionPhoto(fileBlob, originalName); - let image = new Blob([motionPhoto.image]); - - // can run conversion in parellel as video and image - // have different processes - const convertedVideo = ffmpegService.convertToMP4( - motionPhoto.video, - motionPhoto.videoNameTitle - ); - - const typeFromExtension = getFileExtension(motionPhoto.imageNameTitle); - const reader = new FileReader(); - - const mimeType = - (await getFileTypeFromBlob(reader, image))?.mime ?? typeFromExtension; - if (isFileHEIC(mimeType)) { - image = await HEICConverter.convert(image); - } - - const video = new Blob([await convertedVideo]); - - const imageURL = URL.createObjectURL(image); - const videoURL = URL.createObjectURL(video); - return { imageURL, videoURL }; + fileBlob = await convertIfHEIC(file.metadata.title, fileBlob); + return [fileBlob]; } export function fileIsArchived(file: EnteFile) { diff --git a/src/utils/photoFrame/index.ts b/src/utils/photoFrame/index.ts index c079489f5..fdaeb8557 100644 --- a/src/utils/photoFrame/index.ts +++ b/src/utils/photoFrame/index.ts @@ -13,14 +13,3 @@ export async function isPlaybackPossible(url: string): Promise { video.src = url; }); } - -export const livePhotoBtnHTML = ` - - - - LIVE -`; From a4d9526b2286de41d6bb4345509365e57ba30734 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 1 Mar 2022 19:19:39 +0530 Subject: [PATCH 25/45] show fade out in transistion using css --- src/components/PhotoFrame.tsx | 47 ++++++++--------------------------- src/pages/_app.tsx | 2 ++ 2 files changed, 13 insertions(+), 36 deletions(-) diff --git a/src/components/PhotoFrame.tsx b/src/components/PhotoFrame.tsx index 4f19d2bd7..045f48389 100644 --- a/src/components/PhotoFrame.tsx +++ b/src/components/PhotoFrame.tsx @@ -262,56 +262,31 @@ const PhotoFrame = ({ const videoStyle = livePhotoVideo.style as React.CSSProperties; const imageStyle = livePhotoImage.style as React.CSSProperties; - videoStyle.opacity = 0; - imageStyle.opacity = 1; - let videoPlaying = false; - const timer = (ms) => new Promise((res) => setTimeout(res, ms)); - - const showVideoEffect = async (prevOpacity) => { - for (let i = prevOpacity * 100; i < 100; i++) { - videoStyle.opacity = i / 100; - imageStyle.opacity = (100 - i) / 100; - await timer(1); - if (!videoPlaying) { - return false; - } - } - return true; + const showVideoEffect = () => { + videoStyle.opacity = 1; + imageStyle.opacity = 0; }; - const hideVideoEffect = async (prevOpacity) => { - for (let i = prevOpacity * 100; i >= 0; i--) { - videoStyle.opacity = i / 100; - imageStyle.opacity = (100 - i) / 100; - await timer(1); - if (videoPlaying) { - return false; - } - } - return true; + const hideVideoEffect = () => { + videoStyle.opacity = 0; + imageStyle.opacity = 1; }; const playVideo = async () => { if (videoPlaying) return; videoPlaying = true; - if (!(await showVideoEffect(videoStyle.opacity))) { - return; - } - livePhotoVideo.play().catch(async () => { - livePhotoVideo.pause(); - if (!videoPlaying) return; - videoPlaying = false; - await hideVideoEffect(videoStyle.opacity); - }); + showVideoEffect(); + livePhotoVideo.load(); + livePhotoVideo.play().catch(pauseVideo); }; const pauseVideo = async () => { if (!videoPlaying) return; videoPlaying = false; - await hideVideoEffect(videoStyle.opacity); - livePhotoVideo.load(); + livePhotoVideo.pause(); + hideVideoEffect(); }; livePhotoBtn.addEventListener('mouseout', pauseVideo); livePhotoBtn.addEventListener('mouseover', playVideo); diff --git a/src/pages/_app.tsx b/src/pages/_app.tsx index 9ea6e600b..f8a90c729 100644 --- a/src/pages/_app.tsx +++ b/src/pages/_app.tsx @@ -84,6 +84,7 @@ const GlobalStyles = createGlobalStyle` top: 50%; left: 50%; transform: translate(-50%, -50%); + transition: opacity 1s ease; } .live-photo-container > video{ @@ -94,6 +95,7 @@ const GlobalStyles = createGlobalStyle` top: 50%; left: 50%; transform: translate(-50%, -50%); + transition: opacity 1s ease; } .live-photo-container > button { From c83e15a4611131bc7a2bb9ec086f4a8fc23aea65 Mon Sep 17 00:00:00 2001 From: Rushikesh Tote Date: Wed, 2 Mar 2022 11:51:50 +0530 Subject: [PATCH 26/45] fixed live photo event listeners --- src/components/LivePhotoBtn.tsx | 8 +- src/components/PhotoFrame.tsx | 82 +++-------------- src/components/PhotoSwipe/PhotoSwipe.tsx | 112 ++++++++++++++++++++++- 3 files changed, 130 insertions(+), 72 deletions(-) diff --git a/src/components/LivePhotoBtn.tsx b/src/components/LivePhotoBtn.tsx index b43609a00..bb1976db9 100644 --- a/src/components/LivePhotoBtn.tsx +++ b/src/components/LivePhotoBtn.tsx @@ -1,4 +1,6 @@ -export const livePhotoBtnHTML = ` +import React from 'react'; + +export const livePhotoBtnHTML = ( - LIVE -`; + +); diff --git a/src/components/PhotoFrame.tsx b/src/components/PhotoFrame.tsx index 045f48389..1f68153fe 100644 --- a/src/components/PhotoFrame.tsx +++ b/src/components/PhotoFrame.tsx @@ -109,7 +109,7 @@ const PhotoFrame = ({ const filteredDataRef = useRef([]); const filteredData = filteredDataRef?.current ?? []; const router = useRouter(); - const [livePhotoLoaded, setLivePhotoLoaded] = useState(false); + const [isSourceLoaded, setIsSourceLoaded] = useState(false); useEffect(() => { const handleKeyDown = (e: KeyboardEvent) => { if (e.key === 'Shift') { @@ -250,56 +250,6 @@ const PhotoFrame = ({ } }, [open]); - useEffect(() => { - const livePhotoVideo: HTMLVideoElement = - document.querySelector('video'); - const livePhotoBtn: HTMLButtonElement = - document.querySelector('.live-photo-btn'); - const livePhotoImage: HTMLImageElement = - document.querySelector('.live-photo-image'); - - if (livePhotoVideo && livePhotoBtn) { - const videoStyle = livePhotoVideo.style as React.CSSProperties; - const imageStyle = livePhotoImage.style as React.CSSProperties; - - let videoPlaying = false; - - const showVideoEffect = () => { - videoStyle.opacity = 1; - imageStyle.opacity = 0; - }; - - const hideVideoEffect = () => { - videoStyle.opacity = 0; - imageStyle.opacity = 1; - }; - - const playVideo = async () => { - if (videoPlaying) return; - videoPlaying = true; - showVideoEffect(); - livePhotoVideo.load(); - livePhotoVideo.play().catch(pauseVideo); - }; - - const pauseVideo = async () => { - if (!videoPlaying) return; - videoPlaying = false; - livePhotoVideo.pause(); - hideVideoEffect(); - }; - livePhotoBtn.addEventListener('mouseout', pauseVideo); - livePhotoBtn.addEventListener('mouseover', playVideo); - livePhotoBtn.addEventListener('click', playVideo); - - return () => { - livePhotoBtn.removeEventListener('mouseout', pauseVideo); - livePhotoBtn.removeEventListener('mouseover', playVideo); - livePhotoBtn.removeEventListener('click', playVideo); - }; - } - }, [livePhotoLoaded]); - const updateURL = (index: number) => (url: string) => { files[index] = { ...files[index], @@ -378,11 +328,8 @@ const PhotoFrame = ({ if (await isPlaybackPossible(videoURL)) { files[index].html = `
- - -
-
+
${constants.VIDEO_PLAYBACK_FAILED_DOWNLOAD_INSTEAD}
`; } - setLivePhotoLoaded(true); } else { files[index].src = imageURL; } + setIsSourceLoaded(true); setFiles(files); }; @@ -539,36 +486,36 @@ const PhotoFrame = ({ if (!fetching[item.dataIndex]) { try { fetching[item.dataIndex] = true; - let URLs: string[]; + let urls: string[]; if (galleryContext.files.has(item.id)) { const mergedURL = galleryContext.files.get(item.id); - URLs = mergedURL.split(','); + urls = mergedURL.split(','); } else { if ( publicCollectionGalleryContext.accessedThroughSharedURL ) { - URLs = await PublicCollectionDownloadManager.getFile( + urls = await PublicCollectionDownloadManager.getFile( item, publicCollectionGalleryContext.token, publicCollectionGalleryContext.passwordToken, true ); } else { - URLs = await DownloadManager.getFile(item, true); + urls = await DownloadManager.getFile(item, true); } - const mergedURL = URLs.join(','); + const mergedURL = urls.join(','); galleryContext.files.set(item.id, mergedURL); } let imageURL; let videoURL; if (item.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { - [imageURL, videoURL] = URLs; - setLivePhotoLoaded(false); + [imageURL, videoURL] = urls; } else if (item.metadata.fileType === FILE_TYPE.VIDEO) { - [videoURL] = URLs; + [videoURL] = urls; } else { - [imageURL] = URLs; + [imageURL] = urls; } + setIsSourceLoaded(false); await updateSrcURL(item.dataIndex, { imageURL, videoURL }); item.html = files[item.dataIndex].html; item.src = files[item.dataIndex].src; @@ -637,6 +584,7 @@ const PhotoFrame = ({ isSharedCollection={isSharedCollection} isTrashCollection={activeCollection === TRASH_SECTION} enableDownload={enableDownload} + isSourceLoaded={isSourceLoaded} /> )} diff --git a/src/components/PhotoSwipe/PhotoSwipe.tsx b/src/components/PhotoSwipe/PhotoSwipe.tsx index 8757378a3..e60396069 100644 --- a/src/components/PhotoSwipe/PhotoSwipe.tsx +++ b/src/components/PhotoSwipe/PhotoSwipe.tsx @@ -35,6 +35,7 @@ import { Row, Value, } from 'components/Container'; +import { livePhotoBtnHTML } from 'components/LivePhotoBtn'; import { logError } from 'utils/sentry'; import CloseIcon from 'components/icons/CloseIcon'; @@ -43,7 +44,7 @@ import { Formik } from 'formik'; import * as Yup from 'yup'; import EnteSpinner from 'components/EnteSpinner'; import EnteDateTimePicker from 'components/EnteDateTimePicker'; -import { MAX_EDITED_FILE_NAME_LENGTH } from 'constants/file'; +import { MAX_EDITED_FILE_NAME_LENGTH, FILE_TYPE } from 'constants/file'; import { sleep } from 'utils/common'; import { PublicCollectionGalleryContext } from 'utils/publicCollectionGallery'; import { GalleryContext } from 'pages/gallery'; @@ -72,6 +73,7 @@ interface Iprops { isSharedCollection: boolean; isTrashCollection: boolean; enableDownload: boolean; + isSourceLoaded: boolean; } const LegendContainer = styled.div` @@ -90,6 +92,19 @@ const Pre = styled.pre` padding: 7px 15px; `; +const LivePhotoBtn = styled.button` + position: absolute; + top: 6vh; + left: 4vh; + height: 40px; + width: 80px; + background: #d7d7d7; + outline: none; + border: none; + border-radius: 10%; + z-index: 10; +`; + const renderInfoItem = (label: string, value: string | JSX.Element) => ( @@ -492,11 +507,12 @@ function PhotoSwipe(props: Iprops) { const pswpElement = useRef(); const [photoSwipe, setPhotoSwipe] = useState>(); - const { isOpen, items } = props; + const { isOpen, items, isSourceLoaded } = props; const [isFav, setIsFav] = useState(false); const [showInfo, setShowInfo] = useState(false); const [metadata, setMetaData] = useState(null); const [exif, setExif] = useState(null); + const [livePhotoBtn, setLivePhotoBtn] = useState(null); const needUpdate = useRef(false); const publicCollectionGalleryContext = useContext( PublicCollectionGalleryContext @@ -527,6 +543,97 @@ function PhotoSwipe(props: Iprops) { } }, [showInfo]); + useEffect(() => { + if (!isOpen) return; + const item = items[photoSwipe?.getCurrentIndex()]; + if (item && item.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { + let videoPlaying = false; + + const playVideo = async (livePhotoVideo, livePhotoImage) => { + if (videoPlaying) return; + videoPlaying = true; + livePhotoVideo.style.opacity = 1; + livePhotoImage.style.opacity = 0; + livePhotoVideo.load(); + livePhotoVideo.play().catch(() => { + pauseVideo(livePhotoVideo, livePhotoImage); + }); + }; + + const pauseVideo = async (livePhotoVideo, livePhotoImage) => { + if (!videoPlaying) return; + videoPlaying = false; + livePhotoVideo.pause(); + livePhotoVideo.style.opacity = 0; + livePhotoImage.style.opacity = 1; + }; + setLivePhotoBtn( + { + const video = document.getElementsByClassName( + `live-photo-video-${item.id}` + )[0]; + const image = document.getElementsByClassName( + `live-photo-image-${item.id}` + )[0]; + if (video && image) { + playVideo(video, image); + } + }} + onMouseEnter={() => { + const video = document.getElementsByClassName( + `live-photo-video-${item.id}` + )[0]; + const image = document.getElementsByClassName( + `live-photo-image-${item.id}` + )[0]; + if (video && image) { + playVideo(video, image); + } + }} + onMouseLeave={() => { + const video = document.getElementsByClassName( + `live-photo-video-${item.id}` + )[0]; + const image = document.getElementsByClassName( + `live-photo-image-${item.id}` + )[0]; + if (video && image) { + pauseVideo(video, image); + } + }}> + {livePhotoBtnHTML} LIVE + + ); + + const downloadMessageDivs = document.getElementsByClassName( + `download-message` + ) as HTMLCollectionOf; + for (const downloadMessageDiv of downloadMessageDivs) { + if (downloadMessageDiv?.dataset?.id === item.id) { + const downloadLivePhotoBtn = + downloadMessageDiv.firstElementChild as HTMLButtonElement; + + const download = () => { + downloadFileHelper(photoSwipe.currItem); + }; + + downloadLivePhotoBtn.addEventListener('click', download); + return () => { + downloadLivePhotoBtn.removeEventListener( + 'click', + download + ); + }; + } + } + } + + return () => { + setLivePhotoBtn(null); + }; + }, [photoSwipe?.currItem, isOpen, isSourceLoaded]); + function updateFavButton() { setIsFav(isInFav(this?.currItem)); } @@ -726,6 +833,7 @@ function PhotoSwipe(props: Iprops) { ref={pswpElement}>
+ {livePhotoBtn}
From 44eecaf92f1e353b3da97a99c03a7e2a0e599d31 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 2 Mar 2022 12:43:22 +0530 Subject: [PATCH 27/45] Increase timeout for thumbnail gen 10s -> 30s During testing, observed that for multiple copies of the same HEIC files, the thumbnail was blank for some files due to timeout error. Increasing this timeout fixed the issue. --- src/services/upload/thumbnailService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/upload/thumbnailService.ts b/src/services/upload/thumbnailService.ts index b6f1d3903..baa1a9c61 100644 --- a/src/services/upload/thumbnailService.ts +++ b/src/services/upload/thumbnailService.ts @@ -15,7 +15,7 @@ const MAX_THUMBNAIL_SIZE = 100 * 1024; const MIN_QUALITY = 0.5; const MAX_QUALITY = 0.7; -const WAIT_TIME_THUMBNAIL_GENERATION = 10 * 1000; +const WAIT_TIME_THUMBNAIL_GENERATION = 30 * 1000; interface Dimension { width: number; From 1296e961d6922662a5fd093a87564e4266366113 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 2 Mar 2022 15:42:56 +0530 Subject: [PATCH 28/45] allow blob uri as child-src --- configUtil.js | 2 +- public/_headers | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configUtil.js b/configUtil.js index 54b44aff7..9f4eb46e3 100644 --- a/configUtil.js +++ b/configUtil.js @@ -29,7 +29,7 @@ module.exports = { "'self' https://*.ente.io http://localhost:8080 data: blob: https://ente-prod-eu.s3.eu-central-003.backblazeb2.com ", 'base-uri ': "'self'", // to allow worker - 'child-src': "'self'", + 'child-src': "'self' blob:", 'frame-ancestors': " 'none'", 'form-action': "'none'", 'report-uri': ' https://csp-reporter.ente.io/local', diff --git a/public/_headers b/public/_headers index 3a72736de..2104a2445 100644 --- a/public/_headers +++ b/public/_headers @@ -8,5 +8,5 @@ X-Frame-Options: deny X-XSS-Protection: 1; mode=block Referrer-Policy: same-origin - Content-Security-Policy-Report-Only: default-src 'self'; img-src 'self' blob: data:; media-src 'self' blob:; style-src 'self' 'unsafe-inline'; font-src 'self'; script-src 'self' 'unsafe-eval' blob:; manifest-src 'self'; child-src 'self'; connect-src 'self' https://*.ente.io data: blob: https://ente-prod-eu.s3.eu-central-003.backblazeb2.com ; base-uri 'self'; frame-ancestors 'none'; form-action 'none'; report-uri https://csp-reporter.ente.io; report-to https://csp-reporter.ente.io; + Content-Security-Policy-Report-Only: default-src 'self'; img-src 'self' blob: data:; media-src 'self' blob:; style-src 'self' 'unsafe-inline'; font-src 'self'; script-src 'self' 'unsafe-eval' blob:; manifest-src 'self'; child-src 'self' blob:; connect-src 'self' https://*.ente.io data: blob: https://ente-prod-eu.s3.eu-central-003.backblazeb2.com ; base-uri 'self'; frame-ancestors 'none'; form-action 'none'; report-uri https://csp-reporter.ente.io; report-to https://csp-reporter.ente.io; From 7117669491791e975dbc7d5a41a8b203581c35dd Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 2 Mar 2022 15:44:53 +0530 Subject: [PATCH 29/45] block object src to none --- configUtil.js | 1 + public/_headers | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/configUtil.js b/configUtil.js index 9f4eb46e3..274c3d786 100644 --- a/configUtil.js +++ b/configUtil.js @@ -30,6 +30,7 @@ module.exports = { 'base-uri ': "'self'", // to allow worker 'child-src': "'self' blob:", + 'object-src': "'none'", 'frame-ancestors': " 'none'", 'form-action': "'none'", 'report-uri': ' https://csp-reporter.ente.io/local', diff --git a/public/_headers b/public/_headers index 2104a2445..388d4038c 100644 --- a/public/_headers +++ b/public/_headers @@ -8,5 +8,5 @@ X-Frame-Options: deny X-XSS-Protection: 1; mode=block Referrer-Policy: same-origin - Content-Security-Policy-Report-Only: default-src 'self'; img-src 'self' blob: data:; media-src 'self' blob:; style-src 'self' 'unsafe-inline'; font-src 'self'; script-src 'self' 'unsafe-eval' blob:; manifest-src 'self'; child-src 'self' blob:; connect-src 'self' https://*.ente.io data: blob: https://ente-prod-eu.s3.eu-central-003.backblazeb2.com ; base-uri 'self'; frame-ancestors 'none'; form-action 'none'; report-uri https://csp-reporter.ente.io; report-to https://csp-reporter.ente.io; + Content-Security-Policy-Report-Only: default-src 'self'; img-src 'self' blob: data:; media-src 'self' blob:; style-src 'self' 'unsafe-inline'; font-src 'self'; script-src 'self' 'unsafe-eval' blob:; manifest-src 'self'; child-src 'self' blob:; object-src 'none'; connect-src 'self' https://*.ente.io data: blob: https://ente-prod-eu.s3.eu-central-003.backblazeb2.com ; base-uri 'self'; frame-ancestors 'none'; form-action 'none'; report-uri https://csp-reporter.ente.io; report-to https://csp-reporter.ente.io; From ae8db26a002b954f5356cd8b981d7f5affa12522 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 2 Mar 2022 16:07:42 +0530 Subject: [PATCH 30/45] fix video loader not showing issue --- src/components/PhotoFrame.tsx | 83 +++++++++----------- src/components/PhotoSwipe/PhotoSwipe.tsx | 4 +- src/components/pages/gallery/PreviewCard.tsx | 13 +-- src/services/fileService.ts | 26 +++--- src/services/trashService.ts | 14 +++- src/services/upload/uploadManager.ts | 8 +- src/utils/file/index.ts | 27 +++---- 7 files changed, 88 insertions(+), 87 deletions(-) diff --git a/src/components/PhotoFrame.tsx b/src/components/PhotoFrame.tsx index e54d4938b..1daae5b1a 100644 --- a/src/components/PhotoFrame.tsx +++ b/src/components/PhotoFrame.tsx @@ -244,18 +244,18 @@ const PhotoFrame = ({ }, [open]); const updateURL = (index: number) => (url: string) => { - files[index] = { - ...files[index], - msrc: url, - src: files[index].src ? files[index].src : url, - w: window.innerWidth, - h: window.innerHeight, - }; - if ( - files[index].metadata.fileType === FILE_TYPE.VIDEO && - !files[index].html - ) { - files[index].html = ` + setFiles((files) => { + files[index] = { + ...files[index], + msrc: url, + w: window.innerWidth, + h: window.innerHeight, + }; + if ( + files[index].metadata.fileType === FILE_TYPE.VIDEO && + !files[index].html + ) { + files[index].html = `
`; - delete files[index].src; - } - if ( - files[index].metadata.fileType === FILE_TYPE.IMAGE && - !files[index].src - ) { - files[index].src = url; - } - setFiles(files); + } + if ( + files[index].metadata.fileType === FILE_TYPE.IMAGE && + !files[index].src + ) { + files[index].src = url; + } + return [...files]; + }); }; const updateSrcURL = async (index: number, url: string) => { - files[index] = { - ...files[index], - w: window.innerWidth, - h: window.innerHeight, - }; - if (files[index].metadata.fileType === FILE_TYPE.VIDEO) { - if (await isPlaybackPossible(url)) { - files[index].html = ` + const isPlayable = await isPlaybackPossible(url); + setFiles((files) => { + files[index] = { + ...files[index], + w: window.innerWidth, + h: window.innerHeight, + }; + if (files[index].metadata.fileType === FILE_TYPE.VIDEO) { + if (isPlayable) { + files[index].html = ` `; - } else { - files[index].html = ` + } else { + files[index].html = `
@@ -298,11 +300,12 @@ const PhotoFrame = ({
`; + } + } else { + files[index].src = url; } - } else { - files[index].src = url; - } - setFiles(files); + return [...files]; + }); }; const handleClose = (needUpdate) => { @@ -420,11 +423,7 @@ const PhotoFrame = ({ } updateURL(item.dataIndex)(url); item.msrc = url; - if (!item.src) { - item.src = url; - } - item.w = window.innerWidth; - item.h = window.innerHeight; + item.html; try { instance.invalidateCurrItems(); instance.updateSize(true); @@ -457,10 +456,6 @@ const PhotoFrame = ({ galleryContext.files.set(item.id, url); } await updateSrcURL(item.dataIndex, url); - item.html = files[item.dataIndex].html; - item.src = files[item.dataIndex].src; - item.w = files[item.dataIndex].w; - item.h = files[item.dataIndex].h; try { instance.invalidateCurrItems(); instance.updateSize(true); diff --git a/src/components/PhotoSwipe/PhotoSwipe.tsx b/src/components/PhotoSwipe/PhotoSwipe.tsx index 8757378a3..25cb4e49c 100644 --- a/src/components/PhotoSwipe/PhotoSwipe.tsx +++ b/src/components/PhotoSwipe/PhotoSwipe.tsx @@ -490,7 +490,8 @@ function InfoModal({ function PhotoSwipe(props: Iprops) { const pswpElement = useRef(); - const [photoSwipe, setPhotoSwipe] = useState>(); + const [photoSwipe, setPhotoSwipe] = + useState>(); const { isOpen, items } = props; const [isFav, setIsFav] = useState(false); @@ -654,6 +655,7 @@ function PhotoSwipe(props: Iprops) { }; const updateItems = (items = []) => { + console.log('items updated'); if (photoSwipe) { photoSwipe.items.length = 0; items.forEach((item) => { diff --git a/src/components/pages/gallery/PreviewCard.tsx b/src/components/pages/gallery/PreviewCard.tsx index 1a4810528..51a100226 100644 --- a/src/components/pages/gallery/PreviewCard.tsx +++ b/src/components/pages/gallery/PreviewCard.tsx @@ -164,7 +164,7 @@ const Cont = styled.div<{ disabled: boolean; selected: boolean }>` export default function PreviewCard(props: IProps) { const [imgSrc, setImgSrc] = useState(); - const { thumbs, files } = useContext(GalleryContext); + const { thumbs } = useContext(GalleryContext); const { file, onClick, @@ -203,10 +203,6 @@ export default function PreviewCard(props: IProps) { if (isMounted.current) { setImgSrc(url); thumbs.set(file.id, url); - file.msrc = url; - if (!file.src) { - file.src = url; - } updateURL(url); } } catch (e) { @@ -218,13 +214,6 @@ export default function PreviewCard(props: IProps) { const thumbImgSrc = thumbs.get(file.id); setImgSrc(thumbImgSrc); file.msrc = thumbImgSrc; - if (!file.src) { - if (files.has(file.id)) { - file.src = files.get(file.id); - } else { - file.src = thumbImgSrc; - } - } } else { main(); } diff --git a/src/services/fileService.ts b/src/services/fileService.ts index 8d1ad4944..540f0dbfe 100644 --- a/src/services/fileService.ts +++ b/src/services/fileService.ts @@ -6,9 +6,15 @@ import { EncryptionResult } from 'types/upload'; import { Collection } from 'types/collection'; import HTTPService from './HTTPService'; import { logError } from 'utils/sentry'; -import { decryptFile, mergeMetadata, sortFiles } from 'utils/file'; +import { + decryptFile, + mergeMetadata, + preservePhotoswipeProps, + sortFiles, +} from 'utils/file'; import CryptoWorker from 'utils/crypto'; import { EnteFile, TrashRequest, UpdateMagicMetadataRequest } from 'types/file'; +import { SetFiles } from 'types/gallery'; const ENDPOINT = getEndpoint(); const FILES_TABLE = 'files'; @@ -28,13 +34,13 @@ const getCollectionLastSyncTime = async (collection: Collection) => export const syncFiles = async ( collections: Collection[], - setFiles: (files: EnteFile[]) => void + setFiles: SetFiles ) => { const localFiles = await getLocalFiles(); let files = await removeDeletedCollectionFiles(collections, localFiles); if (files.length !== localFiles.length) { await setLocalFiles(files); - setFiles([...sortFiles(mergeMetadata(files))]); + setFiles(preservePhotoswipeProps([...sortFiles(mergeMetadata(files))])); } for (const collection of collections) { if (!getToken()) { @@ -70,7 +76,7 @@ export const syncFiles = async ( `${collection.id}-time`, collection.updationTime ); - setFiles([...sortFiles(mergeMetadata(files))]); + setFiles(preservePhotoswipeProps([...sortFiles(mergeMetadata(files))])); } return sortFiles(mergeMetadata(files)); }; @@ -79,7 +85,7 @@ export const getFiles = async ( collection: Collection, sinceTime: number, files: EnteFile[], - setFiles: (files: EnteFile[]) => void + setFiles: SetFiles ): Promise => { try { const decryptedFiles: EnteFile[] = []; @@ -116,10 +122,12 @@ export const getFiles = async ( time = resp.data.diff.slice(-1)[0].updationTime; } setFiles( - sortFiles( - mergeMetadata( - [...(files || []), ...decryptedFiles].filter( - (item) => !item.isDeleted + preservePhotoswipeProps( + sortFiles( + mergeMetadata( + [...(files || []), ...decryptedFiles].filter( + (item) => !item.isDeleted + ) ) ) ) diff --git a/src/services/trashService.ts b/src/services/trashService.ts index c53413074..4fba7c05e 100644 --- a/src/services/trashService.ts +++ b/src/services/trashService.ts @@ -2,7 +2,12 @@ import { SetFiles } from 'types/gallery'; import { Collection } from 'types/collection'; import { getEndpoint } from 'utils/common/apiUtil'; import { getToken } from 'utils/common/key'; -import { decryptFile, mergeMetadata, sortFiles } from 'utils/file'; +import { + decryptFile, + mergeMetadata, + preservePhotoswipeProps, + sortFiles, +} from 'utils/file'; import { logError } from 'utils/sentry'; import localForage from 'utils/storage/localForage'; import { getCollection } from './collectionService'; @@ -120,7 +125,12 @@ export const updateTrash = async ( updatedTrash = removeRestoredOrDeletedTrashItems(updatedTrash); setFiles( - sortFiles([...(files ?? []), ...getTrashedFiles(updatedTrash)]) + preservePhotoswipeProps( + sortFiles([ + ...(files ?? []), + ...getTrashedFiles(updatedTrash), + ]) + ) ); await localForage.setItem(TRASH, updatedTrash); await localForage.setItem(TRASH_TIME, time); diff --git a/src/services/upload/uploadManager.ts b/src/services/upload/uploadManager.ts index eeeceda0f..e40e83fca 100644 --- a/src/services/upload/uploadManager.ts +++ b/src/services/upload/uploadManager.ts @@ -5,7 +5,7 @@ import { getDedicatedCryptoWorker } from 'utils/crypto'; import { sortFilesIntoCollections, sortFiles, - removeUnnecessaryFileProps, + preservePhotoswipeProps, } from 'utils/file'; import { logError } from 'utils/sentry'; import { getMetadataJSONMapKey, parseMetadataJSON } from './metadataService'; @@ -239,10 +239,8 @@ class UploadManager { if (fileUploadResult === FileUploadResults.UPLOADED) { this.existingFiles.push(file); this.existingFiles = sortFiles(this.existingFiles); - await setLocalFiles( - removeUnnecessaryFileProps(this.existingFiles) - ); - this.setFiles(this.existingFiles); + await setLocalFiles(this.existingFiles); + this.setFiles(preservePhotoswipeProps(this.existingFiles)); if (!this.existingFilesCollectionWise.has(file.collectionID)) { this.existingFilesCollectionWise.set(file.collectionID, []); } diff --git a/src/utils/file/index.ts b/src/utils/file/index.ts index 9666a9c45..242083a7d 100644 --- a/src/utils/file/index.ts +++ b/src/utils/file/index.ts @@ -281,20 +281,19 @@ export async function decryptFile(file: EnteFile, collectionKey: string) { } } -export function removeUnnecessaryFileProps(files: EnteFile[]): EnteFile[] { - const stripedFiles = files.map((file) => { - delete file.src; - delete file.msrc; - delete file.file.objectKey; - delete file.thumbnail.objectKey; - delete file.h; - delete file.html; - delete file.w; - - return file; - }); - return stripedFiles; -} +export const preservePhotoswipeProps = + (newFiles: EnteFile[]) => + (currentFiles: EnteFile[]): EnteFile[] => { + const currentFilesMap = Object.fromEntries( + currentFiles.map((file) => [file.id, file]) + ); + const fileWithPreservedProperty = newFiles.map((file) => { + const currentFile = currentFilesMap[file.id]; + return { ...currentFile, ...file }; + }); + console.log(fileWithPreservedProperty); + return fileWithPreservedProperty; + }; export function fileNameWithoutExtension(filename) { const lastDotPosition = filename.lastIndexOf('.'); From fa2b04a6dafb17b51416b8629c27dd7752559aa7 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 2 Mar 2022 16:18:10 +0530 Subject: [PATCH 31/45] fix srcUrl not updating until changing slide --- src/components/PhotoFrame.tsx | 85 ++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 36 deletions(-) diff --git a/src/components/PhotoFrame.tsx b/src/components/PhotoFrame.tsx index 1daae5b1a..3639adc4b 100644 --- a/src/components/PhotoFrame.tsx +++ b/src/components/PhotoFrame.tsx @@ -244,18 +244,15 @@ const PhotoFrame = ({ }, [open]); const updateURL = (index: number) => (url: string) => { - setFiles((files) => { - files[index] = { - ...files[index], + const updateFile = (file: EnteFile) => { + file = { + ...file, msrc: url, w: window.innerWidth, h: window.innerHeight, }; - if ( - files[index].metadata.fileType === FILE_TYPE.VIDEO && - !files[index].html - ) { - files[index].html = ` + if (file.metadata.fileType === FILE_TYPE.VIDEO && !file.html) { + file.html = `
`; } - if ( - files[index].metadata.fileType === FILE_TYPE.IMAGE && - !files[index].src - ) { - files[index].src = url; + if (file.metadata.fileType === FILE_TYPE.IMAGE && !file.src) { + file.src = url; } + return file; + }; + setFiles((files) => { + files[index] = updateFile(files[index]); return [...files]; }); + return updateFile(files[index]); }; const updateSrcURL = async (index: number, url: string) => { const isPlayable = await isPlaybackPossible(url); - setFiles((files) => { - files[index] = { - ...files[index], + const updateFile = (file: EnteFile) => { + file = { + ...file, w: window.innerWidth, h: window.innerHeight, }; - if (files[index].metadata.fileType === FILE_TYPE.VIDEO) { + if (file.metadata.fileType === FILE_TYPE.VIDEO) { if (isPlayable) { - files[index].html = ` - - `; + file.html = ` + + `; } else { - files[index].html = ` -
- -
- ${constants.VIDEO_PLAYBACK_FAILED_DOWNLOAD_INSTEAD} - Download -
+ file.html = ` +
+ +
+ ${constants.VIDEO_PLAYBACK_FAILED_DOWNLOAD_INSTEAD} + Download
- `; +
+ `; } } else { - files[index].src = url; + file.src = url; } + return file; + }; + setFiles((files) => { + files[index] = updateFile(files[index]); return [...files]; }); + return updateFile(files[index]); }; const handleClose = (needUpdate) => { @@ -421,9 +425,13 @@ const PhotoFrame = ({ } galleryContext.thumbs.set(item.id, url); } - updateURL(item.dataIndex)(url); - item.msrc = url; - item.html; + const newFile = updateURL(item.dataIndex)(url); + item.msrc = newFile.msrc; + item.html = newFile.html; + item.src = newFile.src; + item.w = newFile.w; + item.h = newFile.h; + try { instance.invalidateCurrItems(); instance.updateSize(true); @@ -455,7 +463,12 @@ const PhotoFrame = ({ } galleryContext.files.set(item.id, url); } - await updateSrcURL(item.dataIndex, url); + const newFile = await updateSrcURL(item.dataIndex, url); + item.msrc = newFile.msrc; + item.html = newFile.html; + item.src = newFile.src; + item.w = newFile.w; + item.h = newFile.h; try { instance.invalidateCurrItems(); instance.updateSize(true); From bdb356287eac1f094dbdbcf482358d8cbc7c1b2b Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 2 Mar 2022 16:21:24 +0530 Subject: [PATCH 32/45] remove console log --- src/components/PhotoSwipe/PhotoSwipe.tsx | 1 - src/utils/file/index.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/src/components/PhotoSwipe/PhotoSwipe.tsx b/src/components/PhotoSwipe/PhotoSwipe.tsx index 25cb4e49c..f9c6bb095 100644 --- a/src/components/PhotoSwipe/PhotoSwipe.tsx +++ b/src/components/PhotoSwipe/PhotoSwipe.tsx @@ -655,7 +655,6 @@ function PhotoSwipe(props: Iprops) { }; const updateItems = (items = []) => { - console.log('items updated'); if (photoSwipe) { photoSwipe.items.length = 0; items.forEach((item) => { diff --git a/src/utils/file/index.ts b/src/utils/file/index.ts index 242083a7d..a775a77dc 100644 --- a/src/utils/file/index.ts +++ b/src/utils/file/index.ts @@ -291,7 +291,6 @@ export const preservePhotoswipeProps = const currentFile = currentFilesMap[file.id]; return { ...currentFile, ...file }; }); - console.log(fileWithPreservedProperty); return fileWithPreservedProperty; }; From 70084fdc80201efd0610ab8abea12e2b3f9a4446 Mon Sep 17 00:00:00 2001 From: Rushikesh Tote Date: Wed, 2 Mar 2022 16:31:14 +0530 Subject: [PATCH 33/45] cleanup --- src/components/PhotoFrame.tsx | 4 ---- src/components/PhotoSwipe/PhotoSwipe.tsx | 9 ++++++--- src/pages/_app.tsx | 13 ------------- 3 files changed, 6 insertions(+), 20 deletions(-) diff --git a/src/components/PhotoFrame.tsx b/src/components/PhotoFrame.tsx index 1f68153fe..cded5f043 100644 --- a/src/components/PhotoFrame.tsx +++ b/src/components/PhotoFrame.tsx @@ -17,7 +17,6 @@ import { } from 'constants/collection'; import { isSharedFile } from 'utils/file'; import { isPlaybackPossible } from 'utils/photoFrame'; -import { livePhotoBtnHTML } from './LivePhotoBtn'; import { PhotoList } from './PhotoList'; import { SetFiles, SelectedState, Search, setSearchStats } from 'types/gallery'; import { FILE_TYPE } from 'constants/file'; @@ -277,9 +276,6 @@ const PhotoFrame = ({ ) { files[index].html = `
- `; } - if (file.metadata.fileType === FILE_TYPE.IMAGE && !file.src) { + if (!file.src) { file.src = url; } return file; From f53dfe2c33b8d048d1c293ae1a45535e17b63764 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 2 Mar 2022 19:42:26 +0530 Subject: [PATCH 36/45] fix set src for image and live photo --- src/components/PhotoFrame.tsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/components/PhotoFrame.tsx b/src/components/PhotoFrame.tsx index ba1111bd9..b0460cd7f 100644 --- a/src/components/PhotoFrame.tsx +++ b/src/components/PhotoFrame.tsx @@ -260,8 +260,11 @@ const PhotoFrame = ({
`; - } - if (!file.src) { + } else if ( + (file.metadata.fileType === FILE_TYPE.IMAGE || + file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) && + !file.src + ) { file.src = url; } return file; From c1d7ba500747e7a0e8f5a7d94e8daf4ca26ced96 Mon Sep 17 00:00:00 2001 From: Rushikesh Tote Date: Thu, 3 Mar 2022 09:46:44 +0530 Subject: [PATCH 37/45] refactor live photo button --- src/components/PhotoFrame.tsx | 4 +- src/components/PhotoSwipe/PhotoSwipe.tsx | 136 ++++++++++++----------- 2 files changed, 74 insertions(+), 66 deletions(-) diff --git a/src/components/PhotoFrame.tsx b/src/components/PhotoFrame.tsx index a4ce581d8..1e07439d0 100644 --- a/src/components/PhotoFrame.tsx +++ b/src/components/PhotoFrame.tsx @@ -336,9 +336,9 @@ const PhotoFrame = ({ file.html = `
-
+
${constants.VIDEO_PLAYBACK_FAILED_DOWNLOAD_INSTEAD} - +
`; diff --git a/src/components/PhotoSwipe/PhotoSwipe.tsx b/src/components/PhotoSwipe/PhotoSwipe.tsx index 2c5e7ea19..ea07706c4 100644 --- a/src/components/PhotoSwipe/PhotoSwipe.tsx +++ b/src/components/PhotoSwipe/PhotoSwipe.tsx @@ -105,6 +105,13 @@ const LivePhotoBtn = styled.button` z-index: 10; `; +const livePhotoDefaultOptions = { + click: () => {}, + hide: () => {}, + show: () => {}, + visible: false, +}; + const renderInfoItem = (label: string, value: string | JSX.Element) => ( @@ -513,7 +520,9 @@ function PhotoSwipe(props: Iprops) { const [showInfo, setShowInfo] = useState(false); const [metadata, setMetaData] = useState(null); const [exif, setExif] = useState(null); - const [livePhotoBtn, setLivePhotoBtn] = useState(null); + const [livePhotoBtnOptions, setLivePhotoBtnOptions] = useState( + livePhotoDefaultOptions + ); const needUpdate = useRef(false); const publicCollectionGalleryContext = useContext( PublicCollectionGalleryContext @@ -568,76 +577,65 @@ function PhotoSwipe(props: Iprops) { livePhotoVideo.style.opacity = 0; livePhotoImage.style.opacity = 1; }; - setLivePhotoBtn( - { - const video = document.getElementsByClassName( - `live-photo-video-${item.id}` - )[0]; - const image = document.getElementsByClassName( - `live-photo-image-${item.id}` - )[0]; - if (video && image) { - playVideo(video, image); - } - }} - onMouseEnter={() => { - const video = document.getElementsByClassName( - `live-photo-video-${item.id}` - )[0]; - const image = document.getElementsByClassName( - `live-photo-image-${item.id}` - )[0]; - if (video && image) { - playVideo(video, image); - } - }} - onMouseLeave={() => { - const video = document.getElementsByClassName( - `live-photo-video-${item.id}` - )[0]; - const image = document.getElementsByClassName( - `live-photo-image-${item.id}` - )[0]; - if (video && image) { - pauseVideo(video, image); - } - }}> - {livePhotoBtnHTML} LIVE - - ); - const downloadMessageDivs = document.getElementsByClassName( - `download-message` - ) as HTMLCollectionOf; - for (const downloadMessageDiv of downloadMessageDivs) { - if ( - String(downloadMessageDiv?.dataset?.id) === String(item.id) - ) { - const downloadLivePhotoBtn = - downloadMessageDiv.firstElementChild as HTMLButtonElement; + const getvideoAndImage = () => { + const video = document.getElementsByClassName( + `live-photo-video-${item.id}` + )[0]; + const image = document.getElementsByClassName( + `live-photo-image-${item.id}` + )[0]; + return { video, image }; + }; - const downloadLivePhoto = () => { - downloadFileHelper(photoSwipe.currItem); - }; + setLivePhotoBtnOptions({ + click: () => { + const { video, image } = getvideoAndImage(); + if (video && image) { + playVideo(video, image); + } + }, + hide: () => { + const { video, image } = getvideoAndImage(); + if (video && image) { + pauseVideo(video, image); + } + }, + show: () => { + const { video, image } = getvideoAndImage(); + if (video && image) { + playVideo(video, image); + } + }, + visible: true, + }); - downloadLivePhotoBtn.addEventListener( + const downloadLivePhotoBtn = document.getElementById( + `download-btn-${item.id}` + ) as HTMLButtonElement; + console.log(downloadLivePhotoBtn); + if (downloadLivePhotoBtn) { + const downloadLivePhoto = () => { + downloadFileHelper(photoSwipe.currItem); + }; + + downloadLivePhotoBtn.addEventListener( + 'click', + downloadLivePhoto + ); + return () => { + downloadLivePhotoBtn.removeEventListener( 'click', downloadLivePhoto ); - return () => { - downloadLivePhotoBtn.removeEventListener( - 'click', - downloadLivePhoto - ); - }; - } + setLivePhotoBtnOptions(livePhotoDefaultOptions); + }; } - } - return () => { - setLivePhotoBtn(null); - }; + return () => { + setLivePhotoBtnOptions(livePhotoDefaultOptions); + }; + } }, [photoSwipe?.currItem, isOpen, isSourceLoaded]); function updateFavButton() { @@ -839,7 +837,17 @@ function PhotoSwipe(props: Iprops) { ref={pswpElement}>
- {livePhotoBtn} + + {livePhotoBtnHTML} LIVE +
From 34dd456063c05989f08cc70873c4c597514fd4be Mon Sep 17 00:00:00 2001 From: Rushikesh Tote Date: Thu, 3 Mar 2022 10:18:08 +0530 Subject: [PATCH 38/45] styling live photo button --- src/components/PhotoSwipe/PhotoSwipe.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/PhotoSwipe/PhotoSwipe.tsx b/src/components/PhotoSwipe/PhotoSwipe.tsx index ea07706c4..cfe03dbcb 100644 --- a/src/components/PhotoSwipe/PhotoSwipe.tsx +++ b/src/components/PhotoSwipe/PhotoSwipe.tsx @@ -94,8 +94,8 @@ const Pre = styled.pre` const LivePhotoBtn = styled.button` position: absolute; - top: 6vh; - left: 4vh; + bottom: 6vh; + right: 6vh; height: 40px; width: 80px; background: #d7d7d7; From 81aba874cb3dd3b1989aef996948a85d9d649991 Mon Sep 17 00:00:00 2001 From: Rushikesh Tote Date: Thu, 3 Mar 2022 11:47:55 +0530 Subject: [PATCH 39/45] refactor live photo elements --- src/components/PhotoFrame.tsx | 4 +- src/components/PhotoSwipe/PhotoSwipe.tsx | 75 +++++++++--------------- src/utils/photoFrame/index.ts | 19 ++++++ src/utils/strings/englishConstants.tsx | 1 + 4 files changed, 49 insertions(+), 50 deletions(-) diff --git a/src/components/PhotoFrame.tsx b/src/components/PhotoFrame.tsx index 1e07439d0..160b074cc 100644 --- a/src/components/PhotoFrame.tsx +++ b/src/components/PhotoFrame.tsx @@ -325,8 +325,8 @@ const PhotoFrame = ({ if (isPlayable) { file.html = `
- -