beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

[web] Cast ready to roll (#1671)

Browse source
This commit is contained in:
Manav Rathi 2024-05-10 14:17:00 +05:30 committed by GitHub
parent c29db9bcfb 08c37c69ff
commit 0458b79fc3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
33 changed files with 717 additions and 471 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
desktop/src/main/log.ts
View file

@ -62,7 +62,7 @@ const logError = (message: string, e?: unknown) => {
const logError_ = (message: string) => {
log.error(`[main] [error] ${message}`);
if (isDev) console.error(`[error] ${message}`);
console.error(`[error] ${message}`);
};
const logInfo = (...params: unknown[]) => {
@ -96,8 +96,8 @@ export default {
* any arbitrary object that we obtain, say, when in a try-catch handler (in
* JavaScript any arbitrary value can be thrown).
*
* The log is written to disk. In development builds, the log is also
* printed to the main (Node.js) process console.
* The log is written to disk and printed to the main (Node.js) process's
* console.
*/
error: logError,
/**
@ -120,7 +120,7 @@ export default {
* The function can return an arbitrary value which is serialized before
* being logged.
*
* This log is NOT written to disk. And it is printed to the main (Node.js)
* This log is NOT written to disk. It is printed to the main (Node.js)
* process console, but only on development builds.
*/
debug: logDebug,

56
web/apps/cast/src/components/Slide.tsx
View file

@ -1,56 +0,0 @@
interface SlideViewProps {
/** The URL of the image to show. */
url: string;
/** The URL of the next image that we will transition to. */
nextURL: string;
}
/**
* Show the image at {@link url} in a full screen view.
*
* Also show {@link nextURL} in an hidden image view to prepare the browser for
* an imminent transition to it.
*/
export const SlideView: React.FC<SlideViewProps> = ({ url, nextURL }) => {
return (
<div
style={{
width: "100vw",
height: "100vh",
backgroundImage: `url(${url})`,
backgroundSize: "cover",
backgroundPosition: "center",
backgroundRepeat: "no-repeat",
backgroundBlendMode: "multiply",
backgroundColor: "rgba(0, 0, 0, 0.5)",
}}
>
<div
style={{
height: "100%",
width: "100%",
display: "flex",
justifyContent: "center",
alignItems: "center",
backdropFilter: "blur(10px)",
}}
>
<img
src={nextURL}
style={{
maxWidth: "100%",
maxHeight: "100%",
display: "none",
}}
/>
<img
src={url}
style={{
maxWidth: "100%",
maxHeight: "100%",
}}
/>
</div>
</div>
);
};

2
web/apps/cast/src/pages/_app.tsx
View file

@ -1,4 +1,5 @@
import { CustomHead } from "@/next/components/Head";
import { disableDiskLogs } from "@/next/log";
import { logUnhandledErrorsAndRejections } from "@/next/log-web";
import { APPS, APP_TITLES } from "@ente/shared/apps/constants";
import { getTheme } from "@ente/shared/themes";
@ -11,6 +12,7 @@ import "styles/global.css";
export default function App({ Component, pageProps }: AppProps) {
useEffect(() => {
disableDiskLogs();
logUnhandledErrorsAndRejections(true);
return () => logUnhandledErrorsAndRejections(false);
}, []);

21
web/apps/cast/src/pages/index.tsx
View file

@ -4,19 +4,15 @@ import { styled } from "@mui/material";
import { PairingCode } from "components/PairingCode";
import { useRouter } from "next/router";
import { useEffect, useState } from "react";
import { storeCastData } from "services/cast";
import { advertiseCode, getCastData, register } from "services/pair";
import { castReceiverLoadingIfNeeded } from "../utils/cast-receiver";
import { readCastData, storeCastData } from "services/cast-data";
import { getCastData, register } from "services/pair";
import { advertiseOnChromecast } from "../services/chromecast";
export default function Index() {
const [publicKeyB64, setPublicKeyB64] = useState<string | undefined>();
const [privateKeyB64, setPrivateKeyB64] = useState<string | undefined>();
const [pairingCode, setPairingCode] = useState<string | undefined>();
// Keep a boolean flag to ensure that Cast Receiver starts only once even if
// pairing codes change.
const [haveInitializedCast, setHaveInitializedCast] = useState(false);
const router = useRouter();
useEffect(() => {
@ -27,12 +23,10 @@ export default function Index() {
setPairingCode(r.pairingCode);
});
} else {
if (!haveInitializedCast) {
castReceiverLoadingIfNeeded().then((cast) => {
setHaveInitializedCast(true);
advertiseCode(cast, () => pairingCode);
});
}
advertiseOnChromecast(
() => pairingCode,
() => readCastData()?.collectionID,
);
}
}, [pairingCode]);
@ -52,7 +46,6 @@ export default function Index() {
return;
}
log.info("Pairing complete");
storeCastData(data);
await router.push("/slideshow");
} catch (e) {

138
web/apps/cast/src/pages/slideshow.tsx
View file

@ -1,15 +1,16 @@
import log from "@/next/log";
import { ensure } from "@/utils/ensure";
import { styled } from "@mui/material";
import { FilledCircleCheck } from "components/FilledCircleCheck";
import { SlideView } from "components/Slide";
import { useRouter } from "next/router";
import { useEffect, useState } from "react";
import { readCastData, renderableImageURLs } from "services/cast";
import { readCastData } from "services/cast-data";
import { isChromecast } from "services/chromecast";
import { imageURLGenerator } from "services/render";
export default function Slideshow() {
const [loading, setLoading] = useState(true);
const [imageURL, setImageURL] = useState<string | undefined>();
const [nextImageURL, setNextImageURL] = useState<string | undefined>();
const [isEmpty, setIsEmpty] = useState(false);
const router = useRouter();
@ -22,19 +23,18 @@ export default function Slideshow() {
const loop = async () => {
try {
const urlGenerator = renderableImageURLs(readCastData());
const urlGenerator = imageURLGenerator(ensure(readCastData()));
while (!stop) {
const { value: urls, done } = await urlGenerator.next();
if (done) {
const { value: url, done } = await urlGenerator.next();
if (done || !url) {
// No items in this callection can be shown.
setIsEmpty(true);
// Go back to pairing screen after 3 seconds.
// Go back to pairing screen after 5 seconds.
setTimeout(pair, 5000);
return;
}
setImageURL(urls[0]);
setNextImageURL(urls[1]);
setImageURL(url);
setLoading(false);
}
} catch (e) {
@ -50,12 +50,14 @@ export default function Slideshow() {
};
}, []);
console.log("Rendering slideshow", { loading, imageURL, nextImageURL });
if (loading) return <PairingComplete />;
if (isEmpty) return <NoItems />;
return <SlideView url={imageURL} nextURL={nextImageURL} />;
return isChromecast() ? (
<SlideViewChromecast url={imageURL} />
) : (
<SlideView url={imageURL} />
);
}
const PairingComplete: React.FC = () => {
@ -71,19 +73,13 @@ const PairingComplete: React.FC = () => {
);
};
const Message: React.FC<React.PropsWithChildren> = ({ children }) => {
return (
<Message_>
<MessageItems>{children}</MessageItems>
</Message_>
);
};
const Message_ = styled("div")`
const Message = styled("div")`
display: flex;
min-height: 100svh;
flex-direction: column;
height: 100%;
justify-content: center;
align-items: center;
text-align: center;
line-height: 1.5rem;
@ -92,13 +88,6 @@ const Message_ = styled("div")`
}
`;
const MessageItems = styled("div")`
display: flex;
flex-direction: column;
align-items: center;
text-align: center;
`;
const NoItems: React.FC = () => {
return (
<Message>
@ -110,3 +99,94 @@ const NoItems: React.FC = () => {
</Message>
);
};
interface SlideViewProps {
/** The URL of the image to show. */
url: string;
}
const SlideView: React.FC<SlideViewProps> = ({ url }) => {
return (
<SlideView_ style={{ backgroundImage: `url(${url})` }}>
<img src={url} decoding="sync" alt="" />
</SlideView_>
);
};
const SlideView_ = styled("div")`
width: 100%;
height: 100%;
background-size: cover;
background-position: center;
background-repeat: no-repeat;
background-blend-mode: multiply;
background-color: rgba(0, 0, 0, 0.5);
/* Smooth out the transition a bit.
*
* For the img itself, we set decoding="sync" to have it switch seamlessly.
* But there does not seem to be a way of setting decoding sync for the
* background image, and for large (multi-MB) images the background image
* switch is still visually non-atomic.
*
* As a workaround, add a long transition so that the background image
* transitions in a more "fade-to" manner. This effect might or might not be
* visually the best though.
*
* Does not work in Firefox, but that's fine, this is only a slight tweak,
* not a functional requirement.
*/
transition: all 2s;
img {
width: 100%;
height: 100%;
backdrop-filter: blur(10px);
object-fit: contain;
}
`;
/**
* Variant of {@link SlideView} for use when we're running on Chromecast.
*
* Chromecast devices have trouble with
*
* backdrop-filter: blur(10px);
*
* So emulate a cheaper approximation for use on Chromecast.
*/
const SlideViewChromecast: React.FC<SlideViewProps> = ({ url }) => {
return (
<SlideViewChromecast_>
<img className="svc-bg" src={url} alt="" />
<img className="svc-content" src={url} decoding="sync" alt="" />
</SlideViewChromecast_>
);
};
const SlideViewChromecast_ = styled("div")`
width: 100%;
height: 100%;
/* We can't set opacity of background-image, so use a wrapper */
position: relative;
overflow: hidden;
img.svc-bg {
position: absolute;
left: 0;
top: 0;
width: 100%;
height: 100%;
object-fit: cover;
opacity: 0.1;
}
img.svc-content {
position: relative;
width: 100%;
height: 100%;
object-fit: contain;
}
`;

41
web/apps/cast/src/services/cast-data.ts Normal file
View file

@ -0,0 +1,41 @@
export interface CastData {
/** The ID of the callection we are casting. */
collectionID: string;
/** A key to decrypt the collection we are casting. */
collectionKey: string;
/** A credential to use for fetching media files for this cast session. */
castToken: string;
}
/**
* Save the data received after pairing with a sender into local storage.
*
* We will read in back when we start the slideshow.
*/
export const storeCastData = (payload: unknown) => {
if (!payload || typeof payload != "object")
throw new Error("Unexpected cast data");
// Iterate through all the keys of the payload object and save them to
// localStorage. We don't validate here, we'll validate when we read these
// values back in `readCastData`.
for (const key in payload) {
window.localStorage.setItem(key, payload[key]);
}
};
/**
* Read back the cast data we got after pairing.
*
* Sibling of {@link storeCastData}. It returns undefined if the expected data
* is not present in localStorage.
*/
export const readCastData = (): CastData | undefined => {
const collectionID = localStorage.getItem("collectionID");
const collectionKey = localStorage.getItem("collectionKey");
const castToken = localStorage.getItem("castToken");
return collectionID && collectionKey && castToken
? { collectionID, collectionKey, castToken }
: undefined;
};

227
web/apps/cast/src/services/chromecast.ts Normal file
View file

@ -0,0 +1,227 @@
/// <reference types="chromecast-caf-receiver" />
import log from "@/next/log";
export type Cast = typeof cast;
/**
* A holder for the "cast" global object exposed by the Chromecast SDK,
* alongwith auxiliary state we need around it.
*/
class CastReceiver {
/**
* A reference to the `cast` global object that the Chromecast Web Receiver
* SDK attaches to the window.
*
* https://developers.google.com/cast/docs/web_receiver/basic
*/
cast: Cast | undefined;
/**
* A promise that allows us to ensure multiple requests to load are funneled
* through the same reified load.
*/
loader: Promise<Cast> | undefined;
/**
* True if we have already attached listeners (i.e. if we have "started" the
* Chromecast SDK).
*
* Note that "stopping" the Chromecast SDK causes the Chromecast device to
* reload our tab, so this is a one way flag. The stop is something that'll
* only get triggered when we're actually running on a Chromecast since it
* always happens in response to a message handler.
*/
haveStarted = false;
/**
* Cached result of the isChromecast test.
*/
isChromecast: boolean | undefined;
/**
* A callback to invoke to get the pairing code when we get a new incoming
* pairing request.
*/
pairingCode: (() => string | undefined) | undefined;
/**
* A callback to invoke to get the ID of the collection that is currently
* being shown (if any).
*/
collectionID: (() => string | undefined) | undefined;
}
/** Singleton instance of {@link CastReceiver}. */
const castReceiver = new CastReceiver();
/**
* Listen for incoming messages on the given {@link cast} receiver, replying to
* each of them with a pairing code obtained using the given {@link pairingCode}
* callback. Phase 2 of the pairing protocol.
*
* Calling this function multiple times is fine. The first time around, the
* Chromecast SDK will be loaded and will start listening. Subsequently, each
* time this is call, we'll update the callbacks, but otherwise just return
* immediately (letting the already attached listeners do their thing).
*
* @param pairingCode A callback to invoke to get the pairing code when we get a
* new incoming pairing request.
*
* @param collectionID A callback to invoke to get the ID of the collection that
* is currently being shown (if any).
*
* See: [Note: Pairing protocol].
*/
export const advertiseOnChromecast = (
pairingCode: () => string | undefined,
collectionID: () => string | undefined,
) => {
// Always update the callbacks.
castReceiver.pairingCode = pairingCode;
castReceiver.collectionID = collectionID;
// No-op if we're already running.
if (castReceiver.haveStarted) return;
void loadingChromecastSDKIfNeeded().then((cast) => advertiseCode(cast));
};
/**
* Load the Chromecast Web Receiver SDK and return a reference to the `cast`
* global object that the SDK attaches to the window.
*
* Calling this function multiple times is fine, once the Chromecast SDK is
* loaded it'll thereafter return the reference to the same object always.
*/
const loadingChromecastSDKIfNeeded = async (): Promise<Cast> => {
if (castReceiver.cast) return castReceiver.cast;
if (castReceiver.loader) return await castReceiver.loader;
castReceiver.loader = new Promise((resolve) => {
const script = document.createElement("script");
script.src =
"https://www.gstatic.com/cast/sdk/libs/caf_receiver/v3/cast_receiver_framework.js";
script.addEventListener("load", () => {
castReceiver.cast = cast;
resolve(cast);
});
document.body.appendChild(script);
});
return await castReceiver.loader;
};
const advertiseCode = (cast: Cast) => {
if (castReceiver.haveStarted) {
// Multiple attempts raced to completion, ignore all but the first.
return;
}
castReceiver.haveStarted = true;
// Prepare the Chromecast "context".
const context = cast.framework.CastReceiverContext.getInstance();
const namespace = "urn:x-cast:pair-request";
const options = new cast.framework.CastReceiverOptions();
// We don't use the media features of the Cast SDK.
options.skipPlayersLoad = true;
// Do not stop the casting if the receiver is unreachable. A user should be
// able to start a cast on their phone and then put it away, leaving the
// cast running on their big screen.
options.disableIdleTimeout = true;
type ListenerProps = {
senderId: string;
data: unknown;
};
// Reply with the code that we have if anyone asks over Chromecast.
const incomingMessageListener = ({ senderId, data }: ListenerProps) => {
// The collection ID with is currently paired (if any).
const pairedCollectionID = castReceiver.collectionID?.();
// The collection ID in the request (if any).
const collectionID =
data &&
typeof data == "object" &&
typeof data["collectionID"] == "string"
? data["collectionID"]
: undefined;
// If the request does not have a collectionID (or if we're not showing
// anything currently), forego this check.
if (collectionID && pairedCollectionID) {
// If we get another connection request for a _different_ collection
// ID, stop the app to allow the second device to reconnect using a
// freshly generated pairing code.
if (pairedCollectionID != collectionID) {
log.info(`request for a new collection ${collectionID}`);
context.stop();
} else {
// Duplicate request for same collection that we're already
// showing. Ignore.
}
return;
}
const code = castReceiver.pairingCode?.();
if (!code) {
// No code, but if we're already showing a collection, then ignore.
if (pairedCollectionID) return;
// Our caller waits until it has a pairing code before it calls
// `advertiseCode`, but there is still an edge case where we can
// find ourselves without a pairing code:
//
// 1. The current pairing code expires. We start the process to get
// a new one.
//
// 2. But before that happens, someone connects.
//
// The window where this can happen is short, so if we do find
// ourselves in this scenario, just shutdown.
log.error("got pairing request when refreshing pairing codes");
context.stop();
return;
}
context.sendCustomMessage(namespace, senderId, { code });
};
context.addCustomMessageListener(
namespace,
// We need to cast, the `senderId` is present in the message we get but
// not present in the TypeScript type.
incomingMessageListener as unknown as SystemEventHandler,
);
// Close the (chromecast) tab if the sender disconnects.
//
// Chromecast does a "shutdown" of our cast app when we call `context.stop`.
// This translates into it closing the tab where it is showing our app.
context.addEventListener(
cast.framework.system.EventType.SENDER_DISCONNECTED,
() => context.stop(),
);
// Start listening for Chromecast connections.
context.start(options);
};
/**
* Return true if we're running on a Chromecast device.
*
* This allows changing our app's behaviour when we're running on Chromecast.
* Such checks are needed because during our testing we found that in practice,
* some processing is too heavy for Chromecast hardware (we tested with a 2nd
* gen device, this might not be true for newer variants).
*
* This variable is lazily updated when we enter {@link renderableImageURLs}. It
* is kept at the top level to avoid passing it around.
*/
export const isChromecast = () => {
let isCast = castReceiver.isChromecast;
if (isCast === undefined) {
isCast = window.navigator.userAgent.includes("CrKey");
castReceiver.isChromecast = isCast;
}
return isCast;
};

3
web/apps/cast/src/services/detect-type.ts
View file

@ -9,6 +9,9 @@ import FileType from "file-type";
*
* It first peeks into the file's initial contents to detect the MIME type. If
* that doesn't give any results, it tries to deduce it from the file's name.
*
* For the list of returned extensions, see (for our installed version):
* https://github.com/sindresorhus/file-type/blob/main/core.d.ts
*/
export const detectMediaMIMEType = async (file: File): Promise<string> => {
const chunkSizeForTypeDetection = 4100;

102
web/apps/cast/src/services/pair.ts
View file

@ -1,9 +1,8 @@
import log from "@/next/log";
import { wait } from "@/utils/promise";
import { boxSealOpen, toB64 } from "@ente/shared/crypto/internal/libsodium";
import castGateway from "@ente/shared/network/cast";
import { wait } from "@ente/shared/utils";
import _sodium from "libsodium-wrappers";
import { type Cast } from "../utils/cast-receiver";
export interface Registration {
/** A pairing code shown on the screen. A client can use this to connect. */
@ -100,105 +99,6 @@ export const register = async (): Promise<Registration> => {
return { pairingCode, publicKeyB64, privateKeyB64 };
};
/**
* Listen for incoming messages on the given {@link cast} receiver, replying to
* each of them with a pairing code obtained using the given {@link pairingCode}
* callback. Phase 2 of the pairing protocol.
*
* See: [Note: Pairing protocol].
*/
export const advertiseCode = (
cast: Cast,
pairingCode: () => string | undefined,
) => {
// Prepare the Chromecast "context".
const context = cast.framework.CastReceiverContext.getInstance();
const namespace = "urn:x-cast:pair-request";
const options = new cast.framework.CastReceiverOptions();
// We don't use the media features of the Cast SDK.
options.skipPlayersLoad = true;
// Do not stop the casting if the receiver is unreachable. A user should be
// able to start a cast on their phone and then put it away, leaving the
// cast running on their big screen.
options.disableIdleTimeout = true;
// The collection ID with which we paired. If we get another connection
// request for a different collection ID, restart the app to allow them to
// reconnect using a freshly generated pairing code.
//
// If the request does not have a collectionID, forego this check.
let pairedCollectionID: string | undefined;
type ListenerProps = {
senderId: string;
data: unknown;
};
// Reply with the code that we have if anyone asks over Chromecast.
const incomingMessageListener = ({ senderId, data }: ListenerProps) => {
const restart = (reason: string) => {
log.error(`Restarting app because ${reason}`);
// context.stop will close the tab but it'll get reopened again
// immediately since the client app will reconnect in the scenarios
// where we're calling this function.
context.stop();
};
const collectionID =
data &&
typeof data == "object" &&
typeof data["collectionID"] == "string"
? data["collectionID"]
: undefined;
if (pairedCollectionID && pairedCollectionID != collectionID) {
restart(`incoming request for a new collection ${collectionID}`);
return;
}
pairedCollectionID = collectionID;
const code = pairingCode();
if (!code) {
// Our caller waits until it has a pairing code before it calls
// `advertiseCode`, but there is still an edge case where we can
// find ourselves without a pairing code:
//
// 1. The current pairing code expires. We start the process to get
// a new one.
//
// 2. But before that happens, someone connects.
//
// The window where this can happen is short, so if we do find
// ourselves in this scenario,
restart("we got a pairing request when refreshing pairing codes");
return;
}
context.sendCustomMessage(namespace, senderId, { code });
};
context.addCustomMessageListener(
namespace,
// We need to cast, the `senderId` is present in the message we get but
// not present in the TypeScript type.
incomingMessageListener as unknown as SystemEventHandler,
);
// Close the (chromecast) tab if the sender disconnects.
//
// Chromecast does a "shutdown" of our cast app when we call `context.stop`.
// This translates into it closing the tab where it is showing our app.
context.addEventListener(
cast.framework.system.EventType.SENDER_DISCONNECTED,
() => context.stop(),
);
// Start listening for Chromecast connections.
context.start(options);
};
/**
* Ask museum if anyone has sent a (encrypted) payload corresponding to the
* given pairing code. If so, decrypt it using our private key and return the

206
web/apps/cast/src/services/cast.ts → web/apps/cast/src/services/render.ts
View file

@ -1,14 +1,23 @@
import { FILE_TYPE } from "@/media/file-type";
import { isNonWebImageFileExtension } from "@/media/formats";
import { isHEICExtension, isNonWebImageFileExtension } from "@/media/formats";
import { decodeLivePhoto } from "@/media/live-photo";
import { createHEICConvertComlinkWorker } from "@/media/worker/heic-convert";
import type { DedicatedHEICConvertWorker } from "@/media/worker/heic-convert.worker";
import { nameAndExtension } from "@/next/file";
import log from "@/next/log";
import type { ComlinkWorker } from "@/next/worker/comlink-worker";
import { shuffled } from "@/utils/array";
import { ensure, ensureString } from "@/utils/ensure";
import { wait } from "@/utils/promise";
import ComlinkCryptoWorker from "@ente/shared/crypto";
import { ApiError } from "@ente/shared/error";
import HTTPService from "@ente/shared/network/HTTPService";
import { getCastFileURL, getEndpoint } from "@ente/shared/network/api";
import { wait } from "@ente/shared/utils";
import {
getCastFileURL,
getCastThumbnailURL,
getEndpoint,
} from "@ente/shared/network/api";
import type { AxiosResponse } from "axios";
import type { CastData } from "services/cast-data";
import { detectMediaMIMEType } from "services/detect-type";
import {
EncryptedEnteFile,
@ -16,53 +25,20 @@ import {
FileMagicMetadata,
FilePublicMagicMetadata,
} from "types/file";
import { isChromecast } from "./chromecast";
/**
* Save the data received after pairing with a sender into local storage.
*
* We will read in back when we start the slideshow.
* If we're using HEIC conversion, then this variable caches the comlink web
* worker we're using to perform the actual conversion.
*/
export const storeCastData = (payload: unknown) => {
if (!payload || typeof payload != "object")
throw new Error("Unexpected cast data");
// Iterate through all the keys of the payload object and save them to
// localStorage. We don't validate here, we'll validate when we read these
// values back in `readCastData`.
for (const key in payload) {
window.localStorage.setItem(key, payload[key]);
}
};
interface CastData {
/** A key to decrypt the collection we are casting. */
collectionKey: string;
/** A credential to use for fetching media files for this cast session. */
castToken: string;
}
/**
* Read back the cast data we got after pairing.
*
* Sibling of {@link storeCastData}. It throws an error if the expected data is
* not present in localStorage.
*/
export const readCastData = (): CastData => {
const collectionKey = ensureString(localStorage.getItem("collectionKey"));
const castToken = ensureString(localStorage.getItem("castToken"));
return { collectionKey, castToken };
};
type RenderableImageURLPair = [url: string, nextURL: string];
let heicWorker: ComlinkWorker<typeof DedicatedHEICConvertWorker> | undefined;
/**
* An async generator function that loops through all the files in the
* collection, returning renderable URLs to each that can be displayed in a
* slideshow.
* collection, returning renderable image URLs to each that can be displayed in
* a slideshow.
*
* Each time it resolves with a pair of URLs (a {@link RenderableImageURLPair}),
* one for the next slideshow image, and one for the slideshow image that will
* be displayed after that. It also pre-fetches the next to next URL each time.
* Each time it resolves with a (data) URL for the slideshow image to show next.
*
* If there are no renderable image in the collection, the sequence ends by
* yielding `{done: true}`.
@ -73,14 +49,18 @@ type RenderableImageURLPair = [url: string, nextURL: string];
*
* The generator ignores errors in the fetching and decoding of individual
* images in the collection, skipping the erroneous ones and moving onward to
* the next one. It will however throw if there are errors when getting the
* collection itself. This can happen both the first time, or when we are about
* to loop around to the start of the collection.
* the next one.
*
* - It will however throw if there are errors when getting the collection
* itself. This can happen both the first time, or when we are about to loop
* around to the start of the collection.
*
* - It will also throw if three consecutive image fail.
*
* @param castData The collection to show and credentials to fetch the files
* within it.
*/
export const renderableImageURLs = async function* (castData: CastData) {
export const imageURLGenerator = async function* (castData: CastData) {
const { collectionKey, castToken } = castData;
/**
@ -89,11 +69,8 @@ export const renderableImageURLs = async function* (castData: CastData) {
*/
const previousURLs: string[] = [];
/** The URL pair that we will yield */
const urls: string[] = [];
/** Number of milliseconds to keep the slide on the screen. */
const slideDuration = 10000; /* 10 s */
const slideDuration = 12000; /* 12 s */
/**
* Time when we last yielded.
@ -108,6 +85,14 @@ export const renderableImageURLs = async function* (castData: CastData) {
// bit, for the user to see the checkmark animation as reassurance).
lastYieldTime -= slideDuration - 2500; /* wait at most 2.5 s */
/**
* Number of time we have caught an exception while trying to generate an
* image URL for individual files.
*
* When this happens three times consecutively, we throw.
*/
let consecutiveFailures = 0;
while (true) {
const encryptedFiles = shuffled(
await getEncryptedCollectionFiles(castToken),
@ -118,30 +103,34 @@ export const renderableImageURLs = async function* (castData: CastData) {
for (const encryptedFile of encryptedFiles) {
const file = await decryptEnteFile(encryptedFile, collectionKey);
if (!isFileEligibleForCast(file)) continue;
if (!isFileEligible(file)) continue;
console.log("will start createRenderableURL", new Date());
let url: string;
try {
urls.push(await createRenderableURL(castToken, file));
url = await createRenderableURL(castToken, file);
consecutiveFailures = 0;
haveEligibleFiles = true;
} catch (e) {
consecutiveFailures += 1;
// 1, 2, bang!
if (consecutiveFailures == 3) throw e;
if (e instanceof ApiError && e.httpStatusCode == 401) {
// The token has expired. This can happen, e.g., if the user
// opens the dialog to cast again, causing the client to
// invalidate existing tokens.
//
// Rethrow the error, which will bring us back to the
// pairing page.
throw e;
}
// On all other errors (including temporary network issues),
log.error("Skipping unrenderable file", e);
await wait(100); /* Breathe */
continue;
}
console.log("did end createRenderableURL", new Date());
// Need at least a pair.
//
// There are two scenarios:
//
// - First run: urls will initially be empty, so gobble two.
//
// - Subsequently, urls will have the "next" / "preloaded" URL left
// over from the last time. We'll promote that to being the one
// that'll get displayed, and preload another one.
// if (urls.length < 2) continue;
// The last element of previousURLs is the URL that is currently
// being shown on screen.
//
@ -150,23 +139,14 @@ export const renderableImageURLs = async function* (castData: CastData) {
if (previousURLs.length > 1)
URL.revokeObjectURL(previousURLs.shift());
// The URL that'll now get displayed on screen.
const url = ensure(urls.shift());
// The URL that we're preloading for next time around.
const nextURL = ""; //ensure(urls[0]);
previousURLs.push(url);
const urlPair: RenderableImageURLPair = [url, nextURL];
const elapsedTime = Date.now() - lastYieldTime;
if (elapsedTime > 0 && elapsedTime < slideDuration) {
console.log("waiting", slideDuration - elapsedTime);
if (elapsedTime > 0 && elapsedTime < slideDuration)
await wait(slideDuration - elapsedTime);
}
lastYieldTime = Date.now();
yield urlPair;
yield url;
}
// This collection does not have any files that we can show.
@ -185,7 +165,7 @@ const getEncryptedCollectionFiles = async (
): Promise<EncryptedEnteFile[]> => {
let files: EncryptedEnteFile[] = [];
let sinceTime = 0;
let resp;
let resp: AxiosResponse;
do {
resp = await HTTPService.get(
`${getEndpoint()}/cast/diff`,
@ -269,12 +249,19 @@ const decryptEnteFile = async (
return file;
};
const isFileEligibleForCast = (file: EnteFile) => {
const isFileEligible = (file: EnteFile) => {
if (!isImageOrLivePhoto(file)) return false;
if (file.info.fileSize > 100 * 1024 * 1024) return false;
// This check is fast but potentially incorrect because in practice we do
// encounter files that are incorrectly named and have a misleading
// extension. To detect the actual type, we need to sniff the MIME type, but
// that requires downloading and decrypting the file first.
const [, extension] = nameAndExtension(file.metadata.title);
if (isNonWebImageFileExtension(extension)) return false;
if (isNonWebImageFileExtension(extension)) {
// Of the known non-web types, we support HEIC.
return isHEICExtension(extension);
}
return true;
};
@ -284,6 +271,12 @@ const isImageOrLivePhoto = (file: EnteFile) => {
return fileType == FILE_TYPE.IMAGE || fileType == FILE_TYPE.LIVE_PHOTO;
};
export const heicToJPEG = async (heicBlob: Blob) => {
let worker = heicWorker;
if (!worker) heicWorker = worker = createHEICConvertComlinkWorker();
return await (await worker.remote).heicToJPEG(heicBlob);
};
/**
* Create and return a new data URL that can be used to show the given
* {@link file} in our slideshow image viewer.
@ -291,29 +284,50 @@ const isImageOrLivePhoto = (file: EnteFile) => {
* Once we're done showing the file, the URL should be revoked using
* {@link URL.revokeObjectURL} to free up browser resources.
*/
const createRenderableURL = async (castToken: string, file: EnteFile) =>
URL.createObjectURL(await renderableImageBlob(castToken, file));
const createRenderableURL = async (castToken: string, file: EnteFile) => {
const imageBlob = await renderableImageBlob(castToken, file);
return URL.createObjectURL(imageBlob);
};
const renderableImageBlob = async (castToken: string, file: EnteFile) => {
const fileName = file.metadata.title;
let blob = await downloadFile(castToken, file);
if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
const { imageData } = await decodeLivePhoto(fileName, blob);
const shouldUseThumbnail = isChromecast();
let blob = await downloadFile(castToken, file, shouldUseThumbnail);
let fileName = file.metadata.title;
if (!shouldUseThumbnail && file.metadata.fileType == FILE_TYPE.LIVE_PHOTO) {
const { imageData, imageFileName } = await decodeLivePhoto(
fileName,
blob,
);
fileName = imageFileName;
blob = new Blob([imageData]);
}
// We cannot rely on the file's extension to detect the file type, some
// files are incorrectly named. So use a MIME type sniffer first, but if
// that fails than fallback to the extension.
const mimeType = await detectMediaMIMEType(new File([blob], fileName));
if (!mimeType)
throw new Error(`Could not detect MIME type for file ${fileName}`);
if (mimeType == "image/heif" || mimeType == "image/heic")
blob = await heicToJPEG(blob);
return new Blob([blob], { type: mimeType });
};
const downloadFile = async (castToken: string, file: EnteFile) => {
const downloadFile = async (
castToken: string,
file: EnteFile,
shouldUseThumbnail: boolean,
) => {
if (!isImageOrLivePhoto(file))
throw new Error("Can only cast images and live photos");
const url = getCastFileURL(file.id);
// TODO(MR): Remove if usused eventually
// const url = getCastThumbnailURL(file.id);
const url = shouldUseThumbnail
? getCastThumbnailURL(file.id)
: getCastFileURL(file.id);
const resp = await HTTPService.get(
url,
null,
@ -327,9 +341,11 @@ const downloadFile = async (castToken: string, file: EnteFile) => {
const cryptoWorker = await ComlinkCryptoWorker.getInstance();
const decrypted = await cryptoWorker.decryptFile(
new Uint8Array(resp.data),
await cryptoWorker.fromB64(file.file.decryptionHeader),
// TODO(MR): Remove if usused eventually
// await cryptoWorker.fromB64(file.thumbnail.decryptionHeader),
await cryptoWorker.fromB64(
shouldUseThumbnail
? file.thumbnail.decryptionHeader
: file.file.decryptionHeader,
),
file.key,
);
return new Response(decrypted).blob();

32
web/apps/cast/src/utils/cast-receiver.tsx
View file

@ -1,32 +0,0 @@
/// <reference types="chromecast-caf-receiver" />
export type Cast = typeof cast;
let _cast: Cast | undefined;
let _loader: Promise<Cast> | undefined;
/**
* Load the Chromecast Web Receiver SDK and return a reference to the `cast`
* global object that the SDK attaches to the window.
*
* Calling this function multiple times is fine, once the Chromecast SDK is
* loaded it'll thereafter return the reference to the same object always.
*
* https://developers.google.com/cast/docs/web_receiver/basic
*/
export const castReceiverLoadingIfNeeded = async (): Promise<Cast> => {
if (_cast) return _cast;
if (_loader) return await _loader;
_loader = new Promise((resolve) => {
const script = document.createElement("script");
script.src =
"https://www.gstatic.com/cast/sdk/libs/caf_receiver/v3/cast_receiver_framework.js";
script.addEventListener("load", () => resolve(cast));
document.body.appendChild(script);
});
const c = await _loader;
_cast = c;
return c;
};

1
web/apps/photos/package.json
View file

@ -23,7 +23,6 @@
"ffmpeg-wasm": "file:./thirdparty/ffmpeg-wasm",
"formik": "^2.1.5",
"hdbscan": "0.0.1-alpha.5",
"heic-convert": "^2.0.0",
"idb": "^7.1.1",
"leaflet": "^1.9.4",
"leaflet-defaulticon-compatibility": "^0.1.1",

27
web/apps/photos/src/components/Collections/CollectionOptions/AlbumCastDialog.tsx
View file

@ -32,7 +32,11 @@ declare global {
}
}
export default function AlbumCastDialog(props: Props) {
export default function AlbumCastDialog({
show,
onHide,
currentCollection,
}: Props) {
const [view, setView] = useState<
"choose" | "auto" | "pin" | "auto-cast-error"
>("choose");
@ -51,7 +55,7 @@ export default function AlbumCastDialog(props: Props) {
) => {
try {
await doCast(value.trim());
props.onHide();
onHide();
} catch (e) {
const error = e as Error;
let fieldError: string;
@ -80,8 +84,8 @@ export default function AlbumCastDialog(props: Props) {
// ok, they exist. let's give them the good stuff.
const payload = JSON.stringify({
castToken: castToken,
collectionID: props.currentCollection.id,
collectionKey: props.currentCollection.key,
collectionID: currentCollection.id,
collectionKey: currentCollection.key,
});
const encryptedPayload = await boxSeal(btoa(payload), tvPublicKeyB64);
@ -89,7 +93,7 @@ export default function AlbumCastDialog(props: Props) {
await castGateway.publishCastPayload(
pin,
encryptedPayload,
props.currentCollection.id,
currentCollection.id,
castToken,
);
};
@ -119,7 +123,7 @@ export default function AlbumCastDialog(props: Props) {
doCast(code)
.then(() => {
setView("choose");
props.onHide();
onHide();
})
.catch((e) => {
setView("auto-cast-error");
@ -129,8 +133,9 @@ export default function AlbumCastDialog(props: Props) {
},
);
const collectionID = currentCollection.id;
session
.sendMessage("urn:x-cast:pair-request", {})
.sendMessage("urn:x-cast:pair-request", { collectionID })
.then(() => {
log.debug(() => "Message sent successfully");
})
@ -142,16 +147,16 @@ export default function AlbumCastDialog(props: Props) {
}, [view]);
useEffect(() => {
if (props.show) {
if (show) {
castGateway.revokeAllTokens();
}
}, [props.show]);
}, [show]);
return (
<DialogBoxV2
sx={{ zIndex: 1600 }}
open={props.show}
onClose={props.onHide}
open={show}
onClose={onHide}
attributes={{
title: t("CAST_ALBUM_TO_TV"),
}}

10
web/apps/photos/src/components/Collections/CollectionOptions/SharedCollectionOption.tsx
View file

@ -1,6 +1,7 @@
import { OverflowMenuOption } from "@ente/shared/components/OverflowMenu/option";
import ArchiveOutlined from "@mui/icons-material/ArchiveOutlined";
import LogoutIcon from "@mui/icons-material/Logout";
import TvIcon from "@mui/icons-material/Tv";
import Unarchive from "@mui/icons-material/Unarchive";
import { t } from "i18next";
import { CollectionActions } from ".";
@ -45,6 +46,15 @@ export function SharedCollectionOption({
>
{t("LEAVE_ALBUM")}
</OverflowMenuOption>
<OverflowMenuOption
startIcon={<TvIcon />}
onClick={handleCollectionAction(
CollectionActions.SHOW_ALBUM_CAST_DIALOG,
false,
)}
>
{t("CAST_ALBUM_TO_TV")}
</OverflowMenuOption>
</>
);
}

2
web/apps/photos/src/services/export/index.ts
View file

@ -3,12 +3,12 @@ import { decodeLivePhoto } from "@/media/live-photo";
import type { Metadata } from "@/media/types/file";
import { ensureElectron } from "@/next/electron";
import log from "@/next/log";
import { wait } from "@/utils/promise";
import { CustomError } from "@ente/shared/error";
import { Events, eventBus } from "@ente/shared/events";
import { LS_KEYS, getData, setData } from "@ente/shared/storage/localStorage";
import { formatDateTimeShort } from "@ente/shared/time/format";
import { User } from "@ente/shared/user/types";
import { wait } from "@ente/shared/utils";
import QueueProcessor, {
CancellationStatus,
RequestCanceller,

2
web/apps/photos/src/services/export/migration.ts
View file

@ -3,9 +3,9 @@ import { decodeLivePhoto } from "@/media/live-photo";
import { ensureElectron } from "@/next/electron";
import { nameAndExtension } from "@/next/file";
import log from "@/next/log";
import { wait } from "@/utils/promise";
import { LS_KEYS, getData } from "@ente/shared/storage/localStorage";
import { User } from "@ente/shared/user/types";
import { wait } from "@ente/shared/utils";
import { getLocalCollections } from "services/collectionService";
import downloadManager from "services/download";
import { getAllLocalFiles } from "services/fileService";

13
web/apps/photos/src/services/heic-convert.ts
View file

@ -1,9 +1,10 @@
import { createHEICConvertComlinkWorker } from "@/media/worker/heic-convert";
import type { DedicatedHEICConvertWorker } from "@/media/worker/heic-convert.worker";
import log from "@/next/log";
import { ComlinkWorker } from "@/next/worker/comlink-worker";
import { CustomError } from "@ente/shared/error";
import { retryAsyncFunction } from "@ente/shared/utils";
import QueueProcessor from "@ente/shared/utils/queueProcessor";
import { type DedicatedHEICConvertWorker } from "worker/heic-convert.worker";
/**
* Convert a HEIC image to a JPEG.
@ -29,7 +30,7 @@ class HEICConverter {
if (this.workerPool.length > 0) return;
this.workerPool = [];
for (let i = 0; i < WORKER_POOL_SIZE; i++)
this.workerPool.push(createComlinkWorker());
this.workerPool.push(createHEICConvertComlinkWorker());
}
async convert(fileBlob: Blob): Promise<Blob> {
@ -79,7 +80,7 @@ class HEICConverter {
} catch (e) {
log.error("HEIC conversion failed", e);
convertWorker.terminate();
this.workerPool.push(createComlinkWorker());
this.workerPool.push(createHEICConvertComlinkWorker());
throw e;
}
}, WAIT_TIME_BEFORE_NEXT_ATTEMPT_IN_MICROSECONDS),
@ -99,9 +100,3 @@ class HEICConverter {
/** The singleton instance of {@link HEICConverter}. */
const converter = new HEICConverter();
const createComlinkWorker = () =>
new ComlinkWorker<typeof DedicatedHEICConvertWorker>(
"heic-convert-worker",
new Worker(new URL("worker/heic-convert.worker.ts", import.meta.url)),
);

103
web/apps/photos/src/services/upload/thumbnail.ts
View file

@ -1,7 +1,9 @@
import { FILE_TYPE, type FileTypeInfo } from "@/media/file-type";
import { scaledImageDimensions } from "@/media/image";
import log from "@/next/log";
import { type Electron } from "@/next/types/ipc";
import { withTimeout } from "@ente/shared/utils";
import { ensure } from "@/utils/ensure";
import { withTimeout } from "@/utils/promise";
import * as ffmpeg from "services/ffmpeg";
import { heicToJPEG } from "services/heic-convert";
import { toDataOrPathOrZipEntry, type DesktopUploadItem } from "./types";
@ -30,10 +32,10 @@ export const generateThumbnailWeb = async (
fileTypeInfo: FileTypeInfo,
): Promise<Uint8Array> =>
fileTypeInfo.fileType === FILE_TYPE.IMAGE
? await generateImageThumbnailUsingCanvas(blob, fileTypeInfo)
? await generateImageThumbnailWeb(blob, fileTypeInfo)
: await generateVideoThumbnailWeb(blob);
const generateImageThumbnailUsingCanvas = async (
const generateImageThumbnailWeb = async (
blob: Blob,
{ extension }: FileTypeInfo,
) => {
@ -42,8 +44,12 @@ const generateImageThumbnailUsingCanvas = async (
blob = await heicToJPEG(blob);
}
return generateImageThumbnailUsingCanvas(blob);
};
const generateImageThumbnailUsingCanvas = async (blob: Blob) => {
const canvas = document.createElement("canvas");
const canvasCtx = canvas.getContext("2d");
const canvasCtx = ensure(canvas.getContext("2d"));
const imageURL = URL.createObjectURL(blob);
await withTimeout(
@ -53,7 +59,7 @@ const generateImageThumbnailUsingCanvas = async (
image.onload = () => {
try {
URL.revokeObjectURL(imageURL);
const { width, height } = scaledThumbnailDimensions(
const { width, height } = scaledImageDimensions(
image.width,
image.height,
maxThumbnailDimension,
@ -62,7 +68,7 @@ const generateImageThumbnailUsingCanvas = async (
canvas.height = height;
canvasCtx.drawImage(image, 0, 0, width, height);
resolve(undefined);
} catch (e) {
} catch (e: unknown) {
reject(e);
}
};
@ -73,6 +79,32 @@ const generateImageThumbnailUsingCanvas = async (
return await compressedJPEGData(canvas);
};
const compressedJPEGData = async (canvas: HTMLCanvasElement) => {
let blob: Blob | undefined | null;
let prevSize = Number.MAX_SAFE_INTEGER;
let quality = 0.7;
do {
if (blob) prevSize = blob.size;
blob = await new Promise((resolve) => {
canvas.toBlob((blob) => resolve(blob), "image/jpeg", quality);
});
quality -= 0.1;
} while (
quality >= 0.5 &&
blob &&
blob.size > maxThumbnailSize &&
percentageSizeDiff(blob.size, prevSize) >= 10
);
return new Uint8Array(await ensure(blob).arrayBuffer());
};
const percentageSizeDiff = (
newThumbnailSize: number,
oldThumbnailSize: number,
) => ((oldThumbnailSize - newThumbnailSize) * 100) / oldThumbnailSize;
const generateVideoThumbnailWeb = async (blob: Blob) => {
try {
return await ffmpeg.generateVideoThumbnailWeb(blob);
@ -85,9 +117,9 @@ const generateVideoThumbnailWeb = async (blob: Blob) => {
}
};
const generateVideoThumbnailUsingCanvas = async (blob: Blob) => {
export const generateVideoThumbnailUsingCanvas = async (blob: Blob) => {
const canvas = document.createElement("canvas");
const canvasCtx = canvas.getContext("2d");
const canvasCtx = ensure(canvas.getContext("2d"));
const videoURL = URL.createObjectURL(blob);
await withTimeout(
@ -98,7 +130,7 @@ const generateVideoThumbnailUsingCanvas = async (blob: Blob) => {
video.addEventListener("loadeddata", () => {
try {
URL.revokeObjectURL(videoURL);
const { width, height } = scaledThumbnailDimensions(
const { width, height } = scaledImageDimensions(
video.videoWidth,
video.videoHeight,
maxThumbnailDimension,
@ -118,59 +150,6 @@ const generateVideoThumbnailUsingCanvas = async (blob: Blob) => {
return await compressedJPEGData(canvas);
};
/**
* Compute the size of the thumbnail to create for an image with the given
* {@link width} and {@link height}.
*
* This function calculates a new size of an image for limiting it to maximum
* width and height (both specified by {@link maxDimension}), while maintaining
* aspect ratio.
*
* It returns `{0, 0}` for invalid inputs.
*/
const scaledThumbnailDimensions = (
width: number,
height: number,
maxDimension: number,
): { width: number; height: number } => {
if (width === 0 || height === 0) return { width: 0, height: 0 };
const widthScaleFactor = maxDimension / width;
const heightScaleFactor = maxDimension / height;
const scaleFactor = Math.min(widthScaleFactor, heightScaleFactor);
const thumbnailDimensions = {
width: Math.round(width * scaleFactor),
height: Math.round(height * scaleFactor),
};
if (thumbnailDimensions.width === 0 || thumbnailDimensions.height === 0)
return { width: 0, height: 0 };
return thumbnailDimensions;
};
const compressedJPEGData = async (canvas: HTMLCanvasElement) => {
let blob: Blob;
let prevSize = Number.MAX_SAFE_INTEGER;
let quality = 0.7;
do {
if (blob) prevSize = blob.size;
blob = await new Promise((resolve) => {
canvas.toBlob((blob) => resolve(blob), "image/jpeg", quality);
});
quality -= 0.1;
} while (
quality >= 0.5 &&
blob.size > maxThumbnailSize &&
percentageSizeDiff(blob.size, prevSize) >= 10
);
return new Uint8Array(await blob.arrayBuffer());
};
const percentageSizeDiff = (
newThumbnailSize: number,
oldThumbnailSize: number,
) => ((oldThumbnailSize - newThumbnailSize) * 100) / oldThumbnailSize;
/**
* Generate a JPEG thumbnail for the given file or path using native tools.
*

2
web/apps/photos/src/services/upload/uploadHttpClient.ts
View file

@ -1,9 +1,9 @@
import log from "@/next/log";
import { wait } from "@/utils/promise";
import { CustomError, handleUploadError } from "@ente/shared/error";
import HTTPService from "@ente/shared/network/HTTPService";
import { getEndpoint, getUploadEndpoint } from "@ente/shared/network/api";
import { getToken } from "@ente/shared/storage/localStorage/helpers";
import { wait } from "@ente/shared/utils";
import { EnteFile } from "types/file";
import { MultipartUploadURLs, UploadFile, UploadURL } from "./uploadService";

2
web/apps/photos/src/services/upload/uploadManager.ts
View file

@ -6,11 +6,11 @@ import log from "@/next/log";
import type { Electron } from "@/next/types/ipc";
import { ComlinkWorker } from "@/next/worker/comlink-worker";
import { ensure } from "@/utils/ensure";
import { wait } from "@/utils/promise";
import { getDedicatedCryptoWorker } from "@ente/shared/crypto";
import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker";
import { CustomError } from "@ente/shared/error";
import { Events, eventBus } from "@ente/shared/events";
import { wait } from "@ente/shared/utils";
import { Canceler } from "axios";
import { Remote } from "comlink";
import {

3
web/apps/photos/src/utils/file/index.ts
View file

@ -5,10 +5,11 @@ import { lowercaseExtension } from "@/next/file";
import log from "@/next/log";
import { CustomErrorMessage, type Electron } from "@/next/types/ipc";
import { workerBridge } from "@/next/worker/worker-bridge";
import { withTimeout } from "@/utils/promise";
import ComlinkCryptoWorker from "@ente/shared/crypto";
import { LS_KEYS, getData } from "@ente/shared/storage/localStorage";
import { User } from "@ente/shared/user/types";
import { downloadUsingAnchor, withTimeout } from "@ente/shared/utils";
import { downloadUsingAnchor } from "@ente/shared/utils";
import { t } from "i18next";
import isElectron from "is-electron";
import { moveToHiddenCollection } from "services/collectionService";

8
web/docs/dependencies.md
View file

@ -141,6 +141,14 @@ some cases.
became ESM only - for our limited use case, the custom Webpack configuration
that entails is not worth the upgrade.
- [heic-convert](https://github.com/catdad-experiments/heic-convert) is used
for converting HEIC files (which browsers don't natively support) into JPEG.
## Processing
- [comlink](https://github.com/GoogleChromeLabs/comlink) provides a minimal
layer on top of Web Workers to make them more easier to use.
## Photos app specific
- [react-dropzone](https://github.com/react-dropzone/react-dropzone/) is a

2
web/packages/accounts/components/ChangeEmail.tsx
View file

@ -1,3 +1,4 @@
import { wait } from "@/utils/promise";
import { changeEmail, sendOTTForEmailChange } from "@ente/accounts/api/user";
import { APP_HOMES } from "@ente/shared/apps/constants";
import { PageProps } from "@ente/shared/apps/types";
@ -6,7 +7,6 @@ import FormPaperFooter from "@ente/shared/components/Form/FormPaper/Footer";
import LinkButton from "@ente/shared/components/LinkButton";
import SubmitButton from "@ente/shared/components/SubmitButton";
import { LS_KEYS, getData, setData } from "@ente/shared/storage/localStorage";
import { wait } from "@ente/shared/utils";
import { Alert, Box, TextField } from "@mui/material";
import { Formik, FormikHelpers } from "formik";
import { t } from "i18next";

11
web/packages/accounts/components/two-factor/VerifyForm.tsx
View file

@ -1,16 +1,15 @@
import { Formik, FormikHelpers } from "formik";
import { t } from "i18next";
import { useRef, useState } from "react";
import OtpInput from "react-otp-input";
import { wait } from "@/utils/promise";
import InvalidInputMessage from "@ente/accounts/components/two-factor/InvalidInputMessage";
import {
CenteredFlex,
VerticallyCentered,
} from "@ente/shared/components/Container";
import SubmitButton from "@ente/shared/components/SubmitButton";
import { wait } from "@ente/shared/utils";
import { Box, Typography } from "@mui/material";
import { Formik, FormikHelpers } from "formik";
import { t } from "i18next";
import { useRef, useState } from "react";
import OtpInput from "react-otp-input";
interface formValues {
otp: string;

8
web/packages/media/formats.ts
View file

@ -24,3 +24,11 @@ const nonWebImageFileExtensions = [
*/
export const isNonWebImageFileExtension = (extension: string) =>
nonWebImageFileExtensions.includes(extension.toLowerCase());
/**
* Return `true` if {@link extension} in for an HEIC-like file.
*/
export const isHEICExtension = (extension: string) => {
const ext = extension.toLowerCase();
return ext == "heic" || ext == "heif";
};

33
web/packages/media/image.ts Normal file
View file

@ -0,0 +1,33 @@
/**
* Compute optimal dimensions for a resized version of an image while
* maintaining aspect ratio of the source image.
*
* @param width The width of the source image.
*
* @param height The height of the source image.
*
* @param maxDimension The maximum width of height of the resized image.
*
* This function returns a new size limiting it to maximum width and height
* (both specified by {@link maxDimension}), while maintaining aspect ratio of
* the source {@link width} and {@link height}.
*
* It returns `{0, 0}` for invalid inputs.
*/
export const scaledImageDimensions = (
width: number,
height: number,
maxDimension: number,
): { width: number; height: number } => {
if (width == 0 || height == 0) return { width: 0, height: 0 };
const widthScaleFactor = maxDimension / width;
const heightScaleFactor = maxDimension / height;
const scaleFactor = Math.min(widthScaleFactor, heightScaleFactor);
const resizedDimensions = {
width: Math.round(width * scaleFactor),
height: Math.round(height * scaleFactor),
};
if (resizedDimensions.width == 0 || resizedDimensions.height == 0)
return { width: 0, height: 0 };
return resizedDimensions;
};

4
web/packages/media/package.json
View file

@ -5,6 +5,10 @@
"dependencies": {
"@/next": "*",
"file-type": "16.5.4",
"heic-convert": "^2.1",
"jszip": "^3.10"
},
"devDependencies": {
"@types/heic-convert": "^1.2.3"
}
}

11
web/packages/media/worker/heic-convert.ts Normal file
View file

@ -0,0 +1,11 @@
import { ComlinkWorker } from "@/next/worker/comlink-worker";
import type { DedicatedHEICConvertWorker } from "./heic-convert.worker";
export const createHEICConvertWebWorker = () =>
new Worker(new URL("heic-convert.worker.ts", import.meta.url));
export const createHEICConvertComlinkWorker = () =>
new ComlinkWorker<typeof DedicatedHEICConvertWorker>(
"heic-convert-worker",
createHEICConvertWebWorker(),
);

2
web/apps/photos/src/worker/heic-convert.worker.ts → web/packages/media/worker/heic-convert.worker.ts
View file

@ -7,7 +7,7 @@ export class DedicatedHEICConvertWorker {
}
}
expose(DedicatedHEICConvertWorker, self);
expose(DedicatedHEICConvertWorker);
/**
* Convert a HEIC file to a JPEG file.

42
web/packages/next/log.ts
View file

@ -3,6 +3,19 @@ import { isDevBuild } from "./env";
import { logToDisk as webLogToDisk } from "./log-web";
import { workerBridge } from "./worker/worker-bridge";
/**
* Whether logs go to disk or are always emitted to the console.
*/
let shouldLogToDisk = true;
/**
* By default, logs get saved into a ring buffer in the browser's local storage.
* However, in some contexts, e.g. when we're running as the cast app, there is
* no mechanism for the user to retrieve these logs. So this function exists as
* a way to disable the on disk logging and always use the console.
*/
export const disableDiskLogs = () => (shouldLogToDisk = false);
/**
* Write a {@link message} to the on-disk log.
*
@ -45,14 +58,14 @@ const messageWithError = (message: string, e?: unknown) => {
const logError = (message: string, e?: unknown) => {
const m = `[error] ${messageWithError(message, e)}`;
if (isDevBuild) console.error(m);
logToDisk(m);
console.error(m);
if (shouldLogToDisk) logToDisk(m);
};
const logWarn = (message: string, e?: unknown) => {
const m = `[warn] ${messageWithError(message, e)}`;
if (isDevBuild) console.error(m);
logToDisk(m);
console.error(m);
if (shouldLogToDisk) logToDisk(m);
};
const logInfo = (...params: unknown[]) => {
@ -60,8 +73,8 @@ const logInfo = (...params: unknown[]) => {
.map((p) => (typeof p == "string" ? p : JSON.stringify(p)))
.join(" ");
const m = `[info] ${message}`;
if (isDevBuild) console.log(m);
logToDisk(m);
if (isDevBuild || !shouldLogToDisk) console.log(m);
if (shouldLogToDisk) logToDisk(m);
};
const logDebug = (param: () => unknown) => {
@ -71,8 +84,8 @@ const logDebug = (param: () => unknown) => {
/**
* Ente's logger.
*
* This is an object that provides three functions to log at the corresponding
* levels - error, info or debug.
* This is an object that provides functions to log at the corresponding levels:
* error, warn, info or debug.
*
* Whenever we need to save a log message to disk,
*
@ -89,8 +102,7 @@ export default {
* any arbitrary object that we obtain, say, when in a try-catch handler (in
* JavaScript any arbitrary value can be thrown).
*
* The log is written to disk. In development builds, the log is also
* printed to the browser console.
* The log is written to disk and printed to the browser console.
*/
error: logError,
/**
@ -104,8 +116,10 @@ export default {
* This is meant as a replacement of {@link console.log}, and takes an
* arbitrary number of arbitrary parameters that it then serializes.
*
* The log is written to disk. In development builds, the log is also
* printed to the browser console.
* The log is written to disk. However, if logging to disk is disabled by
* using {@link disableDiskLogs}, then the log is printed to the console.
*
* In development builds, the log is always printed to the browser console.
*/
info: logInfo,
/**
@ -118,8 +132,8 @@ export default {
* The function can return an arbitrary value which is serialized before
* being logged.
*
* This log is NOT written to disk. And it is printed to the browser
* console, but only in development builds.
* This log is NOT written to disk. It is printed to the browser console,
* but only in development builds.
*/
debug: logDebug,
};

29
web/packages/shared/utils/index.ts
View file

@ -1,11 +1,4 @@
/**
* Wait for {@link ms} milliseconds
*
* This function is a promisified `setTimeout`. It returns a promise that
* resolves after {@link ms} milliseconds.
*/
export const wait = (ms: number) =>
new Promise((resolve) => setTimeout(resolve, ms));
import { wait } from "@/utils/promise";
export function downloadAsFile(filename: string, content: string) {
const file = new Blob([content], {
@ -52,23 +45,3 @@ export async function retryAsyncFunction<T>(
}
}
}
/**
* Await the given {@link promise} for {@link timeoutMS} milliseconds. If it
* does not resolve within {@link timeoutMS}, then reject with a timeout error.
*/
export const withTimeout = async <T>(promise: Promise<T>, ms: number) => {
let timeoutId: ReturnType<typeof setTimeout>;
const rejectOnTimeout = new Promise<T>((_, reject) => {
timeoutId = setTimeout(
() => reject(new Error("Operation timed out")),
ms,
);
});
const promiseAndCancelTimeout = async () => {
const result = await promise;
clearTimeout(timeoutId);
return result;
};
return Promise.race([promiseAndCancelTimeout(), rejectOnTimeout]);
};

28
web/packages/utils/promise.ts Normal file
View file

@ -0,0 +1,28 @@
/**
* Wait for {@link ms} milliseconds
*
* This function is a promisified `setTimeout`. It returns a promise that
* resolves after {@link ms} milliseconds.
*/
export const wait = (ms: number) =>
new Promise((resolve) => setTimeout(resolve, ms));
/**
* Await the given {@link promise} for {@link timeoutMS} milliseconds. If it
* does not resolve within {@link timeoutMS}, then reject with a timeout error.
*/
export const withTimeout = async <T>(promise: Promise<T>, ms: number) => {
let timeoutId: ReturnType<typeof setTimeout>;
const rejectOnTimeout = new Promise<T>((_, reject) => {
timeoutId = setTimeout(
() => reject(new Error("Operation timed out")),
ms,
);
});
const promiseAndCancelTimeout = async () => {
const result = await promise;
clearTimeout(timeoutId);
return result;
};
return Promise.race([promiseAndCancelTimeout(), rejectOnTimeout]);
};

9
web/yarn.lock
View file

@ -1015,6 +1015,11 @@
resolved "https://registry.yarnpkg.com/@types/geojson/-/geojson-7946.0.14.tgz#319b63ad6df705ee2a65a73ef042c8271e696613"
integrity sha512-WCfD5Ht3ZesJUsONdhvm84dmzWOiOzOAqOncN0++w0lBw1o8OuDNJF2McvvCef/yBqb/HYRahp1BYtODFQ8bRg==
"@types/heic-convert@^1.2.3":
version "1.2.3"
resolved "https://registry.yarnpkg.com/@types/heic-convert/-/heic-convert-1.2.3.tgz#0705f36e467e7b6180806edd0b3f1e673514ff8c"
integrity sha512-5LJ2fGuVk/gnOLihoT56xJwrXxfnNepGvrHwlW5ZtT3HS4jO1AqBaAHCxXUpnY9UaD3zYcyxXMRM2fNN1AFF/Q==
"@types/hoist-non-react-statics@^3.3.1":
version "3.3.5"
resolved "https://registry.yarnpkg.com/@types/hoist-non-react-statics/-/hoist-non-react-statics-3.3.5.tgz#dab7867ef789d87e2b4b0003c9d65c49cc44a494"
@ -2839,7 +2844,7 @@ hdbscan@0.0.1-alpha.5:
dependencies:
kd-tree-javascript "^1.0.3"
heic-convert@^2.0.0:
heic-convert@^2.1:
version "2.1.0"
resolved "https://registry.yarnpkg.com/heic-convert/-/heic-convert-2.1.0.tgz#7f764529e37591ae263ef49582d1d0c13491526e"
integrity sha512-1qDuRvEHifTVAj3pFIgkqGgJIr0M3X7cxEPjEp0oG4mo8GFjq99DpCo8Eg3kg17Cy0MTjxpFdoBHOatj7ZVKtg==
@ -3321,7 +3326,7 @@ libsodium-wrappers@0.7.9:
dependencies:
libsodium "^0.7.0"
libsodium@0.7.9, libsodium@^0.7.0:
libsodium@^0.7.0:
version "0.7.9"
resolved "https://registry.yarnpkg.com/libsodium/-/libsodium-0.7.9.tgz#4bb7bcbf662ddd920d8795c227ae25bbbfa3821b"
integrity sha512-gfeADtR4D/CM0oRUviKBViMGXZDgnFdMKMzHsvBdqLBHd9ySi6EtYnmuhHVDDYgYpAO8eU8hEY+F8vIUAPh08A==