beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

[desktop] Streaming IPC - Part 2/x (#1454)

Browse source 
The data is getting streamed, but not correctly in some cases. So the
mechanics are in place, need to figure out the readable stream chunks
etc.
This commit is contained in:
Manav Rathi 2024-04-15 20:48:48 +05:30 committed by GitHub
parent 61f05f8eff f2e352c344
commit 2c5ed5bce9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 65 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
desktop/src/main/fs.ts
View file

@ -3,7 +3,6 @@
*/
import { existsSync } from "node:fs";
import fs from "node:fs/promises";
import { writeStream } from "./stream";
export const fsExists = (path: string) => existsSync(path);
@ -25,8 +24,6 @@ export const fsWriteFile = (path: string, contents: string) =>
/* TODO: Audit below this */
export const saveStreamToDisk = writeStream;
export const isFolder = async (dirPath: string) => {
if (!existsSync(dirPath)) return false;
const stats = await fs.stat(dirPath);

7
desktop/src/main/ipc.ts
View file

@ -26,7 +26,6 @@ import {
fsRmdir,
fsWriteFile,
isFolder,
saveStreamToDisk,
} from "./fs";
import { logToDisk } from "./log";
import {
@ -186,12 +185,6 @@ export const attachIPCHandlers = () => {
// - FS Legacy
ipcMain.handle(
"saveStreamToDisk",
(_, path: string, fileStream: ReadableStream) =>
saveStreamToDisk(path, fileStream),
);
ipcMain.handle("isFolder", (_, dirPath: string) => isFolder(dirPath));
// - Upload

8
desktop/src/main/stream.ts
View file

@ -30,12 +30,14 @@ export const registerStreamProtocol = () => {
protocol.handle("stream", async (request: Request) => {
const url = request.url;
const { host, pathname } = new URL(url);
// Convert e.g. "%20" to spaces.
const path = decodeURIComponent(pathname);
switch (host) {
/* stream://write//path/to/file */
/* -host/pathname----- */
/* stream://write/path/to/file */
/* host-pathname----- */
case "write":
try {
await writeStream(pathname, request.body);
await writeStream(path, request.body);
return new Response("", { status: 200 });
} catch (e) {
log.error(`Failed to write stream for ${url}`, e);

6
desktop/src/preload.ts
View file

@ -237,11 +237,6 @@ const updateWatchMappingIgnoredFiles = (
// - FS Legacy
const saveStreamToDisk = (
path: string,
fileStream: ReadableStream,
): Promise<void> => ipcRenderer.invoke("saveStreamToDisk", path, fileStream);
const isFolder = (dirPath: string): Promise<boolean> =>
ipcRenderer.invoke("isFolder", dirPath);
@ -357,7 +352,6 @@ contextBridge.exposeInMainWorld("electron", {
// - FS legacy
// TODO: Move these into fs + document + rename if needed
saveStreamToDisk,
isFolder,
// - Upload

48
web/apps/photos/src/services/export/index.ts
View file

@ -1,5 +1,4 @@
import { ensureElectron } from "@/next/electron";
import { isDevBuild } from "@/next/env";
import log from "@/next/log";
import { CustomError } from "@ente/shared/error";
import { Events, eventBus } from "@ente/shared/events";
@ -35,6 +34,7 @@ import {
mergeMetadata,
} from "utils/file";
import { safeDirectoryName, safeFileName } from "utils/native-fs";
import { writeStream } from "utils/native-stream";
import { getAllLocalCollections } from "../collectionService";
import downloadManager from "../download";
import { getAllLocalFiles } from "../fileService";
@ -993,47 +993,7 @@ class ExportService {
fileExportName,
file,
);
// TODO(MR): Productionalize
if (isDevBuild) {
const testStream = new ReadableStream({
async start(controller) {
await sleep(1000);
controller.enqueue("This ");
await sleep(1000);
controller.enqueue("is ");
await sleep(1000);
controller.enqueue("a ");
await sleep(1000);
controller.enqueue("test");
controller.close();
},
}).pipeThrough(new TextEncoderStream());
console.log({ a: "will send req", updatedFileStream });
// The duplex parameter needs to be set to 'half' when
// streaming requests.
//
// Currently browsers, and specifically in our case,
// since this code runs only within our desktop
// (Electron) app, Chromium, don't support 'full' duplex
// mode (i.e. streaming both the request and the
// response).
//
// https://developer.chrome.com/docs/capabilities/web-apis/fetch-streaming-requests
//
// In another twist, the TypeScript libdom.d.ts does not
// include the "duplex" parameter, so we need to cast to
// get TypeScript to let this code through. e.g. see
// https://github.com/node-fetch/node-fetch/issues/1769
const req = new Request("stream://write/tmp/foo.txt", {
method: "POST",
// body: updatedFileStream,
body: testStream,
duplex: "half",
} as unknown as RequestInit);
const res = await fetch(req);
console.log({ a: "got res", res });
}
await electron.saveStreamToDisk(
await writeStream(
`${collectionExportPath}/${fileExportName}`,
updatedFileStream,
);
@ -1084,7 +1044,7 @@ class ExportService {
imageExportName,
file,
);
await electron.saveStreamToDisk(
await writeStream(
`${collectionExportPath}/${imageExportName}`,
imageStream,
);
@ -1096,7 +1056,7 @@ class ExportService {
file,
);
try {
await electron.saveStreamToDisk(
await writeStream(
`${collectionExportPath}/${videoExportName}`,
videoStream,
);

41
web/apps/photos/src/utils/file/index.ts
View file

@ -53,6 +53,7 @@ import { VISIBILITY_STATE } from "types/magicMetadata";
import { FileTypeInfo } from "types/upload";
import { isArchivedFile, updateMagicMetadata } from "utils/magicMetadata";
import { safeFileName } from "utils/native-fs";
import { writeStream } from "utils/native-stream";
const WAIT_TIME_IMAGE_CONVERSION = 30 * 1000;
@ -798,55 +799,47 @@ async function downloadFileDesktop(
electron: Electron,
fileReader: FileReader,
file: EnteFile,
downloadPath: string,
downloadDir: string,
) {
const fileStream = (await DownloadManager.getFile(
const fs = electron.fs;
const stream = (await DownloadManager.getFile(
file,
)) as ReadableStream<Uint8Array>;
const updatedFileStream = await getUpdatedEXIFFileForDownload(
const updatedStream = await getUpdatedEXIFFileForDownload(
fileReader,
file,
fileStream,
stream,
);
if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
const fileBlob = await new Response(updatedFileStream).blob();
const fileBlob = await new Response(updatedStream).blob();
const livePhoto = await decodeLivePhoto(file, fileBlob);
const imageExportName = await safeFileName(
downloadPath,
downloadDir,
livePhoto.imageNameTitle,
electron.fs.exists,
fs.exists,
);
const imageStream = generateStreamFromArrayBuffer(livePhoto.image);
await electron.saveStreamToDisk(
`${downloadPath}/${imageExportName}`,
imageStream,
);
await writeStream(`${downloadDir}/${imageExportName}`, imageStream);
try {
const videoExportName = await safeFileName(
downloadPath,
downloadDir,
livePhoto.videoNameTitle,
electron.fs.exists,
fs.exists,
);
const videoStream = generateStreamFromArrayBuffer(livePhoto.video);
await electron.saveStreamToDisk(
`${downloadPath}/${videoExportName}`,
videoStream,
);
await writeStream(`${downloadDir}/${videoExportName}`, videoStream);
} catch (e) {
await electron.fs.rm(`${downloadPath}/${imageExportName}`);
await fs.rm(`${downloadDir}/${imageExportName}`);
throw e;
}
} else {
const fileExportName = await safeFileName(
downloadPath,
downloadDir,
file.metadata.title,
electron.fs.exists,
);
await electron.saveStreamToDisk(
`${downloadPath}/${fileExportName}`,
updatedFileStream,
fs.exists,
);
await writeStream(`${downloadDir}/${fileExportName}`, updatedStream);
}
}

39
web/apps/photos/src/utils/native-stream.ts Normal file
View file

@ -0,0 +1,39 @@
/**
* @file Streaming IPC communication with the Node.js layer of our desktop app.
*
* NOTE: These functions only work when we're running in our desktop app.
*/
/**
* Write the given stream to a file on the local machine.
*
* **This only works when we're running in our desktop app**. It uses the
* "stream://" protocol handler exposed by our custom code in the Node.js layer.
* See: [Note: IPC streams].
*
* @param path The path on the local machine where to write the file to.
* @param stream The stream which should be written into the file.
* */
export const writeStream = async (path: string, stream: ReadableStream) => {
// The duplex parameter needs to be set to 'half' when streaming requests.
//
// Currently browsers, and specifically in our case, since this code runs
// only within our desktop (Electron) app, Chromium, don't support 'full'
// duplex mode (i.e. streaming both the request and the response).
// https://developer.chrome.com/docs/capabilities/web-apis/fetch-streaming-requests
//
// In another twist, the TypeScript libdom.d.ts does not include the
// "duplex" parameter, so we need to cast to get TypeScript to let this code
// through. e.g. see https://github.com/node-fetch/node-fetch/issues/1769
const req = new Request(`stream://write${path}`, {
// GET can't have a body
method: "POST",
body: stream,
duplex: "half",
} as unknown as RequestInit);
const res = await fetch(req);
if (!res.ok)
throw new Error(
`Failed to write stream to ${path}: HTTP ${res.status}`,
);
};

4
web/packages/next/types/ipc.ts
View file

@ -311,10 +311,6 @@ export interface Electron {
) => Promise<void>;
// - FS legacy
saveStreamToDisk: (
path: string,
fileStream: ReadableStream,
) => Promise<void>;
isFolder: (dirPath: string) => Promise<boolean>;
// - Upload