beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

teach readstream about zips

Browse source
This commit is contained in:
Manav Rathi 2024-04-29 21:38:21 +05:30
parent 6bcf985390
commit 39737b985b
No known key found for this signature in database
3 changed files with 56 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
web/apps/photos/src/services/upload/takeout.ts
View file

@ -5,6 +5,8 @@ import { nameAndExtension } from "@/next/file";
import log from "@/next/log";
import { NULL_LOCATION } from "constants/upload";
import type { Location } from "types/metadata";
import { readStream } from "utils/native-stream";
import type { UploadItem } from "./uploadManager";
export interface ParsedMetadataJSON {
creationTime: number;
@ -75,21 +77,29 @@ function getFileOriginalName(fileName: string) {
/** Try to parse the contents of a metadata JSON file from a Google Takeout. */
export const tryParseTakeoutMetadataJSON = async (
fileOrPath: File | string,
uploadItem: UploadItem,
): Promise<ParsedMetadataJSON | undefined> => {
try {
const text =
fileOrPath instanceof File
? await fileOrPath.text()
: await ensureElectron().fs.readTextFile(fileOrPath);
return parseMetadataJSONText(text);
return parseMetadataJSONText(await uploadItemText(uploadItem));
} catch (e) {
log.error("Failed to parse takeout metadata JSON", e);
return undefined;
}
};
const uploadItemText = async (uploadItem: UploadItem) => {
if (uploadItem instanceof File) {
return await uploadItem.text();
} else if (typeof uploadItem == "string") {
return await ensureElectron().fs.readTextFile(uploadItem);
} else if (Array.isArray(uploadItem)) {
const { response } = await readStream(ensureElectron(), uploadItem);
return await response.text();
} else {
return await uploadItem.file.text();
}
};
const NULL_PARSED_METADATA_JSON: ParsedMetadataJSON = {
creationTime: null,
modificationTime: null,

53
web/apps/photos/src/services/upload/uploadManager.ts
View file

@ -3,7 +3,7 @@ import { potentialFileTypeFromExtension } from "@/media/live-photo";
import { ensureElectron } from "@/next/electron";
import { lowercaseExtension, nameAndExtension } from "@/next/file";
import log from "@/next/log";
import { ElectronFile, type FileAndPath } from "@/next/types/file";
import { type FileAndPath } from "@/next/types/file";
import type { Electron, ZipEntry } from "@/next/types/ipc";
import { ComlinkWorker } from "@/next/worker/comlink-worker";
import { ensure } from "@/utils/ensure";
@ -437,25 +437,25 @@ class UploadManager {
try {
await this.updateExistingFilesAndCollections(collections);
const namedFiles = itemsWithCollection.map(
const namedItems = itemsWithCollection.map(
makeUploadItemWithCollectionIDAndName,
);
this.uiService.setFiles(namedFiles);
this.uiService.setFiles(namedItems);
const [metadataFiles, mediaFiles] =
splitMetadataAndMediaFiles(namedFiles);
const [metadataItems, mediaItems] =
splitMetadataAndMediaItems(namedItems);
if (metadataFiles.length) {
if (metadataItems.length) {
this.uiService.setUploadStage(
UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES,
);
await this.parseMetadataJSONFiles(metadataFiles);
await this.parseMetadataJSONFiles(metadataItems);
}
if (mediaFiles.length) {
const clusteredMediaFiles = await clusterLivePhotos(mediaFiles);
if (mediaItems.length) {
const clusteredMediaFiles = await clusterLivePhotos(mediaItems);
this.abortIfCancelled();
@ -464,7 +464,7 @@ class UploadManager {
this.uiService.setFiles(clusteredMediaFiles);
this.uiService.setHasLivePhoto(
mediaFiles.length != clusteredMediaFiles.length,
mediaItems.length != clusteredMediaFiles.length,
);
await this.uploadMediaFiles(clusteredMediaFiles);
@ -510,19 +510,17 @@ class UploadManager {
}
private async parseMetadataJSONFiles(
files: UploadItemWithCollectionIDAndName[],
items: UploadItemWithCollectionIDAndName[],
) {
this.uiService.reset(files.length);
this.uiService.reset(items.length);
for (const {
uploadItem: fileOrPath,
fileName,
collectionID,
} of files) {
for (const { uploadItem, fileName, collectionID } of items) {
this.abortIfCancelled();
log.info(`Parsing metadata JSON ${fileName}`);
const metadataJSON = await tryParseTakeoutMetadataJSON(fileOrPath);
const metadataJSON = await tryParseTakeoutMetadataJSON(
ensure(uploadItem),
);
if (metadataJSON) {
this.parsedMetadataJSONMap.set(
getMetadataJSONMapKeyForJSON(collectionID, fileName),
@ -823,7 +821,7 @@ export type UploadableUploadItem = ClusteredUploadItem & {
collection: Collection;
};
const splitMetadataAndMediaFiles = (
const splitMetadataAndMediaItems = (
items: UploadItemWithCollectionIDAndName[],
): [
metadata: UploadItemWithCollectionIDAndName[],
@ -879,13 +877,6 @@ const markUploaded = async (electron: Electron, item: ClusteredUploadItem) => {
}
};
/**
* NOTE: a stop gap measure, only meant to be called by code that is running in
* the context of a desktop app initiated upload
*/
export const getFilePathElectron = (file: File | ElectronFile | string) =>
typeof file == "string" ? file : (file as ElectronFile).path;
const cancelRemainingUploads = () => ensureElectron().clearPendingUploads();
/**
@ -913,13 +904,13 @@ const clusterLivePhotos = async (
fileName: f.fileName,
fileType: fFileType,
collectionID: f.collectionID,
fileOrPath: f.uploadItem,
uploadItem: f.uploadItem,
};
const ga: PotentialLivePhotoAsset = {
fileName: g.fileName,
fileType: gFileType,
collectionID: g.collectionID,
fileOrPath: g.uploadItem,
uploadItem: g.uploadItem,
};
if (await areLivePhotoAssets(fa, ga)) {
const [image, video] =
@ -956,7 +947,7 @@ interface PotentialLivePhotoAsset {
fileName: string;
fileType: FILE_TYPE;
collectionID: number;
fileOrPath: File | string;
uploadItem: UploadItem;
}
const areLivePhotoAssets = async (
@ -999,8 +990,8 @@ const areLivePhotoAssets = async (
// we use doesn't support stream as a input.
const maxAssetSize = 20 * 1024 * 1024; /* 20MB */
const fSize = await uploadItemSize(f.fileOrPath);
const gSize = await uploadItemSize(g.fileOrPath);
const fSize = await uploadItemSize(f.uploadItem);
const gSize = await uploadItemSize(g.uploadItem);
if (fSize > maxAssetSize || gSize > maxAssetSize) {
log.info(
`Not classifying assets with too large sizes ${[fSize, gSize]} as a live photo`,

24
web/apps/photos/src/utils/native-stream.ts
View file

@ -6,10 +6,10 @@
* See: [Note: IPC streams].
*/
import type { Electron } from "@/next/types/ipc";
import type { Electron, ZipEntry } from "@/next/types/ipc";
/**
* Stream the given file from the user's local filesystem.
* Stream the given file or zip entry from the user's local filesystem.
*
* This only works when we're running in our desktop app since it uses the
* "stream://" protocol handler exposed by our custom code in the Node.js layer.
@ -18,8 +18,9 @@ import type { Electron } from "@/next/types/ipc";
* To avoid accidentally invoking it in a non-desktop app context, it requires
* the {@link Electron} object as a parameter (even though it doesn't use it).
*
* @param path The path on the file on the user's local filesystem whose
* contents we want to stream.
* @param pathOrZipEntry Either the path on the file on the user's local
* filesystem whose contents we want to stream. Or a tuple containing the path
* to a zip file and the name of the entry within it.
*
* @return A ({@link Response}, size, lastModifiedMs) triple.
*
@ -34,16 +35,25 @@ import type { Electron } from "@/next/types/ipc";
*/
export const readStream = async (
_: Electron,
path: string,
pathOrZipEntry: string | ZipEntry,
): Promise<{ response: Response; size: number; lastModifiedMs: number }> => {
const req = new Request(`stream://read${path}`, {
let url: URL;
if (typeof pathOrZipEntry == "string") {
url = new URL(`stream://read${pathOrZipEntry}`);
} else {
const [zipPath, entryName] = pathOrZipEntry;
url = new URL(`stream://read${zipPath}`);
url.hash = entryName;
}
const req = new Request(url, {
method: "GET",
});
const res = await fetch(req);
if (!res.ok)
throw new Error(
`Failed to read stream from ${path}: HTTP ${res.status}`,
`Failed to read stream from ${url}: HTTP ${res.status}`,
);
const size = readNumericHeader(res, "Content-Length");