beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Inline

Browse source
This commit is contained in:
Manav Rathi 2024-04-27 17:49:47 +05:30
parent bb2ddec163
commit 17275ed29d
No known key found for this signature in database
11 changed files with 163 additions and 184 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

117
web/apps/photos/src/components/Upload/Uploader.tsx
View file

@ -5,7 +5,7 @@ import { CustomError } from "@ente/shared/error";
import { isPromise } from "@ente/shared/utils";
import DiscFullIcon from "@mui/icons-material/DiscFull";
import UserNameInputDialog from "components/UserNameInputDialog";
import { PICKED_UPLOAD_TYPE, UPLOAD_STAGES } from "constants/upload";
import { UPLOAD_STAGES } from "constants/upload";
import { t } from "i18next";
import isElectron from "is-electron";
import { AppContext } from "pages/_app";
@ -13,6 +13,7 @@ import { GalleryContext } from "pages/gallery";
import { useContext, useEffect, useRef, useState } from "react";
import billingService from "services/billingService";
import { getLatestCollections } from "services/collectionService";
import { exportMetadataDirectoryName } from "services/export";
import {
getPublicCollectionUID,
getPublicCollectionUploaderName,
@ -28,6 +29,7 @@ import type {
import uploadManager, {
setToUploadCollection,
} from "services/upload/uploadManager";
import { fopFileName } from "services/upload/uploadService";
import watcher from "services/watch";
import { NotificationAttributes } from "types/Notification";
import { Collection } from "types/collection";
@ -45,13 +47,6 @@ import {
getDownloadAppMessage,
getRootLevelFileWithFolderNotAllowMessage,
} from "utils/ui";
import {
DEFAULT_IMPORT_SUGGESTION,
getImportSuggestion,
groupFilesBasedOnParentFolder,
pruneHiddenFiles,
type ImportSuggestion,
} from "utils/upload";
import { SetCollectionNamerAttributes } from "../Collections/CollectionNamer";
import { CollectionMappingChoiceModal } from "./CollectionMappingChoiceModal";
import UploadProgress from "./UploadProgress";
@ -59,6 +54,12 @@ import UploadTypeSelector from "./UploadTypeSelector";
const FIRST_ALBUM_NAME = "My First Album";
enum PICKED_UPLOAD_TYPE {
FILES = "files",
FOLDERS = "folders",
ZIPS = "zips",
}
interface Props {
syncWithRemote: (force?: boolean, silent?: boolean) => Promise<void>;
closeCollectionSelector?: () => void;
@ -876,3 +877,103 @@ async function waitAndRun(
}
await task();
}
// This is used to prompt the user the make upload strategy choice
interface ImportSuggestion {
rootFolderName: string;
hasNestedFolders: boolean;
hasRootLevelFileWithFolder: boolean;
}
const DEFAULT_IMPORT_SUGGESTION: ImportSuggestion = {
rootFolderName: "",
hasNestedFolders: false,
hasRootLevelFileWithFolder: false,
};
function getImportSuggestion(
uploadType: PICKED_UPLOAD_TYPE,
paths: string[],
): ImportSuggestion {
if (isElectron() && uploadType === PICKED_UPLOAD_TYPE.FILES) {
return DEFAULT_IMPORT_SUGGESTION;
}
const getCharCount = (str: string) => (str.match(/\//g) ?? []).length;
paths.sort((path1, path2) => getCharCount(path1) - getCharCount(path2));
const firstPath = paths[0];
const lastPath = paths[paths.length - 1];
const L = firstPath.length;
let i = 0;
const firstFileFolder = firstPath.substring(0, firstPath.lastIndexOf("/"));
const lastFileFolder = lastPath.substring(0, lastPath.lastIndexOf("/"));
while (i < L && firstPath.charAt(i) === lastPath.charAt(i)) i++;
let commonPathPrefix = firstPath.substring(0, i);
if (commonPathPrefix) {
commonPathPrefix = commonPathPrefix.substring(
0,
commonPathPrefix.lastIndexOf("/"),
);
if (commonPathPrefix) {
commonPathPrefix = commonPathPrefix.substring(
commonPathPrefix.lastIndexOf("/") + 1,
);
}
}
return {
rootFolderName: commonPathPrefix || null,
hasNestedFolders: firstFileFolder !== lastFileFolder,
hasRootLevelFileWithFolder: firstFileFolder === "",
};
}
// This function groups files that are that have the same parent folder into collections
// For Example, for user files have a directory structure like this
// a
// / | \
// b j c
// /|\ / \
// e f g h i
//
// The files will grouped into 3 collections.
// [a => [j],
// b => [e,f,g],
// c => [h, i]]
const groupFilesBasedOnParentFolder = (fileOrPaths: (File | string)[]) => {
const result = new Map<string, (File | string)[]>();
for (const fileOrPath of fileOrPaths) {
const filePath =
/* TODO(MR): ElectronFile */
typeof fileOrPath == "string"
? fileOrPath
: (fileOrPath["path"] as string);
let folderPath = filePath.substring(0, filePath.lastIndexOf("/"));
// If the parent folder of a file is "metadata"
// we consider it to be part of the parent folder
// For Eg,For FileList -> [a/x.png, a/metadata/x.png.json]
// they will both we grouped into the collection "a"
// This is cluster the metadata json files in the same collection as the file it is for
if (folderPath.endsWith(exportMetadataDirectoryName)) {
folderPath = folderPath.substring(0, folderPath.lastIndexOf("/"));
}
const folderName = folderPath.substring(
folderPath.lastIndexOf("/") + 1,
);
if (!folderName) throw Error("Unexpected empty folder name");
if (!result.has(folderName)) result.set(folderName, []);
result.get(folderName).push(fileOrPath);
}
return result;
};
/**
* Filter out hidden files from amongst {@link fileOrPaths}.
*
* Hidden files are those whose names begin with a "." (dot).
*/
const pruneHiddenFiles = (fileOrPaths: (File | string)[]) =>
fileOrPaths.filter((f) => !fopFileName(f).startsWith("."));

12
web/apps/photos/src/components/WatchFolder.tsx
View file

@ -1,5 +1,5 @@
import { ensureElectron } from "@/next/electron";
import { basename } from "@/next/file";
import { basename, dirname } from "@/next/file";
import type { CollectionMapping, FolderWatch } from "@/next/types/ipc";
import { ensure } from "@/utils/ensure";
import {
@ -32,7 +32,6 @@ import { t } from "i18next";
import { AppContext } from "pages/_app";
import React, { useContext, useEffect, useState } from "react";
import watcher from "services/watch";
import { areAllInSameDirectory } from "utils/upload";
interface WatchFolderProps {
open: boolean;
@ -324,3 +323,12 @@ const EntryOptions: React.FC<EntryOptionsProps> = ({ confirmStopWatching }) => {
</OverflowMenu>
);
};
/**
* Return true if all the paths in the given list are items that belong to the
* same (arbitrary) directory.
*
* Empty list of paths is considered to be in the same directory.
*/
const areAllInSameDirectory = (paths: string[]) =>
new Set(paths.map(dirname)).size == 1;

6
web/apps/photos/src/constants/upload.ts
View file

@ -24,9 +24,3 @@ export enum UPLOAD_RESULT {
UPLOADED_WITH_STATIC_THUMBNAIL,
ADDED_SYMLINK,
}
export enum PICKED_UPLOAD_TYPE {
FILES = "files",
FOLDERS = "folders",
ZIPS = "zips",
}

2
web/apps/photos/src/services/deduplicationService.ts
View file

@ -1,3 +1,4 @@
import { hasFileHash } from "@/media/file";
import { FILE_TYPE } from "@/media/file-type";
import type { Metadata } from "@/media/types/file";
import log from "@/next/log";
@ -5,7 +6,6 @@ import HTTPService from "@ente/shared/network/HTTPService";
import { getEndpoint } from "@ente/shared/network/api";
import { getToken } from "@ente/shared/storage/localStorage/helpers";
import { EnteFile } from "types/file";
import { hasFileHash } from "utils/upload";
const ENDPOINT = getEndpoint();

2
web/apps/photos/src/services/upload/publicUploadHttpClient.ts
View file

@ -3,7 +3,7 @@ import { CustomError, handleUploadError } from "@ente/shared/error";
import HTTPService from "@ente/shared/network/HTTPService";
import { getEndpoint } from "@ente/shared/network/api";
import { EnteFile } from "types/file";
import { retryHTTPCall } from "utils/upload/uploadRetrier";
import { retryHTTPCall } from "./uploadHttpClient";
import { MultipartUploadURLs, UploadFile, UploadURL } from "./uploadService";
const ENDPOINT = getEndpoint();

30
web/apps/photos/src/services/upload/uploadHttpClient.ts
View file

@ -3,8 +3,8 @@ import { CustomError, handleUploadError } from "@ente/shared/error";
import HTTPService from "@ente/shared/network/HTTPService";
import { getEndpoint, getUploadEndpoint } from "@ente/shared/network/api";
import { getToken } from "@ente/shared/storage/localStorage/helpers";
import { wait } from "@ente/shared/utils";
import { EnteFile } from "types/file";
import { retryHTTPCall } from "utils/upload/uploadRetrier";
import { MultipartUploadURLs, UploadFile, UploadURL } from "./uploadService";
const ENDPOINT = getEndpoint();
@ -236,3 +236,31 @@ class UploadHttpClient {
}
export default new UploadHttpClient();
const retrySleepTimeInMilliSeconds = [2000, 5000, 10000];
export async function retryHTTPCall(
func: () => Promise<any>,
checkForBreakingError?: (error) => void,
): Promise<any> {
const retrier = async (
func: () => Promise<any>,
attemptNumber: number = 0,
) => {
try {
const resp = await func();
return resp;
} catch (e) {
if (checkForBreakingError) {
checkForBreakingError(e);
}
if (attemptNumber < retrySleepTimeInMilliSeconds.length) {
await wait(retrySleepTimeInMilliSeconds[attemptNumber]);
return await retrier(func, attemptNumber + 1);
} else {
throw e;
}
}
};
return await retrier(func);
}

9
web/apps/photos/src/services/upload/uploadService.ts
View file

@ -1,3 +1,4 @@
import { hasFileHash } from "@/media/file";
import { FILE_TYPE, type FileTypeInfo } from "@/media/file-type";
import { encodeLivePhoto } from "@/media/live-photo";
import type { Metadata } from "@/media/types/file";
@ -8,13 +9,8 @@ import { CustomErrorMessage } from "@/next/types/ipc";
import { ensure } from "@/utils/ensure";
import { ENCRYPTION_CHUNK_SIZE } from "@ente/shared/crypto/constants";
import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker";
import {
B64EncryptionResult,
EncryptionResult,
LocalFileAttributes,
} from "@ente/shared/crypto/types";
import { B64EncryptionResult } from "@ente/shared/crypto/types";
import { CustomError, handleUploadError } from "@ente/shared/error";
import { isDataStream, type DataStream } from "@ente/shared/utils/data-stream";
import { Remote } from "comlink";
import {
NULL_LOCATION,
@ -43,7 +39,6 @@ import {
updateMagicMetadata,
} from "utils/magicMetadata";
import { readStream } from "utils/native-stream";
import { hasFileHash } from "utils/upload";
import * as convert from "xml-js";
import { detectFileTypeInfoFromChunk } from "../detect-type";
import { getFileStream } from "../readerService";

8
web/apps/photos/src/services/watch.ts
View file

@ -20,7 +20,6 @@ import uploadManager, {
import { Collection } from "types/collection";
import { EncryptedEnteFile } from "types/file";
import { groupFilesBasedOnCollectionID } from "utils/file";
import { isHiddenFile } from "utils/upload";
import { removeFromCollection } from "./collectionService";
import { getLocalFiles } from "./fileService";
@ -596,6 +595,13 @@ const pathsToUpload = (paths: string[], watch: FolderWatch) =>
// Files that are on disk but not yet synced or ignored.
.filter((path) => !isSyncedOrIgnoredPath(path, watch));
/**
* Return true if the file at the given {@link path} is hidden.
*
* Hidden files are those whose names begin with a "." (dot).
*/
const isHiddenFile = (path: string) => basename(path).startsWith(".");
/**
* Return the paths to previously synced files that are no longer on disk and so
* must be removed from the Ente collection.

128
web/apps/photos/src/utils/upload/index.ts
View file

@ -1,128 +0,0 @@
import type { Metadata } from "@/media/types/file";
import { basename, dirname } from "@/next/file";
import { PICKED_UPLOAD_TYPE } from "constants/upload";
import isElectron from "is-electron";
import { exportMetadataDirectoryName } from "services/export";
import { fopFileName } from "services/upload/uploadService";
export const hasFileHash = (file: Metadata) =>
file.hash || (file.imageHash && file.videoHash);
/**
* Return true if all the paths in the given list are items that belong to the
* same (arbitrary) directory.
*
* Empty list of paths is considered to be in the same directory.
*/
export const areAllInSameDirectory = (paths: string[]) =>
new Set(paths.map(dirname)).size == 1;
// This is used to prompt the user the make upload strategy choice
export interface ImportSuggestion {
rootFolderName: string;
hasNestedFolders: boolean;
hasRootLevelFileWithFolder: boolean;
}
export const DEFAULT_IMPORT_SUGGESTION: ImportSuggestion = {
rootFolderName: "",
hasNestedFolders: false,
hasRootLevelFileWithFolder: false,
};
export function getImportSuggestion(
uploadType: PICKED_UPLOAD_TYPE,
paths: string[],
): ImportSuggestion {
if (isElectron() && uploadType === PICKED_UPLOAD_TYPE.FILES) {
return DEFAULT_IMPORT_SUGGESTION;
}
const getCharCount = (str: string) => (str.match(/\//g) ?? []).length;
paths.sort((path1, path2) => getCharCount(path1) - getCharCount(path2));
const firstPath = paths[0];
const lastPath = paths[paths.length - 1];
const L = firstPath.length;
let i = 0;
const firstFileFolder = firstPath.substring(0, firstPath.lastIndexOf("/"));
const lastFileFolder = lastPath.substring(0, lastPath.lastIndexOf("/"));
while (i < L && firstPath.charAt(i) === lastPath.charAt(i)) i++;
let commonPathPrefix = firstPath.substring(0, i);
if (commonPathPrefix) {
commonPathPrefix = commonPathPrefix.substring(
0,
commonPathPrefix.lastIndexOf("/"),
);
if (commonPathPrefix) {
commonPathPrefix = commonPathPrefix.substring(
commonPathPrefix.lastIndexOf("/") + 1,
);
}
}
return {
rootFolderName: commonPathPrefix || null,
hasNestedFolders: firstFileFolder !== lastFileFolder,
hasRootLevelFileWithFolder: firstFileFolder === "",
};
}
// This function groups files that are that have the same parent folder into collections
// For Example, for user files have a directory structure like this
// a
// / | \
// b j c
// /|\ / \
// e f g h i
//
// The files will grouped into 3 collections.
// [a => [j],
// b => [e,f,g],
// c => [h, i]]
export const groupFilesBasedOnParentFolder = (
fileOrPaths: (File | string)[],
) => {
const result = new Map<string, (File | string)[]>();
for (const fileOrPath of fileOrPaths) {
const filePath =
/* TODO(MR): ElectronFile */
typeof fileOrPath == "string"
? fileOrPath
: (fileOrPath["path"] as string);
let folderPath = filePath.substring(0, filePath.lastIndexOf("/"));
// If the parent folder of a file is "metadata"
// we consider it to be part of the parent folder
// For Eg,For FileList -> [a/x.png, a/metadata/x.png.json]
// they will both we grouped into the collection "a"
// This is cluster the metadata json files in the same collection as the file it is for
if (folderPath.endsWith(exportMetadataDirectoryName)) {
folderPath = folderPath.substring(0, folderPath.lastIndexOf("/"));
}
const folderName = folderPath.substring(
folderPath.lastIndexOf("/") + 1,
);
if (!folderName) throw Error("Unexpected empty folder name");
if (!result.has(folderName)) result.set(folderName, []);
result.get(folderName).push(fileOrPath);
}
return result;
};
/**
* Filter out hidden files from amongst {@link fileOrPaths}.
*
* Hidden files are those whose names begin with a "." (dot).
*/
export const pruneHiddenFiles = (fileOrPaths: (File | string)[]) =>
fileOrPaths.filter((f) => !fopFileName(f).startsWith("."));
/**
* Return true if the file at the given {@link path} is hidden.
*
* Hidden files are those whose names begin with a "." (dot).
*/
export const isHiddenFile = (path: string) => basename(path).startsWith(".");

29
web/apps/photos/src/utils/upload/uploadRetrier.ts
View file

@ -1,29 +0,0 @@
import { wait } from "@ente/shared/utils";
const retrySleepTimeInMilliSeconds = [2000, 5000, 10000];
export async function retryHTTPCall(
func: () => Promise<any>,
checkForBreakingError?: (error) => void,
): Promise<any> {
const retrier = async (
func: () => Promise<any>,
attemptNumber: number = 0,
) => {
try {
const resp = await func();
return resp;
} catch (e) {
if (checkForBreakingError) {
checkForBreakingError(e);
}
if (attemptNumber < retrySleepTimeInMilliSeconds.length) {
await wait(retrySleepTimeInMilliSeconds[attemptNumber]);
return await retrier(func, attemptNumber + 1);
} else {
throw e;
}
}
};
return await retrier(func);
}

4
web/packages/media/file.ts Normal file
View file

@ -0,0 +1,4 @@
import type { Metadata } from "./types/file";
export const hasFileHash = (file: Metadata) =>
!!file.hash || (!!file.imageHash && !!file.videoHash);