Inline
This commit is contained in:
parent
bb2ddec163
commit
17275ed29d
|
@ -5,7 +5,7 @@ import { CustomError } from "@ente/shared/error";
|
|||
import { isPromise } from "@ente/shared/utils";
|
||||
import DiscFullIcon from "@mui/icons-material/DiscFull";
|
||||
import UserNameInputDialog from "components/UserNameInputDialog";
|
||||
import { PICKED_UPLOAD_TYPE, UPLOAD_STAGES } from "constants/upload";
|
||||
import { UPLOAD_STAGES } from "constants/upload";
|
||||
import { t } from "i18next";
|
||||
import isElectron from "is-electron";
|
||||
import { AppContext } from "pages/_app";
|
||||
|
@ -13,6 +13,7 @@ import { GalleryContext } from "pages/gallery";
|
|||
import { useContext, useEffect, useRef, useState } from "react";
|
||||
import billingService from "services/billingService";
|
||||
import { getLatestCollections } from "services/collectionService";
|
||||
import { exportMetadataDirectoryName } from "services/export";
|
||||
import {
|
||||
getPublicCollectionUID,
|
||||
getPublicCollectionUploaderName,
|
||||
|
@ -28,6 +29,7 @@ import type {
|
|||
import uploadManager, {
|
||||
setToUploadCollection,
|
||||
} from "services/upload/uploadManager";
|
||||
import { fopFileName } from "services/upload/uploadService";
|
||||
import watcher from "services/watch";
|
||||
import { NotificationAttributes } from "types/Notification";
|
||||
import { Collection } from "types/collection";
|
||||
|
@ -45,13 +47,6 @@ import {
|
|||
getDownloadAppMessage,
|
||||
getRootLevelFileWithFolderNotAllowMessage,
|
||||
} from "utils/ui";
|
||||
import {
|
||||
DEFAULT_IMPORT_SUGGESTION,
|
||||
getImportSuggestion,
|
||||
groupFilesBasedOnParentFolder,
|
||||
pruneHiddenFiles,
|
||||
type ImportSuggestion,
|
||||
} from "utils/upload";
|
||||
import { SetCollectionNamerAttributes } from "../Collections/CollectionNamer";
|
||||
import { CollectionMappingChoiceModal } from "./CollectionMappingChoiceModal";
|
||||
import UploadProgress from "./UploadProgress";
|
||||
|
@ -59,6 +54,12 @@ import UploadTypeSelector from "./UploadTypeSelector";
|
|||
|
||||
const FIRST_ALBUM_NAME = "My First Album";
|
||||
|
||||
enum PICKED_UPLOAD_TYPE {
|
||||
FILES = "files",
|
||||
FOLDERS = "folders",
|
||||
ZIPS = "zips",
|
||||
}
|
||||
|
||||
interface Props {
|
||||
syncWithRemote: (force?: boolean, silent?: boolean) => Promise<void>;
|
||||
closeCollectionSelector?: () => void;
|
||||
|
@ -876,3 +877,103 @@ async function waitAndRun(
|
|||
}
|
||||
await task();
|
||||
}
|
||||
|
||||
// This is used to prompt the user the make upload strategy choice
|
||||
interface ImportSuggestion {
|
||||
rootFolderName: string;
|
||||
hasNestedFolders: boolean;
|
||||
hasRootLevelFileWithFolder: boolean;
|
||||
}
|
||||
|
||||
const DEFAULT_IMPORT_SUGGESTION: ImportSuggestion = {
|
||||
rootFolderName: "",
|
||||
hasNestedFolders: false,
|
||||
hasRootLevelFileWithFolder: false,
|
||||
};
|
||||
|
||||
function getImportSuggestion(
|
||||
uploadType: PICKED_UPLOAD_TYPE,
|
||||
paths: string[],
|
||||
): ImportSuggestion {
|
||||
if (isElectron() && uploadType === PICKED_UPLOAD_TYPE.FILES) {
|
||||
return DEFAULT_IMPORT_SUGGESTION;
|
||||
}
|
||||
|
||||
const getCharCount = (str: string) => (str.match(/\//g) ?? []).length;
|
||||
paths.sort((path1, path2) => getCharCount(path1) - getCharCount(path2));
|
||||
const firstPath = paths[0];
|
||||
const lastPath = paths[paths.length - 1];
|
||||
|
||||
const L = firstPath.length;
|
||||
let i = 0;
|
||||
const firstFileFolder = firstPath.substring(0, firstPath.lastIndexOf("/"));
|
||||
const lastFileFolder = lastPath.substring(0, lastPath.lastIndexOf("/"));
|
||||
|
||||
while (i < L && firstPath.charAt(i) === lastPath.charAt(i)) i++;
|
||||
let commonPathPrefix = firstPath.substring(0, i);
|
||||
|
||||
if (commonPathPrefix) {
|
||||
commonPathPrefix = commonPathPrefix.substring(
|
||||
0,
|
||||
commonPathPrefix.lastIndexOf("/"),
|
||||
);
|
||||
if (commonPathPrefix) {
|
||||
commonPathPrefix = commonPathPrefix.substring(
|
||||
commonPathPrefix.lastIndexOf("/") + 1,
|
||||
);
|
||||
}
|
||||
}
|
||||
return {
|
||||
rootFolderName: commonPathPrefix || null,
|
||||
hasNestedFolders: firstFileFolder !== lastFileFolder,
|
||||
hasRootLevelFileWithFolder: firstFileFolder === "",
|
||||
};
|
||||
}
|
||||
|
||||
// This function groups files that are that have the same parent folder into collections
|
||||
// For Example, for user files have a directory structure like this
|
||||
// a
|
||||
// / | \
|
||||
// b j c
|
||||
// /|\ / \
|
||||
// e f g h i
|
||||
//
|
||||
// The files will grouped into 3 collections.
|
||||
// [a => [j],
|
||||
// b => [e,f,g],
|
||||
// c => [h, i]]
|
||||
const groupFilesBasedOnParentFolder = (fileOrPaths: (File | string)[]) => {
|
||||
const result = new Map<string, (File | string)[]>();
|
||||
for (const fileOrPath of fileOrPaths) {
|
||||
const filePath =
|
||||
/* TODO(MR): ElectronFile */
|
||||
typeof fileOrPath == "string"
|
||||
? fileOrPath
|
||||
: (fileOrPath["path"] as string);
|
||||
|
||||
let folderPath = filePath.substring(0, filePath.lastIndexOf("/"));
|
||||
// If the parent folder of a file is "metadata"
|
||||
// we consider it to be part of the parent folder
|
||||
// For Eg,For FileList -> [a/x.png, a/metadata/x.png.json]
|
||||
// they will both we grouped into the collection "a"
|
||||
// This is cluster the metadata json files in the same collection as the file it is for
|
||||
if (folderPath.endsWith(exportMetadataDirectoryName)) {
|
||||
folderPath = folderPath.substring(0, folderPath.lastIndexOf("/"));
|
||||
}
|
||||
const folderName = folderPath.substring(
|
||||
folderPath.lastIndexOf("/") + 1,
|
||||
);
|
||||
if (!folderName) throw Error("Unexpected empty folder name");
|
||||
if (!result.has(folderName)) result.set(folderName, []);
|
||||
result.get(folderName).push(fileOrPath);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
/**
|
||||
* Filter out hidden files from amongst {@link fileOrPaths}.
|
||||
*
|
||||
* Hidden files are those whose names begin with a "." (dot).
|
||||
*/
|
||||
const pruneHiddenFiles = (fileOrPaths: (File | string)[]) =>
|
||||
fileOrPaths.filter((f) => !fopFileName(f).startsWith("."));
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
import { ensureElectron } from "@/next/electron";
|
||||
import { basename } from "@/next/file";
|
||||
import { basename, dirname } from "@/next/file";
|
||||
import type { CollectionMapping, FolderWatch } from "@/next/types/ipc";
|
||||
import { ensure } from "@/utils/ensure";
|
||||
import {
|
||||
|
@ -32,7 +32,6 @@ import { t } from "i18next";
|
|||
import { AppContext } from "pages/_app";
|
||||
import React, { useContext, useEffect, useState } from "react";
|
||||
import watcher from "services/watch";
|
||||
import { areAllInSameDirectory } from "utils/upload";
|
||||
|
||||
interface WatchFolderProps {
|
||||
open: boolean;
|
||||
|
@ -324,3 +323,12 @@ const EntryOptions: React.FC<EntryOptionsProps> = ({ confirmStopWatching }) => {
|
|||
</OverflowMenu>
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Return true if all the paths in the given list are items that belong to the
|
||||
* same (arbitrary) directory.
|
||||
*
|
||||
* Empty list of paths is considered to be in the same directory.
|
||||
*/
|
||||
const areAllInSameDirectory = (paths: string[]) =>
|
||||
new Set(paths.map(dirname)).size == 1;
|
||||
|
|
|
@ -24,9 +24,3 @@ export enum UPLOAD_RESULT {
|
|||
UPLOADED_WITH_STATIC_THUMBNAIL,
|
||||
ADDED_SYMLINK,
|
||||
}
|
||||
|
||||
export enum PICKED_UPLOAD_TYPE {
|
||||
FILES = "files",
|
||||
FOLDERS = "folders",
|
||||
ZIPS = "zips",
|
||||
}
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
import { hasFileHash } from "@/media/file";
|
||||
import { FILE_TYPE } from "@/media/file-type";
|
||||
import type { Metadata } from "@/media/types/file";
|
||||
import log from "@/next/log";
|
||||
|
@ -5,7 +6,6 @@ import HTTPService from "@ente/shared/network/HTTPService";
|
|||
import { getEndpoint } from "@ente/shared/network/api";
|
||||
import { getToken } from "@ente/shared/storage/localStorage/helpers";
|
||||
import { EnteFile } from "types/file";
|
||||
import { hasFileHash } from "utils/upload";
|
||||
|
||||
const ENDPOINT = getEndpoint();
|
||||
|
||||
|
|
|
@ -3,7 +3,7 @@ import { CustomError, handleUploadError } from "@ente/shared/error";
|
|||
import HTTPService from "@ente/shared/network/HTTPService";
|
||||
import { getEndpoint } from "@ente/shared/network/api";
|
||||
import { EnteFile } from "types/file";
|
||||
import { retryHTTPCall } from "utils/upload/uploadRetrier";
|
||||
import { retryHTTPCall } from "./uploadHttpClient";
|
||||
import { MultipartUploadURLs, UploadFile, UploadURL } from "./uploadService";
|
||||
|
||||
const ENDPOINT = getEndpoint();
|
||||
|
|
|
@ -3,8 +3,8 @@ import { CustomError, handleUploadError } from "@ente/shared/error";
|
|||
import HTTPService from "@ente/shared/network/HTTPService";
|
||||
import { getEndpoint, getUploadEndpoint } from "@ente/shared/network/api";
|
||||
import { getToken } from "@ente/shared/storage/localStorage/helpers";
|
||||
import { wait } from "@ente/shared/utils";
|
||||
import { EnteFile } from "types/file";
|
||||
import { retryHTTPCall } from "utils/upload/uploadRetrier";
|
||||
import { MultipartUploadURLs, UploadFile, UploadURL } from "./uploadService";
|
||||
|
||||
const ENDPOINT = getEndpoint();
|
||||
|
@ -236,3 +236,31 @@ class UploadHttpClient {
|
|||
}
|
||||
|
||||
export default new UploadHttpClient();
|
||||
|
||||
const retrySleepTimeInMilliSeconds = [2000, 5000, 10000];
|
||||
|
||||
export async function retryHTTPCall(
|
||||
func: () => Promise<any>,
|
||||
checkForBreakingError?: (error) => void,
|
||||
): Promise<any> {
|
||||
const retrier = async (
|
||||
func: () => Promise<any>,
|
||||
attemptNumber: number = 0,
|
||||
) => {
|
||||
try {
|
||||
const resp = await func();
|
||||
return resp;
|
||||
} catch (e) {
|
||||
if (checkForBreakingError) {
|
||||
checkForBreakingError(e);
|
||||
}
|
||||
if (attemptNumber < retrySleepTimeInMilliSeconds.length) {
|
||||
await wait(retrySleepTimeInMilliSeconds[attemptNumber]);
|
||||
return await retrier(func, attemptNumber + 1);
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
};
|
||||
return await retrier(func);
|
||||
}
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
import { hasFileHash } from "@/media/file";
|
||||
import { FILE_TYPE, type FileTypeInfo } from "@/media/file-type";
|
||||
import { encodeLivePhoto } from "@/media/live-photo";
|
||||
import type { Metadata } from "@/media/types/file";
|
||||
|
@ -8,13 +9,8 @@ import { CustomErrorMessage } from "@/next/types/ipc";
|
|||
import { ensure } from "@/utils/ensure";
|
||||
import { ENCRYPTION_CHUNK_SIZE } from "@ente/shared/crypto/constants";
|
||||
import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker";
|
||||
import {
|
||||
B64EncryptionResult,
|
||||
EncryptionResult,
|
||||
LocalFileAttributes,
|
||||
} from "@ente/shared/crypto/types";
|
||||
import { B64EncryptionResult } from "@ente/shared/crypto/types";
|
||||
import { CustomError, handleUploadError } from "@ente/shared/error";
|
||||
import { isDataStream, type DataStream } from "@ente/shared/utils/data-stream";
|
||||
import { Remote } from "comlink";
|
||||
import {
|
||||
NULL_LOCATION,
|
||||
|
@ -43,7 +39,6 @@ import {
|
|||
updateMagicMetadata,
|
||||
} from "utils/magicMetadata";
|
||||
import { readStream } from "utils/native-stream";
|
||||
import { hasFileHash } from "utils/upload";
|
||||
import * as convert from "xml-js";
|
||||
import { detectFileTypeInfoFromChunk } from "../detect-type";
|
||||
import { getFileStream } from "../readerService";
|
||||
|
|
|
@ -20,7 +20,6 @@ import uploadManager, {
|
|||
import { Collection } from "types/collection";
|
||||
import { EncryptedEnteFile } from "types/file";
|
||||
import { groupFilesBasedOnCollectionID } from "utils/file";
|
||||
import { isHiddenFile } from "utils/upload";
|
||||
import { removeFromCollection } from "./collectionService";
|
||||
import { getLocalFiles } from "./fileService";
|
||||
|
||||
|
@ -596,6 +595,13 @@ const pathsToUpload = (paths: string[], watch: FolderWatch) =>
|
|||
// Files that are on disk but not yet synced or ignored.
|
||||
.filter((path) => !isSyncedOrIgnoredPath(path, watch));
|
||||
|
||||
/**
|
||||
* Return true if the file at the given {@link path} is hidden.
|
||||
*
|
||||
* Hidden files are those whose names begin with a "." (dot).
|
||||
*/
|
||||
const isHiddenFile = (path: string) => basename(path).startsWith(".");
|
||||
|
||||
/**
|
||||
* Return the paths to previously synced files that are no longer on disk and so
|
||||
* must be removed from the Ente collection.
|
||||
|
|
|
@ -1,128 +0,0 @@
|
|||
import type { Metadata } from "@/media/types/file";
|
||||
import { basename, dirname } from "@/next/file";
|
||||
import { PICKED_UPLOAD_TYPE } from "constants/upload";
|
||||
import isElectron from "is-electron";
|
||||
import { exportMetadataDirectoryName } from "services/export";
|
||||
import { fopFileName } from "services/upload/uploadService";
|
||||
|
||||
export const hasFileHash = (file: Metadata) =>
|
||||
file.hash || (file.imageHash && file.videoHash);
|
||||
|
||||
/**
|
||||
* Return true if all the paths in the given list are items that belong to the
|
||||
* same (arbitrary) directory.
|
||||
*
|
||||
* Empty list of paths is considered to be in the same directory.
|
||||
*/
|
||||
export const areAllInSameDirectory = (paths: string[]) =>
|
||||
new Set(paths.map(dirname)).size == 1;
|
||||
|
||||
// This is used to prompt the user the make upload strategy choice
|
||||
export interface ImportSuggestion {
|
||||
rootFolderName: string;
|
||||
hasNestedFolders: boolean;
|
||||
hasRootLevelFileWithFolder: boolean;
|
||||
}
|
||||
|
||||
export const DEFAULT_IMPORT_SUGGESTION: ImportSuggestion = {
|
||||
rootFolderName: "",
|
||||
hasNestedFolders: false,
|
||||
hasRootLevelFileWithFolder: false,
|
||||
};
|
||||
|
||||
export function getImportSuggestion(
|
||||
uploadType: PICKED_UPLOAD_TYPE,
|
||||
paths: string[],
|
||||
): ImportSuggestion {
|
||||
if (isElectron() && uploadType === PICKED_UPLOAD_TYPE.FILES) {
|
||||
return DEFAULT_IMPORT_SUGGESTION;
|
||||
}
|
||||
|
||||
const getCharCount = (str: string) => (str.match(/\//g) ?? []).length;
|
||||
paths.sort((path1, path2) => getCharCount(path1) - getCharCount(path2));
|
||||
const firstPath = paths[0];
|
||||
const lastPath = paths[paths.length - 1];
|
||||
|
||||
const L = firstPath.length;
|
||||
let i = 0;
|
||||
const firstFileFolder = firstPath.substring(0, firstPath.lastIndexOf("/"));
|
||||
const lastFileFolder = lastPath.substring(0, lastPath.lastIndexOf("/"));
|
||||
|
||||
while (i < L && firstPath.charAt(i) === lastPath.charAt(i)) i++;
|
||||
let commonPathPrefix = firstPath.substring(0, i);
|
||||
|
||||
if (commonPathPrefix) {
|
||||
commonPathPrefix = commonPathPrefix.substring(
|
||||
0,
|
||||
commonPathPrefix.lastIndexOf("/"),
|
||||
);
|
||||
if (commonPathPrefix) {
|
||||
commonPathPrefix = commonPathPrefix.substring(
|
||||
commonPathPrefix.lastIndexOf("/") + 1,
|
||||
);
|
||||
}
|
||||
}
|
||||
return {
|
||||
rootFolderName: commonPathPrefix || null,
|
||||
hasNestedFolders: firstFileFolder !== lastFileFolder,
|
||||
hasRootLevelFileWithFolder: firstFileFolder === "",
|
||||
};
|
||||
}
|
||||
|
||||
// This function groups files that are that have the same parent folder into collections
|
||||
// For Example, for user files have a directory structure like this
|
||||
// a
|
||||
// / | \
|
||||
// b j c
|
||||
// /|\ / \
|
||||
// e f g h i
|
||||
//
|
||||
// The files will grouped into 3 collections.
|
||||
// [a => [j],
|
||||
// b => [e,f,g],
|
||||
// c => [h, i]]
|
||||
export const groupFilesBasedOnParentFolder = (
|
||||
fileOrPaths: (File | string)[],
|
||||
) => {
|
||||
const result = new Map<string, (File | string)[]>();
|
||||
for (const fileOrPath of fileOrPaths) {
|
||||
const filePath =
|
||||
/* TODO(MR): ElectronFile */
|
||||
typeof fileOrPath == "string"
|
||||
? fileOrPath
|
||||
: (fileOrPath["path"] as string);
|
||||
|
||||
let folderPath = filePath.substring(0, filePath.lastIndexOf("/"));
|
||||
// If the parent folder of a file is "metadata"
|
||||
// we consider it to be part of the parent folder
|
||||
// For Eg,For FileList -> [a/x.png, a/metadata/x.png.json]
|
||||
// they will both we grouped into the collection "a"
|
||||
// This is cluster the metadata json files in the same collection as the file it is for
|
||||
if (folderPath.endsWith(exportMetadataDirectoryName)) {
|
||||
folderPath = folderPath.substring(0, folderPath.lastIndexOf("/"));
|
||||
}
|
||||
const folderName = folderPath.substring(
|
||||
folderPath.lastIndexOf("/") + 1,
|
||||
);
|
||||
if (!folderName) throw Error("Unexpected empty folder name");
|
||||
if (!result.has(folderName)) result.set(folderName, []);
|
||||
result.get(folderName).push(fileOrPath);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
/**
|
||||
* Filter out hidden files from amongst {@link fileOrPaths}.
|
||||
*
|
||||
* Hidden files are those whose names begin with a "." (dot).
|
||||
*/
|
||||
|
||||
export const pruneHiddenFiles = (fileOrPaths: (File | string)[]) =>
|
||||
fileOrPaths.filter((f) => !fopFileName(f).startsWith("."));
|
||||
|
||||
/**
|
||||
* Return true if the file at the given {@link path} is hidden.
|
||||
*
|
||||
* Hidden files are those whose names begin with a "." (dot).
|
||||
*/
|
||||
export const isHiddenFile = (path: string) => basename(path).startsWith(".");
|
|
@ -1,29 +0,0 @@
|
|||
import { wait } from "@ente/shared/utils";
|
||||
|
||||
const retrySleepTimeInMilliSeconds = [2000, 5000, 10000];
|
||||
|
||||
export async function retryHTTPCall(
|
||||
func: () => Promise<any>,
|
||||
checkForBreakingError?: (error) => void,
|
||||
): Promise<any> {
|
||||
const retrier = async (
|
||||
func: () => Promise<any>,
|
||||
attemptNumber: number = 0,
|
||||
) => {
|
||||
try {
|
||||
const resp = await func();
|
||||
return resp;
|
||||
} catch (e) {
|
||||
if (checkForBreakingError) {
|
||||
checkForBreakingError(e);
|
||||
}
|
||||
if (attemptNumber < retrySleepTimeInMilliSeconds.length) {
|
||||
await wait(retrySleepTimeInMilliSeconds[attemptNumber]);
|
||||
return await retrier(func, attemptNumber + 1);
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
};
|
||||
return await retrier(func);
|
||||
}
|
4
web/packages/media/file.ts
Normal file
4
web/packages/media/file.ts
Normal file
|
@ -0,0 +1,4 @@
|
|||
import type { Metadata } from "./types/file";
|
||||
|
||||
export const hasFileHash = (file: Metadata) =>
|
||||
!!file.hash || (!!file.imageHash && !!file.videoHash);
|
Loading…
Reference in a new issue