Merge branch 'cancel-upload-without-reload' into watch
This commit is contained in:
commit
0646a908e2
|
@ -17,7 +17,11 @@ import isElectron from 'is-electron';
|
||||||
import { CustomError } from 'utils/error';
|
import { CustomError } from 'utils/error';
|
||||||
import { Collection } from 'types/collection';
|
import { Collection } from 'types/collection';
|
||||||
import { SetLoading, SetFiles } from 'types/gallery';
|
import { SetLoading, SetFiles } from 'types/gallery';
|
||||||
import { AnalysisResult, ElectronFile, FileWithCollection } from 'types/upload';
|
import {
|
||||||
|
ImportSuggestion,
|
||||||
|
ElectronFile,
|
||||||
|
FileWithCollection,
|
||||||
|
} from 'types/upload';
|
||||||
import { isCanvasBlocked } from 'utils/upload/isCanvasBlocked';
|
import { isCanvasBlocked } from 'utils/upload/isCanvasBlocked';
|
||||||
import { downloadApp, waitAndRun } from 'utils/common';
|
import { downloadApp, waitAndRun } from 'utils/common';
|
||||||
import watchFolderService from 'services/watchFolder/watchFolderService';
|
import watchFolderService from 'services/watchFolder/watchFolderService';
|
||||||
|
@ -30,15 +34,19 @@ import {
|
||||||
InProgressUpload,
|
InProgressUpload,
|
||||||
} from 'types/upload/ui';
|
} from 'types/upload/ui';
|
||||||
import {
|
import {
|
||||||
NULL_ANALYSIS_RESULT,
|
DEFAULT_IMPORT_SUGGESTION,
|
||||||
UPLOAD_STAGES,
|
UPLOAD_STAGES,
|
||||||
UPLOAD_STRATEGY,
|
UPLOAD_STRATEGY,
|
||||||
UPLOAD_TYPE,
|
PICKED_UPLOAD_TYPE,
|
||||||
} from 'constants/upload';
|
} from 'constants/upload';
|
||||||
import importService from 'services/importService';
|
import importService from 'services/importService';
|
||||||
import { getDownloadAppMessage } from 'utils/ui';
|
import { getDownloadAppMessage } from 'utils/ui';
|
||||||
import UploadTypeSelector from './UploadTypeSelector';
|
import UploadTypeSelector from './UploadTypeSelector';
|
||||||
import { analyseUploadFiles, getCollectionWiseFiles } from 'utils/upload';
|
import {
|
||||||
|
getImportSuggestion,
|
||||||
|
groupFilesBasedOnParentFolder,
|
||||||
|
} from 'utils/upload';
|
||||||
|
import { getUserOwnedCollections } from 'utils/collection';
|
||||||
|
|
||||||
const FIRST_ALBUM_NAME = 'My First Album';
|
const FIRST_ALBUM_NAME = 'My First Album';
|
||||||
|
|
||||||
|
@ -58,8 +66,8 @@ interface Props {
|
||||||
showSessionExpiredMessage: () => void;
|
showSessionExpiredMessage: () => void;
|
||||||
showUploadFilesDialog: () => void;
|
showUploadFilesDialog: () => void;
|
||||||
showUploadDirsDialog: () => void;
|
showUploadDirsDialog: () => void;
|
||||||
folderSelectorFiles: File[];
|
webFolderSelectorFiles: File[];
|
||||||
fileSelectorFiles: File[];
|
webFileSelectorFiles: File[];
|
||||||
dragAndDropFiles: File[];
|
dragAndDropFiles: File[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -80,15 +88,17 @@ export default function Uploader(props: Props) {
|
||||||
const [hasLivePhotos, setHasLivePhotos] = useState(false);
|
const [hasLivePhotos, setHasLivePhotos] = useState(false);
|
||||||
|
|
||||||
const [choiceModalView, setChoiceModalView] = useState(false);
|
const [choiceModalView, setChoiceModalView] = useState(false);
|
||||||
const [analysisResult, setAnalysisResult] =
|
const [importSuggestion, setImportSuggestion] = useState<ImportSuggestion>(
|
||||||
useState<AnalysisResult>(NULL_ANALYSIS_RESULT);
|
DEFAULT_IMPORT_SUGGESTION
|
||||||
|
);
|
||||||
const appContext = useContext(AppContext);
|
const appContext = useContext(AppContext);
|
||||||
const galleryContext = useContext(GalleryContext);
|
const galleryContext = useContext(GalleryContext);
|
||||||
|
|
||||||
const toUploadFiles = useRef<File[] | ElectronFile[]>(null);
|
const toUploadFiles = useRef<File[] | ElectronFile[]>(null);
|
||||||
const isPendingDesktopUpload = useRef(false);
|
const isPendingDesktopUpload = useRef(false);
|
||||||
const pendingDesktopUploadCollectionName = useRef<string>('');
|
const pendingDesktopUploadCollectionName = useRef<string>('');
|
||||||
const uploadType = useRef<UPLOAD_TYPE>(null);
|
// This is set when the user choses a type to upload from the upload type selector dialog
|
||||||
|
const pickedUploadType = useRef<PICKED_UPLOAD_TYPE>(null);
|
||||||
const zipPaths = useRef<string[]>(null);
|
const zipPaths = useRef<string[]>(null);
|
||||||
const previousUploadPromise = useRef<Promise<void>>(null);
|
const previousUploadPromise = useRef<Promise<void>>(null);
|
||||||
const [electronFiles, setElectronFiles] = useState<ElectronFile[]>(null);
|
const [electronFiles, setElectronFiles] = useState<ElectronFile[]>(null);
|
||||||
|
@ -130,6 +140,9 @@ export default function Uploader(props: Props) {
|
||||||
}
|
}
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
// this handles the change of selectorFiles changes on web when user selects
|
||||||
|
// files for upload through the opened file/folder selector or dragAndDrop them
|
||||||
|
// the webFiles state is update which triggers the upload of those files
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (appContext.watchFolderView) {
|
if (appContext.watchFolderView) {
|
||||||
// if watch folder dialog is open don't catch the dropped file
|
// if watch folder dialog is open don't catch the dropped file
|
||||||
|
@ -137,22 +150,22 @@ export default function Uploader(props: Props) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (
|
if (
|
||||||
uploadType.current === UPLOAD_TYPE.FOLDERS &&
|
pickedUploadType.current === PICKED_UPLOAD_TYPE.FOLDERS &&
|
||||||
props.folderSelectorFiles?.length > 0
|
props.webFolderSelectorFiles?.length > 0
|
||||||
) {
|
) {
|
||||||
setWebFiles(props.folderSelectorFiles);
|
setWebFiles(props.webFolderSelectorFiles);
|
||||||
} else if (
|
} else if (
|
||||||
uploadType.current === UPLOAD_TYPE.FILES &&
|
pickedUploadType.current === PICKED_UPLOAD_TYPE.FILES &&
|
||||||
props.fileSelectorFiles?.length > 0
|
props.webFileSelectorFiles?.length > 0
|
||||||
) {
|
) {
|
||||||
setWebFiles(props.fileSelectorFiles);
|
setWebFiles(props.webFileSelectorFiles);
|
||||||
} else if (props.dragAndDropFiles?.length > 0) {
|
} else if (props.dragAndDropFiles?.length > 0) {
|
||||||
setWebFiles(props.dragAndDropFiles);
|
setWebFiles(props.dragAndDropFiles);
|
||||||
}
|
}
|
||||||
}, [
|
}, [
|
||||||
props.dragAndDropFiles,
|
props.dragAndDropFiles,
|
||||||
props.fileSelectorFiles,
|
props.webFileSelectorFiles,
|
||||||
props.folderSelectorFiles,
|
props.webFolderSelectorFiles,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
@ -198,14 +211,14 @@ export default function Uploader(props: Props) {
|
||||||
toUploadFiles.current = electronFiles;
|
toUploadFiles.current = electronFiles;
|
||||||
setElectronFiles([]);
|
setElectronFiles([]);
|
||||||
}
|
}
|
||||||
const analysisResult = analyseUploadFiles(
|
const importSuggestion = getImportSuggestion(
|
||||||
uploadType.current,
|
pickedUploadType.current,
|
||||||
toUploadFiles.current
|
toUploadFiles.current
|
||||||
);
|
);
|
||||||
setAnalysisResult(analysisResult);
|
setImportSuggestion(importSuggestion);
|
||||||
|
|
||||||
handleCollectionCreationAndUpload(
|
handleCollectionCreationAndUpload(
|
||||||
analysisResult,
|
importSuggestion,
|
||||||
props.isFirstUpload
|
props.isFirstUpload
|
||||||
);
|
);
|
||||||
props.setLoading(false);
|
props.setLoading(false);
|
||||||
|
@ -213,14 +226,14 @@ export default function Uploader(props: Props) {
|
||||||
}, [webFiles, appContext.sharedFiles, electronFiles]);
|
}, [webFiles, appContext.sharedFiles, electronFiles]);
|
||||||
|
|
||||||
const resumeDesktopUpload = async (
|
const resumeDesktopUpload = async (
|
||||||
type: UPLOAD_TYPE,
|
type: PICKED_UPLOAD_TYPE,
|
||||||
electronFiles: ElectronFile[],
|
electronFiles: ElectronFile[],
|
||||||
collectionName: string
|
collectionName: string
|
||||||
) => {
|
) => {
|
||||||
if (electronFiles && electronFiles?.length > 0) {
|
if (electronFiles && electronFiles?.length > 0) {
|
||||||
isPendingDesktopUpload.current = true;
|
isPendingDesktopUpload.current = true;
|
||||||
pendingDesktopUploadCollectionName.current = collectionName;
|
pendingDesktopUploadCollectionName.current = collectionName;
|
||||||
uploadType.current = type;
|
pickedUploadType.current = type;
|
||||||
setElectronFiles(electronFiles);
|
setElectronFiles(electronFiles);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -250,21 +263,29 @@ export default function Uploader(props: Props) {
|
||||||
await preUploadAction();
|
await preUploadAction();
|
||||||
const filesWithCollectionToUpload: FileWithCollection[] = [];
|
const filesWithCollectionToUpload: FileWithCollection[] = [];
|
||||||
const collections: Collection[] = [];
|
const collections: Collection[] = [];
|
||||||
let collectionWiseFiles = new Map<
|
let collectionNameToFilesMap = new Map<
|
||||||
string,
|
string,
|
||||||
(File | ElectronFile)[]
|
(File | ElectronFile)[]
|
||||||
>();
|
>();
|
||||||
if (strategy === UPLOAD_STRATEGY.SINGLE_COLLECTION) {
|
if (strategy === UPLOAD_STRATEGY.SINGLE_COLLECTION) {
|
||||||
collectionWiseFiles.set(collectionName, toUploadFiles.current);
|
collectionNameToFilesMap.set(
|
||||||
|
collectionName,
|
||||||
|
toUploadFiles.current
|
||||||
|
);
|
||||||
} else {
|
} else {
|
||||||
collectionWiseFiles = getCollectionWiseFiles(
|
collectionNameToFilesMap = groupFilesBasedOnParentFolder(
|
||||||
toUploadFiles.current
|
toUploadFiles.current
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
const existingCollection = await syncCollections();
|
const existingCollection = getUserOwnedCollections(
|
||||||
|
await syncCollections()
|
||||||
|
);
|
||||||
let index = 0;
|
let index = 0;
|
||||||
for (const [collectionName, files] of collectionWiseFiles) {
|
for (const [
|
||||||
|
collectionName,
|
||||||
|
files,
|
||||||
|
] of collectionNameToFilesMap) {
|
||||||
const collection = await createAlbum(
|
const collection = await createAlbum(
|
||||||
collectionName,
|
collectionName,
|
||||||
existingCollection
|
existingCollection
|
||||||
|
@ -332,22 +353,26 @@ export default function Uploader(props: Props) {
|
||||||
await ImportService.setToUploadCollection(collections);
|
await ImportService.setToUploadCollection(collections);
|
||||||
if (zipPaths.current) {
|
if (zipPaths.current) {
|
||||||
await ImportService.setToUploadFiles(
|
await ImportService.setToUploadFiles(
|
||||||
UPLOAD_TYPE.ZIPS,
|
PICKED_UPLOAD_TYPE.ZIPS,
|
||||||
zipPaths.current
|
zipPaths.current
|
||||||
);
|
);
|
||||||
zipPaths.current = null;
|
zipPaths.current = null;
|
||||||
}
|
}
|
||||||
await ImportService.setToUploadFiles(
|
await ImportService.setToUploadFiles(
|
||||||
UPLOAD_TYPE.FILES,
|
PICKED_UPLOAD_TYPE.FILES,
|
||||||
filesWithCollectionToUploadIn.map(
|
filesWithCollectionToUploadIn.map(
|
||||||
({ file }) => (file as ElectronFile).path
|
({ file }) => (file as ElectronFile).path
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
const shouldCloseUploadProgress =
|
||||||
await uploadManager.queueFilesForUpload(
|
await uploadManager.queueFilesForUpload(
|
||||||
filesWithCollectionToUploadIn,
|
filesWithCollectionToUploadIn,
|
||||||
collections
|
collections
|
||||||
);
|
);
|
||||||
|
if (shouldCloseUploadProgress) {
|
||||||
|
closeUploadProgress();
|
||||||
|
}
|
||||||
if (isElectron()) {
|
if (isElectron()) {
|
||||||
if (watchFolderService.isUploadRunning()) {
|
if (watchFolderService.isUploadRunning()) {
|
||||||
await watchFolderService.allFileUploadsDone(
|
await watchFolderService.allFileUploadsDone(
|
||||||
|
@ -370,8 +395,13 @@ export default function Uploader(props: Props) {
|
||||||
|
|
||||||
const retryFailed = async () => {
|
const retryFailed = async () => {
|
||||||
try {
|
try {
|
||||||
|
const filesWithCollections =
|
||||||
|
await uploadManager.getFailedFilesWithCollections();
|
||||||
await preUploadAction();
|
await preUploadAction();
|
||||||
await uploadManager.retryFailedFiles();
|
await uploadManager.queueFilesForUpload(
|
||||||
|
filesWithCollections.files,
|
||||||
|
filesWithCollections.collections
|
||||||
|
);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
showUserFacingError(err.message);
|
showUserFacingError(err.message);
|
||||||
closeUploadProgress();
|
closeUploadProgress();
|
||||||
|
@ -435,7 +465,7 @@ export default function Uploader(props: Props) {
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleCollectionCreationAndUpload = (
|
const handleCollectionCreationAndUpload = (
|
||||||
analysisResult: AnalysisResult,
|
importSuggestion: ImportSuggestion,
|
||||||
isFirstUpload: boolean
|
isFirstUpload: boolean
|
||||||
) => {
|
) => {
|
||||||
if (isPendingDesktopUpload.current) {
|
if (isPendingDesktopUpload.current) {
|
||||||
|
@ -452,21 +482,22 @@ export default function Uploader(props: Props) {
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (isElectron() && uploadType.current === UPLOAD_TYPE.ZIPS) {
|
if (
|
||||||
|
isElectron() &&
|
||||||
|
pickedUploadType.current === PICKED_UPLOAD_TYPE.ZIPS
|
||||||
|
) {
|
||||||
uploadFilesToNewCollections(UPLOAD_STRATEGY.COLLECTION_PER_FOLDER);
|
uploadFilesToNewCollections(UPLOAD_STRATEGY.COLLECTION_PER_FOLDER);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (isFirstUpload && !analysisResult.suggestedCollectionName) {
|
if (isFirstUpload && !importSuggestion.rootFolderName) {
|
||||||
analysisResult.suggestedCollectionName = FIRST_ALBUM_NAME;
|
importSuggestion.rootFolderName = FIRST_ALBUM_NAME;
|
||||||
}
|
}
|
||||||
let showNextModal = () => {};
|
let showNextModal = () => {};
|
||||||
if (analysisResult.multipleFolders) {
|
if (importSuggestion.hasNestedFolders) {
|
||||||
showNextModal = () => setChoiceModalView(true);
|
showNextModal = () => setChoiceModalView(true);
|
||||||
} else {
|
} else {
|
||||||
showNextModal = () =>
|
showNextModal = () =>
|
||||||
uploadToSingleNewCollection(
|
uploadToSingleNewCollection(importSuggestion.rootFolderName);
|
||||||
analysisResult.suggestedCollectionName
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
props.setCollectionSelectorAttributes({
|
props.setCollectionSelectorAttributes({
|
||||||
callback: uploadFilesToExistingCollection,
|
callback: uploadFilesToExistingCollection,
|
||||||
|
@ -474,12 +505,12 @@ export default function Uploader(props: Props) {
|
||||||
title: constants.UPLOAD_TO_COLLECTION,
|
title: constants.UPLOAD_TO_COLLECTION,
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
const handleDesktopUpload = async (type: UPLOAD_TYPE) => {
|
const handleDesktopUpload = async (type: PICKED_UPLOAD_TYPE) => {
|
||||||
let files: ElectronFile[];
|
let files: ElectronFile[];
|
||||||
uploadType.current = type;
|
pickedUploadType.current = type;
|
||||||
if (type === UPLOAD_TYPE.FILES) {
|
if (type === PICKED_UPLOAD_TYPE.FILES) {
|
||||||
files = await ImportService.showUploadFilesDialog();
|
files = await ImportService.showUploadFilesDialog();
|
||||||
} else if (type === UPLOAD_TYPE.FOLDERS) {
|
} else if (type === PICKED_UPLOAD_TYPE.FOLDERS) {
|
||||||
files = await ImportService.showUploadDirsDialog();
|
files = await ImportService.showUploadDirsDialog();
|
||||||
} else {
|
} else {
|
||||||
const response = await ImportService.showUploadZipDialog();
|
const response = await ImportService.showUploadZipDialog();
|
||||||
|
@ -492,18 +523,18 @@ export default function Uploader(props: Props) {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleWebUpload = async (type: UPLOAD_TYPE) => {
|
const handleWebUpload = async (type: PICKED_UPLOAD_TYPE) => {
|
||||||
uploadType.current = type;
|
pickedUploadType.current = type;
|
||||||
if (type === UPLOAD_TYPE.FILES) {
|
if (type === PICKED_UPLOAD_TYPE.FILES) {
|
||||||
props.showUploadFilesDialog();
|
props.showUploadFilesDialog();
|
||||||
} else if (type === UPLOAD_TYPE.FOLDERS) {
|
} else if (type === PICKED_UPLOAD_TYPE.FOLDERS) {
|
||||||
props.showUploadDirsDialog();
|
props.showUploadDirsDialog();
|
||||||
} else {
|
} else {
|
||||||
appContext.setDialogMessage(getDownloadAppMessage());
|
appContext.setDialogMessage(getDownloadAppMessage());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const cancelUploads = async () => {
|
const cancelUploads = () => {
|
||||||
uploadManager.cancelRunningUpload();
|
uploadManager.cancelRunningUpload();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -515,9 +546,9 @@ export default function Uploader(props: Props) {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleFileUpload = handleUpload(UPLOAD_TYPE.FILES);
|
const handleFileUpload = handleUpload(PICKED_UPLOAD_TYPE.FILES);
|
||||||
const handleFolderUpload = handleUpload(UPLOAD_TYPE.FOLDERS);
|
const handleFolderUpload = handleUpload(PICKED_UPLOAD_TYPE.FOLDERS);
|
||||||
const handleZipUpload = handleUpload(UPLOAD_TYPE.ZIPS);
|
const handleZipUpload = handleUpload(PICKED_UPLOAD_TYPE.ZIPS);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<>
|
<>
|
||||||
|
@ -525,9 +556,7 @@ export default function Uploader(props: Props) {
|
||||||
open={choiceModalView}
|
open={choiceModalView}
|
||||||
onClose={() => setChoiceModalView(false)}
|
onClose={() => setChoiceModalView(false)}
|
||||||
uploadToSingleCollection={() =>
|
uploadToSingleCollection={() =>
|
||||||
uploadToSingleNewCollection(
|
uploadToSingleNewCollection(importSuggestion.rootFolderName)
|
||||||
analysisResult.suggestedCollectionName
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
uploadToMultipleCollection={() =>
|
uploadToMultipleCollection={() =>
|
||||||
uploadFilesToNewCollections(
|
uploadFilesToNewCollections(
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
export const METADATA_FOLDER_NAME = 'metadata';
|
export const ENTE_METADATA_FOLDER = 'metadata';
|
||||||
|
|
||||||
export enum ExportNotification {
|
export enum ExportNotification {
|
||||||
START = 'export started',
|
START = 'export started',
|
||||||
|
|
|
@ -1,6 +1,10 @@
|
||||||
import { ENCRYPTION_CHUNK_SIZE } from 'constants/crypto';
|
import { ENCRYPTION_CHUNK_SIZE } from 'constants/crypto';
|
||||||
import { FILE_TYPE } from 'constants/file';
|
import { FILE_TYPE } from 'constants/file';
|
||||||
import { Location, ParsedExtractedMetadata } from 'types/upload';
|
import {
|
||||||
|
ImportSuggestion,
|
||||||
|
Location,
|
||||||
|
ParsedExtractedMetadata,
|
||||||
|
} from 'types/upload';
|
||||||
|
|
||||||
// list of format that were missed by type-detection for some files.
|
// list of format that were missed by type-detection for some files.
|
||||||
export const FORMAT_MISSED_BY_FILE_TYPE_LIB = [
|
export const FORMAT_MISSED_BY_FILE_TYPE_LIB = [
|
||||||
|
@ -28,8 +32,8 @@ export enum UPLOAD_STAGES {
|
||||||
READING_GOOGLE_METADATA_FILES,
|
READING_GOOGLE_METADATA_FILES,
|
||||||
EXTRACTING_METADATA,
|
EXTRACTING_METADATA,
|
||||||
UPLOADING,
|
UPLOADING,
|
||||||
|
CANCELLING,
|
||||||
FINISH,
|
FINISH,
|
||||||
PAUSING,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export enum UPLOAD_STRATEGY {
|
export enum UPLOAD_STRATEGY {
|
||||||
|
@ -50,7 +54,7 @@ export enum UPLOAD_RESULT {
|
||||||
CANCELLED,
|
CANCELLED,
|
||||||
}
|
}
|
||||||
|
|
||||||
export enum UPLOAD_TYPE {
|
export enum PICKED_UPLOAD_TYPE {
|
||||||
FILES = 'files',
|
FILES = 'files',
|
||||||
FOLDERS = 'folders',
|
FOLDERS = 'folders',
|
||||||
ZIPS = 'zips',
|
ZIPS = 'zips',
|
||||||
|
@ -69,9 +73,9 @@ export const A_SEC_IN_MICROSECONDS = 1e6;
|
||||||
|
|
||||||
export const USE_CF_PROXY = false;
|
export const USE_CF_PROXY = false;
|
||||||
|
|
||||||
export const NULL_ANALYSIS_RESULT = {
|
export const DEFAULT_IMPORT_SUGGESTION: ImportSuggestion = {
|
||||||
suggestedCollectionName: '',
|
rootFolderName: '',
|
||||||
multipleFolders: false,
|
hasNestedFolders: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
export const BLACK_THUMBNAIL_BASE64 =
|
export const BLACK_THUMBNAIL_BASE64 =
|
||||||
|
|
|
@ -161,14 +161,14 @@ export default function Gallery() {
|
||||||
disabled: uploadInProgress,
|
disabled: uploadInProgress,
|
||||||
});
|
});
|
||||||
const {
|
const {
|
||||||
selectedFiles: fileSelectorFiles,
|
selectedFiles: webFileSelectorFiles,
|
||||||
open: openFileSelector,
|
open: openFileSelector,
|
||||||
getInputProps: getFileSelectorInputProps,
|
getInputProps: getFileSelectorInputProps,
|
||||||
} = useFileInput({
|
} = useFileInput({
|
||||||
directory: false,
|
directory: false,
|
||||||
});
|
});
|
||||||
const {
|
const {
|
||||||
selectedFiles: folderSelectorFiles,
|
selectedFiles: webFolderSelectorFiles,
|
||||||
open: openFolderSelector,
|
open: openFolderSelector,
|
||||||
getInputProps: getFolderSelectorInputProps,
|
getInputProps: getFolderSelectorInputProps,
|
||||||
} = useFileInput({
|
} = useFileInput({
|
||||||
|
@ -672,8 +672,8 @@ export default function Gallery() {
|
||||||
setUploadInProgress={setUploadInProgress}
|
setUploadInProgress={setUploadInProgress}
|
||||||
setFiles={setFiles}
|
setFiles={setFiles}
|
||||||
isFirstUpload={hasNonEmptyCollections(collectionSummaries)}
|
isFirstUpload={hasNonEmptyCollections(collectionSummaries)}
|
||||||
fileSelectorFiles={fileSelectorFiles}
|
webFileSelectorFiles={webFileSelectorFiles}
|
||||||
folderSelectorFiles={folderSelectorFiles}
|
webFolderSelectorFiles={webFolderSelectorFiles}
|
||||||
dragAndDropFiles={dragAndDropFiles}
|
dragAndDropFiles={dragAndDropFiles}
|
||||||
uploadTypeSelectorView={uploadTypeSelectorView}
|
uploadTypeSelectorView={uploadTypeSelectorView}
|
||||||
showUploadFilesDialog={openFileSelector}
|
showUploadFilesDialog={openFileSelector}
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
import { UPLOAD_TYPE } from 'constants/upload';
|
import { PICKED_UPLOAD_TYPE } from 'constants/upload';
|
||||||
import { Collection } from 'types/collection';
|
import { Collection } from 'types/collection';
|
||||||
import { ElectronAPIs } from 'types/electron';
|
import { ElectronAPIs } from 'types/electron';
|
||||||
import { ElectronFile, FileWithCollection } from 'types/upload';
|
import { ElectronFile, FileWithCollection } from 'types/upload';
|
||||||
|
@ -8,7 +8,7 @@ import { logError } from 'utils/sentry';
|
||||||
interface PendingUploads {
|
interface PendingUploads {
|
||||||
files: ElectronFile[];
|
files: ElectronFile[];
|
||||||
collectionName: string;
|
collectionName: string;
|
||||||
type: UPLOAD_TYPE;
|
type: PICKED_UPLOAD_TYPE;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface selectZipResult {
|
interface selectZipResult {
|
||||||
|
@ -88,7 +88,7 @@ class ImportService {
|
||||||
}
|
}
|
||||||
|
|
||||||
async setToUploadFiles(
|
async setToUploadFiles(
|
||||||
type: UPLOAD_TYPE.FILES | UPLOAD_TYPE.ZIPS,
|
type: PICKED_UPLOAD_TYPE.FILES | PICKED_UPLOAD_TYPE.ZIPS,
|
||||||
filePaths: string[]
|
filePaths: string[]
|
||||||
) {
|
) {
|
||||||
if (this.allElectronAPIsExist) {
|
if (this.allElectronAPIsExist) {
|
||||||
|
@ -117,14 +117,14 @@ class ImportService {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
this.setToUploadFiles(UPLOAD_TYPE.FILES, filePaths);
|
this.setToUploadFiles(PICKED_UPLOAD_TYPE.FILES, filePaths);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
cancelRemainingUploads() {
|
cancelRemainingUploads() {
|
||||||
if (this.allElectronAPIsExist) {
|
if (this.allElectronAPIsExist) {
|
||||||
this.ElectronAPIs.setToUploadCollection(null);
|
this.ElectronAPIs.setToUploadCollection(null);
|
||||||
this.ElectronAPIs.setToUploadFiles(UPLOAD_TYPE.ZIPS, []);
|
this.ElectronAPIs.setToUploadFiles(PICKED_UPLOAD_TYPE.ZIPS, []);
|
||||||
this.ElectronAPIs.setToUploadFiles(UPLOAD_TYPE.FILES, []);
|
this.ElectronAPIs.setToUploadFiles(PICKED_UPLOAD_TYPE.FILES, []);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,6 +14,7 @@ import {
|
||||||
import { CustomError } from 'utils/error';
|
import { CustomError } from 'utils/error';
|
||||||
import uploadCancelService from './uploadCancelService';
|
import uploadCancelService from './uploadCancelService';
|
||||||
|
|
||||||
|
const REQUEST_TIMEOUT_TIME = 30 * 1000; // 30 sec;
|
||||||
class UIService {
|
class UIService {
|
||||||
private perFileProgress: number;
|
private perFileProgress: number;
|
||||||
private filesUploaded: number;
|
private filesUploaded: number;
|
||||||
|
@ -75,7 +76,19 @@ class UIService {
|
||||||
this.updateProgressBarUI();
|
this.updateProgressBarUI();
|
||||||
}
|
}
|
||||||
|
|
||||||
updateProgressBarUI() {
|
hasFilesInResultList() {
|
||||||
|
const finishedUploadsList = segregatedFinishedUploadsToList(
|
||||||
|
this.finishedUploads
|
||||||
|
);
|
||||||
|
for (const x of finishedUploadsList.values()) {
|
||||||
|
if (x.length > 0) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
private updateProgressBarUI() {
|
||||||
const {
|
const {
|
||||||
setPercentComplete,
|
setPercentComplete,
|
||||||
setUploadCounter,
|
setUploadCounter,
|
||||||
|
@ -102,10 +115,10 @@ class UIService {
|
||||||
|
|
||||||
setPercentComplete(percentComplete);
|
setPercentComplete(percentComplete);
|
||||||
setInProgressUploads(
|
setInProgressUploads(
|
||||||
this.convertInProgressUploadsToList(this.inProgressUploads)
|
convertInProgressUploadsToList(this.inProgressUploads)
|
||||||
);
|
);
|
||||||
setFinishedUploads(
|
setFinishedUploads(
|
||||||
this.segregatedFinishedUploadsToList(this.finishedUploads)
|
segregatedFinishedUploadsToList(this.finishedUploads)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -115,20 +128,18 @@ class UIService {
|
||||||
index = 0
|
index = 0
|
||||||
) {
|
) {
|
||||||
const cancel: { exec: Canceler } = { exec: () => {} };
|
const cancel: { exec: Canceler } = { exec: () => {} };
|
||||||
|
const cancelTimedOutRequest = () =>
|
||||||
|
cancel.exec(CustomError.REQUEST_TIMEOUT);
|
||||||
|
|
||||||
|
const cancelCancelledUploadRequest = () =>
|
||||||
|
cancel.exec(CustomError.UPLOAD_CANCELLED);
|
||||||
|
|
||||||
let timeout = null;
|
let timeout = null;
|
||||||
const resetTimeout = () => {
|
const resetTimeout = () => {
|
||||||
if (timeout) {
|
if (timeout) {
|
||||||
clearTimeout(timeout);
|
clearTimeout(timeout);
|
||||||
}
|
}
|
||||||
timeout = setTimeout(
|
timeout = setTimeout(cancelTimedOutRequest, REQUEST_TIMEOUT_TIME);
|
||||||
() => cancel.exec(CustomError.REQUEST_TIMEOUT),
|
|
||||||
30 * 1000
|
|
||||||
);
|
|
||||||
};
|
|
||||||
const cancelIfUploadPaused = () => {
|
|
||||||
if (uploadCancelService.isUploadCancelationRequested()) {
|
|
||||||
cancel.exec(CustomError.UPLOAD_CANCELLED);
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
return {
|
return {
|
||||||
cancel,
|
cancel,
|
||||||
|
@ -149,12 +160,17 @@ class UIService {
|
||||||
} else {
|
} else {
|
||||||
resetTimeout();
|
resetTimeout();
|
||||||
}
|
}
|
||||||
cancelIfUploadPaused();
|
if (uploadCancelService.isUploadCancelationRequested()) {
|
||||||
|
cancelCancelledUploadRequest();
|
||||||
|
}
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
convertInProgressUploadsToList(inProgressUploads) {
|
export default new UIService();
|
||||||
|
|
||||||
|
function convertInProgressUploadsToList(inProgressUploads) {
|
||||||
return [...inProgressUploads.entries()].map(
|
return [...inProgressUploads.entries()].map(
|
||||||
([localFileID, progress]) =>
|
([localFileID, progress]) =>
|
||||||
({
|
({
|
||||||
|
@ -162,11 +178,10 @@ class UIService {
|
||||||
progress,
|
progress,
|
||||||
} as InProgressUpload)
|
} as InProgressUpload)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
segregatedFinishedUploadsToList(finishedUploads: FinishedUploads) {
|
function segregatedFinishedUploadsToList(finishedUploads: FinishedUploads) {
|
||||||
const segregatedFinishedUploads =
|
const segregatedFinishedUploads = new Map() as SegregatedFinishedUploads;
|
||||||
new Map() as SegregatedFinishedUploads;
|
|
||||||
for (const [localID, result] of finishedUploads) {
|
for (const [localID, result] of finishedUploads) {
|
||||||
if (!segregatedFinishedUploads.has(result)) {
|
if (!segregatedFinishedUploads.has(result)) {
|
||||||
segregatedFinishedUploads.set(result, []);
|
segregatedFinishedUploads.set(result, []);
|
||||||
|
@ -174,7 +189,4 @@ class UIService {
|
||||||
segregatedFinishedUploads.get(result).push(localID);
|
segregatedFinishedUploads.get(result).push(localID);
|
||||||
}
|
}
|
||||||
return segregatedFinishedUploads;
|
return segregatedFinishedUploads;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export default new UIService();
|
|
||||||
|
|
|
@ -1,22 +1,22 @@
|
||||||
interface UploadCancelStatus {
|
interface UploadCancelStatus {
|
||||||
val: boolean;
|
value: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
class UploadCancelService {
|
class UploadCancelService {
|
||||||
private shouldUploadBeCancelled: UploadCancelStatus = {
|
private shouldUploadBeCancelled: UploadCancelStatus = {
|
||||||
val: false,
|
value: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
reset() {
|
reset() {
|
||||||
this.shouldUploadBeCancelled.val = false;
|
this.shouldUploadBeCancelled.value = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
requestUploadCancelation() {
|
requestUploadCancelation() {
|
||||||
this.shouldUploadBeCancelled.val = true;
|
this.shouldUploadBeCancelled.value = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
isUploadCancelationRequested(): boolean {
|
isUploadCancelationRequested(): boolean {
|
||||||
return this.shouldUploadBeCancelled.val;
|
return this.shouldUploadBeCancelled.value;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -92,14 +92,16 @@ class UploadHttpClient {
|
||||||
progressTracker
|
progressTracker
|
||||||
): Promise<string> {
|
): Promise<string> {
|
||||||
try {
|
try {
|
||||||
await retryHTTPCall(() =>
|
await retryHTTPCall(
|
||||||
|
() =>
|
||||||
HTTPService.put(
|
HTTPService.put(
|
||||||
fileUploadURL.url,
|
fileUploadURL.url,
|
||||||
file,
|
file,
|
||||||
null,
|
null,
|
||||||
null,
|
null,
|
||||||
progressTracker
|
progressTracker
|
||||||
)
|
),
|
||||||
|
handleUploadError
|
||||||
);
|
);
|
||||||
return fileUploadURL.objectKey;
|
return fileUploadURL.objectKey;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
@ -127,7 +129,9 @@ class UploadHttpClient {
|
||||||
);
|
);
|
||||||
return fileUploadURL.objectKey;
|
return fileUploadURL.objectKey;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
if (e.message !== CustomError.UPLOAD_CANCELLED) {
|
||||||
logError(e, 'putFile to dataStore failed ');
|
logError(e, 'putFile to dataStore failed ');
|
||||||
|
}
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -152,10 +156,12 @@ class UploadHttpClient {
|
||||||
throw err;
|
throw err;
|
||||||
}
|
}
|
||||||
return resp;
|
return resp;
|
||||||
});
|
}, handleUploadError);
|
||||||
return response.headers.etag as string;
|
return response.headers.etag as string;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
if (e.message !== CustomError.UPLOAD_CANCELLED) {
|
||||||
logError(e, 'put filePart failed');
|
logError(e, 'put filePart failed');
|
||||||
|
}
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,16 +1,12 @@
|
||||||
import {
|
import { getLocalFiles, updateFileMagicMetadata } from '../fileService';
|
||||||
getLocalFiles,
|
|
||||||
setLocalFiles,
|
|
||||||
updateFileMagicMetadata,
|
|
||||||
} from '../fileService';
|
|
||||||
import { SetFiles } from 'types/gallery';
|
import { SetFiles } from 'types/gallery';
|
||||||
import { getDedicatedCryptoWorker } from 'utils/crypto';
|
import { getDedicatedCryptoWorker } from 'utils/crypto';
|
||||||
import {
|
import {
|
||||||
groupFilesBasedOnCollectionID,
|
|
||||||
sortFiles,
|
sortFiles,
|
||||||
preservePhotoswipeProps,
|
preservePhotoswipeProps,
|
||||||
decryptFile,
|
decryptFile,
|
||||||
appendNewFilePath,
|
appendNewFilePath,
|
||||||
|
getUserOwnedNonTrashedFiles,
|
||||||
} from 'utils/file';
|
} from 'utils/file';
|
||||||
import { logError } from 'utils/sentry';
|
import { logError } from 'utils/sentry';
|
||||||
import { getMetadataJSONMapKey, parseMetadataJSON } from './metadataService';
|
import { getMetadataJSONMapKey, parseMetadataJSON } from './metadataService';
|
||||||
|
@ -58,8 +54,8 @@ class UploadManager {
|
||||||
private filesToBeUploaded: FileWithCollection[];
|
private filesToBeUploaded: FileWithCollection[];
|
||||||
private remainingFiles: FileWithCollection[] = [];
|
private remainingFiles: FileWithCollection[] = [];
|
||||||
private failedFiles: FileWithCollection[];
|
private failedFiles: FileWithCollection[];
|
||||||
private collectionToExistingFilesMap: Map<number, EnteFile[]>;
|
|
||||||
private existingFiles: EnteFile[];
|
private existingFiles: EnteFile[];
|
||||||
|
private userOwnedNonTrashedExistingFiles: EnteFile[];
|
||||||
private setFiles: SetFiles;
|
private setFiles: SetFiles;
|
||||||
private collections: Map<number, Collection>;
|
private collections: Map<number, Collection>;
|
||||||
private uploadInProgress: boolean;
|
private uploadInProgress: boolean;
|
||||||
|
@ -85,12 +81,13 @@ class UploadManager {
|
||||||
prepareForNewUpload() {
|
prepareForNewUpload() {
|
||||||
this.resetState();
|
this.resetState();
|
||||||
UIService.reset();
|
UIService.reset();
|
||||||
|
uploadCancelService.reset();
|
||||||
UIService.setUploadStage(UPLOAD_STAGES.START);
|
UIService.setUploadStage(UPLOAD_STAGES.START);
|
||||||
}
|
}
|
||||||
|
|
||||||
async updateExistingFilesAndCollections(collections: Collection[]) {
|
async updateExistingFilesAndCollections(collections: Collection[]) {
|
||||||
this.existingFiles = await getLocalFiles();
|
this.existingFiles = await getLocalFiles();
|
||||||
this.collectionToExistingFilesMap = groupFilesBasedOnCollectionID(
|
this.userOwnedNonTrashedExistingFiles = getUserOwnedNonTrashedFiles(
|
||||||
this.existingFiles
|
this.existingFiles
|
||||||
);
|
);
|
||||||
this.collections = new Map(
|
this.collections = new Map(
|
||||||
|
@ -203,6 +200,16 @@ class UploadManager {
|
||||||
}
|
}
|
||||||
this.uploadInProgress = false;
|
this.uploadInProgress = false;
|
||||||
}
|
}
|
||||||
|
try {
|
||||||
|
if (!UIService.hasFilesInResultList()) {
|
||||||
|
return true;
|
||||||
|
} else {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
logError(e, ' failed to return shouldCloseProgressBar');
|
||||||
|
return false;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async parseMetadataJSONFiles(metadataFiles: FileWithCollection[]) {
|
private async parseMetadataJSONFiles(metadataFiles: FileWithCollection[]) {
|
||||||
|
@ -216,7 +223,6 @@ class UploadManager {
|
||||||
if (uploadCancelService.isUploadCancelationRequested()) {
|
if (uploadCancelService.isUploadCancelationRequested()) {
|
||||||
throw Error(CustomError.UPLOAD_CANCELLED);
|
throw Error(CustomError.UPLOAD_CANCELLED);
|
||||||
}
|
}
|
||||||
|
|
||||||
addLogLine(
|
addLogLine(
|
||||||
`parsing metadata json file ${getFileNameSize(file)}`
|
`parsing metadata json file ${getFileNameSize(file)}`
|
||||||
);
|
);
|
||||||
|
@ -242,8 +248,8 @@ class UploadManager {
|
||||||
if (e.message === CustomError.UPLOAD_CANCELLED) {
|
if (e.message === CustomError.UPLOAD_CANCELLED) {
|
||||||
throw e;
|
throw e;
|
||||||
} else {
|
} else {
|
||||||
|
// and don't break for subsequent files just log and move on
|
||||||
logError(e, 'parsing failed for a file');
|
logError(e, 'parsing failed for a file');
|
||||||
// and don't break for subsequent files
|
|
||||||
}
|
}
|
||||||
addLogLine(
|
addLogLine(
|
||||||
`failed to parse metadata json file ${getFileNameSize(
|
`failed to parse metadata json file ${getFileNameSize(
|
||||||
|
@ -257,7 +263,6 @@ class UploadManager {
|
||||||
logError(e, 'error seeding MetadataMap');
|
logError(e, 'error seeding MetadataMap');
|
||||||
}
|
}
|
||||||
throw e;
|
throw e;
|
||||||
// silently ignore the error
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -292,8 +297,8 @@ class UploadManager {
|
||||||
if (e.message === CustomError.UPLOAD_CANCELLED) {
|
if (e.message === CustomError.UPLOAD_CANCELLED) {
|
||||||
throw e;
|
throw e;
|
||||||
} else {
|
} else {
|
||||||
|
// and don't break for subsequent files just log and move on
|
||||||
logError(e, 'extractFileTypeAndMetadata failed');
|
logError(e, 'extractFileTypeAndMetadata failed');
|
||||||
// and don't break for subsequent files
|
|
||||||
}
|
}
|
||||||
addLogLine(
|
addLogLine(
|
||||||
`metadata extraction failed ${getFileNameSize(
|
`metadata extraction failed ${getFileNameSize(
|
||||||
|
@ -393,14 +398,11 @@ class UploadManager {
|
||||||
}
|
}
|
||||||
let fileWithCollection = this.filesToBeUploaded.pop();
|
let fileWithCollection = this.filesToBeUploaded.pop();
|
||||||
const { collectionID } = fileWithCollection;
|
const { collectionID } = fileWithCollection;
|
||||||
const collectionExistingFiles =
|
|
||||||
this.collectionToExistingFilesMap.get(collectionID) ?? [];
|
|
||||||
const collection = this.collections.get(collectionID);
|
const collection = this.collections.get(collectionID);
|
||||||
fileWithCollection = { ...fileWithCollection, collection };
|
fileWithCollection = { ...fileWithCollection, collection };
|
||||||
const { fileUploadResult, uploadedFile } = await uploader(
|
const { fileUploadResult, uploadedFile } = await uploader(
|
||||||
worker,
|
worker,
|
||||||
collectionExistingFiles,
|
this.userOwnedNonTrashedExistingFiles,
|
||||||
this.existingFiles,
|
|
||||||
fileWithCollection
|
fileWithCollection
|
||||||
);
|
);
|
||||||
|
|
||||||
|
@ -420,7 +422,7 @@ class UploadManager {
|
||||||
|
|
||||||
async postUploadTask(
|
async postUploadTask(
|
||||||
fileUploadResult: UPLOAD_RESULT,
|
fileUploadResult: UPLOAD_RESULT,
|
||||||
uploadedFile: EnteFile,
|
uploadedFile: EnteFile | null,
|
||||||
fileWithCollection: FileWithCollection
|
fileWithCollection: FileWithCollection
|
||||||
) {
|
) {
|
||||||
try {
|
try {
|
||||||
|
@ -461,8 +463,14 @@ class UploadManager {
|
||||||
default:
|
default:
|
||||||
throw Error('Invalid Upload Result' + fileUploadResult);
|
throw Error('Invalid Upload Result' + fileUploadResult);
|
||||||
}
|
}
|
||||||
if (decryptedFile) {
|
if (
|
||||||
await this.updateExistingFiles(decryptedFile);
|
[
|
||||||
|
UPLOAD_RESULT.ADDED_SYMLINK,
|
||||||
|
UPLOAD_RESULT.UPLOADED,
|
||||||
|
UPLOAD_RESULT.UPLOADED_WITH_STATIC_THUMBNAIL,
|
||||||
|
].includes(fileUploadResult)
|
||||||
|
) {
|
||||||
|
this.updateExistingFiles(decryptedFile);
|
||||||
await this.updateFilePaths(decryptedFile, fileWithCollection);
|
await this.updateFilePaths(decryptedFile, fileWithCollection);
|
||||||
}
|
}
|
||||||
return fileUploadResult;
|
return fileUploadResult;
|
||||||
|
@ -489,35 +497,28 @@ class UploadManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
public cancelRunningUpload() {
|
public cancelRunningUpload() {
|
||||||
UIService.setUploadStage(UPLOAD_STAGES.PAUSING);
|
UIService.setUploadStage(UPLOAD_STAGES.CANCELLING);
|
||||||
uploadCancelService.requestUploadCancelation();
|
uploadCancelService.requestUploadCancelation();
|
||||||
}
|
}
|
||||||
|
|
||||||
async retryFailedFiles() {
|
async getFailedFilesWithCollections() {
|
||||||
await this.queueFilesForUpload(this.failedFiles, [
|
return {
|
||||||
...this.collections.values(),
|
files: this.failedFiles,
|
||||||
]);
|
collections: [...this.collections.values()],
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private updateExistingFileToCollectionMap(decryptedFile: EnteFile) {
|
private updateExistingFiles(decryptedFile: EnteFile) {
|
||||||
if (
|
if (!decryptedFile) {
|
||||||
!this.collectionToExistingFilesMap.has(decryptedFile.collectionID)
|
throw Error("decrypted file can't be undefined");
|
||||||
) {
|
|
||||||
this.collectionToExistingFilesMap.set(
|
|
||||||
decryptedFile.collectionID,
|
|
||||||
[]
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
this.collectionToExistingFilesMap
|
this.userOwnedNonTrashedExistingFiles.push(decryptedFile);
|
||||||
.get(decryptedFile.collectionID)
|
this.updateUIFiles(decryptedFile);
|
||||||
.push(decryptedFile);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async updateExistingFiles(decryptedFile: EnteFile) {
|
private updateUIFiles(decryptedFile: EnteFile) {
|
||||||
this.existingFiles.push(decryptedFile);
|
this.existingFiles.push(decryptedFile);
|
||||||
this.updateExistingFileToCollectionMap(decryptedFile);
|
|
||||||
this.existingFiles = sortFiles(this.existingFiles);
|
this.existingFiles = sortFiles(this.existingFiles);
|
||||||
await setLocalFiles(this.existingFiles);
|
|
||||||
this.setFiles(preservePhotoswipeProps(this.existingFiles));
|
this.setFiles(preservePhotoswipeProps(this.existingFiles));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,14 +1,14 @@
|
||||||
import { EnteFile } from 'types/file';
|
import { EnteFile } from 'types/file';
|
||||||
import { handleUploadError, CustomError } from 'utils/error';
|
import { handleUploadError, CustomError } from 'utils/error';
|
||||||
import { logError } from 'utils/sentry';
|
import { logError } from 'utils/sentry';
|
||||||
import { findMatchingExistingFile } from 'utils/upload';
|
import { findMatchingExistingFiles } from 'utils/upload';
|
||||||
import UploadHttpClient from './uploadHttpClient';
|
import UploadHttpClient from './uploadHttpClient';
|
||||||
import UIService from './uiService';
|
import UIService from './uiService';
|
||||||
import UploadService from './uploadService';
|
import UploadService from './uploadService';
|
||||||
import { FILE_TYPE } from 'constants/file';
|
import { FILE_TYPE } from 'constants/file';
|
||||||
import { UPLOAD_RESULT, MAX_FILE_SIZE_SUPPORTED } from 'constants/upload';
|
import { UPLOAD_RESULT, MAX_FILE_SIZE_SUPPORTED } from 'constants/upload';
|
||||||
import { FileWithCollection, BackupedFile, UploadFile } from 'types/upload';
|
import { FileWithCollection, BackupedFile, UploadFile } from 'types/upload';
|
||||||
import { addLogLine } from 'utils/logging';
|
import { addLocalLog, addLogLine } from 'utils/logging';
|
||||||
import { convertBytesToHumanReadable } from 'utils/file/size';
|
import { convertBytesToHumanReadable } from 'utils/file/size';
|
||||||
import { sleep } from 'utils/common';
|
import { sleep } from 'utils/common';
|
||||||
import { addToCollection } from 'services/collectionService';
|
import { addToCollection } from 'services/collectionService';
|
||||||
|
@ -21,7 +21,6 @@ interface UploadResponse {
|
||||||
|
|
||||||
export default async function uploader(
|
export default async function uploader(
|
||||||
worker: any,
|
worker: any,
|
||||||
existingFilesInCollection: EnteFile[],
|
|
||||||
existingFiles: EnteFile[],
|
existingFiles: EnteFile[],
|
||||||
fileWithCollection: FileWithCollection
|
fileWithCollection: FileWithCollection
|
||||||
): Promise<UploadResponse> {
|
): Promise<UploadResponse> {
|
||||||
|
@ -47,18 +46,35 @@ export default async function uploader(
|
||||||
throw Error(CustomError.NO_METADATA);
|
throw Error(CustomError.NO_METADATA);
|
||||||
}
|
}
|
||||||
|
|
||||||
const existingFile = findMatchingExistingFile(existingFiles, metadata);
|
const matchingExistingFiles = findMatchingExistingFiles(
|
||||||
if (existingFile) {
|
existingFiles,
|
||||||
if (existingFile.collectionID === collection.id) {
|
metadata
|
||||||
|
);
|
||||||
|
addLocalLog(
|
||||||
|
() =>
|
||||||
|
`matchedFileList: ${matchingExistingFiles
|
||||||
|
.map((f) => `${f.id}-${f.metadata.title}`)
|
||||||
|
.join(',')}`
|
||||||
|
);
|
||||||
|
if (matchingExistingFiles?.length) {
|
||||||
|
const matchingExistingFilesCollectionIDs =
|
||||||
|
matchingExistingFiles.map((e) => e.collectionID);
|
||||||
|
addLocalLog(
|
||||||
|
() =>
|
||||||
|
`matched file collectionIDs:${matchingExistingFilesCollectionIDs}
|
||||||
|
and collectionID:${collection.id}`
|
||||||
|
);
|
||||||
|
if (matchingExistingFilesCollectionIDs.includes(collection.id)) {
|
||||||
addLogLine(
|
addLogLine(
|
||||||
`file already present in the collection , skipped upload for ${fileNameSize}`
|
`file already present in the collection , skipped upload for ${fileNameSize}`
|
||||||
);
|
);
|
||||||
return { fileUploadResult: UPLOAD_RESULT.ALREADY_UPLOADED };
|
return { fileUploadResult: UPLOAD_RESULT.ALREADY_UPLOADED };
|
||||||
} else {
|
} else {
|
||||||
addLogLine(
|
addLogLine(
|
||||||
`same file in other collection found for ${fileNameSize}`
|
`same file in ${matchingExistingFilesCollectionIDs.length} collection found for ${fileNameSize}`
|
||||||
);
|
);
|
||||||
const resultFile = Object.assign({}, existingFile);
|
// any of the matching file can used to add a symlink
|
||||||
|
const resultFile = Object.assign({}, matchingExistingFiles[0]);
|
||||||
resultFile.collectionID = collection.id;
|
resultFile.collectionID = collection.id;
|
||||||
await addToCollection(collection, [resultFile]);
|
await addToCollection(collection, [resultFile]);
|
||||||
return {
|
return {
|
||||||
|
|
|
@ -144,7 +144,8 @@ export interface ParsedExtractedMetadata {
|
||||||
creationTime: number;
|
creationTime: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface AnalysisResult {
|
// This is used to prompt the user the make upload strategy choice
|
||||||
suggestedCollectionName: string;
|
export interface ImportSuggestion {
|
||||||
multipleFolders: boolean;
|
rootFolderName: string;
|
||||||
|
hasNestedFolders: boolean;
|
||||||
}
|
}
|
||||||
|
|
|
@ -218,3 +218,11 @@ export const shouldShowOptions = (type: CollectionSummaryType) => {
|
||||||
export const shouldBeShownOnCollectionBar = (type: CollectionSummaryType) => {
|
export const shouldBeShownOnCollectionBar = (type: CollectionSummaryType) => {
|
||||||
return !HIDE_FROM_COLLECTION_BAR_TYPES.has(type);
|
return !HIDE_FROM_COLLECTION_BAR_TYPES.has(type);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const getUserOwnedCollections = (collections: Collection[]) => {
|
||||||
|
const user: User = getData(LS_KEYS.USER);
|
||||||
|
if (!user?.id) {
|
||||||
|
throw Error('user missing');
|
||||||
|
}
|
||||||
|
return collections.filter((collection) => collection.owner.id === user.id);
|
||||||
|
};
|
||||||
|
|
|
@ -83,6 +83,7 @@ export function handleUploadError(error): Error {
|
||||||
case CustomError.SUBSCRIPTION_EXPIRED:
|
case CustomError.SUBSCRIPTION_EXPIRED:
|
||||||
case CustomError.STORAGE_QUOTA_EXCEEDED:
|
case CustomError.STORAGE_QUOTA_EXCEEDED:
|
||||||
case CustomError.SESSION_EXPIRED:
|
case CustomError.SESSION_EXPIRED:
|
||||||
|
case CustomError.UPLOAD_CANCELLED:
|
||||||
throw parsedError;
|
throw parsedError;
|
||||||
}
|
}
|
||||||
return parsedError;
|
return parsedError;
|
||||||
|
|
|
@ -6,7 +6,7 @@ import { EnteFile } from 'types/file';
|
||||||
|
|
||||||
import { Metadata } from 'types/upload';
|
import { Metadata } from 'types/upload';
|
||||||
import { formatDate, splitFilenameAndExtension } from 'utils/file';
|
import { formatDate, splitFilenameAndExtension } from 'utils/file';
|
||||||
import { METADATA_FOLDER_NAME } from 'constants/export';
|
import { ENTE_METADATA_FOLDER } from 'constants/export';
|
||||||
|
|
||||||
export const getExportRecordFileUID = (file: EnteFile) =>
|
export const getExportRecordFileUID = (file: EnteFile) =>
|
||||||
`${file.id}_${file.collectionID}_${file.updationTime}`;
|
`${file.id}_${file.collectionID}_${file.updationTime}`;
|
||||||
|
@ -179,7 +179,7 @@ export const getUniqueCollectionFolderPath = (
|
||||||
};
|
};
|
||||||
|
|
||||||
export const getMetadataFolderPath = (collectionFolderPath: string) =>
|
export const getMetadataFolderPath = (collectionFolderPath: string) =>
|
||||||
`${collectionFolderPath}/${METADATA_FOLDER_NAME}`;
|
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}`;
|
||||||
|
|
||||||
export const getUniqueFileSaveName = (
|
export const getUniqueFileSaveName = (
|
||||||
collectionPath: string,
|
collectionPath: string,
|
||||||
|
@ -211,7 +211,7 @@ export const getOldFileSaveName = (filename: string, fileID: number) =>
|
||||||
export const getFileMetadataSavePath = (
|
export const getFileMetadataSavePath = (
|
||||||
collectionFolderPath: string,
|
collectionFolderPath: string,
|
||||||
fileSaveName: string
|
fileSaveName: string
|
||||||
) => `${collectionFolderPath}/${METADATA_FOLDER_NAME}/${fileSaveName}.json`;
|
) => `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${fileSaveName}.json`;
|
||||||
|
|
||||||
export const getFileSavePath = (
|
export const getFileSavePath = (
|
||||||
collectionFolderPath: string,
|
collectionFolderPath: string,
|
||||||
|
@ -235,6 +235,6 @@ export const getOldFileMetadataSavePath = (
|
||||||
collectionFolderPath: string,
|
collectionFolderPath: string,
|
||||||
file: EnteFile
|
file: EnteFile
|
||||||
) =>
|
) =>
|
||||||
`${collectionFolderPath}/${METADATA_FOLDER_NAME}/${
|
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${
|
||||||
file.id
|
file.id
|
||||||
}_${oldSanitizeName(file.metadata.title)}.json`;
|
}_${oldSanitizeName(file.metadata.title)}.json`;
|
||||||
|
|
|
@ -534,3 +534,11 @@ export const createTypedObjectURL = async (blob: Blob, fileName: string) => {
|
||||||
const type = await getFileType(new File([blob], fileName));
|
const type = await getFileType(new File([blob], fileName));
|
||||||
return URL.createObjectURL(new Blob([blob], { type: type.mimeType }));
|
return URL.createObjectURL(new Blob([blob], { type: type.mimeType }));
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const getUserOwnedNonTrashedFiles = (files: EnteFile[]) => {
|
||||||
|
const user: User = getData(LS_KEYS.USER);
|
||||||
|
if (!user?.id) {
|
||||||
|
throw Error('user missing');
|
||||||
|
}
|
||||||
|
return files.filter((file) => file.isTrashed || file.ownerID === user.id);
|
||||||
|
};
|
||||||
|
|
|
@ -4,12 +4,21 @@ import { formatDateTime } from 'utils/time';
|
||||||
import { saveLogLine, getLogs } from 'utils/storage';
|
import { saveLogLine, getLogs } from 'utils/storage';
|
||||||
|
|
||||||
export function addLogLine(log: string) {
|
export function addLogLine(log: string) {
|
||||||
|
if (!process.env.NEXT_PUBLIC_SENTRY_ENV) {
|
||||||
|
console.log(log);
|
||||||
|
}
|
||||||
saveLogLine({
|
saveLogLine({
|
||||||
timestamp: Date.now(),
|
timestamp: Date.now(),
|
||||||
logLine: log,
|
logLine: log,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const addLocalLog = (getLog: () => string) => {
|
||||||
|
if (!process.env.NEXT_PUBLIC_SENTRY_ENV) {
|
||||||
|
console.log(getLog());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
export function getDebugLogs() {
|
export function getDebugLogs() {
|
||||||
return getLogs().map(
|
return getLogs().map(
|
||||||
(log) => `[${formatDateTime(log.timestamp)}] ${log.logLine}`
|
(log) => `[${formatDateTime(log.timestamp)}] ${log.logLine}`
|
||||||
|
|
|
@ -110,8 +110,8 @@ const englishConstants = {
|
||||||
2: 'Reading file metadata',
|
2: 'Reading file metadata',
|
||||||
3: (fileCounter) =>
|
3: (fileCounter) =>
|
||||||
`${fileCounter.finished} / ${fileCounter.total} files backed up`,
|
`${fileCounter.finished} / ${fileCounter.total} files backed up`,
|
||||||
4: 'Backup complete',
|
4: 'Cancelling remaining uploads',
|
||||||
5: 'Pausing remaining uploads',
|
5: 'Backup complete',
|
||||||
},
|
},
|
||||||
UPLOADING_FILES: 'File upload',
|
UPLOADING_FILES: 'File upload',
|
||||||
FILE_NOT_UPLOADED_LIST: 'The following files were not uploaded',
|
FILE_NOT_UPLOADED_LIST: 'The following files were not uploaded',
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
import {
|
import {
|
||||||
AnalysisResult,
|
ImportSuggestion,
|
||||||
ElectronFile,
|
ElectronFile,
|
||||||
FileWithCollection,
|
FileWithCollection,
|
||||||
Metadata,
|
Metadata,
|
||||||
|
@ -7,45 +7,27 @@ import {
|
||||||
import { EnteFile } from 'types/file';
|
import { EnteFile } from 'types/file';
|
||||||
import {
|
import {
|
||||||
A_SEC_IN_MICROSECONDS,
|
A_SEC_IN_MICROSECONDS,
|
||||||
NULL_ANALYSIS_RESULT,
|
DEFAULT_IMPORT_SUGGESTION,
|
||||||
UPLOAD_TYPE,
|
PICKED_UPLOAD_TYPE,
|
||||||
} from 'constants/upload';
|
} from 'constants/upload';
|
||||||
import { FILE_TYPE } from 'constants/file';
|
import { FILE_TYPE } from 'constants/file';
|
||||||
import { METADATA_FOLDER_NAME } from 'constants/export';
|
import { ENTE_METADATA_FOLDER } from 'constants/export';
|
||||||
import isElectron from 'is-electron';
|
import isElectron from 'is-electron';
|
||||||
|
|
||||||
const TYPE_JSON = 'json';
|
const TYPE_JSON = 'json';
|
||||||
const DEDUPE_COLLECTION = new Set(['icloud library', 'icloudlibrary']);
|
const DEDUPE_COLLECTION = new Set(['icloud library', 'icloudlibrary']);
|
||||||
|
|
||||||
export function findMatchingExistingFile(
|
export function findMatchingExistingFiles(
|
||||||
existingFiles: EnteFile[],
|
existingFiles: EnteFile[],
|
||||||
newFileMetadata: Metadata
|
newFileMetadata: Metadata
|
||||||
): EnteFile {
|
): EnteFile[] {
|
||||||
|
const matchingFiles: EnteFile[] = [];
|
||||||
for (const existingFile of existingFiles) {
|
for (const existingFile of existingFiles) {
|
||||||
if (areFilesSame(existingFile.metadata, newFileMetadata)) {
|
if (areFilesSame(existingFile.metadata, newFileMetadata)) {
|
||||||
return existingFile;
|
matchingFiles.push(existingFile);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return null;
|
return matchingFiles;
|
||||||
}
|
|
||||||
|
|
||||||
export function findSameFileInOtherCollection(
|
|
||||||
existingFiles: EnteFile[],
|
|
||||||
newFileMetadata: Metadata
|
|
||||||
) {
|
|
||||||
if (!hasFileHash(newFileMetadata)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const existingFile of existingFiles) {
|
|
||||||
if (
|
|
||||||
hasFileHash(existingFile.metadata) &&
|
|
||||||
areFilesWithFileHashSame(existingFile.metadata, newFileMetadata)
|
|
||||||
) {
|
|
||||||
return existingFile;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export function shouldDedupeAcrossCollection(collectionName: string): boolean {
|
export function shouldDedupeAcrossCollection(collectionName: string): boolean {
|
||||||
|
@ -128,12 +110,12 @@ export function areFileWithCollectionsSame(
|
||||||
return firstFile.localID === secondFile.localID;
|
return firstFile.localID === secondFile.localID;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function analyseUploadFiles(
|
export function getImportSuggestion(
|
||||||
uploadType: UPLOAD_TYPE,
|
uploadType: PICKED_UPLOAD_TYPE,
|
||||||
toUploadFiles: File[] | ElectronFile[]
|
toUploadFiles: File[] | ElectronFile[]
|
||||||
): AnalysisResult {
|
): ImportSuggestion {
|
||||||
if (isElectron() && uploadType === UPLOAD_TYPE.FILES) {
|
if (isElectron() && uploadType === PICKED_UPLOAD_TYPE.FILES) {
|
||||||
return NULL_ANALYSIS_RESULT;
|
return DEFAULT_IMPORT_SUGGESTION;
|
||||||
}
|
}
|
||||||
|
|
||||||
const paths: string[] = toUploadFiles.map((file) => file['path']);
|
const paths: string[] = toUploadFiles.map((file) => file['path']);
|
||||||
|
@ -161,27 +143,49 @@ export function analyseUploadFiles(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
suggestedCollectionName: commonPathPrefix || null,
|
rootFolderName: commonPathPrefix || null,
|
||||||
multipleFolders: firstFileFolder !== lastFileFolder,
|
hasNestedFolders: firstFileFolder !== lastFileFolder,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getCollectionWiseFiles(toUploadFiles: File[] | ElectronFile[]) {
|
// This function groups files that are that have the same parent folder into collections
|
||||||
const collectionWiseFiles = new Map<string, (File | ElectronFile)[]>();
|
// For Example, for user files have a directory structure like this
|
||||||
|
// a
|
||||||
|
// / | \
|
||||||
|
// b j c
|
||||||
|
// /|\ / \
|
||||||
|
// e f g h i
|
||||||
|
//
|
||||||
|
// The files will grouped into 3 collections.
|
||||||
|
// [a => [j],
|
||||||
|
// b => [e,f,g],
|
||||||
|
// c => [h, i]]
|
||||||
|
export function groupFilesBasedOnParentFolder(
|
||||||
|
toUploadFiles: File[] | ElectronFile[]
|
||||||
|
) {
|
||||||
|
const collectionNameToFilesMap = new Map<string, (File | ElectronFile)[]>();
|
||||||
for (const file of toUploadFiles) {
|
for (const file of toUploadFiles) {
|
||||||
const filePath = file['path'] as string;
|
const filePath = file['path'] as string;
|
||||||
|
|
||||||
let folderPath = filePath.substring(0, filePath.lastIndexOf('/'));
|
let folderPath = filePath.substring(0, filePath.lastIndexOf('/'));
|
||||||
if (folderPath.endsWith(METADATA_FOLDER_NAME)) {
|
// If the parent folder of a file is "metadata"
|
||||||
|
// we consider it to be part of the parent folder
|
||||||
|
// For Eg,For FileList -> [a/x.png, a/metadata/x.png.json]
|
||||||
|
// they will both we grouped into the collection "a"
|
||||||
|
// This is cluster the metadata json files in the same collection as the file it is for
|
||||||
|
if (folderPath.endsWith(ENTE_METADATA_FOLDER)) {
|
||||||
folderPath = folderPath.substring(0, folderPath.lastIndexOf('/'));
|
folderPath = folderPath.substring(0, folderPath.lastIndexOf('/'));
|
||||||
}
|
}
|
||||||
const folderName = folderPath.substring(
|
const folderName = folderPath.substring(
|
||||||
folderPath.lastIndexOf('/') + 1
|
folderPath.lastIndexOf('/') + 1
|
||||||
);
|
);
|
||||||
if (!collectionWiseFiles.has(folderName)) {
|
if (!folderName?.length) {
|
||||||
collectionWiseFiles.set(folderName, []);
|
throw Error("folderName can't be null");
|
||||||
}
|
}
|
||||||
collectionWiseFiles.get(folderName).push(file);
|
if (!collectionNameToFilesMap.has(folderName)) {
|
||||||
|
collectionNameToFilesMap.set(folderName, []);
|
||||||
}
|
}
|
||||||
return collectionWiseFiles;
|
collectionNameToFilesMap.get(folderName).push(file);
|
||||||
|
}
|
||||||
|
return collectionNameToFilesMap;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue