Merge branch 'refactor-upload' into cancel-upload-without-reload
This commit is contained in:
commit
e444177d6a
|
@ -17,7 +17,11 @@ import isElectron from 'is-electron';
|
|||
import { CustomError } from 'utils/error';
|
||||
import { Collection } from 'types/collection';
|
||||
import { SetLoading, SetFiles } from 'types/gallery';
|
||||
import { AnalysisResult, ElectronFile, FileWithCollection } from 'types/upload';
|
||||
import {
|
||||
ImportSuggestion,
|
||||
ElectronFile,
|
||||
FileWithCollection,
|
||||
} from 'types/upload';
|
||||
import { isCanvasBlocked } from 'utils/upload/isCanvasBlocked';
|
||||
import { downloadApp } from 'utils/common';
|
||||
import DiscFullIcon from '@mui/icons-material/DiscFull';
|
||||
|
@ -29,15 +33,19 @@ import {
|
|||
InProgressUpload,
|
||||
} from 'types/upload/ui';
|
||||
import {
|
||||
NULL_ANALYSIS_RESULT,
|
||||
DEFAULT_IMPORT_SUGGESTION,
|
||||
UPLOAD_STAGES,
|
||||
UPLOAD_STRATEGY,
|
||||
UPLOAD_TYPE,
|
||||
PICKED_UPLOAD_TYPE,
|
||||
} from 'constants/upload';
|
||||
import importService from 'services/importService';
|
||||
import { getDownloadAppMessage } from 'utils/ui';
|
||||
import UploadTypeSelector from './UploadTypeSelector';
|
||||
import { analyseUploadFiles, getCollectionWiseFiles } from 'utils/upload';
|
||||
import {
|
||||
getImportSuggestion,
|
||||
groupFilesBasedOnParentFolder,
|
||||
} from 'utils/upload';
|
||||
import { getUserOwnedCollections } from 'utils/collection';
|
||||
|
||||
const FIRST_ALBUM_NAME = 'My First Album';
|
||||
|
||||
|
@ -54,12 +62,11 @@ interface Props {
|
|||
setFiles: SetFiles;
|
||||
isFirstUpload: boolean;
|
||||
uploadTypeSelectorView: boolean;
|
||||
setUploadTypeSelectorView: (open: boolean) => void;
|
||||
showSessionExpiredMessage: () => void;
|
||||
showUploadFilesDialog: () => void;
|
||||
showUploadDirsDialog: () => void;
|
||||
folderSelectorFiles: File[];
|
||||
fileSelectorFiles: File[];
|
||||
webFolderSelectorFiles: File[];
|
||||
webFileSelectorFiles: File[];
|
||||
dragAndDropFiles: File[];
|
||||
}
|
||||
|
||||
|
@ -80,15 +87,17 @@ export default function Uploader(props: Props) {
|
|||
const [hasLivePhotos, setHasLivePhotos] = useState(false);
|
||||
|
||||
const [choiceModalView, setChoiceModalView] = useState(false);
|
||||
const [analysisResult, setAnalysisResult] =
|
||||
useState<AnalysisResult>(NULL_ANALYSIS_RESULT);
|
||||
const [importSuggestion, setImportSuggestion] = useState<ImportSuggestion>(
|
||||
DEFAULT_IMPORT_SUGGESTION
|
||||
);
|
||||
const appContext = useContext(AppContext);
|
||||
const galleryContext = useContext(GalleryContext);
|
||||
|
||||
const toUploadFiles = useRef<File[] | ElectronFile[]>(null);
|
||||
const isPendingDesktopUpload = useRef(false);
|
||||
const pendingDesktopUploadCollectionName = useRef<string>('');
|
||||
const uploadType = useRef<UPLOAD_TYPE>(null);
|
||||
// This is set when the user choses a type to upload from the upload type selector dialog
|
||||
const pickedUploadType = useRef<PICKED_UPLOAD_TYPE>(null);
|
||||
const zipPaths = useRef<string[]>(null);
|
||||
const [electronFiles, setElectronFiles] = useState<ElectronFile[]>(null);
|
||||
const [webFiles, setWebFiles] = useState([]);
|
||||
|
@ -118,24 +127,27 @@ export default function Uploader(props: Props) {
|
|||
}
|
||||
}, []);
|
||||
|
||||
// this handles the change of selectorFiles changes on web when user selects
|
||||
// files for upload through the opened file/folder selector or dragAndDrop them
|
||||
// the webFiles state is update which triggers the upload of those files
|
||||
useEffect(() => {
|
||||
if (
|
||||
uploadType.current === UPLOAD_TYPE.FOLDERS &&
|
||||
props.folderSelectorFiles?.length > 0
|
||||
pickedUploadType.current === PICKED_UPLOAD_TYPE.FOLDERS &&
|
||||
props.webFolderSelectorFiles?.length > 0
|
||||
) {
|
||||
setWebFiles(props.folderSelectorFiles);
|
||||
setWebFiles(props.webFolderSelectorFiles);
|
||||
} else if (
|
||||
uploadType.current === UPLOAD_TYPE.FILES &&
|
||||
props.fileSelectorFiles?.length > 0
|
||||
pickedUploadType.current === PICKED_UPLOAD_TYPE.FILES &&
|
||||
props.webFileSelectorFiles?.length > 0
|
||||
) {
|
||||
setWebFiles(props.fileSelectorFiles);
|
||||
setWebFiles(props.webFileSelectorFiles);
|
||||
} else if (props.dragAndDropFiles?.length > 0) {
|
||||
setWebFiles(props.dragAndDropFiles);
|
||||
}
|
||||
}, [
|
||||
props.dragAndDropFiles,
|
||||
props.fileSelectorFiles,
|
||||
props.folderSelectorFiles,
|
||||
props.webFileSelectorFiles,
|
||||
props.webFolderSelectorFiles,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
|
@ -176,14 +188,14 @@ export default function Uploader(props: Props) {
|
|||
toUploadFiles.current = electronFiles;
|
||||
setElectronFiles([]);
|
||||
}
|
||||
const analysisResult = analyseUploadFiles(
|
||||
uploadType.current,
|
||||
const importSuggestion = getImportSuggestion(
|
||||
pickedUploadType.current,
|
||||
toUploadFiles.current
|
||||
);
|
||||
setAnalysisResult(analysisResult);
|
||||
setImportSuggestion(importSuggestion);
|
||||
|
||||
handleCollectionCreationAndUpload(
|
||||
analysisResult,
|
||||
importSuggestion,
|
||||
props.isFirstUpload
|
||||
);
|
||||
props.setLoading(false);
|
||||
|
@ -191,14 +203,14 @@ export default function Uploader(props: Props) {
|
|||
}, [webFiles, appContext.sharedFiles, electronFiles]);
|
||||
|
||||
const resumeDesktopUpload = async (
|
||||
type: UPLOAD_TYPE,
|
||||
type: PICKED_UPLOAD_TYPE,
|
||||
electronFiles: ElectronFile[],
|
||||
collectionName: string
|
||||
) => {
|
||||
if (electronFiles && electronFiles?.length > 0) {
|
||||
isPendingDesktopUpload.current = true;
|
||||
pendingDesktopUploadCollectionName.current = collectionName;
|
||||
uploadType.current = type;
|
||||
pickedUploadType.current = type;
|
||||
setElectronFiles(electronFiles);
|
||||
}
|
||||
};
|
||||
|
@ -226,21 +238,29 @@ export default function Uploader(props: Props) {
|
|||
await preUploadAction();
|
||||
const filesWithCollectionToUpload: FileWithCollection[] = [];
|
||||
const collections: Collection[] = [];
|
||||
let collectionWiseFiles = new Map<
|
||||
let collectionNameToFilesMap = new Map<
|
||||
string,
|
||||
(File | ElectronFile)[]
|
||||
>();
|
||||
if (strategy === UPLOAD_STRATEGY.SINGLE_COLLECTION) {
|
||||
collectionWiseFiles.set(collectionName, toUploadFiles.current);
|
||||
collectionNameToFilesMap.set(
|
||||
collectionName,
|
||||
toUploadFiles.current
|
||||
);
|
||||
} else {
|
||||
collectionWiseFiles = getCollectionWiseFiles(
|
||||
collectionNameToFilesMap = groupFilesBasedOnParentFolder(
|
||||
toUploadFiles.current
|
||||
);
|
||||
}
|
||||
try {
|
||||
const existingCollection = await syncCollections();
|
||||
const existingCollection = getUserOwnedCollections(
|
||||
await syncCollections()
|
||||
);
|
||||
let index = 0;
|
||||
for (const [collectionName, files] of collectionWiseFiles) {
|
||||
for (const [
|
||||
collectionName,
|
||||
files,
|
||||
] of collectionNameToFilesMap) {
|
||||
const collection = await createAlbum(
|
||||
collectionName,
|
||||
existingCollection
|
||||
|
@ -295,13 +315,13 @@ export default function Uploader(props: Props) {
|
|||
await ImportService.setToUploadCollection(collections);
|
||||
if (zipPaths.current) {
|
||||
await ImportService.setToUploadFiles(
|
||||
UPLOAD_TYPE.ZIPS,
|
||||
PICKED_UPLOAD_TYPE.ZIPS,
|
||||
zipPaths.current
|
||||
);
|
||||
zipPaths.current = null;
|
||||
}
|
||||
await ImportService.setToUploadFiles(
|
||||
UPLOAD_TYPE.FILES,
|
||||
PICKED_UPLOAD_TYPE.FILES,
|
||||
filesWithCollectionToUploadIn.map(
|
||||
({ file }) => (file as ElectronFile).path
|
||||
)
|
||||
|
@ -326,8 +346,13 @@ export default function Uploader(props: Props) {
|
|||
|
||||
const retryFailed = async () => {
|
||||
try {
|
||||
const filesWithCollections =
|
||||
await uploadManager.getFailedFilesWithCollections();
|
||||
await preUploadAction();
|
||||
await uploadManager.retryFailedFiles();
|
||||
await uploadManager.queueFilesForUpload(
|
||||
filesWithCollections.files,
|
||||
filesWithCollections.collections
|
||||
);
|
||||
} catch (err) {
|
||||
showUserFacingError(err.message);
|
||||
closeUploadProgress();
|
||||
|
@ -391,7 +416,7 @@ export default function Uploader(props: Props) {
|
|||
};
|
||||
|
||||
const handleCollectionCreationAndUpload = (
|
||||
analysisResult: AnalysisResult,
|
||||
importSuggestion: ImportSuggestion,
|
||||
isFirstUpload: boolean
|
||||
) => {
|
||||
if (isPendingDesktopUpload.current) {
|
||||
|
@ -408,21 +433,22 @@ export default function Uploader(props: Props) {
|
|||
}
|
||||
return;
|
||||
}
|
||||
if (isElectron() && uploadType.current === UPLOAD_TYPE.ZIPS) {
|
||||
if (
|
||||
isElectron() &&
|
||||
pickedUploadType.current === PICKED_UPLOAD_TYPE.ZIPS
|
||||
) {
|
||||
uploadFilesToNewCollections(UPLOAD_STRATEGY.COLLECTION_PER_FOLDER);
|
||||
return;
|
||||
}
|
||||
if (isFirstUpload && !analysisResult.suggestedCollectionName) {
|
||||
analysisResult.suggestedCollectionName = FIRST_ALBUM_NAME;
|
||||
if (isFirstUpload && !importSuggestion.rootFolderName) {
|
||||
importSuggestion.rootFolderName = FIRST_ALBUM_NAME;
|
||||
}
|
||||
let showNextModal = () => {};
|
||||
if (analysisResult.multipleFolders) {
|
||||
if (importSuggestion.hasNestedFolders) {
|
||||
showNextModal = () => setChoiceModalView(true);
|
||||
} else {
|
||||
showNextModal = () =>
|
||||
uploadToSingleNewCollection(
|
||||
analysisResult.suggestedCollectionName
|
||||
);
|
||||
uploadToSingleNewCollection(importSuggestion.rootFolderName);
|
||||
}
|
||||
props.setCollectionSelectorAttributes({
|
||||
callback: uploadFilesToExistingCollection,
|
||||
|
@ -430,12 +456,12 @@ export default function Uploader(props: Props) {
|
|||
title: constants.UPLOAD_TO_COLLECTION,
|
||||
});
|
||||
};
|
||||
const handleDesktopUpload = async (type: UPLOAD_TYPE) => {
|
||||
const handleDesktopUpload = async (type: PICKED_UPLOAD_TYPE) => {
|
||||
let files: ElectronFile[];
|
||||
uploadType.current = type;
|
||||
if (type === UPLOAD_TYPE.FILES) {
|
||||
pickedUploadType.current = type;
|
||||
if (type === PICKED_UPLOAD_TYPE.FILES) {
|
||||
files = await ImportService.showUploadFilesDialog();
|
||||
} else if (type === UPLOAD_TYPE.FOLDERS) {
|
||||
} else if (type === PICKED_UPLOAD_TYPE.FOLDERS) {
|
||||
files = await ImportService.showUploadDirsDialog();
|
||||
} else {
|
||||
const response = await ImportService.showUploadZipDialog();
|
||||
|
@ -444,15 +470,15 @@ export default function Uploader(props: Props) {
|
|||
}
|
||||
if (files?.length > 0) {
|
||||
setElectronFiles(files);
|
||||
props.setUploadTypeSelectorView(false);
|
||||
props.closeUploadTypeSelector();
|
||||
}
|
||||
};
|
||||
|
||||
const handleWebUpload = async (type: UPLOAD_TYPE) => {
|
||||
uploadType.current = type;
|
||||
if (type === UPLOAD_TYPE.FILES) {
|
||||
const handleWebUpload = async (type: PICKED_UPLOAD_TYPE) => {
|
||||
pickedUploadType.current = type;
|
||||
if (type === PICKED_UPLOAD_TYPE.FILES) {
|
||||
props.showUploadFilesDialog();
|
||||
} else if (type === UPLOAD_TYPE.FOLDERS) {
|
||||
} else if (type === PICKED_UPLOAD_TYPE.FOLDERS) {
|
||||
props.showUploadDirsDialog();
|
||||
} else {
|
||||
appContext.setDialogMessage(getDownloadAppMessage());
|
||||
|
@ -471,9 +497,9 @@ export default function Uploader(props: Props) {
|
|||
}
|
||||
};
|
||||
|
||||
const handleFileUpload = handleUpload(UPLOAD_TYPE.FILES);
|
||||
const handleFolderUpload = handleUpload(UPLOAD_TYPE.FOLDERS);
|
||||
const handleZipUpload = handleUpload(UPLOAD_TYPE.ZIPS);
|
||||
const handleFileUpload = handleUpload(PICKED_UPLOAD_TYPE.FILES);
|
||||
const handleFolderUpload = handleUpload(PICKED_UPLOAD_TYPE.FOLDERS);
|
||||
const handleZipUpload = handleUpload(PICKED_UPLOAD_TYPE.ZIPS);
|
||||
|
||||
return (
|
||||
<>
|
||||
|
@ -481,9 +507,7 @@ export default function Uploader(props: Props) {
|
|||
open={choiceModalView}
|
||||
onClose={() => setChoiceModalView(false)}
|
||||
uploadToSingleCollection={() =>
|
||||
uploadToSingleNewCollection(
|
||||
analysisResult.suggestedCollectionName
|
||||
)
|
||||
uploadToSingleNewCollection(importSuggestion.rootFolderName)
|
||||
}
|
||||
uploadToMultipleCollection={() =>
|
||||
uploadFilesToNewCollections(
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
export const METADATA_FOLDER_NAME = 'metadata';
|
||||
export const ENTE_METADATA_FOLDER = 'metadata';
|
||||
|
||||
export enum ExportNotification {
|
||||
START = 'export started',
|
||||
|
|
|
@ -1,6 +1,10 @@
|
|||
import { ENCRYPTION_CHUNK_SIZE } from 'constants/crypto';
|
||||
import { FILE_TYPE } from 'constants/file';
|
||||
import { Location, ParsedExtractedMetadata } from 'types/upload';
|
||||
import {
|
||||
ImportSuggestion,
|
||||
Location,
|
||||
ParsedExtractedMetadata,
|
||||
} from 'types/upload';
|
||||
|
||||
// list of format that were missed by type-detection for some files.
|
||||
export const FORMAT_MISSED_BY_FILE_TYPE_LIB = [
|
||||
|
@ -50,7 +54,7 @@ export enum UPLOAD_STRATEGY {
|
|||
COLLECTION_PER_FOLDER,
|
||||
}
|
||||
|
||||
export enum UPLOAD_TYPE {
|
||||
export enum PICKED_UPLOAD_TYPE {
|
||||
FILES = 'files',
|
||||
FOLDERS = 'folders',
|
||||
ZIPS = 'zips',
|
||||
|
@ -69,9 +73,9 @@ export const A_SEC_IN_MICROSECONDS = 1e6;
|
|||
|
||||
export const USE_CF_PROXY = false;
|
||||
|
||||
export const NULL_ANALYSIS_RESULT = {
|
||||
suggestedCollectionName: '',
|
||||
multipleFolders: false,
|
||||
export const DEFAULT_IMPORT_SUGGESTION: ImportSuggestion = {
|
||||
rootFolderName: '',
|
||||
hasNestedFolders: false,
|
||||
};
|
||||
|
||||
export const BLACK_THUMBNAIL_BASE64 =
|
||||
|
|
|
@ -160,14 +160,14 @@ export default function Gallery() {
|
|||
disabled: uploadInProgress,
|
||||
});
|
||||
const {
|
||||
selectedFiles: fileSelectorFiles,
|
||||
selectedFiles: webFileSelectorFiles,
|
||||
open: openFileSelector,
|
||||
getInputProps: getFileSelectorInputProps,
|
||||
} = useFileInput({
|
||||
directory: false,
|
||||
});
|
||||
const {
|
||||
selectedFiles: folderSelectorFiles,
|
||||
selectedFiles: webFolderSelectorFiles,
|
||||
open: openFolderSelector,
|
||||
getInputProps: getFolderSelectorInputProps,
|
||||
} = useFileInput({
|
||||
|
@ -671,11 +671,10 @@ export default function Gallery() {
|
|||
setUploadInProgress={setUploadInProgress}
|
||||
setFiles={setFiles}
|
||||
isFirstUpload={hasNonEmptyCollections(collectionSummaries)}
|
||||
fileSelectorFiles={fileSelectorFiles}
|
||||
folderSelectorFiles={folderSelectorFiles}
|
||||
webFileSelectorFiles={webFileSelectorFiles}
|
||||
webFolderSelectorFiles={webFolderSelectorFiles}
|
||||
dragAndDropFiles={dragAndDropFiles}
|
||||
uploadTypeSelectorView={uploadTypeSelectorView}
|
||||
setUploadTypeSelectorView={setUploadTypeSelectorView}
|
||||
showUploadFilesDialog={openFileSelector}
|
||||
showUploadDirsDialog={openFolderSelector}
|
||||
showSessionExpiredMessage={showSessionExpiredMessage}
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import { UPLOAD_TYPE } from 'constants/upload';
|
||||
import { PICKED_UPLOAD_TYPE } from 'constants/upload';
|
||||
import { Collection } from 'types/collection';
|
||||
import { ElectronAPIs } from 'types/electron';
|
||||
import { ElectronFile, FileWithCollection } from 'types/upload';
|
||||
|
@ -8,7 +8,7 @@ import { logError } from 'utils/sentry';
|
|||
interface PendingUploads {
|
||||
files: ElectronFile[];
|
||||
collectionName: string;
|
||||
type: UPLOAD_TYPE;
|
||||
type: PICKED_UPLOAD_TYPE;
|
||||
}
|
||||
|
||||
interface selectZipResult {
|
||||
|
@ -88,7 +88,7 @@ class ImportService {
|
|||
}
|
||||
|
||||
async setToUploadFiles(
|
||||
type: UPLOAD_TYPE.FILES | UPLOAD_TYPE.ZIPS,
|
||||
type: PICKED_UPLOAD_TYPE.FILES | PICKED_UPLOAD_TYPE.ZIPS,
|
||||
filePaths: string[]
|
||||
) {
|
||||
if (this.allElectronAPIsExist) {
|
||||
|
@ -117,14 +117,14 @@ class ImportService {
|
|||
);
|
||||
}
|
||||
}
|
||||
this.setToUploadFiles(UPLOAD_TYPE.FILES, filePaths);
|
||||
this.setToUploadFiles(PICKED_UPLOAD_TYPE.FILES, filePaths);
|
||||
}
|
||||
}
|
||||
cancelRemainingUploads() {
|
||||
if (this.allElectronAPIsExist) {
|
||||
this.ElectronAPIs.setToUploadCollection(null);
|
||||
this.ElectronAPIs.setToUploadFiles(UPLOAD_TYPE.ZIPS, []);
|
||||
this.ElectronAPIs.setToUploadFiles(UPLOAD_TYPE.FILES, []);
|
||||
this.ElectronAPIs.setToUploadFiles(PICKED_UPLOAD_TYPE.ZIPS, []);
|
||||
this.ElectronAPIs.setToUploadFiles(PICKED_UPLOAD_TYPE.FILES, []);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,11 +1,11 @@
|
|||
import { getLocalFiles, setLocalFiles } from '../fileService';
|
||||
import { getLocalFiles } from '../fileService';
|
||||
import { SetFiles } from 'types/gallery';
|
||||
import { getDedicatedCryptoWorker } from 'utils/crypto';
|
||||
import {
|
||||
groupFilesBasedOnCollectionID,
|
||||
sortFiles,
|
||||
preservePhotoswipeProps,
|
||||
decryptFile,
|
||||
getUserOwnedNonTrashedFiles,
|
||||
} from 'utils/file';
|
||||
import { logError } from 'utils/sentry';
|
||||
import { getMetadataJSONMapKey, parseMetadataJSON } from './metadataService';
|
||||
|
@ -52,8 +52,8 @@ class UploadManager {
|
|||
private filesToBeUploaded: FileWithCollection[];
|
||||
private remainingFiles: FileWithCollection[] = [];
|
||||
private failedFiles: FileWithCollection[];
|
||||
private collectionToExistingFilesMap: Map<number, EnteFile[]>;
|
||||
private existingFiles: EnteFile[];
|
||||
private userOwnedNonTrashedExistingFiles: EnteFile[];
|
||||
private setFiles: SetFiles;
|
||||
private collections: Map<number, Collection>;
|
||||
private uploadInProgress: boolean;
|
||||
|
@ -85,7 +85,7 @@ class UploadManager {
|
|||
|
||||
async updateExistingFilesAndCollections(collections: Collection[]) {
|
||||
this.existingFiles = await getLocalFiles();
|
||||
this.collectionToExistingFilesMap = groupFilesBasedOnCollectionID(
|
||||
this.userOwnedNonTrashedExistingFiles = getUserOwnedNonTrashedFiles(
|
||||
this.existingFiles
|
||||
);
|
||||
this.collections = new Map(
|
||||
|
@ -390,14 +390,11 @@ class UploadManager {
|
|||
}
|
||||
let fileWithCollection = this.filesToBeUploaded.pop();
|
||||
const { collectionID } = fileWithCollection;
|
||||
const collectionExistingFiles =
|
||||
this.collectionToExistingFilesMap.get(collectionID) ?? [];
|
||||
const collection = this.collections.get(collectionID);
|
||||
fileWithCollection = { ...fileWithCollection, collection };
|
||||
const { fileUploadResult, uploadedFile } = await uploader(
|
||||
worker,
|
||||
collectionExistingFiles,
|
||||
this.existingFiles,
|
||||
this.userOwnedNonTrashedExistingFiles,
|
||||
fileWithCollection
|
||||
);
|
||||
|
||||
|
@ -417,7 +414,7 @@ class UploadManager {
|
|||
|
||||
async postUploadTask(
|
||||
fileUploadResult: UPLOAD_RESULT,
|
||||
uploadedFile: EnteFile,
|
||||
uploadedFile: EnteFile | null,
|
||||
fileWithCollection: FileWithCollection
|
||||
) {
|
||||
try {
|
||||
|
@ -449,7 +446,13 @@ class UploadManager {
|
|||
default:
|
||||
throw Error('Invalid Upload Result' + fileUploadResult);
|
||||
}
|
||||
if (decryptedFile) {
|
||||
if (
|
||||
[
|
||||
UPLOAD_RESULT.ADDED_SYMLINK,
|
||||
UPLOAD_RESULT.UPLOADED,
|
||||
UPLOAD_RESULT.UPLOADED_WITH_STATIC_THUMBNAIL,
|
||||
].includes(fileUploadResult)
|
||||
) {
|
||||
await this.updateExistingFiles(decryptedFile);
|
||||
}
|
||||
return fileUploadResult;
|
||||
|
@ -468,31 +471,24 @@ class UploadManager {
|
|||
uploadCancelService.requestUploadCancelation();
|
||||
}
|
||||
|
||||
async retryFailedFiles() {
|
||||
await this.queueFilesForUpload(this.failedFiles, [
|
||||
...this.collections.values(),
|
||||
]);
|
||||
}
|
||||
|
||||
private updateExistingFileToCollectionMap(decryptedFile: EnteFile) {
|
||||
if (
|
||||
!this.collectionToExistingFilesMap.has(decryptedFile.collectionID)
|
||||
) {
|
||||
this.collectionToExistingFilesMap.set(
|
||||
decryptedFile.collectionID,
|
||||
[]
|
||||
);
|
||||
}
|
||||
this.collectionToExistingFilesMap
|
||||
.get(decryptedFile.collectionID)
|
||||
.push(decryptedFile);
|
||||
async getFailedFilesWithCollections() {
|
||||
return {
|
||||
files: this.failedFiles,
|
||||
collections: [...this.collections.values()],
|
||||
};
|
||||
}
|
||||
|
||||
private async updateExistingFiles(decryptedFile: EnteFile) {
|
||||
if (!decryptedFile) {
|
||||
throw Error("decrypted file can't be undefined");
|
||||
}
|
||||
this.userOwnedNonTrashedExistingFiles.push(decryptedFile);
|
||||
await this.updateUIFiles(decryptedFile);
|
||||
}
|
||||
|
||||
private async updateUIFiles(decryptedFile: EnteFile) {
|
||||
this.existingFiles.push(decryptedFile);
|
||||
this.updateExistingFileToCollectionMap(decryptedFile);
|
||||
this.existingFiles = sortFiles(this.existingFiles);
|
||||
await setLocalFiles(this.existingFiles);
|
||||
this.setFiles(preservePhotoswipeProps(this.existingFiles));
|
||||
}
|
||||
|
||||
|
|
|
@ -1,14 +1,14 @@
|
|||
import { EnteFile } from 'types/file';
|
||||
import { handleUploadError, CustomError } from 'utils/error';
|
||||
import { logError } from 'utils/sentry';
|
||||
import { findMatchingExistingFile } from 'utils/upload';
|
||||
import { findMatchingExistingFiles } from 'utils/upload';
|
||||
import UploadHttpClient from './uploadHttpClient';
|
||||
import UIService from './uiService';
|
||||
import UploadService from './uploadService';
|
||||
import { FILE_TYPE } from 'constants/file';
|
||||
import { UPLOAD_RESULT, MAX_FILE_SIZE_SUPPORTED } from 'constants/upload';
|
||||
import { FileWithCollection, BackupedFile, UploadFile } from 'types/upload';
|
||||
import { addLogLine } from 'utils/logging';
|
||||
import { addLocalLog, addLogLine } from 'utils/logging';
|
||||
import { convertBytesToHumanReadable } from 'utils/file/size';
|
||||
import { sleep } from 'utils/common';
|
||||
import { addToCollection } from 'services/collectionService';
|
||||
|
@ -21,7 +21,6 @@ interface UploadResponse {
|
|||
|
||||
export default async function uploader(
|
||||
worker: any,
|
||||
existingFilesInCollection: EnteFile[],
|
||||
existingFiles: EnteFile[],
|
||||
fileWithCollection: FileWithCollection
|
||||
): Promise<UploadResponse> {
|
||||
|
@ -47,18 +46,35 @@ export default async function uploader(
|
|||
throw Error(CustomError.NO_METADATA);
|
||||
}
|
||||
|
||||
const existingFile = findMatchingExistingFile(existingFiles, metadata);
|
||||
if (existingFile) {
|
||||
if (existingFile.collectionID === collection.id) {
|
||||
const matchingExistingFiles = findMatchingExistingFiles(
|
||||
existingFiles,
|
||||
metadata
|
||||
);
|
||||
addLocalLog(
|
||||
() =>
|
||||
`matchedFileList: ${matchingExistingFiles
|
||||
.map((f) => `${f.id}-${f.metadata.title}`)
|
||||
.join(',')}`
|
||||
);
|
||||
if (matchingExistingFiles?.length) {
|
||||
const matchingExistingFilesCollectionIDs =
|
||||
matchingExistingFiles.map((e) => e.collectionID);
|
||||
addLocalLog(
|
||||
() =>
|
||||
`matched file collectionIDs:${matchingExistingFilesCollectionIDs}
|
||||
and collectionID:${collection.id}`
|
||||
);
|
||||
if (matchingExistingFilesCollectionIDs.includes(collection.id)) {
|
||||
addLogLine(
|
||||
`file already present in the collection , skipped upload for ${fileNameSize}`
|
||||
);
|
||||
return { fileUploadResult: UPLOAD_RESULT.ALREADY_UPLOADED };
|
||||
} else {
|
||||
addLogLine(
|
||||
`same file in other collection found for ${fileNameSize}`
|
||||
`same file in ${matchingExistingFilesCollectionIDs.length} collection found for ${fileNameSize}`
|
||||
);
|
||||
const resultFile = Object.assign({}, existingFile);
|
||||
// any of the matching file can used to add a symlink
|
||||
const resultFile = Object.assign({}, matchingExistingFiles[0]);
|
||||
resultFile.collectionID = collection.id;
|
||||
await addToCollection(collection, [resultFile]);
|
||||
return {
|
||||
|
|
|
@ -143,7 +143,8 @@ export interface ParsedExtractedMetadata {
|
|||
creationTime: number;
|
||||
}
|
||||
|
||||
export interface AnalysisResult {
|
||||
suggestedCollectionName: string;
|
||||
multipleFolders: boolean;
|
||||
// This is used to prompt the user the make upload strategy choice
|
||||
export interface ImportSuggestion {
|
||||
rootFolderName: string;
|
||||
hasNestedFolders: boolean;
|
||||
}
|
||||
|
|
|
@ -218,3 +218,11 @@ export const shouldShowOptions = (type: CollectionSummaryType) => {
|
|||
export const shouldBeShownOnCollectionBar = (type: CollectionSummaryType) => {
|
||||
return !HIDE_FROM_COLLECTION_BAR_TYPES.has(type);
|
||||
};
|
||||
|
||||
export const getUserOwnedCollections = (collections: Collection[]) => {
|
||||
const user: User = getData(LS_KEYS.USER);
|
||||
if (!user?.id) {
|
||||
throw Error('user missing');
|
||||
}
|
||||
return collections.filter((collection) => collection.owner.id === user.id);
|
||||
};
|
||||
|
|
|
@ -6,7 +6,7 @@ import { EnteFile } from 'types/file';
|
|||
|
||||
import { Metadata } from 'types/upload';
|
||||
import { formatDate, splitFilenameAndExtension } from 'utils/file';
|
||||
import { METADATA_FOLDER_NAME } from 'constants/export';
|
||||
import { ENTE_METADATA_FOLDER } from 'constants/export';
|
||||
|
||||
export const getExportRecordFileUID = (file: EnteFile) =>
|
||||
`${file.id}_${file.collectionID}_${file.updationTime}`;
|
||||
|
@ -179,7 +179,7 @@ export const getUniqueCollectionFolderPath = (
|
|||
};
|
||||
|
||||
export const getMetadataFolderPath = (collectionFolderPath: string) =>
|
||||
`${collectionFolderPath}/${METADATA_FOLDER_NAME}`;
|
||||
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}`;
|
||||
|
||||
export const getUniqueFileSaveName = (
|
||||
collectionPath: string,
|
||||
|
@ -211,7 +211,7 @@ export const getOldFileSaveName = (filename: string, fileID: number) =>
|
|||
export const getFileMetadataSavePath = (
|
||||
collectionFolderPath: string,
|
||||
fileSaveName: string
|
||||
) => `${collectionFolderPath}/${METADATA_FOLDER_NAME}/${fileSaveName}.json`;
|
||||
) => `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${fileSaveName}.json`;
|
||||
|
||||
export const getFileSavePath = (
|
||||
collectionFolderPath: string,
|
||||
|
@ -235,6 +235,6 @@ export const getOldFileMetadataSavePath = (
|
|||
collectionFolderPath: string,
|
||||
file: EnteFile
|
||||
) =>
|
||||
`${collectionFolderPath}/${METADATA_FOLDER_NAME}/${
|
||||
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${
|
||||
file.id
|
||||
}_${oldSanitizeName(file.metadata.title)}.json`;
|
||||
|
|
|
@ -508,3 +508,11 @@ export const createTypedObjectURL = async (blob: Blob, fileName: string) => {
|
|||
const type = await getFileType(new File([blob], fileName));
|
||||
return URL.createObjectURL(new Blob([blob], { type: type.mimeType }));
|
||||
};
|
||||
|
||||
export const getUserOwnedNonTrashedFiles = (files: EnteFile[]) => {
|
||||
const user: User = getData(LS_KEYS.USER);
|
||||
if (!user?.id) {
|
||||
throw Error('user missing');
|
||||
}
|
||||
return files.filter((file) => file.isTrashed || file.ownerID === user.id);
|
||||
};
|
||||
|
|
|
@ -4,12 +4,21 @@ import { formatDateTime } from 'utils/time';
|
|||
import { saveLogLine, getLogs } from 'utils/storage';
|
||||
|
||||
export function addLogLine(log: string) {
|
||||
if (!process.env.NEXT_PUBLIC_SENTRY_ENV) {
|
||||
console.log(log);
|
||||
}
|
||||
saveLogLine({
|
||||
timestamp: Date.now(),
|
||||
logLine: log,
|
||||
});
|
||||
}
|
||||
|
||||
export const addLocalLog = (getLog: () => string) => {
|
||||
if (!process.env.NEXT_PUBLIC_SENTRY_ENV) {
|
||||
console.log(getLog());
|
||||
}
|
||||
};
|
||||
|
||||
export function getDebugLogs() {
|
||||
return getLogs().map(
|
||||
(log) => `[${formatDateTime(log.timestamp)}] ${log.logLine}`
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
import {
|
||||
AnalysisResult,
|
||||
ImportSuggestion,
|
||||
ElectronFile,
|
||||
FileWithCollection,
|
||||
Metadata,
|
||||
|
@ -7,45 +7,27 @@ import {
|
|||
import { EnteFile } from 'types/file';
|
||||
import {
|
||||
A_SEC_IN_MICROSECONDS,
|
||||
NULL_ANALYSIS_RESULT,
|
||||
UPLOAD_TYPE,
|
||||
DEFAULT_IMPORT_SUGGESTION,
|
||||
PICKED_UPLOAD_TYPE,
|
||||
} from 'constants/upload';
|
||||
import { FILE_TYPE } from 'constants/file';
|
||||
import { METADATA_FOLDER_NAME } from 'constants/export';
|
||||
import { ENTE_METADATA_FOLDER } from 'constants/export';
|
||||
import isElectron from 'is-electron';
|
||||
|
||||
const TYPE_JSON = 'json';
|
||||
const DEDUPE_COLLECTION = new Set(['icloud library', 'icloudlibrary']);
|
||||
|
||||
export function findMatchingExistingFile(
|
||||
export function findMatchingExistingFiles(
|
||||
existingFiles: EnteFile[],
|
||||
newFileMetadata: Metadata
|
||||
): EnteFile {
|
||||
): EnteFile[] {
|
||||
const matchingFiles: EnteFile[] = [];
|
||||
for (const existingFile of existingFiles) {
|
||||
if (areFilesSame(existingFile.metadata, newFileMetadata)) {
|
||||
return existingFile;
|
||||
matchingFiles.push(existingFile);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function findSameFileInOtherCollection(
|
||||
existingFiles: EnteFile[],
|
||||
newFileMetadata: Metadata
|
||||
) {
|
||||
if (!hasFileHash(newFileMetadata)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
for (const existingFile of existingFiles) {
|
||||
if (
|
||||
hasFileHash(existingFile.metadata) &&
|
||||
areFilesWithFileHashSame(existingFile.metadata, newFileMetadata)
|
||||
) {
|
||||
return existingFile;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
return matchingFiles;
|
||||
}
|
||||
|
||||
export function shouldDedupeAcrossCollection(collectionName: string): boolean {
|
||||
|
@ -132,12 +114,12 @@ export function areFileWithCollectionsSame(
|
|||
return firstFile.localID === secondFile.localID;
|
||||
}
|
||||
|
||||
export function analyseUploadFiles(
|
||||
uploadType: UPLOAD_TYPE,
|
||||
export function getImportSuggestion(
|
||||
uploadType: PICKED_UPLOAD_TYPE,
|
||||
toUploadFiles: File[] | ElectronFile[]
|
||||
): AnalysisResult {
|
||||
if (isElectron() && uploadType === UPLOAD_TYPE.FILES) {
|
||||
return NULL_ANALYSIS_RESULT;
|
||||
): ImportSuggestion {
|
||||
if (isElectron() && uploadType === PICKED_UPLOAD_TYPE.FILES) {
|
||||
return DEFAULT_IMPORT_SUGGESTION;
|
||||
}
|
||||
|
||||
const paths: string[] = toUploadFiles.map((file) => file['path']);
|
||||
|
@ -165,26 +147,49 @@ export function analyseUploadFiles(
|
|||
}
|
||||
}
|
||||
return {
|
||||
suggestedCollectionName: commonPathPrefix || null,
|
||||
multipleFolders: firstFileFolder !== lastFileFolder,
|
||||
rootFolderName: commonPathPrefix || null,
|
||||
hasNestedFolders: firstFileFolder !== lastFileFolder,
|
||||
};
|
||||
}
|
||||
export function getCollectionWiseFiles(toUploadFiles: File[] | ElectronFile[]) {
|
||||
const collectionWiseFiles = new Map<string, (File | ElectronFile)[]>();
|
||||
|
||||
// This function groups files that are that have the same parent folder into collections
|
||||
// For Example, for user files have a directory structure like this
|
||||
// a
|
||||
// / | \
|
||||
// b j c
|
||||
// /|\ / \
|
||||
// e f g h i
|
||||
//
|
||||
// The files will grouped into 3 collections.
|
||||
// [a => [j],
|
||||
// b => [e,f,g],
|
||||
// c => [h, i]]
|
||||
export function groupFilesBasedOnParentFolder(
|
||||
toUploadFiles: File[] | ElectronFile[]
|
||||
) {
|
||||
const collectionNameToFilesMap = new Map<string, (File | ElectronFile)[]>();
|
||||
for (const file of toUploadFiles) {
|
||||
const filePath = file['path'] as string;
|
||||
|
||||
let folderPath = filePath.substring(0, filePath.lastIndexOf('/'));
|
||||
if (folderPath.endsWith(METADATA_FOLDER_NAME)) {
|
||||
// If the parent folder of a file is "metadata"
|
||||
// we consider it to be part of the parent folder
|
||||
// For Eg,For FileList -> [a/x.png, a/metadata/x.png.json]
|
||||
// they will both we grouped into the collection "a"
|
||||
// This is cluster the metadata json files in the same collection as the file it is for
|
||||
if (folderPath.endsWith(ENTE_METADATA_FOLDER)) {
|
||||
folderPath = folderPath.substring(0, folderPath.lastIndexOf('/'));
|
||||
}
|
||||
const folderName = folderPath.substring(
|
||||
folderPath.lastIndexOf('/') + 1
|
||||
);
|
||||
if (!collectionWiseFiles.has(folderName)) {
|
||||
collectionWiseFiles.set(folderName, []);
|
||||
if (!folderName?.length) {
|
||||
throw Error("folderName can't be null");
|
||||
}
|
||||
collectionWiseFiles.get(folderName).push(file);
|
||||
if (!collectionNameToFilesMap.has(folderName)) {
|
||||
collectionNameToFilesMap.set(folderName, []);
|
||||
}
|
||||
collectionNameToFilesMap.get(folderName).push(file);
|
||||
}
|
||||
return collectionWiseFiles;
|
||||
return collectionNameToFilesMap;
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue