beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'cancel-upload-without-reload' into watch

Browse source
This commit is contained in:
Abhinav 2022-09-01 17:44:07 +05:30
parent 42e5081fca e444177d6a
commit 0646a908e2
18 changed files with 321 additions and 222 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

137
src/components/Upload/Uploader.tsx
View file

@ -17,7 +17,11 @@ import isElectron from 'is-electron';
import { CustomError } from 'utils/error';
import { Collection } from 'types/collection';
import { SetLoading, SetFiles } from 'types/gallery';
import { AnalysisResult, ElectronFile, FileWithCollection } from 'types/upload';
import {
ImportSuggestion,
ElectronFile,
FileWithCollection,
} from 'types/upload';
import { isCanvasBlocked } from 'utils/upload/isCanvasBlocked';
import { downloadApp, waitAndRun } from 'utils/common';
import watchFolderService from 'services/watchFolder/watchFolderService';
@ -30,15 +34,19 @@ import {
InProgressUpload,
} from 'types/upload/ui';
import {
NULL_ANALYSIS_RESULT,
DEFAULT_IMPORT_SUGGESTION,
UPLOAD_STAGES,
UPLOAD_STRATEGY,
UPLOAD_TYPE,
PICKED_UPLOAD_TYPE,
} from 'constants/upload';
import importService from 'services/importService';
import { getDownloadAppMessage } from 'utils/ui';
import UploadTypeSelector from './UploadTypeSelector';
import { analyseUploadFiles, getCollectionWiseFiles } from 'utils/upload';
import {
getImportSuggestion,
groupFilesBasedOnParentFolder,
} from 'utils/upload';
import { getUserOwnedCollections } from 'utils/collection';
const FIRST_ALBUM_NAME = 'My First Album';
@ -58,8 +66,8 @@ interface Props {
showSessionExpiredMessage: () => void;
showUploadFilesDialog: () => void;
showUploadDirsDialog: () => void;
folderSelectorFiles: File[];
fileSelectorFiles: File[];
webFolderSelectorFiles: File[];
webFileSelectorFiles: File[];
dragAndDropFiles: File[];
}
@ -80,15 +88,17 @@ export default function Uploader(props: Props) {
const [hasLivePhotos, setHasLivePhotos] = useState(false);
const [choiceModalView, setChoiceModalView] = useState(false);
const [analysisResult, setAnalysisResult] =
useState<AnalysisResult>(NULL_ANALYSIS_RESULT);
const [importSuggestion, setImportSuggestion] = useState<ImportSuggestion>(
DEFAULT_IMPORT_SUGGESTION
);
const appContext = useContext(AppContext);
const galleryContext = useContext(GalleryContext);
const toUploadFiles = useRef<File[] | ElectronFile[]>(null);
const isPendingDesktopUpload = useRef(false);
const pendingDesktopUploadCollectionName = useRef<string>('');
const uploadType = useRef<UPLOAD_TYPE>(null);
// This is set when the user choses a type to upload from the upload type selector dialog
const pickedUploadType = useRef<PICKED_UPLOAD_TYPE>(null);
const zipPaths = useRef<string[]>(null);
const previousUploadPromise = useRef<Promise<void>>(null);
const [electronFiles, setElectronFiles] = useState<ElectronFile[]>(null);
@ -130,6 +140,9 @@ export default function Uploader(props: Props) {
}
}, []);
// this handles the change of selectorFiles changes on web when user selects
// files for upload through the opened file/folder selector or dragAndDrop them
// the webFiles state is update which triggers the upload of those files
useEffect(() => {
if (appContext.watchFolderView) {
// if watch folder dialog is open don't catch the dropped file
@ -137,22 +150,22 @@ export default function Uploader(props: Props) {
return;
}
if (
uploadType.current === UPLOAD_TYPE.FOLDERS &&
props.folderSelectorFiles?.length > 0
pickedUploadType.current === PICKED_UPLOAD_TYPE.FOLDERS &&
props.webFolderSelectorFiles?.length > 0
) {
setWebFiles(props.folderSelectorFiles);
setWebFiles(props.webFolderSelectorFiles);
} else if (
uploadType.current === UPLOAD_TYPE.FILES &&
props.fileSelectorFiles?.length > 0
pickedUploadType.current === PICKED_UPLOAD_TYPE.FILES &&
props.webFileSelectorFiles?.length > 0
) {
setWebFiles(props.fileSelectorFiles);
setWebFiles(props.webFileSelectorFiles);
} else if (props.dragAndDropFiles?.length > 0) {
setWebFiles(props.dragAndDropFiles);
}
}, [
props.dragAndDropFiles,
props.fileSelectorFiles,
props.folderSelectorFiles,
props.webFileSelectorFiles,
props.webFolderSelectorFiles,
]);
useEffect(() => {
@ -198,14 +211,14 @@ export default function Uploader(props: Props) {
toUploadFiles.current = electronFiles;
setElectronFiles([]);
}
const analysisResult = analyseUploadFiles(
uploadType.current,
const importSuggestion = getImportSuggestion(
pickedUploadType.current,
toUploadFiles.current
);
setAnalysisResult(analysisResult);
setImportSuggestion(importSuggestion);
handleCollectionCreationAndUpload(
analysisResult,
importSuggestion,
props.isFirstUpload
);
props.setLoading(false);
@ -213,14 +226,14 @@ export default function Uploader(props: Props) {
}, [webFiles, appContext.sharedFiles, electronFiles]);
const resumeDesktopUpload = async (
type: UPLOAD_TYPE,
type: PICKED_UPLOAD_TYPE,
electronFiles: ElectronFile[],
collectionName: string
) => {
if (electronFiles && electronFiles?.length > 0) {
isPendingDesktopUpload.current = true;
pendingDesktopUploadCollectionName.current = collectionName;
uploadType.current = type;
pickedUploadType.current = type;
setElectronFiles(electronFiles);
}
};
@ -250,21 +263,29 @@ export default function Uploader(props: Props) {
await preUploadAction();
const filesWithCollectionToUpload: FileWithCollection[] = [];
const collections: Collection[] = [];
let collectionWiseFiles = new Map<
let collectionNameToFilesMap = new Map<
string,
(File | ElectronFile)[]
>();
if (strategy === UPLOAD_STRATEGY.SINGLE_COLLECTION) {
collectionWiseFiles.set(collectionName, toUploadFiles.current);
collectionNameToFilesMap.set(
collectionName,
toUploadFiles.current
);
} else {
collectionWiseFiles = getCollectionWiseFiles(
collectionNameToFilesMap = groupFilesBasedOnParentFolder(
toUploadFiles.current
);
}
try {
const existingCollection = await syncCollections();
const existingCollection = getUserOwnedCollections(
await syncCollections()
);
let index = 0;
for (const [collectionName, files] of collectionWiseFiles) {
for (const [
collectionName,
files,
] of collectionNameToFilesMap) {
const collection = await createAlbum(
collectionName,
existingCollection
@ -332,22 +353,26 @@ export default function Uploader(props: Props) {
await ImportService.setToUploadCollection(collections);
if (zipPaths.current) {
await ImportService.setToUploadFiles(
UPLOAD_TYPE.ZIPS,
PICKED_UPLOAD_TYPE.ZIPS,
zipPaths.current
);
zipPaths.current = null;
}
await ImportService.setToUploadFiles(
UPLOAD_TYPE.FILES,
PICKED_UPLOAD_TYPE.FILES,
filesWithCollectionToUploadIn.map(
({ file }) => (file as ElectronFile).path
)
);
}
const shouldCloseUploadProgress =
await uploadManager.queueFilesForUpload(
filesWithCollectionToUploadIn,
collections
);
if (shouldCloseUploadProgress) {
closeUploadProgress();
}
if (isElectron()) {
if (watchFolderService.isUploadRunning()) {
await watchFolderService.allFileUploadsDone(
@ -370,8 +395,13 @@ export default function Uploader(props: Props) {
const retryFailed = async () => {
try {
const filesWithCollections =
await uploadManager.getFailedFilesWithCollections();
await preUploadAction();
await uploadManager.retryFailedFiles();
await uploadManager.queueFilesForUpload(
filesWithCollections.files,
filesWithCollections.collections
);
} catch (err) {
showUserFacingError(err.message);
closeUploadProgress();
@ -435,7 +465,7 @@ export default function Uploader(props: Props) {
};
const handleCollectionCreationAndUpload = (
analysisResult: AnalysisResult,
importSuggestion: ImportSuggestion,
isFirstUpload: boolean
) => {
if (isPendingDesktopUpload.current) {
@ -452,21 +482,22 @@ export default function Uploader(props: Props) {
}
return;
}
if (isElectron() && uploadType.current === UPLOAD_TYPE.ZIPS) {
if (
isElectron() &&
pickedUploadType.current === PICKED_UPLOAD_TYPE.ZIPS
) {
uploadFilesToNewCollections(UPLOAD_STRATEGY.COLLECTION_PER_FOLDER);
return;
}
if (isFirstUpload && !analysisResult.suggestedCollectionName) {
analysisResult.suggestedCollectionName = FIRST_ALBUM_NAME;
if (isFirstUpload && !importSuggestion.rootFolderName) {
importSuggestion.rootFolderName = FIRST_ALBUM_NAME;
}
let showNextModal = () => {};
if (analysisResult.multipleFolders) {
if (importSuggestion.hasNestedFolders) {
showNextModal = () => setChoiceModalView(true);
} else {
showNextModal = () =>
uploadToSingleNewCollection(
analysisResult.suggestedCollectionName
);
uploadToSingleNewCollection(importSuggestion.rootFolderName);
}
props.setCollectionSelectorAttributes({
callback: uploadFilesToExistingCollection,
@ -474,12 +505,12 @@ export default function Uploader(props: Props) {
title: constants.UPLOAD_TO_COLLECTION,
});
};
const handleDesktopUpload = async (type: UPLOAD_TYPE) => {
const handleDesktopUpload = async (type: PICKED_UPLOAD_TYPE) => {
let files: ElectronFile[];
uploadType.current = type;
if (type === UPLOAD_TYPE.FILES) {
pickedUploadType.current = type;
if (type === PICKED_UPLOAD_TYPE.FILES) {
files = await ImportService.showUploadFilesDialog();
} else if (type === UPLOAD_TYPE.FOLDERS) {
} else if (type === PICKED_UPLOAD_TYPE.FOLDERS) {
files = await ImportService.showUploadDirsDialog();
} else {
const response = await ImportService.showUploadZipDialog();
@ -492,18 +523,18 @@ export default function Uploader(props: Props) {
}
};
const handleWebUpload = async (type: UPLOAD_TYPE) => {
uploadType.current = type;
if (type === UPLOAD_TYPE.FILES) {
const handleWebUpload = async (type: PICKED_UPLOAD_TYPE) => {
pickedUploadType.current = type;
if (type === PICKED_UPLOAD_TYPE.FILES) {
props.showUploadFilesDialog();
} else if (type === UPLOAD_TYPE.FOLDERS) {
} else if (type === PICKED_UPLOAD_TYPE.FOLDERS) {
props.showUploadDirsDialog();
} else {
appContext.setDialogMessage(getDownloadAppMessage());
}
};
const cancelUploads = async () => {
const cancelUploads = () => {
uploadManager.cancelRunningUpload();
};
@ -515,9 +546,9 @@ export default function Uploader(props: Props) {
}
};
const handleFileUpload = handleUpload(UPLOAD_TYPE.FILES);
const handleFolderUpload = handleUpload(UPLOAD_TYPE.FOLDERS);
const handleZipUpload = handleUpload(UPLOAD_TYPE.ZIPS);
const handleFileUpload = handleUpload(PICKED_UPLOAD_TYPE.FILES);
const handleFolderUpload = handleUpload(PICKED_UPLOAD_TYPE.FOLDERS);
const handleZipUpload = handleUpload(PICKED_UPLOAD_TYPE.ZIPS);
return (
<>
@ -525,9 +556,7 @@ export default function Uploader(props: Props) {
open={choiceModalView}
onClose={() => setChoiceModalView(false)}
uploadToSingleCollection={() =>
uploadToSingleNewCollection(
analysisResult.suggestedCollectionName
)
uploadToSingleNewCollection(importSuggestion.rootFolderName)
}
uploadToMultipleCollection={() =>
uploadFilesToNewCollections(

2
src/constants/export/index.ts
View file

@ -1,4 +1,4 @@
export const METADATA_FOLDER_NAME = 'metadata';
export const ENTE_METADATA_FOLDER = 'metadata';
export enum ExportNotification {
START = 'export started',

16
src/constants/upload/index.ts
View file

@ -1,6 +1,10 @@
import { ENCRYPTION_CHUNK_SIZE } from 'constants/crypto';
import { FILE_TYPE } from 'constants/file';
import { Location, ParsedExtractedMetadata } from 'types/upload';
import {
ImportSuggestion,
Location,
ParsedExtractedMetadata,
} from 'types/upload';
// list of format that were missed by type-detection for some files.
export const FORMAT_MISSED_BY_FILE_TYPE_LIB = [
@ -28,8 +32,8 @@ export enum UPLOAD_STAGES {
READING_GOOGLE_METADATA_FILES,
EXTRACTING_METADATA,
UPLOADING,
CANCELLING,
FINISH,
PAUSING,
}
export enum UPLOAD_STRATEGY {
@ -50,7 +54,7 @@ export enum UPLOAD_RESULT {
CANCELLED,
}
export enum UPLOAD_TYPE {
export enum PICKED_UPLOAD_TYPE {
FILES = 'files',
FOLDERS = 'folders',
ZIPS = 'zips',
@ -69,9 +73,9 @@ export const A_SEC_IN_MICROSECONDS = 1e6;
export const USE_CF_PROXY = false;
export const NULL_ANALYSIS_RESULT = {
suggestedCollectionName: '',
multipleFolders: false,
export const DEFAULT_IMPORT_SUGGESTION: ImportSuggestion = {
rootFolderName: '',
hasNestedFolders: false,
};
export const BLACK_THUMBNAIL_BASE64 =

8
src/pages/gallery/index.tsx
View file

@ -161,14 +161,14 @@ export default function Gallery() {
disabled: uploadInProgress,
});
const {
selectedFiles: fileSelectorFiles,
selectedFiles: webFileSelectorFiles,
open: openFileSelector,
getInputProps: getFileSelectorInputProps,
} = useFileInput({
directory: false,
});
const {
selectedFiles: folderSelectorFiles,
selectedFiles: webFolderSelectorFiles,
open: openFolderSelector,
getInputProps: getFolderSelectorInputProps,
} = useFileInput({
@ -672,8 +672,8 @@ export default function Gallery() {
setUploadInProgress={setUploadInProgress}
setFiles={setFiles}
isFirstUpload={hasNonEmptyCollections(collectionSummaries)}
fileSelectorFiles={fileSelectorFiles}
folderSelectorFiles={folderSelectorFiles}
webFileSelectorFiles={webFileSelectorFiles}
webFolderSelectorFiles={webFolderSelectorFiles}
dragAndDropFiles={dragAndDropFiles}
uploadTypeSelectorView={uploadTypeSelectorView}
showUploadFilesDialog={openFileSelector}

12
src/services/importService.ts
View file

@ -1,4 +1,4 @@
import { UPLOAD_TYPE } from 'constants/upload';
import { PICKED_UPLOAD_TYPE } from 'constants/upload';
import { Collection } from 'types/collection';
import { ElectronAPIs } from 'types/electron';
import { ElectronFile, FileWithCollection } from 'types/upload';
@ -8,7 +8,7 @@ import { logError } from 'utils/sentry';
interface PendingUploads {
files: ElectronFile[];
collectionName: string;
type: UPLOAD_TYPE;
type: PICKED_UPLOAD_TYPE;
}
interface selectZipResult {
@ -88,7 +88,7 @@ class ImportService {
}
async setToUploadFiles(
type: UPLOAD_TYPE.FILES | UPLOAD_TYPE.ZIPS,
type: PICKED_UPLOAD_TYPE.FILES | PICKED_UPLOAD_TYPE.ZIPS,
filePaths: string[]
) {
if (this.allElectronAPIsExist) {
@ -117,14 +117,14 @@ class ImportService {
);
}
}
this.setToUploadFiles(UPLOAD_TYPE.FILES, filePaths);
this.setToUploadFiles(PICKED_UPLOAD_TYPE.FILES, filePaths);
}
}
cancelRemainingUploads() {
if (this.allElectronAPIsExist) {
this.ElectronAPIs.setToUploadCollection(null);
this.ElectronAPIs.setToUploadFiles(UPLOAD_TYPE.ZIPS, []);
this.ElectronAPIs.setToUploadFiles(UPLOAD_TYPE.FILES, []);
this.ElectronAPIs.setToUploadFiles(PICKED_UPLOAD_TYPE.ZIPS, []);
this.ElectronAPIs.setToUploadFiles(PICKED_UPLOAD_TYPE.FILES, []);
}
}
}

54
src/services/upload/uiService.ts
View file

@ -14,6 +14,7 @@ import {
import { CustomError } from 'utils/error';
import uploadCancelService from './uploadCancelService';
const REQUEST_TIMEOUT_TIME = 30 * 1000; // 30 sec;
class UIService {
private perFileProgress: number;
private filesUploaded: number;
@ -75,7 +76,19 @@ class UIService {
this.updateProgressBarUI();
}
updateProgressBarUI() {
hasFilesInResultList() {
const finishedUploadsList = segregatedFinishedUploadsToList(
this.finishedUploads
);
for (const x of finishedUploadsList.values()) {
if (x.length > 0) {
return true;
}
}
return false;
}
private updateProgressBarUI() {
const {
setPercentComplete,
setUploadCounter,
@ -102,10 +115,10 @@ class UIService {
setPercentComplete(percentComplete);
setInProgressUploads(
this.convertInProgressUploadsToList(this.inProgressUploads)
convertInProgressUploadsToList(this.inProgressUploads)
);
setFinishedUploads(
this.segregatedFinishedUploadsToList(this.finishedUploads)
segregatedFinishedUploadsToList(this.finishedUploads)
);
}
@ -115,20 +128,18 @@ class UIService {
index = 0
) {
const cancel: { exec: Canceler } = { exec: () => {} };
const cancelTimedOutRequest = () =>
cancel.exec(CustomError.REQUEST_TIMEOUT);
const cancelCancelledUploadRequest = () =>
cancel.exec(CustomError.UPLOAD_CANCELLED);
let timeout = null;
const resetTimeout = () => {
if (timeout) {
clearTimeout(timeout);
}
timeout = setTimeout(
() => cancel.exec(CustomError.REQUEST_TIMEOUT),
30 * 1000
);
};
const cancelIfUploadPaused = () => {
if (uploadCancelService.isUploadCancelationRequested()) {
cancel.exec(CustomError.UPLOAD_CANCELLED);
}
timeout = setTimeout(cancelTimedOutRequest, REQUEST_TIMEOUT_TIME);
};
return {
cancel,
@ -149,12 +160,17 @@ class UIService {
} else {
resetTimeout();
}
cancelIfUploadPaused();
if (uploadCancelService.isUploadCancelationRequested()) {
cancelCancelledUploadRequest();
}
},
};
}
}
convertInProgressUploadsToList(inProgressUploads) {
export default new UIService();
function convertInProgressUploadsToList(inProgressUploads) {
return [...inProgressUploads.entries()].map(
([localFileID, progress]) =>
({
@ -162,11 +178,10 @@ class UIService {
progress,
} as InProgressUpload)
);
}
}
segregatedFinishedUploadsToList(finishedUploads: FinishedUploads) {
const segregatedFinishedUploads =
new Map() as SegregatedFinishedUploads;
function segregatedFinishedUploadsToList(finishedUploads: FinishedUploads) {
const segregatedFinishedUploads = new Map() as SegregatedFinishedUploads;
for (const [localID, result] of finishedUploads) {
if (!segregatedFinishedUploads.has(result)) {
segregatedFinishedUploads.set(result, []);
@ -174,7 +189,4 @@ class UIService {
segregatedFinishedUploads.get(result).push(localID);
}
return segregatedFinishedUploads;
}
}
export default new UIService();

10
src/services/upload/uploadCancelService.ts
View file

@ -1,22 +1,22 @@
interface UploadCancelStatus {
val: boolean;
value: boolean;
}
class UploadCancelService {
private shouldUploadBeCancelled: UploadCancelStatus = {
val: false,
value: false,
};
reset() {
this.shouldUploadBeCancelled.val = false;
this.shouldUploadBeCancelled.value = false;
}
requestUploadCancelation() {
this.shouldUploadBeCancelled.val = true;
this.shouldUploadBeCancelled.value = true;
}
isUploadCancelationRequested(): boolean {
return this.shouldUploadBeCancelled.val;
return this.shouldUploadBeCancelled.value;
}
}

12
src/services/upload/uploadHttpClient.ts
View file

@ -92,14 +92,16 @@ class UploadHttpClient {
progressTracker
): Promise<string> {
try {
await retryHTTPCall(() =>
await retryHTTPCall(
() =>
HTTPService.put(
fileUploadURL.url,
file,
null,
null,
progressTracker
)
),
handleUploadError
);
return fileUploadURL.objectKey;
} catch (e) {
@ -127,7 +129,9 @@ class UploadHttpClient {
);
return fileUploadURL.objectKey;
} catch (e) {
if (e.message !== CustomError.UPLOAD_CANCELLED) {
logError(e, 'putFile to dataStore failed ');
}
throw e;
}
}
@ -152,10 +156,12 @@ class UploadHttpClient {
throw err;
}
return resp;
});
}, handleUploadError);
return response.headers.etag as string;
} catch (e) {
if (e.message !== CustomError.UPLOAD_CANCELLED) {
logError(e, 'put filePart failed');
}
throw e;
}
}

77
src/services/upload/uploadManager.ts
View file

@ -1,16 +1,12 @@
import {
getLocalFiles,
setLocalFiles,
updateFileMagicMetadata,
} from '../fileService';
import { getLocalFiles, updateFileMagicMetadata } from '../fileService';
import { SetFiles } from 'types/gallery';
import { getDedicatedCryptoWorker } from 'utils/crypto';
import {
groupFilesBasedOnCollectionID,
sortFiles,
preservePhotoswipeProps,
decryptFile,
appendNewFilePath,
getUserOwnedNonTrashedFiles,
} from 'utils/file';
import { logError } from 'utils/sentry';
import { getMetadataJSONMapKey, parseMetadataJSON } from './metadataService';
@ -58,8 +54,8 @@ class UploadManager {
private filesToBeUploaded: FileWithCollection[];
private remainingFiles: FileWithCollection[] = [];
private failedFiles: FileWithCollection[];
private collectionToExistingFilesMap: Map<number, EnteFile[]>;
private existingFiles: EnteFile[];
private userOwnedNonTrashedExistingFiles: EnteFile[];
private setFiles: SetFiles;
private collections: Map<number, Collection>;
private uploadInProgress: boolean;
@ -85,12 +81,13 @@ class UploadManager {
prepareForNewUpload() {
this.resetState();
UIService.reset();
uploadCancelService.reset();
UIService.setUploadStage(UPLOAD_STAGES.START);
}
async updateExistingFilesAndCollections(collections: Collection[]) {
this.existingFiles = await getLocalFiles();
this.collectionToExistingFilesMap = groupFilesBasedOnCollectionID(
this.userOwnedNonTrashedExistingFiles = getUserOwnedNonTrashedFiles(
this.existingFiles
);
this.collections = new Map(
@ -203,6 +200,16 @@ class UploadManager {
}
this.uploadInProgress = false;
}
try {
if (!UIService.hasFilesInResultList()) {
return true;
} else {
return false;
}
} catch (e) {
logError(e, ' failed to return shouldCloseProgressBar');
return false;
}
}
private async parseMetadataJSONFiles(metadataFiles: FileWithCollection[]) {
@ -216,7 +223,6 @@ class UploadManager {
if (uploadCancelService.isUploadCancelationRequested()) {
throw Error(CustomError.UPLOAD_CANCELLED);
}
addLogLine(
`parsing metadata json file ${getFileNameSize(file)}`
);
@ -242,8 +248,8 @@ class UploadManager {
if (e.message === CustomError.UPLOAD_CANCELLED) {
throw e;
} else {
// and don't break for subsequent files just log and move on
logError(e, 'parsing failed for a file');
// and don't break for subsequent files
}
addLogLine(
`failed to parse metadata json file ${getFileNameSize(
@ -257,7 +263,6 @@ class UploadManager {
logError(e, 'error seeding MetadataMap');
}
throw e;
// silently ignore the error
}
}
@ -292,8 +297,8 @@ class UploadManager {
if (e.message === CustomError.UPLOAD_CANCELLED) {
throw e;
} else {
// and don't break for subsequent files just log and move on
logError(e, 'extractFileTypeAndMetadata failed');
// and don't break for subsequent files
}
addLogLine(
`metadata extraction failed ${getFileNameSize(
@ -393,14 +398,11 @@ class UploadManager {
}
let fileWithCollection = this.filesToBeUploaded.pop();
const { collectionID } = fileWithCollection;
const collectionExistingFiles =
this.collectionToExistingFilesMap.get(collectionID) ?? [];
const collection = this.collections.get(collectionID);
fileWithCollection = { ...fileWithCollection, collection };
const { fileUploadResult, uploadedFile } = await uploader(
worker,
collectionExistingFiles,
this.existingFiles,
this.userOwnedNonTrashedExistingFiles,
fileWithCollection
);
@ -420,7 +422,7 @@ class UploadManager {
async postUploadTask(
fileUploadResult: UPLOAD_RESULT,
uploadedFile: EnteFile,
uploadedFile: EnteFile | null,
fileWithCollection: FileWithCollection
) {
try {
@ -461,8 +463,14 @@ class UploadManager {
default:
throw Error('Invalid Upload Result' + fileUploadResult);
}
if (decryptedFile) {
await this.updateExistingFiles(decryptedFile);
if (
[
UPLOAD_RESULT.ADDED_SYMLINK,
UPLOAD_RESULT.UPLOADED,
UPLOAD_RESULT.UPLOADED_WITH_STATIC_THUMBNAIL,
].includes(fileUploadResult)
) {
this.updateExistingFiles(decryptedFile);
await this.updateFilePaths(decryptedFile, fileWithCollection);
}
return fileUploadResult;
@ -489,35 +497,28 @@ class UploadManager {
}
public cancelRunningUpload() {
UIService.setUploadStage(UPLOAD_STAGES.PAUSING);
UIService.setUploadStage(UPLOAD_STAGES.CANCELLING);
uploadCancelService.requestUploadCancelation();
}
async retryFailedFiles() {
await this.queueFilesForUpload(this.failedFiles, [
...this.collections.values(),
]);
async getFailedFilesWithCollections() {
return {
files: this.failedFiles,
collections: [...this.collections.values()],
};
}
private updateExistingFileToCollectionMap(decryptedFile: EnteFile) {
if (
!this.collectionToExistingFilesMap.has(decryptedFile.collectionID)
) {
this.collectionToExistingFilesMap.set(
decryptedFile.collectionID,
[]
);
private updateExistingFiles(decryptedFile: EnteFile) {
if (!decryptedFile) {
throw Error("decrypted file can't be undefined");
}
this.collectionToExistingFilesMap
.get(decryptedFile.collectionID)
.push(decryptedFile);
this.userOwnedNonTrashedExistingFiles.push(decryptedFile);
this.updateUIFiles(decryptedFile);
}
private async updateExistingFiles(decryptedFile: EnteFile) {
private updateUIFiles(decryptedFile: EnteFile) {
this.existingFiles.push(decryptedFile);
this.updateExistingFileToCollectionMap(decryptedFile);
this.existingFiles = sortFiles(this.existingFiles);
await setLocalFiles(this.existingFiles);
this.setFiles(preservePhotoswipeProps(this.existingFiles));
}

32
src/services/upload/uploader.ts
View file

@ -1,14 +1,14 @@
import { EnteFile } from 'types/file';
import { handleUploadError, CustomError } from 'utils/error';
import { logError } from 'utils/sentry';
import { findMatchingExistingFile } from 'utils/upload';
import { findMatchingExistingFiles } from 'utils/upload';
import UploadHttpClient from './uploadHttpClient';
import UIService from './uiService';
import UploadService from './uploadService';
import { FILE_TYPE } from 'constants/file';
import { UPLOAD_RESULT, MAX_FILE_SIZE_SUPPORTED } from 'constants/upload';
import { FileWithCollection, BackupedFile, UploadFile } from 'types/upload';
import { addLogLine } from 'utils/logging';
import { addLocalLog, addLogLine } from 'utils/logging';
import { convertBytesToHumanReadable } from 'utils/file/size';
import { sleep } from 'utils/common';
import { addToCollection } from 'services/collectionService';
@ -21,7 +21,6 @@ interface UploadResponse {
export default async function uploader(
worker: any,
existingFilesInCollection: EnteFile[],
existingFiles: EnteFile[],
fileWithCollection: FileWithCollection
): Promise<UploadResponse> {
@ -47,18 +46,35 @@ export default async function uploader(
throw Error(CustomError.NO_METADATA);
}
const existingFile = findMatchingExistingFile(existingFiles, metadata);
if (existingFile) {
if (existingFile.collectionID === collection.id) {
const matchingExistingFiles = findMatchingExistingFiles(
existingFiles,
metadata
);
addLocalLog(
() =>
`matchedFileList: ${matchingExistingFiles
.map((f) => `${f.id}-${f.metadata.title}`)
.join(',')}`
);
if (matchingExistingFiles?.length) {
const matchingExistingFilesCollectionIDs =
matchingExistingFiles.map((e) => e.collectionID);
addLocalLog(
() =>
`matched file collectionIDs:${matchingExistingFilesCollectionIDs}
and collectionID:${collection.id}`
);
if (matchingExistingFilesCollectionIDs.includes(collection.id)) {
addLogLine(
`file already present in the collection , skipped upload for ${fileNameSize}`
);
return { fileUploadResult: UPLOAD_RESULT.ALREADY_UPLOADED };
} else {
addLogLine(
`same file in other collection found for ${fileNameSize}`
`same file in ${matchingExistingFilesCollectionIDs.length} collection found for ${fileNameSize}`
);
const resultFile = Object.assign({}, existingFile);
// any of the matching file can used to add a symlink
const resultFile = Object.assign({}, matchingExistingFiles[0]);
resultFile.collectionID = collection.id;
await addToCollection(collection, [resultFile]);
return {

7
src/types/upload/index.ts
View file

@ -144,7 +144,8 @@ export interface ParsedExtractedMetadata {
creationTime: number;
}
export interface AnalysisResult {
suggestedCollectionName: string;
multipleFolders: boolean;
// This is used to prompt the user the make upload strategy choice
export interface ImportSuggestion {
rootFolderName: string;
hasNestedFolders: boolean;
}

8
src/utils/collection/index.ts
View file

@ -218,3 +218,11 @@ export const shouldShowOptions = (type: CollectionSummaryType) => {
export const shouldBeShownOnCollectionBar = (type: CollectionSummaryType) => {
return !HIDE_FROM_COLLECTION_BAR_TYPES.has(type);
};
export const getUserOwnedCollections = (collections: Collection[]) => {
const user: User = getData(LS_KEYS.USER);
if (!user?.id) {
throw Error('user missing');
}
return collections.filter((collection) => collection.owner.id === user.id);
};

1
src/utils/error/index.ts
View file

@ -83,6 +83,7 @@ export function handleUploadError(error): Error {
case CustomError.SUBSCRIPTION_EXPIRED:
case CustomError.STORAGE_QUOTA_EXCEEDED:
case CustomError.SESSION_EXPIRED:
case CustomError.UPLOAD_CANCELLED:
throw parsedError;
}
return parsedError;

8
src/utils/export/index.ts
View file

@ -6,7 +6,7 @@ import { EnteFile } from 'types/file';
import { Metadata } from 'types/upload';
import { formatDate, splitFilenameAndExtension } from 'utils/file';
import { METADATA_FOLDER_NAME } from 'constants/export';
import { ENTE_METADATA_FOLDER } from 'constants/export';
export const getExportRecordFileUID = (file: EnteFile) =>
`${file.id}_${file.collectionID}_${file.updationTime}`;
@ -179,7 +179,7 @@ export const getUniqueCollectionFolderPath = (
};
export const getMetadataFolderPath = (collectionFolderPath: string) =>
`${collectionFolderPath}/${METADATA_FOLDER_NAME}`;
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}`;
export const getUniqueFileSaveName = (
collectionPath: string,
@ -211,7 +211,7 @@ export const getOldFileSaveName = (filename: string, fileID: number) =>
export const getFileMetadataSavePath = (
collectionFolderPath: string,
fileSaveName: string
) => `${collectionFolderPath}/${METADATA_FOLDER_NAME}/${fileSaveName}.json`;
) => `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${fileSaveName}.json`;
export const getFileSavePath = (
collectionFolderPath: string,
@ -235,6 +235,6 @@ export const getOldFileMetadataSavePath = (
collectionFolderPath: string,
file: EnteFile
) =>
`${collectionFolderPath}/${METADATA_FOLDER_NAME}/${
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${
file.id
}_${oldSanitizeName(file.metadata.title)}.json`;

8
src/utils/file/index.ts
View file

@ -534,3 +534,11 @@ export const createTypedObjectURL = async (blob: Blob, fileName: string) => {
const type = await getFileType(new File([blob], fileName));
return URL.createObjectURL(new Blob([blob], { type: type.mimeType }));
};
export const getUserOwnedNonTrashedFiles = (files: EnteFile[]) => {
const user: User = getData(LS_KEYS.USER);
if (!user?.id) {
throw Error('user missing');
}
return files.filter((file) => file.isTrashed || file.ownerID === user.id);
};

9
src/utils/logging/index.ts
View file

@ -4,12 +4,21 @@ import { formatDateTime } from 'utils/time';
import { saveLogLine, getLogs } from 'utils/storage';
export function addLogLine(log: string) {
if (!process.env.NEXT_PUBLIC_SENTRY_ENV) {
console.log(log);
}
saveLogLine({
timestamp: Date.now(),
logLine: log,
});
}
export const addLocalLog = (getLog: () => string) => {
if (!process.env.NEXT_PUBLIC_SENTRY_ENV) {
console.log(getLog());
}
};
export function getDebugLogs() {
return getLogs().map(
(log) => `[${formatDateTime(log.timestamp)}] ${log.logLine}`

4
src/utils/strings/englishConstants.tsx
View file

@ -110,8 +110,8 @@ const englishConstants = {
2: 'Reading file metadata',
3: (fileCounter) =>
`${fileCounter.finished} / ${fileCounter.total} files backed up`,
4: 'Backup complete',
5: 'Pausing remaining uploads',
4: 'Cancelling remaining uploads',
5: 'Backup complete',
},
UPLOADING_FILES: 'File upload',
FILE_NOT_UPLOADED_LIST: 'The following files were not uploaded',

86
src/utils/upload/index.ts
View file

@ -1,5 +1,5 @@
import {
AnalysisResult,
ImportSuggestion,
ElectronFile,
FileWithCollection,
Metadata,
@ -7,45 +7,27 @@ import {
import { EnteFile } from 'types/file';
import {
A_SEC_IN_MICROSECONDS,
NULL_ANALYSIS_RESULT,
UPLOAD_TYPE,
DEFAULT_IMPORT_SUGGESTION,
PICKED_UPLOAD_TYPE,
} from 'constants/upload';
import { FILE_TYPE } from 'constants/file';
import { METADATA_FOLDER_NAME } from 'constants/export';
import { ENTE_METADATA_FOLDER } from 'constants/export';
import isElectron from 'is-electron';
const TYPE_JSON = 'json';
const DEDUPE_COLLECTION = new Set(['icloud library', 'icloudlibrary']);
export function findMatchingExistingFile(
export function findMatchingExistingFiles(
existingFiles: EnteFile[],
newFileMetadata: Metadata
): EnteFile {
): EnteFile[] {
const matchingFiles: EnteFile[] = [];
for (const existingFile of existingFiles) {
if (areFilesSame(existingFile.metadata, newFileMetadata)) {
return existingFile;
matchingFiles.push(existingFile);
}
}
return null;
}
export function findSameFileInOtherCollection(
existingFiles: EnteFile[],
newFileMetadata: Metadata
) {
if (!hasFileHash(newFileMetadata)) {
return null;
}
for (const existingFile of existingFiles) {
if (
hasFileHash(existingFile.metadata) &&
areFilesWithFileHashSame(existingFile.metadata, newFileMetadata)
) {
return existingFile;
}
}
return null;
return matchingFiles;
}
export function shouldDedupeAcrossCollection(collectionName: string): boolean {
@ -128,12 +110,12 @@ export function areFileWithCollectionsSame(
return firstFile.localID === secondFile.localID;
}
export function analyseUploadFiles(
uploadType: UPLOAD_TYPE,
export function getImportSuggestion(
uploadType: PICKED_UPLOAD_TYPE,
toUploadFiles: File[] | ElectronFile[]
): AnalysisResult {
if (isElectron() && uploadType === UPLOAD_TYPE.FILES) {
return NULL_ANALYSIS_RESULT;
): ImportSuggestion {
if (isElectron() && uploadType === PICKED_UPLOAD_TYPE.FILES) {
return DEFAULT_IMPORT_SUGGESTION;
}
const paths: string[] = toUploadFiles.map((file) => file['path']);
@ -161,27 +143,49 @@ export function analyseUploadFiles(
}
}
return {
suggestedCollectionName: commonPathPrefix || null,
multipleFolders: firstFileFolder !== lastFileFolder,
rootFolderName: commonPathPrefix || null,
hasNestedFolders: firstFileFolder !== lastFileFolder,
};
}
export function getCollectionWiseFiles(toUploadFiles: File[] | ElectronFile[]) {
const collectionWiseFiles = new Map<string, (File | ElectronFile)[]>();
// This function groups files that are that have the same parent folder into collections
// For Example, for user files have a directory structure like this
// a
// / | \
// b j c
// /|\ / \
// e f g h i
//
// The files will grouped into 3 collections.
// [a => [j],
// b => [e,f,g],
// c => [h, i]]
export function groupFilesBasedOnParentFolder(
toUploadFiles: File[] | ElectronFile[]
) {
const collectionNameToFilesMap = new Map<string, (File | ElectronFile)[]>();
for (const file of toUploadFiles) {
const filePath = file['path'] as string;
let folderPath = filePath.substring(0, filePath.lastIndexOf('/'));
if (folderPath.endsWith(METADATA_FOLDER_NAME)) {
// If the parent folder of a file is "metadata"
// we consider it to be part of the parent folder
// For Eg,For FileList -> [a/x.png, a/metadata/x.png.json]
// they will both we grouped into the collection "a"
// This is cluster the metadata json files in the same collection as the file it is for
if (folderPath.endsWith(ENTE_METADATA_FOLDER)) {
folderPath = folderPath.substring(0, folderPath.lastIndexOf('/'));
}
const folderName = folderPath.substring(
folderPath.lastIndexOf('/') + 1
);
if (!collectionWiseFiles.has(folderName)) {
collectionWiseFiles.set(folderName, []);
if (!folderName?.length) {
throw Error("folderName can't be null");
}
collectionWiseFiles.get(folderName).push(file);
if (!collectionNameToFilesMap.has(folderName)) {
collectionNameToFilesMap.set(folderName, []);
}
return collectionWiseFiles;
collectionNameToFilesMap.get(folderName).push(file);
}
return collectionNameToFilesMap;
}