beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #693 from ente-io/refactor-upload

Browse source 
Refactor upload
This commit is contained in:
Abhinav Kumar 2022-09-05 15:44:00 +05:30 committed by GitHub
parent d2d40e747f 0a8f2f798b
commit 02afc0b9cb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 664 additions and 420 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

358
src/components/Upload/Uploader.tsx
View file

@ -14,12 +14,14 @@ import UploadManager from 'services/upload/uploadManager';
import uploadManager from 'services/upload/uploadManager';
import ImportService from 'services/importService';
import isElectron from 'is-electron';
import { METADATA_FOLDER_NAME } from 'constants/export';
import { CustomError } from 'utils/error';
import { Collection } from 'types/collection';
import { SetLoading, SetFiles } from 'types/gallery';
import { ElectronFile, FileWithCollection } from 'types/upload';
import Router from 'next/router';
import {
ImportSuggestion,
ElectronFile,
FileWithCollection,
} from 'types/upload';
import { isCanvasBlocked } from 'utils/upload/isCanvasBlocked';
import { downloadApp } from 'utils/common';
import DiscFullIcon from '@mui/icons-material/DiscFull';
@ -30,16 +32,27 @@ import {
SegregatedFinishedUploads,
InProgressUpload,
} from 'types/upload/ui';
import { UPLOAD_STAGES } from 'constants/upload';
import {
DEFAULT_IMPORT_SUGGESTION,
UPLOAD_STAGES,
UPLOAD_STRATEGY,
PICKED_UPLOAD_TYPE,
} from 'constants/upload';
import importService from 'services/importService';
import { getDownloadAppMessage } from 'utils/ui';
import UploadTypeSelector from './UploadTypeSelector';
import {
getImportSuggestion,
groupFilesBasedOnParentFolder,
} from 'utils/upload';
import { getUserOwnedCollections } from 'utils/collection';
const FIRST_ALBUM_NAME = 'My First Album';
interface Props {
syncWithRemote: (force?: boolean, silent?: boolean) => Promise<void>;
closeCollectionSelector: () => void;
closeUploadTypeSelector: () => void;
setCollectionSelectorAttributes: SetCollectionSelectorAttributes;
setCollectionNamerAttributes: SetCollectionNamerAttributes;
setLoading: SetLoading;
@ -48,38 +61,15 @@ interface Props {
showCollectionSelector: () => void;
setFiles: SetFiles;
isFirstUpload: boolean;
electronFiles: ElectronFile[];
setElectronFiles: (files: ElectronFile[]) => void;
webFiles: File[];
setWebFiles: (files: File[]) => void;
uploadTypeSelectorView: boolean;
setUploadTypeSelectorView: (open: boolean) => void;
showSessionExpiredMessage: () => void;
showUploadFilesDialog: () => void;
showUploadDirsDialog: () => void;
webFolderSelectorFiles: File[];
webFileSelectorFiles: File[];
dragAndDropFiles: File[];
}
enum UPLOAD_STRATEGY {
SINGLE_COLLECTION,
COLLECTION_PER_FOLDER,
}
export enum UPLOAD_TYPE {
FILES = 'files',
FOLDERS = 'folders',
ZIPS = 'zips',
}
interface AnalysisResult {
suggestedCollectionName: string;
multipleFolders: boolean;
}
const NULL_ANALYSIS_RESULT = {
suggestedCollectionName: '',
multipleFolders: false,
};
export default function Uploader(props: Props) {
const [uploadProgressView, setUploadProgressView] = useState(false);
const [uploadStage, setUploadStage] = useState<UPLOAD_STAGES>();
@ -97,21 +87,25 @@ export default function Uploader(props: Props) {
const [hasLivePhotos, setHasLivePhotos] = useState(false);
const [choiceModalView, setChoiceModalView] = useState(false);
const [analysisResult, setAnalysisResult] =
useState<AnalysisResult>(NULL_ANALYSIS_RESULT);
const [importSuggestion, setImportSuggestion] = useState<ImportSuggestion>(
DEFAULT_IMPORT_SUGGESTION
);
const appContext = useContext(AppContext);
const galleryContext = useContext(GalleryContext);
const toUploadFiles = useRef<File[] | ElectronFile[]>(null);
const isPendingDesktopUpload = useRef(false);
const pendingDesktopUploadCollectionName = useRef<string>('');
const uploadType = useRef<UPLOAD_TYPE>(null);
// This is set when the user choses a type to upload from the upload type selector dialog
const pickedUploadType = useRef<PICKED_UPLOAD_TYPE>(null);
const zipPaths = useRef<string[]>(null);
const [electronFiles, setElectronFiles] = useState<ElectronFile[]>(null);
const [webFiles, setWebFiles] = useState([]);
const closeUploadProgress = () => setUploadProgressView(false);
useEffect(() => {
UploadManager.initUploader(
UploadManager.init(
{
setPercentComplete,
setUploadCounter,
@ -133,16 +127,41 @@ export default function Uploader(props: Props) {
}
}, []);
// this handles the change of selectorFiles changes on web when user selects
// files for upload through the opened file/folder selector or dragAndDrop them
// the webFiles state is update which triggers the upload of those files
useEffect(() => {
if (
props.electronFiles?.length > 0 ||
props.webFiles?.length > 0 ||
pickedUploadType.current === PICKED_UPLOAD_TYPE.FOLDERS &&
props.webFolderSelectorFiles?.length > 0
) {
setWebFiles(props.webFolderSelectorFiles);
} else if (
pickedUploadType.current === PICKED_UPLOAD_TYPE.FILES &&
props.webFileSelectorFiles?.length > 0
) {
setWebFiles(props.webFileSelectorFiles);
} else if (props.dragAndDropFiles?.length > 0) {
setWebFiles(props.dragAndDropFiles);
}
}, [
props.dragAndDropFiles,
props.webFileSelectorFiles,
props.webFolderSelectorFiles,
]);
useEffect(() => {
if (
electronFiles?.length > 0 ||
webFiles?.length > 0 ||
appContext.sharedFiles?.length > 0
) {
if (props.uploadInProgress) {
// no-op
// a upload is already in progress
} else if (isCanvasBlocked()) {
return;
}
if (isCanvasBlocked()) {
appContext.setDialogMessage({
title: constants.CANVAS_BLOCKED_TITLE,
@ -154,119 +173,51 @@ export default function Uploader(props: Props) {
variant: 'accent',
},
});
} else {
props.setLoading(true);
if (props.webFiles?.length > 0) {
// File selection by drag and drop or selection of file.
toUploadFiles.current = props.webFiles;
props.setWebFiles([]);
} else if (appContext.sharedFiles?.length > 0) {
toUploadFiles.current = appContext.sharedFiles;
appContext.resetSharedFiles();
} else if (props.electronFiles?.length > 0) {
// File selection from desktop app
toUploadFiles.current = props.electronFiles;
props.setElectronFiles([]);
}
const analysisResult = analyseUploadFiles();
setAnalysisResult(analysisResult);
handleCollectionCreationAndUpload(
analysisResult,
props.isFirstUpload
);
props.setLoading(false);
return;
}
}
}, [props.webFiles, appContext.sharedFiles, props.electronFiles]);
props.setLoading(true);
if (webFiles?.length > 0) {
// File selection by drag and drop or selection of file.
toUploadFiles.current = webFiles;
setWebFiles([]);
} else if (appContext.sharedFiles?.length > 0) {
toUploadFiles.current = appContext.sharedFiles;
appContext.resetSharedFiles();
} else if (electronFiles?.length > 0) {
// File selection from desktop app
toUploadFiles.current = electronFiles;
setElectronFiles([]);
}
const importSuggestion = getImportSuggestion(
pickedUploadType.current,
toUploadFiles.current
);
setImportSuggestion(importSuggestion);
const uploadInit = function () {
setUploadStage(UPLOAD_STAGES.START);
setUploadCounter({ finished: 0, total: 0 });
setInProgressUploads([]);
setFinishedUploads(new Map());
setPercentComplete(0);
props.closeCollectionSelector();
setUploadProgressView(true);
};
handleCollectionCreationAndUpload(
importSuggestion,
props.isFirstUpload
);
props.setLoading(false);
}
}, [webFiles, appContext.sharedFiles, electronFiles]);
const resumeDesktopUpload = async (
type: UPLOAD_TYPE,
type: PICKED_UPLOAD_TYPE,
electronFiles: ElectronFile[],
collectionName: string
) => {
if (electronFiles && electronFiles?.length > 0) {
isPendingDesktopUpload.current = true;
pendingDesktopUploadCollectionName.current = collectionName;
uploadType.current = type;
props.setElectronFiles(electronFiles);
pickedUploadType.current = type;
setElectronFiles(electronFiles);
}
};
function analyseUploadFiles(): AnalysisResult {
if (isElectron() && uploadType.current === UPLOAD_TYPE.FILES) {
return NULL_ANALYSIS_RESULT;
}
const paths: string[] = toUploadFiles.current.map(
(file) => file['path']
);
const getCharCount = (str: string) => (str.match(/\//g) ?? []).length;
paths.sort((path1, path2) => getCharCount(path1) - getCharCount(path2));
const firstPath = paths[0];
const lastPath = paths[paths.length - 1];
const L = firstPath.length;
let i = 0;
const firstFileFolder = firstPath.substring(
0,
firstPath.lastIndexOf('/')
);
const lastFileFolder = lastPath.substring(0, lastPath.lastIndexOf('/'));
while (i < L && firstPath.charAt(i) === lastPath.charAt(i)) i++;
let commonPathPrefix = firstPath.substring(0, i);
if (commonPathPrefix) {
commonPathPrefix = commonPathPrefix.substring(
0,
commonPathPrefix.lastIndexOf('/')
);
if (commonPathPrefix) {
commonPathPrefix = commonPathPrefix.substring(
commonPathPrefix.lastIndexOf('/') + 1
);
}
}
return {
suggestedCollectionName: commonPathPrefix || null,
multipleFolders: firstFileFolder !== lastFileFolder,
};
}
function getCollectionWiseFiles() {
const collectionWiseFiles = new Map<string, (File | ElectronFile)[]>();
for (const file of toUploadFiles.current) {
const filePath = file['path'] as string;
let folderPath = filePath.substring(0, filePath.lastIndexOf('/'));
if (folderPath.endsWith(METADATA_FOLDER_NAME)) {
folderPath = folderPath.substring(
0,
folderPath.lastIndexOf('/')
);
}
const folderName = folderPath.substring(
folderPath.lastIndexOf('/') + 1
);
if (!collectionWiseFiles.has(folderName)) {
collectionWiseFiles.set(folderName, []);
}
collectionWiseFiles.get(folderName).push(file);
}
return collectionWiseFiles;
}
const uploadFilesToExistingCollection = async (collection: Collection) => {
try {
await preUploadAction();
const filesWithCollectionToUpload: FileWithCollection[] =
toUploadFiles.current.map((file, index) => ({
file,
@ -284,21 +235,32 @@ export default function Uploader(props: Props) {
collectionName?: string
) => {
try {
await preUploadAction();
const filesWithCollectionToUpload: FileWithCollection[] = [];
const collections: Collection[] = [];
let collectionWiseFiles = new Map<
let collectionNameToFilesMap = new Map<
string,
(File | ElectronFile)[]
>();
if (strategy === UPLOAD_STRATEGY.SINGLE_COLLECTION) {
collectionWiseFiles.set(collectionName, toUploadFiles.current);
collectionNameToFilesMap.set(
collectionName,
toUploadFiles.current
);
} else {
collectionWiseFiles = getCollectionWiseFiles();
collectionNameToFilesMap = groupFilesBasedOnParentFolder(
toUploadFiles.current
);
}
try {
const existingCollection = await syncCollections();
const existingCollection = getUserOwnedCollections(
await syncCollections()
);
let index = 0;
for (const [collectionName, files] of collectionWiseFiles) {
for (const [
collectionName,
files,
] of collectionNameToFilesMap) {
const collection = await createAlbum(
collectionName,
existingCollection
@ -330,58 +292,72 @@ export default function Uploader(props: Props) {
}
};
const preUploadAction = async () => {
props.closeCollectionSelector();
props.closeUploadTypeSelector();
uploadManager.prepareForNewUpload();
setUploadProgressView(true);
props.setUploadInProgress(true);
await props.syncWithRemote(true, true);
};
function postUploadAction() {
props.setUploadInProgress(false);
props.syncWithRemote();
}
const uploadFiles = async (
filesWithCollectionToUpload: FileWithCollection[],
filesWithCollectionToUploadIn: FileWithCollection[],
collections: Collection[]
) => {
try {
uploadInit();
props.setUploadInProgress(true);
props.closeCollectionSelector();
await props.syncWithRemote(true, true);
if (isElectron() && !isPendingDesktopUpload.current) {
await ImportService.setToUploadCollection(collections);
if (zipPaths.current) {
await ImportService.setToUploadFiles(
UPLOAD_TYPE.ZIPS,
PICKED_UPLOAD_TYPE.ZIPS,
zipPaths.current
);
zipPaths.current = null;
}
await ImportService.setToUploadFiles(
UPLOAD_TYPE.FILES,
filesWithCollectionToUpload.map(
PICKED_UPLOAD_TYPE.FILES,
filesWithCollectionToUploadIn.map(
({ file }) => (file as ElectronFile).path
)
);
}
await uploadManager.queueFilesForUpload(
filesWithCollectionToUpload,
collections
);
const shouldCloseUploadProgress =
await uploadManager.queueFilesForUpload(
filesWithCollectionToUploadIn,
collections
);
if (shouldCloseUploadProgress) {
closeUploadProgress();
}
} catch (err) {
showUserFacingError(err.message);
closeUploadProgress();
throw err;
} finally {
props.setUploadInProgress(false);
props.syncWithRemote();
postUploadAction();
}
};
const retryFailed = async () => {
try {
props.setUploadInProgress(true);
uploadInit();
await props.syncWithRemote(true, true);
await uploadManager.retryFailedFiles();
const filesWithCollections =
await uploadManager.getFailedFilesWithCollections();
await preUploadAction();
await uploadManager.queueFilesForUpload(
filesWithCollections.files,
filesWithCollections.collections
);
} catch (err) {
showUserFacingError(err.message);
closeUploadProgress();
} finally {
props.setUploadInProgress(false);
props.syncWithRemote();
postUploadAction();
}
};
@ -440,7 +416,7 @@ export default function Uploader(props: Props) {
};
const handleCollectionCreationAndUpload = (
analysisResult: AnalysisResult,
importSuggestion: ImportSuggestion,
isFirstUpload: boolean
) => {
if (isPendingDesktopUpload.current) {
@ -457,21 +433,22 @@ export default function Uploader(props: Props) {
}
return;
}
if (isElectron() && uploadType.current === UPLOAD_TYPE.ZIPS) {
if (
isElectron() &&
pickedUploadType.current === PICKED_UPLOAD_TYPE.ZIPS
) {
uploadFilesToNewCollections(UPLOAD_STRATEGY.COLLECTION_PER_FOLDER);
return;
}
if (isFirstUpload && !analysisResult.suggestedCollectionName) {
analysisResult.suggestedCollectionName = FIRST_ALBUM_NAME;
if (isFirstUpload && !importSuggestion.rootFolderName) {
importSuggestion.rootFolderName = FIRST_ALBUM_NAME;
}
let showNextModal = () => {};
if (analysisResult.multipleFolders) {
if (importSuggestion.hasNestedFolders) {
showNextModal = () => setChoiceModalView(true);
} else {
showNextModal = () =>
uploadToSingleNewCollection(
analysisResult.suggestedCollectionName
);
uploadToSingleNewCollection(importSuggestion.rootFolderName);
}
props.setCollectionSelectorAttributes({
callback: uploadFilesToExistingCollection,
@ -479,12 +456,12 @@ export default function Uploader(props: Props) {
title: constants.UPLOAD_TO_COLLECTION,
});
};
const handleDesktopUpload = async (type: UPLOAD_TYPE) => {
const handleDesktopUpload = async (type: PICKED_UPLOAD_TYPE) => {
let files: ElectronFile[];
uploadType.current = type;
if (type === UPLOAD_TYPE.FILES) {
pickedUploadType.current = type;
if (type === PICKED_UPLOAD_TYPE.FILES) {
files = await ImportService.showUploadFilesDialog();
} else if (type === UPLOAD_TYPE.FOLDERS) {
} else if (type === PICKED_UPLOAD_TYPE.FOLDERS) {
files = await ImportService.showUploadDirsDialog();
} else {
const response = await ImportService.showUploadZipDialog();
@ -492,29 +469,24 @@ export default function Uploader(props: Props) {
zipPaths.current = response.zipPaths;
}
if (files?.length > 0) {
props.setElectronFiles(files);
props.setUploadTypeSelectorView(false);
setElectronFiles(files);
props.closeUploadTypeSelector();
}
};
const handleWebUpload = async (type: UPLOAD_TYPE) => {
uploadType.current = type;
if (type === UPLOAD_TYPE.FILES) {
const handleWebUpload = async (type: PICKED_UPLOAD_TYPE) => {
pickedUploadType.current = type;
if (type === PICKED_UPLOAD_TYPE.FILES) {
props.showUploadFilesDialog();
} else if (type === UPLOAD_TYPE.FOLDERS) {
} else if (type === PICKED_UPLOAD_TYPE.FOLDERS) {
props.showUploadDirsDialog();
} else {
appContext.setDialogMessage(getDownloadAppMessage());
}
};
const cancelUploads = async () => {
closeUploadProgress();
if (isElectron()) {
ImportService.cancelRemainingUploads();
}
props.setUploadInProgress(false);
Router.reload();
const cancelUploads = () => {
uploadManager.cancelRunningUpload();
};
const handleUpload = (type) => () => {
@ -525,11 +497,9 @@ export default function Uploader(props: Props) {
}
};
const handleFileUpload = handleUpload(UPLOAD_TYPE.FILES);
const handleFolderUpload = handleUpload(UPLOAD_TYPE.FOLDERS);
const handleZipUpload = handleUpload(UPLOAD_TYPE.ZIPS);
const closeUploadTypeSelector = () =>
props.setUploadTypeSelectorView(false);
const handleFileUpload = handleUpload(PICKED_UPLOAD_TYPE.FILES);
const handleFolderUpload = handleUpload(PICKED_UPLOAD_TYPE.FOLDERS);
const handleZipUpload = handleUpload(PICKED_UPLOAD_TYPE.ZIPS);
return (
<>
@ -537,9 +507,7 @@ export default function Uploader(props: Props) {
open={choiceModalView}
onClose={() => setChoiceModalView(false)}
uploadToSingleCollection={() =>
uploadToSingleNewCollection(
analysisResult.suggestedCollectionName
)
uploadToSingleNewCollection(importSuggestion.rootFolderName)
}
uploadToMultipleCollection={() =>
uploadFilesToNewCollections(
@ -549,7 +517,7 @@ export default function Uploader(props: Props) {
/>
<UploadTypeSelector
show={props.uploadTypeSelectorView}
onHide={closeUploadTypeSelector}
onHide={props.closeUploadTypeSelector}
uploadFiles={handleFileUpload}
uploadFolders={handleFolderUpload}
uploadGoogleTakeoutZips={handleZipUpload}

2
src/constants/export/index.ts
View file

@ -1,4 +1,4 @@
export const METADATA_FOLDER_NAME = 'metadata';
export const ENTE_METADATA_FOLDER = 'metadata';
export enum ExportNotification {
START = 'export started',

25
src/constants/upload/index.ts
View file

@ -1,6 +1,10 @@
import { ENCRYPTION_CHUNK_SIZE } from 'constants/crypto';
import { FILE_TYPE } from 'constants/file';
import { Location, ParsedExtractedMetadata } from 'types/upload';
import {
ImportSuggestion,
Location,
ParsedExtractedMetadata,
} from 'types/upload';
// list of format that were missed by type-detection for some files.
export const FORMAT_MISSED_BY_FILE_TYPE_LIB = [
@ -28,6 +32,7 @@ export enum UPLOAD_STAGES {
READING_GOOGLE_METADATA_FILES,
EXTRACTING_METADATA,
UPLOADING,
CANCELLING,
FINISH,
}
@ -40,6 +45,19 @@ export enum UPLOAD_RESULT {
LARGER_THAN_AVAILABLE_STORAGE,
UPLOADED,
UPLOADED_WITH_STATIC_THUMBNAIL,
ADDED_SYMLINK,
CANCELLED,
}
export enum UPLOAD_STRATEGY {
SINGLE_COLLECTION,
COLLECTION_PER_FOLDER,
}
export enum PICKED_UPLOAD_TYPE {
FILES = 'files',
FOLDERS = 'folders',
ZIPS = 'zips',
}
export const MAX_FILE_SIZE_SUPPORTED = 4 * 1024 * 1024 * 1024; // 4 GB
@ -55,6 +73,11 @@ export const A_SEC_IN_MICROSECONDS = 1e6;
export const USE_CF_PROXY = false;
export const DEFAULT_IMPORT_SUGGESTION: ImportSuggestion = {
rootFolderName: '',
hasNestedFolders: false,
};
export const BLACK_THUMBNAIL_BASE64 =
'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEB' +
'AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQ' +

35
src/pages/gallery/index.tsx
View file

@ -90,7 +90,6 @@ import { EnteFile } from 'types/file';
import { GalleryContextType, SelectedState } from 'types/gallery';
import { VISIBILITY_STATE } from 'types/magicMetadata';
import Notification from 'components/Notification';
import { ElectronFile } from 'types/upload';
import Collections from 'components/Collections';
import { GalleryNavbar } from 'components/pages/gallery/Navbar';
import { Search, SearchResultSummary, UpdateSearch } from 'types/search';
@ -161,14 +160,14 @@ export default function Gallery() {
disabled: uploadInProgress,
});
const {
selectedFiles: fileSelectorFiles,
selectedFiles: webFileSelectorFiles,
open: openFileSelector,
getInputProps: getFileSelectorInputProps,
} = useFileInput({
directory: false,
});
const {
selectedFiles: folderSelectorFiles,
selectedFiles: webFolderSelectorFiles,
open: openFolderSelector,
getInputProps: getFolderSelectorInputProps,
} = useFileInput({
@ -202,8 +201,6 @@ export default function Gallery() {
const showPlanSelectorModal = () => setPlanModalView(true);
const [electronFiles, setElectronFiles] = useState<ElectronFile[]>(null);
const [webFiles, setWebFiles] = useState([]);
const [uploadTypeSelectorView, setUploadTypeSelectorView] = useState(false);
const [sidebarView, setSidebarView] = useState(false);
@ -285,16 +282,6 @@ export default function Gallery() {
[notificationAttributes]
);
useEffect(() => {
if (dragAndDropFiles?.length > 0) {
setWebFiles(dragAndDropFiles);
} else if (folderSelectorFiles?.length > 0) {
setWebFiles(folderSelectorFiles);
} else if (fileSelectorFiles?.length > 0) {
setWebFiles(fileSelectorFiles);
}
}, [dragAndDropFiles, fileSelectorFiles, folderSelectorFiles]);
useEffect(() => {
if (typeof activeCollection === 'undefined') {
return;
@ -575,7 +562,11 @@ export default function Gallery() {
setSetSearchResultSummary(null);
};
const openUploader = () => setUploadTypeSelectorView(true);
const openUploader = () => {
if (!uploadInProgress) {
setUploadTypeSelectorView(true);
}
};
return (
<GalleryContext.Provider
@ -663,6 +654,10 @@ export default function Gallery() {
null,
true
)}
closeUploadTypeSelector={setUploadTypeSelectorView.bind(
null,
false
)}
setCollectionSelectorAttributes={
setCollectionSelectorAttributes
}
@ -676,12 +671,10 @@ export default function Gallery() {
setUploadInProgress={setUploadInProgress}
setFiles={setFiles}
isFirstUpload={hasNonEmptyCollections(collectionSummaries)}
electronFiles={electronFiles}
setElectronFiles={setElectronFiles}
webFiles={webFiles}
setWebFiles={setWebFiles}
webFileSelectorFiles={webFileSelectorFiles}
webFolderSelectorFiles={webFolderSelectorFiles}
dragAndDropFiles={dragAndDropFiles}
uploadTypeSelectorView={uploadTypeSelectorView}
setUploadTypeSelectorView={setUploadTypeSelectorView}
showUploadFilesDialog={openFileSelector}
showUploadDirsDialog={openFolderSelector}
showSessionExpiredMessage={showSessionExpiredMessage}

12
src/services/importService.ts
View file

@ -1,4 +1,4 @@
import { UPLOAD_TYPE } from 'components/Upload/Uploader';
import { PICKED_UPLOAD_TYPE } from 'constants/upload';
import { Collection } from 'types/collection';
import { ElectronAPIs } from 'types/electron';
import { ElectronFile, FileWithCollection } from 'types/upload';
@ -8,7 +8,7 @@ import { logError } from 'utils/sentry';
interface PendingUploads {
files: ElectronFile[];
collectionName: string;
type: UPLOAD_TYPE;
type: PICKED_UPLOAD_TYPE;
}
interface selectZipResult {
@ -88,7 +88,7 @@ class ImportService {
}
async setToUploadFiles(
type: UPLOAD_TYPE.FILES | UPLOAD_TYPE.ZIPS,
type: PICKED_UPLOAD_TYPE.FILES | PICKED_UPLOAD_TYPE.ZIPS,
filePaths: string[]
) {
if (this.allElectronAPIsExist) {
@ -117,14 +117,14 @@ class ImportService {
);
}
}
this.setToUploadFiles(UPLOAD_TYPE.FILES, filePaths);
this.setToUploadFiles(PICKED_UPLOAD_TYPE.FILES, filePaths);
}
}
cancelRemainingUploads() {
if (this.allElectronAPIsExist) {
this.ElectronAPIs.setToUploadCollection(null);
this.ElectronAPIs.setToUploadFiles(UPLOAD_TYPE.ZIPS, []);
this.ElectronAPIs.setToUploadFiles(UPLOAD_TYPE.FILES, []);
this.ElectronAPIs.setToUploadFiles(PICKED_UPLOAD_TYPE.ZIPS, []);
this.ElectronAPIs.setToUploadFiles(PICKED_UPLOAD_TYPE.FILES, []);
}
}
}

4
src/services/upload/multiPartUploadService.ts
View file

@ -8,6 +8,7 @@ import UploadHttpClient from './uploadHttpClient';
import * as convert from 'xml-js';
import { CustomError } from 'utils/error';
import { DataStream, MultipartUploadURLs } from 'types/upload';
import uploadCancelService from './uploadCancelService';
interface PartEtag {
PartNumber: number;
@ -51,6 +52,9 @@ export async function uploadStreamInParts(
index,
fileUploadURL,
] of multipartUploadURLs.partURLs.entries()) {
if (uploadCancelService.isUploadCancelationRequested()) {
throw Error(CustomError.UPLOAD_CANCELLED);
}
const uploadChunk = await combineChunksToFormUploadPart(streamReader);
const progressTracker = UIService.trackUploadProgress(
fileLocalID,

90
src/services/upload/uiService.ts
View file

@ -10,7 +10,11 @@ import {
ProgressUpdater,
SegregatedFinishedUploads,
} from 'types/upload/ui';
import { Canceler } from 'axios';
import { CustomError } from 'utils/error';
import uploadCancelService from './uploadCancelService';
const REQUEST_TIMEOUT_TIME = 30 * 1000; // 30 sec;
class UIService {
private perFileProgress: number;
private filesUploaded: number;
@ -23,7 +27,7 @@ class UIService {
this.progressUpdater = progressUpdater;
}
reset(count: number) {
reset(count = 0) {
this.setTotalFileCount(count);
this.filesUploaded = 0;
this.inProgressUploads = new Map<number, number>();
@ -33,7 +37,11 @@ class UIService {
setTotalFileCount(count: number) {
this.totalFileCount = count;
this.perFileProgress = 100 / this.totalFileCount;
if (count > 0) {
this.perFileProgress = 100 / this.totalFileCount;
} else {
this.perFileProgress = 0;
}
}
setFileProgress(key: number, progress: number) {
@ -68,14 +76,26 @@ class UIService {
this.updateProgressBarUI();
}
updateProgressBarUI() {
hasFilesInResultList() {
const finishedUploadsList = segregatedFinishedUploadsToList(
this.finishedUploads
);
for (const x of finishedUploadsList.values()) {
if (x.length > 0) {
return true;
}
}
return false;
}
private updateProgressBarUI() {
const {
setPercentComplete,
setUploadCounter: setFileCounter,
setUploadCounter,
setInProgressUploads,
setFinishedUploads,
} = this.progressUpdater;
setFileCounter({
setUploadCounter({
finished: this.filesUploaded,
total: this.totalFileCount,
});
@ -95,10 +115,10 @@ class UIService {
setPercentComplete(percentComplete);
setInProgressUploads(
this.convertInProgressUploadsToList(this.inProgressUploads)
convertInProgressUploadsToList(this.inProgressUploads)
);
setFinishedUploads(
this.segregatedFinishedUploadsToList(this.finishedUploads)
segregatedFinishedUploadsToList(this.finishedUploads)
);
}
@ -107,13 +127,19 @@ class UIService {
percentPerPart = RANDOM_PERCENTAGE_PROGRESS_FOR_PUT(),
index = 0
) {
const cancel = { exec: null };
const cancel: { exec: Canceler } = { exec: () => {} };
const cancelTimedOutRequest = () =>
cancel.exec(CustomError.REQUEST_TIMEOUT);
const cancelCancelledUploadRequest = () =>
cancel.exec(CustomError.UPLOAD_CANCELLED);
let timeout = null;
const resetTimeout = () => {
if (timeout) {
clearTimeout(timeout);
}
timeout = setTimeout(() => cancel.exec(), 30 * 1000);
timeout = setTimeout(cancelTimedOutRequest, REQUEST_TIMEOUT_TIME);
};
return {
cancel,
@ -134,31 +160,33 @@ class UIService {
} else {
resetTimeout();
}
if (uploadCancelService.isUploadCancelationRequested()) {
cancelCancelledUploadRequest();
}
},
};
}
convertInProgressUploadsToList(inProgressUploads) {
return [...inProgressUploads.entries()].map(
([localFileID, progress]) =>
({
localFileID,
progress,
} as InProgressUpload)
);
}
segregatedFinishedUploadsToList(finishedUploads: FinishedUploads) {
const segregatedFinishedUploads =
new Map() as SegregatedFinishedUploads;
for (const [localID, result] of finishedUploads) {
if (!segregatedFinishedUploads.has(result)) {
segregatedFinishedUploads.set(result, []);
}
segregatedFinishedUploads.get(result).push(localID);
}
return segregatedFinishedUploads;
}
}
export default new UIService();
function convertInProgressUploadsToList(inProgressUploads) {
return [...inProgressUploads.entries()].map(
([localFileID, progress]) =>
({
localFileID,
progress,
} as InProgressUpload)
);
}
function segregatedFinishedUploadsToList(finishedUploads: FinishedUploads) {
const segregatedFinishedUploads = new Map() as SegregatedFinishedUploads;
for (const [localID, result] of finishedUploads) {
if (!segregatedFinishedUploads.has(result)) {
segregatedFinishedUploads.set(result, []);
}
segregatedFinishedUploads.get(result).push(localID);
}
return segregatedFinishedUploads;
}

23
src/services/upload/uploadCancelService.ts Normal file
View file

@ -0,0 +1,23 @@
interface UploadCancelStatus {
value: boolean;
}
class UploadCancelService {
private shouldUploadBeCancelled: UploadCancelStatus = {
value: false,
};
reset() {
this.shouldUploadBeCancelled.value = false;
}
requestUploadCancelation() {
this.shouldUploadBeCancelled.value = true;
}
isUploadCancelationRequested(): boolean {
return this.shouldUploadBeCancelled.value;
}
}
export default new UploadCancelService();

36
src/services/upload/uploadHttpClient.ts
View file

@ -92,18 +92,22 @@ class UploadHttpClient {
progressTracker
): Promise<string> {
try {
await retryHTTPCall(() =>
HTTPService.put(
fileUploadURL.url,
file,
null,
null,
progressTracker
)
await retryHTTPCall(
() =>
HTTPService.put(
fileUploadURL.url,
file,
null,
null,
progressTracker
),
handleUploadError
);
return fileUploadURL.objectKey;
} catch (e) {
logError(e, 'putFile to dataStore failed ');
if (e.message !== CustomError.UPLOAD_CANCELLED) {
logError(e, 'putFile to dataStore failed ');
}
throw e;
}
}
@ -127,7 +131,9 @@ class UploadHttpClient {
);
return fileUploadURL.objectKey;
} catch (e) {
logError(e, 'putFile to dataStore failed ');
if (e.message !== CustomError.UPLOAD_CANCELLED) {
logError(e, 'putFile to dataStore failed ');
}
throw e;
}
}
@ -152,10 +158,12 @@ class UploadHttpClient {
throw err;
}
return resp;
});
}, handleUploadError);
return response.headers.etag as string;
} catch (e) {
logError(e, 'put filePart failed');
if (e.message !== CustomError.UPLOAD_CANCELLED) {
logError(e, 'put filePart failed');
}
throw e;
}
}
@ -185,7 +193,9 @@ class UploadHttpClient {
});
return response.data.etag as string;
} catch (e) {
logError(e, 'put filePart failed');
if (e.message !== CustomError.UPLOAD_CANCELLED) {
logError(e, 'put filePart failed');
}
throw e;
}
}

230
src/services/upload/uploadManager.ts
View file

@ -1,11 +1,11 @@
import { getLocalFiles, setLocalFiles } from '../fileService';
import { getLocalFiles } from '../fileService';
import { SetFiles } from 'types/gallery';
import { getDedicatedCryptoWorker } from 'utils/crypto';
import {
groupFilesBasedOnCollectionID,
sortFiles,
preservePhotoswipeProps,
decryptFile,
getUserOwnedNonTrashedFiles,
} from 'utils/file';
import { logError } from 'utils/sentry';
import { getMetadataJSONMapKey, parseMetadataJSON } from './metadataService';
@ -40,6 +40,7 @@ import { addLogLine, getFileNameSize } from 'utils/logging';
import isElectron from 'is-electron';
import ImportService from 'services/importService';
import { ProgressUpdater } from 'types/upload/ui';
import uploadCancelService from './uploadCancelService';
const MAX_CONCURRENT_UPLOADS = 4;
const FILE_UPLOAD_COMPLETED = 100;
@ -51,11 +52,15 @@ class UploadManager {
private filesToBeUploaded: FileWithCollection[];
private remainingFiles: FileWithCollection[] = [];
private failedFiles: FileWithCollection[];
private existingFilesCollectionWise: Map<number, EnteFile[]>;
private existingFiles: EnteFile[];
private userOwnedNonTrashedExistingFiles: EnteFile[];
private setFiles: SetFiles;
private collections: Map<number, Collection>;
public initUploader(progressUpdater: ProgressUpdater, setFiles: SetFiles) {
private uploadInProgress: boolean;
public async init(progressUpdater: ProgressUpdater, setFiles: SetFiles) {
UIService.init(progressUpdater);
this.setFiles = setFiles;
UIService.init(progressUpdater);
this.setFiles = setFiles;
}
@ -71,10 +76,16 @@ class UploadManager {
>();
}
private async init(collections: Collection[]) {
prepareForNewUpload() {
this.resetState();
UIService.reset();
uploadCancelService.reset();
UIService.setUploadStage(UPLOAD_STAGES.START);
}
async updateExistingFilesAndCollections(collections: Collection[]) {
this.existingFiles = await getLocalFiles();
this.existingFilesCollectionWise = groupFilesBasedOnCollectionID(
this.userOwnedNonTrashedExistingFiles = getUserOwnedNonTrashedFiles(
this.existingFiles
);
this.collections = new Map(
@ -83,16 +94,20 @@ class UploadManager {
}
public async queueFilesForUpload(
fileWithCollectionToBeUploaded: FileWithCollection[],
filesWithCollectionToUploadIn: FileWithCollection[],
collections: Collection[]
) {
try {
await this.init(collections);
if (this.uploadInProgress) {
throw Error("can't run multiple uploads at once");
}
this.uploadInProgress = true;
await this.updateExistingFilesAndCollections(collections);
addLogLine(
`received ${fileWithCollectionToBeUploaded.length} files to upload`
`received ${filesWithCollectionToUploadIn.length} files to upload`
);
const { metadataJSONFiles, mediaFiles } =
segregateMetadataAndMediaFiles(fileWithCollectionToBeUploaded);
segregateMetadataAndMediaFiles(filesWithCollectionToUploadIn);
addLogLine(`has ${metadataJSONFiles.length} metadata json files`);
addLogLine(`has ${mediaFiles.length} media files`);
if (metadataJSONFiles.length) {
@ -111,7 +126,6 @@ class UploadManager {
this.metadataAndFileTypeInfoMap
);
UIService.setUploadStage(UPLOAD_STAGES.START);
addLogLine(`clusterLivePhotoFiles called`);
// filter out files whose metadata detection failed or those that have been skipped because the files are too large,
@ -161,19 +175,36 @@ class UploadManager {
await this.uploadMediaFiles(allFiles);
}
} catch (e) {
if (e.message === CustomError.UPLOAD_CANCELLED) {
if (isElectron()) {
ImportService.cancelRemainingUploads();
}
} else {
logError(e, 'uploading failed with error');
addLogLine(
`uploading failed with error -> ${e.message}
${(e as Error).stack}`
);
throw e;
}
} finally {
UIService.setUploadStage(UPLOAD_STAGES.FINISH);
UIService.setPercentComplete(FILE_UPLOAD_COMPLETED);
} catch (e) {
logError(e, 'uploading failed with error');
addLogLine(
`uploading failed with error -> ${e.message}
${(e as Error).stack}`
);
throw e;
} finally {
for (let i = 0; i < MAX_CONCURRENT_UPLOADS; i++) {
this.cryptoWorkers[i]?.worker.terminate();
}
this.uploadInProgress = false;
}
try {
if (!UIService.hasFilesInResultList()) {
return true;
} else {
return false;
}
} catch (e) {
logError(e, ' failed to return shouldCloseProgressBar');
return false;
}
}
@ -185,6 +216,9 @@ class UploadManager {
for (const { file, collectionID } of metadataFiles) {
try {
if (uploadCancelService.isUploadCancelationRequested()) {
throw Error(CustomError.UPLOAD_CANCELLED);
}
addLogLine(
`parsing metadata json file ${getFileNameSize(file)}`
);
@ -207,7 +241,12 @@ class UploadManager {
)}`
);
} catch (e) {
logError(e, 'parsing failed for a file');
if (e.message === CustomError.UPLOAD_CANCELLED) {
throw e;
} else {
// and don't break for subsequent files just log and move on
logError(e, 'parsing failed for a file');
}
addLogLine(
`failed to parse metadata json file ${getFileNameSize(
file
@ -216,8 +255,10 @@ class UploadManager {
}
}
} catch (e) {
logError(e, 'error seeding MetadataMap');
// silently ignore the error
if (e.message !== CustomError.UPLOAD_CANCELLED) {
logError(e, 'error seeding MetadataMap');
}
throw e;
}
}
@ -226,6 +267,9 @@ class UploadManager {
addLogLine(`extractMetadataFromFiles executed`);
UIService.reset(mediaFiles.length);
for (const { file, localID, collectionID } of mediaFiles) {
if (uploadCancelService.isUploadCancelationRequested()) {
throw Error(CustomError.UPLOAD_CANCELLED);
}
let fileTypeInfo = null;
let metadata = null;
try {
@ -244,7 +288,12 @@ class UploadManager {
)} `
);
} catch (e) {
logError(e, 'extractFileTypeAndMetadata failed');
if (e.message === CustomError.UPLOAD_CANCELLED) {
throw e;
} else {
// and don't break for subsequent files just log and move on
logError(e, 'extractFileTypeAndMetadata failed');
}
addLogLine(
`metadata extraction failed ${getFileNameSize(
file
@ -258,7 +307,9 @@ class UploadManager {
UIService.increaseFileUploaded();
}
} catch (e) {
logError(e, 'error extracting metadata');
if (e.message !== CustomError.UPLOAD_CANCELLED) {
logError(e, 'error extracting metadata');
}
throw e;
}
}
@ -334,25 +385,25 @@ class UploadManager {
private async uploadNextFileInQueue(worker: any) {
while (this.filesToBeUploaded.length > 0) {
if (uploadCancelService.isUploadCancelationRequested()) {
throw Error(CustomError.UPLOAD_CANCELLED);
}
let fileWithCollection = this.filesToBeUploaded.pop();
const { collectionID } = fileWithCollection;
const existingFilesInCollection =
this.existingFilesCollectionWise.get(collectionID) ?? [];
const collection = this.collections.get(collectionID);
fileWithCollection = { ...fileWithCollection, collection };
const { fileUploadResult, uploadedFile, skipDecryption } =
await uploader(
worker,
existingFilesInCollection,
this.existingFiles,
fileWithCollection
);
const { fileUploadResult, uploadedFile } = await uploader(
worker,
this.userOwnedNonTrashedExistingFiles,
fileWithCollection
);
const finalUploadResult = await this.postUploadTask(
fileUploadResult,
uploadedFile,
skipDecryption,
fileWithCollection
);
UIService.moveFileToResultList(
fileWithCollection.localID,
finalUploadResult
@ -363,54 +414,46 @@ class UploadManager {
async postUploadTask(
fileUploadResult: UPLOAD_RESULT,
uploadedFile: EnteFile,
skipDecryption: boolean,
uploadedFile: EnteFile | null,
fileWithCollection: FileWithCollection
) {
try {
let decryptedFile: EnteFile;
addLogLine(`uploadedFile ${JSON.stringify(uploadedFile)}`);
if (
(fileUploadResult === UPLOAD_RESULT.UPLOADED ||
fileUploadResult ===
UPLOAD_RESULT.UPLOADED_WITH_STATIC_THUMBNAIL) &&
!skipDecryption
) {
const decryptedFile = await decryptFile(
uploadedFile,
fileWithCollection.collection.key
);
this.existingFiles.push(decryptedFile);
this.existingFiles = sortFiles(this.existingFiles);
await setLocalFiles(this.existingFiles);
this.setFiles(preservePhotoswipeProps(this.existingFiles));
if (
!this.existingFilesCollectionWise.has(
decryptedFile.collectionID
)
) {
this.existingFilesCollectionWise.set(
decryptedFile.collectionID,
[]
this.updateElectronRemainingFiles(fileWithCollection);
switch (fileUploadResult) {
case UPLOAD_RESULT.FAILED:
case UPLOAD_RESULT.BLOCKED:
this.failedFiles.push(fileWithCollection);
break;
case UPLOAD_RESULT.ADDED_SYMLINK:
decryptedFile = uploadedFile;
fileUploadResult = UPLOAD_RESULT.UPLOADED;
break;
case UPLOAD_RESULT.UPLOADED:
case UPLOAD_RESULT.UPLOADED_WITH_STATIC_THUMBNAIL:
decryptedFile = await decryptFile(
uploadedFile,
fileWithCollection.collection.key
);
}
this.existingFilesCollectionWise
.get(decryptedFile.collectionID)
.push(decryptedFile);
break;
case UPLOAD_RESULT.ALREADY_UPLOADED:
case UPLOAD_RESULT.UNSUPPORTED:
case UPLOAD_RESULT.TOO_LARGE:
case UPLOAD_RESULT.CANCELLED:
// no-op
break;
default:
throw Error('Invalid Upload Result' + fileUploadResult);
}
if (
fileUploadResult === UPLOAD_RESULT.FAILED ||
fileUploadResult === UPLOAD_RESULT.BLOCKED
[
UPLOAD_RESULT.ADDED_SYMLINK,
UPLOAD_RESULT.UPLOADED,
UPLOAD_RESULT.UPLOADED_WITH_STATIC_THUMBNAIL,
].includes(fileUploadResult)
) {
this.failedFiles.push(fileWithCollection);
}
if (isElectron()) {
this.remainingFiles = this.remainingFiles.filter(
(file) =>
!areFileWithCollectionsSame(file, fileWithCollection)
);
ImportService.updatePendingUploads(this.remainingFiles);
await this.updateExistingFiles(decryptedFile);
}
return fileUploadResult;
} catch (e) {
@ -423,10 +466,41 @@ class UploadManager {
}
}
async retryFailedFiles() {
await this.queueFilesForUpload(this.failedFiles, [
...this.collections.values(),
]);
public cancelRunningUpload() {
UIService.setUploadStage(UPLOAD_STAGES.CANCELLING);
uploadCancelService.requestUploadCancelation();
}
async getFailedFilesWithCollections() {
return {
files: this.failedFiles,
collections: [...this.collections.values()],
};
}
private async updateExistingFiles(decryptedFile: EnteFile) {
if (!decryptedFile) {
throw Error("decrypted file can't be undefined");
}
this.userOwnedNonTrashedExistingFiles.push(decryptedFile);
await this.updateUIFiles(decryptedFile);
}
private async updateUIFiles(decryptedFile: EnteFile) {
this.existingFiles.push(decryptedFile);
this.existingFiles = sortFiles(this.existingFiles);
this.setFiles(preservePhotoswipeProps(this.existingFiles));
}
private updateElectronRemainingFiles(
fileWithCollection: FileWithCollection
) {
if (isElectron()) {
this.remainingFiles = this.remainingFiles.filter(
(file) => !areFileWithCollectionsSame(file, fileWithCollection)
);
ImportService.updatePendingUploads(this.remainingFiles);
}
}
}

6
src/services/upload/uploadService.ts
View file

@ -3,7 +3,6 @@ import { logError } from 'utils/sentry';
import UploadHttpClient from './uploadHttpClient';
import { extractFileMetadata, getFilename } from './fileService';
import { getFileType } from '../typeDetectionService';
import { handleUploadError } from 'utils/error';
import {
B64EncryptionResult,
BackupedFile,
@ -33,6 +32,7 @@ import { encryptFile, getFileSize, readFile } from './fileService';
import { uploadStreamUsingMultipart } from './multiPartUploadService';
import UIService from './uiService';
import { USE_CF_PROXY } from 'constants/upload';
import { CustomError, handleUploadError } from 'utils/error';
class UploadService {
private uploadURLs: UploadURL[] = [];
@ -185,7 +185,9 @@ class UploadService {
};
return backupedFile;
} catch (e) {
logError(e, 'error uploading to bucket');
if (e.message !== CustomError.UPLOAD_CANCELLED) {
logError(e, 'error uploading to bucket');
}
throw e;
}
}

92
src/services/upload/uploader.ts
View file

@ -1,30 +1,26 @@
import { EnteFile } from 'types/file';
import { handleUploadError, CustomError } from 'utils/error';
import { logError } from 'utils/sentry';
import {
fileAlreadyInCollection,
findSameFileInOtherCollection,
shouldDedupeAcrossCollection,
} from 'utils/upload';
import { findMatchingExistingFiles } from 'utils/upload';
import UploadHttpClient from './uploadHttpClient';
import UIService from './uiService';
import UploadService from './uploadService';
import { FILE_TYPE } from 'constants/file';
import { UPLOAD_RESULT, MAX_FILE_SIZE_SUPPORTED } from 'constants/upload';
import { FileWithCollection, BackupedFile, UploadFile } from 'types/upload';
import { addLogLine } from 'utils/logging';
import { addLocalLog, addLogLine } from 'utils/logging';
import { convertBytesToHumanReadable } from 'utils/file/size';
import { sleep } from 'utils/common';
import { addToCollection } from 'services/collectionService';
import uploadCancelService from './uploadCancelService';
interface UploadResponse {
fileUploadResult: UPLOAD_RESULT;
uploadedFile?: EnteFile;
skipDecryption?: boolean;
}
export default async function uploader(
worker: any,
existingFilesInCollection: EnteFile[],
existingFiles: EnteFile[],
fileWithCollection: FileWithCollection
): Promise<UploadResponse> {
@ -50,40 +46,47 @@ export default async function uploader(
throw Error(CustomError.NO_METADATA);
}
if (fileAlreadyInCollection(existingFilesInCollection, metadata)) {
addLogLine(`skipped upload for ${fileNameSize}`);
return { fileUploadResult: UPLOAD_RESULT.ALREADY_UPLOADED };
}
const sameFileInOtherCollection = findSameFileInOtherCollection(
const matchingExistingFiles = findMatchingExistingFiles(
existingFiles,
metadata
);
if (sameFileInOtherCollection) {
addLogLine(
`same file in other collection found for ${fileNameSize}`
addLocalLog(
() =>
`matchedFileList: ${matchingExistingFiles
.map((f) => `${f.id}-${f.metadata.title}`)
.join(',')}`
);
if (matchingExistingFiles?.length) {
const matchingExistingFilesCollectionIDs =
matchingExistingFiles.map((e) => e.collectionID);
addLocalLog(
() =>
`matched file collectionIDs:${matchingExistingFilesCollectionIDs}
and collectionID:${collection.id}`
);
const resultFile = Object.assign({}, sameFileInOtherCollection);
resultFile.collectionID = collection.id;
await addToCollection(collection, [resultFile]);
return {
fileUploadResult: UPLOAD_RESULT.UPLOADED,
uploadedFile: resultFile,
skipDecryption: true,
};
if (matchingExistingFilesCollectionIDs.includes(collection.id)) {
addLogLine(
`file already present in the collection , skipped upload for ${fileNameSize}`
);
return { fileUploadResult: UPLOAD_RESULT.ALREADY_UPLOADED };
} else {
addLogLine(
`same file in ${matchingExistingFilesCollectionIDs.length} collection found for ${fileNameSize}`
);
// any of the matching file can used to add a symlink
const resultFile = Object.assign({}, matchingExistingFiles[0]);
resultFile.collectionID = collection.id;
await addToCollection(collection, [resultFile]);
return {
fileUploadResult: UPLOAD_RESULT.ADDED_SYMLINK,
uploadedFile: resultFile,
};
}
}
if (uploadCancelService.isUploadCancelationRequested()) {
throw Error(CustomError.UPLOAD_CANCELLED);
}
// iOS exports via album doesn't export files without collection and if user exports all photos, album info is not preserved.
// This change allow users to export by albums, upload to ente. And export all photos -> upload files which are not already uploaded
// as part of the albums
if (
shouldDedupeAcrossCollection(fileWithCollection.collection.name) &&
fileAlreadyInCollection(existingFiles, metadata)
) {
addLogLine(`deduped upload for ${fileNameSize}`);
return { fileUploadResult: UPLOAD_RESULT.ALREADY_UPLOADED };
}
addLogLine(`reading asset ${fileNameSize}`);
const file = await UploadService.readAsset(fileTypeInfo, uploadAsset);
@ -97,6 +100,9 @@ export default async function uploader(
thumbnail: file.thumbnail,
metadata,
};
if (uploadCancelService.isUploadCancelationRequested()) {
throw Error(CustomError.UPLOAD_CANCELLED);
}
addLogLine(`encryptAsset ${fileNameSize}`);
const encryptedFile = await UploadService.encryptAsset(
@ -105,6 +111,9 @@ export default async function uploader(
collection.key
);
if (uploadCancelService.isUploadCancelationRequested()) {
throw Error(CustomError.UPLOAD_CANCELLED);
}
addLogLine(`uploadToBucket ${fileNameSize}`);
const backupedFile: BackupedFile = await UploadService.uploadToBucket(
@ -131,12 +140,15 @@ export default async function uploader(
};
} catch (e) {
addLogLine(`upload failed for ${fileNameSize} ,error: ${e.message}`);
logError(e, 'file upload failed', {
fileFormat: fileTypeInfo?.exactType,
});
if (e.message !== CustomError.UPLOAD_CANCELLED) {
logError(e, 'file upload failed', {
fileFormat: fileTypeInfo?.exactType,
});
}
const error = handleUploadError(e);
switch (error.message) {
case CustomError.UPLOAD_CANCELLED:
return { fileUploadResult: UPLOAD_RESULT.CANCELLED };
case CustomError.ETAG_MISSING:
return { fileUploadResult: UPLOAD_RESULT.BLOCKED };
case CustomError.UNSUPPORTED_FILE_FORMAT:

6
src/types/upload/index.ts
View file

@ -142,3 +142,9 @@ export interface ParsedExtractedMetadata {
location: Location;
creationTime: number;
}
// This is used to prompt the user the make upload strategy choice
export interface ImportSuggestion {
rootFolderName: string;
hasNestedFolders: boolean;
}

8
src/utils/collection/index.ts
View file

@ -218,3 +218,11 @@ export const shouldShowOptions = (type: CollectionSummaryType) => {
export const shouldBeShownOnCollectionBar = (type: CollectionSummaryType) => {
return !HIDE_FROM_COLLECTION_BAR_TYPES.has(type);
};
export const getUserOwnedCollections = (collections: Collection[]) => {
const user: User = getData(LS_KEYS.USER);
if (!user?.id) {
throw Error('user missing');
}
return collections.filter((collection) => collection.owner.id === user.id);
};

3
src/utils/error/index.ts
View file

@ -45,6 +45,8 @@ export enum CustomError {
FILE_ID_NOT_FOUND = 'file with id not found',
WEAK_DEVICE = 'password decryption failed on the device',
INCORRECT_PASSWORD = 'incorrect password',
UPLOAD_CANCELLED = 'upload cancelled',
REQUEST_TIMEOUT = 'request taking too long',
}
function parseUploadErrorCodes(error) {
@ -81,6 +83,7 @@ export function handleUploadError(error): Error {
case CustomError.SUBSCRIPTION_EXPIRED:
case CustomError.STORAGE_QUOTA_EXCEEDED:
case CustomError.SESSION_EXPIRED:
case CustomError.UPLOAD_CANCELLED:
throw parsedError;
}
return parsedError;

8
src/utils/export/index.ts
View file

@ -6,7 +6,7 @@ import { EnteFile } from 'types/file';
import { Metadata } from 'types/upload';
import { formatDate, splitFilenameAndExtension } from 'utils/file';
import { METADATA_FOLDER_NAME } from 'constants/export';
import { ENTE_METADATA_FOLDER } from 'constants/export';
export const getExportRecordFileUID = (file: EnteFile) =>
`${file.id}_${file.collectionID}_${file.updationTime}`;
@ -179,7 +179,7 @@ export const getUniqueCollectionFolderPath = (
};
export const getMetadataFolderPath = (collectionFolderPath: string) =>
`${collectionFolderPath}/${METADATA_FOLDER_NAME}`;
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}`;
export const getUniqueFileSaveName = (
collectionPath: string,
@ -211,7 +211,7 @@ export const getOldFileSaveName = (filename: string, fileID: number) =>
export const getFileMetadataSavePath = (
collectionFolderPath: string,
fileSaveName: string
) => `${collectionFolderPath}/${METADATA_FOLDER_NAME}/${fileSaveName}.json`;
) => `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${fileSaveName}.json`;
export const getFileSavePath = (
collectionFolderPath: string,
@ -235,6 +235,6 @@ export const getOldFileMetadataSavePath = (
collectionFolderPath: string,
file: EnteFile
) =>
`${collectionFolderPath}/${METADATA_FOLDER_NAME}/${
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${
file.id
}_${oldSanitizeName(file.metadata.title)}.json`;

8
src/utils/file/index.ts
View file

@ -508,3 +508,11 @@ export const createTypedObjectURL = async (blob: Blob, fileName: string) => {
const type = await getFileType(new File([blob], fileName));
return URL.createObjectURL(new Blob([blob], { type: type.mimeType }));
};
export const getUserOwnedNonTrashedFiles = (files: EnteFile[]) => {
const user: User = getData(LS_KEYS.USER);
if (!user?.id) {
throw Error('user missing');
}
return files.filter((file) => file.isTrashed || file.ownerID === user.id);
};

9
src/utils/logging/index.ts
View file

@ -4,12 +4,21 @@ import { formatDateTime } from 'utils/time';
import { saveLogLine, getLogs } from 'utils/storage';
export function addLogLine(log: string) {
if (!process.env.NEXT_PUBLIC_SENTRY_ENV) {
console.log(log);
}
saveLogLine({
timestamp: Date.now(),
logLine: log,
});
}
export const addLocalLog = (getLog: () => string) => {
if (!process.env.NEXT_PUBLIC_SENTRY_ENV) {
console.log(getLog());
}
};
export function getDebugLogs() {
return getLogs().map(
(log) => `[${formatDateTime(log.timestamp)}] ${log.logLine}`

4
src/utils/strings/englishConstants.tsx
View file

@ -110,8 +110,8 @@ const englishConstants = {
2: 'Reading file metadata',
3: (fileCounter) =>
`${fileCounter.finished} / ${fileCounter.total} files backed up`,
4: 'Backup complete',
5: 'Cancelling remaining uploads',
4: 'Cancelling remaining uploads',
5: 'Backup complete',
},
UPLOADING_FILES: 'File upload',
FILE_NOT_UPLOADED_LIST: 'The following files were not uploaded',

125
src/utils/upload/index.ts
View file

@ -1,40 +1,33 @@
import { FileWithCollection, Metadata } from 'types/upload';
import {
ImportSuggestion,
ElectronFile,
FileWithCollection,
Metadata,
} from 'types/upload';
import { EnteFile } from 'types/file';
import { A_SEC_IN_MICROSECONDS } from 'constants/upload';
import {
A_SEC_IN_MICROSECONDS,
DEFAULT_IMPORT_SUGGESTION,
PICKED_UPLOAD_TYPE,
} from 'constants/upload';
import { FILE_TYPE } from 'constants/file';
import { ENTE_METADATA_FOLDER } from 'constants/export';
import isElectron from 'is-electron';
const TYPE_JSON = 'json';
const DEDUPE_COLLECTION = new Set(['icloud library', 'icloudlibrary']);
export function fileAlreadyInCollection(
existingFilesInCollection: EnteFile[],
newFileMetadata: Metadata
): boolean {
for (const existingFile of existingFilesInCollection) {
if (areFilesSame(existingFile.metadata, newFileMetadata)) {
return true;
}
}
return false;
}
export function findSameFileInOtherCollection(
export function findMatchingExistingFiles(
existingFiles: EnteFile[],
newFileMetadata: Metadata
) {
if (!hasFileHash(newFileMetadata)) {
return null;
}
): EnteFile[] {
const matchingFiles: EnteFile[] = [];
for (const existingFile of existingFiles) {
if (
hasFileHash(existingFile.metadata) &&
areFilesWithFileHashSame(existingFile.metadata, newFileMetadata)
) {
return existingFile;
if (areFilesSame(existingFile.metadata, newFileMetadata)) {
matchingFiles.push(existingFile);
}
}
return null;
return matchingFiles;
}
export function shouldDedupeAcrossCollection(collectionName: string): boolean {
@ -120,3 +113,83 @@ export function areFileWithCollectionsSame(
): boolean {
return firstFile.localID === secondFile.localID;
}
export function getImportSuggestion(
uploadType: PICKED_UPLOAD_TYPE,
toUploadFiles: File[] | ElectronFile[]
): ImportSuggestion {
if (isElectron() && uploadType === PICKED_UPLOAD_TYPE.FILES) {
return DEFAULT_IMPORT_SUGGESTION;
}
const paths: string[] = toUploadFiles.map((file) => file['path']);
const getCharCount = (str: string) => (str.match(/\//g) ?? []).length;
paths.sort((path1, path2) => getCharCount(path1) - getCharCount(path2));
const firstPath = paths[0];
const lastPath = paths[paths.length - 1];
const L = firstPath.length;
let i = 0;
const firstFileFolder = firstPath.substring(0, firstPath.lastIndexOf('/'));
const lastFileFolder = lastPath.substring(0, lastPath.lastIndexOf('/'));
while (i < L && firstPath.charAt(i) === lastPath.charAt(i)) i++;
let commonPathPrefix = firstPath.substring(0, i);
if (commonPathPrefix) {
commonPathPrefix = commonPathPrefix.substring(
0,
commonPathPrefix.lastIndexOf('/')
);
if (commonPathPrefix) {
commonPathPrefix = commonPathPrefix.substring(
commonPathPrefix.lastIndexOf('/') + 1
);
}
}
return {
rootFolderName: commonPathPrefix || null,
hasNestedFolders: firstFileFolder !== lastFileFolder,
};
}
// This function groups files that are that have the same parent folder into collections
// For Example, for user files have a directory structure like this
// a
// / | \
// b j c
// /|\ / \
// e f g h i
//
// The files will grouped into 3 collections.
// [a => [j],
// b => [e,f,g],
// c => [h, i]]
export function groupFilesBasedOnParentFolder(
toUploadFiles: File[] | ElectronFile[]
) {
const collectionNameToFilesMap = new Map<string, (File | ElectronFile)[]>();
for (const file of toUploadFiles) {
const filePath = file['path'] as string;
let folderPath = filePath.substring(0, filePath.lastIndexOf('/'));
// If the parent folder of a file is "metadata"
// we consider it to be part of the parent folder
// For Eg,For FileList -> [a/x.png, a/metadata/x.png.json]
// they will both we grouped into the collection "a"
// This is cluster the metadata json files in the same collection as the file it is for
if (folderPath.endsWith(ENTE_METADATA_FOLDER)) {
folderPath = folderPath.substring(0, folderPath.lastIndexOf('/'));
}
const folderName = folderPath.substring(
folderPath.lastIndexOf('/') + 1
);
if (!folderName?.length) {
throw Error("folderName can't be null");
}
if (!collectionNameToFilesMap.has(folderName)) {
collectionNameToFilesMap.set(folderName, []);
}
collectionNameToFilesMap.get(folderName).push(file);
}
return collectionNameToFilesMap;
}