beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'refactor-upload' into watch

Browse source
This commit is contained in:
Abhinav 2022-08-31 12:45:28 +05:30
parent cfcfbb876f 96cffbe70f
commit 4dd8b3b49d
11 changed files with 239 additions and 250 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

155
src/components/Upload/Uploader.tsx
View file

@ -14,17 +14,10 @@ import UploadManager from 'services/upload/uploadManager';
import uploadManager from 'services/upload/uploadManager';
import ImportService from 'services/importService';
import isElectron from 'is-electron';
import { METADATA_FOLDER_NAME } from 'constants/export';
import { CustomError } from 'utils/error';
import { Collection } from 'types/collection';
import { SetLoading, SetFiles } from 'types/gallery';
import {
AnalysisResult,
ElectronFile,
FileWithCollection,
NULL_ANALYSIS_RESULT,
UPLOAD_TYPE,
} from 'types/upload';
import { AnalysisResult, ElectronFile, FileWithCollection } from 'types/upload';
import { isCanvasBlocked } from 'utils/upload/isCanvasBlocked';
import { downloadApp, waitAndRun } from 'utils/common';
import watchFolderService from 'services/watchFolder/watchFolderService';
@ -36,17 +29,23 @@ import {
SegregatedFinishedUploads,
InProgressUpload,
} from 'types/upload/ui';
import { UPLOAD_STAGES, UPLOAD_STRATEGY } from 'constants/upload';
import {
NULL_ANALYSIS_RESULT,
UPLOAD_STAGES,
UPLOAD_STRATEGY,
UPLOAD_TYPE,
} from 'constants/upload';
import importService from 'services/importService';
import { getDownloadAppMessage } from 'utils/ui';
import UploadTypeSelector from './UploadTypeSelector';
import { analyseUploadFiles } from 'utils/upload/fs';
import { analyseUploadFiles, getCollectionWiseFiles } from 'utils/upload';
const FIRST_ALBUM_NAME = 'My First Album';
interface Props {
syncWithRemote: (force?: boolean, silent?: boolean) => Promise<void>;
closeCollectionSelector: () => void;
closeUploadTypeSelector: () => void;
setCollectionSelectorAttributes: SetCollectionSelectorAttributes;
setCollectionNamerAttributes: SetCollectionNamerAttributes;
setLoading: SetLoading;
@ -55,15 +54,13 @@ interface Props {
showCollectionSelector: () => void;
setFiles: SetFiles;
isFirstUpload: boolean;
electronFiles: ElectronFile[];
setElectronFiles: (files: ElectronFile[]) => void;
webFiles: File[];
setWebFiles: (files: File[]) => void;
uploadTypeSelectorView: boolean;
setUploadTypeSelectorView: (open: boolean) => void;
showSessionExpiredMessage: () => void;
showUploadFilesDialog: () => void;
showUploadDirsDialog: () => void;
folderSelectorFiles: File[];
fileSelectorFiles: File[];
dragAndDropFiles: File[];
}
export default function Uploader(props: Props) {
@ -94,6 +91,8 @@ export default function Uploader(props: Props) {
const uploadType = useRef<UPLOAD_TYPE>(null);
const zipPaths = useRef<string[]>(null);
const previousUploadPromise = useRef<Promise<void>>(null);
const [electronFiles, setElectronFiles] = useState<ElectronFile[]>(null);
const [webFiles, setWebFiles] = useState([]);
const closeUploadProgress = () => setUploadProgressView(false);
@ -123,7 +122,7 @@ export default function Uploader(props: Props) {
}
);
watchFolderService.init(
props.setElectronFiles,
setElectronFiles,
setCollectionName,
props.syncWithRemote,
appContext.setIsFolderSyncRunning
@ -132,9 +131,34 @@ export default function Uploader(props: Props) {
}, []);
useEffect(() => {
if (appContext.watchFolderView) {
// if watch folder dialog is open don't catch the dropped file
// as they are folder being dropped for watching
return;
}
if (
props.electronFiles?.length > 0 ||
props.webFiles?.length > 0 ||
uploadType.current === UPLOAD_TYPE.FOLDERS &&
props.folderSelectorFiles?.length > 0
) {
setWebFiles(props.folderSelectorFiles);
} else if (
uploadType.current === UPLOAD_TYPE.FILES &&
props.fileSelectorFiles?.length > 0
) {
setWebFiles(props.fileSelectorFiles);
} else if (props.dragAndDropFiles?.length > 0) {
setWebFiles(props.dragAndDropFiles);
}
}, [
props.dragAndDropFiles,
props.fileSelectorFiles,
props.folderSelectorFiles,
]);
useEffect(() => {
if (
electronFiles?.length > 0 ||
webFiles?.length > 0 ||
appContext.sharedFiles?.length > 0
) {
if (props.uploadInProgress) {
@ -162,21 +186,21 @@ export default function Uploader(props: Props) {
return;
}
props.setLoading(true);
if (props.webFiles?.length > 0) {
if (webFiles?.length > 0) {
// File selection by drag and drop or selection of file.
toUploadFiles.current = props.webFiles;
props.setWebFiles([]);
toUploadFiles.current = webFiles;
setWebFiles([]);
} else if (appContext.sharedFiles?.length > 0) {
toUploadFiles.current = appContext.sharedFiles;
appContext.resetSharedFiles();
} else if (props.electronFiles?.length > 0) {
} else if (electronFiles?.length > 0) {
// File selection from desktop app
toUploadFiles.current = props.electronFiles;
props.setElectronFiles([]);
toUploadFiles.current = electronFiles;
setElectronFiles([]);
}
const analysisResult = analyseUploadFiles(
toUploadFiles.current,
uploadType.current
uploadType.current,
toUploadFiles.current
);
setAnalysisResult(analysisResult);
@ -186,19 +210,7 @@ export default function Uploader(props: Props) {
);
props.setLoading(false);
}
}, [props.webFiles, appContext.sharedFiles, props.electronFiles]);
const preUploadAction = async () => {
setUploadStage(UPLOAD_STAGES.START);
setUploadCounter({ finished: 0, total: 0 });
setInProgressUploads([]);
setFinishedUploads(new Map());
setPercentComplete(0);
props.closeCollectionSelector();
setUploadProgressView(true);
props.setUploadInProgress(true);
await props.syncWithRemote(true, true);
};
}, [webFiles, appContext.sharedFiles, electronFiles]);
const resumeDesktopUpload = async (
type: UPLOAD_TYPE,
@ -209,36 +221,13 @@ export default function Uploader(props: Props) {
isPendingDesktopUpload.current = true;
pendingDesktopUploadCollectionName.current = collectionName;
uploadType.current = type;
props.setElectronFiles(electronFiles);
setElectronFiles(electronFiles);
}
};
function getCollectionWiseFiles() {
const collectionWiseFiles = new Map<string, (File | ElectronFile)[]>();
for (const file of toUploadFiles.current) {
const filePath = file['path'] as string;
let folderPath = filePath.substring(0, filePath.lastIndexOf('/'));
if (folderPath.endsWith(METADATA_FOLDER_NAME)) {
folderPath = folderPath.substring(
0,
folderPath.lastIndexOf('/')
);
}
const folderName = folderPath.substring(
folderPath.lastIndexOf('/') + 1
);
if (!collectionWiseFiles.has(folderName)) {
collectionWiseFiles.set(folderName, []);
}
collectionWiseFiles.get(folderName).push(file);
}
return collectionWiseFiles;
}
const uploadFilesToExistingCollection = async (collection: Collection) => {
try {
props.closeCollectionSelector();
await preUploadAction();
const filesWithCollectionToUpload: FileWithCollection[] =
toUploadFiles.current.map((file, index) => ({
file,
@ -258,7 +247,7 @@ export default function Uploader(props: Props) {
collectionName?: string
) => {
try {
props.closeCollectionSelector();
await preUploadAction();
const filesWithCollectionToUpload: FileWithCollection[] = [];
const collections: Collection[] = [];
let collectionWiseFiles = new Map<
@ -268,7 +257,9 @@ export default function Uploader(props: Props) {
if (strategy === UPLOAD_STRATEGY.SINGLE_COLLECTION) {
collectionWiseFiles.set(collectionName, toUploadFiles.current);
} else {
collectionWiseFiles = getCollectionWiseFiles();
collectionWiseFiles = getCollectionWiseFiles(
toUploadFiles.current
);
}
try {
const existingCollection = await syncCollections();
@ -318,12 +309,25 @@ export default function Uploader(props: Props) {
);
};
const preUploadAction = async () => {
props.closeCollectionSelector();
props.closeUploadTypeSelector();
uploadManager.prepareForNewUpload();
setUploadProgressView(true);
props.setUploadInProgress(true);
await props.syncWithRemote(true, true);
};
function postUploadAction() {
props.setUploadInProgress(false);
props.syncWithRemote();
}
const uploadFiles = async (
filesWithCollectionToUploadIn: FileWithCollection[],
collections: Collection[]
) => {
try {
preUploadAction();
if (isElectron() && !isPendingDesktopUpload.current) {
await ImportService.setToUploadCollection(collections);
if (zipPaths.current) {
@ -360,22 +364,19 @@ export default function Uploader(props: Props) {
closeUploadProgress();
throw err;
} finally {
props.setUploadInProgress(false);
props.syncWithRemote();
postUploadAction();
}
};
const retryFailed = async () => {
try {
preUploadAction();
await preUploadAction();
await uploadManager.retryFailedFiles();
} catch (err) {
showUserFacingError(err.message);
closeUploadProgress();
} finally {
props.setUploadInProgress(false);
props.syncWithRemote();
postUploadAction();
}
};
@ -486,8 +487,8 @@ export default function Uploader(props: Props) {
zipPaths.current = response.zipPaths;
}
if (files?.length > 0) {
props.setElectronFiles(files);
props.setUploadTypeSelectorView(false);
setElectronFiles(files);
props.closeUploadTypeSelector();
}
};
@ -517,8 +518,6 @@ export default function Uploader(props: Props) {
const handleFileUpload = handleUpload(UPLOAD_TYPE.FILES);
const handleFolderUpload = handleUpload(UPLOAD_TYPE.FOLDERS);
const handleZipUpload = handleUpload(UPLOAD_TYPE.ZIPS);
const closeUploadTypeSelector = () =>
props.setUploadTypeSelectorView(false);
return (
<>
@ -538,7 +537,7 @@ export default function Uploader(props: Props) {
/>
<UploadTypeSelector
show={props.uploadTypeSelectorView}
onHide={closeUploadTypeSelector}
onHide={props.closeUploadTypeSelector}
uploadFiles={handleFileUpload}
uploadFolders={handleFolderUpload}
uploadGoogleTakeoutZips={handleZipUpload}

12
src/components/WatchFolder/index.tsx
View file

@ -8,9 +8,9 @@ import constants from 'utils/strings/constants';
import DialogTitleWithCloseButton from 'components/DialogBox/TitleWithCloseButton';
import UploadStrategyChoiceModal from 'components/Upload/UploadStrategyChoiceModal';
import { UPLOAD_STRATEGY } from 'constants/upload';
import { analyseUploadFiles } from 'utils/upload/fs';
import { analyseUploadFiles } from 'utils/upload';
import electronFSService from 'services/electron/fs';
import { UPLOAD_TYPE } from 'types/upload';
import { UPLOAD_TYPE } from 'constants/upload';
interface Iprops {
open: boolean;
@ -44,8 +44,8 @@ export default function WatchFolder({ open, onClose }: Iprops) {
setInputFolderPath(path);
const files = await electronFSService.getDirFiles(path);
const analysisResult = analyseUploadFiles(
files,
UPLOAD_TYPE.FOLDERS
UPLOAD_TYPE.FOLDERS,
files
);
if (analysisResult.multipleFolders) {
setChoiceModalOpen(true);
@ -69,8 +69,8 @@ export default function WatchFolder({ open, onClose }: Iprops) {
setInputFolderPath(folderPath);
const files = await electronFSService.getDirFiles(folderPath);
const analysisResult = analyseUploadFiles(
files,
UPLOAD_TYPE.FOLDERS
UPLOAD_TYPE.FOLDERS,
files
);
if (analysisResult.multipleFolders) {
setChoiceModalOpen(true);

11
src/constants/upload/index.ts
View file

@ -50,6 +50,12 @@ export enum UPLOAD_RESULT {
CANCELLED,
}
export enum UPLOAD_TYPE {
FILES = 'files',
FOLDERS = 'folders',
ZIPS = 'zips',
}
export const MAX_FILE_SIZE_SUPPORTED = 4 * 1024 * 1024 * 1024; // 4 GB
export const LIVE_PHOTO_ASSET_SIZE_LIMIT = 20 * 1024 * 1024; // 20MB
@ -63,6 +69,11 @@ export const A_SEC_IN_MICROSECONDS = 1e6;
export const USE_CF_PROXY = false;
export const NULL_ANALYSIS_RESULT = {
suggestedCollectionName: '',
multipleFolders: false,
};
export const BLACK_THUMBNAIL_BASE64 =
'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEB' +
'AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQ' +

33
src/pages/gallery/index.tsx
View file

@ -90,7 +90,6 @@ import { EnteFile } from 'types/file';
import { GalleryContextType, SelectedState } from 'types/gallery';
import { VISIBILITY_STATE } from 'types/magicMetadata';
import Notification from 'components/Notification';
import { ElectronFile } from 'types/upload';
import Collections from 'components/Collections';
import { GalleryNavbar } from 'components/pages/gallery/Navbar';
import { Search, SearchResultSummary, UpdateSearch } from 'types/search';
@ -203,8 +202,6 @@ export default function Gallery() {
const showPlanSelectorModal = () => setPlanModalView(true);
const [electronFiles, setElectronFiles] = useState<ElectronFile[]>(null);
const [webFiles, setWebFiles] = useState([]);
const [uploadTypeSelectorView, setUploadTypeSelectorView] = useState(false);
const [sidebarView, setSidebarView] = useState(false);
@ -286,18 +283,6 @@ export default function Gallery() {
[notificationAttributes]
);
useEffect(() => {
if (!appContext.watchFolderView) {
if (dragAndDropFiles?.length > 0) {
setWebFiles(dragAndDropFiles);
} else if (folderSelectorFiles?.length > 0) {
setWebFiles(folderSelectorFiles);
} else if (fileSelectorFiles?.length > 0) {
setWebFiles(fileSelectorFiles);
}
}
}, [dragAndDropFiles, fileSelectorFiles, folderSelectorFiles]);
useEffect(() => {
if (typeof activeCollection === 'undefined') {
return;
@ -578,7 +563,11 @@ export default function Gallery() {
setSetSearchResultSummary(null);
};
const openUploader = () => setUploadTypeSelectorView(true);
const openUploader = () => {
if (!uploadInProgress) {
setUploadTypeSelectorView(true);
}
};
return (
<GalleryContext.Provider
@ -666,6 +655,10 @@ export default function Gallery() {
null,
true
)}
closeUploadTypeSelector={setUploadTypeSelectorView.bind(
null,
false
)}
setCollectionSelectorAttributes={
setCollectionSelectorAttributes
}
@ -679,12 +672,10 @@ export default function Gallery() {
setUploadInProgress={setUploadInProgress}
setFiles={setFiles}
isFirstUpload={hasNonEmptyCollections(collectionSummaries)}
electronFiles={electronFiles}
setElectronFiles={setElectronFiles}
webFiles={webFiles}
setWebFiles={setWebFiles}
fileSelectorFiles={fileSelectorFiles}
folderSelectorFiles={folderSelectorFiles}
dragAndDropFiles={dragAndDropFiles}
uploadTypeSelectorView={uploadTypeSelectorView}
setUploadTypeSelectorView={setUploadTypeSelectorView}
showUploadFilesDialog={openFileSelector}
showUploadDirsDialog={openFolderSelector}
showSessionExpiredMessage={showSessionExpiredMessage}

3
src/services/importService.ts
View file

@ -1,6 +1,7 @@
import { UPLOAD_TYPE } from 'constants/upload';
import { Collection } from 'types/collection';
import { ElectronAPIs } from 'types/electron';
import { ElectronFile, FileWithCollection, UPLOAD_TYPE } from 'types/upload';
import { ElectronFile, FileWithCollection } from 'types/upload';
import { runningInBrowser } from 'utils/common';
import { logError } from 'utils/sentry';

4
src/services/upload/uiService.ts
View file

@ -78,11 +78,11 @@ class UIService {
updateProgressBarUI() {
const {
setPercentComplete,
setUploadCounter: setFileCounter,
setUploadCounter,
setInProgressUploads,
setFinishedUploads,
} = this.progressUpdater;
setFileCounter({
setUploadCounter({
finished: this.filesUploaded,
total: this.totalFileCount,
});

67
src/services/upload/uploadManager.ts
View file

@ -58,16 +58,15 @@ class UploadManager {
private filesToBeUploaded: FileWithCollection[];
private remainingFiles: FileWithCollection[] = [];
private failedFiles: FileWithCollection[];
private existingFilesCollectionWise: Map<number, EnteFile[]>;
private collectionToExistingFilesMap: Map<number, EnteFile[]>;
private existingFiles: EnteFile[];
private setFiles: SetFiles;
private collections: Map<number, Collection>;
private uploadInProgress: boolean;
async init(progressUpdater: ProgressUpdater, setFiles: SetFiles) {
this.existingFiles = await getLocalFiles();
this.existingFilesCollectionWise = groupFilesBasedOnCollectionID(
this.existingFiles
);
public async init(progressUpdater: ProgressUpdater, setFiles: SetFiles) {
UIService.init(progressUpdater);
this.setFiles = setFiles;
UIService.init(progressUpdater);
this.setFiles = setFiles;
}
@ -83,10 +82,17 @@ class UploadManager {
>();
}
private async prepareForNewUpload(collections: Collection[]) {
prepareForNewUpload() {
this.resetState();
UIService.reset();
uploadCancelService.reset();
UIService.setUploadStage(UPLOAD_STAGES.START);
}
async updateExistingFilesAndCollections(collections: Collection[]) {
this.existingFiles = await getLocalFiles();
this.collectionToExistingFilesMap = groupFilesBasedOnCollectionID(
this.existingFiles
);
this.collections = new Map(
collections.map((collection) => [collection.id, collection])
);
@ -97,7 +103,11 @@ class UploadManager {
collections: Collection[]
) {
try {
await this.prepareForNewUpload(collections);
if (this.uploadInProgress) {
throw Error("can't run multiple uploads at once");
}
this.uploadInProgress = true;
await this.updateExistingFilesAndCollections(collections);
addLogLine(
`received ${filesWithCollectionToUploadIn.length} files to upload`
);
@ -123,7 +133,6 @@ class UploadManager {
this.metadataAndFileTypeInfoMap
);
UIService.setUploadStage(UPLOAD_STAGES.START);
addLogLine(`clusterLivePhotoFiles called`);
// filter out files whose metadata detection failed or those that have been skipped because the files are too large,
@ -192,6 +201,7 @@ class UploadManager {
for (let i = 0; i < MAX_CONCURRENT_UPLOADS; i++) {
this.cryptoWorkers[i]?.worker.terminate();
}
this.uploadInProgress = false;
}
}
@ -383,13 +393,13 @@ class UploadManager {
}
let fileWithCollection = this.filesToBeUploaded.pop();
const { collectionID } = fileWithCollection;
const existingFilesInCollection =
this.existingFilesCollectionWise.get(collectionID) ?? [];
const collectionExistingFiles =
this.collectionToExistingFilesMap.get(collectionID) ?? [];
const collection = this.collections.get(collectionID);
fileWithCollection = { ...fileWithCollection, collection };
const { fileUploadResult, uploadedFile } = await uploader(
worker,
existingFilesInCollection,
collectionExistingFiles,
this.existingFiles,
fileWithCollection
);
@ -436,6 +446,7 @@ class UploadManager {
break;
case UPLOAD_RESULT.ADDED_SYMLINK:
decryptedFile = uploadedFile;
fileUploadResult = UPLOAD_RESULT.UPLOADED;
break;
case UPLOAD_RESULT.UPLOADED:
case UPLOAD_RESULT.UPLOADED_WITH_STATIC_THUMBNAIL:
@ -448,15 +459,10 @@ class UploadManager {
// no-op
break;
default:
throw Error('Invalid Upload Result');
throw Error('Invalid Upload Result' + fileUploadResult);
}
if (decryptedFile) {
await this.updateExistingFiles(decryptedFile);
this.updateExistingCollections(decryptedFile);
await this.watchFolderCallback(
fileWithCollection,
uploadedFile
);
await this.updateFilePaths(decryptedFile, fileWithCollection);
}
return fileUploadResult;
@ -487,20 +493,29 @@ class UploadManager {
uploadCancelService.requestUploadCancelation();
}
private updateExistingCollections(decryptedFile: EnteFile) {
if (!this.existingFilesCollectionWise.has(decryptedFile.collectionID)) {
this.existingFilesCollectionWise.set(
async retryFailedFiles() {
await this.queueFilesForUpload(this.failedFiles, [
...this.collections.values(),
]);
}
private updateExistingFileToCollectionMap(decryptedFile: EnteFile) {
if (
!this.collectionToExistingFilesMap.has(decryptedFile.collectionID)
) {
this.collectionToExistingFilesMap.set(
decryptedFile.collectionID,
[]
);
}
this.existingFilesCollectionWise
this.collectionToExistingFilesMap
.get(decryptedFile.collectionID)
.push(decryptedFile);
}
private async updateExistingFiles(decryptedFile: EnteFile) {
this.existingFiles.push(decryptedFile);
this.updateExistingFileToCollectionMap(decryptedFile);
this.existingFiles = sortFiles(this.existingFiles);
await setLocalFiles(this.existingFiles);
this.setFiles(preservePhotoswipeProps(this.existingFiles));
@ -528,12 +543,6 @@ class UploadManager {
const updatedFile = await appendNewFilePath(decryptedFile, filePath);
await updateFileMagicMetadata([updatedFile]);
}
async retryFailedFiles() {
await this.queueFilesForUpload(this.failedFiles, [
...this.collections.values(),
]);
}
}
export default new UploadManager();

64
src/services/upload/uploader.ts
View file

@ -1,11 +1,7 @@
import { EnteFile } from 'types/file';
import { handleUploadError, CustomError } from 'utils/error';
import { logError } from 'utils/sentry';
import {
findSameFileInCollection,
findSameFileInOtherCollection,
shouldDedupeAcrossCollection,
} from 'utils/upload';
import { findMatchingExistingFile } from 'utils/upload';
import UploadHttpClient from './uploadHttpClient';
import UIService from './uiService';
import UploadService from './uploadService';
@ -51,49 +47,23 @@ export default async function uploader(
throw Error(CustomError.NO_METADATA);
}
const sameFileInSameCollection = findSameFileInCollection(
existingFilesInCollection,
metadata
);
if (sameFileInSameCollection) {
addLogLine(`skipped upload for ${fileNameSize}`);
return {
fileUploadResult: UPLOAD_RESULT.ALREADY_UPLOADED,
uploadedFile: sameFileInSameCollection,
};
}
const sameFileInOtherCollection = findSameFileInOtherCollection(
existingFiles,
metadata
);
if (sameFileInOtherCollection) {
addLogLine(
`same file in other collection found for ${fileNameSize}`
);
const resultFile = Object.assign({}, sameFileInOtherCollection);
resultFile.collectionID = collection.id;
await addToCollection(collection, [resultFile]);
return {
fileUploadResult: UPLOAD_RESULT.ADDED_SYMLINK,
uploadedFile: resultFile,
};
}
// iOS exports via album doesn't export files without collection and if user exports all photos, album info is not preserved.
// This change allow users to export by albums, upload to ente. And export all photos -> upload files which are not already uploaded
// as part of the albums
if (shouldDedupeAcrossCollection(fileWithCollection.collection.name)) {
addLogLine(`deduped upload for ${fileNameSize}`);
const sameFileInOtherCollection = findSameFileInCollection(
existingFiles,
metadata
);
if (sameFileInOtherCollection) {
const existingFile = findMatchingExistingFile(existingFiles, metadata);
if (existingFile) {
if (existingFile.collectionID === collection.id) {
addLogLine(
`file already present in the collection , skipped upload for ${fileNameSize}`
);
return { fileUploadResult: UPLOAD_RESULT.ALREADY_UPLOADED };
} else {
addLogLine(
`same file in other collection found for ${fileNameSize}`
);
const resultFile = Object.assign({}, existingFile);
resultFile.collectionID = collection.id;
await addToCollection(collection, [resultFile]);
return {
fileUploadResult: UPLOAD_RESULT.ALREADY_UPLOADED,
uploadedFile: sameFileInOtherCollection,
fileUploadResult: UPLOAD_RESULT.ADDED_SYMLINK,
uploadedFile: resultFile,
};
}
}

16
src/types/upload/index.ts
View file

@ -144,23 +144,7 @@ export interface ParsedExtractedMetadata {
creationTime: number;
}
export enum UPLOAD_STRATEGY {
SINGLE_COLLECTION,
COLLECTION_PER_FOLDER,
}
export enum UPLOAD_TYPE {
FILES = 'files',
FOLDERS = 'folders',
ZIPS = 'zips',
}
export interface AnalysisResult {
suggestedCollectionName: string;
multipleFolders: boolean;
}
export const NULL_ANALYSIS_RESULT = {
suggestedCollectionName: '',
multipleFolders: false,
};

45
src/utils/upload/fs.ts
View file

@ -1,45 +0,0 @@
import isElectron from 'is-electron';
import { ElectronFile } from 'types/upload';
import {
AnalysisResult,
UPLOAD_TYPE,
NULL_ANALYSIS_RESULT,
} from 'types/upload';
export function analyseUploadFiles(
uploadFiles: File[] | ElectronFile[],
uploadType: UPLOAD_TYPE
): AnalysisResult {
if (isElectron() && uploadType === UPLOAD_TYPE.FILES) {
return NULL_ANALYSIS_RESULT;
}
const paths: string[] = uploadFiles.map((file) => file['path']);
const getCharCount = (str: string) => (str.match(/\//g) ?? []).length;
paths.sort((path1, path2) => getCharCount(path1) - getCharCount(path2));
const firstPath = paths[0];
const lastPath = paths[paths.length - 1];
const L = firstPath.length;
let i = 0;
const firstFileFolder = firstPath.substring(0, firstPath.lastIndexOf('/'));
const lastFileFolder = lastPath.substring(0, lastPath.lastIndexOf('/'));
while (i < L && firstPath.charAt(i) === lastPath.charAt(i)) i++;
let commonPathPrefix = firstPath.substring(0, i);
if (commonPathPrefix) {
commonPathPrefix = commonPathPrefix.substring(
0,
commonPathPrefix.lastIndexOf('/')
);
if (commonPathPrefix) {
commonPathPrefix = commonPathPrefix.substring(
commonPathPrefix.lastIndexOf('/') + 1
);
}
}
return {
suggestedCollectionName: commonPathPrefix || null,
multipleFolders: firstFileFolder !== lastFileFolder,
};
}

79
src/utils/upload/index.ts
View file

@ -1,16 +1,27 @@
import { FileWithCollection, Metadata } from 'types/upload';
import {
AnalysisResult,
ElectronFile,
FileWithCollection,
Metadata,
} from 'types/upload';
import { EnteFile } from 'types/file';
import { A_SEC_IN_MICROSECONDS } from 'constants/upload';
import {
A_SEC_IN_MICROSECONDS,
NULL_ANALYSIS_RESULT,
UPLOAD_TYPE,
} from 'constants/upload';
import { FILE_TYPE } from 'constants/file';
import { METADATA_FOLDER_NAME } from 'constants/export';
import isElectron from 'is-electron';
const TYPE_JSON = 'json';
const DEDUPE_COLLECTION = new Set(['icloud library', 'icloudlibrary']);
export function findSameFileInCollection(
existingFilesInCollection: EnteFile[],
export function findMatchingExistingFile(
existingFiles: EnteFile[],
newFileMetadata: Metadata
): EnteFile {
for (const existingFile of existingFilesInCollection) {
for (const existingFile of existingFiles) {
if (areFilesSame(existingFile.metadata, newFileMetadata)) {
return existingFile;
}
@ -116,3 +127,61 @@ export function areFileWithCollectionsSame(
): boolean {
return firstFile.localID === secondFile.localID;
}
export function analyseUploadFiles(
uploadType: UPLOAD_TYPE,
toUploadFiles: File[] | ElectronFile[]
): AnalysisResult {
if (isElectron() && uploadType === UPLOAD_TYPE.FILES) {
return NULL_ANALYSIS_RESULT;
}
const paths: string[] = toUploadFiles.map((file) => file['path']);
const getCharCount = (str: string) => (str.match(/\//g) ?? []).length;
paths.sort((path1, path2) => getCharCount(path1) - getCharCount(path2));
const firstPath = paths[0];
const lastPath = paths[paths.length - 1];
const L = firstPath.length;
let i = 0;
const firstFileFolder = firstPath.substring(0, firstPath.lastIndexOf('/'));
const lastFileFolder = lastPath.substring(0, lastPath.lastIndexOf('/'));
while (i < L && firstPath.charAt(i) === lastPath.charAt(i)) i++;
let commonPathPrefix = firstPath.substring(0, i);
if (commonPathPrefix) {
commonPathPrefix = commonPathPrefix.substring(
0,
commonPathPrefix.lastIndexOf('/')
);
if (commonPathPrefix) {
commonPathPrefix = commonPathPrefix.substring(
commonPathPrefix.lastIndexOf('/') + 1
);
}
}
return {
suggestedCollectionName: commonPathPrefix || null,
multipleFolders: firstFileFolder !== lastFileFolder,
};
}
export function getCollectionWiseFiles(toUploadFiles: File[] | ElectronFile[]) {
const collectionWiseFiles = new Map<string, (File | ElectronFile)[]>();
for (const file of toUploadFiles) {
const filePath = file['path'] as string;
let folderPath = filePath.substring(0, filePath.lastIndexOf('/'));
if (folderPath.endsWith(METADATA_FOLDER_NAME)) {
folderPath = folderPath.substring(0, folderPath.lastIndexOf('/'));
}
const folderName = folderPath.substring(
folderPath.lastIndexOf('/') + 1
);
if (!collectionWiseFiles.has(folderName)) {
collectionWiseFiles.set(folderName, []);
}
collectionWiseFiles.get(folderName).push(file);
}
return collectionWiseFiles;
}