From f8b0706d81c886771b05754abeb9217a0593e7cf Mon Sep 17 00:00:00 2001 From: Abhinav Date: Sat, 3 Sep 2022 20:38:25 +0530 Subject: [PATCH] add files that can't be uploaded to ignored files --- src/services/upload/uploadManager.ts | 20 +- .../watchFolder/watchFolderService.ts | 242 ++++++++++++------ src/types/electron/index.ts | 8 +- src/types/watchFolder/index.ts | 11 +- 4 files changed, 177 insertions(+), 104 deletions(-) diff --git a/src/services/upload/uploadManager.ts b/src/services/upload/uploadManager.ts index 6ebd4a319..e4c50c6d4 100644 --- a/src/services/upload/uploadManager.ts +++ b/src/services/upload/uploadManager.ts @@ -469,19 +469,11 @@ class UploadManager { ) { this.updateExistingFiles(decryptedFile); } - if ( - [ - UPLOAD_RESULT.ADDED_SYMLINK, - UPLOAD_RESULT.UPLOADED, - UPLOAD_RESULT.UPLOADED_WITH_STATIC_THUMBNAIL, - UPLOAD_RESULT.ALREADY_UPLOADED, - ].includes(fileUploadResult) - ) { - await this.watchFolderCallback( - fileWithCollection, - uploadedFile - ); - } + await this.watchFolderCallback( + fileUploadResult, + fileWithCollection, + uploadedFile + ); return fileUploadResult; } catch (e) { logError(e, 'failed to do post file upload action'); @@ -494,11 +486,13 @@ class UploadManager { } private async watchFolderCallback( + fileUploadResult: UPLOAD_RESULT, fileWithCollection: FileWithCollection, uploadedFile: EnteFile ) { if (isElectron()) { await watchFolderService.onFileUpload( + fileUploadResult, fileWithCollection, uploadedFile ); diff --git a/src/services/watchFolder/watchFolderService.ts b/src/services/watchFolder/watchFolderService.ts index 17f97bd39..2d9ac7ae1 100644 --- a/src/services/watchFolder/watchFolderService.ts +++ b/src/services/watchFolder/watchFolderService.ts @@ -17,7 +17,7 @@ import { diskFolderRemovedCallback, } from './watchFolderEventHandlers'; import { getParentFolderName } from './utils'; -import { UPLOAD_STRATEGY } from 'constants/upload'; +import { UPLOAD_RESULT, UPLOAD_STRATEGY } from 'constants/upload'; import uploadManager from 'services/upload/uploadManager'; import { addLocalLog, addLogLine } from 'utils/logging'; @@ -26,10 +26,12 @@ class watchFolderService { private allElectronAPIsExist: boolean = false; private eventQueue: EventQueueItem[] = []; private currentEvent: EventQueueItem; + private currentlySyncedMapping: WatchMapping; private trashingDirQueue: string[] = []; private isEventRunning: boolean = false; private uploadRunning: boolean = false; - private pathToIDMap = new Map(); + private filePathToUploadedFileIDMap = new Map(); + private unUploadableFilePaths = new Set(); private isPaused = false; private setElectronFiles: (files: ElectronFile[]) => void; private setCollectionName: (collectionName: string) => void; @@ -108,8 +110,10 @@ class watchFolderService { filesOnDisk: ElectronFile[] ) { const filesToUpload = filesOnDisk.filter((electronFile) => { - return !mapping.files.find( - (file) => file.path === electronFile.path + return ( + !mapping.syncedFiles.find( + (file) => file.path === electronFile.path + ) && !mapping.ignoredFiles.includes(electronFile.path) ); }); @@ -133,7 +137,7 @@ class watchFolderService { mapping: WatchMapping, filesOnDisk: ElectronFile[] ) { - const filesToRemove = mapping.files.filter((file) => { + const filesToRemove = mapping.syncedFiles.filter((file) => { return !filesOnDisk.find( (electronFile) => electronFile.path === file.path ); @@ -238,27 +242,40 @@ class watchFolderService { } private async runNextEvent() { - addLogLine( - `mappings, + try { + addLogLine( + `mappings, ${this.getWatchMappings().map((m) => JSON.stringify(m))}` - ); + ); - if ( - this.eventQueue.length === 0 || - this.isEventRunning || - this.isPaused - ) { - return; - } + if ( + this.eventQueue.length === 0 || + this.isEventRunning || + this.isPaused + ) { + return; + } - this.setIsEventRunning(true); - const event = this.clubSameCollectionEvents(); - this.currentEvent = event; - addLogLine(`running event', ${JSON.stringify(event)}`); - if (event.type === 'upload') { - this.processUploadEvent(); - } else { - await this.processTrashEvent(); + this.setIsEventRunning(true); + const event = this.clubSameCollectionEvents(); + this.currentEvent = event; + const mappings = this.getWatchMappings(); + const mapping = mappings.find( + (mapping) => mapping.folderPath === this.currentEvent.folderPath + ); + if (!mapping) { + throw Error('no Mapping found for event'); + } + this.currentlySyncedMapping = mapping; + addLogLine(`running event', ${JSON.stringify(event)}`); + if (event.type === 'upload') { + this.processUploadEvent(); + } else { + await this.processTrashEvent(); + } + } catch (e) { + logError(e, 'runNextEvent failed'); + } finally { this.setIsEventRunning(false); this.runNextEvent(); } @@ -275,25 +292,59 @@ class watchFolderService { } } - async onFileUpload(fileWithCollection: FileWithCollection, file: EnteFile) { + async onFileUpload( + fileUploadResult: UPLOAD_RESULT, + fileWithCollection: FileWithCollection, + file: EnteFile + ) { addLocalLog(() => `onFileUpload called`); if (!this.isUploadRunning) { return; } - if (fileWithCollection.isLivePhoto) { - this.pathToIDMap.set( - (fileWithCollection.livePhotoAssets.image as ElectronFile).path, - file.id - ); - this.pathToIDMap.set( - (fileWithCollection.livePhotoAssets.video as ElectronFile).path, - file.id - ); - } else { - this.pathToIDMap.set( - (fileWithCollection.file as ElectronFile).path, - file.id - ); + if ( + [ + UPLOAD_RESULT.ADDED_SYMLINK, + UPLOAD_RESULT.UPLOADED, + UPLOAD_RESULT.UPLOADED_WITH_STATIC_THUMBNAIL, + UPLOAD_RESULT.ALREADY_UPLOADED, + ].includes(fileUploadResult) + ) { + if (fileWithCollection.isLivePhoto) { + this.filePathToUploadedFileIDMap.set( + (fileWithCollection.livePhotoAssets.image as ElectronFile) + .path, + file.id + ); + this.filePathToUploadedFileIDMap.set( + (fileWithCollection.livePhotoAssets.video as ElectronFile) + .path, + file.id + ); + } else { + this.filePathToUploadedFileIDMap.set( + (fileWithCollection.file as ElectronFile).path, + file.id + ); + } + } else if ( + [UPLOAD_RESULT.UNSUPPORTED, UPLOAD_RESULT.TOO_LARGE].includes( + fileUploadResult + ) + ) { + if (fileWithCollection.isLivePhoto) { + this.unUploadableFilePaths.add( + (fileWithCollection.livePhotoAssets.image as ElectronFile) + .path + ); + this.unUploadableFilePaths.add( + (fileWithCollection.livePhotoAssets.video as ElectronFile) + .path + ); + } else { + this.unUploadableFilePaths.add( + (fileWithCollection.file as ElectronFile).path + ); + } } } @@ -321,29 +372,41 @@ class watchFolderService { return; } - const uploadedFiles: WatchMapping['files'] = []; + const syncedFiles: WatchMapping['syncedFiles'] = []; + const ignoredFiles: WatchMapping['ignoredFiles'] = []; for (const fileWithCollection of filesWithCollection) { - this.handleUploadedFile(fileWithCollection, uploadedFiles); + this.handleUploadedFile( + fileWithCollection, + syncedFiles, + ignoredFiles + ); } + addLocalLog(() => `syncedFiles ${JSON.stringify(syncedFiles)}`); addLocalLog( - () => `uploadedFiles ${JSON.stringify(uploadedFiles)}` + () => `ignoredFiles ${JSON.stringify(ignoredFiles)}` ); - if (uploadedFiles.length > 0) { - const mappings = this.getWatchMappings(); - const mapping = mappings.find( - (mapping) => - mapping.folderPath === this.currentEvent.folderPath + if (syncedFiles.length > 0) { + this.currentlySyncedMapping.syncedFiles = [ + ...this.currentlySyncedMapping.syncedFiles, + ...syncedFiles, + ]; + this.ElectronAPIs.updateWatchMappingSyncedFiles( + this.currentlySyncedMapping.folderPath, + this.currentlySyncedMapping.syncedFiles + ); + } + if (ignoredFiles.length > 0) { + this.currentlySyncedMapping.ignoredFiles = [ + ...this.currentlySyncedMapping.ignoredFiles, + ...ignoredFiles, + ]; + this.ElectronAPIs.updateWatchMappingIgnoredFiles( + this.currentlySyncedMapping.folderPath, + this.currentlySyncedMapping.ignoredFiles ); - if (mapping) { - mapping.files = [...mapping.files, ...uploadedFiles]; - this.ElectronAPIs.updateWatchMappingFiles( - mapping.folderPath, - mapping.files - ); - } } this.runPostUploadsAction(); @@ -361,7 +424,8 @@ class watchFolderService { private handleUploadedFile( fileWithCollection: FileWithCollection, - uploadedFiles: { path: string; id: number }[] + syncedFiles: { path: string; id: number }[], + ignoredFiles: string[] ) { if (fileWithCollection.isLivePhoto) { const imagePath = ( @@ -372,40 +436,55 @@ class watchFolderService { ).path; if ( - this.pathToIDMap.has(imagePath) && - this.pathToIDMap.has(videoPath) + this.filePathToUploadedFileIDMap.has(imagePath) && + this.filePathToUploadedFileIDMap.has(videoPath) ) { const imageFile = { path: imagePath, - id: this.pathToIDMap.get(imagePath), + id: this.filePathToUploadedFileIDMap.get(imagePath), }; const videoFile = { path: videoPath, - id: this.pathToIDMap.get(videoPath), + id: this.filePathToUploadedFileIDMap.get(videoPath), }; - uploadedFiles.push(imageFile); - uploadedFiles.push(videoFile); + syncedFiles.push(imageFile); + syncedFiles.push(videoFile); addLocalLog( () => `added image ${JSON.stringify( imageFile - )} and video file ${JSON.stringify(videoFile)}` + )} and video file ${JSON.stringify( + videoFile + )} to uploadedFiles` + ); + } else if ( + this.unUploadableFilePaths.has(imagePath) && + this.unUploadableFilePaths.has(videoPath) + ) { + ignoredFiles.push(imagePath); + ignoredFiles.push(videoPath); + addLocalLog( + () => + `added image ${imagePath} and video file ${videoPath} to rejectedFiles` ); - this.pathToIDMap.delete(imagePath); - this.pathToIDMap.delete(videoPath); } + this.filePathToUploadedFileIDMap.delete(imagePath); + this.filePathToUploadedFileIDMap.delete(videoPath); } else { const filePath = (fileWithCollection.file as ElectronFile).path; - if (this.pathToIDMap.has(filePath)) { + if (this.filePathToUploadedFileIDMap.has(filePath)) { const file = { path: filePath, - id: this.pathToIDMap.get(filePath), + id: this.filePathToUploadedFileIDMap.get(filePath), }; - uploadedFiles.push(file); + syncedFiles.push(file); addLocalLog(() => `added file ${JSON.stringify(file)} `); - this.pathToIDMap.delete(filePath); + } else if (this.unUploadableFilePaths.has(filePath)) { + ignoredFiles.push(filePath); + addLocalLog(() => `added file ${filePath} to rejectedFiles`); } + this.filePathToUploadedFileIDMap.delete(filePath); } } @@ -415,29 +494,22 @@ class watchFolderService { return; } - const { collectionName, folderPath, paths } = this.currentEvent; + const { collectionName, paths } = this.currentEvent; const filePathsToRemove = new Set(paths); - const mappings = this.getWatchMappings(); - const mappingIdx = mappings.findIndex( - (mapping) => mapping.folderPath === folderPath - ); - if (mappingIdx === -1) { - return; - } - - const files = mappings[mappingIdx].files.filter((file) => - filePathsToRemove.has(file.path) + const files = this.currentlySyncedMapping.syncedFiles.filter( + (file) => filePathsToRemove.has(file.path) ); await this.trashByIDs(files, collectionName); - mappings[mappingIdx].files = mappings[mappingIdx].files.filter( - (file) => !filePathsToRemove.has(file.path) - ); - this.ElectronAPIs.updateWatchMappingFiles( - mappings[mappingIdx].folderPath, - mappings[mappingIdx].files + this.currentlySyncedMapping.syncedFiles = + this.currentlySyncedMapping.syncedFiles.filter( + (file) => !filePathsToRemove.has(file.path) + ); + this.ElectronAPIs.updateWatchMappingSyncedFiles( + this.currentlySyncedMapping.folderPath, + this.currentlySyncedMapping.syncedFiles ); } catch (e) { logError(e, 'error while running next trash'); @@ -445,7 +517,7 @@ class watchFolderService { } private async trashByIDs( - toTrashFiles: WatchMapping['files'], + toTrashFiles: WatchMapping['syncedFiles'], collectionName: string ) { try { diff --git a/src/types/electron/index.ts b/src/types/electron/index.ts index 59615347b..3bb4f59bc 100644 --- a/src/types/electron/index.ts +++ b/src/types/electron/index.ts @@ -37,9 +37,13 @@ export interface ElectronAPIs { setToUploadCollection: (collectionName: string) => void; getDirFiles: (dirPath: string) => Promise; getWatchMappings: () => WatchMapping[]; - updateWatchMappingFiles: ( + updateWatchMappingSyncedFiles: ( folderPath: string, - files: WatchMapping['files'] + files: WatchMapping['syncedFiles'] + ) => void; + updateWatchMappingIgnoredFiles: ( + folderPath: string, + files: WatchMapping['ignoredFiles'] ) => void; addWatchMapping: ( collectionName: string, diff --git a/src/types/watchFolder/index.ts b/src/types/watchFolder/index.ts index 97fa96dce..d191d7358 100644 --- a/src/types/watchFolder/index.ts +++ b/src/types/watchFolder/index.ts @@ -1,14 +1,17 @@ import { UPLOAD_STRATEGY } from 'constants/upload'; import { ElectronFile } from 'types/upload'; +interface WatchMappingSyncedFile { + path: string; + id: number; +} + export interface WatchMapping { rootFolderName: string; folderPath: string; uploadStrategy: UPLOAD_STRATEGY; - files: { - path: string; - id: number; - }[]; + syncedFiles: WatchMappingSyncedFile[]; + ignoredFiles: string[]; } export interface EventQueueItem {