dedupe files to be sent for uploads

src/services/watchFolder/watchFolderService.ts

@@ -20,7 +20,7 @@ import { getParentFolderName } from './utils';
 import { UPLOAD_RESULT, UPLOAD_STRATEGY } from 'constants/upload';
 import uploadManager from 'services/upload/uploadManager';
 import { addLocalLog, addLogLine } from 'utils/logging';
-import { filterOutSystemFiles } from 'utils/upload';
+import { getValidFilesToUpload } from 'utils/watch';
 
 class watchFolderService {
     private electronAPIs: ElectronAPIs;
@@ -108,15 +108,7 @@ class watchFolderService {
         mapping: WatchMapping,
         filesOnDisk: ElectronFile[]
     ) {
-        const filesToUpload = filterOutSystemFiles(
-            filesOnDisk.filter((electronFile) => {
-                return (
-                    !mapping.syncedFiles.find(
-                        (file) => file.path === electronFile.path
-                    ) && !mapping.ignoredFiles.includes(electronFile.path)
-                );
-            })
-        ) as ElectronFile[];
+        const filesToUpload = getValidFilesToUpload(filesOnDisk, mapping);
 
         if (filesToUpload.length > 0) {
             for (const file of filesToUpload) {
@@ -266,15 +258,7 @@ class watchFolderService {
                 throw Error('no Mapping found for event');
             }
             if (event.type === 'upload') {
-                event.files = filterOutSystemFiles(
-                    event.files.filter(
-                        (file) =>
-                            !mapping.ignoredFiles.includes(file.path) &&
-                            !mapping.syncedFiles.find(
-                                (f) => f.path === file.path
-                            )
-                    )
-                ) as ElectronFile[];
+                event.files = getValidFilesToUpload(event.files, mapping);
                 if (event.files.length === 0) {
                     return;
                 }

src/utils/upload/index.ts

@@ -203,12 +203,16 @@ export function filterOutSystemFiles(files: File[] | ElectronFile[]) {
     if (files[0] instanceof File) {
         const browserFiles = files as File[];
         return browserFiles.filter((file) => {
-            return !SYSTEM_FILES.includes(file.name);
+            return !isSystemFile(file);
         });
     } else {
         const electronFiles = files as ElectronFile[];
         return electronFiles.filter((file) => {
-            return !SYSTEM_FILES.includes(file.name);
+            return !isSystemFile(file);
         });
     }
 }
+
+export function isSystemFile(file: File | ElectronFile) {
+    return SYSTEM_FILES.includes(file.name);
+}

src/utils/watch/index.ts

@@ -0,0 +1,26 @@
+import { ElectronFile } from 'types/upload';
+import { WatchMapping } from 'types/watchFolder';
+import { isSystemFile } from 'utils/upload';
+
+export function isNotSyncedOrIgnoredFile(
+    file: ElectronFile,
+    mapping: WatchMapping
+) {
+    return (
+        !mapping.ignoredFiles.includes(file.path) &&
+        !mapping.syncedFiles.find((f) => f.path === file.path)
+    );
+}
+
+export function getValidFilesToUpload(
+    files: ElectronFile[],
+    mapping: WatchMapping
+) {
+    const uniqueFilePaths = new Set<string>();
+    return files.filter(
+        (file) =>
+            !isSystemFile(file) &&
+            !isNotSyncedOrIgnoredFile(file, mapping) &&
+            !uniqueFilePaths.has(file.path)
+    );
+}