Merge pull request #447 from ente-io/rewrite_device_sync_remote

[Part-2] Rewrite Device Sync

lib/db/device_files_db.dart

@@ -236,6 +236,24 @@ extension DeviceFiles on FilesDB {
     }
   }
 
+  // getDeviceSyncCollectionIDs returns the collectionIDs for the
+  // deviceCollections which are marked for auto-backup
+  Future<Set<int>> getDeviceSyncCollectionIDs() async {
+    final Database db = await database;
+    final rows = await db.rawQuery(
+      '''
+      SELECT collection_id FROM device_collections where should_backup = 
+      $_sqlBoolTrue 
+      and collection_id != -1;
+      ''',
+    );
+    final Set<int> result = <int>{};
+    for (final row in rows) {
+      result.add(row['collection_id']);
+    }
+    return result;
+  }
+
   Future<void> updateDevicePathSyncStatus(Map<String, bool> syncStatus) async {
     final db = await database;
     var batch = db.batch();
@@ -296,7 +314,8 @@ extension DeviceFiles on FilesDB {
         (limit != null ? ' limit $limit;' : ';');
     final results = await db.rawQuery(rawQuery);
     final files = convertToFiles(results);
-    return FileLoadResult(files, files.length == limit);
+    final dedupe = deduplicateByLocalID(files);
+    return FileLoadResult(dedupe, files.length == limit);
   }
 
   Future<List<DeviceCollection>> getDeviceCollections({

lib/db/files_db.dart

@@ -455,7 +455,7 @@ class FilesDB {
     return BackedUpFileIDs(localIDs.toList(), uploadedIDs.toList());
   }
 
-  Future<FileLoadResult> getAllUploadedFiles(
+  Future<FileLoadResult> getAllPendingOrUploadedFiles(
     int startTime,
     int endTime,
     int ownerID, {
@@ -469,7 +469,7 @@ class FilesDB {
     final results = await db.query(
       filesTable,
       where:
-          '$columnCreationTime >= ? AND $columnCreationTime <= ? AND  $columnOwnerID = ? AND ($columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1)'
+          '$columnCreationTime >= ? AND $columnCreationTime <= ? AND  ($columnOwnerID IS NULL OR $columnOwnerID = ?) AND ($columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1)'
           ' AND $columnMMdVisibility = ?',
       whereArgs: [startTime, endTime, ownerID, visibility],
       orderBy:
@@ -528,39 +528,7 @@ class FilesDB {
     return FileLoadResult(deduplicatedFiles, files.length == limit);
   }
 
-  Future<FileLoadResult> getImportantFiles(
+  List<File> deduplicateByLocalID(List<File> files) {
-    int startTime,
-    int endTime,
-    int ownerID,
-    List<String> paths, {
-    int limit,
-    bool asc,
-    Set<int> ignoredCollectionIDs,
-  }) async {
-    final db = await instance.database;
-    String inParam = "";
-    for (final path in paths) {
-      inParam += "'" + path.replaceAll("'", "''") + "',";
-    }
-    inParam = inParam.substring(0, inParam.length - 1);
-    final order = (asc ?? false ? 'ASC' : 'DESC');
-    final results = await db.query(
-      filesTable,
-      where:
-          '$columnCreationTime >= ? AND $columnCreationTime <= ? AND ($columnOwnerID IS NULL OR $columnOwnerID = ?) AND ($columnMMdVisibility IS NULL OR $columnMMdVisibility = ?)'
-          'AND (($columnLocalID IS NOT NULL AND $columnDeviceFolder IN ($inParam)) OR ($columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1))',
-      whereArgs: [startTime, endTime, ownerID, kVisibilityVisible],
-      orderBy:
-          '$columnCreationTime ' + order + ', $columnModificationTime ' + order,
-      limit: limit,
-    );
-    final files = convertToFiles(results);
-    final List<File> deduplicatedFiles =
-        _deduplicatedAndFilterIgnoredFiles(files, ignoredCollectionIDs);
-    return FileLoadResult(deduplicatedFiles, files.length == limit);
-  }
-
-  List<File> _deduplicateByLocalID(List<File> files) {
     final localIDs = <String>{};
     final List<File> deduplicatedFiles = [];
     for (final file in files) {
@@ -630,29 +598,6 @@ class FilesDB {
     return FileLoadResult(files, files.length == limit);
   }
 
-  Future<FileLoadResult> getFilesInPath(
-    String path,
-    int startTime,
-    int endTime, {
-    int limit,
-    bool asc,
-  }) async {
-    final db = await instance.database;
-    final order = (asc ?? false ? 'ASC' : 'DESC');
-    final results = await db.query(
-      filesTable,
-      where:
-          '$columnDeviceFolder = ? AND $columnCreationTime >= ? AND $columnCreationTime <= ? AND $columnLocalID IS NOT NULL',
-      whereArgs: [path, startTime, endTime],
-      orderBy:
-          '$columnCreationTime ' + order + ', $columnModificationTime ' + order,
-      groupBy: columnLocalID,
-      limit: limit,
-    );
-    final files = convertToFiles(results);
-    return FileLoadResult(files, files.length == limit);
-  }
-
   Future<FileLoadResult> getLocalDeviceFiles(
     int startTime,
     int endTime, {
@@ -671,32 +616,10 @@ class FilesDB {
       limit: limit,
     );
     final files = convertToFiles(results);
-    final result = _deduplicateByLocalID(files);
+    final result = deduplicateByLocalID(files);
     return FileLoadResult(result, files.length == limit);
   }
 
-  Future<List<File>> getAllVideos() async {
-    final db = await instance.database;
-    final results = await db.query(
-      filesTable,
-      where: '$columnLocalID IS NOT NULL AND $columnFileType = 1',
-      orderBy: '$columnCreationTime DESC',
-    );
-    return convertToFiles(results);
-  }
-
-  Future<List<File>> getAllInPath(String path) async {
-    final db = await instance.database;
-    final results = await db.query(
-      filesTable,
-      where: '$columnLocalID IS NOT NULL AND $columnDeviceFolder = ?',
-      whereArgs: [path],
-      orderBy: '$columnCreationTime DESC',
-      groupBy: columnLocalID,
-    );
-    return convertToFiles(results);
-  }
-
   Future<List<File>> getFilesCreatedWithinDurations(
     List<List<int>> durations,
     Set<int> ignoredCollectionIDs, {
@@ -724,28 +647,6 @@ class FilesDB {
     return _deduplicatedAndFilterIgnoredFiles(files, ignoredCollectionIDs);
   }
 
-  Future<List<File>> getFilesToBeUploadedWithinFolders(
-    Set<String> folders,
-  ) async {
-    if (folders.isEmpty) {
-      return [];
-    }
-    final db = await instance.database;
-    String inParam = "";
-    for (final folder in folders) {
-      inParam += "'" + folder.replaceAll("'", "''") + "',";
-    }
-    inParam = inParam.substring(0, inParam.length - 1);
-    final results = await db.query(
-      filesTable,
-      where:
-          '($columnUploadedFileID IS NULL OR $columnUploadedFileID IS -1) AND $columnDeviceFolder IN ($inParam)',
-      orderBy: '$columnCreationTime DESC',
-      groupBy: columnLocalID,
-    );
-    return convertToFiles(results);
-  }
-
   // Files which user added to a collection manually but they are not uploaded yet.
   Future<List<File>> getPendingManualUploads() async {
     final db = await instance.database;
@@ -782,25 +683,16 @@ class FilesDB {
     return convertToFiles(results);
   }
 
-  Future<List<File>> getEditedRemoteFiles() async {
+  Future<List<int>> getUploadedFileIDsToBeUpdated(int ownerID) async {
-    final db = await instance.database;
-    final results = await db.query(
-      filesTable,
-      where:
-          '($columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1) AND ($columnUploadedFileID IS NULL OR $columnUploadedFileID IS -1)',
-      orderBy: '$columnCreationTime DESC',
-      groupBy: columnLocalID,
-    );
-    return convertToFiles(results);
-  }
-
-  Future<List<int>> getUploadedFileIDsToBeUpdated() async {
     final db = await instance.database;
     final rows = await db.query(
       filesTable,
       columns: [columnUploadedFileID],
-      where:
+      where: '($columnLocalID IS NOT NULL AND $columnOwnerID = ? AND '
-          '($columnLocalID IS NOT NULL AND ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) AND $columnUpdationTime IS NULL)',
+          '($columnUploadedFileID '
+          'IS NOT '
+          'NULL AND $columnUploadedFileID IS NOT -1) AND $columnUpdationTime IS NULL)',
+      whereArgs: [ownerID],
       orderBy: '$columnCreationTime DESC',
       distinct: true,
     );
@@ -842,6 +734,62 @@ class FilesDB {
     return result;
   }
 
+  Future<Set<String>> getLocalIDsMarkedForOrAlreadyUploaded(int ownerID) async {
+    final db = await instance.database;
+    final rows = await db.query(
+      filesTable,
+      columns: [columnLocalID],
+      distinct: true,
+      where: '$columnLocalID IS NOT NULL AND ($columnCollectionID IS NOT NULL '
+          'AND '
+          '$columnCollectionID != -1) AND ($columnOwnerID = ? OR '
+          '$columnOwnerID IS NULL)',
+      whereArgs: [ownerID],
+    );
+    final result = <String>{};
+    for (final row in rows) {
+      result.add(row[columnLocalID]);
+    }
+    return result;
+  }
+
+  Future<Set<String>> getLocalFileIDsForCollection(int collectionID) async {
+    final db = await instance.database;
+    final rows = await db.query(
+      filesTable,
+      columns: [columnLocalID],
+      where: '$columnLocalID IS NOT NULL AND $columnCollectionID = ?',
+      whereArgs: [collectionID],
+    );
+    final result = <String>{};
+    for (final row in rows) {
+      result.add(row[columnLocalID]);
+    }
+    return result;
+  }
+
+  // Sets the collectionID for the files with given LocalIDs if the
+  // corresponding file entries are not already mapped to some other collection
+  Future<int> setCollectionIDForUnMappedLocalFiles(
+    int collectionID,
+    Set<String> localIDs,
+  ) async {
+    final db = await instance.database;
+    String inParam = "";
+    for (final localID in localIDs) {
+      inParam += "'" + localID + "',";
+    }
+    inParam = inParam.substring(0, inParam.length - 1);
+    return await db.rawUpdate(
+      '''
+      UPDATE $filesTable
+      SET $columnCollectionID = $collectionID
+      WHERE $columnLocalID IN ($inParam) AND ($columnCollectionID IS NULL OR 
+      $columnCollectionID = -1);
+    ''',
+    );
+  }
+
   Future<int> getNumberOfUploadedFiles() async {
     final db = await instance.database;
     final rows = await db.query(
@@ -954,7 +902,6 @@ class FilesDB {
     if (fileType == FileType.livePhoto && hashData.zipHash != null) {
       inParam += ",'${hashData.zipHash}'";
     }
-
     final db = await instance.database;
     final rows = await db.query(
       filesTable,
@@ -1026,6 +973,17 @@ class FilesDB {
     );
   }
 
+  Future<int> deleteMultipleByGeneratedIDs(List<int> generatedIDs) async {
+    if (generatedIDs.isEmpty) {
+      return 0;
+    }
+    final db = await instance.database;
+    return await db.delete(
+      filesTable,
+      where: '$columnGeneratedID IN (${generatedIDs.join(', ')})',
+    );
+  }
+
   Future<int> deleteLocalFile(File file) async {
     final db = await instance.database;
     if (file.localID != null) {
@@ -1149,6 +1107,42 @@ class FilesDB {
     );
   }
 
+  Future<List<File>> getPendingUploadForCollection(int collectionID) async {
+    final db = await instance.database;
+    final results = await db.query(
+      filesTable,
+      where: '$columnCollectionID = ? AND ($columnUploadedFileID IS NULL OR '
+          '$columnUploadedFileID = -1)',
+      whereArgs: [collectionID],
+    );
+    return convertToFiles(results);
+  }
+
+  Future<Set<String>> getLocalIDsPresentInEntries(
+    List<File> existingFiles,
+    int collectionID,
+  ) async {
+    String inParam = "";
+    for (final existingFile in existingFiles) {
+      inParam += "'" + existingFile.localID + "',";
+    }
+    inParam = inParam.substring(0, inParam.length - 1);
+    final db = await instance.database;
+    final rows = await db.rawQuery(
+      '''
+      SELECT $columnLocalID
+      FROM $filesTable
+      WHERE $columnLocalID IN ($inParam) AND $columnCollectionID != 
+      $collectionID AND $columnLocalID IS NOT NULL;
+    ''',
+    );
+    final result = <String>{};
+    for (final row in rows) {
+      result.add(row[columnLocalID]);
+    }
+    return result;
+  }
+
   Future<List<File>> getLatestLocalFiles() async {
     final db = await instance.database;
     final rows = await db.rawQuery(
@@ -1210,9 +1204,9 @@ class FilesDB {
         ) latest_files
         ON $filesTable.$columnCollectionID = latest_files.$columnCollectionID
         AND $filesTable.$columnCreationTime = latest_files.max_creation_time;
+
   ''';
     }
-
     final db = await instance.database;
     final rows = await db.rawQuery(
       query,

lib/services/local/local_sync_util.dart

@@ -237,12 +237,13 @@ Future<List<AssetPathEntity>> _getGalleryList({
     type: RequestType.common,
     filterOption: filterOptionGroup,
   );
-
-  // todo: assetCount will be deprecated in the new version.
-  // disable sorting and either try to evaluate if it's required or yolo
   galleryList.sort((s1, s2) {
-    return s2.assetCount.compareTo(s1.assetCount);
+    if (s1.isAll) {
+      return 1;
+    }
+    return 0;
   });
+
   return galleryList;
 }
 

lib/services/local_file_update_service.dart

@@ -100,7 +100,7 @@ class LocalFileUpdateService {
   ) async {
     _logger.info("files to process ${localIDsToProcess.length} for reupload");
     final List<ente.File> localFiles =
-        (await FilesDB.instance.getLocalFiles(localIDsToProcess));
+        await FilesDB.instance.getLocalFiles(localIDsToProcess);
     final Set<String> processedIDs = {};
     for (ente.File file in localFiles) {
       if (processedIDs.contains(file.localID)) {

lib/services/local_sync_service.dart

@@ -5,6 +5,7 @@ import 'dart:io';
 
 import 'package:computer/computer.dart';
 import 'package:flutter/foundation.dart';
+import 'package:flutter/widgets.dart';
 import 'package:logging/logging.dart';
 import 'package:photo_manager/photo_manager.dart';
 import 'package:photos/core/configuration.dart';
@@ -12,6 +13,7 @@ import 'package:photos/core/event_bus.dart';
 import 'package:photos/db/device_files_db.dart';
 import 'package:photos/db/file_updation_db.dart';
 import 'package:photos/db/files_db.dart';
+import 'package:photos/db/ignored_files_db.dart';
 import 'package:photos/events/backup_folders_updated_event.dart';
 import 'package:photos/events/local_photos_updated_event.dart';
 import 'package:photos/events/sync_status_update_event.dart';
@@ -147,9 +149,27 @@ class LocalSyncService {
     if (hasUpdated) {
       Bus.instance.fire(BackupFoldersUpdatedEvent());
     }
+    // migrate the backed up folder settings
+    if (!_prefs.containsKey(hasImportedDeviceCollections)) {
+      final pathsToBackUp = Configuration.instance.getPathsToBackUp();
+      final entriesToBackUp = Map.fromEntries(
+        result
+            .where((element) => pathsToBackUp.contains(element.item1.name))
+            .map((e) => MapEntry(e.item1.id, true)),
+      );
+      if (entriesToBackUp.isNotEmpty) {
+        await _db.updateDevicePathSyncStatus(entriesToBackUp);
+        Bus.instance.fire(BackupFoldersUpdatedEvent());
+      }
+      await _prefs.setBool(hasImportedDeviceCollections, true);
+    }
     return hasUpdated;
   }
 
+  bool isDeviceFileMigrationDone() {
+    return _prefs.containsKey(hasImportedDeviceCollections);
+  }
+
   Future<bool> syncAll() async {
     final stopwatch = Stopwatch()..start();
     final localAssets = await getAllLocalAssets();
@@ -273,12 +293,15 @@ class LocalSyncService {
   Future<void> resetLocalSync() async {
     assert(kDebugMode, "only available in debug mode");
     await FilesDB.instance.deleteDB();
+    await IgnoredFilesDB.instance.clearTable();
     for (var element in [
       kHasCompletedFirstImportKey,
       hasImportedDeviceCollections,
       kDbUpdationTimeKey,
       kDownloadedFileIDsKey,
-      kEditedFileIDsKey
+      kEditedFileIDsKey,
+      "has_synced_edit_time",
+      "has_selected_all_folders_for_backup",
     ]) {
       await _prefs.remove(element);
     }

lib/services/remote_sync_service.dart

@@ -5,6 +5,7 @@ import 'dart:io';
 import 'dart:math';
 
 import 'package:flutter/foundation.dart';
+import 'package:flutter/widgets.dart';
 import 'package:logging/logging.dart';
 import 'package:photos/core/configuration.dart';
 import 'package:photos/core/errors.dart';
@@ -20,11 +21,13 @@ import 'package:photos/events/sync_status_update_event.dart';
 import 'package:photos/models/device_collection.dart';
 import 'package:photos/models/file.dart';
 import 'package:photos/models/file_type.dart';
+import 'package:photos/models/upload_strategy.dart';
 import 'package:photos/services/app_lifecycle_service.dart';
 import 'package:photos/services/collections_service.dart';
 import 'package:photos/services/ignored_files_service.dart';
 import 'package:photos/services/local_file_update_service.dart';
 import 'package:photos/services/local_sync_service.dart';
+import 'package:photos/services/sync_service.dart';
 import 'package:photos/services/trash_sync_service.dart';
 import 'package:photos/utils/diff_fetcher.dart';
 import 'package:photos/utils/file_uploader.dart';
@@ -102,7 +105,7 @@ class RemoteSyncService {
       final filesToBeUploaded = await _getFilesToBeUploaded();
       if (kDebugMode) {
         debugPrint("Skip upload for testing");
-        filesToBeUploaded.clear();
+        // filesToBeUploaded.clear();
       }
       final hasUploadedFiles = await _uploadFiles(filesToBeUploaded);
       if (hasUploadedFiles) {
@@ -233,9 +236,132 @@ class RemoteSyncService {
   }
 
   Future<void> _syncDeviceCollectionFilesForUpload() async {
-    final deviceCollections = await FilesDB.instance.getDeviceCollections();
+    final int ownerID = Configuration.instance.getUserID();
+    final FilesDB filesDB = FilesDB.instance;
+    final deviceCollections = await filesDB.getDeviceCollections();
     deviceCollections.removeWhere((element) => !element.shouldBackup);
+    // Sort by count to ensure that photos in iOS are first inserted in
+    // smallest album marked for backup. This is to ensure that photo is
+    // first attempted to upload in a non-recent album.
+    deviceCollections.sort((a, b) => a.count.compareTo(b.count));
     await _createCollectionsForDevicePath(deviceCollections);
+    final Map<String, Set<String>> pathIdToLocalIDs =
+        await filesDB.getDevicePathIDToLocalIDMap();
+    for (final deviceCollection in deviceCollections) {
+      _logger.fine("processing ${deviceCollection.name}");
+      final Set<String> localIDsToSync =
+          pathIdToLocalIDs[deviceCollection.id] ?? {};
+      if (deviceCollection.uploadStrategy == UploadStrategy.ifMissing) {
+        final Set<String> alreadyClaimedLocalIDs =
+            await filesDB.getLocalIDsMarkedForOrAlreadyUploaded(ownerID);
+        localIDsToSync.removeAll(alreadyClaimedLocalIDs);
+      }
+
+      if (localIDsToSync.isEmpty || deviceCollection.collectionID == -1) {
+        continue;
+      }
+
+      await filesDB.setCollectionIDForUnMappedLocalFiles(
+        deviceCollection.collectionID,
+        localIDsToSync,
+      );
+
+      // mark IDs as already synced if corresponding entry is present in
+      // the collection. This can happen when a user has marked a folder
+      // for sync, then un-synced it and again tries to mark if for sync.
+      final Set<String> existingMapping = await filesDB
+          .getLocalFileIDsForCollection(deviceCollection.collectionID);
+      final Set<String> commonElements =
+          localIDsToSync.intersection(existingMapping);
+      if (commonElements.isNotEmpty) {
+        debugPrint(
+          "${commonElements.length} files already existing in "
+          "collection ${deviceCollection.collectionID} for ${deviceCollection.name}",
+        );
+        localIDsToSync.removeAll(commonElements);
+      }
+
+      // At this point, the remaining localIDsToSync will need to create
+      // new file entries, where we can store mapping for localID and
+      // corresponding collection ID
+      if (localIDsToSync.isNotEmpty) {
+        debugPrint(
+          'Adding new entries for ${localIDsToSync.length} files'
+          ' for ${deviceCollection.name}',
+        );
+        final filesWithCollectionID =
+            await filesDB.getLocalFiles(localIDsToSync.toList());
+        final List<File> newFilesToInsert = [];
+        final Set<String> fileFoundForLocalIDs = {};
+        for (var existingFile in filesWithCollectionID) {
+          final String localID = existingFile.localID;
+          if (!fileFoundForLocalIDs.contains(localID)) {
+            existingFile.generatedID = null;
+            existingFile.collectionID = deviceCollection.collectionID;
+            existingFile.uploadedFileID = null;
+            existingFile.ownerID = null;
+            newFilesToInsert.add(existingFile);
+            fileFoundForLocalIDs.add(localID);
+          }
+        }
+        await filesDB.insertMultiple(newFilesToInsert);
+        if (fileFoundForLocalIDs.length != localIDsToSync.length) {
+          _logger.warning(
+            "mismatch in num of filesToSync ${localIDsToSync.length} to "
+            "fileSynced ${fileFoundForLocalIDs.length}",
+          );
+        }
+      }
+    }
+  }
+
+  Future<void> updateDeviceFolderSyncStatus(
+    Map<String, bool> syncStatusUpdate,
+  ) async {
+    final Set<int> oldCollectionIDsForAutoSync =
+        await _db.getDeviceSyncCollectionIDs();
+    await _db.updateDevicePathSyncStatus(syncStatusUpdate);
+    final Set<int> newCollectionIDsForAutoSync =
+        await _db.getDeviceSyncCollectionIDs();
+    SyncService.instance.onDeviceCollectionSet(newCollectionIDsForAutoSync);
+    // remove all collectionIDs which are still marked for backup
+    oldCollectionIDsForAutoSync.removeAll(newCollectionIDsForAutoSync);
+    await removeFilesQueuedForUpload(oldCollectionIDsForAutoSync.toList());
+    Bus.instance.fire(LocalPhotosUpdatedEvent(<File>[]));
+  }
+
+  Future<void> removeFilesQueuedForUpload(List<int> collectionIDs) async {
+    /*
+      For each collection, perform following action
+      1) Get List of all files not uploaded yet
+      2) Delete files who localIDs is also present in other collections.
+      3) For Remaining files, set the collectionID as -1
+     */
+    debugPrint("Removing files for collections $collectionIDs");
+    for (int collectionID in collectionIDs) {
+      final List<File> pendingUploads =
+          await _db.getPendingUploadForCollection(collectionID);
+      if (pendingUploads.isEmpty) {
+        continue;
+      }
+      final Set<String> localIDsInOtherFileEntries =
+          await _db.getLocalIDsPresentInEntries(
+        pendingUploads,
+        collectionID,
+      );
+      final List<File> entriesToUpdate = [];
+      final List<int> entriesToDelete = [];
+      for (File pendingUpload in pendingUploads) {
+        if (localIDsInOtherFileEntries.contains(pendingUpload.localID)) {
+          entriesToDelete.add(pendingUpload.generatedID);
+        } else {
+          pendingUpload.collectionID = -1;
+          entriesToUpdate.add(pendingUpload);
+        }
+      }
+      await _db.deleteMultipleByGeneratedIDs(entriesToDelete);
+      await _db.insertMultiple(entriesToUpdate);
+    }
   }
 
   Future<void> _createCollectionsForDevicePath(
@@ -267,14 +393,12 @@ class RemoteSyncService {
   Future<List<File>> _getFilesToBeUploaded() async {
     final deviceCollections = await FilesDB.instance.getDeviceCollections();
     deviceCollections.removeWhere((element) => !element.shouldBackup);
-    final foldersToBackUp = Configuration.instance.getPathsToBackUp();
     List<File> filesToBeUploaded;
     if (LocalSyncService.instance.hasGrantedLimitedPermissions() &&
-        foldersToBackUp.isEmpty) {
+        deviceCollections.isEmpty) {
       filesToBeUploaded = await _db.getUnUploadedLocalFiles();
     } else {
-      filesToBeUploaded =
+      filesToBeUploaded = await _db.getPendingManualUploads();
-          await _db.getFilesToBeUploadedWithinFolders(foldersToBackUp);
     }
     if (!Configuration.instance.shouldBackupVideos() || _shouldThrottleSync()) {
       filesToBeUploaded
@@ -294,11 +418,6 @@ class RemoteSyncService {
         );
       }
     }
-    if (filesToBeUploaded.isEmpty) {
-      // look for files which user manually tried to back up but they are not
-      // uploaded yet. These files should ignore video backup & ignored files filter
-      filesToBeUploaded = await _db.getPendingManualUploads();
-    }
     _sortByTimeAndType(filesToBeUploaded);
     _logger.info(
       filesToBeUploaded.length.toString() + " new files to be uploaded.",
@@ -307,16 +426,14 @@ class RemoteSyncService {
   }
 
   Future<bool> _uploadFiles(List<File> filesToBeUploaded) async {
-    final updatedFileIDs = await _db.getUploadedFileIDsToBeUpdated();
+    final int ownerID = Configuration.instance.getUserID();
-    _logger.info(updatedFileIDs.length.toString() + " files updated.");
+    final updatedFileIDs = await _db.getUploadedFileIDsToBeUpdated(ownerID);
-
+    if (updatedFileIDs.isNotEmpty) {
-    final editedFiles = await _db.getEditedRemoteFiles();
+      _logger.info("Identified ${updatedFileIDs.length} files for reupload");
-    _logger.info(editedFiles.length.toString() + " files edited.");
+    }
 
     _completedUploads = 0;
-    final int toBeUploaded =
+    final int toBeUploaded = filesToBeUploaded.length + updatedFileIDs.length;
-        filesToBeUploaded.length + updatedFileIDs.length + editedFiles.length;
-
     if (toBeUploaded > 0) {
       Bus.instance.fire(SyncStatusUpdate(SyncStatus.preparingForUpload));
       // verify if files upload is allowed based on their subscription plan and
@@ -351,15 +468,6 @@ class RemoteSyncService {
       _uploadFile(file, collectionID, futures);
     }
 
-    for (final file in editedFiles) {
-      if (_shouldThrottleSync() &&
-          futures.length >= kMaximumPermissibleUploadsInThrottledMode) {
-        _logger.info("Skipping some edited files as we are throttling uploads");
-        break;
-      }
-      _uploadFile(file, file.collectionID, futures);
-    }
-
     try {
       await Future.wait(futures);
     } on InvalidFileError {

lib/services/sync_service.dart

@@ -179,6 +179,15 @@ class SyncService {
     );
   }
 
+  void onDeviceCollectionSet(Set<int> collectionIDs) {
+    _uploader.removeFromQueueWhere(
+      (file) {
+        return !collectionIDs.contains(file.collectionID);
+      },
+      UserCancelledUploadError(),
+    );
+  }
+
   void onVideoBackupPaused() {
     _uploader.removeFromQueueWhere(
       (file) {

lib/ui/backup_folder_selection_page.dart

@@ -16,6 +16,7 @@ import 'package:photos/ente_theme_data.dart';
 import 'package:photos/events/backup_folders_updated_event.dart';
 import 'package:photos/models/device_collection.dart';
 import 'package:photos/models/file.dart';
+import 'package:photos/services/remote_sync_service.dart';
 import 'package:photos/ui/common/loading_widget.dart';
 import 'package:photos/ui/viewer/file/thumbnail_widget.dart';
 
@@ -181,8 +182,8 @@ class _BackupFolderSelectionPageState extends State<BackupFolderSelectionPage> {
                               syncStatus[pathID] =
                                   _selectedDevicePathIDs.contains(pathID);
                             }
-                            await FilesDB.instance
+                            await RemoteSyncService.instance
-                                .updateDevicePathSyncStatus(syncStatus);
+                                .updateDeviceFolderSyncStatus(syncStatus);
                             await Configuration.instance
                                 .setSelectAllFoldersForBackup(
                               _allDevicePathIDs.length ==

lib/ui/collections/device_folders_grid_view_widget.dart

@@ -2,6 +2,7 @@
 
 import 'package:flutter/material.dart';
 import 'package:photos/models/device_collection.dart';
+import 'package:photos/services/local_sync_service.dart';
 import 'package:photos/ui/collections/device_folder_icon_widget.dart';
 import 'package:photos/ui/viewer/gallery/empte_state.dart';
 
@@ -15,6 +16,8 @@ class DeviceFoldersGridViewWidget extends StatelessWidget {
 
   @override
   Widget build(BuildContext context) {
+    final bool isMigrationDone =
+        LocalSyncService.instance.isDeviceFileMigrationDone();
     return Padding(
       padding: const EdgeInsets.symmetric(horizontal: 8),
       child: SizedBox(
@@ -22,7 +25,11 @@ class DeviceFoldersGridViewWidget extends StatelessWidget {
         child: Align(
           alignment: Alignment.centerLeft,
           child: deviceCollections.isEmpty
+              ? (isMigrationDone
                   ? const EmptyState()
+                  : const EmptyState(
+                      text: "Importing....",
+                    ))
               : ListView.builder(
                   shrinkWrap: true,
                   scrollDirection: Axis.horizontal,

lib/ui/home_widget.dart

@@ -392,22 +392,10 @@ class _HomeWidgetState extends State<HomeWidget> {
     }
     final gallery = Gallery(
       asyncLoader: (creationStartTime, creationEndTime, {limit, asc}) async {
-        final importantPaths = Configuration.instance.getPathsToBackUp();
         final ownerID = Configuration.instance.getUserID();
         final archivedCollectionIds =
             CollectionsService.instance.getArchivedCollections();
         FileLoadResult result;
-        if (importantPaths.isNotEmpty) {
-          result = await FilesDB.instance.getImportantFiles(
-            creationStartTime,
-            creationEndTime,
-            ownerID,
-            importantPaths.toList(),
-            limit: limit,
-            asc: asc,
-            ignoredCollectionIDs: archivedCollectionIds,
-          );
-        } else {
         if (LocalSyncService.instance.hasGrantedLimitedPermissions()) {
           result = await FilesDB.instance.getAllLocalAndUploadedFiles(
             creationStartTime,
@@ -418,7 +406,7 @@ class _HomeWidgetState extends State<HomeWidget> {
             ignoredCollectionIDs: archivedCollectionIds,
           );
         } else {
-            result = await FilesDB.instance.getAllUploadedFiles(
+          result = await FilesDB.instance.getAllPendingOrUploadedFiles(
             creationStartTime,
             creationEndTime,
             ownerID,
@@ -427,7 +415,7 @@ class _HomeWidgetState extends State<HomeWidget> {
             ignoredCollectionIDs: archivedCollectionIds,
           );
         }
-        }
+
         // hide ignored files from home page UI
         final ignoredIDs = await IgnoredFilesService.instance.ignoredIDs;
         result.files.removeWhere(

lib/ui/viewer/gallery/archive_page.dart

@@ -29,7 +29,7 @@ class ArchivePage extends StatelessWidget {
   Widget build(Object context) {
     final gallery = Gallery(
       asyncLoader: (creationStartTime, creationEndTime, {limit, asc}) {
-        return FilesDB.instance.getAllUploadedFiles(
+        return FilesDB.instance.getAllPendingOrUploadedFiles(
           creationStartTime,
           creationEndTime,
           Configuration.instance.getUserID(),

lib/ui/viewer/gallery/device_folder_page.dart

@@ -12,6 +12,7 @@ import 'package:photos/events/local_photos_updated_event.dart';
 import 'package:photos/models/device_collection.dart';
 import 'package:photos/models/gallery_type.dart';
 import 'package:photos/models/selected_files.dart';
+import 'package:photos/services/remote_sync_service.dart';
 import 'package:photos/ui/viewer/gallery/gallery.dart';
 import 'package:photos/ui/viewer/gallery/gallery_app_bar_widget.dart';
 import 'package:photos/ui/viewer/gallery/gallery_overlay_widget.dart';
@@ -114,7 +115,7 @@ class _BackupConfigurationHeaderWidgetState
           Switch(
             value: _isBackedUp,
             onChanged: (value) async {
-              await FilesDB.instance.updateDevicePathSyncStatus(
+              await RemoteSyncService.instance.updateDeviceFolderSyncStatus(
                 {widget.deviceCollection.id: value},
               );
               _isBackedUp = value;