Merge branch 'main' into remove_getCollectionsWithThumbnails_shared

lib/db/files_db.dart

@@ -11,6 +11,7 @@ import 'package:photos/models/file_load_result.dart';
 import 'package:photos/models/file_type.dart';
 import 'package:photos/models/location/location.dart';
 import "package:photos/models/metadata/common_keys.dart";
+import "package:photos/services/filter/db_filters.dart";
 import 'package:photos/utils/file_uploader_util.dart';
 import 'package:sqflite/sqflite.dart';
 import 'package:sqflite_migration/sqflite_migration.dart';
@@ -507,7 +508,7 @@ class FilesDB {
     int? limit,
     bool? asc,
     int visibility = visibleVisibility,
-    Set<int>? ignoredCollectionIDs,
+    DBFilterOptions? filterOptions,
     bool applyOwnerCheck = false,
   }) async {
     final stopWatch = EnteWatch('getAllPendingOrUploadedFiles')..start();
@@ -539,13 +540,10 @@ class FilesDB {
     stopWatch.log('queryDone');
     final files = convertToFiles(results);
     stopWatch.log('convertDone');
-    final List<File> deduplicatedFiles =
-        _deduplicatedAndFilterIgnoredFiles(files, ignoredCollectionIDs);
-    stopWatch.log(
-      "dedupeDone: for ${files.length} files.",
-    );
+    final filteredFiles = await applyDBFilters(files, filterOptions);
+    stopWatch.log('filteringDone');
     stopWatch.stop();
-    return FileLoadResult(deduplicatedFiles, files.length == limit);
+    return FileLoadResult(filteredFiles, files.length == limit);
   }
 
   Future<FileLoadResult> getAllLocalAndUploadedFiles(
@@ -554,7 +552,7 @@ class FilesDB {
     int ownerID, {
     int? limit,
     bool? asc,
-    Set<int>? ignoredCollectionIDs,
+    required DBFilterOptions filterOptions,
   }) async {
     final db = await instance.database;
     final order = (asc ?? false ? 'ASC' : 'DESC');
@@ -569,9 +567,8 @@ class FilesDB {
       limit: limit,
     );
     final files = convertToFiles(results);
-    final List<File> deduplicatedFiles =
-        _deduplicatedAndFilterIgnoredFiles(files, ignoredCollectionIDs);
-    return FileLoadResult(deduplicatedFiles, files.length == limit);
+    final List<File> filteredFiles = await applyDBFilters(files, filterOptions);
+    return FileLoadResult(filteredFiles, files.length == limit);
   }
 
   List<File> deduplicateByLocalID(List<File> files) {
@@ -591,43 +588,6 @@ class FilesDB {
     return deduplicatedFiles;
   }
 
-  List<File> _deduplicatedAndFilterIgnoredFiles(
-    List<File> files,
-    Set<int>? ignoredCollectionIDs,
-  ) {
-    final Set<int> uploadedFileIDs = <int>{};
-    // ignoredFileUploadIDs is to keep a track of files which are part of
-    // archived collection
-    final Set<int> ignoredFileUploadIDs = <int>{};
-    final List<File> deduplicatedFiles = [];
-    for (final file in files) {
-      final id = file.uploadedFileID;
-      final bool isFileUploaded = id != null && id != -1;
-      final bool isCollectionIgnored = ignoredCollectionIDs != null &&
-          ignoredCollectionIDs.contains(file.collectionID);
-      if (isCollectionIgnored || ignoredFileUploadIDs.contains(id)) {
-        if (isFileUploaded) {
-          ignoredFileUploadIDs.add(id);
-          // remove the file from the list of deduplicated files
-          if (uploadedFileIDs.contains(id)) {
-            deduplicatedFiles
-                .removeWhere((element) => element.uploadedFileID == id);
-            uploadedFileIDs.remove(id);
-          }
-        }
-        continue;
-      }
-      if (isFileUploaded && uploadedFileIDs.contains(id)) {
-        continue;
-      }
-      if (isFileUploaded) {
-        uploadedFileIDs.add(id);
-      }
-      deduplicatedFiles.add(file);
-    }
-    return deduplicatedFiles;
-  }
-
   Future<FileLoadResult> getFilesInCollection(
     int collectionID,
     int startTime,
@@ -699,7 +659,8 @@ class FilesDB {
       limit: limit,
     );
     final files = convertToFiles(results);
-    final dedupeResult = _deduplicatedAndFilterIgnoredFiles(files, {});
+    final dedupeResult =
+        await applyDBFilters(files, DBFilterOptions.dedupeOption);
     _logger.info("Fetched " + dedupeResult.length.toString() + " files");
     return FileLoadResult(files, files.length == limit);
   }
@@ -731,7 +692,10 @@ class FilesDB {
       orderBy: '$columnCreationTime ' + order,
     );
     final files = convertToFiles(results);
-    return _deduplicatedAndFilterIgnoredFiles(files, ignoredCollectionIDs);
+    return applyDBFilters(
+      files,
+      DBFilterOptions(ignoredCollectionIDs: ignoredCollectionIDs),
+    );
   }
 
   // Files which user added to a collection manually but they are not
@@ -1470,22 +1434,6 @@ class FilesDB {
     return result;
   }
 
-  // For given list of localIDs and ownerID, get a list of uploaded files
-  // owned by given user
-  Future<List<File>> getFilesForLocalIDs(
-    List<String> localIDs,
-    int ownerID,
-  ) async {
-    final db = await instance.database;
-    final rows = await db.query(
-      filesTable,
-      where:
-          '$columnLocalID IN (${localIDs.map((e) => "'$e'").join(',')}) AND $columnOwnerID = ?',
-      whereArgs: [ownerID],
-    );
-    return _deduplicatedAndFilterIgnoredFiles(convertToFiles(rows), {});
-  }
-
   // For a given userID, return unique uploadedFileId for the given userID
   Future<List<int>> getUploadIDsWithMissingSize(int userId) async {
     final db = await instance.database;
@@ -1529,8 +1477,10 @@ class FilesDB {
     final List<Map<String, dynamic>> result =
         await db.query(filesTable, orderBy: '$columnCreationTime DESC');
     final List<File> files = convertToFiles(result);
-    final List<File> deduplicatedFiles =
-        _deduplicatedAndFilterIgnoredFiles(files, collectionsToIgnore);
+    final List<File> deduplicatedFiles = await applyDBFilters(
+      files,
+      DBFilterOptions(ignoredCollectionIDs: collectionsToIgnore),
+    );
     return deduplicatedFiles;
   }
 
@@ -1554,7 +1504,7 @@ class FilesDB {
     int endTime, {
     int? limit,
     bool? asc,
-    Set<int>? ignoredCollectionIDs,
+    required DBFilterOptions? filterOptions,
   }) async {
     final db = await instance.database;
     final order = (asc ?? false ? 'ASC' : 'DESC');
@@ -1570,9 +1520,8 @@ class FilesDB {
       limit: limit,
     );
     final files = convertToFiles(results);
-    final List<File> deduplicatedFiles =
-        _deduplicatedAndFilterIgnoredFiles(files, ignoredCollectionIDs);
-    return FileLoadResult(deduplicatedFiles, files.length == limit);
+    final List<File> filteredFiles = await applyDBFilters(files, filterOptions);
+    return FileLoadResult(filteredFiles, files.length == limit);
   }
 
   Map<String, dynamic> _getRowForFile(File file) {

lib/services/filter/collection_ignore.dart

@@ -0,0 +1,30 @@
+import "package:photos/models/file.dart";
+import "package:photos/services/filter/filter.dart";
+
+class CollectionsIgnoreFilter extends Filter {
+  final Set<int> collectionIDs;
+
+  Set<int>? _ignoredUploadIDs;
+
+  CollectionsIgnoreFilter(this.collectionIDs, List<File> files) : super() {
+    init(files);
+  }
+
+  void init(List<File> files) {
+    _ignoredUploadIDs = {};
+    if (collectionIDs.isEmpty) return;
+    for (var file in files) {
+      if (file.collectionID != null &&
+          file.isUploaded &&
+          collectionIDs.contains(file.collectionID!)) {
+        _ignoredUploadIDs!.add(file.uploadedFileID!);
+      }
+    }
+  }
+
+  @override
+  bool filter(File file) {
+    return file.isUploaded &&
+        !_ignoredUploadIDs!.contains(file.uploadedFileID!);
+  }
+}

lib/services/filter/db_filters.dart

@@ -0,0 +1,58 @@
+import "package:photos/models/file.dart";
+import "package:photos/services/filter/collection_ignore.dart";
+import "package:photos/services/filter/dedupe_by_upload_id.dart";
+import "package:photos/services/filter/filter.dart";
+import "package:photos/services/filter/upload_ignore.dart";
+import "package:photos/services/ignored_files_service.dart";
+
+class DBFilterOptions {
+  // typically used for filtering out all files which are present in hidden
+  // (searchable files result) or archived collections or both (ex: home
+  // timeline)
+  Set<int>? ignoredCollectionIDs;
+  bool dedupeUploadID;
+  bool hideIgnoredForUpload;
+
+  DBFilterOptions({
+    this.ignoredCollectionIDs,
+    this.hideIgnoredForUpload = false,
+    this.dedupeUploadID = true,
+  });
+
+  static DBFilterOptions dedupeOption = DBFilterOptions(
+    dedupeUploadID: true,
+  );
+}
+
+Future<List<File>> applyDBFilters(
+  List<File> files,
+  DBFilterOptions? options,
+) async {
+  if (options == null) {
+    return files;
+  }
+  final List<Filter> filters = [];
+  if (options.hideIgnoredForUpload) {
+    final Set<String> ignoredIDs =
+        await IgnoredFilesService.instance.ignoredIDs;
+    if (ignoredIDs.isNotEmpty) {
+      filters.add(UploadIgnoreFilter(ignoredIDs));
+    }
+  }
+  if (options.dedupeUploadID) {
+    filters.add(DedupeUploadIDFilter());
+  }
+  if (options.ignoredCollectionIDs != null &&
+      options.ignoredCollectionIDs!.isNotEmpty) {
+    final collectionIgnoreFilter =
+        CollectionsIgnoreFilter(options.ignoredCollectionIDs!, files);
+    filters.add(collectionIgnoreFilter);
+  }
+  final List<File> filterFiles = [];
+  for (final file in files) {
+    if (filters.every((f) => f.filter(file))) {
+      filterFiles.add(file);
+    }
+  }
+  return filterFiles;
+}

lib/services/filter/dedupe_by_upload_id.dart

@@ -0,0 +1,20 @@
+import "package:photos/models/file.dart";
+import "package:photos/services/filter/filter.dart";
+
+// DedupeUploadIDFilter will filter out files where were previously filtered
+// during the same filtering session
+class DedupeUploadIDFilter extends Filter {
+  final Set<int> trackedUploadIDs = {};
+
+  @override
+  bool filter(File file) {
+    if (!file.isUploaded) {
+      return true;
+    }
+    if (trackedUploadIDs.contains(file.uploadedFileID!)) {
+      return false;
+    }
+    trackedUploadIDs.add(file.uploadedFileID!);
+    return true;
+  }
+}

lib/services/filter/filter.dart

@@ -0,0 +1,5 @@
+import "package:photos/models/file.dart";
+
+abstract class Filter {
+  bool filter(File file);
+}

lib/services/filter/type_filter.dart

@@ -0,0 +1,18 @@
+import "package:photos/models/file.dart";
+import "package:photos/models/file_type.dart";
+import "package:photos/services/filter/filter.dart";
+
+class TypeFilter extends Filter {
+  final FileType type;
+  final bool reverse;
+
+  TypeFilter(
+    this.type, {
+    this.reverse = false,
+  });
+
+  @override
+  bool filter(File file) {
+    return reverse ? file.fileType != type : file.fileType == type;
+  }
+}

lib/services/filter/upload_ignore.dart

@@ -0,0 +1,18 @@
+import "package:photos/models/file.dart";
+import "package:photos/services/filter/filter.dart";
+import "package:photos/services/ignored_files_service.dart";
+
+// UploadIgnoreFilter hides the unuploaded files that are ignored from for
+// upload
+class UploadIgnoreFilter extends Filter {
+  Set<String> ignoredIDs;
+
+  UploadIgnoreFilter(this.ignoredIDs) : super();
+
+  @override
+  bool filter(File file) {
+    // Already uploaded files pass the filter
+    if (file.isUploaded) return true;
+    return !IgnoredFilesService.instance.shouldSkipUpload(ignoredIDs, file);
+  }
+}

lib/ui/home/home_gallery_widget.dart

@@ -10,7 +10,7 @@ import 'package:photos/models/file_load_result.dart';
 import 'package:photos/models/gallery_type.dart';
 import 'package:photos/models/selected_files.dart';
 import 'package:photos/services/collections_service.dart';
-import 'package:photos/services/ignored_files_service.dart';
+import "package:photos/services/filter/db_filters.dart";
 import 'package:photos/ui/viewer/actions/file_selection_overlay_bar.dart';
 import 'package:photos/ui/viewer/gallery/gallery.dart';
 
@@ -37,6 +37,11 @@ class HomeGalleryWidget extends StatelessWidget {
         final collectionsToHide =
             CollectionsService.instance.archivedOrHiddenCollections();
         FileLoadResult result;
+        final DBFilterOptions filterOptions = DBFilterOptions(
+          hideIgnoredForUpload: true,
+          dedupeUploadID: true,
+          ignoredCollectionIDs: collectionsToHide,
+        );
         if (hasSelectedAllForBackup) {
           result = await FilesDB.instance.getAllLocalAndUploadedFiles(
             creationStartTime,
@@ -44,7 +49,7 @@ class HomeGalleryWidget extends StatelessWidget {
             ownerID!,
             limit: limit,
             asc: asc,
-            ignoredCollectionIDs: collectionsToHide,
+            filterOptions: filterOptions,
           );
         } else {
           result = await FilesDB.instance.getAllPendingOrUploadedFiles(
@@ -53,17 +58,10 @@ class HomeGalleryWidget extends StatelessWidget {
             ownerID!,
             limit: limit,
             asc: asc,
-            ignoredCollectionIDs: collectionsToHide,
+            filterOptions: filterOptions,
           );
         }
 
-        // hide ignored files from home page UI
-        final ignoredIDs = await IgnoredFilesService.instance.ignoredIDs;
-        result.files.removeWhere(
-          (f) =>
-              f.uploadedFileID == null &&
-              IgnoredFilesService.instance.shouldSkipUpload(ignoredIDs, f),
-        );
         return result;
       },
       reloadEvent: Bus.instance.on<LocalPhotosUpdatedEvent>(),

lib/ui/viewer/gallery/archive_page.dart

@@ -9,6 +9,7 @@ import 'package:photos/models/gallery_type.dart';
 import "package:photos/models/metadata/common_keys.dart";
 import 'package:photos/models/selected_files.dart';
 import 'package:photos/services/collections_service.dart';
+import "package:photos/services/filter/db_filters.dart";
 import "package:photos/ui/collections/album/horizontal_list.dart";
 import 'package:photos/ui/viewer/actions/file_selection_overlay_bar.dart';
 import "package:photos/ui/viewer/gallery/empty_state.dart";
@@ -41,7 +42,11 @@ class ArchivePage extends StatelessWidget {
           visibility: archiveVisibility,
           limit: limit,
           asc: asc,
-          ignoredCollectionIDs: hiddenCollectionIDs,
+          filterOptions: DBFilterOptions(
+            hideIgnoredForUpload: true,
+            dedupeUploadID: true,
+            ignoredCollectionIDs: hiddenCollectionIDs,
+          ),
           applyOwnerCheck: true,
         );
       },

lib/ui/viewer/location/dynamic_location_gallery_widget.dart

@@ -7,7 +7,7 @@ import "package:photos/db/files_db.dart";
 import "package:photos/models/file.dart";
 import "package:photos/models/file_load_result.dart";
 import "package:photos/services/collections_service.dart";
-import "package:photos/services/files_service.dart";
+import "package:photos/services/filter/db_filters.dart";
 import "package:photos/services/location_service.dart";
 import 'package:photos/states/location_state.dart';
 import "package:photos/ui/viewer/gallery/gallery.dart";
@@ -32,7 +32,6 @@ class DynamicLocationGalleryWidget extends StatefulWidget {
 class _DynamicLocationGalleryWidgetState
     extends State<DynamicLocationGalleryWidget> {
   late final Future<FileLoadResult> fileLoadResult;
-  late Future<void> removeIgnoredFiles;
   double heightOfGallery = 0;
 
   @override
@@ -45,10 +44,12 @@ class _DynamicLocationGalleryWidgetState
       galleryLoadEndTime,
       limit: null,
       asc: false,
-      ignoredCollectionIDs: collectionsToHide,
+      filterOptions: DBFilterOptions(
+        ignoredCollectionIDs: collectionsToHide,
+        hideIgnoredForUpload: true,
+      ),
     );
-    removeIgnoredFiles =
-        FilesService.instance.removeIgnoredFiles(fileLoadResult);
+
     super.initState();
   }
 
@@ -58,8 +59,6 @@ class _DynamicLocationGalleryWidgetState
     final selectedRadius = InheritedLocationTagData.of(context).selectedRadius;
     Future<FileLoadResult> filterFiles() async {
       final FileLoadResult result = await fileLoadResult;
-      //wait for ignored files to be removed after init
-      await removeIgnoredFiles;
       final stopWatch = Stopwatch()..start();
       final copyOfFiles = List<File>.from(result.files);
       copyOfFiles.removeWhere((f) {

lib/ui/viewer/location/location_screen.dart

@@ -13,7 +13,7 @@ import "package:photos/models/file_load_result.dart";
 import "package:photos/models/gallery_type.dart";
 import "package:photos/models/selected_files.dart";
 import "package:photos/services/collections_service.dart";
-import "package:photos/services/files_service.dart";
+import "package:photos/services/filter/db_filters.dart";
 import "package:photos/services/location_service.dart";
 import "package:photos/states/location_screen_state.dart";
 import "package:photos/theme/colors.dart";
@@ -134,7 +134,7 @@ class LocationGalleryWidget extends StatefulWidget {
 
 class _LocationGalleryWidgetState extends State<LocationGalleryWidget> {
   late final Future<FileLoadResult> fileLoadResult;
-  late Future<void> removeIgnoredFiles;
+
   late Widget galleryHeaderWidget;
   final _selectedFiles = SelectedFiles();
   @override
@@ -147,10 +147,11 @@ class _LocationGalleryWidgetState extends State<LocationGalleryWidget> {
       galleryLoadEndTime,
       limit: null,
       asc: false,
-      ignoredCollectionIDs: collectionsToHide,
+      filterOptions: DBFilterOptions(
+        ignoredCollectionIDs: collectionsToHide,
+        hideIgnoredForUpload: true,
+      ),
     );
-    removeIgnoredFiles =
-        FilesService.instance.removeIgnoredFiles(fileLoadResult);
     galleryHeaderWidget = const GalleryHeaderWidget();
     super.initState();
   }
@@ -172,7 +173,6 @@ class _LocationGalleryWidgetState extends State<LocationGalleryWidget> {
     Future<FileLoadResult> filterFiles() async {
       final FileLoadResult result = await fileLoadResult;
       //wait for ignored files to be removed after init
-      await removeIgnoredFiles;
       final stopWatch = Stopwatch()..start();
       final copyOfFiles = List<File>.from(result.files);
       copyOfFiles.removeWhere((f) {

lib/ui/viewer/location/pick_center_point_widget.dart

@@ -13,7 +13,7 @@ import "package:photos/models/local_entity_data.dart";
 import "package:photos/models/location_tag/location_tag.dart";
 import "package:photos/models/selected_files.dart";
 import "package:photos/services/collections_service.dart";
-import "package:photos/services/ignored_files_service.dart";
+import "package:photos/services/filter/db_filters.dart";
 import "package:photos/theme/colors.dart";
 import "package:photos/theme/ente_theme.dart";
 import "package:photos/ui/components/bottom_of_title_bar_widget.dart";
@@ -101,17 +101,10 @@ class PickCenterPointWidget extends StatelessWidget {
                                 galleryLoadEndTime,
                                 limit: null,
                                 asc: false,
-                                ignoredCollectionIDs: collectionsToHide,
-                              );
-
-                              // hide ignored files from UI
-                              final ignoredIDs =
-                                  await IgnoredFilesService.instance.ignoredIDs;
-                              result.files.removeWhere(
-                                (f) =>
-                                    f.uploadedFileID == null &&
-                                    IgnoredFilesService.instance
-                                        .shouldSkipUpload(ignoredIDs, f),
+                                filterOptions: DBFilterOptions(
+                                  ignoredCollectionIDs: collectionsToHide,
+                                  hideIgnoredForUpload: true,
+                                ),
                               );
                               return result;
                             },