From 113d3e57098cc3e6b78159556e41f5fb28bf334a Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 23 May 2023 16:13:07 +0530 Subject: [PATCH 1/3] Perf: Avoid reading entire collections while sync diff --- lib/db/collections_db.dart | 18 ++++++++++++++++++ lib/services/collections_service.dart | 17 ++++++++++------- lib/services/remote_sync_service.dart | 17 +++++++++++------ 3 files changed, 39 insertions(+), 13 deletions(-) diff --git a/lib/db/collections_db.dart b/lib/db/collections_db.dart index f69820246..b66c01704 100644 --- a/lib/db/collections_db.dart +++ b/lib/db/collections_db.dart @@ -186,6 +186,24 @@ class CollectionsDB { return collections; } + // getActiveCollectionIDsAndUpdationTime returns map of collectionID to + // updationTime for non-deleted collections + Future> getActiveIDsAndRemoteUpdateTime() async { + final db = await instance.database; + final rows = await db.query( + table, + where: '($columnIsDeleted = ? OR $columnIsDeleted IS NULL)', + whereArgs: [_sqlBoolFalse], + columns: [columnID, columnUpdationTime], + ); + final collectionIDsAndUpdationTime = {}; + for (final row in rows) { + collectionIDsAndUpdationTime[row[columnID] as int] = + int.parse(row[columnUpdationTime] as String); + } + return collectionIDsAndUpdationTime; + } + Future deleteCollection(int collectionID) async { final db = await instance.database; return db.delete( diff --git a/lib/services/collections_service.dart b/lib/services/collections_service.dart index 5be3ef487..b7c779297 100644 --- a/lib/services/collections_service.dart +++ b/lib/services/collections_service.dart @@ -165,15 +165,18 @@ class CollectionsService { _cachedKeys.clear(); } - Future> getCollectionsToBeSynced() async { - final collections = await _db.getAllCollections(); - final updatedCollections = []; - for (final c in collections) { - if (c.updationTime > getCollectionSyncTime(c.id) && !c.isDeleted) { - updatedCollections.add(c); + Future> getCollectionIDsToBeSynced() async { + final idsToRemoveUpdateTimeMap = + await _db.getActiveIDsAndRemoteUpdateTime(); + final result = {}; + for (final MapEntry e in idsToRemoveUpdateTimeMap.entries) { + final int cid = e.key; + final int remoteUpdateTime = e.value; + if (remoteUpdateTime > getCollectionSyncTime(cid)) { + result[cid] = remoteUpdateTime; } } - return updatedCollections; + return result; } Set getArchivedCollections() { diff --git a/lib/services/remote_sync_service.dart b/lib/services/remote_sync_service.dart index 5de50955a..dba521ed0 100644 --- a/lib/services/remote_sync_service.dart +++ b/lib/services/remote_sync_service.dart @@ -181,14 +181,19 @@ class RemoteSyncService { } Future _syncUpdatedCollections() async { - final updatedCollections = - await _collectionsService.getCollectionsToBeSynced(); - for (final c in updatedCollections) { + final idsToRemoteUpdationTimeMap = + await _collectionsService.getCollectionIDsToBeSynced(); + for (final cid in idsToRemoteUpdationTimeMap.keys) { await _syncCollectionDiff( - c.id, - _collectionsService.getCollectionSyncTime(c.id), + cid, + _collectionsService.getCollectionSyncTime(cid), ); - await _collectionsService.setCollectionSyncTime(c.id, c.updationTime); + // update syncTime for the collection in sharedPrefs. Note: the + // syncTime can change on remote but we might not get a diff for the + // collection if there are not changes in the file, but the collection + // metadata (name, archive status, sharing etc) has changed. + final remoteUpdateTime = idsToRemoteUpdationTimeMap[cid]; + await _collectionsService.setCollectionSyncTime(cid, remoteUpdateTime); } _logger.info("All updated collections synced"); } From 6d8d4aeddf7bca8c0a875a1724270bd751139dbf Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 23 May 2023 16:57:35 +0530 Subject: [PATCH 2/3] CollectionDiff: Avoid redundant event for delete from delete --- lib/services/remote_sync_service.dart | 43 ++++++++++++++++----------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/lib/services/remote_sync_service.dart b/lib/services/remote_sync_service.dart index dba521ed0..8c29e0621 100644 --- a/lib/services/remote_sync_service.dart +++ b/lib/services/remote_sync_service.dart @@ -221,24 +221,7 @@ class RemoteSyncService { final diff = await _diffFetcher.getEncryptedFilesDiff(collectionID, sinceTime); if (diff.deletedFiles.isNotEmpty) { - final fileIDs = diff.deletedFiles.map((f) => f.uploadedFileID!).toList(); - final deletedFiles = (await _db.getFilesFromIDs(fileIDs)).values.toList(); - await _db.deleteFilesFromCollection(collectionID, fileIDs); - Bus.instance.fire( - CollectionUpdatedEvent( - collectionID, - deletedFiles, - "syncDeleteFromRemote", - type: EventType.deletedFromRemote, - ), - ); - Bus.instance.fire( - LocalPhotosUpdatedEvent( - deletedFiles, - type: EventType.deletedFromRemote, - source: "syncDeleteFromRemote", - ), - ); + await _syncCollectionDiffDelete(diff, collectionID); } if (diff.updatedFiles.isNotEmpty) { await _storeDiff(diff.updatedFiles, collectionID); @@ -279,6 +262,30 @@ class RemoteSyncService { } } + Future _syncCollectionDiffDelete(Diff diff, int collectionID) async { + final fileIDs = diff.deletedFiles.map((f) => f.uploadedFileID!).toList(); + final deletedFiles = (await _db.getFilesFromIDs(fileIDs)).values.toList(); + final localDeleteCount = + await _db.deleteFilesFromCollection(collectionID, fileIDs); + if (localDeleteCount > 0) { + Bus.instance.fire( + CollectionUpdatedEvent( + collectionID, + deletedFiles, + "syncDeleteFromRemote", + type: EventType.deletedFromRemote, + ), + ); + Bus.instance.fire( + LocalPhotosUpdatedEvent( + deletedFiles, + type: EventType.deletedFromRemote, + source: "syncDeleteFromRemote", + ), + ); + } + } + Future syncDeviceCollectionFilesForUpload() async { _logger.info("Syncing device collections to be uploaded"); final int ownerID = _config.getUserID()!; From 4c6ba5c851a1e3ef4b90b7cbdc620a5ba5d336bf Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 23 May 2023 16:59:50 +0530 Subject: [PATCH 3/3] Fix: Fire event with files for given collection --- lib/services/remote_sync_service.dart | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/services/remote_sync_service.dart b/lib/services/remote_sync_service.dart index 8c29e0621..3a272297c 100644 --- a/lib/services/remote_sync_service.dart +++ b/lib/services/remote_sync_service.dart @@ -264,21 +264,23 @@ class RemoteSyncService { Future _syncCollectionDiffDelete(Diff diff, int collectionID) async { final fileIDs = diff.deletedFiles.map((f) => f.uploadedFileID!).toList(); - final deletedFiles = (await _db.getFilesFromIDs(fileIDs)).values.toList(); final localDeleteCount = await _db.deleteFilesFromCollection(collectionID, fileIDs); if (localDeleteCount > 0) { + final collectionFiles = + (await _db.getFilesFromIDs(fileIDs)).values.toList(); + collectionFiles.removeWhere((f) => f.collectionID != collectionID); Bus.instance.fire( CollectionUpdatedEvent( collectionID, - deletedFiles, + collectionFiles, "syncDeleteFromRemote", type: EventType.deletedFromRemote, ), ); Bus.instance.fire( LocalPhotosUpdatedEvent( - deletedFiles, + collectionFiles, type: EventType.deletedFromRemote, source: "syncDeleteFromRemote", ),