From 6d5c80fef615cc7cb12fe0cbc015ac9e8a9b656e Mon Sep 17 00:00:00 2001 From: vishnukvmd Date: Thu, 23 Sep 2021 18:42:50 +0530 Subject: [PATCH 1/6] Remove unnecessary optimization --- lib/utils/diff_fetcher.dart | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/lib/utils/diff_fetcher.dart b/lib/utils/diff_fetcher.dart index f0bff9cc6..98f9330a0 100644 --- a/lib/utils/diff_fetcher.dart +++ b/lib/utils/diff_fetcher.dart @@ -49,21 +49,13 @@ class DiffFetcher { if (existingFiles.contains(file.uploadedFileID)) { await FilesDB.instance.deleteFromCollection( file.uploadedFileID, file.collectionID); - Bus.instance.fire( - CollectionUpdatedEvent(file.collectionID, [file])); + Bus.instance + .fire(CollectionUpdatedEvent(file.collectionID, [file])); Bus.instance.fire(LocalPhotosUpdatedEvent([file])); } continue; } file.updationTime = item["updationTime"]; - if (existingFiles.contains(file.uploadedFileID)) { - final existingFile = await FilesDB.instance - .getUploadedFile(file.uploadedFileID, file.collectionID); - if (existingFile != null && - existingFile.updationTime == file.updationTime) { - continue; - } - } file.ownerID = item["ownerID"]; file.encryptedKey = item["encryptedKey"]; file.keyDecryptionNonce = item["keyDecryptionNonce"]; @@ -84,11 +76,13 @@ class DiffFetcher { file.applyMetadata(metadata); if (item['magicMetadata'] != null) { final utfEncodedMmd = await CryptoUtil.decryptChaCha( - Sodium.base642bin(item['magicMetadata']['data']), fileDecryptionKey, + Sodium.base642bin(item['magicMetadata']['data']), + fileDecryptionKey, Sodium.base642bin(item['magicMetadata']['header'])); file.mMdEncodedJson = utf8.decode(utfEncodedMmd); file.mMdVersion = item['magicMetadata']['version']; - file.magicMetadata = MagicMetadata.fromEncodedJson(file.mMdEncodedJson); + file.magicMetadata = + MagicMetadata.fromEncodedJson(file.mMdEncodedJson); } files.add(file); } From 283ac899b2721016774d510d45ca06d6e3f43a3e Mon Sep 17 00:00:00 2001 From: vishnukvmd Date: Thu, 23 Sep 2021 19:18:24 +0530 Subject: [PATCH 2/6] Force pull since the time archive was released --- lib/services/collections_service.dart | 7 ++- lib/services/remote_sync_service.dart | 68 +++++++++++++++++++++------ 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/lib/services/collections_service.dart b/lib/services/collections_service.dart index 53ea6147e..4e7ba3b49 100644 --- a/lib/services/collections_service.dart +++ b/lib/services/collections_service.dart @@ -233,7 +233,8 @@ class CollectionsService { Future rename(Collection collection, String newName) async { try { - final encryptedName = CryptoUtil.encryptSync(utf8.encode(newName), getCollectionKey(collection.id)); + final encryptedName = CryptoUtil.encryptSync( + utf8.encode(newName), getCollectionKey(collection.id)); await _dio.post( Configuration.instance.getHttpEndpoint() + "/collections/rename", data: { @@ -471,6 +472,10 @@ class CollectionsService { Sodium.base642bin(collection.attributes.pathDecryptionNonce))); } + bool hasSyncedCollections() { + return _prefs.containsKey(_collectionsSyncTimeKey); + } + Collection _getCollectionWithDecryptedName(Collection collection) { if (collection.encryptedName != null && collection.encryptedName.isNotEmpty) { diff --git a/lib/services/remote_sync_service.dart b/lib/services/remote_sync_service.dart index 592c57a1a..40eb85c86 100644 --- a/lib/services/remote_sync_service.dart +++ b/lib/services/remote_sync_service.dart @@ -1,5 +1,6 @@ import 'dart:async'; import 'dart:io'; +import 'dart:math'; import 'package:logging/logging.dart'; import 'package:photos/core/configuration.dart'; @@ -16,6 +17,7 @@ import 'package:photos/services/local_sync_service.dart'; import 'package:photos/utils/diff_fetcher.dart'; import 'package:photos/utils/file_uploader.dart'; import 'package:photos/utils/file_util.dart'; +import 'package:shared_preferences/shared_preferences.dart'; class RemoteSyncService { final _logger = Logger("RemoteSyncService"); @@ -24,22 +26,46 @@ class RemoteSyncService { final _collectionsService = CollectionsService.instance; final _diffFetcher = DiffFetcher(); int _completedUploads = 0; + SharedPreferences _prefs; static const kDiffLimit = 2500; + static const kHasSyncedArchiveKey = "has_synced_archive"; + static const kArchiveFeatureReleaseTime = 1632100000000000; static final RemoteSyncService instance = RemoteSyncService._privateConstructor(); RemoteSyncService._privateConstructor(); - Future init() async {} + Future init() async { + _prefs = await SharedPreferences.getInstance(); + } Future sync({bool silently = false}) async { if (!Configuration.instance.hasConfiguredAccount()) { _logger.info("Skipping remote sync since account is not configured"); return; } + + bool isFirstSync = !_collectionsService.hasSyncedCollections(); await _collectionsService.sync(); + + if (isFirstSync || _hasSyncedArchive()) { + await _syncUpdatedCollections(silently); + } else { + await _resyncAllCollectionsSinceTime(kArchiveFeatureReleaseTime); + } + if (!_hasSyncedArchive()) { + await _markArchiveAsSynced(); + } + + bool hasUploadedFiles = await _uploadDiff(); + if (hasUploadedFiles) { + sync(silently: true); + } + } + + Future _syncUpdatedCollections(bool silently) async { final updatedCollections = await _collectionsService.getCollectionsToBeSynced(); @@ -47,21 +73,24 @@ class RemoteSyncService { Bus.instance.fire(SyncStatusUpdate(SyncStatus.applying_remote_diff)); } for (final c in updatedCollections) { - await _syncCollectionDiff(c.id); - _collectionsService.setCollectionSyncTime(c.id, c.updationTime); - } - bool hasUploadedFiles = await _uploadDiff(); - if (hasUploadedFiles) { - sync(silently: true); + await _syncCollectionDiff( + c.id, _collectionsService.getCollectionSyncTime(c.id)); + await _collectionsService.setCollectionSyncTime(c.id, c.updationTime); } } - Future _syncCollectionDiff(int collectionID) async { + Future _resyncAllCollectionsSinceTime(int sinceTime) async { + final collections = _collectionsService.getCollections(); + for (final c in collections) { + await _syncCollectionDiff(c.id, + min(_collectionsService.getCollectionSyncTime(c.id), sinceTime)); + await _collectionsService.setCollectionSyncTime(c.id, c.updationTime); + } + } + + Future _syncCollectionDiff(int collectionID, int sinceTime) async { final diff = await _diffFetcher.getEncryptedFilesDiff( - collectionID, - _collectionsService.getCollectionSyncTime(collectionID), - kDiffLimit, - ); + collectionID, sinceTime, kDiffLimit); if (diff.updatedFiles.isNotEmpty) { await _storeDiff(diff.updatedFiles, collectionID); _logger.info("Updated " + @@ -71,9 +100,10 @@ class RemoteSyncService { Bus.instance.fire(LocalPhotosUpdatedEvent(diff.updatedFiles)); Bus.instance .fire(CollectionUpdatedEvent(collectionID, diff.updatedFiles)); - if (diff.fetchCount == kDiffLimit) { - return await _syncCollectionDiff(collectionID); - } + } + if (diff.fetchCount == kDiffLimit) { + return await _syncCollectionDiff(collectionID, + _collectionsService.getCollectionSyncTime(collectionID)); } } @@ -253,4 +283,12 @@ class RemoteSyncService { " was uploaded from device but added to a new collection on remote.", ); } + + bool _hasSyncedArchive() { + return _prefs.containsKey(kHasSyncedArchiveKey); + } + + Future _markArchiveAsSynced() { + return _prefs.setBool(kHasSyncedArchiveKey, true); + } } From f5643b2f1c88ab0b5b59e258b4b7c55b488190f0 Mon Sep 17 00:00:00 2001 From: vishnukvmd Date: Thu, 23 Sep 2021 20:25:08 +0530 Subject: [PATCH 3/6] Dedupe existing files fetched from remote --- lib/utils/diff_fetcher.dart | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/utils/diff_fetcher.dart b/lib/utils/diff_fetcher.dart index 98f9330a0..a853ef352 100644 --- a/lib/utils/diff_fetcher.dart +++ b/lib/utils/diff_fetcher.dart @@ -55,6 +55,13 @@ class DiffFetcher { } continue; } + if (existingFiles.contains(file.uploadedFileID)) { + final existingFile = await FilesDB.instance + .getUploadedFile(file.uploadedFileID, file.collectionID); + if (existingFile != null) { + file.generatedID = existingFile.generatedID; + } + } file.updationTime = item["updationTime"]; file.ownerID = item["ownerID"]; file.encryptedKey = item["encryptedKey"]; From 511b129d40f768522340c67acc231ec300a9be32 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 23 Sep 2021 20:54:20 +0530 Subject: [PATCH 4/6] Skip files that are already part of the collection --- lib/services/collections_service.dart | 37 +++++++++++++++++++-------- lib/ui/create_collection_page.dart | 1 - 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/lib/services/collections_service.dart b/lib/services/collections_service.dart index 4e7ba3b49..41dfbf6ac 100644 --- a/lib/services/collections_service.dart +++ b/lib/services/collections_service.dart @@ -331,7 +331,22 @@ class CollectionsService { return collection; } - Future addToCollection(int collectionID, List files) { + Future addToCollection(int collectionID, List files) async { + final shouldDedupeWithExistingUploadIDs = files.firstWhere( + (element) => element.uploadedFileID != null, + orElse: () => null) != null; + if (shouldDedupeWithExistingUploadIDs) { + final existingUploadedIDs = + await FilesDB.instance.getUploadedFileIDs(collectionID); + files.removeWhere((element) => + element.uploadedFileID != null && + existingUploadedIDs.contains(element.uploadedFileID)); + if (files.isEmpty) { + _logger.info("nothing to add to the collection"); + return; + } + } + final params = {}; params["collectionID"] = collectionID; for (final file in files) { @@ -349,17 +364,19 @@ class CollectionsService { file.uploadedFileID, file.encryptedKey, file.keyDecryptionNonce) .toMap()); } - return _dio - .post( - Configuration.instance.getHttpEndpoint() + "/collections/add-files", - data: params, - options: - Options(headers: {"X-Auth-Token": Configuration.instance.getToken()}), - ) - .then((value) async { + + try { + await _dio.post( + Configuration.instance.getHttpEndpoint() + "/collections/add-files", + data: params, + options: Options( + headers: {"X-Auth-Token": Configuration.instance.getToken()}), + ); await _filesDB.insertMultiple(files); Bus.instance.fire(CollectionUpdatedEvent(collectionID, files)); - }); + } catch (e) { + rethrow; + } } Future move( diff --git a/lib/ui/create_collection_page.dart b/lib/ui/create_collection_page.dart index 3164573b6..136fc157b 100644 --- a/lib/ui/create_collection_page.dart +++ b/lib/ui/create_collection_page.dart @@ -272,7 +272,6 @@ class _CreateCollectionPageState extends State { await FilesDB.instance.insertMultiple(localFiles); } else { for (final file in widget.selectedFiles.files) { - await FilesDB.instance.insert(file); final currentFile = await FilesDB.instance.getFile(file.generatedID); if (currentFile.uploadedFileID == null) { final uploadedFile = (await FileUploader.instance From 98022ff334a271081840421c3bce98e378adcd5a Mon Sep 17 00:00:00 2001 From: vishnukvmd Date: Fri, 24 Sep 2021 11:33:46 +0530 Subject: [PATCH 5/6] Ignore files during move if they are already present in the target collection --- lib/services/collections_service.dart | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/services/collections_service.dart b/lib/services/collections_service.dart index 41dfbf6ac..339018ca5 100644 --- a/lib/services/collections_service.dart +++ b/lib/services/collections_service.dart @@ -333,8 +333,9 @@ class CollectionsService { Future addToCollection(int collectionID, List files) async { final shouldDedupeWithExistingUploadIDs = files.firstWhere( - (element) => element.uploadedFileID != null, - orElse: () => null) != null; + (element) => element.uploadedFileID != null, + orElse: () => null) != + null; if (shouldDedupeWithExistingUploadIDs) { final existingUploadedIDs = await FilesDB.instance.getUploadedFileIDs(collectionID); @@ -380,8 +381,17 @@ class CollectionsService { } Future move( - int toCollectionID, int fromCollectionID, List files) { + int toCollectionID, int fromCollectionID, List files) async { _validateMoveRequest(toCollectionID, fromCollectionID, files); + final existingUploadedIDs = + await FilesDB.instance.getUploadedFileIDs(toCollectionID); + files.removeWhere((element) => + element.uploadedFileID != null && + existingUploadedIDs.contains(element.uploadedFileID)); + if (files.isEmpty) { + _logger.info("nothing to move to collection"); + return; + } final params = {}; params["toCollectionID"] = toCollectionID; params["fromCollectionID"] = fromCollectionID; From c974befed5dfcaa19703ac45fef76a0b7c9aba92 Mon Sep 17 00:00:00 2001 From: vishnukvmd Date: Fri, 24 Sep 2021 11:35:13 +0530 Subject: [PATCH 6/6] Minor refactor --- lib/services/collections_service.dart | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/services/collections_service.dart b/lib/services/collections_service.dart index 339018ca5..b4e3b15cb 100644 --- a/lib/services/collections_service.dart +++ b/lib/services/collections_service.dart @@ -332,16 +332,16 @@ class CollectionsService { } Future addToCollection(int collectionID, List files) async { - final shouldDedupeWithExistingUploadIDs = files.firstWhere( + final containsUploadedFile = files.firstWhere( (element) => element.uploadedFileID != null, orElse: () => null) != null; - if (shouldDedupeWithExistingUploadIDs) { - final existingUploadedIDs = + if (containsUploadedFile) { + final existingFileIDsInCollection = await FilesDB.instance.getUploadedFileIDs(collectionID); files.removeWhere((element) => element.uploadedFileID != null && - existingUploadedIDs.contains(element.uploadedFileID)); + existingFileIDsInCollection.contains(element.uploadedFileID)); if (files.isEmpty) { _logger.info("nothing to add to the collection"); return;