beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'rewrite_device_sync' into migrate-to-null-safety

Browse source
This commit is contained in:
ashilkn 2022-09-13 15:31:45 +05:30
parent eecc664196 a7766d3b30
commit 7b049cf3b9
13 changed files with 207 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
lib/core/configuration.dart
View file

@ -11,6 +11,7 @@ import 'package:logging/logging.dart';
import 'package:path_provider/path_provider.dart';
import 'package:photos/core/constants.dart';
import 'package:photos/core/error-reporting/super_logging.dart';
import 'package:photos/core/errors.dart';
import 'package:photos/core/event_bus.dart';
import 'package:photos/db/collections_db.dart';
import 'package:photos/db/files_db.dart';
@ -257,7 +258,10 @@ class Configuration {
Sodium.base642bin(attributes.kekSalt),
attributes.memLimit,
attributes.opsLimit,
);
).onError((e, s) {
_logger.severe('deriveKey failed', e, s);
throw KeyDerivationError();
});
_logger.info('user-key done');
Uint8List key;

2
lib/core/errors.dart
View file

@ -40,3 +40,5 @@ class UnauthorizedEditError extends AssertionError {}
class InvalidStateError extends AssertionError {
InvalidStateError(String message) : super(message);
}
class KeyDerivationError extends Error {}

4
lib/db/files_db.dart
View file

@ -1164,7 +1164,9 @@ class FilesDB {
(
SELECT $columnCollectionID, MAX($columnCreationTime) AS max_creation_time
FROM $filesTable
WHERE ($columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1 AND $columnMMdVisibility = $kVisibilityVisible)
WHERE ($columnCollectionID IS NOT NULL AND $columnCollectionID IS
NOT -1 AND $columnMMdVisibility = $kVisibilityVisible AND
$columnUploadedFileID IS NOT -1)
GROUP BY $columnCollectionID
) latest_files
ON $filesTable.$columnCollectionID = latest_files.$columnCollectionID

3
lib/events/trash_updated_event.dart Normal file
View file

@ -0,0 +1,3 @@
import 'package:photos/events/event.dart';
class TrashUpdatedEvent extends Event {}

37
lib/services/collections_service.dart
View file

@ -1,5 +1,6 @@
// @dart=2.9
import 'dart:async';
import 'dart:convert';
import 'dart:math';
import 'dart:typed_data';
@ -13,6 +14,7 @@ import 'package:photos/core/errors.dart';
import 'package:photos/core/event_bus.dart';
import 'package:photos/core/network.dart';
import 'package:photos/db/collections_db.dart';
import 'package:photos/db/device_files_db.dart';
import 'package:photos/db/files_db.dart';
import 'package:photos/db/trash_db.dart';
import 'package:photos/events/collection_updated_event.dart';
@ -25,6 +27,7 @@ import 'package:photos/models/file.dart';
import 'package:photos/models/magic_metadata.dart';
import 'package:photos/services/app_lifecycle_service.dart';
import 'package:photos/services/file_magic_service.dart';
import 'package:photos/services/local_sync_service.dart';
import 'package:photos/services/remote_sync_service.dart';
import 'package:photos/utils/crypto_util.dart';
import 'package:photos/utils/file_download_util.dart';
@ -241,6 +244,40 @@ class CollectionsService {
RemoteSyncService.instance.sync(silently: true);
}
Future<void> trashCollection(Collection collection) async {
try {
final deviceCollections = await _filesDB.getDeviceCollections();
final Map<String, bool> deivcePathIDsToUnsync = Map.fromEntries(
deviceCollections
.where((e) => e.shouldBackup && e.collectionID == collection.id)
.map((e) => MapEntry(e.id, false)),
);
if (deivcePathIDsToUnsync.isNotEmpty) {
_logger.info(
'turning off backup status for folders $deivcePathIDsToUnsync',
);
await RemoteSyncService.instance
.updateDeviceFolderSyncStatus(deivcePathIDsToUnsync);
}
await _dio.delete(
Configuration.instance.getHttpEndpoint() +
"/collections/v2/${collection.id}",
options: Options(
headers: {"X-Auth-Token": Configuration.instance.getToken()},
),
);
await _filesDB.deleteCollection(collection.id);
final deletedCollection = collection.copyWith(isDeleted: true);
_collectionIDToCollections[collection.id] = deletedCollection;
_db.insert([deletedCollection]);
unawaited(LocalSyncService.instance.syncAll());
} catch (e) {
_logger.severe('failed to trash collection', e);
rethrow;
}
}
Uint8List getCollectionKey(int collectionID) {
if (!_cachedKeys.containsKey(collectionID)) {
final collection = _collectionIDToCollections[collectionID];

59
lib/services/remote_sync_service.dart
View file

@ -13,6 +13,7 @@ import 'package:photos/core/event_bus.dart';
import 'package:photos/db/device_files_db.dart';
import 'package:photos/db/file_updation_db.dart';
import 'package:photos/db/files_db.dart';
import 'package:photos/events/backup_folders_updated_event.dart';
import 'package:photos/events/collection_updated_event.dart';
import 'package:photos/events/files_updated_event.dart';
import 'package:photos/events/force_reload_home_gallery_event.dart';
@ -36,9 +37,10 @@ import 'package:shared_preferences/shared_preferences.dart';
class RemoteSyncService {
final _logger = Logger("RemoteSyncService");
final _db = FilesDB.instance;
final _uploader = FileUploader.instance;
final _collectionsService = CollectionsService.instance;
final _diffFetcher = DiffFetcher();
final FileUploader _uploader = FileUploader.instance;
final Configuration _config = Configuration.instance;
final CollectionsService _collectionsService = CollectionsService.instance;
final DiffFetcher _diffFetcher = DiffFetcher();
final LocalFileUpdateService _localFileUpdateService =
LocalFileUpdateService.instance;
int _completedUploads = 0;
@ -75,7 +77,7 @@ class RemoteSyncService {
}
Future<void> sync({bool silently = false}) async {
if (!Configuration.instance.hasConfiguredAccount()) {
if (!_config.hasConfiguredAccount()) {
_logger.info("Skipping remote sync since account is not configured");
return;
}
@ -186,9 +188,8 @@ class RemoteSyncService {
await _diffFetcher.getEncryptedFilesDiff(collectionID, sinceTime);
if (diff.deletedFiles.isNotEmpty) {
final fileIDs = diff.deletedFiles.map((f) => f.uploadedFileID).toList();
final deletedFiles =
(await FilesDB.instance.getFilesFromIDs(fileIDs)).values.toList();
await FilesDB.instance.deleteFilesFromCollection(collectionID, fileIDs);
final deletedFiles = (await _db.getFilesFromIDs(fileIDs)).values.toList();
await _db.deleteFilesFromCollection(collectionID, fileIDs);
Bus.instance.fire(
CollectionUpdatedEvent(
collectionID,
@ -231,9 +232,8 @@ class RemoteSyncService {
}
Future<void> _syncDeviceCollectionFilesForUpload() async {
final int ownerID = Configuration.instance.getUserID();
final FilesDB filesDB = FilesDB.instance;
final deviceCollections = await filesDB.getDeviceCollections();
final int ownerID = _config.getUserID();
final deviceCollections = await _db.getDeviceCollections();
deviceCollections.removeWhere((element) => !element.shouldBackup);
// Sort by count to ensure that photos in iOS are first inserted in
// smallest album marked for backup. This is to ensure that photo is
@ -241,14 +241,14 @@ class RemoteSyncService {
deviceCollections.sort((a, b) => a.count.compareTo(b.count));
await _createCollectionsForDevicePath(deviceCollections);
final Map<String, Set<String>> pathIdToLocalIDs =
await filesDB.getDevicePathIDToLocalIDMap();
await _db.getDevicePathIDToLocalIDMap();
for (final deviceCollection in deviceCollections) {
_logger.fine("processing ${deviceCollection.name}");
final Set<String> localIDsToSync =
pathIdToLocalIDs[deviceCollection.id] ?? {};
if (deviceCollection.uploadStrategy == UploadStrategy.ifMissing) {
final Set<String> alreadyClaimedLocalIDs =
await filesDB.getLocalIDsMarkedForOrAlreadyUploaded(ownerID);
await _db.getLocalIDsMarkedForOrAlreadyUploaded(ownerID);
localIDsToSync.removeAll(alreadyClaimedLocalIDs);
}
@ -256,7 +256,7 @@ class RemoteSyncService {
continue;
}
await filesDB.setCollectionIDForUnMappedLocalFiles(
await _db.setCollectionIDForUnMappedLocalFiles(
deviceCollection.collectionID,
localIDsToSync,
);
@ -264,8 +264,8 @@ class RemoteSyncService {
// mark IDs as already synced if corresponding entry is present in
// the collection. This can happen when a user has marked a folder
// for sync, then un-synced it and again tries to mark if for sync.
final Set<String> existingMapping = await filesDB
.getLocalFileIDsForCollection(deviceCollection.collectionID);
final Set<String> existingMapping =
await _db.getLocalFileIDsForCollection(deviceCollection.collectionID);
final Set<String> commonElements =
localIDsToSync.intersection(existingMapping);
if (commonElements.isNotEmpty) {
@ -285,7 +285,7 @@ class RemoteSyncService {
' for ${deviceCollection.name}',
);
final filesWithCollectionID =
await filesDB.getLocalFiles(localIDsToSync.toList());
await _db.getLocalFiles(localIDsToSync.toList());
final List<File> newFilesToInsert = [];
final Set<String> fileFoundForLocalIDs = {};
for (var existingFile in filesWithCollectionID) {
@ -299,7 +299,7 @@ class RemoteSyncService {
fileFoundForLocalIDs.add(localID);
}
}
await filesDB.insertMultiple(newFilesToInsert);
await _db.insertMultiple(newFilesToInsert);
if (fileFoundForLocalIDs.length != localIDsToSync.length) {
_logger.warning(
"mismatch in num of filesToSync ${localIDsToSync.length} to "
@ -323,6 +323,7 @@ class RemoteSyncService {
oldCollectionIDsForAutoSync.removeAll(newCollectionIDsForAutoSync);
await removeFilesQueuedForUpload(oldCollectionIDsForAutoSync.toList());
Bus.instance.fire(LocalPhotosUpdatedEvent(<File>[]));
Bus.instance.fire(BackupFoldersUpdatedEvent());
}
Future<void> removeFilesQueuedForUpload(List<int> collectionIDs) async {
@ -366,7 +367,7 @@ class RemoteSyncService {
int deviceCollectionID = deviceCollection.collectionID;
if (deviceCollectionID != -1) {
final collectionByID =
CollectionsService.instance.getCollectionByID(deviceCollectionID);
_collectionsService.getCollectionByID(deviceCollectionID);
if (collectionByID == null || collectionByID.isDeleted) {
_logger.info(
"Collection $deviceCollectionID either deleted or missing "
@ -376,20 +377,19 @@ class RemoteSyncService {
}
}
if (deviceCollectionID == -1) {
final collection = await CollectionsService.instance
.getOrCreateForPath(deviceCollection.name);
await FilesDB.instance
.updateDeviceCollection(deviceCollection.id, collection.id);
final collection =
await _collectionsService.getOrCreateForPath(deviceCollection.name);
await _db.updateDeviceCollection(deviceCollection.id, collection.id);
deviceCollection.collectionID = collection.id;
}
}
}
Future<List<File>> _getFilesToBeUploaded() async {
final deviceCollections = await FilesDB.instance.getDeviceCollections();
final deviceCollections = await _db.getDeviceCollections();
deviceCollections.removeWhere((element) => !element.shouldBackup);
final List<File> filesToBeUploaded = await _db.getFilesPendingForUpload();
if (!Configuration.instance.shouldBackupVideos() || _shouldThrottleSync()) {
if (!_config.shouldBackupVideos() || _shouldThrottleSync()) {
filesToBeUploaded
.removeWhere((element) => element.fileType == FileType.video);
}
@ -415,7 +415,7 @@ class RemoteSyncService {
}
Future<bool> _uploadFiles(List<File> filesToBeUploaded) async {
final int ownerID = Configuration.instance.getUserID();
final int ownerID = _config.getUserID();
final updatedFileIDs = await _db.getUploadedFileIDsToBeUpdated(ownerID);
if (updatedFileIDs.isNotEmpty) {
_logger.info("Identified ${updatedFileIDs.length} files for reupload");
@ -451,9 +451,7 @@ class RemoteSyncService {
// prefer existing collection ID for manually uploaded files.
// See https://github.com/ente-io/frame/pull/187
final collectionID = file.collectionID ??
(await CollectionsService.instance
.getOrCreateForPath(file.deviceFolder))
.id;
(await _collectionsService.getOrCreateForPath(file.deviceFolder)).id;
_uploadFile(file, collectionID, futures);
}
@ -487,8 +485,7 @@ class RemoteSyncService {
Future<void> _onFileUploaded(File file) async {
Bus.instance.fire(CollectionUpdatedEvent(file.collectionID, [file]));
_completedUploads++;
final toBeUploadedInThisSession =
FileUploader.instance.getCurrentSessionUploadCount();
final toBeUploadedInThisSession = _uploader.getCurrentSessionUploadCount();
if (toBeUploadedInThisSession == 0) {
return;
}
@ -535,7 +532,7 @@ class RemoteSyncService {
localUploadedFromDevice = 0,
localButUpdatedOnDevice = 0,
remoteNewFile = 0;
final int userID = Configuration.instance.getUserID();
final int userID = _config.getUserID();
bool needsGalleryReload = false;
// this is required when same file is uploaded twice in the same
// collection. Without this check, if both remote files are part of same

24
lib/services/trash_sync_service.dart
View file

@ -1,5 +1,7 @@
// @dart=2.9
import 'dart:async';
import 'package:dio/dio.dart';
import 'package:logging/logging.dart';
import 'package:photos/core/configuration.dart';
@ -8,6 +10,7 @@ import 'package:photos/core/network.dart';
import 'package:photos/db/trash_db.dart';
import 'package:photos/events/collection_updated_event.dart';
import 'package:photos/events/force_reload_trash_page_event.dart';
import 'package:photos/events/trash_updated_event.dart';
import 'package:photos/models/file.dart';
import 'package:photos/models/ignored_file.dart';
import 'package:photos/models/trash_file.dart';
@ -36,20 +39,24 @@ class TrashSyncService {
Future<void> syncTrash() async {
final lastSyncTime = _getSyncTime();
bool isLocalTrashUpdated = false;
_logger.fine('sync trash sinceTime : $lastSyncTime');
final diff = await _diffFetcher.getTrashFilesDiff(lastSyncTime);
if (diff.trashedFiles.isNotEmpty) {
isLocalTrashUpdated = true;
_logger.fine("inserting ${diff.trashedFiles.length} items in trash");
await _trashDB.insertMultiple(diff.trashedFiles);
}
if (diff.deletedUploadIDs.isNotEmpty) {
_logger.fine("discard ${diff.deletedUploadIDs.length} deleted items");
await _trashDB.delete(diff.deletedUploadIDs);
final itemsDeleted = await _trashDB.delete(diff.deletedUploadIDs);
isLocalTrashUpdated = isLocalTrashUpdated || itemsDeleted > 0;
}
if (diff.restoredFiles.isNotEmpty) {
_logger.fine("discard ${diff.restoredFiles.length} restored items");
await _trashDB
final itemsDeleted = await _trashDB
.delete(diff.restoredFiles.map((e) => e.uploadedFileID).toList());
isLocalTrashUpdated = isLocalTrashUpdated || itemsDeleted > 0;
}
await _updateIgnoredFiles(diff);
@ -57,6 +64,11 @@ class TrashSyncService {
if (diff.lastSyncedTimeStamp != 0) {
await _setSyncTime(diff.lastSyncedTimeStamp);
}
if (isLocalTrashUpdated) {
_logger
.fine('local trash updated, fire ${(TrashUpdatedEvent).toString()}');
Bus.instance.fire(TrashUpdatedEvent());
}
if (diff.hasMore) {
return await syncTrash();
} else if (diff.trashedFiles.isNotEmpty ||
@ -145,8 +157,10 @@ class TrashSyncService {
),
data: params,
);
_trashDB.delete(uniqueFileIds);
syncTrash();
await _trashDB.delete(uniqueFileIds);
Bus.instance.fire(TrashUpdatedEvent());
// no need to await on syncing trash from remote
unawaited(syncTrash());
} catch (e, s) {
_logger.severe("failed to delete from trash", e, s);
rethrow;
@ -167,6 +181,8 @@ class TrashSyncService {
data: params,
);
await _trashDB.clearTable();
unawaited(syncTrash());
Bus.instance.fire(TrashUpdatedEvent());
Bus.instance.fire(ForceReloadTrashPageEvent());
} catch (e, s) {
_logger.severe("failed to empty trash", e, s);

26
lib/ui/account/password_reentry_page.dart
View file

@ -3,6 +3,7 @@
import 'package:flutter/material.dart';
import 'package:logging/logging.dart';
import 'package:photos/core/configuration.dart';
import 'package:photos/core/errors.dart';
import 'package:photos/core/event_bus.dart';
import 'package:photos/events/subscription_purchased_event.dart';
import 'package:photos/models/key_attributes.dart';
@ -77,6 +78,31 @@ class _PasswordReentryPageState extends State<PasswordReentryPage> {
_passwordController.text,
Configuration.instance.getKeyAttributes(),
);
} on KeyDerivationError catch (e, s) {
_logger.severe("Password verification failed", e, s);
await dialog.hide();
final dialogUserChoice = await showChoiceDialog(
context,
"Recreate password",
"The current device is not powerful enough to verify your "
"password, so we need to regenerate it once in a way that "
"works with all devices. \n\nPlease login using your "
"recovery key and regenerate your password (you can use the same one again if you wish).",
firstAction: "Cancel",
firstActionColor: Theme.of(context).colorScheme.primary,
secondAction: "Use recovery key",
secondActionColor: Theme.of(context).colorScheme.primary,
);
if (dialogUserChoice == DialogUserChoice.secondChoice) {
Navigator.of(context).push(
MaterialPageRoute(
builder: (BuildContext context) {
return const RecoveryPage();
},
),
);
}
return;
} catch (e, s) {
_logger.severe("Password verification failed", e, s);
await dialog.hide();

3
lib/ui/backup_folder_selection_page.dart
View file

@ -9,11 +9,9 @@ import 'package:implicitly_animated_reorderable_list/implicitly_animated_reorder
import 'package:implicitly_animated_reorderable_list/transitions.dart';
import 'package:logging/logging.dart';
import 'package:photos/core/configuration.dart';
import 'package:photos/core/event_bus.dart';
import 'package:photos/db/device_files_db.dart';
import 'package:photos/db/files_db.dart';
import 'package:photos/ente_theme_data.dart';
import 'package:photos/events/backup_folders_updated_event.dart';
import 'package:photos/models/device_collection.dart';
import 'package:photos/models/file.dart';
import 'package:photos/services/remote_sync_service.dart';
@ -193,7 +191,6 @@ class _BackupFolderSelectionPageState extends State<BackupFolderSelectionPage> {
_allDevicePathIDs.length ==
_selectedDevicePathIDs.length,
);
Bus.instance.fire(BackupFoldersUpdatedEvent());
Navigator.of(context).pop();
},
child: Text(widget.buttonText),

34
lib/ui/collections/trash_button_widget.dart
View file

@ -1,11 +1,15 @@
// @dart=2.9
import 'dart:async';
import 'package:flutter/material.dart';
import 'package:photos/core/event_bus.dart';
import 'package:photos/db/trash_db.dart';
import 'package:photos/events/trash_updated_event.dart';
import 'package:photos/ui/viewer/gallery/trash_page.dart';
import 'package:photos/utils/navigation_util.dart';
class TrashButtonWidget extends StatelessWidget {
class TrashButtonWidget extends StatefulWidget {
const TrashButtonWidget(
this.textStyle, {
Key key,
@ -13,6 +17,30 @@ class TrashButtonWidget extends StatelessWidget {
final TextStyle textStyle;
@override
State<TrashButtonWidget> createState() => _TrashButtonWidgetState();
}
class _TrashButtonWidgetState extends State<TrashButtonWidget> {
StreamSubscription<TrashUpdatedEvent> _trashUpdatedEventSubscription;
@override
void initState() {
super.initState();
_trashUpdatedEventSubscription =
Bus.instance.on<TrashUpdatedEvent>().listen((event) {
if (mounted) {
setState(() {});
}
});
}
@override
void dispose() {
_trashUpdatedEventSubscription?.cancel();
super.dispose();
}
@override
Widget build(BuildContext context) {
return OutlinedButton(
@ -48,7 +76,7 @@ class TrashButtonWidget extends StatelessWidget {
if (snapshot.hasData && snapshot.data > 0) {
return RichText(
text: TextSpan(
style: textStyle,
style: widget.textStyle,
children: [
TextSpan(
text: "Trash",
@ -65,7 +93,7 @@ class TrashButtonWidget extends StatelessWidget {
} else {
return RichText(
text: TextSpan(
style: textStyle,
style: widget.textStyle,
children: [
TextSpan(
text: "Trash",

2
lib/ui/viewer/gallery/device_folder_page.dart
View file

@ -6,7 +6,6 @@ import 'package:photos/core/event_bus.dart';
import 'package:photos/db/device_files_db.dart';
import 'package:photos/db/files_db.dart';
import 'package:photos/ente_theme_data.dart';
import 'package:photos/events/backup_folders_updated_event.dart';
import 'package:photos/events/files_updated_event.dart';
import 'package:photos/events/local_photos_updated_event.dart';
import 'package:photos/models/device_collection.dart';
@ -120,7 +119,6 @@ class _BackupConfigurationHeaderWidgetState
);
_isBackedUp = value;
setState(() {});
Bus.instance.fire(BackupFoldersUpdatedEvent());
},
),
],

52
lib/ui/viewer/gallery/gallery_app_bar_widget.dart
View file

@ -21,6 +21,7 @@ import 'package:photos/ui/common/rename_dialog.dart';
import 'package:photos/ui/sharing/share_collection_widget.dart';
import 'package:photos/utils/dialog_util.dart';
import 'package:photos/utils/magic_util.dart';
import 'package:photos/utils/toast_util.dart';
class GalleryAppBarWidget extends StatefulWidget {
final GalleryType type;
@ -193,13 +194,26 @@ class _GalleryAppBarWidgetState extends State<GalleryAppBarWidget> {
),
),
);
items.add(
PopupMenuItem(
value: 3,
child: Row(
children: const [
Icon(Icons.delete_sweep_outlined),
Padding(
padding: EdgeInsets.all(8),
),
Text("Delete Album"),
],
),
),
);
return items;
},
onSelected: (value) async {
if (value == 1) {
await _renameAlbum(context);
}
if (value == 2) {
} else if (value == 2) {
await changeCollectionVisibility(
context,
widget.collection,
@ -207,6 +221,8 @@ class _GalleryAppBarWidgetState extends State<GalleryAppBarWidget> {
? kVisibilityVisible
: kVisibilityArchive,
);
} else if (value == 3) {
await _trashCollection();
}
},
),
@ -215,6 +231,38 @@ class _GalleryAppBarWidgetState extends State<GalleryAppBarWidget> {
return actions;
}
Future<void> _trashCollection() async {
final result = await showChoiceDialog(
context,
"Delete album?",
"Files that are unique to this album "
"will be moved to trash, and this album will be deleted.",
firstAction: "Cancel",
secondAction: "Delete album",
secondActionColor: Colors.red,
);
if (result != DialogUserChoice.secondChoice) {
return;
}
final dialog = createProgressDialog(
context,
"Please wait, deleting album",
);
await dialog.show();
try {
await CollectionsService.instance.trashCollection(widget.collection);
showShortToast(context, "Successfully deleted album");
await dialog.hide();
Navigator.of(context).pop();
} catch (e, s) {
_logger.severe("failed to trash collection", e, s);
await dialog.hide();
showGenericErrorDialog(context);
rethrow;
}
}
Future<void> _showShareCollectionDialog() async {
var collection = widget.collection;
final dialog = createProgressDialog(context, "Please wait...");

2
lib/utils/toast_util.dart
View file

@ -11,6 +11,7 @@ Future<void> showToast(
BuildContext context,
String message, {
toastLength = Toast.LENGTH_LONG,
iOSDismissOnTap = true,
}) async {
if (Platform.isAndroid) {
await Fluttertoast.cancel();
@ -34,6 +35,7 @@ Future<void> showToast(
message,
duration: Duration(seconds: (toastLength == Toast.LENGTH_LONG ? 5 : 2)),
toastPosition: EasyLoadingToastPosition.bottom,
dismissOnTap: iOSDismissOnTap,
);
}
}