Merge pull request #447 from ente-io/rewrite_device_sync_remote
[Part-2] Rewrite Device Sync
This commit is contained in:
Neeraj Gupta
GitHub
commit
5700ac9ae5
|
|
@ -236,6 +236,24 @@ extension DeviceFiles on FilesDB {
|
|||
}
|
||||
}
|
||||
|
||||
// getDeviceSyncCollectionIDs returns the collectionIDs for the
|
||||
// deviceCollections which are marked for auto-backup
|
||||
Future<Set<int>> getDeviceSyncCollectionIDs() async {
|
||||
final Database db = await database;
|
||||
final rows = await db.rawQuery(
|
||||
'''
|
||||
SELECT collection_id FROM device_collections where should_backup =
|
||||
$_sqlBoolTrue
|
||||
and collection_id != -1;
|
||||
''',
|
||||
);
|
||||
final Set<int> result = <int>{};
|
||||
for (final row in rows) {
|
||||
result.add(row['collection_id']);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
Future<void> updateDevicePathSyncStatus(Map<String, bool> syncStatus) async {
|
||||
final db = await database;
|
||||
var batch = db.batch();
|
||||
|
|
@ -296,7 +314,8 @@ extension DeviceFiles on FilesDB {
|
|||
(limit != null ? ' limit $limit;' : ';');
|
||||
final results = await db.rawQuery(rawQuery);
|
||||
final files = convertToFiles(results);
|
||||
return FileLoadResult(files, files.length == limit);
|
||||
final dedupe = deduplicateByLocalID(files);
|
||||
return FileLoadResult(dedupe, files.length == limit);
|
||||
}
|
||||
|
||||
Future<List<DeviceCollection>> getDeviceCollections({
|
||||
|
|
|
|||
|
|
@ -455,7 +455,7 @@ class FilesDB {
|
|||
return BackedUpFileIDs(localIDs.toList(), uploadedIDs.toList());
|
||||
}
|
||||
|
||||
Future<FileLoadResult> getAllUploadedFiles(
|
||||
Future<FileLoadResult> getAllPendingOrUploadedFiles(
|
||||
int startTime,
|
||||
int endTime,
|
||||
int ownerID, {
|
||||
|
|
@ -469,7 +469,7 @@ class FilesDB {
|
|||
final results = await db.query(
|
||||
filesTable,
|
||||
where:
|
||||
'$columnCreationTime >= ? AND $columnCreationTime <= ? AND $columnOwnerID = ? AND ($columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1)'
|
||||
'$columnCreationTime >= ? AND $columnCreationTime <= ? AND ($columnOwnerID IS NULL OR $columnOwnerID = ?) AND ($columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1)'
|
||||
' AND $columnMMdVisibility = ?',
|
||||
whereArgs: [startTime, endTime, ownerID, visibility],
|
||||
orderBy:
|
||||
|
|
@ -528,39 +528,7 @@ class FilesDB {
|
|||
return FileLoadResult(deduplicatedFiles, files.length == limit);
|
||||
}
|
||||
|
||||
Future<FileLoadResult> getImportantFiles(
|
||||
int startTime,
|
||||
int endTime,
|
||||
int ownerID,
|
||||
List<String> paths, {
|
||||
int limit,
|
||||
bool asc,
|
||||
Set<int> ignoredCollectionIDs,
|
||||
}) async {
|
||||
final db = await instance.database;
|
||||
String inParam = "";
|
||||
for (final path in paths) {
|
||||
inParam += "'" + path.replaceAll("'", "''") + "',";
|
||||
}
|
||||
inParam = inParam.substring(0, inParam.length - 1);
|
||||
final order = (asc ?? false ? 'ASC' : 'DESC');
|
||||
final results = await db.query(
|
||||
filesTable,
|
||||
where:
|
||||
'$columnCreationTime >= ? AND $columnCreationTime <= ? AND ($columnOwnerID IS NULL OR $columnOwnerID = ?) AND ($columnMMdVisibility IS NULL OR $columnMMdVisibility = ?)'
|
||||
'AND (($columnLocalID IS NOT NULL AND $columnDeviceFolder IN ($inParam)) OR ($columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1))',
|
||||
whereArgs: [startTime, endTime, ownerID, kVisibilityVisible],
|
||||
orderBy:
|
||||
'$columnCreationTime ' + order + ', $columnModificationTime ' + order,
|
||||
limit: limit,
|
||||
);
|
||||
final files = convertToFiles(results);
|
||||
final List<File> deduplicatedFiles =
|
||||
_deduplicatedAndFilterIgnoredFiles(files, ignoredCollectionIDs);
|
||||
return FileLoadResult(deduplicatedFiles, files.length == limit);
|
||||
}
|
||||
|
||||
List<File> _deduplicateByLocalID(List<File> files) {
|
||||
List<File> deduplicateByLocalID(List<File> files) {
|
||||
final localIDs = <String>{};
|
||||
final List<File> deduplicatedFiles = [];
|
||||
for (final file in files) {
|
||||
|
|
@ -630,29 +598,6 @@ class FilesDB {
|
|||
return FileLoadResult(files, files.length == limit);
|
||||
}
|
||||
|
||||
Future<FileLoadResult> getFilesInPath(
|
||||
String path,
|
||||
int startTime,
|
||||
int endTime, {
|
||||
int limit,
|
||||
bool asc,
|
||||
}) async {
|
||||
final db = await instance.database;
|
||||
final order = (asc ?? false ? 'ASC' : 'DESC');
|
||||
final results = await db.query(
|
||||
filesTable,
|
||||
where:
|
||||
'$columnDeviceFolder = ? AND $columnCreationTime >= ? AND $columnCreationTime <= ? AND $columnLocalID IS NOT NULL',
|
||||
whereArgs: [path, startTime, endTime],
|
||||
orderBy:
|
||||
'$columnCreationTime ' + order + ', $columnModificationTime ' + order,
|
||||
groupBy: columnLocalID,
|
||||
limit: limit,
|
||||
);
|
||||
final files = convertToFiles(results);
|
||||
return FileLoadResult(files, files.length == limit);
|
||||
}
|
||||
|
||||
Future<FileLoadResult> getLocalDeviceFiles(
|
||||
int startTime,
|
||||
int endTime, {
|
||||
|
|
@ -671,32 +616,10 @@ class FilesDB {
|
|||
limit: limit,
|
||||
);
|
||||
final files = convertToFiles(results);
|
||||
final result = _deduplicateByLocalID(files);
|
||||
final result = deduplicateByLocalID(files);
|
||||
return FileLoadResult(result, files.length == limit);
|
||||
}
|
||||
|
||||
Future<List<File>> getAllVideos() async {
|
||||
final db = await instance.database;
|
||||
final results = await db.query(
|
||||
filesTable,
|
||||
where: '$columnLocalID IS NOT NULL AND $columnFileType = 1',
|
||||
orderBy: '$columnCreationTime DESC',
|
||||
);
|
||||
return convertToFiles(results);
|
||||
}
|
||||
|
||||
Future<List<File>> getAllInPath(String path) async {
|
||||
final db = await instance.database;
|
||||
final results = await db.query(
|
||||
filesTable,
|
||||
where: '$columnLocalID IS NOT NULL AND $columnDeviceFolder = ?',
|
||||
whereArgs: [path],
|
||||
orderBy: '$columnCreationTime DESC',
|
||||
groupBy: columnLocalID,
|
||||
);
|
||||
return convertToFiles(results);
|
||||
}
|
||||
|
||||
Future<List<File>> getFilesCreatedWithinDurations(
|
||||
List<List<int>> durations,
|
||||
Set<int> ignoredCollectionIDs, {
|
||||
|
|
@ -724,28 +647,6 @@ class FilesDB {
|
|||
return _deduplicatedAndFilterIgnoredFiles(files, ignoredCollectionIDs);
|
||||
}
|
||||
|
||||
Future<List<File>> getFilesToBeUploadedWithinFolders(
|
||||
Set<String> folders,
|
||||
) async {
|
||||
if (folders.isEmpty) {
|
||||
return [];
|
||||
}
|
||||
final db = await instance.database;
|
||||
String inParam = "";
|
||||
for (final folder in folders) {
|
||||
inParam += "'" + folder.replaceAll("'", "''") + "',";
|
||||
}
|
||||
inParam = inParam.substring(0, inParam.length - 1);
|
||||
final results = await db.query(
|
||||
filesTable,
|
||||
where:
|
||||
'($columnUploadedFileID IS NULL OR $columnUploadedFileID IS -1) AND $columnDeviceFolder IN ($inParam)',
|
||||
orderBy: '$columnCreationTime DESC',
|
||||
groupBy: columnLocalID,
|
||||
);
|
||||
return convertToFiles(results);
|
||||
}
|
||||
|
||||
// Files which user added to a collection manually but they are not uploaded yet.
|
||||
Future<List<File>> getPendingManualUploads() async {
|
||||
final db = await instance.database;
|
||||
|
|
@ -782,25 +683,16 @@ class FilesDB {
|
|||
return convertToFiles(results);
|
||||
}
|
||||
|
||||
Future<List<File>> getEditedRemoteFiles() async {
|
||||
final db = await instance.database;
|
||||
final results = await db.query(
|
||||
filesTable,
|
||||
where:
|
||||
'($columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1) AND ($columnUploadedFileID IS NULL OR $columnUploadedFileID IS -1)',
|
||||
orderBy: '$columnCreationTime DESC',
|
||||
groupBy: columnLocalID,
|
||||
);
|
||||
return convertToFiles(results);
|
||||
}
|
||||
|
||||
Future<List<int>> getUploadedFileIDsToBeUpdated() async {
|
||||
Future<List<int>> getUploadedFileIDsToBeUpdated(int ownerID) async {
|
||||
final db = await instance.database;
|
||||
final rows = await db.query(
|
||||
filesTable,
|
||||
columns: [columnUploadedFileID],
|
||||
where:
|
||||
'($columnLocalID IS NOT NULL AND ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) AND $columnUpdationTime IS NULL)',
|
||||
where: '($columnLocalID IS NOT NULL AND $columnOwnerID = ? AND '
|
||||
'($columnUploadedFileID '
|
||||
'IS NOT '
|
||||
'NULL AND $columnUploadedFileID IS NOT -1) AND $columnUpdationTime IS NULL)',
|
||||
whereArgs: [ownerID],
|
||||
orderBy: '$columnCreationTime DESC',
|
||||
distinct: true,
|
||||
);
|
||||
|
|
@ -842,6 +734,62 @@ class FilesDB {
|
|||
return result;
|
||||
}
|
||||
|
||||
Future<Set<String>> getLocalIDsMarkedForOrAlreadyUploaded(int ownerID) async {
|
||||
final db = await instance.database;
|
||||
final rows = await db.query(
|
||||
filesTable,
|
||||
columns: [columnLocalID],
|
||||
distinct: true,
|
||||
where: '$columnLocalID IS NOT NULL AND ($columnCollectionID IS NOT NULL '
|
||||
'AND '
|
||||
'$columnCollectionID != -1) AND ($columnOwnerID = ? OR '
|
||||
'$columnOwnerID IS NULL)',
|
||||
whereArgs: [ownerID],
|
||||
);
|
||||
final result = <String>{};
|
||||
for (final row in rows) {
|
||||
result.add(row[columnLocalID]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
Future<Set<String>> getLocalFileIDsForCollection(int collectionID) async {
|
||||
final db = await instance.database;
|
||||
final rows = await db.query(
|
||||
filesTable,
|
||||
columns: [columnLocalID],
|
||||
where: '$columnLocalID IS NOT NULL AND $columnCollectionID = ?',
|
||||
whereArgs: [collectionID],
|
||||
);
|
||||
final result = <String>{};
|
||||
for (final row in rows) {
|
||||
result.add(row[columnLocalID]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Sets the collectionID for the files with given LocalIDs if the
|
||||
// corresponding file entries are not already mapped to some other collection
|
||||
Future<int> setCollectionIDForUnMappedLocalFiles(
|
||||
int collectionID,
|
||||
Set<String> localIDs,
|
||||
) async {
|
||||
final db = await instance.database;
|
||||
String inParam = "";
|
||||
for (final localID in localIDs) {
|
||||
inParam += "'" + localID + "',";
|
||||
}
|
||||
inParam = inParam.substring(0, inParam.length - 1);
|
||||
return await db.rawUpdate(
|
||||
'''
|
||||
UPDATE $filesTable
|
||||
SET $columnCollectionID = $collectionID
|
||||
WHERE $columnLocalID IN ($inParam) AND ($columnCollectionID IS NULL OR
|
||||
$columnCollectionID = -1);
|
||||
''',
|
||||
);
|
||||
}
|
||||
|
||||
Future<int> getNumberOfUploadedFiles() async {
|
||||
final db = await instance.database;
|
||||
final rows = await db.query(
|
||||
|
|
@ -954,7 +902,6 @@ class FilesDB {
|
|||
if (fileType == FileType.livePhoto && hashData.zipHash != null) {
|
||||
inParam += ",'${hashData.zipHash}'";
|
||||
}
|
||||
|
||||
final db = await instance.database;
|
||||
final rows = await db.query(
|
||||
filesTable,
|
||||
|
|
@ -1026,6 +973,17 @@ class FilesDB {
|
|||
);
|
||||
}
|
||||
|
||||
Future<int> deleteMultipleByGeneratedIDs(List<int> generatedIDs) async {
|
||||
if (generatedIDs.isEmpty) {
|
||||
return 0;
|
||||
}
|
||||
final db = await instance.database;
|
||||
return await db.delete(
|
||||
filesTable,
|
||||
where: '$columnGeneratedID IN (${generatedIDs.join(', ')})',
|
||||
);
|
||||
}
|
||||
|
||||
Future<int> deleteLocalFile(File file) async {
|
||||
final db = await instance.database;
|
||||
if (file.localID != null) {
|
||||
|
|
@ -1149,6 +1107,42 @@ class FilesDB {
|
|||
);
|
||||
}
|
||||
|
||||
Future<List<File>> getPendingUploadForCollection(int collectionID) async {
|
||||
final db = await instance.database;
|
||||
final results = await db.query(
|
||||
filesTable,
|
||||
where: '$columnCollectionID = ? AND ($columnUploadedFileID IS NULL OR '
|
||||
'$columnUploadedFileID = -1)',
|
||||
whereArgs: [collectionID],
|
||||
);
|
||||
return convertToFiles(results);
|
||||
}
|
||||
|
||||
Future<Set<String>> getLocalIDsPresentInEntries(
|
||||
List<File> existingFiles,
|
||||
int collectionID,
|
||||
) async {
|
||||
String inParam = "";
|
||||
for (final existingFile in existingFiles) {
|
||||
inParam += "'" + existingFile.localID + "',";
|
||||
}
|
||||
inParam = inParam.substring(0, inParam.length - 1);
|
||||
final db = await instance.database;
|
||||
final rows = await db.rawQuery(
|
||||
'''
|
||||
SELECT $columnLocalID
|
||||
FROM $filesTable
|
||||
WHERE $columnLocalID IN ($inParam) AND $columnCollectionID !=
|
||||
$collectionID AND $columnLocalID IS NOT NULL;
|
||||
''',
|
||||
);
|
||||
final result = <String>{};
|
||||
for (final row in rows) {
|
||||
result.add(row[columnLocalID]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
Future<List<File>> getLatestLocalFiles() async {
|
||||
final db = await instance.database;
|
||||
final rows = await db.rawQuery(
|
||||
|
|
@ -1210,9 +1204,9 @@ class FilesDB {
|
|||
) latest_files
|
||||
ON $filesTable.$columnCollectionID = latest_files.$columnCollectionID
|
||||
AND $filesTable.$columnCreationTime = latest_files.max_creation_time;
|
||||
''';
|
||||
}
|
||||
|
||||
''';
|
||||
}
|
||||
final db = await instance.database;
|
||||
final rows = await db.rawQuery(
|
||||
query,
|
||||
|
|
|
|||
|
|
@ -237,12 +237,13 @@ Future<List<AssetPathEntity>> _getGalleryList({
|
|||
type: RequestType.common,
|
||||
filterOption: filterOptionGroup,
|
||||
);
|
||||
|
||||
// todo: assetCount will be deprecated in the new version.
|
||||
// disable sorting and either try to evaluate if it's required or yolo
|
||||
galleryList.sort((s1, s2) {
|
||||
return s2.assetCount.compareTo(s1.assetCount);
|
||||
if (s1.isAll) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
});
|
||||
|
||||
return galleryList;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@ class LocalFileUpdateService {
|
|||
) async {
|
||||
_logger.info("files to process ${localIDsToProcess.length} for reupload");
|
||||
final List<ente.File> localFiles =
|
||||
(await FilesDB.instance.getLocalFiles(localIDsToProcess));
|
||||
await FilesDB.instance.getLocalFiles(localIDsToProcess);
|
||||
final Set<String> processedIDs = {};
|
||||
for (ente.File file in localFiles) {
|
||||
if (processedIDs.contains(file.localID)) {
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import 'dart:io';
|
|||
|
||||
import 'package:computer/computer.dart';
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:flutter/widgets.dart';
|
||||
import 'package:logging/logging.dart';
|
||||
import 'package:photo_manager/photo_manager.dart';
|
||||
import 'package:photos/core/configuration.dart';
|
||||
|
|
@ -12,6 +13,7 @@ import 'package:photos/core/event_bus.dart';
|
|||
import 'package:photos/db/device_files_db.dart';
|
||||
import 'package:photos/db/file_updation_db.dart';
|
||||
import 'package:photos/db/files_db.dart';
|
||||
import 'package:photos/db/ignored_files_db.dart';
|
||||
import 'package:photos/events/backup_folders_updated_event.dart';
|
||||
import 'package:photos/events/local_photos_updated_event.dart';
|
||||
import 'package:photos/events/sync_status_update_event.dart';
|
||||
|
|
@ -147,9 +149,27 @@ class LocalSyncService {
|
|||
if (hasUpdated) {
|
||||
Bus.instance.fire(BackupFoldersUpdatedEvent());
|
||||
}
|
||||
// migrate the backed up folder settings
|
||||
if (!_prefs.containsKey(hasImportedDeviceCollections)) {
|
||||
final pathsToBackUp = Configuration.instance.getPathsToBackUp();
|
||||
final entriesToBackUp = Map.fromEntries(
|
||||
result
|
||||
.where((element) => pathsToBackUp.contains(element.item1.name))
|
||||
.map((e) => MapEntry(e.item1.id, true)),
|
||||
);
|
||||
if (entriesToBackUp.isNotEmpty) {
|
||||
await _db.updateDevicePathSyncStatus(entriesToBackUp);
|
||||
Bus.instance.fire(BackupFoldersUpdatedEvent());
|
||||
}
|
||||
await _prefs.setBool(hasImportedDeviceCollections, true);
|
||||
}
|
||||
return hasUpdated;
|
||||
}
|
||||
|
||||
bool isDeviceFileMigrationDone() {
|
||||
return _prefs.containsKey(hasImportedDeviceCollections);
|
||||
}
|
||||
|
||||
Future<bool> syncAll() async {
|
||||
final stopwatch = Stopwatch()..start();
|
||||
final localAssets = await getAllLocalAssets();
|
||||
|
|
@ -273,12 +293,15 @@ class LocalSyncService {
|
|||
Future<void> resetLocalSync() async {
|
||||
assert(kDebugMode, "only available in debug mode");
|
||||
await FilesDB.instance.deleteDB();
|
||||
await IgnoredFilesDB.instance.clearTable();
|
||||
for (var element in [
|
||||
kHasCompletedFirstImportKey,
|
||||
hasImportedDeviceCollections,
|
||||
kDbUpdationTimeKey,
|
||||
kDownloadedFileIDsKey,
|
||||
kEditedFileIDsKey
|
||||
kEditedFileIDsKey,
|
||||
"has_synced_edit_time",
|
||||
"has_selected_all_folders_for_backup",
|
||||
]) {
|
||||
await _prefs.remove(element);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import 'dart:io';
|
|||
import 'dart:math';
|
||||
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:flutter/widgets.dart';
|
||||
import 'package:logging/logging.dart';
|
||||
import 'package:photos/core/configuration.dart';
|
||||
import 'package:photos/core/errors.dart';
|
||||
|
|
@ -20,11 +21,13 @@ import 'package:photos/events/sync_status_update_event.dart';
|
|||
import 'package:photos/models/device_collection.dart';
|
||||
import 'package:photos/models/file.dart';
|
||||
import 'package:photos/models/file_type.dart';
|
||||
import 'package:photos/models/upload_strategy.dart';
|
||||
import 'package:photos/services/app_lifecycle_service.dart';
|
||||
import 'package:photos/services/collections_service.dart';
|
||||
import 'package:photos/services/ignored_files_service.dart';
|
||||
import 'package:photos/services/local_file_update_service.dart';
|
||||
import 'package:photos/services/local_sync_service.dart';
|
||||
import 'package:photos/services/sync_service.dart';
|
||||
import 'package:photos/services/trash_sync_service.dart';
|
||||
import 'package:photos/utils/diff_fetcher.dart';
|
||||
import 'package:photos/utils/file_uploader.dart';
|
||||
|
|
@ -102,7 +105,7 @@ class RemoteSyncService {
|
|||
final filesToBeUploaded = await _getFilesToBeUploaded();
|
||||
if (kDebugMode) {
|
||||
debugPrint("Skip upload for testing");
|
||||
filesToBeUploaded.clear();
|
||||
// filesToBeUploaded.clear();
|
||||
}
|
||||
final hasUploadedFiles = await _uploadFiles(filesToBeUploaded);
|
||||
if (hasUploadedFiles) {
|
||||
|
|
@ -233,9 +236,132 @@ class RemoteSyncService {
|
|||
}
|
||||
|
||||
Future<void> _syncDeviceCollectionFilesForUpload() async {
|
||||
final deviceCollections = await FilesDB.instance.getDeviceCollections();
|
||||
final int ownerID = Configuration.instance.getUserID();
|
||||
final FilesDB filesDB = FilesDB.instance;
|
||||
final deviceCollections = await filesDB.getDeviceCollections();
|
||||
deviceCollections.removeWhere((element) => !element.shouldBackup);
|
||||
// Sort by count to ensure that photos in iOS are first inserted in
|
||||
// smallest album marked for backup. This is to ensure that photo is
|
||||
// first attempted to upload in a non-recent album.
|
||||
deviceCollections.sort((a, b) => a.count.compareTo(b.count));
|
||||
await _createCollectionsForDevicePath(deviceCollections);
|
||||
final Map<String, Set<String>> pathIdToLocalIDs =
|
||||
await filesDB.getDevicePathIDToLocalIDMap();
|
||||
for (final deviceCollection in deviceCollections) {
|
||||
_logger.fine("processing ${deviceCollection.name}");
|
||||
final Set<String> localIDsToSync =
|
||||
pathIdToLocalIDs[deviceCollection.id] ?? {};
|
||||
if (deviceCollection.uploadStrategy == UploadStrategy.ifMissing) {
|
||||
final Set<String> alreadyClaimedLocalIDs =
|
||||
await filesDB.getLocalIDsMarkedForOrAlreadyUploaded(ownerID);
|
||||
localIDsToSync.removeAll(alreadyClaimedLocalIDs);
|
||||
}
|
||||
|
||||
if (localIDsToSync.isEmpty || deviceCollection.collectionID == -1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await filesDB.setCollectionIDForUnMappedLocalFiles(
|
||||
deviceCollection.collectionID,
|
||||
localIDsToSync,
|
||||
);
|
||||
|
||||
// mark IDs as already synced if corresponding entry is present in
|
||||
// the collection. This can happen when a user has marked a folder
|
||||
// for sync, then un-synced it and again tries to mark if for sync.
|
||||
final Set<String> existingMapping = await filesDB
|
||||
.getLocalFileIDsForCollection(deviceCollection.collectionID);
|
||||
final Set<String> commonElements =
|
||||
localIDsToSync.intersection(existingMapping);
|
||||
if (commonElements.isNotEmpty) {
|
||||
debugPrint(
|
||||
"${commonElements.length} files already existing in "
|
||||
"collection ${deviceCollection.collectionID} for ${deviceCollection.name}",
|
||||
);
|
||||
localIDsToSync.removeAll(commonElements);
|
||||
}
|
||||
|
||||
// At this point, the remaining localIDsToSync will need to create
|
||||
// new file entries, where we can store mapping for localID and
|
||||
// corresponding collection ID
|
||||
if (localIDsToSync.isNotEmpty) {
|
||||
debugPrint(
|
||||
'Adding new entries for ${localIDsToSync.length} files'
|
||||
' for ${deviceCollection.name}',
|
||||
);
|
||||
final filesWithCollectionID =
|
||||
await filesDB.getLocalFiles(localIDsToSync.toList());
|
||||
final List<File> newFilesToInsert = [];
|
||||
final Set<String> fileFoundForLocalIDs = {};
|
||||
for (var existingFile in filesWithCollectionID) {
|
||||
final String localID = existingFile.localID;
|
||||
if (!fileFoundForLocalIDs.contains(localID)) {
|
||||
existingFile.generatedID = null;
|
||||
existingFile.collectionID = deviceCollection.collectionID;
|
||||
existingFile.uploadedFileID = null;
|
||||
existingFile.ownerID = null;
|
||||
newFilesToInsert.add(existingFile);
|
||||
fileFoundForLocalIDs.add(localID);
|
||||
}
|
||||
}
|
||||
await filesDB.insertMultiple(newFilesToInsert);
|
||||
if (fileFoundForLocalIDs.length != localIDsToSync.length) {
|
||||
_logger.warning(
|
||||
"mismatch in num of filesToSync ${localIDsToSync.length} to "
|
||||
"fileSynced ${fileFoundForLocalIDs.length}",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> updateDeviceFolderSyncStatus(
|
||||
Map<String, bool> syncStatusUpdate,
|
||||
) async {
|
||||
final Set<int> oldCollectionIDsForAutoSync =
|
||||
await _db.getDeviceSyncCollectionIDs();
|
||||
await _db.updateDevicePathSyncStatus(syncStatusUpdate);
|
||||
final Set<int> newCollectionIDsForAutoSync =
|
||||
await _db.getDeviceSyncCollectionIDs();
|
||||
SyncService.instance.onDeviceCollectionSet(newCollectionIDsForAutoSync);
|
||||
// remove all collectionIDs which are still marked for backup
|
||||
oldCollectionIDsForAutoSync.removeAll(newCollectionIDsForAutoSync);
|
||||
await removeFilesQueuedForUpload(oldCollectionIDsForAutoSync.toList());
|
||||
Bus.instance.fire(LocalPhotosUpdatedEvent(<File>[]));
|
||||
}
|
||||
|
||||
Future<void> removeFilesQueuedForUpload(List<int> collectionIDs) async {
|
||||
/*
|
||||
For each collection, perform following action
|
||||
1) Get List of all files not uploaded yet
|
||||
2) Delete files who localIDs is also present in other collections.
|
||||
3) For Remaining files, set the collectionID as -1
|
||||
*/
|
||||
debugPrint("Removing files for collections $collectionIDs");
|
||||
for (int collectionID in collectionIDs) {
|
||||
final List<File> pendingUploads =
|
||||
await _db.getPendingUploadForCollection(collectionID);
|
||||
if (pendingUploads.isEmpty) {
|
||||
continue;
|
||||
}
|
||||
final Set<String> localIDsInOtherFileEntries =
|
||||
await _db.getLocalIDsPresentInEntries(
|
||||
pendingUploads,
|
||||
collectionID,
|
||||
);
|
||||
final List<File> entriesToUpdate = [];
|
||||
final List<int> entriesToDelete = [];
|
||||
for (File pendingUpload in pendingUploads) {
|
||||
if (localIDsInOtherFileEntries.contains(pendingUpload.localID)) {
|
||||
entriesToDelete.add(pendingUpload.generatedID);
|
||||
} else {
|
||||
pendingUpload.collectionID = -1;
|
||||
entriesToUpdate.add(pendingUpload);
|
||||
}
|
||||
}
|
||||
await _db.deleteMultipleByGeneratedIDs(entriesToDelete);
|
||||
await _db.insertMultiple(entriesToUpdate);
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _createCollectionsForDevicePath(
|
||||
|
|
@ -267,14 +393,12 @@ class RemoteSyncService {
|
|||
Future<List<File>> _getFilesToBeUploaded() async {
|
||||
final deviceCollections = await FilesDB.instance.getDeviceCollections();
|
||||
deviceCollections.removeWhere((element) => !element.shouldBackup);
|
||||
final foldersToBackUp = Configuration.instance.getPathsToBackUp();
|
||||
List<File> filesToBeUploaded;
|
||||
if (LocalSyncService.instance.hasGrantedLimitedPermissions() &&
|
||||
foldersToBackUp.isEmpty) {
|
||||
deviceCollections.isEmpty) {
|
||||
filesToBeUploaded = await _db.getUnUploadedLocalFiles();
|
||||
} else {
|
||||
filesToBeUploaded =
|
||||
await _db.getFilesToBeUploadedWithinFolders(foldersToBackUp);
|
||||
filesToBeUploaded = await _db.getPendingManualUploads();
|
||||
}
|
||||
if (!Configuration.instance.shouldBackupVideos() || _shouldThrottleSync()) {
|
||||
filesToBeUploaded
|
||||
|
|
@ -294,11 +418,6 @@ class RemoteSyncService {
|
|||
);
|
||||
}
|
||||
}
|
||||
if (filesToBeUploaded.isEmpty) {
|
||||
// look for files which user manually tried to back up but they are not
|
||||
// uploaded yet. These files should ignore video backup & ignored files filter
|
||||
filesToBeUploaded = await _db.getPendingManualUploads();
|
||||
}
|
||||
_sortByTimeAndType(filesToBeUploaded);
|
||||
_logger.info(
|
||||
filesToBeUploaded.length.toString() + " new files to be uploaded.",
|
||||
|
|
@ -307,16 +426,14 @@ class RemoteSyncService {
|
|||
}
|
||||
|
||||
Future<bool> _uploadFiles(List<File> filesToBeUploaded) async {
|
||||
final updatedFileIDs = await _db.getUploadedFileIDsToBeUpdated();
|
||||
_logger.info(updatedFileIDs.length.toString() + " files updated.");
|
||||
|
||||
final editedFiles = await _db.getEditedRemoteFiles();
|
||||
_logger.info(editedFiles.length.toString() + " files edited.");
|
||||
final int ownerID = Configuration.instance.getUserID();
|
||||
final updatedFileIDs = await _db.getUploadedFileIDsToBeUpdated(ownerID);
|
||||
if (updatedFileIDs.isNotEmpty) {
|
||||
_logger.info("Identified ${updatedFileIDs.length} files for reupload");
|
||||
}
|
||||
|
||||
_completedUploads = 0;
|
||||
final int toBeUploaded =
|
||||
filesToBeUploaded.length + updatedFileIDs.length + editedFiles.length;
|
||||
|
||||
final int toBeUploaded = filesToBeUploaded.length + updatedFileIDs.length;
|
||||
if (toBeUploaded > 0) {
|
||||
Bus.instance.fire(SyncStatusUpdate(SyncStatus.preparingForUpload));
|
||||
// verify if files upload is allowed based on their subscription plan and
|
||||
|
|
@ -351,15 +468,6 @@ class RemoteSyncService {
|
|||
_uploadFile(file, collectionID, futures);
|
||||
}
|
||||
|
||||
for (final file in editedFiles) {
|
||||
if (_shouldThrottleSync() &&
|
||||
futures.length >= kMaximumPermissibleUploadsInThrottledMode) {
|
||||
_logger.info("Skipping some edited files as we are throttling uploads");
|
||||
break;
|
||||
}
|
||||
_uploadFile(file, file.collectionID, futures);
|
||||
}
|
||||
|
||||
try {
|
||||
await Future.wait(futures);
|
||||
} on InvalidFileError {
|
||||
|
|
|
|||
|
|
@ -179,6 +179,15 @@ class SyncService {
|
|||
);
|
||||
}
|
||||
|
||||
void onDeviceCollectionSet(Set<int> collectionIDs) {
|
||||
_uploader.removeFromQueueWhere(
|
||||
(file) {
|
||||
return !collectionIDs.contains(file.collectionID);
|
||||
},
|
||||
UserCancelledUploadError(),
|
||||
);
|
||||
}
|
||||
|
||||
void onVideoBackupPaused() {
|
||||
_uploader.removeFromQueueWhere(
|
||||
(file) {
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import 'package:photos/ente_theme_data.dart';
|
|||
import 'package:photos/events/backup_folders_updated_event.dart';
|
||||
import 'package:photos/models/device_collection.dart';
|
||||
import 'package:photos/models/file.dart';
|
||||
import 'package:photos/services/remote_sync_service.dart';
|
||||
import 'package:photos/ui/common/loading_widget.dart';
|
||||
import 'package:photos/ui/viewer/file/thumbnail_widget.dart';
|
||||
|
||||
|
|
@ -181,8 +182,8 @@ class _BackupFolderSelectionPageState extends State<BackupFolderSelectionPage> {
|
|||
syncStatus[pathID] =
|
||||
_selectedDevicePathIDs.contains(pathID);
|
||||
}
|
||||
await FilesDB.instance
|
||||
.updateDevicePathSyncStatus(syncStatus);
|
||||
await RemoteSyncService.instance
|
||||
.updateDeviceFolderSyncStatus(syncStatus);
|
||||
await Configuration.instance
|
||||
.setSelectAllFoldersForBackup(
|
||||
_allDevicePathIDs.length ==
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:photos/models/device_collection.dart';
|
||||
import 'package:photos/services/local_sync_service.dart';
|
||||
import 'package:photos/ui/collections/device_folder_icon_widget.dart';
|
||||
import 'package:photos/ui/viewer/gallery/empte_state.dart';
|
||||
|
||||
|
|
@ -15,6 +16,8 @@ class DeviceFoldersGridViewWidget extends StatelessWidget {
|
|||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
final bool isMigrationDone =
|
||||
LocalSyncService.instance.isDeviceFileMigrationDone();
|
||||
return Padding(
|
||||
padding: const EdgeInsets.symmetric(horizontal: 8),
|
||||
child: SizedBox(
|
||||
|
|
@ -22,7 +25,11 @@ class DeviceFoldersGridViewWidget extends StatelessWidget {
|
|||
child: Align(
|
||||
alignment: Alignment.centerLeft,
|
||||
child: deviceCollections.isEmpty
|
||||
? const EmptyState()
|
||||
? (isMigrationDone
|
||||
? const EmptyState()
|
||||
: const EmptyState(
|
||||
text: "Importing....",
|
||||
))
|
||||
: ListView.builder(
|
||||
shrinkWrap: true,
|
||||
scrollDirection: Axis.horizontal,
|
||||
|
|
|
|||
|
|
@ -392,42 +392,30 @@ class _HomeWidgetState extends State<HomeWidget> {
|
|||
}
|
||||
final gallery = Gallery(
|
||||
asyncLoader: (creationStartTime, creationEndTime, {limit, asc}) async {
|
||||
final importantPaths = Configuration.instance.getPathsToBackUp();
|
||||
final ownerID = Configuration.instance.getUserID();
|
||||
final archivedCollectionIds =
|
||||
CollectionsService.instance.getArchivedCollections();
|
||||
FileLoadResult result;
|
||||
if (importantPaths.isNotEmpty) {
|
||||
result = await FilesDB.instance.getImportantFiles(
|
||||
if (LocalSyncService.instance.hasGrantedLimitedPermissions()) {
|
||||
result = await FilesDB.instance.getAllLocalAndUploadedFiles(
|
||||
creationStartTime,
|
||||
creationEndTime,
|
||||
ownerID,
|
||||
importantPaths.toList(),
|
||||
limit: limit,
|
||||
asc: asc,
|
||||
ignoredCollectionIDs: archivedCollectionIds,
|
||||
);
|
||||
} else {
|
||||
if (LocalSyncService.instance.hasGrantedLimitedPermissions()) {
|
||||
result = await FilesDB.instance.getAllLocalAndUploadedFiles(
|
||||
creationStartTime,
|
||||
creationEndTime,
|
||||
ownerID,
|
||||
limit: limit,
|
||||
asc: asc,
|
||||
ignoredCollectionIDs: archivedCollectionIds,
|
||||
);
|
||||
} else {
|
||||
result = await FilesDB.instance.getAllUploadedFiles(
|
||||
creationStartTime,
|
||||
creationEndTime,
|
||||
ownerID,
|
||||
limit: limit,
|
||||
asc: asc,
|
||||
ignoredCollectionIDs: archivedCollectionIds,
|
||||
);
|
||||
}
|
||||
result = await FilesDB.instance.getAllPendingOrUploadedFiles(
|
||||
creationStartTime,
|
||||
creationEndTime,
|
||||
ownerID,
|
||||
limit: limit,
|
||||
asc: asc,
|
||||
ignoredCollectionIDs: archivedCollectionIds,
|
||||
);
|
||||
}
|
||||
|
||||
// hide ignored files from home page UI
|
||||
final ignoredIDs = await IgnoredFilesService.instance.ignoredIDs;
|
||||
result.files.removeWhere(
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ class ArchivePage extends StatelessWidget {
|
|||
Widget build(Object context) {
|
||||
final gallery = Gallery(
|
||||
asyncLoader: (creationStartTime, creationEndTime, {limit, asc}) {
|
||||
return FilesDB.instance.getAllUploadedFiles(
|
||||
return FilesDB.instance.getAllPendingOrUploadedFiles(
|
||||
creationStartTime,
|
||||
creationEndTime,
|
||||
Configuration.instance.getUserID(),
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import 'package:photos/events/local_photos_updated_event.dart';
|
|||
import 'package:photos/models/device_collection.dart';
|
||||
import 'package:photos/models/gallery_type.dart';
|
||||
import 'package:photos/models/selected_files.dart';
|
||||
import 'package:photos/services/remote_sync_service.dart';
|
||||
import 'package:photos/ui/viewer/gallery/gallery.dart';
|
||||
import 'package:photos/ui/viewer/gallery/gallery_app_bar_widget.dart';
|
||||
import 'package:photos/ui/viewer/gallery/gallery_overlay_widget.dart';
|
||||
|
|
@ -114,7 +115,7 @@ class _BackupConfigurationHeaderWidgetState
|
|||
Switch(
|
||||
value: _isBackedUp,
|
||||
onChanged: (value) async {
|
||||
await FilesDB.instance.updateDevicePathSyncStatus(
|
||||
await RemoteSyncService.instance.updateDeviceFolderSyncStatus(
|
||||
{widget.deviceCollection.id: value},
|
||||
);
|
||||
_isBackedUp = value;
|
||||
|
|
|
|||
Loading…
Reference in a new issue