import 'dart:io'; import 'dart:math'; import 'package:computer/computer.dart'; import 'package:logging/logging.dart'; import 'package:photo_manager/photo_manager.dart'; import 'package:photos/core/event_bus.dart'; import 'package:photos/events/local_import_progress.dart'; import 'package:photos/models/file/file.dart'; import 'package:tuple/tuple.dart'; final _logger = Logger("FileSyncUtil"); const ignoreSizeConstraint = SizeConstraint(ignoreSize: true); const assetFetchPageSize = 2000; Future, List>> getLocalPathAssetsAndFiles( int fromTime, int toTime, ) async { final pathEntities = await _getGalleryList( updateFromTime: fromTime, updateToTime: toTime, ); final List localPathAssets = []; // alreadySeenLocalIDs is used to track and ignore file with particular // localID if it's already present in another album. This only impacts iOS // devices where a file can belong to multiple final Set alreadySeenLocalIDs = {}; final List uniqueFiles = []; for (AssetPathEntity pathEntity in pathEntities) { final List assetsInPath = await _getAllAssetLists(pathEntity); late Tuple2, List> result; if (assetsInPath.isEmpty) { result = const Tuple2({}, []); } else { result = await Computer.shared().compute( _getLocalIDsAndFilesFromAssets, param: { "pathEntity": pathEntity, "fromTime": fromTime, "alreadySeenLocalIDs": alreadySeenLocalIDs, "assetList": assetsInPath, }, taskName: "getLocalPathAssetsAndFiles-${pathEntity.name}-count-${assetsInPath.length}", ); alreadySeenLocalIDs.addAll(result.item1); uniqueFiles.addAll(result.item2); } localPathAssets.add( LocalPathAsset( localIDs: result.item1, pathName: pathEntity.name, pathID: pathEntity.id, ), ); } return Tuple2(localPathAssets, uniqueFiles); } // getDeviceFolderWithCountAndLatestFile returns a tuple of AssetPathEntity and // latest file's localID in the assetPath, along with modifiedPath time and // total count of assets in a Asset Path. // We use this result to update the latest thumbnail for deviceFolder and // identify (in future) which AssetPath needs to be re-synced again. Future>> getDeviceFolderWithCountAndCoverID() async { final List> result = []; final pathEntities = await _getGalleryList( needsTitle: false, containsModifiedPath: true, orderOption: const OrderOption(type: OrderOptionType.createDate, asc: false), ); for (AssetPathEntity pathEntity in pathEntities) { final latestEntity = await pathEntity.getAssetListPaged( page: 0, size: 1, ); final String localCoverID = latestEntity.isEmpty ? '' : latestEntity.first.id; result.add(Tuple2(pathEntity, localCoverID)); } return result; } Future> getAllLocalAssets() async { final filterOptionGroup = FilterOptionGroup(); filterOptionGroup.setOption( AssetType.image, const FilterOption(sizeConstraint: ignoreSizeConstraint), ); filterOptionGroup.setOption( AssetType.video, const FilterOption(sizeConstraint: ignoreSizeConstraint), ); filterOptionGroup.createTimeCond = DateTimeCond.def().copyWith(ignore: true); final assetPaths = await PhotoManager.getAssetPathList( hasAll: !Platform.isAndroid, type: RequestType.common, filterOption: filterOptionGroup, ); final List localPathAssets = []; for (final assetPath in assetPaths) { final Set localIDs = {}; for (final asset in await _getAllAssetLists(assetPath)) { localIDs.add(asset.id); } localPathAssets.add( LocalPathAsset( localIDs: localIDs, pathName: assetPath.name, pathID: assetPath.id, ), ); } return localPathAssets; } Future getDiffWithLocal( List assets, // current set of assets available on device Set existingIDs, // localIDs of files already imported in app Map> pathToLocalIDs, Set invalidIDs, ) async { final Map args = {}; args['assets'] = assets; args['existingIDs'] = existingIDs; args['invalidIDs'] = invalidIDs; args['pathToLocalIDs'] = pathToLocalIDs; final LocalDiffResult diffResult = await Computer.shared().compute( _getLocalAssetsDiff, param: args, taskName: "getLocalAssetsDiff", ); if (diffResult.localPathAssets != null) { diffResult.uniqueLocalFiles = await _convertLocalAssetsToUniqueFiles(diffResult.localPathAssets!); } return diffResult; } // _getLocalAssetsDiff compares local db with the file system and compute // the files which needs to be added or removed from device collection. LocalDiffResult _getLocalAssetsDiff(Map args) { final List onDeviceLocalPathAsset = args['assets']; final Set existingIDs = args['existingIDs']; final Set invalidIDs = args['invalidIDs']; final Map> pathToLocalIDs = args['pathToLocalIDs']; final Map> newPathToLocalIDs = >{}; final Map> removedPathToLocalIDs = >{}; final List unsyncedAssets = []; for (final localPathAsset in onDeviceLocalPathAsset) { final String pathID = localPathAsset.pathID; // Start identifying pathID to localID mapping changes which needs to be // synced final Set candidateLocalIDsForRemoval = pathToLocalIDs[pathID] ?? {}; final Set missingLocalIDsInPath = {}; for (final String localID in localPathAsset.localIDs) { if (candidateLocalIDsForRemoval.contains(localID)) { // remove the localID after checking. Any pending existing ID indicates // the the local file was removed from the path. candidateLocalIDsForRemoval.remove(localID); } else { missingLocalIDsInPath.add(localID); } } if (candidateLocalIDsForRemoval.isNotEmpty) { removedPathToLocalIDs[pathID] = candidateLocalIDsForRemoval; } if (missingLocalIDsInPath.isNotEmpty) { newPathToLocalIDs[pathID] = missingLocalIDsInPath; } // End localPathAsset.localIDs.removeAll(existingIDs); localPathAsset.localIDs.removeAll(invalidIDs); if (localPathAsset.localIDs.isNotEmpty) { unsyncedAssets.add(localPathAsset); } } return LocalDiffResult( localPathAssets: unsyncedAssets, newPathToLocalIDs: newPathToLocalIDs, deletePathToLocalIDs: removedPathToLocalIDs, ); } Future> _convertLocalAssetsToUniqueFiles( List assets, ) async { final Set alreadySeenLocalIDs = {}; final List files = []; for (LocalPathAsset localPathAsset in assets) { final String localPathName = localPathAsset.pathName; for (final String localID in localPathAsset.localIDs) { if (!alreadySeenLocalIDs.contains(localID)) { final assetEntity = await AssetEntity.fromId(localID); if (assetEntity == null) { _logger.warning('Failed to fetch asset with id $localID'); continue; } files.add( await EnteFile.fromAsset(localPathName, assetEntity), ); alreadySeenLocalIDs.add(localID); } } } return files; } /// returns a list of AssetPathEntity with relevant filter operations. /// [needTitle] impacts the performance for fetching the actual [AssetEntity] /// in iOS. Same is true for [containsModifiedPath] Future> _getGalleryList({ final int? updateFromTime, final int? updateToTime, final bool containsModifiedPath = false, // in iOS fetching the AssetEntity title impacts performance final bool needsTitle = true, final OrderOption? orderOption, }) async { final filterOptionGroup = FilterOptionGroup(); filterOptionGroup.setOption( AssetType.image, FilterOption(needTitle: needsTitle, sizeConstraint: ignoreSizeConstraint), ); filterOptionGroup.setOption( AssetType.video, FilterOption(needTitle: needsTitle, sizeConstraint: ignoreSizeConstraint), ); if (orderOption != null) { filterOptionGroup.addOrderOption(orderOption); } if (updateFromTime != null && updateToTime != null) { filterOptionGroup.updateTimeCond = DateTimeCond( min: DateTime.fromMillisecondsSinceEpoch(updateFromTime ~/ 1000), max: DateTime.fromMillisecondsSinceEpoch(updateToTime ~/ 1000), ); } filterOptionGroup.containsPathModified = containsModifiedPath; final galleryList = await PhotoManager.getAssetPathList( hasAll: !Platform.isAndroid, type: RequestType.common, filterOption: filterOptionGroup, ); galleryList.sort((s1, s2) { if (s1.isAll) { return 1; } return 0; }); return galleryList; } Future> _getAllAssetLists(AssetPathEntity pathEntity) async { final List result = []; int currentPage = 0; List currentPageResult = []; do { currentPageResult = await pathEntity.getAssetListPaged( page: currentPage, size: assetFetchPageSize, ); Bus.instance.fire( LocalImportProgressEvent( pathEntity.name, currentPage * assetFetchPageSize + currentPageResult.length, ), ); result.addAll(currentPageResult); currentPage = currentPage + 1; } while (currentPageResult.length >= assetFetchPageSize); return result; } // review: do we need to run this inside compute, after making File.FromAsset // sync. If yes, update the method documentation with reason. Future, List>> _getLocalIDsAndFilesFromAssets( Map args, ) async { final pathEntity = args["pathEntity"] as AssetPathEntity; final assetList = args["assetList"]; final fromTime = args["fromTime"]; final alreadySeenLocalIDs = args["alreadySeenLocalIDs"] as Set; final List files = []; final Set localIDs = {}; for (AssetEntity entity in assetList) { localIDs.add(entity.id); final bool assetCreatedOrUpdatedAfterGivenTime = max( entity.createDateTime.millisecondsSinceEpoch, entity.modifiedDateTime.millisecondsSinceEpoch, ) >= (fromTime / ~1000); if (!alreadySeenLocalIDs.contains(entity.id) && assetCreatedOrUpdatedAfterGivenTime) { try { final file = await EnteFile.fromAsset(pathEntity.name, entity); files.add(file); } catch (e) { _logger.severe(e); } } } return Tuple2(localIDs, files); } class LocalPathAsset { final Set localIDs; final String pathID; final String pathName; LocalPathAsset({ required this.localIDs, required this.pathName, required this.pathID, }); } class LocalDiffResult { // unique localPath Assets. final List? localPathAssets; // set of File object created from localPathAssets List? uniqueLocalFiles; // newPathToLocalIDs represents new entries which needs to be synced to // the local db final Map>? newPathToLocalIDs; final Map>? deletePathToLocalIDs; LocalDiffResult({ this.uniqueLocalFiles, this.localPathAssets, this.newPathToLocalIDs, this.deletePathToLocalIDs, }); }