// @dart = 2.9 import 'dart:io'; import 'dart:math'; import 'package:computer/computer.dart'; import 'package:flutter/foundation.dart'; import 'package:logging/logging.dart'; import 'package:photo_manager/photo_manager.dart'; import 'package:photos/core/event_bus.dart'; import 'package:photos/events/local_import_progress.dart'; import 'package:photos/models/file.dart'; import 'package:tuple/tuple.dart'; final _logger = Logger("FileSyncUtil"); const ignoreSizeConstraint = SizeConstraint(ignoreSize: true); const assetFetchPageSize = 2000; Future, List>> getLocalPathAssetsAndFiles( int fromTime, int toTime, Computer computer, ) async { final pathEntities = await _getGalleryList( updateFromTime: fromTime, updateToTime: toTime, ); final List localPathAssets = []; // alreadySeenLocalIDs is used to track and ignore file with particular // localID if it's already present in another album. This only impacts iOS // devices where a file can belong to multiple final Set alreadySeenLocalIDs = {}; final List uniqueFiles = []; for (AssetPathEntity pathEntity in pathEntities) { final List assetsInPath = await _getAllAssetLists(pathEntity); final Tuple2, List> result = await computer.compute( _getLocalIDsAndFilesFromAssets, param: { "pathEntity": pathEntity, "fromTime": fromTime, "alreadySeenLocalIDs": alreadySeenLocalIDs, "assetList": assetsInPath, }, ); alreadySeenLocalIDs.addAll(result.item1); uniqueFiles.addAll(result.item2); localPathAssets.add( LocalPathAsset( localIDs: result.item1, pathName: pathEntity.name, pathID: pathEntity.id, ), ); } return Tuple2(localPathAssets, uniqueFiles); } // getDeviceFolderWithCountAndLatestFile returns a tuple of AssetPathEntity and // latest file's localID in the assetPath, along with modifiedPath time and // total count of assets in a Asset Path. // We use this result to update the latest thumbnail for deviceFolder and // identify (in future) which AssetPath needs to be re-synced again. Future>> getDeviceFolderWithCountAndCoverID() async { final List> result = []; final pathEntities = await _getGalleryList( needsTitle: false, containsModifiedPath: true, orderOption: const OrderOption(type: OrderOptionType.createDate, asc: false), ); for (AssetPathEntity pathEntity in pathEntities) { //todo: test and handle empty album case final latestEntity = await pathEntity.getAssetListPaged( page: 0, size: 1, ); final String localCoverID = latestEntity.first.id; result.add(Tuple2(pathEntity, localCoverID)); } return result; } Future> getAllLocalAssets() async { final filterOptionGroup = FilterOptionGroup(); filterOptionGroup.setOption( AssetType.image, const FilterOption(sizeConstraint: ignoreSizeConstraint), ); filterOptionGroup.setOption( AssetType.video, const FilterOption(sizeConstraint: ignoreSizeConstraint), ); filterOptionGroup.createTimeCond = DateTimeCond.def().copyWith(ignore: true); final assetPaths = await PhotoManager.getAssetPathList( hasAll: !Platform.isAndroid, type: RequestType.common, filterOption: filterOptionGroup, ); final List localPathAssets = []; for (final assetPath in assetPaths) { final Set localIDs = {}; for (final asset in await _getAllAssetLists(assetPath)) { localIDs.add(asset.id); } localPathAssets.add( LocalPathAsset( localIDs: localIDs, pathName: assetPath.name, pathID: assetPath.id, ), ); } return localPathAssets; } Future getDiffWithLocal( List assets, // current set of assets available on device Set existingIDs, // localIDs of files already imported in app Map> pathToLocalIDs, Set invalidIDs, Computer computer, ) async { final Map args = {}; args['assets'] = assets; args['existingIDs'] = existingIDs; args['invalidIDs'] = invalidIDs; args['pathToLocalIDs'] = pathToLocalIDs; final LocalDiffResult diffResult = await computer.compute(_getLocalAssetsDiff, param: args); diffResult.uniqueLocalFiles = await _convertLocalAssetsToUniqueFiles(diffResult.localPathAssets); return diffResult; } // _getLocalAssetsDiff compares local db with the file system and compute // the files which needs to be added or removed from device collection. LocalDiffResult _getLocalAssetsDiff(Map args) { final List onDeviceLocalPathAsset = args['assets']; final Set existingIDs = args['existingIDs']; final Set invalidIDs = args['invalidIDs']; final Map> pathToLocalIDs = args['pathToLocalIDs']; final Map> newPathToLocalIDs = >{}; final Map> removedPathToLocalIDs = >{}; final List unsyncedAssets = []; for (final localPathAsset in onDeviceLocalPathAsset) { final String pathID = localPathAsset.pathID; // Start identifying pathID to localID mapping changes which needs to be // synced final Set candidateLocalIDsForRemoval = pathToLocalIDs[pathID] ?? {}; final Set missingLocalIDsInPath = {}; for (final String localID in localPathAsset.localIDs) { if (candidateLocalIDsForRemoval.contains(localID)) { // remove the localID after checking. Any pending existing ID indicates // the the local file was removed from the path. candidateLocalIDsForRemoval.remove(localID); } else { missingLocalIDsInPath.add(localID); } } if (candidateLocalIDsForRemoval.isNotEmpty) { removedPathToLocalIDs[pathID] = candidateLocalIDsForRemoval; } if (missingLocalIDsInPath.isNotEmpty) { newPathToLocalIDs[pathID] = missingLocalIDsInPath; } // End localPathAsset.localIDs.removeAll(existingIDs); localPathAsset.localIDs.removeAll(invalidIDs); if (localPathAsset.localIDs.isNotEmpty) { unsyncedAssets.add(localPathAsset); } } return LocalDiffResult( localPathAssets: unsyncedAssets, newPathToLocalIDs: newPathToLocalIDs, deletePathToLocalIDs: removedPathToLocalIDs, ); } Future> _convertLocalAssetsToUniqueFiles( List assets, ) async { final Set alreadySeenLocalIDs = {}; final List files = []; for (LocalPathAsset localPathAsset in assets) { final String localPathName = localPathAsset.pathName; for (final String localID in localPathAsset.localIDs) { if (!alreadySeenLocalIDs.contains(localID)) { final assetEntity = await AssetEntity.fromId(localID); files.add( await File.fromAsset(localPathName, assetEntity), ); alreadySeenLocalIDs.add(localID); } } } return files; } /// returns a list of AssetPathEntity with relevant filter operations. /// [needTitle] impacts the performance for fetching the actual [AssetEntity] /// in iOS. Same is true for [containsModifiedPath] Future> _getGalleryList({ final int updateFromTime, final int updateToTime, final bool containsModifiedPath = false, // in iOS fetching the AssetEntity title impacts performance final bool needsTitle = true, final OrderOption orderOption, }) async { final filterOptionGroup = FilterOptionGroup(); filterOptionGroup.setOption( AssetType.image, FilterOption(needTitle: needsTitle, sizeConstraint: ignoreSizeConstraint), ); filterOptionGroup.setOption( AssetType.video, FilterOption(needTitle: needsTitle, sizeConstraint: ignoreSizeConstraint), ); if (orderOption != null) { filterOptionGroup.addOrderOption(orderOption); } if (updateFromTime != null && updateToTime != null) { filterOptionGroup.updateTimeCond = DateTimeCond( min: DateTime.fromMicrosecondsSinceEpoch(updateFromTime), max: DateTime.fromMicrosecondsSinceEpoch(updateToTime), ); } filterOptionGroup.containsPathModified = containsModifiedPath; final galleryList = await PhotoManager.getAssetPathList( hasAll: !Platform.isAndroid, type: RequestType.common, filterOption: filterOptionGroup, ); galleryList.sort((s1, s2) { if (s1.isAll) { return 1; } return 0; }); return galleryList; } Future> _getAllAssetLists(AssetPathEntity pathEntity) async { final List result = []; int currentPage = 0; List currentPageResult = []; do { currentPageResult = await pathEntity.getAssetListPaged( page: currentPage, size: assetFetchPageSize, ); Bus.instance.fire( LocalImportProgressEvent(pathEntity.name, currentPage * assetFetchPageSize + currentPageResult.length), ); result.addAll(currentPageResult); currentPage = currentPage + 1; } while (currentPageResult.length >= assetFetchPageSize); return result; } // review: do we need to run this inside compute, after making File.FromAsset // sync. If yes, update the method documentation with reason. Future, List>> _getLocalIDsAndFilesFromAssets( Map args, ) async { final pathEntity = args["pathEntity"] as AssetPathEntity; final assetList = args["assetList"]; final fromTime = args["fromTime"]; final alreadySeenLocalIDs = args["alreadySeenLocalIDs"] as Set; final List files = []; final Set localIDs = {}; for (AssetEntity entity in assetList) { localIDs.add(entity.id); final bool assetCreatedOrUpdatedAfterGivenTime = max( entity.createDateTime.microsecondsSinceEpoch, entity.modifiedDateTime.microsecondsSinceEpoch, ) > fromTime; if (!alreadySeenLocalIDs.contains(entity.id) && assetCreatedOrUpdatedAfterGivenTime) { try { final file = await File.fromAsset(pathEntity.name, entity); files.add(file); } catch (e) { _logger.severe(e); } } } return Tuple2(localIDs, files); } class LocalPathAsset { final Set localIDs; final String pathID; final String pathName; LocalPathAsset({ @required this.localIDs, @required this.pathName, @required this.pathID, }); } class LocalDiffResult { // unique localPath Assets. final List localPathAssets; // set of File object created from localPathAssets List uniqueLocalFiles; // newPathToLocalIDs represents new entries which needs to be synced to // the local db final Map> newPathToLocalIDs; final Map> deletePathToLocalIDs; LocalDiffResult({ this.uniqueLocalFiles, this.localPathAssets, this.newPathToLocalIDs, this.deletePathToLocalIDs, }); }