import 'package:logging/logging.dart'; import "package:photos/core/configuration.dart"; import 'package:photos/core/errors.dart'; import 'package:photos/core/network/network.dart'; import 'package:photos/db/files_db.dart'; import 'package:photos/models/duplicate_files.dart'; import 'package:photos/models/file.dart'; import "package:photos/services/collections_service.dart"; import "package:photos/services/files_service.dart"; class DeduplicationService { final _logger = Logger("DeduplicationService"); final _enteDio = NetworkClient.instance.enteDio; DeduplicationService._privateConstructor(); static final DeduplicationService instance = DeduplicationService._privateConstructor(); Future> getDuplicateFiles() async { try { final bool hasFileSizes = await FilesService.instance.hasMigratedSizes(); if (hasFileSizes) { final List result = await _getDuplicateFilesFromLocal(); return result; } final DuplicateFilesResponse dupes = await _fetchDuplicateFileIDs(); final ids = []; for (final dupe in dupes.duplicates) { ids.addAll(dupe.fileIDs); } final fileMap = await FilesDB.instance.getFilesFromIDs(ids); final result = []; final missingFileIDs = []; for (final dupe in dupes.duplicates) { final files = []; for (final id in dupe.fileIDs) { final file = fileMap[id]; if (file != null) { files.add(file); } else { missingFileIDs.add(id); } } // Place files that are available locally at first to minimize the chances // of a deletion followed by a re-upload files.sort((first, second) { if (first.localID != null && second.localID == null) { return -1; } else if (first.localID == null && second.localID != null) { return 1; } return 0; }); if (files.length > 1) { result.add(DuplicateFiles(files, dupe.size)); } } if (missingFileIDs.isNotEmpty) { _logger.severe( "Missing files", InvalidStateError( "Could not find " + missingFileIDs.length.toString() + " files in local DB: " + missingFileIDs.toString(), ), ); } return result; } catch (e, s) { _logger.severe("failed to get dedupeFile", e, s); rethrow; } } List clubDuplicates( List dupesBySize, { required String? Function(File) clubbingKey, }) { final dupesBySizeAndClubKey = []; for (final sizeBasedDupe in dupesBySize) { final Map> clubKeyToFilesMap = {}; for (final file in sizeBasedDupe.files) { final String? clubKey = clubbingKey(file); if (clubKey == null || clubKey.isEmpty) { continue; } if (!clubKeyToFilesMap.containsKey(clubKey)) { clubKeyToFilesMap[clubKey] = []; } clubKeyToFilesMap[clubKey]!.add(file); } for (final clubbingKey in clubKeyToFilesMap.keys) { final clubbedFiles = clubKeyToFilesMap[clubbingKey]!; if (clubbedFiles.length > 1) { dupesBySizeAndClubKey.add( DuplicateFiles(clubbedFiles, sizeBasedDupe.size), ); } } } return dupesBySizeAndClubKey; } Future> _getDuplicateFilesFromLocal() async { final List allFiles = await FilesDB.instance .getAllFilesFromDB(CollectionsService.instance.getHiddenCollections()); final int ownerID = Configuration.instance.getUserID()!; allFiles.removeWhere( (f) => !f.isUploaded || (f.ownerID ?? 0) != ownerID || (f.fileSize ?? 0) <= 0, ); final Map> sizeToFilesMap = {}; for (final file in allFiles) { if (!sizeToFilesMap.containsKey(file.fileSize)) { sizeToFilesMap[file.fileSize!] = []; } sizeToFilesMap[file.fileSize]!.add(file); } final List dupesBySize = []; for (final size in sizeToFilesMap.keys) { final List files = sizeToFilesMap[size]!; if (files.length > 1) { dupesBySize.add(DuplicateFiles(files, size)); } } return dupesBySize; } Future _fetchDuplicateFileIDs() async { final response = await _enteDio.get("/files/duplicates"); return DuplicateFilesResponse.fromMap(response.data); } }