import 'dart:async'; import 'dart:core'; import 'dart:io'; import 'package:logging/logging.dart'; import "package:photo_manager/photo_manager.dart"; import "package:photos/core/configuration.dart"; import 'package:photos/core/errors.dart'; import 'package:photos/db/file_updation_db.dart'; import 'package:photos/db/files_db.dart'; import "package:photos/extensions/list.dart"; import "package:photos/extensions/stop_watch.dart"; import "package:photos/models/file/extensions/file_props.dart"; import 'package:photos/models/file/file.dart'; import 'package:photos/models/file/file_type.dart'; import "package:photos/services/files_service.dart"; import 'package:photos/utils/file_uploader_util.dart'; import 'package:photos/utils/file_util.dart'; import 'package:shared_preferences/shared_preferences.dart'; // LocalFileUpdateService tracks all the potential local file IDs which have // changed/modified on the device and needed to be uploaded again. class LocalFileUpdateService { late FileUpdationDB _fileUpdationDB; late SharedPreferences _prefs; late Logger _logger; final String _iosLivePhotoSizeMigrationDone = 'fm_ios_live_photo_check'; final String _doneLivePhotoImport = 'fm_import_ios_live_photo_check'; final String _androidMissingGPSImportDone = 'fm_android_missing_gps_import_done'; final String _androidMissingGPSCheckDone = 'fm_android_missing_gps_check_done'; static int twoHundredKb = 200 * 1024; final List _oldMigrationKeys = [ 'fm_badCreationTime', 'fm_badCreationTimeCompleted', 'fm_missingLocationV2ImportDone', 'fm_missingLocationV2MigrationDone', 'fm_badLocationImportDone', 'fm_badLocationMigrationDone', 'fm_ios_live_photo_size', 'fm_import_ios_live_photo_size', ]; Completer? _existingMigration; LocalFileUpdateService._privateConstructor() { _logger = Logger((LocalFileUpdateService).toString()); _fileUpdationDB = FileUpdationDB.instance; } void init(SharedPreferences preferences) { _prefs = preferences; } static LocalFileUpdateService instance = LocalFileUpdateService._privateConstructor(); Future markUpdatedFilesForReUpload() async { if (_existingMigration != null) { _logger.info("migration is already in progress, skipping"); return _existingMigration!.future; } _existingMigration = Completer(); try { await _markFilesWhichAreActuallyUpdated(); _cleanUpOlderMigration().ignore(); if (!Platform.isAndroid) { await _handleLivePhotosSizedCheck(); } if (Platform.isAndroid) { await _androidMissingGPSCheck(); } } catch (e, s) { _logger.severe('failed to perform migration', e, s); } finally { _existingMigration?.complete(); _existingMigration = null; } } Future _cleanUpOlderMigration() async { // check if any old_migration_keys are present in shared preferences bool hasOldMigrationKey = false; for (String key in _oldMigrationKeys) { if (_prefs.containsKey(key)) { hasOldMigrationKey = true; break; } } if (hasOldMigrationKey) { await _fileUpdationDB.deleteByReasons([ 'missing_location', 'badCreationTime', 'missingLocationV2', 'badLocationCord', 'livePhotoSize', ]); for (var element in _oldMigrationKeys) { await _prefs.remove(element); } } } // This method analyses all of local files for which the file // modification/update time was changed. It checks if the existing fileHash // is different from the hash of uploaded file. If fileHash are different, // then it marks the file for file update. Future _markFilesWhichAreActuallyUpdated() async { final sTime = DateTime.now().microsecondsSinceEpoch; // singleRunLimit indicates number of files to check during single // invocation of this method. The limit act as a crude way to limit the // resource consumed by the method const int singleRunLimit = 10; final localIDsToProcess = await _fileUpdationDB.getLocalIDsForPotentialReUpload( singleRunLimit, FileUpdationDB.modificationTimeUpdated, ); if (localIDsToProcess.isNotEmpty) { await _checkAndMarkFilesWithDifferentHashForFileUpdate( localIDsToProcess, ); final eTime = DateTime.now().microsecondsSinceEpoch; final d = Duration(microseconds: eTime - sTime); _logger.info( 'Performed hashCheck for ${localIDsToProcess.length} updated files ' 'completed in ${d.inSeconds.toString()} secs', ); } } Future _checkAndMarkFilesWithDifferentHashForFileUpdate( List localIDsToProcess, ) async { final int userID = Configuration.instance.getUserID()!; final List result = await FilesDB.instance.getLocalFiles(localIDsToProcess); final List localFilesForUser = []; final Set localIDsWithFile = {}; for (EnteFile file in result) { if (file.ownerID == null || file.ownerID == userID) { localFilesForUser.add(file); localIDsWithFile.add(file.localID!); } } final Set processedIDs = {}; // if a file for localID doesn't exist, then mark it as processed // otherwise the app will be stuck in retrying same set of ids for (String localID in localIDsToProcess) { if (!localIDsWithFile.contains(localID)) { processedIDs.add(localID); } } _logger.info("files to process ${localIDsToProcess.length} for reupload, " "missing localFile cnt ${processedIDs.length}"); for (EnteFile file in localFilesForUser) { if (processedIDs.contains(file.localID)) { continue; } MediaUploadData uploadData; try { uploadData = await getUploadData(file); if (uploadData.hashData != null && file.hash != null && (file.hash == uploadData.hashData!.fileHash || file.hash == uploadData.hashData!.zipHash)) { _logger.info("Skip file update as hash matched ${file.tag}"); } else { _logger.info( "Marking for file update as hash did not match ${file.tag}", ); await clearCache(file); await FilesDB.instance.markFilesForReUpload( userID, file.localID!, file.title, file.location, file.creationTime!, file.modificationTime!, file.fileType, ); } processedIDs.add(file.localID!); } on InvalidFileError catch (e) { if (e.reason == InvalidReason.livePhotoToImageTypeChanged || e.reason == InvalidReason.imageToLivePhotoTypeChanged) { late FileType fileType; if (e.reason == InvalidReason.livePhotoToImageTypeChanged) { fileType = FileType.image; } else if (e.reason == InvalidReason.imageToLivePhotoTypeChanged) { fileType = FileType.livePhoto; } final int count = await FilesDB.instance.markFilesForReUpload( userID, file.localID!, file.title, file.location, file.creationTime!, file.modificationTime!, fileType, ); _logger.fine('fileType changed for ${file.tag} to ${e.reason} for ' '$count files'); } else { _logger.severe("failed to check hash: invalid file ${file.tag}", e); } processedIDs.add(file.localID!); } catch (e) { _logger.severe("Failed to check hash", e); } finally {} } await _fileUpdationDB.deleteByLocalIDs( processedIDs.toList(), FileUpdationDB.modificationTimeUpdated, ); } Future checkLivePhoto(EnteFile file) async { if (file.localID == null || file.localID!.isEmpty || !file.isUploaded || file.fileType != FileType.livePhoto || !file.isOwner) { return; } if (_prefs.containsKey(_iosLivePhotoSizeMigrationDone)) { return; } final hasEntry = await _fileUpdationDB.isExisting( file.localID!, FileUpdationDB.livePhotoCheck, ); if (hasEntry) { _logger.info('eager checkLivePhoto ${file.tag}'); await _checkLivePhotoWithLowOrUnknownSize([file.localID!]); } } Future _handleLivePhotosSizedCheck() async { try { if (_prefs.containsKey(_iosLivePhotoSizeMigrationDone)) { return; } await _importLivePhotoReUploadCandidates(); // singleRunLimit indicates number of files to check during single // invocation of this method. The limit act as a crude way to limit the // resource consumed by the method const int singleRunLimit = 500; final localIDsToProcess = await _fileUpdationDB.getLocalIDsForPotentialReUpload( singleRunLimit, FileUpdationDB.livePhotoCheck, ); if (localIDsToProcess.isNotEmpty) { final chunksOf50 = localIDsToProcess.chunks(50); for (final chunk in chunksOf50) { final sTime = DateTime.now().microsecondsSinceEpoch; final List futures = []; final chunkOf10 = chunk.chunks(10); for (final smallChunk in chunkOf10) { futures.add(_checkLivePhotoWithLowOrUnknownSize(smallChunk)); } await Future.wait(futures); final eTime = DateTime.now().microsecondsSinceEpoch; final d = Duration(microseconds: eTime - sTime); _logger.info( 'Performed hashCheck for ${chunk.length} livePhoto files ' 'completed in ${d.inSeconds.toString()} secs', ); } } else { await _prefs.setBool(_iosLivePhotoSizeMigrationDone, true); } } catch (e, s) { _logger.severe('error while checking livePhotoSize check', e, s); } } Future _checkLivePhotoWithLowOrUnknownSize( List localIDsToProcess, ) async { final int userID = Configuration.instance.getUserID()!; final List result = await FilesDB.instance.getLocalFiles(localIDsToProcess); final List localFilesForUser = []; final Set localIDsWithFile = {}; final Set missingSizeIDs = {}; final Set processedIDs = {}; for (EnteFile file in result) { if (file.ownerID == null || file.ownerID == userID) { localFilesForUser.add(file); localIDsWithFile.add(file.localID!); if (file.isUploaded && file.fileSize == null) { missingSizeIDs.add(file.uploadedFileID!); } if (file.isUploaded && file.updationTime == null) { // file already queued for re-upload processedIDs.add(file.localID!); } } } if (missingSizeIDs.isNotEmpty) { await FilesService.instance.backFillSizes(missingSizeIDs.toList()); _logger.info('sizes back fill for ${missingSizeIDs.length} files'); // return early, let the check run in the next batch return; } // if a file for localID doesn't exist, then mark it as processed // otherwise the app will be stuck in retrying same set of ids for (String localID in localIDsToProcess) { if (!localIDsWithFile.contains(localID)) { processedIDs.add(localID); } } _logger.info(" check ${localIDsToProcess.length} files for livePhotoSize, " "missing file cnt ${processedIDs.length}"); for (EnteFile file in localFilesForUser) { if (file.fileSize == null) { _logger.info('fileSize still null, skip this file'); continue; } else if (file.fileType != FileType.livePhoto) { _logger.severe('fileType is not livePhoto, skip this file'); processedIDs.add(file.localID!); continue; } if (processedIDs.contains(file.localID)) { continue; } try { late MediaUploadData uploadData; late int mediaUploadSize; (uploadData, mediaUploadSize) = await getUploadDataWithSizeSize(file); if ((file.fileSize! - mediaUploadSize).abs() > twoHundredKb) { _logger.info( 'Re-upload livePhoto localHash ${uploadData.hashData?.fileHash ?? "null"} & localSize: $mediaUploadSize' ' and remoteHash ${file.hash ?? "null"} & removeSize: ${file.fileSize!}', ); await FilesDB.instance.markFilesForReUpload( userID, file.localID!, file.title, file.location, file.creationTime!, file.modificationTime!, file.fileType, ); } processedIDs.add(file.localID!); } on InvalidFileError catch (e) { if (e.reason == InvalidReason.livePhotoToImageTypeChanged || e.reason == InvalidReason.imageToLivePhotoTypeChanged) { // let existing file update check handle this case _fileUpdationDB.insertMultiple( [file.localID!], FileUpdationDB.modificationTimeUpdated, ).ignore(); } else { _logger.severe("livePhoto check failed: invalid file ${file.tag}", e); } processedIDs.add(file.localID!); } catch (e) { _logger.severe("livePhoto check failed", e); } finally {} } _logger.info('completed check for ${localIDsToProcess.length} files'); await _fileUpdationDB.deleteByLocalIDs( processedIDs.toList(), FileUpdationDB.livePhotoCheck, ); } Future _importLivePhotoReUploadCandidates() async { if (_prefs.containsKey(_doneLivePhotoImport)) { return; } _logger.info('_importLivePhotoReUploadCandidates'); final EnteWatch watch = EnteWatch("_importLivePhotoReUploadCandidates"); final int ownerID = Configuration.instance.getUserID()!; final List localIDs = await FilesDB.instance.getLivePhotosForUser(ownerID); await _fileUpdationDB.insertMultiple( localIDs, FileUpdationDB.livePhotoCheck, ); watch.log("imported ${localIDs.length} files"); await _prefs.setBool(_doneLivePhotoImport, true); } //#region Android Missing GPS specific methods ### Future _androidMissingGPSCheck() async { if (_prefs.containsKey(_androidMissingGPSCheckDone)) { return; } await _importAndroidBadGPSCandidate(); // singleRunLimit indicates number of files to check during single // invocation of this method. The limit act as a crude way to limit the // resource consumed by the method const int singleRunLimit = 500; final localIDsToProcess = await _fileUpdationDB.getLocalIDsForPotentialReUpload( singleRunLimit, FileUpdationDB.androidMissingGPS, ); if (localIDsToProcess.isNotEmpty) { final chunksOf50 = localIDsToProcess.chunks(50); for (final chunk in chunksOf50) { final sTime = DateTime.now().microsecondsSinceEpoch; final List futures = []; final chunkOf10 = chunk.chunks(10); for (final smallChunk in chunkOf10) { futures.add(_checkForMissingGPS(smallChunk)); } await Future.wait(futures); final eTime = DateTime.now().microsecondsSinceEpoch; final d = Duration(microseconds: eTime - sTime); _logger.info( 'Performed missing GPS Location check for ${chunk.length} files ' 'completed in ${d.inSeconds.toString()} secs', ); } } else { _logger.info('Completed android missing GPS check'); await _prefs.setBool(_androidMissingGPSCheckDone, true); } } Future _checkForMissingGPS(List localIDs) async { try { final List localFiles = await FilesDB.instance.getLocalFiles(localIDs); final ownerID = Configuration.instance.getUserID()!; final Set localIDsWithFile = {}; final Set reuploadCandidate = {}; final Set processedIDs = {}; for (EnteFile file in localFiles) { if (file.localID == null) continue; // ignore files that are not uploaded or have different owner if (!file.isUploaded || file.ownerID! != ownerID) { processedIDs.add(file.localID!); } if (file.hasLocation) { processedIDs.add(file.localID!); } } for (EnteFile enteFile in localFiles) { try { if (enteFile.localID == null || processedIDs.contains(enteFile.localID!)) { continue; } final localID = enteFile.localID!; localIDsWithFile.add(localID); final AssetEntity? entity = await AssetEntity.fromId(localID); if (entity == null) { processedIDs.add(localID); } else { final latLng = await entity.latlngAsync(); if ((latLng.longitude ?? 0) == 0 || (latLng.latitude ?? 0) == 0) { processedIDs.add(localID); } else { reuploadCandidate.add(localID); processedIDs.add(localID); } } } catch (e, s) { processedIDs.add(enteFile.localID!); _logger.severe('lat/long check file ${enteFile.toString()}', e, s); } } for (String id in localIDs) { // if the file with given localID doesn't exist, consider it as done. if (!localIDsWithFile.contains(id)) { processedIDs.add(id); } } await FileUpdationDB.instance.insertMultiple( reuploadCandidate.toList(), FileUpdationDB.modificationTimeUpdated, ); await FileUpdationDB.instance.deleteByLocalIDs( processedIDs.toList(), FileUpdationDB.androidMissingGPS, ); } catch (e, s) { _logger.severe('error while checking missing GPS', e, s); } } Future _importAndroidBadGPSCandidate() async { if (_prefs.containsKey(_androidMissingGPSImportDone)) { return; } final sTime = DateTime.now().microsecondsSinceEpoch; _logger.info('importing files without missing GPS'); final int ownerID = Configuration.instance.getUserID()!; final fileLocalIDs = await FilesDB.instance.getLocalFilesBackedUpWithoutLocation(ownerID); await _fileUpdationDB.insertMultiple( fileLocalIDs, FileUpdationDB.androidMissingGPS, ); final eTime = DateTime.now().microsecondsSinceEpoch; final d = Duration(microseconds: eTime - sTime); _logger.info( 'importing completed, total files count ${fileLocalIDs.length} and took ${d.inSeconds.toString()} seconds', ); await _prefs.setBool(_androidMissingGPSImportDone, true); } //#endregion Android Missing GPS specific methods ### Future getUploadData(EnteFile file) async { final mediaUploadData = await getUploadDataFromEnteFile(file); // delete the file from app's internal cache if it was copied to app // for upload. Shared Media should only be cleared when the upload // succeeds. if (Platform.isIOS && mediaUploadData.sourceFile != null) { await mediaUploadData.sourceFile?.delete(); } return mediaUploadData; } Future<(MediaUploadData, int)> getUploadDataWithSizeSize( EnteFile file, ) async { final mediaUploadData = await getUploadDataFromEnteFile(file); final int size = await mediaUploadData.sourceFile!.length(); // delete the file from app's internal cache if it was copied to app // for upload. Shared Media should only be cleared when the upload // succeeds. if (Platform.isIOS && mediaUploadData.sourceFile != null) { await mediaUploadData.sourceFile?.delete(); } return (mediaUploadData, size); } }