diff --git a/lib/db/file_updation_db.dart b/lib/db/file_updation_db.dart index 5d69fe8b4..22f317924 100644 --- a/lib/db/file_updation_db.dart +++ b/lib/db/file_updation_db.dart @@ -14,16 +14,8 @@ class FileUpdationDB { static const tableName = 're_upload_tracker'; static const columnLocalID = 'local_id'; static const columnReason = 'reason'; - static const missingLocation = 'missing_location'; - static const modificationTimeUpdated = 'modificationTimeUpdated'; - static const badCreationTime = 'badCreationTime'; - // refers to the patching of files which had location in exif but the app - // did not extracted the location to include it in the file metadata - static const missingLocationV2 = 'missingLocationV2'; - // refers to the patching of files which had location in exif but the app - // mapped the location to a wrong location - static const badLocationCord = 'badLocationCord'; + static const modificationTimeUpdated = 'modificationTimeUpdated'; // SQL code to create the database table static List _createTable() { @@ -43,7 +35,7 @@ class FileUpdationDB { ALTER TABLE $tableName ADD COLUMN $columnReason TEXT; ''', ''' - UPDATE $tableName SET $columnReason = '$missingLocation'; + UPDATE $tableName SET $columnReason = '$modificationTimeUpdated'; ''', ]; } @@ -153,6 +145,25 @@ class FileUpdationDB { return result; } + // delete entries for given list of reasons + Future deleteByReasons(List reasons) async { + if (reasons.isEmpty) { + return; + } + String inParam = ""; + for (final reason in reasons) { + inParam += "'" + reason + "',"; + } + inParam = inParam.substring(0, inParam.length - 1); + final db = await instance.database; + await db.rawQuery( + ''' + DELETE FROM $tableName + WHERE $columnReason IN ($inParam); + ''', + ); + } + Map _getRowForReUploadTable(String localID, String reason) { final row = {}; row[columnLocalID] = localID; diff --git a/lib/services/files_service.dart b/lib/services/files_service.dart index 41544594a..44aef45ee 100644 --- a/lib/services/files_service.dart +++ b/lib/services/files_service.dart @@ -42,6 +42,8 @@ class FilesService { } } + // Note: this method is not used anywhere, but it is kept for future + // reference when we add bulk EditTime feature Future bulkEditTime( List files, EditTimeSource source, diff --git a/lib/services/local_file_update_service.dart b/lib/services/local_file_update_service.dart index 663fec160..1ad4f0f56 100644 --- a/lib/services/local_file_update_service.dart +++ b/lib/services/local_file_update_service.dart @@ -2,21 +2,12 @@ import 'dart:async'; import 'dart:core'; import 'dart:io'; -import "package:collection/collection.dart"; import 'package:flutter/foundation.dart'; import 'package:logging/logging.dart'; -import 'package:photos/core/configuration.dart'; -import 'package:photos/core/constants.dart'; import 'package:photos/core/errors.dart'; import 'package:photos/db/file_updation_db.dart'; import 'package:photos/db/files_db.dart'; -import 'package:photos/extensions/stop_watch.dart'; import 'package:photos/models/file.dart' as ente; -import "package:photos/models/location/location.dart"; -import "package:photos/models/magic_metadata.dart"; -import "package:photos/services/file_magic_service.dart"; -import 'package:photos/services/files_service.dart'; -import "package:photos/utils/exif_util.dart"; import 'package:photos/utils/file_uploader_util.dart'; import 'package:photos/utils/file_util.dart'; import 'package:shared_preferences/shared_preferences.dart'; @@ -27,15 +18,14 @@ class LocalFileUpdateService { late FileUpdationDB _fileUpdationDB; late SharedPreferences _prefs; late Logger _logger; - static const isBadCreationTimeImportDone = 'fm_badCreationTime'; - static const isBadCreationTimeMigrationComplete = - 'fm_badCreationTimeCompleted'; - static const isMissingLocationV2ImportDone = "fm_missingLocationV2ImportDone"; - static const isMissingLocationV2MigrationDone = - "fm_missingLocationV2MigrationDone"; - - static const isBadLocationCordImportDone = "fm_badLocationImportDone"; - static const isBadLocationCordMigrationDone = "fm_badLocationMigrationDone"; + final List _oldMigrationKeys = [ + 'fm_badCreationTime', + 'fm_badCreationTimeCompleted', + 'fm_missingLocationV2ImportDone', + 'fm_missingLocationV2MigrationDone', + 'fm_badLocationImportDone', + 'fm_badLocationMigrationDone', + ]; Completer? _existingMigration; @@ -51,10 +41,6 @@ class LocalFileUpdateService { static LocalFileUpdateService instance = LocalFileUpdateService._privateConstructor(); - bool isBadCreationMigrationCompleted() { - return (_prefs.getBool(isBadCreationTimeMigrationComplete) ?? false); - } - Future markUpdatedFilesForReUpload() async { if (_existingMigration != null) { _logger.info("migration is already in progress, skipping"); @@ -64,9 +50,7 @@ class LocalFileUpdateService { try { await _markFilesWhichAreActuallyUpdated(); if (Platform.isAndroid) { - await _migrationForFixingBadCreationTime(); - await _migrationFilesWithMissingLocationV2(); - await _migrationFilesWithBadLocationCord(); + _cleanUpOlderMigration().ignore(); } } catch (e, s) { _logger.severe('failed to perform migration', e, s); @@ -76,6 +60,28 @@ class LocalFileUpdateService { } } + Future _cleanUpOlderMigration() async { + // check if any old_migration_keys are present in shared preferences + bool hasOldMigrationKey = false; + for (String key in _oldMigrationKeys) { + if (_prefs.containsKey(key)) { + hasOldMigrationKey = true; + break; + } + } + if (hasOldMigrationKey) { + for (var element in _oldMigrationKeys) { + _prefs.remove(element); + } + await _fileUpdationDB.deleteByReasons([ + 'missing_location', + 'badCreationTime', + 'missingLocationV2', + 'badLocationCord', + ]); + } + } + // This method analyses all of local files for which the file // modification/update time was changed. It checks if the existing fileHash // is different from the hash of uploaded file. If fileHash are different, @@ -162,241 +168,4 @@ class LocalFileUpdateService { } return mediaUploadData; } - - Future _migrationForFixingBadCreationTime() async { - if (_prefs.containsKey(isBadCreationTimeMigrationComplete)) { - return; - } - await _importFilesWithBadCreationTime(); - const int singleRunLimit = 100; - try { - final generatedIDs = - await _fileUpdationDB.getLocalIDsForPotentialReUpload( - singleRunLimit, - FileUpdationDB.badCreationTime, - ); - if (generatedIDs.isNotEmpty) { - final List genIdIntList = []; - for (String genIdString in generatedIDs) { - final int? genIdInt = int.tryParse(genIdString); - if (genIdInt != null) { - genIdIntList.add(genIdInt); - } - } - - final filesWithBadTime = - (await FilesDB.instance.getFilesFromGeneratedIDs(genIdIntList)) - .values - .toList(); - filesWithBadTime.removeWhere( - (e) => e.isUploaded && e.pubMagicMetadata?.editedTime != null, - ); - await FilesService.instance - .bulkEditTime(filesWithBadTime, EditTimeSource.fileName); - } else { - // everything is done - await _prefs.setBool(isBadCreationTimeMigrationComplete, true); - } - await _fileUpdationDB.deleteByLocalIDs( - generatedIDs, - FileUpdationDB.badCreationTime, - ); - } catch (e) { - _logger.severe("Failed to fix bad creationTime", e); - } - } - - Future _importFilesWithBadCreationTime() async { - if (_prefs.containsKey(isBadCreationTimeImportDone)) { - return; - } - _logger.info('_importFilesWithBadCreationTime'); - final EnteWatch watch = EnteWatch("_importFilesWithBadCreationTime"); - final int ownerID = Configuration.instance.getUserID()!; - final filesGeneratedID = await FilesDB.instance - .getGeneratedIDForFilesOlderThan(jan011981Time, ownerID); - await _fileUpdationDB.insertMultiple( - filesGeneratedID, - FileUpdationDB.badCreationTime, - ); - watch.log("imported ${filesGeneratedID.length} files"); - _prefs.setBool(isBadCreationTimeImportDone, true); - } - - Future _migrationFilesWithMissingLocationV2() async { - if (_prefs.containsKey(isMissingLocationV2MigrationDone)) { - return; - } - await _importForMissingLocationV2(); - const int singleRunLimit = 10; - final List processedIDs = []; - try { - final localIDs = await _fileUpdationDB.getLocalIDsForPotentialReUpload( - singleRunLimit, - FileUpdationDB.missingLocationV2, - ); - if (localIDs.isEmpty) { - // everything is done - await _prefs.setBool(isMissingLocationV2MigrationDone, true); - return; - } - - final List enteFiles = await FilesDB.instance - .getFilesForLocalIDs(localIDs, Configuration.instance.getUserID()!); - // fine localIDs which are not present in enteFiles - final List missingLocalIDs = []; - for (String localID in localIDs) { - if (enteFiles.firstWhereOrNull((e) => e.localID == localID) == null) { - missingLocalIDs.add(localID); - } - } - processedIDs.addAll(missingLocalIDs); - - final List remoteFilesToUpdate = []; - final Map> fileIDToUpdateMetadata = {}; - - for (ente.File file in enteFiles) { - final Location? location = await tryLocationFromExif(file); - if (location != null && Location.isValidLocation(location)) { - remoteFilesToUpdate.add(file); - fileIDToUpdateMetadata[file.uploadedFileID!] = { - pubMagicKeyLat: location.latitude!, - pubMagicKeyLong: location.longitude! - }; - } else if (file.localID != null) { - processedIDs.add(file.localID!); - } - } - if (remoteFilesToUpdate.isNotEmpty) { - await FileMagicService.instance.updatePublicMagicMetadata( - remoteFilesToUpdate, - null, - metadataUpdateMap: fileIDToUpdateMetadata, - ); - for (ente.File file in remoteFilesToUpdate) { - if (file.localID != null) { - processedIDs.add(file.localID!); - } - } - } - } catch (e) { - _logger.severe("Failed to fix bad creationTime", e); - } finally { - await _fileUpdationDB.deleteByLocalIDs( - processedIDs, - FileUpdationDB.missingLocationV2, - ); - } - } - - Future _importForMissingLocationV2() async { - if (_prefs.containsKey(isMissingLocationV2ImportDone)) { - return; - } - _logger.info('_importForMissingLocationV2'); - final EnteWatch watch = EnteWatch("_importForMissingLocationV2"); - final int ownerID = Configuration.instance.getUserID()!; - final List localIDs = - await FilesDB.instance.getLocalIDsForFilesWithoutLocation(ownerID); - - await _fileUpdationDB.insertMultiple( - localIDs, - FileUpdationDB.missingLocationV2, - ); - watch.log("imported ${localIDs.length} files"); - await _prefs.setBool(isMissingLocationV2ImportDone, true); - } - - Future _migrationFilesWithBadLocationCord() async { - if (_prefs.containsKey(isBadLocationCordMigrationDone)) { - return; - } - await _importForBadLocationCord(); - const int singleRunLimit = 10; - final List processedIDs = []; - try { - final localIDs = await _fileUpdationDB.getLocalIDsForPotentialReUpload( - singleRunLimit, - FileUpdationDB.badLocationCord, - ); - if (localIDs.isEmpty) { - // everything is done - await _prefs.setBool(isBadLocationCordMigrationDone, true); - return; - } - - final List enteFiles = await FilesDB.instance - .getFilesForLocalIDs(localIDs, Configuration.instance.getUserID()!); - // fine localIDs which are not present in enteFiles - final List missingLocalIDs = []; - for (String localID in localIDs) { - if (enteFiles.firstWhereOrNull((e) => e.localID == localID) == null) { - missingLocalIDs.add(localID); - } - } - processedIDs.addAll(missingLocalIDs); - - final List remoteFilesToUpdate = []; - final Map> fileIDToUpdateMetadata = {}; - - for (ente.File file in enteFiles) { - final Location? location = await tryLocationFromExif(file); - if (location != null && - (location.latitude ?? 0) != 0.0 && - (location.longitude ?? 0) != 0.0) { - // check if the location is already correct - if (file.location != null && - file.location?.longitude == location.latitude && - file.location?.longitude == location.longitude) { - processedIDs.add(file.localID!); - } else { - remoteFilesToUpdate.add(file); - fileIDToUpdateMetadata[file.uploadedFileID!] = { - pubMagicKeyLat: location.latitude!, - pubMagicKeyLong: location.longitude! - }; - } - } else if (file.localID != null) { - processedIDs.add(file.localID!); - } - } - if (remoteFilesToUpdate.isNotEmpty) { - await FileMagicService.instance.updatePublicMagicMetadata( - remoteFilesToUpdate, - null, - metadataUpdateMap: fileIDToUpdateMetadata, - ); - for (ente.File file in remoteFilesToUpdate) { - if (file.localID != null) { - processedIDs.add(file.localID!); - } - } - } - } catch (e) { - _logger.severe("Failed to fix bad location cord", e); - } finally { - await _fileUpdationDB.deleteByLocalIDs( - processedIDs, - FileUpdationDB.badLocationCord, - ); - } - } - - Future _importForBadLocationCord() async { - if (_prefs.containsKey(isBadLocationCordImportDone)) { - return; - } - _logger.info('_importForBadLocationCord'); - final EnteWatch watch = EnteWatch("_importForBadLocationCord"); - final int ownerID = Configuration.instance.getUserID()!; - final List localIDs = await FilesDB.instance - .getFilesWithLocationUploadedBtw20AprTo15May2023(ownerID); - - await _fileUpdationDB.insertMultiple( - localIDs, - FileUpdationDB.badLocationCord, - ); - watch.log("imported ${localIDs.length} files"); - await _prefs.setBool(isBadLocationCordImportDone, true); - } }