Merge branch 'master' into redesign

lib/core/constants.dart

@@ -36,4 +36,5 @@ class FFDefault {
   static const bool enableStripe = true;
   static const bool disableUrlSharing = false;
   static const bool disableCFWorker = false;
+  static const bool enableMissingLocationMigration = false;
 }

lib/db/file_migration_db.dart

@@ -0,0 +1,111 @@
+import 'dart:io';
+
+import 'package:logging/logging.dart';
+import 'package:path/path.dart';
+import 'package:path_provider/path_provider.dart';
+import 'package:sqflite/sqflite.dart';
+
+class FilesMigrationDB {
+  static final _databaseName = "ente.files_migration.db";
+  static final _databaseVersion = 1;
+  static final Logger _logger = Logger((FilesMigrationDB).toString());
+  static final tableName = 're_upload_tracker';
+
+  static final columnLocalID = 'local_id';
+
+  Future _onCreate(Database db, int version) async {
+    await db.execute('''
+        CREATE TABLE $tableName (
+        $columnLocalID TEXT NOT NULL,
+          UNIQUE($columnLocalID)
+        );
+      ''');
+  }
+
+  FilesMigrationDB._privateConstructor();
+
+  static final FilesMigrationDB instance =
+      FilesMigrationDB._privateConstructor();
+
+  // only have a single app-wide reference to the database
+  static Future<Database> _dbFuture;
+
+  Future<Database> get database async {
+    // lazily instantiate the db the first time it is accessed
+    _dbFuture ??= _initDatabase();
+    return _dbFuture;
+  }
+
+  // this opens the database (and creates it if it doesn't exist)
+  Future<Database> _initDatabase() async {
+    Directory documentsDirectory = await getApplicationDocumentsDirectory();
+    String path = join(documentsDirectory.path, _databaseName);
+    return await openDatabase(
+      path,
+      version: _databaseVersion,
+      onCreate: _onCreate,
+    );
+  }
+
+  Future<void> clearTable() async {
+    final db = await instance.database;
+    await db.delete(tableName);
+  }
+
+  Future<void> insertMultiple(List<String> fileLocalIDs) async {
+    final startTime = DateTime.now();
+    final db = await instance.database;
+    var batch = db.batch();
+    int batchCounter = 0;
+    for (String localID in fileLocalIDs) {
+      if (batchCounter == 400) {
+        await batch.commit(noResult: true);
+        batch = db.batch();
+        batchCounter = 0;
+      }
+      batch.insert(
+        tableName,
+        _getRowForReUploadTable(localID),
+        conflictAlgorithm: ConflictAlgorithm.replace,
+      );
+      batchCounter++;
+    }
+    await batch.commit(noResult: true);
+    final endTime = DateTime.now();
+    final duration = Duration(
+        microseconds:
+            endTime.microsecondsSinceEpoch - startTime.microsecondsSinceEpoch);
+    _logger.info("Batch insert of ${fileLocalIDs.length} "
+        "took ${duration.inMilliseconds} ms.");
+  }
+
+  Future<int> deleteByLocalIDs(List<String> localIDs) async {
+    String inParam = "";
+    for (final localID in localIDs) {
+      inParam += "'" + localID + "',";
+    }
+    inParam = inParam.substring(0, inParam.length - 1);
+    final db = await instance.database;
+    return await db.delete(
+      tableName,
+      where: '$columnLocalID IN (${localIDs.join(', ')})',
+    );
+  }
+
+  Future<List<String>> getLocalIDsForPotentialReUpload(int limit) async {
+    final db = await instance.database;
+    final rows = await db.query(tableName, limit: limit);
+    final result = <String>[];
+    for (final row in rows) {
+      result.add(row[columnLocalID]);
+    }
+    return result;
+  }
+
+  Map<String, dynamic> _getRowForReUploadTable(String localID) {
+    assert(localID != null);
+    final row = <String, dynamic>{};
+    row[columnLocalID] = localID;
+    return row;
+  }
+}

lib/db/files_db.dart

@@ -978,6 +978,41 @@ class FilesDB {
     return result;
   }
 
+  Future<List<String>> getLocalFilesBackedUpWithoutLocation() async {
+    final db = await instance.database;
+    final rows = await db.query(
+      table,
+      columns: [columnLocalID],
+      distinct: true,
+      where:
+          '$columnLocalID IS NOT NULL AND ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) '
+          'AND ($columnLatitude IS NULL OR $columnLongitude IS NULL OR $columnLongitude = 0.0 or $columnLongitude = 0.0)',
+    );
+    final result = <String>[];
+    for (final row in rows) {
+      result.add(row[columnLocalID]);
+    }
+    return result;
+  }
+
+  Future<void> markForReUploadIfLocationMissing(List<String> localIDs) async {
+    if (localIDs.isEmpty) {
+      return;
+    }
+    String inParam = "";
+    for (final localID in localIDs) {
+      inParam += "'" + localID + "',";
+    }
+    inParam = inParam.substring(0, inParam.length - 1);
+    final db = await instance.database;
+    await db.rawUpdate('''
+      UPDATE $table
+      SET $columnUpdationTime = NULL
+      WHERE $columnLocalID IN ($inParam)
+      AND ($columnLatitude IS NULL OR $columnLongitude IS NULL OR $columnLongitude = 0.0 or $columnLongitude = 0.0);
+    ''');
+  }
+
   Future<bool> doesFileExistInCollection(
       int uploadedFileID, int collectionID) async {
     final db = await instance.database;

lib/main.dart

@@ -19,6 +19,7 @@ import 'package:photos/services/app_lifecycle_service.dart';
 import 'package:photos/services/billing_service.dart';
 import 'package:photos/services/collections_service.dart';
 import 'package:photos/services/feature_flag_service.dart';
+import 'package:photos/services/file_migration_service.dart';
 import 'package:photos/services/local_sync_service.dart';
 import 'package:photos/services/memories_service.dart';
 import 'package:photos/services/notification_service.dart';
@@ -134,6 +135,7 @@ Future<void> _init(bool isBackground, {String via = ''}) async {
   await SyncService.instance.init();
   await MemoriesService.instance.init();
   await LocalSettings.instance.init();
+  await FileMigrationService.instance.init();
   if (Platform.isIOS) {
     PushService.instance.init().then((_) {
       FirebaseMessaging.onBackgroundMessage(

lib/services/feature_flag_service.dart

@@ -58,6 +58,19 @@ class FeatureFlagService {
     }
   }
 
+  bool enableMissingLocationMigration() {
+    // only needs to be enabled for android
+    if (!Platform.isAndroid) {
+      return false;
+    }
+    try {
+      return _getFeatureFlags().enableMissingLocationMigration;
+    } catch (e) {
+      _logger.severe(e);
+      return FFDefault.enableMissingLocationMigration;
+    }
+  }
+
   bool enableStripe() {
     if (Platform.isIOS) {
       return false;
@@ -91,6 +104,7 @@ class FeatureFlags {
       disableCFWorker: FFDefault.disableCFWorker,
       disableUrlSharing: FFDefault.disableUrlSharing,
       enableStripe: FFDefault.enableStripe);
+<<<<<<< HEAD
 
   final bool disableCFWorker;
   final bool disableUrlSharing;
@@ -101,12 +115,26 @@ class FeatureFlags {
     @required this.disableUrlSharing,
     @required this.enableStripe,
   });
+=======
+
+  final bool disableCFWorker;
+  final bool disableUrlSharing;
+  final bool enableStripe;
+  final bool enableMissingLocationMigration;
+
+  FeatureFlags(
+      {@required this.disableCFWorker,
+      @required this.disableUrlSharing,
+      @required this.enableStripe,
+      @required this.enableMissingLocationMigration});
+>>>>>>> master
 
   Map<String, dynamic> toMap() {
     return {
       "disableCFWorker": disableCFWorker,
       "disableUrlSharing": disableUrlSharing,
       "enableStripe": enableStripe,
+      "enableMissingLocationMigration": enableMissingLocationMigration,
     };
   }
 
@@ -121,6 +149,11 @@ class FeatureFlags {
       disableUrlSharing:
           json["disableUrlSharing"] ?? FFDefault.disableUrlSharing,
       enableStripe: json["enableStripe"] ?? FFDefault.enableStripe,
+<<<<<<< HEAD
+=======
+      enableMissingLocationMigration: json["enableMissingLocationMigration"] ??
+          FFDefault.enableMissingLocationMigration,
+>>>>>>> master
     );
   }
 

lib/services/file_migration_service.dart

@@ -0,0 +1,133 @@
+import 'dart:async';
+import 'dart:core';
+import 'dart:io';
+
+import 'package:logging/logging.dart';
+import 'package:photo_manager/photo_manager.dart';
+import 'package:photos/db/file_migration_db.dart';
+import 'package:photos/db/files_db.dart';
+import 'package:shared_preferences/shared_preferences.dart';
+
+class FileMigrationService {
+  FilesDB _filesDB;
+  FilesMigrationDB _filesMigrationDB;
+  SharedPreferences _prefs;
+  Logger _logger;
+  static const isLocationMigrationComplete = "fm_isLocationMigrationComplete";
+  static const isLocalImportDone = "fm_IsLocalImportDone";
+  Completer<void> _existingMigration;
+
+  FileMigrationService._privateConstructor() {
+    _logger = Logger((FileMigrationService).toString());
+    _filesDB = FilesDB.instance;
+    _filesMigrationDB = FilesMigrationDB.instance;
+  }
+
+  Future<void> init() async {
+    _prefs = await SharedPreferences.getInstance();
+  }
+
+  static FileMigrationService instance =
+      FileMigrationService._privateConstructor();
+
+  Future<bool> _markLocationMigrationAsCompleted() async {
+    _logger.info('marking migration as completed');
+    return _prefs.setBool(isLocationMigrationComplete, true);
+  }
+
+  bool isLocationMigrationCompleted() {
+    return _prefs.get(isLocationMigrationComplete) ?? false;
+  }
+
+  Future<void> runMigration() async {
+    if (_existingMigration != null) {
+      _logger.info("migration is already in progress, skipping");
+      return _existingMigration.future;
+    }
+    _logger.info("start migration");
+    _existingMigration = Completer<void>();
+    try {
+      await _runMigrationForFilesWithMissingLocation();
+      _existingMigration.complete();
+      _existingMigration = null;
+    } catch (e, s) {
+      _logger.severe('failed to perform migration', e, s);
+      _existingMigration.complete();
+      _existingMigration = null;
+    }
+  }
+
+  Future<void> _runMigrationForFilesWithMissingLocation() async {
+    if (!Platform.isAndroid) {
+      return;
+    }
+    // migration only needs to run if Android API Level is 29 or higher
+    final int version = int.parse(await PhotoManager.systemVersion());
+    bool isMigrationRequired = version >= 29;
+    if (isMigrationRequired) {
+      await _importLocalFilesForMigration();
+      final sTime = DateTime.now().microsecondsSinceEpoch;
+      bool hasData = true;
+      final int limitInBatch = 100;
+      while (hasData) {
+        var localIDsToProcess = await _filesMigrationDB
+            .getLocalIDsForPotentialReUpload(limitInBatch);
+        if (localIDsToProcess.isEmpty) {
+          hasData = false;
+        } else {
+          await _checkAndMarkFilesForReUpload(localIDsToProcess);
+        }
+      }
+      final eTime = DateTime.now().microsecondsSinceEpoch;
+      final d = Duration(microseconds: eTime - sTime);
+      _logger.info(
+          'filesWithMissingLocation migration completed in ${d.inSeconds.toString()} seconds');
+    }
+    await _markLocationMigrationAsCompleted();
+  }
+
+  Future<void> _checkAndMarkFilesForReUpload(
+      List<String> localIDsToProcess) async {
+    _logger.info("files to process ${localIDsToProcess.length}");
+    var localIDsWithLocation = <String>[];
+    for (var localID in localIDsToProcess) {
+      bool hasLocation = false;
+      try {
+        var assetEntity = await AssetEntity.fromId(localID);
+        if (assetEntity == null) {
+          continue;
+        }
+        var latLng = await assetEntity.latlngAsync();
+        if ((latLng.longitude ?? 0.0) != 0.0 ||
+            (latLng.longitude ?? 0.0) != 0.0) {
+          _logger.finest(
+              'found lat/long ${latLng.longitude}/${latLng.longitude} for  ${assetEntity.title} ${assetEntity.relativePath} with id : $localID');
+          hasLocation = true;
+        }
+      } catch (e, s) {
+        _logger.severe('failed to get asset entity with id $localID', e, s);
+      }
+      if (hasLocation) {
+        localIDsWithLocation.add(localID);
+      }
+    }
+    _logger.info('marking ${localIDsWithLocation.length} files for re-upload');
+    await _filesDB.markForReUploadIfLocationMissing(localIDsWithLocation);
+    await _filesMigrationDB.deleteByLocalIDs(localIDsToProcess);
+  }
+
+  Future<void> _importLocalFilesForMigration() async {
+    if (_prefs.containsKey(isLocalImportDone)) {
+      return;
+    }
+    final sTime = DateTime.now().microsecondsSinceEpoch;
+    _logger.info('importing files without location info');
+    var fileLocalIDs = await _filesDB.getLocalFilesBackedUpWithoutLocation();
+    await _filesMigrationDB.insertMultiple(fileLocalIDs);
+    final eTime = DateTime.now().microsecondsSinceEpoch;
+    final d = Duration(microseconds: eTime - sTime);
+    _logger.info(
+        'importing completed, total files count ${fileLocalIDs.length} and took ${d.inSeconds.toString()} seconds');
+    _prefs.setBool(isLocalImportDone, true);
+  }
+}

lib/services/remote_sync_service.dart

@@ -16,6 +16,8 @@ import 'package:photos/models/file.dart';
 import 'package:photos/models/file_type.dart';
 import 'package:photos/services/app_lifecycle_service.dart';
 import 'package:photos/services/collections_service.dart';
+import 'package:photos/services/feature_flag_service.dart';
+import 'package:photos/services/file_migration_service.dart';
 import 'package:photos/services/ignored_files_service.dart';
 import 'package:photos/services/local_sync_service.dart';
 import 'package:photos/services/trash_sync_service.dart';
@@ -30,6 +32,8 @@ class RemoteSyncService {
   final _uploader = FileUploader.instance;
   final _collectionsService = CollectionsService.instance;
   final _diffFetcher = DiffFetcher();
+  final FileMigrationService _fileMigrationService =
+      FileMigrationService.instance;
   int _completedUploads = 0;
   SharedPreferences _prefs;
   Completer<void> _existingSync;
@@ -128,6 +132,10 @@ class RemoteSyncService {
     if (!_hasReSynced()) {
       await _markReSyncAsDone();
     }
+    if (FeatureFlagService.instance.enableMissingLocationMigration() &&
+        !_fileMigrationService.isLocationMigrationCompleted()) {
+      _fileMigrationService.runMigration();
+    }
   }
 
   Future<void> _syncUpdatedCollections(bool silently) async {

lib/ui/settings/backup_section_widget.dart

@@ -99,7 +99,15 @@ class BackupSectionWidgetState extends State<BackupSectionWidget> {
           onTap: () async {
             final dialog = createProgressDialog(context, "Calculating...");
             await dialog.show();
-            final status = await SyncService.instance.getBackupStatus();
+            BackupStatus status;
+            try {
+              status = await SyncService.instance.getBackupStatus();
+            } catch (e, s) {
+              await dialog.hide();
+              showGenericErrorDialog(context);
+              return;
+            }
+
             await dialog.hide();
             if (status.localIDs.isEmpty) {
               showErrorDialog(context, "✨ All clear",

pubspec.yaml

@@ -11,8 +11,7 @@ description: ente photos application
 # In iOS, build-name is used as CFBundleShortVersionString while build-number used as CFBundleVersion.
 # Read more about iOS versioning at
 # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html
-
-version: 0.5.35+315
+version: 0.5.37+317
 
 environment:
   sdk: ">=2.10.0 <3.0.0"