import 'dart:io'; import 'package:flutter/foundation.dart'; import 'package:logging/logging.dart'; import 'package:path/path.dart'; import 'package:path_provider/path_provider.dart'; import 'package:photos/models/backup_status.dart'; import 'package:photos/models/file.dart'; import 'package:photos/models/file_load_result.dart'; import 'package:photos/models/file_type.dart'; import 'package:photos/models/location.dart'; import 'package:photos/models/magic_metadata.dart'; import 'package:sqflite/sqflite.dart'; import 'package:sqflite_migration/sqflite_migration.dart'; class FilesDB { /* Note: columnUploadedFileID and columnCollectionID have to be compared against both NULL and -1 because older clients might have entries where the DEFAULT was unset, and a migration script to set the DEFAULT would break in case of duplicate entries for un-uploaded files that were created due to a collision in background and foreground syncs. */ static const _databaseName = "ente.files.db"; static final Logger _logger = Logger("FilesDB"); static const table = 'files'; static const tempTable = 'temp_files'; static const columnGeneratedID = '_id'; static const columnUploadedFileID = 'uploaded_file_id'; static const columnOwnerID = 'owner_id'; static const columnCollectionID = 'collection_id'; static const columnLocalID = 'local_id'; static const columnTitle = 'title'; static const columnDeviceFolder = 'device_folder'; static const columnLatitude = 'latitude'; static const columnLongitude = 'longitude'; static const columnFileType = 'file_type'; static const columnFileSubType = 'file_sub_type'; static const columnDuration = 'duration'; static const columnExif = 'exif'; static const columnHash = 'hash'; static const columnMetadataVersion = 'metadata_version'; static const columnIsDeleted = 'is_deleted'; static const columnCreationTime = 'creation_time'; static const columnModificationTime = 'modification_time'; static const columnUpdationTime = 'updation_time'; static const columnEncryptedKey = 'encrypted_key'; static const columnKeyDecryptionNonce = 'key_decryption_nonce'; static const columnFileDecryptionHeader = 'file_decryption_header'; static const columnThumbnailDecryptionHeader = 'thumbnail_decryption_header'; static const columnMetadataDecryptionHeader = 'metadata_decryption_header'; // MMD -> Magic Metadata static const columnMMdEncodedJson = 'mmd_encoded_json'; static const columnMMdVersion = 'mmd_ver'; static const columnPubMMdEncodedJson = 'pub_mmd_encoded_json'; static const columnPubMMdVersion = 'pub_mmd_ver'; // part of magic metadata // Only parse & store selected fields from JSON in separate columns if // we need to write query based on that field static const columnMMdVisibility = 'mmd_visibility'; static final initializationScript = [...createTable(table)]; static final migrationScripts = [ ...alterDeviceFolderToAllowNULL(), ...alterTimestampColumnTypes(), ...addIndices(), ...addMetadataColumns(), ...addMagicMetadataColumns(), ...addUniqueConstraintOnCollectionFiles(), ...addPubMagicMetadataColumns() ]; final dbConfig = MigrationConfig( initializationScript: initializationScript, migrationScripts: migrationScripts, ); // make this a singleton class FilesDB._privateConstructor(); static final FilesDB instance = FilesDB._privateConstructor(); // only have a single app-wide reference to the database static Future _dbFuture; Future get database async { // lazily instantiate the db the first time it is accessed _dbFuture ??= _initDatabase(); return _dbFuture; } // this opens the database (and creates it if it doesn't exist) Future _initDatabase() async { Directory documentsDirectory = await getApplicationDocumentsDirectory(); String path = join(documentsDirectory.path, _databaseName); _logger.info("DB path " + path); return await openDatabaseWithMigration(path, dbConfig); } // SQL code to create the database table static List createTable(String tableName) { return [ ''' CREATE TABLE $tableName ( $columnGeneratedID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, $columnLocalID TEXT, $columnUploadedFileID INTEGER DEFAULT -1, $columnOwnerID INTEGER, $columnCollectionID INTEGER DEFAULT -1, $columnTitle TEXT NOT NULL, $columnDeviceFolder TEXT, $columnLatitude REAL, $columnLongitude REAL, $columnFileType INTEGER, $columnModificationTime TEXT NOT NULL, $columnEncryptedKey TEXT, $columnKeyDecryptionNonce TEXT, $columnFileDecryptionHeader TEXT, $columnThumbnailDecryptionHeader TEXT, $columnMetadataDecryptionHeader TEXT, $columnIsDeleted INTEGER DEFAULT 0, $columnCreationTime TEXT NOT NULL, $columnUpdationTime TEXT, UNIQUE($columnLocalID, $columnUploadedFileID, $columnCollectionID) ); ''', ]; } static List addIndices() { return [ ''' CREATE INDEX IF NOT EXISTS collection_id_index ON $table($columnCollectionID); ''', ''' CREATE INDEX IF NOT EXISTS device_folder_index ON $table($columnDeviceFolder); ''', ''' CREATE INDEX IF NOT EXISTS creation_time_index ON $table($columnCreationTime); ''', ''' CREATE INDEX IF NOT EXISTS updation_time_index ON $table($columnUpdationTime); ''' ]; } static List alterDeviceFolderToAllowNULL() { return [ ...createTable(tempTable), ''' INSERT INTO $tempTable SELECT * FROM $table; DROP TABLE $table; ALTER TABLE $tempTable RENAME TO $table; ''' ]; } static List alterTimestampColumnTypes() { return [ ''' DROP TABLE IF EXISTS $tempTable; ''', ''' CREATE TABLE $tempTable ( $columnGeneratedID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, $columnLocalID TEXT, $columnUploadedFileID INTEGER DEFAULT -1, $columnOwnerID INTEGER, $columnCollectionID INTEGER DEFAULT -1, $columnTitle TEXT NOT NULL, $columnDeviceFolder TEXT, $columnLatitude REAL, $columnLongitude REAL, $columnFileType INTEGER, $columnModificationTime INTEGER NOT NULL, $columnEncryptedKey TEXT, $columnKeyDecryptionNonce TEXT, $columnFileDecryptionHeader TEXT, $columnThumbnailDecryptionHeader TEXT, $columnMetadataDecryptionHeader TEXT, $columnCreationTime INTEGER NOT NULL, $columnUpdationTime INTEGER, UNIQUE($columnLocalID, $columnUploadedFileID, $columnCollectionID) ); ''', ''' INSERT INTO $tempTable SELECT $columnGeneratedID, $columnLocalID, $columnUploadedFileID, $columnOwnerID, $columnCollectionID, $columnTitle, $columnDeviceFolder, $columnLatitude, $columnLongitude, $columnFileType, CAST($columnModificationTime AS INTEGER), $columnEncryptedKey, $columnKeyDecryptionNonce, $columnFileDecryptionHeader, $columnThumbnailDecryptionHeader, $columnMetadataDecryptionHeader, CAST($columnCreationTime AS INTEGER), CAST($columnUpdationTime AS INTEGER) FROM $table; ''', ''' DROP TABLE $table; ''', ''' ALTER TABLE $tempTable RENAME TO $table; ''', ]; } static List addMetadataColumns() { return [ ''' ALTER TABLE $table ADD COLUMN $columnFileSubType INTEGER; ''', ''' ALTER TABLE $table ADD COLUMN $columnDuration INTEGER; ''', ''' ALTER TABLE $table ADD COLUMN $columnExif TEXT; ''', ''' ALTER TABLE $table ADD COLUMN $columnHash TEXT; ''', ''' ALTER TABLE $table ADD COLUMN $columnMetadataVersion INTEGER; ''', ]; } static List addMagicMetadataColumns() { return [ ''' ALTER TABLE $table ADD COLUMN $columnMMdEncodedJson TEXT DEFAULT '{}'; ''', ''' ALTER TABLE $table ADD COLUMN $columnMMdVersion INTEGER DEFAULT 0; ''', ''' ALTER TABLE $table ADD COLUMN $columnMMdVisibility INTEGER DEFAULT $kVisibilityVisible; ''' ]; } static List addUniqueConstraintOnCollectionFiles() { return [ ''' DELETE from $table where $columnCollectionID || '-' || $columnUploadedFileID IN (SELECT $columnCollectionID || '-' || $columnUploadedFileID from $table WHERE $columnCollectionID is not NULL AND $columnUploadedFileID is NOT NULL AND $columnCollectionID != -1 AND $columnUploadedFileID != -1 GROUP BY ($columnCollectionID || '-' || $columnUploadedFileID) HAVING count(*) > 1) AND ($columnCollectionID || '-' || $columnUploadedFileID || '-' || $columnGeneratedID) NOT IN (SELECT $columnCollectionID || '-' || $columnUploadedFileID || '-' || max($columnGeneratedID) from $table WHERE $columnCollectionID is not NULL AND $columnUploadedFileID is NOT NULL AND $columnCollectionID != -1 AND $columnUploadedFileID != -1 GROUP BY ($columnCollectionID || '-' || $columnUploadedFileID) HAVING count(*) > 1); ''', ''' CREATE UNIQUE INDEX IF NOT EXISTS cid_uid ON $table ($columnCollectionID, $columnUploadedFileID) WHERE $columnCollectionID is not NULL AND $columnUploadedFileID is not NULL AND $columnCollectionID != -1 AND $columnUploadedFileID != -1; ''' ]; } static List addPubMagicMetadataColumns() { return [ ''' ALTER TABLE $table ADD COLUMN $columnPubMMdEncodedJson TEXT DEFAULT '{}'; ''', ''' ALTER TABLE $table ADD COLUMN $columnPubMMdVersion INTEGER DEFAULT 0; ''' ]; } Future clearTable() async { final db = await instance.database; await db.delete(table); } Future insertMultiple(List files) async { final startTime = DateTime.now(); final db = await instance.database; var batch = db.batch(); int batchCounter = 0; for (File file in files) { if (batchCounter == 400) { await batch.commit(noResult: true); batch = db.batch(); batchCounter = 0; } batch.insert( table, _getRowForFile(file), conflictAlgorithm: ConflictAlgorithm.replace, ); batchCounter++; } await batch.commit(noResult: true); final endTime = DateTime.now(); final duration = Duration( microseconds: endTime.microsecondsSinceEpoch - startTime.microsecondsSinceEpoch, ); _logger.info( "Batch insert of " + files.length.toString() + " took " + duration.inMilliseconds.toString() + "ms.", ); } Future insert(File file) async { final db = await instance.database; return db.insert( table, _getRowForFile(file), conflictAlgorithm: ConflictAlgorithm.replace, ); } Future getFile(int generatedID) async { final db = await instance.database; final results = await db.query( table, where: '$columnGeneratedID = ?', whereArgs: [generatedID], ); if (results.isEmpty) { return null; } return _convertToFiles(results)[0]; } Future getUploadedFile(int uploadedID, int collectionID) async { final db = await instance.database; final results = await db.query( table, where: '$columnUploadedFileID = ? AND $columnCollectionID = ?', whereArgs: [ uploadedID, collectionID, ], ); if (results.isEmpty) { return null; } return _convertToFiles(results)[0]; } Future> getUploadedFileIDs(int collectionID) async { final db = await instance.database; final results = await db.query( table, columns: [columnUploadedFileID], where: '$columnCollectionID = ?', whereArgs: [ collectionID, ], ); final ids = {}; for (final result in results) { ids.add(result[columnUploadedFileID]); } return ids; } Future getBackedUpIDs() async { final db = await instance.database; final results = await db.query( table, columns: [columnLocalID, columnUploadedFileID], where: '$columnLocalID IS NOT NULL AND ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1)', ); final localIDs = {}; final uploadedIDs = {}; for (final result in results) { localIDs.add(result[columnLocalID]); uploadedIDs.add(result[columnUploadedFileID]); } return BackedUpFileIDs(localIDs.toList(), uploadedIDs.toList()); } Future getAllUploadedFiles( int startTime, int endTime, int ownerID, { int limit, bool asc, int visibility = kVisibilityVisible, Set ignoredCollectionIDs, }) async { final db = await instance.database; final order = (asc ?? false ? 'ASC' : 'DESC'); final results = await db.query( table, where: '$columnCreationTime >= ? AND $columnCreationTime <= ? AND $columnOwnerID = ? AND ($columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1)' ' AND $columnMMdVisibility = ?', whereArgs: [startTime, endTime, ownerID, visibility], orderBy: '$columnCreationTime ' + order + ', $columnModificationTime ' + order, limit: limit, ); final files = _convertToFiles(results); List deduplicatedFiles = _deduplicatedAndFilterIgnoredFiles(files, ignoredCollectionIDs); return FileLoadResult(deduplicatedFiles, files.length == limit); } Future getAllLocalAndUploadedFiles( int startTime, int endTime, int ownerID, { int limit, bool asc, Set ignoredCollectionIDs, }) async { final db = await instance.database; final order = (asc ?? false ? 'ASC' : 'DESC'); final results = await db.query( table, where: '$columnCreationTime >= ? AND $columnCreationTime <= ? AND ($columnOwnerID IS NULL OR $columnOwnerID = ?) AND ($columnMMdVisibility IS NULL OR $columnMMdVisibility = ?)' ' AND ($columnLocalID IS NOT NULL OR ($columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1))', whereArgs: [startTime, endTime, ownerID, kVisibilityVisible], orderBy: '$columnCreationTime ' + order + ', $columnModificationTime ' + order, limit: limit, ); final files = _convertToFiles(results); List deduplicatedFiles = _deduplicatedAndFilterIgnoredFiles(files, ignoredCollectionIDs); return FileLoadResult(deduplicatedFiles, files.length == limit); } Future getImportantFiles( int startTime, int endTime, int ownerID, List paths, { int limit, bool asc, Set ignoredCollectionIDs, }) async { final db = await instance.database; String inParam = ""; for (final path in paths) { inParam += "'" + path.replaceAll("'", "''") + "',"; } inParam = inParam.substring(0, inParam.length - 1); final order = (asc ?? false ? 'ASC' : 'DESC'); final results = await db.query( table, where: '$columnCreationTime >= ? AND $columnCreationTime <= ? AND ($columnOwnerID IS NULL OR $columnOwnerID = ?) AND ($columnMMdVisibility IS NULL OR $columnMMdVisibility = ?)' 'AND (($columnLocalID IS NOT NULL AND $columnDeviceFolder IN ($inParam)) OR ($columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1))', whereArgs: [startTime, endTime, ownerID, kVisibilityVisible], orderBy: '$columnCreationTime ' + order + ', $columnModificationTime ' + order, limit: limit, ); final files = _convertToFiles(results); List deduplicatedFiles = _deduplicatedAndFilterIgnoredFiles(files, ignoredCollectionIDs); return FileLoadResult(deduplicatedFiles, files.length == limit); } List _deduplicateByLocalID(List files) { final localIDs = {}; final List deduplicatedFiles = []; for (final file in files) { final id = file.localID; if (id != null && localIDs.contains(id)) { continue; } localIDs.add(id); deduplicatedFiles.add(file); } return deduplicatedFiles; } List _deduplicatedAndFilterIgnoredFiles( List files, Set ignoredCollectionIDs, ) { final uploadedFileIDs = {}; final List deduplicatedFiles = []; for (final file in files) { final id = file.uploadedFileID; if (ignoredCollectionIDs != null && ignoredCollectionIDs.contains(file.collectionID)) { continue; } if (id != null && id != -1 && uploadedFileIDs.contains(id)) { continue; } uploadedFileIDs.add(id); deduplicatedFiles.add(file); } return deduplicatedFiles; } Future getFilesInCollection( int collectionID, int startTime, int endTime, { int limit, bool asc, }) async { final db = await instance.database; final order = (asc ?? false ? 'ASC' : 'DESC'); final results = await db.query( table, where: '$columnCollectionID = ? AND $columnCreationTime >= ? AND $columnCreationTime <= ?', whereArgs: [collectionID, startTime, endTime], orderBy: '$columnCreationTime ' + order + ', $columnModificationTime ' + order, limit: limit, ); final files = _convertToFiles(results); _logger.info("Fetched " + files.length.toString() + " files"); return FileLoadResult(files, files.length == limit); } Future getFilesInPath( String path, int startTime, int endTime, { int limit, bool asc, }) async { final db = await instance.database; final order = (asc ?? false ? 'ASC' : 'DESC'); final results = await db.query( table, where: '$columnDeviceFolder = ? AND $columnCreationTime >= ? AND $columnCreationTime <= ? AND $columnLocalID IS NOT NULL', whereArgs: [path, startTime, endTime], orderBy: '$columnCreationTime ' + order + ', $columnModificationTime ' + order, groupBy: columnLocalID, limit: limit, ); final files = _convertToFiles(results); return FileLoadResult(files, files.length == limit); } Future getLocalDeviceFiles( int startTime, int endTime, { int limit, bool asc, }) async { final db = await instance.database; final order = (asc ?? false ? 'ASC' : 'DESC'); final results = await db.query( table, where: '$columnCreationTime >= ? AND $columnCreationTime <= ? AND $columnLocalID IS NOT NULL', whereArgs: [startTime, endTime], orderBy: '$columnCreationTime ' + order + ', $columnModificationTime ' + order, limit: limit, ); final files = _convertToFiles(results); final result = _deduplicateByLocalID(files); return FileLoadResult(result, files.length == limit); } Future> getAllVideos() async { final db = await instance.database; final results = await db.query( table, where: '$columnLocalID IS NOT NULL AND $columnFileType = 1', orderBy: '$columnCreationTime DESC', ); return _convertToFiles(results); } Future> getAllInPath(String path) async { final db = await instance.database; final results = await db.query( table, where: '$columnLocalID IS NOT NULL AND $columnDeviceFolder = ?', whereArgs: [path], orderBy: '$columnCreationTime DESC', groupBy: columnLocalID, ); return _convertToFiles(results); } Future> getFilesCreatedWithinDurations( List> durations, Set ignoredCollectionIDs, ) async { final db = await instance.database; String whereClause = "( "; for (int index = 0; index < durations.length; index++) { whereClause += "($columnCreationTime > " + durations[index][0].toString() + " AND $columnCreationTime < " + durations[index][1].toString() + ")"; if (index != durations.length - 1) { whereClause += " OR "; } } whereClause += ") AND $columnMMdVisibility = $kVisibilityVisible"; final results = await db.query( table, where: whereClause, orderBy: '$columnCreationTime ASC', ); final files = _convertToFiles(results); return _deduplicatedAndFilterIgnoredFiles(files, ignoredCollectionIDs); } Future> getFilesToBeUploadedWithinFolders( Set folders, ) async { if (folders.isEmpty) { return []; } final db = await instance.database; String inParam = ""; for (final folder in folders) { inParam += "'" + folder.replaceAll("'", "''") + "',"; } inParam = inParam.substring(0, inParam.length - 1); final results = await db.query( table, where: '($columnUploadedFileID IS NULL OR $columnUploadedFileID IS -1) AND $columnDeviceFolder IN ($inParam)', orderBy: '$columnCreationTime DESC', groupBy: columnLocalID, ); return _convertToFiles(results); } // Files which user added to a collection manually but they are not uploaded yet. Future> getPendingManualUploads() async { final db = await instance.database; final results = await db.query( table, where: '($columnUploadedFileID IS NULL OR $columnUploadedFileID IS -1) AND ' '$columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1 AND ' '$columnLocalID IS NOT NULL AND $columnLocalID IS NOT -1', orderBy: '$columnCreationTime DESC', groupBy: columnLocalID, ); var files = _convertToFiles(results); // future-safe filter just to ensure that the query doesn't end up returning files // which should not be backed up files.removeWhere( (e) => e.collectionID == null || e.localID == null || e.uploadedFileID != null, ); return files; } Future> getAllLocalFiles() async { final db = await instance.database; final results = await db.query( table, where: '($columnUploadedFileID IS NULL OR $columnUploadedFileID IS -1) AND $columnLocalID IS NOT NULL', orderBy: '$columnCreationTime DESC', groupBy: columnLocalID, ); return _convertToFiles(results); } Future> getEditedRemoteFiles() async { final db = await instance.database; final results = await db.query( table, where: '($columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1) AND ($columnUploadedFileID IS NULL OR $columnUploadedFileID IS -1)', orderBy: '$columnCreationTime DESC', groupBy: columnLocalID, ); return _convertToFiles(results); } Future> getUploadedFileIDsToBeUpdated() async { final db = await instance.database; final rows = await db.query( table, columns: [columnUploadedFileID], where: '($columnLocalID IS NOT NULL AND ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) AND $columnUpdationTime IS NULL)', orderBy: '$columnCreationTime DESC', distinct: true, ); final uploadedFileIDs = []; for (final row in rows) { uploadedFileIDs.add(row[columnUploadedFileID]); } return uploadedFileIDs; } Future getUploadedFileInAnyCollection(int uploadedFileID) async { final db = await instance.database; final results = await db.query( table, where: '$columnUploadedFileID = ?', whereArgs: [ uploadedFileID, ], limit: 1, ); if (results.isEmpty) { return null; } return _convertToFiles(results)[0]; } Future> getExistingLocalFileIDs() async { final db = await instance.database; final rows = await db.query( table, columns: [columnLocalID], distinct: true, where: '$columnLocalID IS NOT NULL', ); final result = {}; for (final row in rows) { result.add(row[columnLocalID]); } return result; } Future getNumberOfUploadedFiles() async { final db = await instance.database; final rows = await db.query( table, columns: [columnUploadedFileID], where: '($columnLocalID IS NOT NULL AND ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) AND $columnUpdationTime IS NOT NULL)', distinct: true, ); return rows.length; } Future updateUploadedFile( String localID, String title, Location location, int creationTime, int modificationTime, int updationTime, ) async { final db = await instance.database; return await db.update( table, { columnTitle: title, columnLatitude: location.latitude, columnLongitude: location.longitude, columnCreationTime: creationTime, columnModificationTime: modificationTime, columnUpdationTime: updationTime, }, where: '$columnLocalID = ?', whereArgs: [localID], ); } Future> getMatchingFiles( String title, String deviceFolder, ) async { final db = await instance.database; final rows = await db.query( table, where: '''$columnTitle=? AND $columnDeviceFolder=?''', whereArgs: [ title, deviceFolder, ], ); if (rows.isNotEmpty) { return _convertToFiles(rows); } else { return null; } } Future> getUploadedFilesWithHashes( List hash, FileType fileType, int ownerID, ) async { final db = await instance.database; String rawQuery = 'SELECT * from files where ($columnUploadedFileID != ' 'NULL OR $columnUploadedFileID != -1) AND $columnOwnerID = $ownerID ' 'AND ($columnHash = "${hash.first}" OR $columnHash = "${hash.last}")'; final rows = await db.rawQuery(rawQuery, []); if (rows.isNotEmpty) { return _convertToFiles(rows); } else { debugPrint(rawQuery); return []; } } Future update(File file) async { final db = await instance.database; return await db.update( table, _getRowForFile(file), where: '$columnGeneratedID = ?', whereArgs: [file.generatedID], ); } Future updateUploadedFileAcrossCollections(File file) async { final db = await instance.database; return await db.update( table, _getRowForFileWithoutCollection(file), where: '$columnUploadedFileID = ?', whereArgs: [file.uploadedFileID], ); } Future delete(int uploadedFileID) async { final db = await instance.database; return db.delete( table, where: '$columnUploadedFileID =?', whereArgs: [uploadedFileID], ); } Future deleteByGeneratedID(int genID) async { final db = await instance.database; return db.delete( table, where: '$columnGeneratedID =?', whereArgs: [genID], ); } Future deleteMultipleUploadedFiles(List uploadedFileIDs) async { final db = await instance.database; return await db.delete( table, where: '$columnUploadedFileID IN (${uploadedFileIDs.join(', ')})', ); } Future deleteLocalFile(File file) async { final db = await instance.database; if (file.localID != null) { // delete all files with same local ID return db.delete( table, where: '$columnLocalID =?', whereArgs: [file.localID], ); } else { return db.delete( table, where: '$columnGeneratedID =?', whereArgs: [file.generatedID], ); } } Future deleteLocalFiles(List localIDs) async { String inParam = ""; for (final localID in localIDs) { inParam += "'" + localID + "',"; } inParam = inParam.substring(0, inParam.length - 1); final db = await instance.database; await db.rawQuery( ''' UPDATE $table SET $columnLocalID = NULL WHERE $columnLocalID IN ($inParam); ''', ); } Future> getLocalFiles(List localIDs) async { String inParam = ""; for (final localID in localIDs) { inParam += "'" + localID + "',"; } inParam = inParam.substring(0, inParam.length - 1); final db = await instance.database; final results = await db.query( table, where: '$columnLocalID IN ($inParam)', ); return _convertToFiles(results); } Future deleteUnSyncedLocalFiles(List localIDs) async { String inParam = ""; for (final localID in localIDs) { inParam += "'" + localID + "',"; } inParam = inParam.substring(0, inParam.length - 1); final db = await instance.database; return db.delete( table, where: '($columnUploadedFileID is NULL OR $columnUploadedFileID = -1 ) AND $columnLocalID IN ($inParam)', ); } Future deleteFromCollection(int uploadedFileID, int collectionID) async { final db = await instance.database; return db.delete( table, where: '$columnUploadedFileID = ? AND $columnCollectionID = ?', whereArgs: [uploadedFileID, collectionID], ); } Future deleteFilesFromCollection( int collectionID, List uploadedFileIDs, ) async { final db = await instance.database; return db.delete( table, where: '$columnCollectionID = ? AND $columnUploadedFileID IN (${uploadedFileIDs.join(', ')})', whereArgs: [collectionID], ); } Future collectionFileCount(int collectionID) async { final db = await instance.database; var count = Sqflite.firstIntValue( await db.rawQuery( 'SELECT COUNT(*) FROM $table where $columnCollectionID = $collectionID', ), ); return count; } Future fileCountWithVisibility(int visibility, int ownerID) async { final db = await instance.database; var count = Sqflite.firstIntValue( await db.rawQuery( 'SELECT COUNT(*) FROM $table where $columnMMdVisibility = $visibility AND $columnOwnerID = $ownerID', ), ); return count; } Future deleteCollection(int collectionID) async { final db = await instance.database; return db.delete( table, where: '$columnCollectionID = ?', whereArgs: [collectionID], ); } Future removeFromCollection(int collectionID, List fileIDs) async { final db = await instance.database; return db.delete( table, where: '$columnCollectionID =? AND $columnUploadedFileID IN (${fileIDs.join(', ')})', whereArgs: [collectionID], ); } Future> getLatestLocalFiles() async { final db = await instance.database; final rows = await db.rawQuery( ''' SELECT $table.* FROM $table INNER JOIN ( SELECT $columnDeviceFolder, MAX($columnCreationTime) AS max_creation_time FROM $table WHERE $table.$columnLocalID IS NOT NULL GROUP BY $columnDeviceFolder ) latest_files ON $table.$columnDeviceFolder = latest_files.$columnDeviceFolder AND $table.$columnCreationTime = latest_files.max_creation_time; ''', ); final files = _convertToFiles(rows); // TODO: Do this de-duplication within the SQL Query final folderMap = {}; for (final file in files) { if (folderMap.containsKey(file.deviceFolder)) { if (folderMap[file.deviceFolder].updationTime < file.updationTime) { continue; } } folderMap[file.deviceFolder] = file; } return folderMap.values.toList(); } Future> getLatestCollectionFiles() async { final db = await instance.database; final rows = await db.rawQuery( ''' SELECT $table.* FROM $table INNER JOIN ( SELECT $columnCollectionID, MAX($columnCreationTime) AS max_creation_time FROM $table WHERE ($columnCollectionID IS NOT NULL AND $columnCollectionID IS NOT -1) GROUP BY $columnCollectionID ) latest_files ON $table.$columnCollectionID = latest_files.$columnCollectionID AND $table.$columnCreationTime = latest_files.max_creation_time; ''', ); final files = _convertToFiles(rows); // TODO: Do this de-duplication within the SQL Query final collectionMap = {}; for (final file in files) { if (collectionMap.containsKey(file.collectionID)) { if (collectionMap[file.collectionID].updationTime < file.updationTime) { continue; } } collectionMap[file.collectionID] = file; } return collectionMap.values.toList(); } Future> getFileCountInDeviceFolders() async { final db = await instance.database; final rows = await db.rawQuery( ''' SELECT COUNT($columnGeneratedID) as count, $columnDeviceFolder FROM $table WHERE $columnLocalID IS NOT NULL GROUP BY $columnDeviceFolder ''', ); final result = {}; for (final row in rows) { result[row[columnDeviceFolder]] = row["count"]; } return result; } Future> getLocalFilesBackedUpWithoutLocation() async { final db = await instance.database; final rows = await db.query( table, columns: [columnLocalID], distinct: true, where: '$columnLocalID IS NOT NULL AND ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) ' 'AND ($columnLatitude IS NULL OR $columnLongitude IS NULL OR $columnLongitude = 0.0 or $columnLongitude = 0.0)', ); final result = []; for (final row in rows) { result.add(row[columnLocalID]); } return result; } Future markForReUploadIfLocationMissing(List localIDs) async { if (localIDs.isEmpty) { return; } String inParam = ""; for (final localID in localIDs) { inParam += "'" + localID + "',"; } inParam = inParam.substring(0, inParam.length - 1); final db = await instance.database; await db.rawUpdate( ''' UPDATE $table SET $columnUpdationTime = NULL WHERE $columnLocalID IN ($inParam) AND ($columnLatitude IS NULL OR $columnLongitude IS NULL OR $columnLongitude = 0.0 or $columnLongitude = 0.0); ''', ); } Future doesFileExistInCollection( int uploadedFileID, int collectionID, ) async { final db = await instance.database; final rows = await db.query( table, where: '$columnUploadedFileID = ? AND $columnCollectionID = ?', whereArgs: [uploadedFileID, collectionID], limit: 1, ); return rows.isNotEmpty; } Future> getFilesFromIDs(List ids) async { final result = {}; if (ids.isEmpty) { return result; } String inParam = ""; for (final id in ids) { inParam += "'" + id.toString() + "',"; } inParam = inParam.substring(0, inParam.length - 1); final db = await instance.database; final results = await db.query( table, where: '$columnUploadedFileID IN ($inParam)', ); final files = _convertToFiles(results); for (final file in files) { result[file.uploadedFileID] = file; } return result; } List _convertToFiles(List> results) { final List files = []; for (final result in results) { files.add(_getFileFromRow(result)); } return files; } Map _getRowForFile(File file) { final row = {}; if (file.generatedID != null) { row[columnGeneratedID] = file.generatedID; } row[columnLocalID] = file.localID; row[columnUploadedFileID] = file.uploadedFileID ?? -1; row[columnOwnerID] = file.ownerID; row[columnCollectionID] = file.collectionID ?? -1; row[columnTitle] = file.title; row[columnDeviceFolder] = file.deviceFolder; if (file.location != null) { row[columnLatitude] = file.location.latitude; row[columnLongitude] = file.location.longitude; } row[columnFileType] = getInt(file.fileType); row[columnCreationTime] = file.creationTime; row[columnModificationTime] = file.modificationTime; row[columnUpdationTime] = file.updationTime; row[columnEncryptedKey] = file.encryptedKey; row[columnKeyDecryptionNonce] = file.keyDecryptionNonce; row[columnFileDecryptionHeader] = file.fileDecryptionHeader; row[columnThumbnailDecryptionHeader] = file.thumbnailDecryptionHeader; row[columnMetadataDecryptionHeader] = file.metadataDecryptionHeader; row[columnFileSubType] = file.fileSubType ?? -1; row[columnDuration] = file.duration ?? 0; row[columnExif] = file.exif; row[columnHash] = file.hash; row[columnMetadataVersion] = file.metadataVersion; row[columnMMdVersion] = file.mMdVersion ?? 0; row[columnMMdEncodedJson] = file.mMdEncodedJson ?? '{}'; row[columnMMdVisibility] = file.magicMetadata?.visibility ?? kVisibilityVisible; row[columnPubMMdVersion] = file.pubMmdVersion ?? 0; row[columnPubMMdEncodedJson] = file.pubMmdEncodedJson ?? '{}'; if (file.pubMagicMetadata != null && file.pubMagicMetadata.editedTime != null) { // override existing creationTime to avoid re-writing all queries related // to loading the gallery row[columnCreationTime] = file.pubMagicMetadata.editedTime; } return row; } Map _getRowForFileWithoutCollection(File file) { final row = {}; row[columnLocalID] = file.localID; row[columnUploadedFileID] = file.uploadedFileID ?? -1; row[columnOwnerID] = file.ownerID; row[columnTitle] = file.title; row[columnDeviceFolder] = file.deviceFolder; if (file.location != null) { row[columnLatitude] = file.location.latitude; row[columnLongitude] = file.location.longitude; } row[columnFileType] = getInt(file.fileType); row[columnCreationTime] = file.creationTime; row[columnModificationTime] = file.modificationTime; row[columnUpdationTime] = file.updationTime; row[columnFileDecryptionHeader] = file.fileDecryptionHeader; row[columnThumbnailDecryptionHeader] = file.thumbnailDecryptionHeader; row[columnMetadataDecryptionHeader] = file.metadataDecryptionHeader; row[columnFileSubType] = file.fileSubType ?? -1; row[columnDuration] = file.duration ?? 0; row[columnExif] = file.exif; row[columnHash] = file.hash; row[columnMetadataVersion] = file.metadataVersion; row[columnMMdVersion] = file.mMdVersion ?? 0; row[columnMMdEncodedJson] = file.mMdEncodedJson ?? '{}'; row[columnMMdVisibility] = file.magicMetadata?.visibility ?? kVisibilityVisible; row[columnPubMMdVersion] = file.pubMmdVersion ?? 0; row[columnPubMMdEncodedJson] = file.pubMmdEncodedJson ?? '{}'; if (file.pubMagicMetadata != null && file.pubMagicMetadata.editedTime != null) { // override existing creationTime to avoid re-writing all queries related // to loading the gallery row[columnCreationTime] = file.pubMagicMetadata.editedTime; } return row; } File _getFileFromRow(Map row) { final file = File(); file.generatedID = row[columnGeneratedID]; file.localID = row[columnLocalID]; file.uploadedFileID = row[columnUploadedFileID] == -1 ? null : row[columnUploadedFileID]; file.ownerID = row[columnOwnerID]; file.collectionID = row[columnCollectionID] == -1 ? null : row[columnCollectionID]; file.title = row[columnTitle]; file.deviceFolder = row[columnDeviceFolder]; if (row[columnLatitude] != null && row[columnLongitude] != null) { file.location = Location(row[columnLatitude], row[columnLongitude]); } file.fileType = getFileType(row[columnFileType]); file.creationTime = row[columnCreationTime]; file.modificationTime = row[columnModificationTime]; file.updationTime = row[columnUpdationTime] ?? -1; file.encryptedKey = row[columnEncryptedKey]; file.keyDecryptionNonce = row[columnKeyDecryptionNonce]; file.fileDecryptionHeader = row[columnFileDecryptionHeader]; file.thumbnailDecryptionHeader = row[columnThumbnailDecryptionHeader]; file.metadataDecryptionHeader = row[columnMetadataDecryptionHeader]; file.fileSubType = row[columnFileSubType] ?? -1; file.duration = row[columnDuration] ?? 0; file.exif = row[columnExif]; file.hash = row[columnHash]; file.metadataVersion = row[columnMetadataVersion] ?? 0; file.mMdVersion = row[columnMMdVersion] ?? 0; file.mMdEncodedJson = row[columnMMdEncodedJson] ?? '{}'; file.pubMmdVersion = row[columnPubMMdVersion] ?? 0; file.pubMmdEncodedJson = row[columnPubMMdEncodedJson] ?? '{}'; return file; } }