import 'dart:io'; import 'package:logging/logging.dart'; import 'package:photos/models/file_type.dart'; import 'package:photos/models/location.dart'; import 'package:photos/models/file.dart'; import 'package:path/path.dart'; import 'package:sqflite/sqflite.dart'; import 'package:path_provider/path_provider.dart'; import 'package:sqflite_migration/sqflite_migration.dart'; class FilesDB { static final _databaseName = "ente.files.db"; static final Logger _logger = Logger("FilesDB"); static final table = 'files'; static final tempTable = 'temp_files'; static final columnGeneratedID = '_id'; static final columnUploadedFileID = 'uploaded_file_id'; static final columnOwnerID = 'owner_id'; static final columnCollectionID = 'collection_id'; static final columnLocalID = 'local_id'; static final columnTitle = 'title'; static final columnDeviceFolder = 'device_folder'; static final columnLatitude = 'latitude'; static final columnLongitude = 'longitude'; static final columnFileType = 'file_type'; static final columnIsDeleted = 'is_deleted'; static final columnCreationTime = 'creation_time'; static final columnModificationTime = 'modification_time'; static final columnUpdationTime = 'updation_time'; static final columnEncryptedKey = 'encrypted_key'; static final columnKeyDecryptionNonce = 'key_decryption_nonce'; static final columnFileDecryptionHeader = 'file_decryption_header'; static final columnThumbnailDecryptionHeader = 'thumbnail_decryption_header'; static final columnMetadataDecryptionHeader = 'metadata_decryption_header'; static final intitialScript = [...createTable(table), ...addIndex()]; static final migrationScripts = [...alterDeviceFolderToAllowNULL()]; final dbConfig = MigrationConfig( initializationScript: intitialScript, migrationScripts: migrationScripts); // make this a singleton class FilesDB._privateConstructor(); static final FilesDB instance = FilesDB._privateConstructor(); // only have a single app-wide reference to the database static Database _database; Future get database async { if (_database != null) return _database; // lazily instantiate the db the first time it is accessed _database = await _initDatabase(); return _database; } // this opens the database (and creates it if it doesn't exist) _initDatabase() async { Directory documentsDirectory = await getApplicationDocumentsDirectory(); String path = join(documentsDirectory.path, _databaseName); return await openDatabaseWithMigration(path, dbConfig); } // SQL code to create the database table static List createTable(String tableName) { return [ ''' CREATE TABLE $tableName ( $columnGeneratedID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, $columnLocalID TEXT, $columnUploadedFileID INTEGER, $columnOwnerID INTEGER, $columnCollectionID INTEGER, $columnTitle TEXT NOT NULL, $columnDeviceFolder TEXT, $columnLatitude REAL, $columnLongitude REAL, $columnFileType INTEGER, $columnModificationTime TEXT NOT NULL, $columnEncryptedKey TEXT, $columnKeyDecryptionNonce TEXT, $columnFileDecryptionHeader TEXT, $columnThumbnailDecryptionHeader TEXT, $columnMetadataDecryptionHeader TEXT, $columnIsDeleted INTEGER DEFAULT 0, $columnCreationTime TEXT NOT NULL, $columnUpdationTime TEXT, UNIQUE($columnLocalID, $columnUploadedFileID, $columnCollectionID) ); ''', ]; } static List addIndex() { return [ ''' CREATE INDEX collection_id_index ON $table($columnCollectionID); CREATE INDEX device_folder_index ON $table($columnDeviceFolder); CREATE INDEX creation_time_index ON $table($columnCreationTime); CREATE INDEX updation_time_index ON $table($columnUpdationTime); ''' ]; } static List alterDeviceFolderToAllowNULL() { return [ ...createTable(tempTable), ''' INSERT INTO $tempTable SELECT * FROM $table; DROP TABLE $table; ALTER TABLE $tempTable RENAME TO $table; ''' ]; } Future clearTable() async { final db = await instance.database; await db.delete(table); } Future insert(File file) async { final db = await instance.database; return await db.insert(table, _getRowForFile(file), conflictAlgorithm: ConflictAlgorithm.replace); } Future insertMultiple(List files) async { final startTime = DateTime.now(); final db = await instance.database; var batch = db.batch(); int batchCounter = 0; for (File file in files) { if (batchCounter == 400) { await batch.commit(); batch = db.batch(); batchCounter = 0; } batch.insert( table, _getRowForFile(file), conflictAlgorithm: ConflictAlgorithm.replace, ); batchCounter++; } await batch.commit(noResult: true); final endTime = DateTime.now(); final duration = Duration( microseconds: endTime.microsecondsSinceEpoch - startTime.microsecondsSinceEpoch); _logger.info("Batch insert of " + files.length.toString() + " took " + duration.inMilliseconds.toString() + "ms."); } Future getFile(int generatedID) async { final db = await instance.database; final results = await db.query(table, where: '$columnGeneratedID = ?', whereArgs: [generatedID]); if (results.isEmpty) { return null; } return _convertToFiles(results)[0]; } Future getUploadedFile(int uploadedID, int collectionID) async { final db = await instance.database; final results = await db.query( table, where: '$columnUploadedFileID = ? AND $columnCollectionID = ?', whereArgs: [ uploadedID, collectionID, ], ); if (results.isEmpty) { return null; } return _convertToFiles(results)[0]; } Future> getUploadedFileIDs(int collectionID) async { final db = await instance.database; final results = await db.query( table, columns: [columnUploadedFileID], where: '$columnCollectionID = ?', whereArgs: [ collectionID, ], ); final ids = Set(); ; for (final result in results) { ids.add(result[columnUploadedFileID]); } return ids; } Future> getAllFiles(int startTime, int endTime, {int limit}) async { final db = await instance.database; final results = await db.query( table, where: '$columnCreationTime >= ? AND $columnCreationTime <= ? AND $columnIsDeleted = 0 AND ($columnLocalID IS NOT NULL OR $columnUploadedFileID IS NOT NULL)', whereArgs: [startTime, endTime], orderBy: '$columnCreationTime DESC', limit: limit, ); return _convertToFiles(results); } Future> getFilesInPaths( int startTime, int endTime, List paths, {int limit}) async { final db = await instance.database; final results = await db.query( table, where: '$columnCreationTime >= ? AND $columnCreationTime <= ? AND $columnIsDeleted = 0 AND (($columnLocalID IS NOT NULL AND $columnDeviceFolder IN (?)) OR $columnUploadedFileID IS NOT NULL)', whereArgs: [startTime, endTime, paths.join(", ")], orderBy: '$columnCreationTime DESC', limit: limit, ); return _convertToFiles(results); } Future> getFilesInCollection( int collectionID, int startTime, int endTime, {int limit}) async { final db = await instance.database; final results = await db.query( table, where: '$columnCollectionID = ? AND $columnCreationTime >= ? AND $columnCreationTime <= ? AND $columnIsDeleted = 0', whereArgs: [collectionID, startTime, endTime], orderBy: '$columnCreationTime DESC', limit: limit, ); final files = _convertToFiles(results); _logger.info("Fetched " + files.length.toString() + " files"); return files; } Future> getFilesInPath(String path, int startTime, int endTime, {int limit}) async { final db = await instance.database; final results = await db.query( table, where: '$columnDeviceFolder = ? AND $columnCreationTime >= ? AND $columnCreationTime <= ? AND $columnLocalID IS NOT NULL AND $columnIsDeleted = 0', whereArgs: [path, startTime, endTime], orderBy: '$columnCreationTime DESC', groupBy: '$columnLocalID', limit: limit, ); return _convertToFiles(results); } Future> getAllVideos() async { final db = await instance.database; final results = await db.query( table, where: '$columnLocalID IS NOT NULL AND $columnFileType = 1 AND $columnIsDeleted = 0', orderBy: '$columnCreationTime DESC', ); return _convertToFiles(results); } Future> getAllInPath(String path) async { final db = await instance.database; final results = await db.query( table, where: '$columnLocalID IS NOT NULL AND $columnDeviceFolder = ? AND $columnIsDeleted = 0', whereArgs: [path], orderBy: '$columnCreationTime DESC', groupBy: '$columnLocalID', ); return _convertToFiles(results); } Future> getFilesCreatedWithinDurations( List> durations) async { final db = await instance.database; String whereClause = ""; for (int index = 0; index < durations.length; index++) { whereClause += "($columnCreationTime > " + durations[index][0].toString() + " AND $columnCreationTime < " + durations[index][1].toString() + ")"; if (index != durations.length - 1) { whereClause += " OR "; } } final results = await db.query( table, where: whereClause + " AND $columnIsDeleted = 0", orderBy: '$columnCreationTime ASC', ); return _convertToFiles(results); } Future> getDeletedFileIDs() async { final db = await instance.database; final rows = await db.query( table, columns: [columnUploadedFileID], distinct: true, where: '$columnIsDeleted = 1', orderBy: '$columnCreationTime DESC', ); final result = List(); for (final row in rows) { result.add(row[columnUploadedFileID]); } return result; } Future> getFilesToBeUploadedWithinFolders( Set folders) async { if (folders.isEmpty) { return []; } final db = await instance.database; String inParam = ""; for (final folder in folders) { inParam += "'" + folder + "',"; } inParam = inParam.substring(0, inParam.length - 1); final results = await db.query( table, where: '$columnUploadedFileID IS NULL AND $columnDeviceFolder IN ($inParam)', orderBy: '$columnCreationTime DESC', groupBy: '$columnLocalID', ); return _convertToFiles(results); } Future> getUploadedFileIDsToBeUpdated() async { final db = await instance.database; final rows = await db.query( table, columns: [columnUploadedFileID], where: '($columnLocalID IS NOT NULL AND $columnUploadedFileID IS NOT NULL AND $columnUpdationTime IS NULL AND $columnIsDeleted = 0)', orderBy: '$columnCreationTime DESC', distinct: true, ); final uploadedFileIDs = List(); for (final row in rows) { uploadedFileIDs.add(row[columnUploadedFileID]); } return uploadedFileIDs; } Future getUploadedFileInAnyCollection(int uploadedFileID) async { final db = await instance.database; final results = await db.query( table, where: '$columnUploadedFileID = ?', whereArgs: [ uploadedFileID, ], limit: 1, ); if (results.isEmpty) { return null; } return _convertToFiles(results)[0]; } Future> getExistingLocalFileIDs() async { final db = await instance.database; final rows = await db.query( table, columns: [columnLocalID], distinct: true, where: '$columnLocalID IS NOT NULL', ); final result = Set(); for (final row in rows) { result.add(row[columnLocalID]); } return result; } Future getNumberOfUploadedFiles() async { final db = await instance.database; final rows = await db.query( table, columns: [columnUploadedFileID], where: '($columnLocalID IS NOT NULL AND $columnUploadedFileID IS NOT NULL AND $columnUpdationTime IS NOT NULL AND $columnIsDeleted = 0)', orderBy: '$columnCreationTime DESC', distinct: true, ); return rows.length; } Future updateUploadedFile( String localID, String title, Location location, int creationTime, int modificationTime, int updationTime, ) async { final db = await instance.database; return await db.update( table, { columnTitle: title, columnLatitude: location.latitude, columnLongitude: location.longitude, columnCreationTime: creationTime, columnModificationTime: modificationTime, columnUpdationTime: updationTime, }, where: '$columnLocalID = ?', whereArgs: [localID], ); } Future> getMatchingFiles( String title, String deviceFolder, int creationTime, ) async { final db = await instance.database; var query; if (deviceFolder != null) { query = db.query( table, where: '''$columnTitle=? AND $columnDeviceFolder=? AND $columnCreationTime=?''', whereArgs: [ title, deviceFolder, creationTime, ], ); } else { query = db.query( table, where: '''$columnTitle=? AND $columnCreationTime=?''', whereArgs: [ title, creationTime, ], ); } final rows = await query; if (rows.isNotEmpty) { return _convertToFiles(rows); } else { return null; } } Future update(File file) async { final db = await instance.database; return await db.update( table, _getRowForFile(file), where: '$columnGeneratedID = ?', whereArgs: [file.generatedID], ); } Future updateUploadedFileAcrossCollections(File file) async { final db = await instance.database; return await db.update( table, _getRowForFileWithoutCollection(file), where: '$columnUploadedFileID = ?', whereArgs: [file.uploadedFileID], ); } Future delete(int uploadedFileID) async { final db = await instance.database; return db.delete( table, where: '$columnUploadedFileID =?', whereArgs: [uploadedFileID], ); } Future deleteMultipleUploadedFiles(List uploadedFileIDs) async { final db = await instance.database; return await db.delete( table, where: '$columnUploadedFileID IN (${uploadedFileIDs.join(', ')})', ); } Future deleteLocalFile(String localID) async { final db = await instance.database; return db.delete( table, where: '$columnLocalID =?', whereArgs: [localID], ); } Future deleteFromCollection(int uploadedFileID, int collectionID) async { final db = await instance.database; return db.delete( table, where: '$columnUploadedFileID = ? AND $columnCollectionID = ?', whereArgs: [uploadedFileID, collectionID], ); } Future deleteCollection(int collectionID) async { final db = await instance.database; return db.delete( table, where: '$columnCollectionID = ?', whereArgs: [collectionID], ); } Future removeFromCollection(int collectionID, List fileIDs) async { final db = await instance.database; return db.delete( table, where: '$columnCollectionID =? AND $columnUploadedFileID IN (${fileIDs.join(', ')})', whereArgs: [collectionID], ); } Future> getLatestLocalFiles() async { final db = await instance.database; final rows = await db.rawQuery(''' SELECT $table.* FROM $table INNER JOIN ( SELECT $columnDeviceFolder, MAX($columnCreationTime) AS max_creation_time FROM $table WHERE $table.$columnLocalID IS NOT NULL GROUP BY $columnDeviceFolder ) latest_files ON $table.$columnDeviceFolder = latest_files.$columnDeviceFolder AND $table.$columnCreationTime = latest_files.max_creation_time; '''); final files = _convertToFiles(rows); // TODO: Do this de-duplication within the SQL Query final folderMap = Map(); for (final file in files) { if (folderMap.containsKey(file.deviceFolder)) { if (folderMap[file.deviceFolder].updationTime < file.updationTime) { continue; } } folderMap[file.deviceFolder] = file; } return folderMap.values.toList(); } Future> getLatestCollectionFiles() async { final db = await instance.database; final rows = await db.rawQuery(''' SELECT $table.* FROM $table INNER JOIN ( SELECT $columnCollectionID, MAX($columnCreationTime) AS max_creation_time FROM $table GROUP BY $columnCollectionID ) latest_files ON $table.$columnCollectionID = latest_files.$columnCollectionID AND $table.$columnCreationTime = latest_files.max_creation_time; '''); final files = _convertToFiles(rows); // TODO: Do this de-duplication within the SQL Query final collectionMap = Map(); for (final file in files) { if (collectionMap.containsKey(file.collectionID)) { if (collectionMap[file.collectionID].updationTime < file.updationTime) { continue; } } collectionMap[file.collectionID] = file; } return collectionMap.values.toList(); } Future getLastModifiedFileInCollection(int collectionID) async { final db = await instance.database; final rows = await db.query( table, where: '$columnCollectionID = ? AND $columnIsDeleted = 0', whereArgs: [collectionID], orderBy: '$columnUpdationTime DESC', limit: 1, ); if (rows.isNotEmpty) { return _getFileFromRow(rows[0]); } else { return null; } } Future doesFileExistInCollection( int uploadedFileID, int collectionID) async { final db = await instance.database; final rows = await db.query( table, where: '$columnUploadedFileID = ? AND $columnCollectionID = ?', whereArgs: [uploadedFileID, collectionID], limit: 1, ); return rows.isNotEmpty; } List _convertToFiles(List> results) { final List files = []; for (final result in results) { files.add(_getFileFromRow(result)); } return files; } Map _getRowForFile(File file) { final row = new Map(); if (file.generatedID != null) { row[columnGeneratedID] = file.generatedID; } row[columnLocalID] = file.localID; row[columnUploadedFileID] = file.uploadedFileID; row[columnOwnerID] = file.ownerID; row[columnCollectionID] = file.collectionID; row[columnTitle] = file.title; row[columnDeviceFolder] = file.deviceFolder; if (file.location != null) { row[columnLatitude] = file.location.latitude; row[columnLongitude] = file.location.longitude; } switch (file.fileType) { case FileType.image: row[columnFileType] = 0; break; case FileType.video: row[columnFileType] = 1; break; default: row[columnFileType] = -1; } row[columnCreationTime] = file.creationTime; row[columnModificationTime] = file.modificationTime; row[columnUpdationTime] = file.updationTime; row[columnEncryptedKey] = file.encryptedKey; row[columnKeyDecryptionNonce] = file.keyDecryptionNonce; row[columnFileDecryptionHeader] = file.fileDecryptionHeader; row[columnThumbnailDecryptionHeader] = file.thumbnailDecryptionHeader; row[columnMetadataDecryptionHeader] = file.metadataDecryptionHeader; return row; } Map _getRowForFileWithoutCollection(File file) { final row = new Map(); row[columnLocalID] = file.localID; row[columnUploadedFileID] = file.uploadedFileID; row[columnOwnerID] = file.ownerID; row[columnTitle] = file.title; row[columnDeviceFolder] = file.deviceFolder; if (file.location != null) { row[columnLatitude] = file.location.latitude; row[columnLongitude] = file.location.longitude; } switch (file.fileType) { case FileType.image: row[columnFileType] = 0; break; case FileType.video: row[columnFileType] = 1; break; default: row[columnFileType] = -1; } row[columnCreationTime] = file.creationTime; row[columnModificationTime] = file.modificationTime; row[columnUpdationTime] = file.updationTime; row[columnFileDecryptionHeader] = file.fileDecryptionHeader; row[columnThumbnailDecryptionHeader] = file.thumbnailDecryptionHeader; row[columnMetadataDecryptionHeader] = file.metadataDecryptionHeader; return row; } File _getFileFromRow(Map row) { final file = File(); file.generatedID = row[columnGeneratedID]; file.localID = row[columnLocalID]; file.uploadedFileID = row[columnUploadedFileID]; file.ownerID = row[columnOwnerID]; file.collectionID = row[columnCollectionID]; file.title = row[columnTitle]; file.deviceFolder = row[columnDeviceFolder]; if (row[columnLatitude] != null && row[columnLongitude] != null) { file.location = Location(row[columnLatitude], row[columnLongitude]); } file.fileType = getFileType(row[columnFileType]); file.creationTime = int.parse(row[columnCreationTime]); file.modificationTime = int.parse(row[columnModificationTime]); file.updationTime = row[columnUpdationTime] == null ? -1 : int.parse(row[columnUpdationTime]); file.encryptedKey = row[columnEncryptedKey]; file.keyDecryptionNonce = row[columnKeyDecryptionNonce]; file.fileDecryptionHeader = row[columnFileDecryptionHeader]; file.thumbnailDecryptionHeader = row[columnThumbnailDecryptionHeader]; file.metadataDecryptionHeader = row[columnMetadataDecryptionHeader]; return file; } }