From 628d13ea530509139312de5a789be4024e660f25 Mon Sep 17 00:00:00 2001
From: Ashil <77285023+ashilkn@users.noreply.github.com>
Date: Tue, 9 Apr 2024 15:17:48 +0530
Subject: [PATCH] [mobile][photos] Use sqlite async for fetching all files for
 search (#1391)

## Description

Using [sqlite_async](https://pub.dev/packages/sqlite_async) has
increased query speed by about 5x and has removed GC logs like:
`Background concurrent copying GC freed 424845(20MB) AllocSpace objects,
183(4932KB) LOS objects, 37% free, 39MB/63MB, paused 144us,44us total
128.048ms`, which has improved performance of the app.

Tried using [sqlite3](https://pub.dev/packages/sqlite3), which reduced
query speed by 10x and removed the GC log, but introduced some jank
since it blocks the UI.

Converting the fetched rows to `EnteFile` now runs on an isolate to
avoid blocking the UI.

## Tests

Did manual testing to see difference in jank.
---
 mobile/lib/db/files_db.dart | 36 ++++++++++++++++++++++++++++++++----
 mobile/pubspec.lock         | 12 ++++++++++--
 mobile/pubspec.yaml         |  3 ++-
 3 files changed, 44 insertions(+), 7 deletions(-)

diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart
index 62122a29b..5ffd710e5 100644
--- a/mobile/lib/db/files_db.dart
+++ b/mobile/lib/db/files_db.dart
@@ -1,5 +1,6 @@
 import "dart:io";
 
+import "package:computer/computer.dart";
 import 'package:flutter/foundation.dart';
 import 'package:logging/logging.dart';
 import 'package:path/path.dart';
@@ -16,6 +17,7 @@ import 'package:photos/utils/file_uploader_util.dart';
 import 'package:sqflite/sqflite.dart';
 import 'package:sqflite_migration/sqflite_migration.dart';
 import 'package:sqlite3/sqlite3.dart' as sqlite3;
+import 'package:sqlite_async/sqlite_async.dart' as sqlite_async;
 
 class FilesDB {
   /*
@@ -102,6 +104,7 @@ class FilesDB {
   // only have a single app-wide reference to the database
   static Future<Database>? _dbFuture;
   static Future<sqlite3.Database>? _ffiDBFuture;
+  static Future<sqlite_async.SqliteDatabase>? _sqliteAsyncDBFuture;
 
   Future<Database> get database async {
     // lazily instantiate the db the first time it is accessed
@@ -114,6 +117,11 @@ class FilesDB {
     return _ffiDBFuture!;
   }
 
+  Future<sqlite_async.SqliteDatabase> get sqliteAsyncDB async {
+    _sqliteAsyncDBFuture ??= _initSqliteAsyncDatabase();
+    return _sqliteAsyncDBFuture!;
+  }
+
   // this opens the database (and creates it if it doesn't exist)
   Future<Database> _initDatabase() async {
     final Directory documentsDirectory =
@@ -131,6 +139,14 @@ class FilesDB {
     return sqlite3.sqlite3.open(path);
   }
 
+  Future<sqlite_async.SqliteDatabase> _initSqliteAsyncDatabase() async {
+    final Directory documentsDirectory =
+        await getApplicationDocumentsDirectory();
+    final String path = join(documentsDirectory.path, _databaseName);
+    _logger.info("DB path " + path);
+    return sqlite_async.SqliteDatabase(path: path);
+  }
+
   // SQL code to create the database table
   static List<String> createTable(String tableName) {
     return [
@@ -1450,6 +1466,14 @@ class FilesDB {
     return collectionIDsOfFile;
   }
 
+  List<EnteFile> convertToFilesForIsolate(Map args) {
+    final List<EnteFile> files = [];
+    for (final result in args["result"]) {
+      files.add(_getFileFromRow(result));
+    }
+    return files;
+  }
+
   List<EnteFile> convertToFiles(List<Map<String, dynamic>> results) {
     final List<EnteFile> files = [];
     for (final result in results) {
@@ -1576,10 +1600,14 @@ class FilesDB {
     Set<int> collectionsToIgnore, {
     bool dedupeByUploadId = true,
   }) async {
-    final db = await instance.database;
-    final List<Map<String, dynamic>> result =
-        await db.query(filesTable, orderBy: '$columnCreationTime DESC');
-    final List<EnteFile> files = convertToFiles(result);
+    final db = await instance.sqliteAsyncDB;
+
+    final result = await db.getAll(
+      'SELECT * FROM $filesTable ORDER BY $columnCreationTime DESC',
+    );
+    final List<EnteFile> files = await Computer.shared()
+        .compute(convertToFilesForIsolate, param: {"result": result});
+
     final List<EnteFile> deduplicatedFiles = await applyDBFilters(
       files,
       DBFilterOptions(
diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock
index a4d34d05a..cb89d90d7 100644
--- a/mobile/pubspec.lock
+++ b/mobile/pubspec.lock
@@ -2019,10 +2019,18 @@ packages:
     dependency: "direct main"
     description:
       name: sqlite3_flutter_libs
-      sha256: "90963b515721d6a71e96f438175cf43c979493ed14822860a300b69694c74eb6"
+      sha256: d6c31c8511c441d1f12f20b607343df1afe4eddf24a1cf85021677c8eea26060
       url: "https://pub.dev"
     source: hosted
-    version: "0.5.19+1"
+    version: "0.5.20"
+  sqlite_async:
+    dependency: "direct main"
+    description:
+      name: sqlite_async
+      sha256: b252fd3a53766460b2f240e082517d3bc1cce7682a1550817f0f799d4a7a4087
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.5.2"
   stack_trace:
     dependency: transitive
     description:
diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml
index 1c28bc1ff..104c17ba6 100644
--- a/mobile/pubspec.yaml
+++ b/mobile/pubspec.yaml
@@ -146,7 +146,8 @@ dependencies:
   sqflite: ^2.3.0
   sqflite_migration: ^0.3.0
   sqlite3: ^2.1.0
-  sqlite3_flutter_libs: ^0.5.19+1
+  sqlite3_flutter_libs: ^0.5.20
+  sqlite_async: ^0.5.2
   step_progress_indicator: ^1.0.2
   styled_text: ^7.0.0
   syncfusion_flutter_core: ^19.2.49