import "dart:async"; import "dart:io"; import "package:clip_ggml/clip_ggml.dart"; import "package:computer/computer.dart"; import "package:flutter/services.dart"; import "package:logging/logging.dart"; import "package:path_provider/path_provider.dart"; import "package:photos/db/files_db.dart"; import "package:photos/models/embedding.dart"; import "package:photos/models/file/file.dart"; import "package:photos/utils/thumbnail_util.dart"; class SemanticSearchService { SemanticSearchService._privateConstructor(); static final SemanticSearchService instance = SemanticSearchService._privateConstructor(); static final Computer _computer = Computer.shared(); static const int batchSize = 1; bool hasLoaded = false; final _logger = Logger("SemanticSearchService"); Future>? _ongoingRequest; PendingQuery? _nextQuery; Future init() async { await _loadModel(); _computeMissingEmbeddings(); } Future> search(String query) async { if (_ongoingRequest == null) { _ongoingRequest = getMatchingFiles(query).then((result) { _ongoingRequest = null; if (_nextQuery != null) { final next = _nextQuery; _nextQuery = null; search(next!.query).then((nextResult) { next.completer.complete(nextResult); }); } return result; }); return _ongoingRequest!; } else { // If there's an ongoing request, create or replace the nextCompleter. _nextQuery?.completer.future .timeout(const Duration(seconds: 0)); // Cancels the previous future. _nextQuery = PendingQuery(query, Completer>()); return _nextQuery!.completer.future; } } Future> getMatchingFiles(String query) async { _logger.info("Searching for " + query); var startTime = DateTime.now(); final textEmbedding = await _computer.compute( createTextEmbedding, param: { "text": query, }, taskName: "createTextEmbedding", ); var endTime = DateTime.now(); _logger.info( "createTextEmbedding took: " + (endTime.millisecondsSinceEpoch - startTime.millisecondsSinceEpoch) .toString() + "ms", ); startTime = DateTime.now(); final embeddings = await FilesDB.instance.getAllEmbeddings(); endTime = DateTime.now(); _logger.info( "Fetching embeddings took: " + (endTime.millisecondsSinceEpoch - startTime.millisecondsSinceEpoch) .toString() + "ms", ); startTime = DateTime.now(); final queryResults = []; for (final embedding in embeddings) { final score = computeScore({ "imageEmbedding": embedding.embedding, "textEmbedding": textEmbedding, }); queryResults.add(QueryResult(embedding.id, score)); } queryResults.sort((first, second) => second.score.compareTo(first.score)); queryResults.removeWhere((element) => element.score < 0.25); endTime = DateTime.now(); _logger.info( "computingScores took: " + (endTime.millisecondsSinceEpoch - startTime.millisecondsSinceEpoch) .toString() + "ms", ); startTime = DateTime.now(); final filesMap = await FilesDB.instance .getFilesFromGeneratedIDs(queryResults.map((e) => e.id).toList()); final results = []; for (final result in queryResults) { if (filesMap.containsKey(result.id)) { results.add(filesMap[result.id]!); } } endTime = DateTime.now(); _logger.info( "Fetching files took: " + (endTime.millisecondsSinceEpoch - startTime.millisecondsSinceEpoch) .toString() + "ms", ); _logger.info(results.length.toString() + " results"); return results; } Future _loadModel() async { const modelPath = "assets/models/clip/openai_clip-vit-base-patch32.ggmlv0.f16.bin"; final path = await _getAccessiblePathForAsset(modelPath, "model.bin"); final startTime = DateTime.now(); CLIP.loadModel(path); final endTime = DateTime.now(); _logger.info( "Loading model took: " + (endTime.millisecondsSinceEpoch - startTime.millisecondsSinceEpoch) .toString() + "ms", ); hasLoaded = true; } Future _getAccessiblePathForAsset( String assetPath, String tempName, ) async { final byteData = await rootBundle.load(assetPath); return _writeToFile(byteData.buffer.asUint8List(), tempName); } Future _writeToFile(Uint8List bytes, String fileName) async { final tempDir = await getTemporaryDirectory(); final file = await File('${tempDir.path}/$fileName').writeAsBytes(bytes); return file.path; } Future _computeMissingEmbeddings() async { final files = await FilesDB.instance.getFilesWithoutEmbeddings(); _logger.info(files.length.toString() + " pending to be embedded"); int counter = 0; final List batch = []; for (final file in files) { if (counter < batchSize) { batch.add(file); counter++; } else { await _computeImageEmbeddings(batch); counter = 0; batch.clear(); } } } Future _computeImageEmbeddings(List files) async { if (!hasLoaded) { return; } final List filePaths = []; for (final file in files) { filePaths.add((await getThumbnailFile(file))!.path); } _logger.info("Running clip over " + files.length.toString() + " items"); final startTime = DateTime.now(); final List> imageEmbeddings = []; if (filePaths.length == 1) { final result = await _computer.compute( createImageEmbedding, param: { "imagePath": filePaths.first, }, taskName: "createImageEmbedding", ) as List; imageEmbeddings.add(result); } else { final result = await _computer.compute( createImageEmbeddings, param: { "imagePaths": filePaths, }, taskName: "createImageEmbeddings", ) as List>; imageEmbeddings.addAll(result); } for (int i = 0; i < imageEmbeddings.length; i++) { await FilesDB.instance.insertEmbedding( Embedding( files[i].generatedID!, imageEmbeddings[i], -1, ), ); } final endTime = DateTime.now(); _logger.info( "createImageEmbeddings took: " + (endTime.millisecondsSinceEpoch - startTime.millisecondsSinceEpoch) .toString() + "ms for " + imageEmbeddings.length.toString() + " items", ); } } List> createImageEmbeddings(Map args) { return CLIP.createBatchImageEmbedding(args["imagePaths"]); } List createImageEmbedding(Map args) { return CLIP.createImageEmbedding(args["imagePath"]); } List createTextEmbedding(Map args) { return CLIP.createTextEmbedding(args["text"]); } double computeScore(Map args) { return CLIP.computeScore( args["imageEmbedding"] as List, args["textEmbedding"] as List, ); } class QueryResult { final int id; final double score; QueryResult(this.id, this.score); } class PendingQuery { final String query; final Completer> completer; PendingQuery(this.query, this.completer); }