Fetch and cache embeddings
This commit is contained in:
parent
8f4078fc5d
commit
a5e2fff272
|
@ -5,6 +5,10 @@ import "package:computer/computer.dart";
|
||||||
import "package:flutter/services.dart";
|
import "package:flutter/services.dart";
|
||||||
import "package:logging/logging.dart";
|
import "package:logging/logging.dart";
|
||||||
import "package:path_provider/path_provider.dart";
|
import "package:path_provider/path_provider.dart";
|
||||||
|
import "package:photos/db/files_db.dart";
|
||||||
|
import "package:photos/models/embedding.dart";
|
||||||
|
import "package:photos/models/file/file.dart";
|
||||||
|
import "package:photos/utils/thumbnail_util.dart";
|
||||||
|
|
||||||
class SemanticSearchService {
|
class SemanticSearchService {
|
||||||
SemanticSearchService._privateConstructor();
|
SemanticSearchService._privateConstructor();
|
||||||
|
@ -21,22 +25,37 @@ class SemanticSearchService {
|
||||||
await _loadModel();
|
await _loadModel();
|
||||||
}
|
}
|
||||||
|
|
||||||
Future<void> runInference(Uint8List image, String text) async {
|
Future<void> runInference(EnteFile file, String text) async {
|
||||||
if (!hasLoaded || isRunning) {
|
if (!hasLoaded || isRunning) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
isRunning = true;
|
isRunning = true;
|
||||||
_logger.info("Running clip");
|
_logger.info("Running clip");
|
||||||
final imagePath = await _writeToFile(image, "input.jpg");
|
final imagePath = (await getThumbnailFile(file))!.path;
|
||||||
|
|
||||||
var startTime = DateTime.now();
|
var startTime = DateTime.now();
|
||||||
final imageEmbedding = await _computer.compute(
|
// ignore: prefer_typing_uninitialized_variables
|
||||||
|
var imageEmbedding;
|
||||||
|
final embeddings = await FilesDB.instance.getAllEmbeddings();
|
||||||
|
bool hasCachedEmbedding = false;
|
||||||
|
for (final embedding in embeddings) {
|
||||||
|
if (embedding.id == file.generatedID) {
|
||||||
|
imageEmbedding = embedding.embedding;
|
||||||
|
hasCachedEmbedding = true;
|
||||||
|
_logger.info("Found cached embedding");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!hasCachedEmbedding) {
|
||||||
|
imageEmbedding ??= await _computer.compute(
|
||||||
createImageEmbedding,
|
createImageEmbedding,
|
||||||
param: {
|
param: {
|
||||||
"imagePath": imagePath,
|
"imagePath": imagePath,
|
||||||
},
|
},
|
||||||
taskName: "createImageEmbedding",
|
taskName: "createImageEmbedding",
|
||||||
);
|
);
|
||||||
|
await FilesDB.instance
|
||||||
|
.insertEmbedding(Embedding(file.generatedID!, imageEmbedding, -1));
|
||||||
|
}
|
||||||
var endTime = DateTime.now();
|
var endTime = DateTime.now();
|
||||||
_logger.info(
|
_logger.info(
|
||||||
"createImageEmbedding took: " +
|
"createImageEmbedding took: " +
|
||||||
|
|
|
@ -60,7 +60,7 @@ class _ZoomableImageState extends State<ZoomableImage>
|
||||||
@override
|
@override
|
||||||
void initState() {
|
void initState() {
|
||||||
_photo = widget.photo;
|
_photo = widget.photo;
|
||||||
_test();
|
SemanticSearchService.instance.runInference(_photo, "desert with a tree");
|
||||||
_logger = Logger("ZoomableImage");
|
_logger = Logger("ZoomableImage");
|
||||||
_logger.info('initState for ${_photo.generatedID} with tag ${_photo.tag}');
|
_logger.info('initState for ${_photo.generatedID} with tag ${_photo.tag}');
|
||||||
_scaleStateChangedCallback = (value) {
|
_scaleStateChangedCallback = (value) {
|
||||||
|
@ -80,13 +80,6 @@ class _ZoomableImageState extends State<ZoomableImage>
|
||||||
super.dispose();
|
super.dispose();
|
||||||
}
|
}
|
||||||
|
|
||||||
void _test() {
|
|
||||||
final thumb = ThumbnailInMemoryLruCache.get(_photo);
|
|
||||||
if (thumb != null) {
|
|
||||||
SemanticSearchService.instance.runInference(thumb, "desert with a tree");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@override
|
@override
|
||||||
Widget build(BuildContext context) {
|
Widget build(BuildContext context) {
|
||||||
if (_photo.isRemoteFile) {
|
if (_photo.isRemoteFile) {
|
||||||
|
|
|
@ -42,6 +42,18 @@ Future<Uint8List?> getThumbnail(EnteFile file) async {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Future<File?> getThumbnailFile(EnteFile file) async {
|
||||||
|
final thumbnail = await getThumbnail(file);
|
||||||
|
if (thumbnail != null) {
|
||||||
|
final cachedThumbnail = cachedThumbnailPath(file);
|
||||||
|
if (!await cachedThumbnail.exists()) {
|
||||||
|
await cachedThumbnail.writeAsBytes(thumbnail, flush: true);
|
||||||
|
}
|
||||||
|
return cachedThumbnail;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
Future<Uint8List> getThumbnailFromServer(EnteFile file) async {
|
Future<Uint8List> getThumbnailFromServer(EnteFile file) async {
|
||||||
final cachedThumbnail = cachedThumbnailPath(file);
|
final cachedThumbnail = cachedThumbnailPath(file);
|
||||||
if (await cachedThumbnail.exists()) {
|
if (await cachedThumbnail.exists()) {
|
||||||
|
|
Loading…
Reference in a new issue