2023-02-08 13:40:18 +00:00
|
|
|
import "dart:isolate";
|
2023-05-11 08:44:34 +00:00
|
|
|
import "dart:math";
|
2023-02-08 13:40:18 +00:00
|
|
|
import "dart:typed_data";
|
|
|
|
|
|
|
|
import "package:logging/logging.dart";
|
|
|
|
import "package:photos/services/object_detection/models/predictions.dart";
|
|
|
|
import 'package:photos/services/object_detection/models/recognition.dart';
|
2023-03-14 09:41:49 +00:00
|
|
|
import 'package:photos/services/object_detection/tflite/cocossd_classifier.dart';
|
|
|
|
import "package:photos/services/object_detection/tflite/mobilenet_classifier.dart";
|
2023-03-15 06:58:25 +00:00
|
|
|
import "package:photos/services/object_detection/tflite/scene_classifier.dart";
|
2023-02-08 13:40:18 +00:00
|
|
|
import "package:photos/services/object_detection/utils/isolate_utils.dart";
|
|
|
|
|
|
|
|
class ObjectDetectionService {
|
2023-05-11 07:38:02 +00:00
|
|
|
static const scoreThreshold = 0.35;
|
2023-02-12 13:59:09 +00:00
|
|
|
|
2023-02-08 13:40:18 +00:00
|
|
|
final _logger = Logger("ObjectDetectionService");
|
|
|
|
|
2023-03-14 09:41:49 +00:00
|
|
|
late CocoSSDClassifier _objectClassifier;
|
|
|
|
late MobileNetClassifier _mobileNetClassifier;
|
2023-03-15 06:58:25 +00:00
|
|
|
late SceneClassifier _sceneClassifier;
|
2023-02-08 13:40:18 +00:00
|
|
|
|
|
|
|
late IsolateUtils _isolateUtils;
|
|
|
|
|
|
|
|
ObjectDetectionService._privateConstructor();
|
2023-05-08 17:05:24 +00:00
|
|
|
bool inInitiated = false;
|
2023-02-08 13:40:18 +00:00
|
|
|
|
|
|
|
Future<void> init() async {
|
|
|
|
_isolateUtils = IsolateUtils();
|
|
|
|
await _isolateUtils.start();
|
2023-03-31 11:06:44 +00:00
|
|
|
try {
|
|
|
|
_objectClassifier = CocoSSDClassifier();
|
|
|
|
} catch (e, s) {
|
|
|
|
_logger.severe("Could not initialize cocossd", e, s);
|
|
|
|
}
|
|
|
|
try {
|
|
|
|
_mobileNetClassifier = MobileNetClassifier();
|
|
|
|
} catch (e, s) {
|
|
|
|
_logger.severe("Could not initialize mobilenet", e, s);
|
|
|
|
}
|
|
|
|
try {
|
|
|
|
_sceneClassifier = SceneClassifier();
|
|
|
|
} catch (e, s) {
|
|
|
|
_logger.severe("Could not initialize sceneclassifier", e, s);
|
|
|
|
}
|
2023-05-08 17:05:24 +00:00
|
|
|
inInitiated = true;
|
2023-02-08 13:40:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static ObjectDetectionService instance =
|
|
|
|
ObjectDetectionService._privateConstructor();
|
|
|
|
|
2023-05-11 08:44:34 +00:00
|
|
|
Future<Map<String, double>> predict(Uint8List bytes) async {
|
2023-02-08 13:40:18 +00:00
|
|
|
try {
|
2023-05-08 17:05:24 +00:00
|
|
|
if (!inInitiated) {
|
|
|
|
return Future.error("ObjectDetectionService init is not completed");
|
|
|
|
}
|
2023-05-11 08:44:34 +00:00
|
|
|
final results = <String, double>{};
|
|
|
|
final methods = [_getObjects, _getMobileNetResults, _getSceneResults];
|
|
|
|
|
|
|
|
for (var method in methods) {
|
|
|
|
final methodResults = await method(bytes);
|
|
|
|
methodResults.forEach((key, value) {
|
|
|
|
results.update(
|
|
|
|
key,
|
|
|
|
(existingValue) => max(existingValue, value),
|
|
|
|
ifAbsent: () => value,
|
|
|
|
);
|
|
|
|
});
|
|
|
|
}
|
|
|
|
return results;
|
2023-02-08 13:40:18 +00:00
|
|
|
} catch (e, s) {
|
|
|
|
_logger.severe(e, s);
|
|
|
|
rethrow;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-05-11 08:44:34 +00:00
|
|
|
Future<Map<String, double>> _getObjects(Uint8List bytes) async {
|
2023-03-31 11:06:44 +00:00
|
|
|
try {
|
|
|
|
final isolateData = IsolateData(
|
|
|
|
bytes,
|
|
|
|
_objectClassifier.interpreter.address,
|
|
|
|
_objectClassifier.labels,
|
|
|
|
ClassifierType.cocossd,
|
|
|
|
);
|
|
|
|
return _getPredictions(isolateData);
|
|
|
|
} catch (e, s) {
|
|
|
|
_logger.severe("Could not run cocossd", e, s);
|
|
|
|
}
|
2023-05-11 08:44:34 +00:00
|
|
|
return {};
|
2023-03-14 09:41:49 +00:00
|
|
|
}
|
|
|
|
|
2023-05-11 08:44:34 +00:00
|
|
|
Future<Map<String, double>> _getMobileNetResults(Uint8List bytes) async {
|
2023-03-31 11:06:44 +00:00
|
|
|
try {
|
|
|
|
final isolateData = IsolateData(
|
|
|
|
bytes,
|
|
|
|
_mobileNetClassifier.interpreter.address,
|
|
|
|
_mobileNetClassifier.labels,
|
|
|
|
ClassifierType.mobilenet,
|
|
|
|
);
|
|
|
|
return _getPredictions(isolateData);
|
|
|
|
} catch (e, s) {
|
|
|
|
_logger.severe("Could not run mobilenet", e, s);
|
|
|
|
}
|
2023-05-11 08:44:34 +00:00
|
|
|
return {};
|
2023-03-15 06:58:25 +00:00
|
|
|
}
|
|
|
|
|
2023-05-11 08:44:34 +00:00
|
|
|
Future<Map<String, double>> _getSceneResults(Uint8List bytes) async {
|
2023-03-31 11:06:44 +00:00
|
|
|
try {
|
|
|
|
final isolateData = IsolateData(
|
|
|
|
bytes,
|
|
|
|
_sceneClassifier.interpreter.address,
|
|
|
|
_sceneClassifier.labels,
|
|
|
|
ClassifierType.scenes,
|
|
|
|
);
|
|
|
|
return _getPredictions(isolateData);
|
|
|
|
} catch (e, s) {
|
|
|
|
_logger.severe("Could not run scene detection", e, s);
|
|
|
|
}
|
2023-05-11 08:44:34 +00:00
|
|
|
return {};
|
2023-03-15 06:58:25 +00:00
|
|
|
}
|
|
|
|
|
2023-05-11 08:44:34 +00:00
|
|
|
Future<Map<String, double>> _getPredictions(IsolateData isolateData) async {
|
2023-03-14 09:41:49 +00:00
|
|
|
final predictions = await _inference(isolateData);
|
2023-05-11 08:44:34 +00:00
|
|
|
final Map<String, double> results = {};
|
|
|
|
|
2023-03-31 12:33:43 +00:00
|
|
|
if (predictions.error == null) {
|
|
|
|
for (final Recognition result in predictions.recognitions!) {
|
|
|
|
if (result.score > scoreThreshold) {
|
2023-05-11 08:44:34 +00:00
|
|
|
// Update the result score only if it's higher than the current score
|
|
|
|
if (!results.containsKey(result.label) ||
|
|
|
|
results[result.label]! < result.score) {
|
|
|
|
results[result.label] = result.score;
|
|
|
|
}
|
2023-03-31 12:33:43 +00:00
|
|
|
}
|
2023-03-14 09:41:49 +00:00
|
|
|
}
|
2023-05-11 08:44:34 +00:00
|
|
|
|
2023-03-31 12:33:43 +00:00
|
|
|
_logger.info(
|
2023-05-11 08:44:34 +00:00
|
|
|
"Time taken for ${isolateData.type}: ${predictions.stats!.totalElapsedTime}ms",
|
2023-03-31 12:33:43 +00:00
|
|
|
);
|
|
|
|
} else {
|
|
|
|
_logger.severe(
|
2023-05-11 08:44:34 +00:00
|
|
|
"Error while fetching predictions for ${isolateData.type}",
|
2023-03-31 12:33:43 +00:00
|
|
|
predictions.error,
|
|
|
|
);
|
2023-03-14 09:41:49 +00:00
|
|
|
}
|
2023-05-11 08:44:34 +00:00
|
|
|
|
|
|
|
return results;
|
2023-03-14 09:41:49 +00:00
|
|
|
}
|
|
|
|
|
2023-02-08 13:40:18 +00:00
|
|
|
/// Runs inference in another isolate
|
|
|
|
Future<Predictions> _inference(IsolateData isolateData) async {
|
|
|
|
final responsePort = ReceivePort();
|
|
|
|
_isolateUtils.sendPort.send(
|
|
|
|
isolateData..responsePort = responsePort.sendPort,
|
|
|
|
);
|
|
|
|
return await responsePort.first;
|
|
|
|
}
|
|
|
|
}
|