From 1f8ea9c2f467abdaf5d7db06c110f5d28e1ce35b Mon Sep 17 00:00:00 2001 From: vishnukvmd Date: Tue, 26 Dec 2023 23:21:56 +0530 Subject: [PATCH 1/8] Create a service to load and cache remote assets --- lib/services/remote_assets_service.dart | 57 +++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 lib/services/remote_assets_service.dart diff --git a/lib/services/remote_assets_service.dart b/lib/services/remote_assets_service.dart new file mode 100644 index 000000000..8a11f84a4 --- /dev/null +++ b/lib/services/remote_assets_service.dart @@ -0,0 +1,57 @@ +import "dart:io"; + +import "package:logging/logging.dart"; +import "package:path_provider/path_provider.dart"; +import "package:photos/core/network/network.dart"; + +class RemoteAssetsService { + static final _logger = Logger("RemoteAssetsService"); + + RemoteAssetsService._privateConstructor(); + + static final RemoteAssetsService instance = + RemoteAssetsService._privateConstructor(); + + Future getAsset(String remotePath) async { + final path = await _getLocalPath(remotePath); + final file = File(path); + if (await file.exists()) { + _logger.info("Returning cached file for $remotePath"); + return file; + } else { + final tempFile = File(path + ".temp"); + await _downloadFile(remotePath, tempFile.path); + await tempFile.rename(path); + return File(path); + } + } + + Future _getLocalPath(String remotePath) async { + return (await getTemporaryDirectory()).path + + "/assets/" + + _urlToFileName(remotePath); + } + + String _urlToFileName(String url) { + // Remove the protocol part (http:// or https://) + String fileName = url + .replaceAll(RegExp(r'https?://'), '') + // Replace all non-alphanumeric characters except for underscores and periods with an underscore + .replaceAll(RegExp(r'[^\w\.]'), '_'); + // Optionally, you might want to trim the resulting string to a certain length + + // Replace periods with underscores for better readability, if desired + fileName = fileName.replaceAll('.', '_'); + + return fileName; + } + + Future _downloadFile(String url, String savePath) async { + _logger.info("Downloading " + url); + final existingFile = File(savePath); + if (await existingFile.exists()) { + await existingFile.delete(); + } + await NetworkClient.instance.getDio().download(url, savePath); + } +} From 5d25785eddfc8e4a93429da13a2eebf58a620480 Mon Sep 17 00:00:00 2001 From: vishnukvmd Date: Tue, 26 Dec 2023 23:22:09 +0530 Subject: [PATCH 2/8] Load and parse cities --- lib/services/location_service.dart | 49 ++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/lib/services/location_service.dart b/lib/services/location_service.dart index 10e3bfd72..516adff01 100644 --- a/lib/services/location_service.dart +++ b/lib/services/location_service.dart @@ -10,11 +10,13 @@ import "package:photos/models/local_entity_data.dart"; import "package:photos/models/location/location.dart"; import 'package:photos/models/location_tag/location_tag.dart'; import "package:photos/services/entity_service.dart"; +import "package:photos/services/remote_assets_service.dart"; import "package:shared_preferences/shared_preferences.dart"; class LocationService { late SharedPreferences prefs; final Logger _logger = Logger((LocationService).toString()); + final List _cities = []; LocationService._privateConstructor(); @@ -22,6 +24,7 @@ class LocationService { void init(SharedPreferences preferences) { prefs = preferences; + _loadCities(); } Future>> _getStoredLocationTags() async { @@ -203,6 +206,52 @@ class LocationService { rethrow; } } + + Future _loadCities() async { + try { + final data = await RemoteAssetsService.instance + .getAsset("https://assets.ente.io/world_cities.json"); + final citiesJson = json.decode(await data.readAsString()); + final List jsonData = citiesJson['data']; + final cities = + jsonData.map((jsonItem) => City.fromMap(jsonItem)).toList(); + _cities.clear(); + _cities.addAll(cities); + _logger.info("Loaded cities"); + } catch (e, s) { + _logger.severe("Failed to load cities", e, s); + } + } +} + +class City { + final String city; + final String country; + final double lat; + final double lng; + + City({ + required this.city, + required this.country, + required this.lat, + required this.lng, + }); + + factory City.fromMap(Map map) { + return City( + city: map['city'] ?? '', + country: map['country'] ?? '', + lat: map['lat']?.toDouble() ?? 0.0, + lng: map['lng']?.toDouble() ?? 0.0, + ); + } + + factory City.fromJson(String source) => City.fromMap(json.decode(source)); + + @override + String toString() { + return 'City(city: $city, country: $country, lat: $lat, lng: $lng)'; + } } class GPSData { From b16ee22dea069011cc8a944caae18a2ab793cd85 Mon Sep 17 00:00:00 2001 From: vishnukvmd Date: Tue, 26 Dec 2023 23:23:46 +0530 Subject: [PATCH 3/8] Add attribution for Simple Maps --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index bd750fdef..d6d9fba28 100644 --- a/README.md +++ b/README.md @@ -105,3 +105,7 @@ If you're interested in helping out with translation, please visit our [Crowdin Follow us on [Twitter](https://twitter.com/enteio), join [r/enteio](https://reddit.com/r/enteio) or hang out on our [Discord](https://ente.io/discord) to get regular updates, connect with other customers, and discuss your ideas. An important part of our journey is to build better software by consistently listening to community feedback. Please feel free to [share your thoughts](mailto:feedback@ente.io) with us at any time. + +## 🙇 Attributions + +- [Simple Maps](https://simplemaps.com/data/world-cities) From fe2bc0bc975b80d275582be4204f1af736a770ab Mon Sep 17 00:00:00 2001 From: vishnukvmd Date: Tue, 26 Dec 2023 23:52:47 +0530 Subject: [PATCH 4/8] Render city results for internal users --- lib/core/constants.dart | 2 ++ lib/services/location_service.dart | 9 ++++- lib/services/search_service.dart | 48 +++++++++++++++++++++++++ lib/ui/viewer/search/search_widget.dart | 12 ++++++- 4 files changed, 69 insertions(+), 2 deletions(-) diff --git a/lib/core/constants.dart b/lib/core/constants.dart index 9ae2e8b66..5c70e218f 100644 --- a/lib/core/constants.dart +++ b/lib/core/constants.dart @@ -61,6 +61,8 @@ const defaultRadiusValues = [1, 2, 10, 20, 40, 80, 200, 400, 1200]; const defaultRadiusValue = 40.0; +const defaultCityRadius = 10.0; + const galleryGridSpacing = 2.0; const searchSectionLimit = 7; diff --git a/lib/services/location_service.dart b/lib/services/location_service.dart index 516adff01..4f61e5f99 100644 --- a/lib/services/location_service.dart +++ b/lib/services/location_service.dart @@ -10,6 +10,7 @@ import "package:photos/models/local_entity_data.dart"; import "package:photos/models/location/location.dart"; import 'package:photos/models/location_tag/location_tag.dart'; import "package:photos/services/entity_service.dart"; +import "package:photos/services/feature_flag_service.dart"; import "package:photos/services/remote_assets_service.dart"; import "package:shared_preferences/shared_preferences.dart"; @@ -24,7 +25,9 @@ class LocationService { void init(SharedPreferences preferences) { prefs = preferences; - _loadCities(); + if (!FeatureFlagService.instance.isInternalUserOrDebugBuild()) { + _loadCities(); + } } Future>> _getStoredLocationTags() async { @@ -34,6 +37,10 @@ class LocationService { ); } + List getAllCities() { + return _cities; + } + Future>> getLocationTags() { return _getStoredLocationTags(); } diff --git a/lib/services/search_service.dart b/lib/services/search_service.dart index 335864840..7b831480a 100644 --- a/lib/services/search_service.dart +++ b/lib/services/search_service.dart @@ -3,6 +3,7 @@ import "dart:math"; import "package:flutter/cupertino.dart"; import "package:intl/intl.dart"; import 'package:logging/logging.dart'; +import "package:photos/core/constants.dart"; import 'package:photos/core/event_bus.dart'; import 'package:photos/data/holidays.dart'; import 'package:photos/data/months.dart'; @@ -17,6 +18,7 @@ import "package:photos/models/file/extensions/file_props.dart"; import 'package:photos/models/file/file.dart'; import 'package:photos/models/file/file_type.dart'; import "package:photos/models/local_entity_data.dart"; +import "package:photos/models/location/location.dart"; import "package:photos/models/location_tag/location_tag.dart"; import 'package:photos/models/search/album_search_result.dart'; import 'package:photos/models/search/generic_search_result.dart'; @@ -681,6 +683,52 @@ class SearchService { return searchResults; } + Future> getCityResults(String query) async { + final startTime = DateTime.now().microsecondsSinceEpoch; + final List searchResults = []; + final cities = LocationService.instance.getAllCities(); + final matchingCities = []; + final queryLower = query.toLowerCase(); + for (City city in cities) { + if (city.city.toLowerCase().startsWith(queryLower)) { + matchingCities.add(city); + } + } + final files = await getAllFiles(); + final Map> results = {}; + for (final city in matchingCities) { + final List matchingFiles = []; + final cityLocation = Location(latitude: city.lat, longitude: city.lng); + for (final file in files) { + if (file.hasLocation) { + if (LocationService.instance.isFileInsideLocationTag( + cityLocation, + file.location!, + defaultCityRadius, + )) { + matchingFiles.add(file); + } + } + } + if (matchingFiles.isNotEmpty) { + results[city] = matchingFiles; + } + } + for (final entry in results.entries) { + searchResults.add( + GenericSearchResult( + ResultType.location, + entry.key.city, + entry.value, + ), + ); + } + final endTime = DateTime.now().microsecondsSinceEpoch; + _logger + .info("Time taken " + ((endTime - startTime) / 1000).toString() + "ms"); + return searchResults; + } + Future> getAllLocationTags(int? limit) async { try { final Map, List> tagToItemsMap = {}; diff --git a/lib/ui/viewer/search/search_widget.dart b/lib/ui/viewer/search/search_widget.dart index de28c78dc..daf3d9f8b 100644 --- a/lib/ui/viewer/search/search_widget.dart +++ b/lib/ui/viewer/search/search_widget.dart @@ -278,7 +278,7 @@ class SearchWidgetState extends State { String query, ) { int resultCount = 0; - final maxResultCount = _isYearValid(query) ? 11 : 10; + final maxResultCount = _isYearValid(query) ? 12 : 11; final streamController = StreamController>(); if (query.isEmpty) { @@ -346,6 +346,16 @@ class SearchWidgetState extends State { }, ); + _searchService.getCityResults(query).then( + (results) { + streamController.sink.add(results); + resultCount++; + if (resultCount == maxResultCount) { + streamController.close(); + } + }, + ); + _searchService.getCollectionSearchResults(query).then( (collectionResults) { streamController.sink.add(collectionResults); From 4b06f58059fbc063cfdecbf698dc301f1e5fe7d2 Mon Sep 17 00:00:00 2001 From: vishnukvmd Date: Tue, 26 Dec 2023 23:58:06 +0530 Subject: [PATCH 5/8] Flip the flag --- lib/services/location_service.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/location_service.dart b/lib/services/location_service.dart index 4f61e5f99..6ca82a9b8 100644 --- a/lib/services/location_service.dart +++ b/lib/services/location_service.dart @@ -25,7 +25,7 @@ class LocationService { void init(SharedPreferences preferences) { prefs = preferences; - if (!FeatureFlagService.instance.isInternalUserOrDebugBuild()) { + if (FeatureFlagService.instance.isInternalUserOrDebugBuild()) { _loadCities(); } } From 42bbe05d61ef3f447a2b0e5d446115df8d4d4dc6 Mon Sep 17 00:00:00 2001 From: vishnukvmd Date: Wed, 27 Dec 2023 00:00:16 +0530 Subject: [PATCH 6/8] Refactor SearchWidget --- lib/ui/viewer/search/search_widget.dart | 81 +++++++------------------ 1 file changed, 21 insertions(+), 60 deletions(-) diff --git a/lib/ui/viewer/search/search_widget.dart b/lib/ui/viewer/search/search_widget.dart index daf3d9f8b..293025ee8 100644 --- a/lib/ui/viewer/search/search_widget.dart +++ b/lib/ui/viewer/search/search_widget.dart @@ -286,123 +286,84 @@ class SearchWidgetState extends State { streamController.close(); return streamController.stream; } + + void onResultsReceived(List results) { + streamController.sink.add(results); + resultCount++; + if (resultCount == maxResultCount) { + streamController.close(); + } + } + if (_isYearValid(query)) { _searchService.getYearSearchResults(query).then((yearSearchResults) { - streamController.sink.add(yearSearchResults); - resultCount++; - if (resultCount == maxResultCount) { - streamController.close(); - } + onResultsReceived(yearSearchResults); }); } _searchService.getHolidaySearchResults(context, query).then( (holidayResults) { - streamController.sink.add(holidayResults); - resultCount++; - if (resultCount == maxResultCount) { - streamController.close(); - } + onResultsReceived(holidayResults); }, ); _searchService.getFileTypeResults(context, query).then( (fileTypeSearchResults) { - streamController.sink.add(fileTypeSearchResults); - resultCount++; - if (resultCount == maxResultCount) { - streamController.close(); - } + onResultsReceived(fileTypeSearchResults); }, ); _searchService.getCaptionAndNameResults(query).then( (captionAndDisplayNameResult) { - streamController.sink.add(captionAndDisplayNameResult); - resultCount++; - if (resultCount == maxResultCount) { - streamController.close(); - } + onResultsReceived(captionAndDisplayNameResult); }, ); _searchService.getFileExtensionResults(query).then( (fileExtnResult) { - streamController.sink.add(fileExtnResult); - resultCount++; - if (resultCount == maxResultCount) { - streamController.close(); - } + onResultsReceived(fileExtnResult); }, ); _searchService.getLocationResults(query).then( (locationResult) { - streamController.sink.add(locationResult); - resultCount++; - if (resultCount == maxResultCount) { - streamController.close(); - } + onResultsReceived(locationResult); }, ); _searchService.getCityResults(query).then( (results) { - streamController.sink.add(results); - resultCount++; - if (resultCount == maxResultCount) { - streamController.close(); - } + onResultsReceived(results); }, ); _searchService.getCollectionSearchResults(query).then( (collectionResults) { - streamController.sink.add(collectionResults); - resultCount++; - if (resultCount == maxResultCount) { - streamController.close(); - } + onResultsReceived(collectionResults); }, ); _searchService.getMonthSearchResults(context, query).then( (monthResults) { - streamController.sink.add(monthResults); - resultCount++; - if (resultCount == maxResultCount) { - streamController.close(); - } + onResultsReceived(monthResults); }, ); _searchService.getDateResults(context, query).then( (possibleEvents) { - streamController.sink.add(possibleEvents); - resultCount++; - if (resultCount == maxResultCount) { - streamController.close(); - } + onResultsReceived(possibleEvents); }, ); _searchService.getMagicSearchResults(context, query).then( (magicResults) { - streamController.sink.add(magicResults); - resultCount++; - if (resultCount == maxResultCount) { - streamController.close(); - } + onResultsReceived(magicResults); }, ); _searchService.getContactSearchResults(query).then( (contactResults) { - streamController.sink.add(contactResults); - resultCount++; - if (resultCount == maxResultCount) { - streamController.close(); - } + onResultsReceived(contactResults); }, ); From ea35b94d794726d09906a31d627184012fb95c35 Mon Sep 17 00:00:00 2001 From: vishnukvmd Date: Wed, 27 Dec 2023 01:18:39 +0530 Subject: [PATCH 7/8] Extract constant --- lib/services/location_service.dart | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/services/location_service.dart b/lib/services/location_service.dart index 6ca82a9b8..2da7092e4 100644 --- a/lib/services/location_service.dart +++ b/lib/services/location_service.dart @@ -23,6 +23,8 @@ class LocationService { static final LocationService instance = LocationService._privateConstructor(); + static const kCitiesRemotePath = "https://assets.ente.io/world_cities.json"; + void init(SharedPreferences preferences) { prefs = preferences; if (FeatureFlagService.instance.isInternalUserOrDebugBuild()) { @@ -216,8 +218,8 @@ class LocationService { Future _loadCities() async { try { - final data = await RemoteAssetsService.instance - .getAsset("https://assets.ente.io/world_cities.json"); + final data = + await RemoteAssetsService.instance.getAsset(kCitiesRemotePath); final citiesJson = json.decode(await data.readAsString()); final List jsonData = citiesJson['data']; final cities = From ce276bbc07874b7328259fb3a69ca0f50fb4a030 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Wed, 27 Dec 2023 08:29:39 +0530 Subject: [PATCH 8/8] check if Semantic search has been initialized before calling ObjectBox.instance.clearTable() to avoid LateInitializationError of 'store' --- lib/core/configuration.dart | 5 ++++- lib/services/semantic_search/semantic_search_service.dart | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/core/configuration.dart b/lib/core/configuration.dart index fa96f1fd5..a77099dc3 100644 --- a/lib/core/configuration.dart +++ b/lib/core/configuration.dart @@ -27,6 +27,7 @@ import 'package:photos/services/favorites_service.dart'; import 'package:photos/services/ignored_files_service.dart'; import 'package:photos/services/memories_service.dart'; import 'package:photos/services/search_service.dart'; +import "package:photos/services/semantic_search/semantic_search_service.dart"; import 'package:photos/services/sync_service.dart'; import 'package:photos/utils/crypto_util.dart'; import 'package:photos/utils/file_uploader.dart'; @@ -157,7 +158,9 @@ class Configuration { _cachedToken = null; _secretKey = null; await FilesDB.instance.clearTable(); - await ObjectBox.instance.clearTable(); + SemanticSearchService.instance.hasInitialized + ? await ObjectBox.instance.clearTable() + : null; await CollectionsDB.instance.clearTable(); await MemoriesDB.instance.clearTable(); await PublicKeysDB.instance.clearTable(); diff --git a/lib/services/semantic_search/semantic_search_service.dart b/lib/services/semantic_search/semantic_search_service.dart index 8087d28de..23daecd61 100644 --- a/lib/services/semantic_search/semantic_search_service.dart +++ b/lib/services/semantic_search/semantic_search_service.dart @@ -46,6 +46,8 @@ class SemanticSearchService { Future>? _ongoingRequest; PendingQuery? _nextQuery; + get hasInitialized => _hasInitialized; + Future init() async { if (!LocalSettings.instance.hasEnabledMagicSearch()) { return;