diff --git a/eodag/config.py b/eodag/config.py index cc46501668..79bc296072 100644 --- a/eodag/config.py +++ b/eodag/config.py @@ -191,7 +191,7 @@ class DiscoverCollections(TypedDict, total=False): #: Mapping for collection properties which can be parsed from the result and are not collection metadata generic_collection_parsable_properties: dict[str, str] #: Mapping for collection properties which cannot be parsed from the result and are not collection metadata - generic_collection_unparsable_properties: dict[str, str] + generic_collection_unparsable_properties: dict[str, Any] #: URL to fetch data for a single collection single_collection_fetch_url: str #: Query string to be added to the fetch_url to filter for a collection diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index 314022c6b8..2902aaf1a5 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -374,6 +374,16 @@ def __init__(self, provider: str, config: PluginConfig) -> None: "single_collection_parsable_metadata" ] ) + if "metadata_mapping" in self.config.discover_collections.get( + "generic_collection_unparsable_properties", {} + ): + self.config.discover_collections[ + "generic_collection_unparsable_properties" + ]["metadata_mapping"] = mtd_cfg_as_conversion_and_querypath( + self.config.discover_collections[ + "generic_collection_unparsable_properties" + ]["metadata_mapping"] + ) # parse jsonpath on init: queryables discovery if ( @@ -597,13 +607,29 @@ def conf_update_from_collection_result( generic_collection_id = extracted_mapping.pop( "generic_collection_id" ) + unparsable_properties = deepcopy( + self.config.discover_collections.get( + "generic_collection_unparsable_properties", {} + ) + ) + if "metadata_mapping" in unparsable_properties: + # merge with default metadata mapping + merged_metadata_mapping = deepcopy( + self.config.metadata_mapping + ) + for metadata, mapping in unparsable_properties[ + "metadata_mapping" + ].items(): + merged_metadata_mapping.pop(metadata, None) + merged_metadata_mapping[metadata] = mapping + unparsable_properties[ + "metadata_mapping" + ] = merged_metadata_mapping conf_update_dict["providers_config"][ generic_collection_id ] = dict( extracted_mapping, - **self.config.discover_collections.get( - "generic_collection_unparsable_properties", {} - ), + **unparsable_properties, ) # collections_config extraction collection_properties = properties_from_json( diff --git a/tests/units/test_search_plugins.py b/tests/units/test_search_plugins.py index de385a69cd..dbbdb37b1a 100644 --- a/tests/units/test_search_plugins.py +++ b/tests/units/test_search_plugins.py @@ -462,6 +462,59 @@ def test_plugins_search_querystringsearch_discover_collections(self, mock__reque # restore configuration search_plugin.config.discover_collections["results_entry"] = results_entry + @mock.patch( + "eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True + ) + def test_plugins_search_querystringsearch_discover_collections_unparsable_metadata_mapping( + self, mock__request + ): + """QueryStringSearch.discover_collections must merge unparsable metadata_mapping with default""" + provider = "earth_search" + search_plugin = self.get_search_plugin(self.collection, provider) + + # backup and set unparsable_properties with a metadata_mapping override + discover_collections_conf = copy_deepcopy( + search_plugin.config.discover_collections + ) + from eodag.api.product.metadata_mapping import ( + mtd_cfg_as_conversion_and_querypath, + ) + + search_plugin.config.discover_collections[ + "generic_collection_unparsable_properties" + ] = { + "metadata_mapping": mtd_cfg_as_conversion_and_querypath( + {"eo:cloud_cover": "$.null"} + ) + } + + mock__request.return_value = mock.Mock() + mock__request.return_value.json.return_value = { + "collections": [ + { + "id": "foo_collection", + "title": "The FOO collection", + }, + ] + } + conf_update_dict = search_plugin.discover_collections() + providers_config = conf_update_dict["providers_config"]["foo_collection"] + + # unparsable metadata_mapping should be merged with default metadata_mapping + self.assertIn("metadata_mapping", providers_config) + merged_mapping = providers_config["metadata_mapping"] + + # should contain the override from unparsable_properties + self.assertIn("eo:cloud_cover", merged_mapping) + + # should also contain keys from the default metadata_mapping + self.assertIn("id", merged_mapping) + self.assertIn("geometry", merged_mapping) + self.assertIn("start_datetime", merged_mapping) + + # restore configuration + search_plugin.config.discover_collections = discover_collections_conf + def test_plugins_search_querystringsearch_discover_collections_paginated(self): """QueryStringSearch.discover_collections must handle pagination""" # One of the providers that has a QueryStringSearch Search plugin and discover_collections configured