fix(plugins): merged generic_collection_unparsable_properties on fetched collections

eodag/config.py

@@ -191,7 +191,7 @@ class DiscoverCollections(TypedDict, total=False):
         #: Mapping for collection properties which can be parsed from the result and are not collection metadata
         generic_collection_parsable_properties: dict[str, str]
         #: Mapping for collection properties which cannot be parsed from the result and are not collection metadata
-        generic_collection_unparsable_properties: dict[str, str]
+        generic_collection_unparsable_properties: dict[str, Any]
         #: URL to fetch data for a single collection
         single_collection_fetch_url: str
         #: Query string to be added to the fetch_url to filter for a collection

eodag/plugins/search/qssearch.py

@@ -374,6 +374,16 @@ def __init__(self, provider: str, config: PluginConfig) -> None:
                         "single_collection_parsable_metadata"
                     ]
                 )
+            if "metadata_mapping" in self.config.discover_collections.get(
+                "generic_collection_unparsable_properties", {}
+            ):
+                self.config.discover_collections[
+                    "generic_collection_unparsable_properties"
+                ]["metadata_mapping"] = mtd_cfg_as_conversion_and_querypath(
+                    self.config.discover_collections[
+                        "generic_collection_unparsable_properties"
+                    ]["metadata_mapping"]
+                )
 
         # parse jsonpath on init: queryables discovery
         if (
@@ -597,13 +607,29 @@ def conf_update_from_collection_result(
                         generic_collection_id = extracted_mapping.pop(
                             "generic_collection_id"
                         )
+                        unparsable_properties = deepcopy(
+                            self.config.discover_collections.get(
+                                "generic_collection_unparsable_properties", {}
+                            )
+                        )
+                        if "metadata_mapping" in unparsable_properties:
+                            # merge with default metadata mapping
+                            merged_metadata_mapping = deepcopy(
+                                self.config.metadata_mapping
+                            )
+                            for metadata, mapping in unparsable_properties[
+                                "metadata_mapping"
+                            ].items():
+                                merged_metadata_mapping.pop(metadata, None)
+                                merged_metadata_mapping[metadata] = mapping
+                            unparsable_properties[
+                                "metadata_mapping"
+                            ] = merged_metadata_mapping
                         conf_update_dict["providers_config"][
                             generic_collection_id
                         ] = dict(
                             extracted_mapping,
-                            **self.config.discover_collections.get(
-                                "generic_collection_unparsable_properties", {}
-                            ),
+                            **unparsable_properties,
                         )
                         # collections_config extraction
                         collection_properties = properties_from_json(

tests/units/test_search_plugins.py

@@ -462,6 +462,59 @@ def test_plugins_search_querystringsearch_discover_collections(self, mock__reque
         # restore configuration
         search_plugin.config.discover_collections["results_entry"] = results_entry
 
+    @mock.patch(
+        "eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True
+    )
+    def test_plugins_search_querystringsearch_discover_collections_unparsable_metadata_mapping(
+        self, mock__request
+    ):
+        """QueryStringSearch.discover_collections must merge unparsable metadata_mapping with default"""
+        provider = "earth_search"
+        search_plugin = self.get_search_plugin(self.collection, provider)
+
+        # backup and set unparsable_properties with a metadata_mapping override
+        discover_collections_conf = copy_deepcopy(
+            search_plugin.config.discover_collections
+        )
+        from eodag.api.product.metadata_mapping import (
+            mtd_cfg_as_conversion_and_querypath,
+        )
+
+        search_plugin.config.discover_collections[
+            "generic_collection_unparsable_properties"
+        ] = {
+            "metadata_mapping": mtd_cfg_as_conversion_and_querypath(
+                {"eo:cloud_cover": "$.null"}
+            )
+        }
+
+        mock__request.return_value = mock.Mock()
+        mock__request.return_value.json.return_value = {
+            "collections": [
+                {
+                    "id": "foo_collection",
+                    "title": "The FOO collection",
+                },
+            ]
+        }
+        conf_update_dict = search_plugin.discover_collections()
+        providers_config = conf_update_dict["providers_config"]["foo_collection"]
+
+        # unparsable metadata_mapping should be merged with default metadata_mapping
+        self.assertIn("metadata_mapping", providers_config)
+        merged_mapping = providers_config["metadata_mapping"]
+
+        # should contain the override from unparsable_properties
+        self.assertIn("eo:cloud_cover", merged_mapping)
+
+        # should also contain keys from the default metadata_mapping
+        self.assertIn("id", merged_mapping)
+        self.assertIn("geometry", merged_mapping)
+        self.assertIn("start_datetime", merged_mapping)
+
+        # restore configuration
+        search_plugin.config.discover_collections = discover_collections_conf
+
     def test_plugins_search_querystringsearch_discover_collections_paginated(self):
         """QueryStringSearch.discover_collections must handle pagination"""
         # One of the providers that has a QueryStringSearch Search plugin and discover_collections configured