feat: download cache for assets and asset eodag:download_link creation

eodag/api/core.py

@@ -1853,6 +1853,9 @@ def _do_search(
                     else:
                         eo_product.collection = guesses[0].id
 
+                # remove property "eodag:download_link" which may have been useful to create its matching asset
+                eo_product.properties.pop("eodag:download_link", None)
+
                 if eo_product.search_intersection is not None:
                     eo_product._register_downloader_from_manager(self._plugins_manager)
 

eodag/api/product/_assets.py

@@ -17,10 +17,14 @@
 # limitations under the License.
 from __future__ import annotations
 
+import logging
 import re
 from collections import UserDict
-from typing import TYPE_CHECKING, Any, Optional
+from typing import TYPE_CHECKING, Any, Optional, Union
 
+from eodag.plugins.apis.base import Api
+from eodag.plugins.authentication.base import Authentication
+from eodag.plugins.download.base import Download
 from eodag.utils.exceptions import NotAvailableError
 from eodag.utils.repr import dict_to_html_table
 
@@ -30,6 +34,9 @@
     from eodag.utils import Unpack
 
 
+logger = logging.getLogger("eodag.assets")
+
+
 class AssetsDict(UserDict):
     """A UserDict object which values are :class:`~eodag.api.product._assets.Asset`
     contained in a :class:`~eodag.api.product._product.EOProduct` resulting from a
@@ -164,15 +171,26 @@ class Asset(UserDict):
     {'href': 'http://somewhere/something'}
     """
 
+    #: EOProduct the asset belongs to
     product: EOProduct
+    #: size of the asset
     size: int
+    #: name of the asset file
     filename: Optional[str]
+    #: relative path of the asset
     rel_path: str
+    #: The path to the asset, either remote or local if downloaded
+    location: str
+    #: The remote path to the asset
+    remote_location: str
 
     def __init__(self, product: EOProduct, key: str, *args: Any, **kwargs: Any) -> None:
         self.product = product
         self.key = key
+        self.downloader: Optional[Union[Api, Download]] = None
+        self.downloader_auth: Optional[Authentication] = None
         super(Asset, self).__init__(*args, **kwargs)
+        self.location = self.remote_location = self.data.get("href", "")
 
     def as_dict(self) -> dict[str, Any]:
         """Builds a representation of Asset to enable its serialization
@@ -201,3 +219,33 @@ def _repr_html_(self):
                 </details>
                 </td></tr>
             </table>"""
+
+    def register_downloader(
+        self, downloader: Union[Api, Download], authenticator: Optional[Authentication]
+    ) -> None:
+        """Give to the asset the information needed to download itself.
+
+        :param downloader: The download method that it can use
+                          :class:`~eodag.plugins.download.base.Download` or
+                          :class:`~eodag.plugins.api.base.Api`
+        :param authenticator: The authentication method needed to perform the download
+                             :class:`~eodag.plugins.authentication.base.Authentication`
+        """
+        self.downloader = downloader
+        self.downloader_auth = authenticator
+
+        # resolve locations and properties if needed with downloader configuration
+        location_attrs = ("location", "remote_location")
+        for location_attr in location_attrs:
+            if "%(" in getattr(self, location_attr):
+                try:
+                    setattr(
+                        self,
+                        location_attr,
+                        getattr(self, location_attr) % vars(self.downloader.config),
+                    )
+                except ValueError as e:
+                    logger.debug(
+                        f"Could not resolve asset.{location_attr} ({getattr(self, location_attr)})"
+                        f" in register_downloader: {str(e)}"
+                    )

eodag/api/product/_product.py

@@ -141,7 +141,7 @@ def __init__(
             or properties.get("_collection")
         )
         self.location = self.remote_location = properties.get("eodag:download_link", "")
-        self.assets = AssetsDict(self)
+        self.assets = AssetsDict(self, properties.pop("assets"))
         self.properties = {
             key: value
             for key, value in properties.items()
@@ -336,14 +336,24 @@ def _register_downloader_from_manager(self, plugins_manager: PluginManager) -> N
                                 the download and authentication plugins.
         """
         download_plugin = plugins_manager.get_download_plugin(self)
-        if len(self.assets) > 0:
-            matching_url = next(iter(self.assets.values()))["href"]
-        elif self.properties.get("order:status") != ONLINE_STATUS:
-            matching_url = self.properties.get(
-                "eodag:order_link"
-            ) or self.properties.get("eodag:download_link")
+
+        assets_values = self.assets.values()
+        is_there_download_link = any(
+            assets_val.key == "eodag:download_link" for assets_val in assets_values
+        )
+
+        # check url of property "order:status" and asset "eodag:download_link" first
+        # since other assets can have paths not matching plugin matching_url pattern
+        if self.properties.get("order:status") != ONLINE_STATUS and (
+            (order_link := self.properties.get("eodag:order_link")) is not None
+        ):
+            matching_url = order_link
+        elif not assets_values:
+            matching_url = None
+        elif is_there_download_link:
+            matching_url = self.assets["eodag:download_link"]["href"]
         else:
-            matching_url = self.properties.get("eodag:download_link")
+            matching_url = next(iter(assets_values))["href"]
 
         try:
             auth_plugin = next(

eodag/api/product/metadata_mapping.py

@@ -165,6 +165,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
         - ``literalize_unicode``: convert a string to its raw Unicode literal form
         - ``not_available``: replace value with "Not Available"
         - ``recursive_sub_str``: recursively substitue in the structure (e.g. dict) values matching a regex
+        - ``dict_update``: update a dictionary with a list converted to a dictionary
         - ``remove_extension``: on a string that contains dots, only take the first part of the list obtained by
           splitting the string on dots
         - ``replace_str``: execute "string".replace(old, new)
@@ -189,6 +190,9 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
         - ``to_rounded_wkt``: simplify the WKT of a geometry
         - ``to_title``: Convert a string to title case
         - ``to_upper``: Convert a string to uppercase
+        - ``assets_list_to_dict``: convert a list of assets into a dictionary
+        - ``assets_list_to_dict_and_update``: convert a list of assets into a dictionary and update it with
+          another dictionary
 
     :param search_param: The string to be formatted
     :param args: (optional) Additional arguments to use in the formatting process
@@ -695,8 +699,12 @@ def convert_recursive_sub_str(
         def convert_dict_update(
             input_dict: dict[Any, Any], args: str
         ) -> dict[Any, Any]:
-            """Converts"""
-            new_items_list = ast.literal_eval(args)
+            """Updates a dictionary with a list converted to a dictionary"""
+            # if the value was not found, consider it as an empty dictionary
+            if input_dict == NOT_AVAILABLE:
+                input_dict = {}
+
+            new_items_list = ast.literal_eval(args.strip())
 
             new_items_dict = nested_pairs2dict(new_items_list)
 
@@ -706,7 +714,7 @@ def convert_dict_update(
         def convert_dict_filter(
             input_dict: dict[Any, Any], jsonpath_filter_str: str
         ) -> dict[Any, Any]:
-            """Fitlers dict items using jsonpath"""
+            """Filters dict items using jsonpath"""
 
             jsonpath_filter = string_to_jsonpath(jsonpath_filter_str, force=True)
             if isinstance(jsonpath_filter, str) or not isinstance(input_dict, dict):
@@ -1022,52 +1030,105 @@ def convert_assets_list_to_dict(
                 {"href": "foo", "title": "asset1", "name": "foo-name"},
                 {"href": "bar", "title": "path/to/asset1", "name": "bar-name"},
                 {"href": "baz", "title": "path/to/asset2", "name": "baz-name"},
-                {"href": "qux", "title": "asset3", "name": "qux-name"},
+                {"href": "qux", "title": "asset3", "name": "qux-name"}
             ] and asset_name_key == "title" => {
                 "asset1": {"href": "foo", "title": "asset1", "name": "foo-name"},
                 "path/to/asset1": {"href": "bar", "title": "path/to/asset1", "name": "bar-name"},
                 "asset2": {"href": "baz", "title": "path/to/asset2", "name": "baz-name"},
-                "asset3": {"href": "qux", "title": "asset3", "name": "qux-name"},
+                "asset3": {"href": "qux", "title": "asset3", "name": "qux-name"}
             }
             assets_list == [
                 {"href": "foo", "title": "foo-title", "name": "asset1"},
                 {"href": "bar", "title": "bar-title", "name": "path/to/asset1"},
                 {"href": "baz", "title": "baz-title", "name": "path/to/asset2"},
-                {"href": "qux", "title": "qux-title", "name": "asset3"},
+                {"href": "qux", "title": "qux-title", "name": "asset3"}
             ] and asset_name_key == "name" => {
                 "asset1": {"href": "foo", "title": "foo-title", "name": "asset1"},
                 "path/to/asset1": {"href": "bar", "title": "bar-title", "name": "path/to/asset1"},
                 "asset2": {"href": "baz", "title": "baz-title", "name": "path/to/asset2"},
-                "asset3": {"href": "qux", "title": "qux-title", "name": "asset3"},
+                "asset3": {"href": "qux", "title": "qux-title", "name": "asset3"}
             }
             """
             asset_names: list[str] = []
             assets_dict: dict[str, dict[str, str]] = {}
 
+            # create dictionary with assets full name
             for asset in assets_list:
                 asset_name = asset[asset_name_key]
                 asset_names.append(asset_name)
                 assets_dict[asset_name] = asset
 
-            # we only keep the equivalent of the path basename in the case where the
-            # asset name has a path pattern and this basename is only found once
+            # when an asset name has a path pattern, we update its value with its basename if
+            # this basename is found for the first time. Otherwise, we keep it as a full path
             immutable_asset_indexes: list[int] = []
             for i, asset_name in enumerate(asset_names):
                 if i in immutable_asset_indexes:
                     continue
-                change_asset_name = True
+                update_asset_name = True
                 asset_basename = asset_name.split("/")[-1]
                 j = i + 1
-                while change_asset_name and j < len(asset_names):
+                while update_asset_name and j < len(asset_names):
                     asset_tmp_basename = asset_names[j].split("/")[-1]
                     if asset_basename == asset_tmp_basename:
-                        change_asset_name = False
+                        update_asset_name = False
                         immutable_asset_indexes.extend([i, j])
                     j += 1
-                if change_asset_name:
+                if update_asset_name:
                     assets_dict[asset_basename] = assets_dict.pop(asset_name)
             return assets_dict
 
+        @staticmethod
+        def convert_assets_list_to_dict_and_update(
+            assets_list: list[dict[str, str]], args: str, asset_name_key: str = "title"
+        ) -> dict[str, dict[str, str]]:
+            """Combine two MetadataFormatter class methods in the following order:
+            - convert_assets_list_to_dict()
+            - convert_dict_update()
+
+            assets_list == [
+                {"href": "foo", "title": "asset1", "name": "foo-name"},
+                {"href": "bar", "title": "path/to/asset1", "name": "bar-name"},
+                {"href": "baz", "title": "path/to/asset2", "name": "baz-name"},
+                {"href": "qux", "title": "asset3", "name": "qux-name"},
+            ], asset_name_key == "title" and args == '[["eodag:download_link",[
+                ["title","Full product download"],
+                ["href","https://downloadlink.foo"],
+                ["roles",["data"]],
+                ["type","application/zip"]
+            ]]]' => {
+                "asset1": {"href": "foo", "title": "asset1", "name": "foo-name"},
+                "path/to/asset1": {"href": "bar", "title": "path/to/asset1", "name": "bar-name"},
+                "asset2": {"href": "baz", "title": "path/to/asset2", "name": "baz-name"},
+                "asset3": {"href": "qux", "title": "asset3", "name": "qux-name"},
+                "eodag:download_link": {"title": "Full product download", "href": "https://downloadlink.foo",
+                    "roles": ["data"], "type": "application/zip"
+                }
+            }
+            assets_list == [
+                {"href": "foo", "title": "foo-title", "name": "asset1"},
+                {"href": "bar", "title": "bar-title", "name": "path/to/asset1"},
+                {"href": "baz", "title": "baz-title", "name": "path/to/asset2"},
+                {"href": "qux", "title": "qux-title", "name": "asset3"},
+            ], asset_name_key == "name" and args == '[["eodag:download_link",[
+                ["title","Full product download"],
+                ["href","https://downloadlink.foo"],
+                ["roles",["data"]],
+                ["type","application/zip"]
+            ]]]' => {
+                "asset1": {"href": "foo", "title": "foo-title", "name": "asset1"},
+                "path/to/asset1": {"href": "bar", "title": "bar-title", "name": "path/to/asset1"},
+                "asset2": {"href": "baz", "title": "baz-title", "name": "path/to/asset2"},
+                "asset3": {"href": "qux", "title": "qux-title", "name": "asset3"},
+                "eodag:download_link": {"title": "Full product download", "href": "https://downloadlink.foo",
+                    "roles": ["data"], "type": "application/zip"
+                }
+            }
+            """
+            assets_dict = MetadataFormatter.convert_assets_list_to_dict(
+                assets_list, asset_name_key
+            )
+            return MetadataFormatter.convert_dict_update(assets_dict, args)
+
         @staticmethod
         def convert_wekeo_to_cop_collection(val: str, prefix: str) -> str:
             """Converts the name of a collection from the WEkEO format to the Copernicus format."""

eodag/config.py

@@ -430,9 +430,6 @@ class MetadataPreMapping(TypedDict, total=False):
     flatten_top_dirs: bool
     #: :class:`~eodag.plugins.download.base.Download` Level in extracted path tree where to find data
     archive_depth: int
-    #: :class:`~eodag.plugins.download.base.Download` Whether ignore assets and download using ``eodag:download_link``
-    #: or not
-    ignore_assets: bool
     #: :class:`~eodag.plugins.download.base.Download` Collection specific configuration
     products: dict[str, dict[str, Any]]
     #: :class:`~eodag.plugins.download.base.Download` Number of maximum workers allowed for parallel downloads

eodag/plugins/apis/ecmwf.py

@@ -199,17 +199,19 @@ def download(
         )
 
         # Prepare download
-        fs_path, record_filename = self._prepare_download(
+        fs_path, record_filenames = self._prepare_download(
             product,
             progress_callback=progress_callback,
             **kwargs,
         )
 
-        if not fs_path or not record_filename:
+        if not fs_path or not record_filenames:
             if fs_path:
                 product.location = path_to_uri(fs_path)
             return fs_path
 
+        record_filename = record_filenames["eodag:download_link"]
+
         new_fs_path = os.path.join(
             os.path.dirname(fs_path), sanitize(product.properties["title"])
         )