diff --git a/eodag/__init__.py b/eodag/__init__.py index 366f3ecc7d..c8ac1b47db 100644 --- a/eodag/__init__.py +++ b/eodag/__init__.py @@ -22,12 +22,8 @@ # depends on numpy and ship with a pre-built version of numpy that is older than 1.15.1 (where the warning is silenced # exactly as below) """EODAG package""" -from importlib.metadata import PackageNotFoundError, version -from .api.core import EODataAccessGateway -from .api.product import EOProduct -from .api.search_result import SearchResult -from .utils.logging import setup_logging +from importlib.metadata import PackageNotFoundError, version try: __version__ = version(__name__) @@ -41,3 +37,27 @@ "SearchResult", "setup_logging", ] + +# Lazy imports (PEP 562) — avoid loading heavy dependencies on ``import eodag`` +_LAZY_IMPORTS: dict[str, tuple[str, str]] = { + "EODataAccessGateway": (".api.core", "EODataAccessGateway"), + "EOProduct": (".api.product", "EOProduct"), + "SearchResult": (".api.search_result", "SearchResult"), + "setup_logging": (".utils.logging", "setup_logging"), +} + + +def __getattr__(name: str): + if name in _LAZY_IMPORTS: + from importlib import import_module + + module_path, attr_name = _LAZY_IMPORTS[name] + module = import_module(module_path, __name__) + value = getattr(module, attr_name) + globals()[name] = value + return value + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__(): + return __all__ + ["__version__"] diff --git a/eodag/api/core.py b/eodag/api/core.py index 67cbce504d..2189dc7207 100644 --- a/eodag/api/core.py +++ b/eodag/api/core.py @@ -85,7 +85,6 @@ ValidationError, ) from eodag.utils.free_text_search import compile_free_text_query -from eodag.utils.stac_reader import fetch_stac_items if TYPE_CHECKING: from concurrent.futures import ThreadPoolExecutor @@ -2075,6 +2074,8 @@ def serialize( return filename collections = set(p.collection for p in search_result) for collection in collections: + if collection is None: + continue collection_obj = search_result._dag.collections_config.get( collection, Collection(id=collection) ) @@ -2393,6 +2394,8 @@ def import_stac_items(self, items_urls: list[str]) -> SearchResult: :param items_urls: A list of STAC items URLs to import :returns: A SearchResult containing the imported STAC items """ + from eodag.utils.stac_reader import fetch_stac_items + json_items = [] for item_url in items_urls: json_items.extend(fetch_stac_items(item_url)) diff --git a/eodag/api/product/__init__.py b/eodag/api/product/__init__.py index a682bdc500..e58a569de3 100644 --- a/eodag/api/product/__init__.py +++ b/eodag/api/product/__init__.py @@ -18,21 +18,57 @@ # """EODAG product package""" +from __future__ import annotations + from typing import TYPE_CHECKING, Any, Optional if TYPE_CHECKING: + from eodag.api.product._assets import Asset, AssetsDict + from eodag.api.product._product import EOProduct from eodag.plugins.manager import PluginManager -try: - # import from eodag-cube if installed - from eodag_cube.api.product import ( # pyright: ignore[reportMissingImports] - Asset, - AssetsDict, - EOProduct, - ) -except ImportError: - from ._assets import Asset, AssetsDict # type: ignore[assignment] - from ._product import EOProduct # type: ignore[assignment] +# exportable content +__all__ = ["Asset", "AssetsDict", "EOProduct", "unregistered_product_from_item"] + +# Lazy imports (PEP 562) — defer heavy _product / _assets loading +_LAZY_IMPORTS: dict[str, tuple[str, str]] = { + "EOProduct": ("._product", "EOProduct"), + "Asset": ("._assets", "Asset"), + "AssetsDict": ("._assets", "AssetsDict"), +} + + +def _resolve(name: str): + """Resolve a lazy import, trying eodag-cube first for EOProduct/Asset/AssetsDict.""" + module_path, attr_name = _LAZY_IMPORTS[name] + from importlib import import_module + + # Try eodag_cube first (provides extended classes with xarray/rasterio support). + # Users just need ``pip install eodag_cube`` — no other change required. + try: + cube_module = import_module("eodag_cube.api.product") + value = getattr(cube_module, attr_name, None) + if value is not None: + globals()[name] = value + return value + except ImportError: + pass + + # Fallback to eodag's own modules + module = import_module(module_path, __name__) + value = getattr(module, attr_name) + globals()[name] = value + return value + + +def __getattr__(name: str): + if name in _LAZY_IMPORTS: + return _resolve(name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__(): + return __all__ def unregistered_product_from_item( diff --git a/eodag/api/product/_assets.py b/eodag/api/product/_assets.py index b07b58a869..c470e3d5c5 100644 --- a/eodag/api/product/_assets.py +++ b/eodag/api/product/_assets.py @@ -188,7 +188,8 @@ def download(self, **kwargs: Unpack[DownloadConf]) -> str: :param kwargs: (optional) Additional named-arguments passed to `plugin.download()` :returns: The absolute path to the downloaded product on the local filesystem """ - return self.product.download(asset=self.key, **kwargs) + kwargs["asset"] = self.key + return self.product.download(**kwargs) def _repr_html_(self): thead = f""" diff --git a/eodag/api/product/_product.py b/eodag/api/product/_product.py index 32a46ead4e..2c842460e6 100644 --- a/eodag/api/product/_product.py +++ b/eodag/api/product/_product.py @@ -26,49 +26,25 @@ from typing import TYPE_CHECKING, Any, Iterable, Optional, Union, cast import orjson -import requests -from requests import RequestException -from requests.auth import AuthBase -from shapely import geometry -from shapely.errors import ShapelyError - -from eodag.types.queryables import CommonStacMetadata -from eodag.types.stac_metadata import create_stac_metadata_model - -try: - # import from eodag-cube if installed - from eodag_cube.api.product import ( # pyright: ignore[reportMissingImports] - AssetsDict, - ) -except ImportError: - from eodag.api.product._assets import AssetsDict -from eodag.api.product.drivers import DRIVERS -from eodag.api.product.drivers.generic import GenericDriver -from eodag.api.product.metadata_mapping import ( - DEFAULT_GEOMETRY, - NOT_AVAILABLE, - NOT_MAPPED, - ONLINE_STATUS, -) from eodag.utils import ( DEFAULT_DOWNLOAD_TIMEOUT, DEFAULT_DOWNLOAD_WAIT, - DEFAULT_SHAPELY_GEOMETRY, DEFAULT_STREAM_REQUESTS_TIMEOUT, STAC_VERSION, USER_AGENT, ProgressCallback, format_string, - get_geometry_from_various, ) from eodag.utils.exceptions import DownloadError, MisconfiguredError, ValidationError -from eodag.utils.repr import dict_to_html_table if TYPE_CHECKING: + import requests from concurrent.futures import ThreadPoolExecutor + from requests.auth import AuthBase from shapely.geometry.base import BaseGeometry + from eodag.api.product._assets import AssetsDict from eodag.api.product.drivers.base import DatasetDriver from eodag.plugins.apis.base import Api from eodag.plugins.authentication.base import Authentication @@ -78,6 +54,17 @@ from eodag.utils import Unpack +def _get_assets_dict_cls(): + """Get AssetsDict class, preferring eodag-cube if installed.""" + try: + from eodag_cube.api.product import ( # pyright: ignore[reportMissingImports] + AssetsDict, + ) + except ImportError: + from eodag.api.product._assets import AssetsDict + return AssetsDict + + logger = logging.getLogger("eodag.product") @@ -134,6 +121,15 @@ class EOProduct: def __init__( self, provider: str, properties: dict[str, Any], **kwargs: Any ) -> None: + from shapely.errors import ShapelyError + + from eodag.api.product.metadata_mapping import ( + DEFAULT_GEOMETRY, + NOT_AVAILABLE, + NOT_MAPPED, + ) + from eodag.utils import DEFAULT_SHAPELY_GEOMETRY, get_geometry_from_various + self.provider = provider self.collection = ( kwargs.get("collection") @@ -141,6 +137,7 @@ def __init__( or properties.get("_collection") ) self.location = self.remote_location = properties.get("eodag:download_link", "") + AssetsDict = _get_assets_dict_cls() self.assets = AssetsDict(self) self.properties = { key: value @@ -215,6 +212,9 @@ def as_dict(self) -> dict[str, Any]: :returns: The representation of a :class:`~eodag.api.product._product.EOProduct` as a Python dict """ + from eodag.types.queryables import CommonStacMetadata + from eodag.types.stac_metadata import create_stac_metadata_model + search_intersection = None if self.search_intersection is not None: search_intersection = orjson.loads( @@ -296,6 +296,8 @@ def from_geojson(cls, feature: dict[str, Any]) -> EOProduct: :raises: :class:`~eodag.utils.exceptions.ValidationError` """ try: + from shapely import geometry + collection = feature.get("collection") properties = feature["properties"] properties["geometry"] = feature["geometry"] @@ -309,6 +311,7 @@ def from_geojson(cls, feature: dict[str, Any]) -> EOProduct: ) from e obj = cls(provider, properties, collection=collection) obj.search_intersection = geometry.shape(search_intersection) + AssetsDict = _get_assets_dict_cls() obj.assets = AssetsDict(obj, feature.get("assets", {})) return obj @@ -335,6 +338,8 @@ def _register_downloader_from_manager(self, plugins_manager: PluginManager) -> N :param plugins_manager: The plugins manager instance to use for retrieving the download and authentication plugins. """ + from eodag.api.product.metadata_mapping import ONLINE_STATUS + download_plugin = plugins_manager.get_download_plugin(self) if len(self.assets) > 0: matching_url = next(iter(self.assets.values()))["href"] @@ -527,6 +532,8 @@ def _download_quicklook( HTTP request if the resource requires authentication. :raises HTTPError: If the HTTP request to the quicklook URL fails. """ + import requests + with requests.get( self.properties["eodag:quicklook"], stream=True, @@ -639,6 +646,9 @@ def format_quicklook_address() -> None: if self.downloader_auth is not None else None ) + from requests import RequestException + from requests.auth import AuthBase # noqa: F811 + if not isinstance(auth, AuthBase): auth = None # Read the ssl_verify parameter used on the provider config @@ -675,8 +685,10 @@ def format_quicklook_address() -> None: def get_driver(self) -> DatasetDriver: """Get the most appropriate driver""" - for driver_conf in DRIVERS: + from eodag.api.product.drivers import DRIVERS + from eodag.api.product.drivers.generic import GenericDriver + for driver_conf in DRIVERS: # Select a driver if all criterias match match = True for criteria in driver_conf["criteria"]: @@ -689,6 +701,8 @@ def get_driver(self) -> DatasetDriver: return GenericDriver() def _repr_html_(self): + from eodag.utils.repr import dict_to_html_table + thumbnail = self.properties.get("eodag:thumbnail") or self.properties.get( "eodag:quicklook" ) diff --git a/eodag/cli.py b/eodag/cli.py index d63ad62a0e..aab6255826 100755 --- a/eodag/cli.py +++ b/eodag/cli.py @@ -48,13 +48,8 @@ from urllib.parse import parse_qs import click -from concurrent.futures import ThreadPoolExecutor -from eodag.api.collection import CollectionsList -from eodag.api.core import EODataAccessGateway, SearchResult from eodag.utils import DEFAULT_LIMIT, DEFAULT_PAGE -from eodag.utils.exceptions import NoMatchingCollection, UnsupportedProvider -from eodag.utils.logging import setup_logging if TYPE_CHECKING: from click import Context @@ -310,6 +305,9 @@ def search_crunch(ctx: Context, **kwargs: Any) -> None: click.echo(search_crunch.get_help(ctx)) sys.exit(-1) + from eodag.api.core import EODataAccessGateway + from eodag.utils.logging import setup_logging + setup_logging(verbose=ctx.obj["verbosity"]) if kwargs["box"] != (None,) * 4: @@ -442,6 +440,11 @@ def search_crunch(ctx: Context, **kwargs: Any) -> None: @click.pass_context def list_col(ctx: Context, **kwargs: Any) -> None: """Print the list of supported collections""" + from eodag.api.collection import CollectionsList + from eodag.api.core import EODataAccessGateway + from eodag.utils.exceptions import NoMatchingCollection, UnsupportedProvider + from eodag.utils.logging import setup_logging + setup_logging(verbose=ctx.obj["verbosity"]) dag = EODataAccessGateway() provider = kwargs.pop("provider") @@ -517,6 +520,9 @@ def list_col(ctx: Context, **kwargs: Any) -> None: @click.pass_context def discover_col(ctx: Context, **kwargs: Any) -> None: """Fetch external collections configuration and save result""" + from eodag.api.core import EODataAccessGateway + from eodag.utils.logging import setup_logging + setup_logging(verbose=ctx.obj["verbosity"]) dag = EODataAccessGateway() provider = kwargs.pop("provider") @@ -588,6 +594,13 @@ def download(ctx: Context, **kwargs: Any) -> None: click.echo("Nothing to do (no search results file or stac item provided)") click.echo(download.get_help(ctx)) sys.exit(1) + + from concurrent.futures import ThreadPoolExecutor + + from eodag.api.core import EODataAccessGateway + from eodag.api.search_result import SearchResult + from eodag.utils.logging import setup_logging + setup_logging(verbose=ctx.obj["verbosity"]) conf_file = kwargs.pop("conf") if conf_file: diff --git a/eodag/config.py b/eodag/config.py index cc46501668..5806aafe5c 100644 --- a/eodag/config.py +++ b/eodag/config.py @@ -22,8 +22,6 @@ from importlib.resources import files as res_files from typing import TYPE_CHECKING, Annotated, Any, Literal, Optional, Union -import orjson -import requests import yaml import yaml.parser from annotated_types import Gt @@ -758,6 +756,8 @@ def get_ext_collections_conf( if conf_uri.lower().startswith("http"): # read from remote try: + import requests + response = requests.get( conf_uri, headers=USER_AGENT, timeout=HTTP_REQ_TIMEOUT ) @@ -774,6 +774,8 @@ def get_ext_collections_conf( # read from local try: + import orjson + with open(conf_uri, "rb") as f: return orjson.loads(f.read()) except (orjson.JSONDecodeError, FileNotFoundError) as e: diff --git a/eodag/plugins/apis/usgs.py b/eodag/plugins/apis/usgs.py index 8816897e11..9a0db937e8 100644 --- a/eodag/plugins/apis/usgs.py +++ b/eodag/plugins/apis/usgs.py @@ -325,7 +325,8 @@ def download( output_extension = cast( str, self.config.products.get( # type: ignore - product.collection, self.config.products[GENERIC_COLLECTION] # type: ignore + product.collection or "", + self.config.products[GENERIC_COLLECTION], # type: ignore ).get("output_extension", ".tar.gz"), ) kwargs["output_extension"] = kwargs.get("output_extension", output_extension) @@ -346,7 +347,7 @@ def download( raise NotAvailableError( f"No USGS products found for {product.properties['id']}" ) - usgs_dataset = self.config.products.get(product.collection, {}).get( + usgs_dataset = self.config.products.get(product.collection or "", {}).get( "_collection", GENERIC_COLLECTION ) download_request_results = api.download_request( diff --git a/eodag/plugins/download/aws.py b/eodag/plugins/download/aws.py index d66aa2e14e..2bca150fe0 100644 --- a/eodag/plugins/download/aws.py +++ b/eodag/plugins/download/aws.py @@ -602,7 +602,7 @@ def _get_bucket_names_and_prefixes( rf"No asset key matching re.fullmatch(r'{asset_filter}') was found in {product}" ) else: - assets_values = product.assets.values() + assets_values = list(product.assets.values()) bucket_names_and_prefixes = [] for complementary_url in assets_values: @@ -808,9 +808,8 @@ def _stream_download_dict( re.sub(rf"^{common_path}/?", "", rel_path), ) - data_type = assets_by_path.get(f"{obj.bucket_name}/{obj.key}", {}).get( - "type" - ) + asset_match = assets_by_path.get(f"{obj.bucket_name}/{obj.key}") + data_type = asset_match.get("type") if asset_match else None file_info = S3FileInfo( key=obj.key, diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py index d49f6a8615..6b53280437 100644 --- a/eodag/plugins/download/http.py +++ b/eodag/plugins/download/http.py @@ -800,7 +800,7 @@ def _stream_download_dict( max_workers=getattr(self.config, "max_workers", None) ) try: - assets_values = product.assets.get_values(kwargs.get("asset")) + assets_values = product.assets.get_values(kwargs.get("asset") or "") with executor: assets_stream_list = self._stream_download_assets( product, @@ -865,7 +865,7 @@ def _stream_download_dict( return StreamResponse( content=chain(iter([first_chunk]), chunk_iterator), - headers=product.headers, + headers=getattr(product, "headers", {}), filename=getattr(product, "filename", None), size=getattr(product, "size", None), ) @@ -1060,17 +1060,17 @@ def _stream_download( self._process_exception(None, product, ordered_message) stream_size = self._check_stream_size(product) or None - product.headers = product._stream.headers + product.headers = product._stream.headers # type: ignore[attr-defined] filename = self._check_product_filename(product) - content_type = product.headers.get("Content-Type") + content_type = product.headers.get("Content-Type") # type: ignore[attr-defined] guessed_content_type = ( guess_file_type(filename) if filename and not content_type else None ) if guessed_content_type is not None: - product.headers["Content-Type"] = guessed_content_type + product.headers["Content-Type"] = guessed_content_type # type: ignore[attr-defined] progress_callback.reset(total=stream_size) - product.size = stream_size + product.size = stream_size # type: ignore[attr-defined] product.filename = filename return product._stream.iter_content(chunk_size=64 * 1024) @@ -1250,7 +1250,7 @@ def _download_assets( if not assets_urls: raise NotAvailableError("No assets available for %s" % product) - assets_values = product.assets.get_values(kwargs.get("asset")) + assets_values = product.assets.get_values(kwargs.get("asset") or "") assets_stream_list = self._stream_download_assets( product, executor, auth, progress_callback, assets_values, **kwargs diff --git a/eodag/plugins/search/build_search_result.py b/eodag/plugins/search/build_search_result.py index 489a02d0a0..e3401ae261 100644 --- a/eodag/plugins/search/build_search_result.py +++ b/eodag/plugins/search/build_search_result.py @@ -339,7 +339,7 @@ def ecmwf_format(v: str, alias: bool = True) -> str: def ecmwf_temporal_to_eodag( - params: dict[str, Any] + params: dict[str, Any], ) -> tuple[Optional[str], Optional[str]]: """ Converts ECMWF temporal parameters to EODAG temporal parameters. @@ -1282,9 +1282,9 @@ def normalize_results( ) # backup original register_downloader to register_downloader_only - product.register_downloader_only = product.register_downloader + product.register_downloader_only = product.register_downloader # type: ignore[attr-defined] # patched register_downloader that will also update properties - product.register_downloader = MethodType(patched_register_downloader, product) + product.register_downloader = MethodType(patched_register_downloader, product) # type: ignore[method-assign] return [product] @@ -1314,6 +1314,8 @@ def _check_id(product: EOProduct) -> EOProduct: if "ORDERABLE" in product_id: return product + if product.downloader is None: + return product on_response_mm = getattr(product.downloader.config, "order_on_response", {}).get( "metadata_mapping", {} ) diff --git a/eodag/plugins/search/creodias_s3.py b/eodag/plugins/search/creodias_s3.py index 9b07421d5d..89e7092dac 100644 --- a/eodag/plugins/search/creodias_s3.py +++ b/eodag/plugins/search/creodias_s3.py @@ -84,9 +84,9 @@ def normalize_results( for product in products: # backup original register_downloader to register_downloader_only - product.register_downloader_only = product.register_downloader + product.register_downloader_only = product.register_downloader # type: ignore[attr-defined] # patched register_downloader that will also update assets - product.register_downloader = MethodType( + product.register_downloader = MethodType( # type: ignore[method-assign] patched_register_downloader, product ) diff --git a/eodag/plugins/search/csw.py b/eodag/plugins/search/csw.py index 6f25f2f0d4..442b560078 100644 --- a/eodag/plugins/search/csw.py +++ b/eodag/plugins/search/csw.py @@ -227,14 +227,13 @@ def __build_product(self, rec: Any, collection: str, **kwargs: Any) -> EOProduct else: properties["geometry"] = wkt.loads(properties["geometry"]) return EOProduct( - collection, - self.provider, + provider=self.provider, # TODO: EOProduct has no more *args in its __init__ (search_args attribute removed) # Not sure why download_url was here in the first place, needs to be updated, # possibly by having instead 'eodag:download_link' in the properties # download_url, - properties, - searched_bbox=kwargs.get("footprints"), + properties=properties, + collection=collection, ) def __convert_query_params( diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index 314022c6b8..47603fc842 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -387,7 +387,6 @@ def __init__(self, provider: str, config: PluginConfig) -> None: # parse jsonpath on init: collection specific metadata-mapping for collection in self.config.products.keys(): - collection_metadata_mapping = {} # collection specific metadata-mapping if any( @@ -1134,7 +1133,6 @@ def do_search( ): del prep.total_items_nb if limit is not None and len(results) == limit: - raw_search_results = self._build_raw_search_results( results, resp_as_json, kwargs, limit, prep ) @@ -1800,14 +1798,13 @@ def normalize_results( if "eodag:download_link" in product.properties: decoded_link = unquote(product.properties["eodag:download_link"]) if decoded_link[0] == "{": # not a url but a dict - default_values = deepcopy( - self.config.products.get(product.collection, {}) - ) + collection = product.collection or "" + default_values = deepcopy(self.config.products.get(collection, {})) default_values.pop("metadata_mapping", None) searched_values = orjson.loads(decoded_link) _dc_qs = orjson.dumps( format_query_params( - product.collection, + collection, self.config, {**default_values, **searched_values}, ) diff --git a/eodag/plugins/search/stac_list_assets.py b/eodag/plugins/search/stac_list_assets.py index 05944f2298..f8a98dfeb1 100644 --- a/eodag/plugins/search/stac_list_assets.py +++ b/eodag/plugins/search/stac_list_assets.py @@ -76,9 +76,9 @@ def normalize_results( for product in products: # backup original register_downloader to register_downloader_only - product.register_downloader_only = product.register_downloader + product.register_downloader_only = product.register_downloader # type: ignore[attr-defined] # patched register_downloader that will also update assets - product.register_downloader = MethodType( + product.register_downloader = MethodType( # type: ignore[method-assign] patched_register_downloader, product ) diff --git a/eodag/utils/__init__.py b/eodag/utils/__init__.py index 5a50322128..9c4e85d6f5 100644 --- a/eodag/utils/__init__.py +++ b/eodag/utils/__init__.py @@ -63,31 +63,21 @@ from urllib.parse import urlparse, urlsplit from urllib.request import url2pathname -from pydantic import ValidationError as PydanticValidationError - if sys.version_info >= (3, 12): from typing import Unpack # type: ignore # noqa else: from typing_extensions import Unpack # noqa -import click import orjson -import shapefile -import shapely.wkt -import yaml -from jsonpath_ng import jsonpath -from jsonpath_ng.ext import parse -from jsonpath_ng.jsonpath import Child, Fields, Index, Root, Slice -from requests import HTTPError, Response -from shapely.geometry import Polygon, box, shape -from shapely.geometry.base import GEOMETRY_TYPES, BaseGeometry from tqdm.auto import tqdm from eodag.utils import logging as eodag_logging from eodag.utils.exceptions import MisconfiguredError if TYPE_CHECKING: - from jsonpath_ng import JSONPath + from jsonpath_ng import JSONPath, jsonpath + from pydantic import ValidationError as PydanticValidationError + from shapely.geometry.base import BaseGeometry from eodag.api.product._product import EOProduct @@ -166,7 +156,7 @@ #: default collections start date DEFAULT_MISSION_START_DATE = "2015-01-01T00:00:00.000Z" #: default geometry / whole world bounding box -DEFAULT_SHAPELY_GEOMETRY = box(-180, -90, 180, 90) +# DEFAULT_SHAPELY_GEOMETRY is lazily computed via __getattr__ to defer shapely import #: Online status value for ``order:status`` property ONLINE_STATUS = "succeeded" @@ -189,6 +179,25 @@ mimetypes.add_type("image/jp2", ".jp2") +# --------------------------------------------------------------------------- +# PEP 562 lazy module attributes +# --------------------------------------------------------------------------- + +_DEFAULT_SHAPELY_GEOMETRY = None + + +def __getattr__(name: str): + if name == "DEFAULT_SHAPELY_GEOMETRY": + global _DEFAULT_SHAPELY_GEOMETRY + if _DEFAULT_SHAPELY_GEOMETRY is None: + from shapely.geometry import box + + _DEFAULT_SHAPELY_GEOMETRY = box(-180, -90, 180, 90) + globals()["DEFAULT_SHAPELY_GEOMETRY"] = _DEFAULT_SHAPELY_GEOMETRY + return _DEFAULT_SHAPELY_GEOMETRY + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + def _deprecated(reason: str = "", version: Optional[str] = None) -> Callable[..., Any]: """Simple decorator to mark functions/methods/classes as deprecated. @@ -224,58 +233,6 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: return decorator -class FloatRange(click.types.FloatParamType): - """A parameter that works similar to :data:`click.FLOAT` but restricts the - value to fit into a range. Fails if the value doesn't fit into the range. - """ - - name = "percentage" - - def __init__( - self, min: Optional[float] = None, max: Optional[float] = None - ) -> None: - self.min = min - self.max = max - - def convert( - self, - value: Any, - param: Optional[click.core.Parameter], - ctx: Optional[click.core.Context], - ) -> Any: - """Convert value""" - rv = click.types.FloatParamType.convert(self, value, param, ctx) - if ( - self.min is not None - and rv < self.min - or self.max is not None - and rv > self.max - ): - if self.min is None: - self.fail( - "%s is bigger than the maximum valid value %s." % (rv, self.max), - param, - ctx, - ) - elif self.max is None: - self.fail( - "%s is smaller than the minimum valid value %s." % (rv, self.min), - param, - ctx, - ) - else: - self.fail( - "%s is not in the valid range of %s to %s." - % (rv, self.min, self.max), - param, - ctx, - ) - return rv - - def __repr__(self) -> str: - return "FloatRange(%r, %r)" % (self.min, self.max) - - def slugify(value: Any, allow_unicode: bool = False) -> str: """Copied from Django Source code, only modifying last line (no need for safe strings). @@ -644,6 +601,7 @@ def jsonpath_parse_dict_items( """Recursively parse :class:`jsonpath_ng.JSONPath` elements in dict >>> import jsonpath_ng.ext as jsonpath + >>> from jsonpath_ng.ext import parse >>> jsonpath_parse_dict_items( ... {"foo": {"bar": parse("$.a.b")}, "qux": [parse("$.c"), parse("$.c")]}, ... {"a":{"b":"baz"}, "c":"quux"} @@ -928,6 +886,7 @@ def list_items_recursive_sort(config_list: list[Any]) -> list[Any]: def string_to_jsonpath(*args: Any, force: bool = False) -> Union[str, JSONPath]: """Get :class:`jsonpath_ng.JSONPath` for ``$.foo.bar`` like string + >>> from jsonpath_ng.jsonpath import Child, Fields, Index, Root, Slice >>> string_to_jsonpath(None, "$.foo.bar") Child(Child(Root(), Fields('foo')), Fields('bar')) >>> string_to_jsonpath("$.foo.bar") @@ -946,6 +905,8 @@ def string_to_jsonpath(*args: Any, force: bool = False) -> Union[str, JSONPath]: :param force: force conversion even if input string is not detected as a :class:`jsonpath_ng.JSONPath` :returns: Parsed value """ + from jsonpath_ng.jsonpath import Child, Fields, Index, Root, Slice + path_str: str = args[-1] if JSONPATH_MATCH.match(str(path_str)) or force: try: @@ -1070,6 +1031,7 @@ def parse_jsonpath( """Parse jsonpah in ``jsonpath_obj`` using ``values_dict`` >>> import jsonpath_ng.ext as jsonpath + >>> from jsonpath_ng.ext import parse >>> parse_jsonpath(None, parse("$.foo.bar"), **{"foo": {"bar": "baz"}}) 'baz' @@ -1078,6 +1040,8 @@ def parse_jsonpath( :param values_dict: Values used as args for parsing :returns: Parsed value """ + from jsonpath_ng import jsonpath + if isinstance(jsonpath_obj, jsonpath.Child): match = jsonpath_obj.find(values_dict) return match[0].value if len(match) == 1 else None @@ -1118,6 +1082,11 @@ def get_geometry_from_various( :raises TypeError: Unexpected geometry type :raises ValueError: Location name is wrong or its value does not match """ + import shapefile + import shapely.wkt + from shapely.geometry import Polygon, shape + from shapely.geometry.base import GEOMETRY_TYPES, BaseGeometry + geom = None if "geometry" in query_args: @@ -1211,6 +1180,8 @@ def get_geometry_from_ecmwf_feature(geom: dict[str, Any]) -> BaseGeometry: if not isinstance(geom["shape"], list): raise TypeError("Geometry shape must be a list") + from shapely.geometry import Polygon + shape: list = geom["shape"] polygon_args = [(p[1], p[0]) for p in shape] return Polygon(polygon_args) @@ -1233,7 +1204,7 @@ def get_geometry_from_ecmwf_area(area: list[float]) -> Optional[BaseGeometry]: def get_geometry_from_ecmwf_location( - location: dict[str, float] + location: dict[str, float], ) -> Optional[BaseGeometry]: """ Creates a ``shapely.geometry`` from a single location. @@ -1283,6 +1254,8 @@ def __exit__(self, exc_type, exc_val, exc_tb): def raise_for_status(self) -> None: """raises an exception when the status is not ok""" if self.status_code != 200: + from requests import HTTPError, Response + response = Response() response.status_code = self.status_code raise HTTPError(response=response) @@ -1338,11 +1311,15 @@ def cached_parse(str_to_parse: str) -> JSONPath: :param str_to_parse: string to parse as :class:`jsonpath_ng.JSONPath` :returns: parsed :class:`jsonpath_ng.JSONPath` """ + from jsonpath_ng.ext import parse + return parse(str_to_parse) @functools.lru_cache() def _mutable_cached_yaml_load(config_path: str) -> Any: + import yaml + with open( os.path.abspath(os.path.realpath(config_path)), mode="r", encoding="utf-8" ) as fh: @@ -1360,6 +1337,8 @@ def cached_yaml_load(config_path: str) -> dict[str, Any]: @functools.lru_cache() def _mutable_cached_yaml_load_all(config_path: str) -> list[Any]: + import yaml + with open(config_path, "r") as fh: return list(yaml.load_all(fh, Loader=yaml.Loader)) @@ -1743,14 +1722,14 @@ def concat_loc_names(location: tuple): return ".".join(str_loc) error_messages = [ - f'{concat_loc_names(err["loc"])}: {err["msg"]}' if err["loc"] else err["msg"] + f"{concat_loc_names(err['loc'])}: {err['msg']}" if err["loc"] else err["msg"] for err in e.errors() ] return error_header + "; ".join(set(error_messages)) def get_collection_dates( - collection_dict: dict[str, Any] + collection_dict: dict[str, Any], ) -> tuple[Optional[str], Optional[str]]: """Extract mission start and end dates from collection configuration. diff --git a/tests/integration/test_core_search_results.py b/tests/integration/test_core_search_results.py index 0c91a8efde..f93dc66e87 100644 --- a/tests/integration/test_core_search_results.py +++ b/tests/integration/test_core_search_results.py @@ -515,7 +515,7 @@ def test_core_search_with_count(self, mock_urlopen): self.assertIsNotNone(search_results.number_matched) @mock.patch( - "eodag.api.core.fetch_stac_items", + "eodag.utils.stac_reader.fetch_stac_items", autospec=True, side_effect=[ [ @@ -638,7 +638,7 @@ def test_core_import_stac_items_from_eodag_server(self, mock_fetch_stac_items): ) self.assertIsInstance(results[1].downloader, Download) - @mock.patch("eodag.api.core.fetch_stac_items", autospec=True) + @mock.patch("eodag.utils.stac_reader.fetch_stac_items", autospec=True) def test_core_import_stac_items_from_known_provider( self, mock_fetch_stac_items, @@ -666,7 +666,7 @@ def test_core_import_stac_items_from_known_provider( ) self.assertIsInstance(results[0].downloader, Download) - @mock.patch("eodag.api.core.fetch_stac_items", autospec=True) + @mock.patch("eodag.utils.stac_reader.fetch_stac_items", autospec=True) def test_core_import_stac_items_from_unknown_provider(self, mock_fetch_stac_items): """The core api must import STAC items from an unknwown provider""" stac_singlefile = os.path.join(TEST_RESOURCES_PATH, "stac_singlefile.json") diff --git a/tests/test_cli.py b/tests/test_cli.py index 050e912966..dc1fe40eea 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -225,7 +225,7 @@ def test_eodag_search_bbox_invalid(self): self.assertNotEqual(exit_code, 0) self.assertIsInstance(error, SystemExit) - @mock.patch("eodag.cli.EODataAccessGateway", autospec=True) + @mock.patch("eodag.api.core.EODataAccessGateway", autospec=True) def test_eodag_search_bbox_valid(self, dag): """Calling eodag search with --bbox argument valid""" with self.user_conf() as conf_file: @@ -269,7 +269,7 @@ def test_eodag_search_geom_wkt_invalid(self): # GEOSException for shapely >= 2.0 self.assertTrue(isinstance(error, shapely.errors.GEOSException)) - @mock.patch("eodag.cli.EODataAccessGateway", autospec=True) + @mock.patch("eodag.api.core.EODataAccessGateway", autospec=True) def test_eodag_search_geom_wkt_valid(self, dag): """Calling eodag search with --geom WKT argument valid""" with self.user_conf() as conf_file: @@ -334,7 +334,7 @@ def test_eodag_search_bbox_geom_mutually_exclusive(self): self.assertNotEqual(exit_code, 0) self.assertIsInstance(error, SystemExit) - @mock.patch("eodag.cli.EODataAccessGateway", autospec=True) + @mock.patch("eodag.api.core.EODataAccessGateway", autospec=True) def test_eodag_search_storage_arg(self, dag): """Calling eodag search with specified result filename without .geojson extension""" # noqa with self.user_conf() as conf_file: @@ -359,7 +359,7 @@ def test_eodag_search_storage_arg(self, dag): api_obj.search.return_value, filename="results.geojson" ) - @mock.patch("eodag.cli.EODataAccessGateway", autospec=True) + @mock.patch("eodag.api.core.EODataAccessGateway", autospec=True) def test_eodag_search_with_cruncher(self, dag): """Calling eodag search with --cruncher arg should call crunch method of search result""" # noqa with self.user_conf() as conf_file: @@ -437,7 +437,7 @@ def test_eodag_search_with_cruncher(self, dag): **{cruncher: {"minimum_overlap": "10"}}, ) - @mock.patch("eodag.cli.EODataAccessGateway", autospec=True) + @mock.patch("eodag.api.core.EODataAccessGateway", autospec=True) def test_eodag_search_and_download(self, dag): """Calling eodag search with --download argument should directly download search results""" with self.user_conf() as conf_file: @@ -479,7 +479,7 @@ def test_eodag_search_and_download(self, dag): search_results, output_dir=None, executor=mock.ANY ) - @mock.patch("eodag.cli.EODataAccessGateway", autospec=True) + @mock.patch("eodag.api.core.EODataAccessGateway", autospec=True) def test_eodag_search_all(self, dag): """Calling eodag search with --bbox argument valid""" with self.user_conf() as conf_file: @@ -520,7 +520,7 @@ def test_eodag_search_all(self, dag): }, ) - @mock.patch("eodag.cli.EODataAccessGateway", autospec=True) + @mock.patch("eodag.api.core.EODataAccessGateway", autospec=True) def test_eodag_search_query(self, dag): """Calling eodag search with --query argument""" with self.user_conf() as conf_file: @@ -564,7 +564,7 @@ def test_eodag_search_query(self, dag): }, ) - @mock.patch("eodag.cli.EODataAccessGateway", autospec=True) + @mock.patch("eodag.api.core.EODataAccessGateway", autospec=True) def test_eodag_search_locations(self, dag): """Calling eodag search with --locations argument""" with self.user_conf() as conf_file: @@ -606,7 +606,7 @@ def test_eodag_search_locations(self, dag): }, ) - @mock.patch("eodag.cli.EODataAccessGateway", autospec=True) + @mock.patch("eodag.api.core.EODataAccessGateway", autospec=True) def test_eodag_search_start_date(self, dag): """Calling eodag search with --start argument""" with self.user_conf() as conf_file: @@ -652,7 +652,7 @@ def test_eodag_search_start_date(self, dag): }, ) - @mock.patch("eodag.cli.EODataAccessGateway", autospec=True) + @mock.patch("eodag.api.core.EODataAccessGateway", autospec=True) def test_eodag_search_stop_date(self, dag): """Calling eodag search with --end argument""" with self.user_conf() as conf_file: @@ -699,7 +699,7 @@ def test_eodag_search_stop_date(self, dag): }, ) - @mock.patch("eodag.cli.EODataAccessGateway", autospec=True) + @mock.patch("eodag.api.core.EODataAccessGateway", autospec=True) def test_eodag_search_locs(self, dag): """Calling eodag search with --locs argument""" with self.user_conf() as conf_file: @@ -774,7 +774,9 @@ def test_eodag_list_collection_with_provider_ko(self): output, ) - @mock.patch("eodag.cli.EODataAccessGateway.fetch_collections_list", autospec=True) + @mock.patch( + "eodag.api.core.EODataAccessGateway.fetch_collections_list", autospec=True + ) def test_eodag_list_collection_fetch(self, mock_fetch_collections_list): """Calling eodag list should fetch for new collections depending on passed option""" @@ -800,7 +802,7 @@ def test_eodag_list_collection_fetch(self, mock_fetch_collections_list): mock_fetch_collections_list.assert_called_with(mock.ANY, provider="peps") self.assertEqual(mock_fetch_collections_list.call_count, 2) - @mock.patch("eodag.cli.EODataAccessGateway", autospec=True) + @mock.patch("eodag.api.core.EODataAccessGateway", autospec=True) def test_eodag_guess_collection_ok(self, dag): """Calling eodag list with one or several valid collection feature(s) should return all supported collections with this (these) feature(s) among the ones of its provider @@ -836,7 +838,7 @@ def test_eodag_guess_collection_ok(self, dag): ) @mock.patch( - "eodag.cli.EODataAccessGateway.guess_collection", + "eodag.api.core.EODataAccessGateway.guess_collection", autospec=True, side_effect=NoMatchingCollection(), ) @@ -855,7 +857,7 @@ def test_eodag_guess_collection_ko(self, mock_guess_collection): ) @mock.patch( - "eodag.cli.EODataAccessGateway.discover_collections", + "eodag.api.core.EODataAccessGateway.discover_collections", autospec=True, return_value={}, ) @@ -942,7 +944,7 @@ def test_eodag_download_no_search_results_arg(self): ), ) - @mock.patch("eodag.cli.EODataAccessGateway", autospec=True) + @mock.patch("eodag.api.core.EODataAccessGateway", autospec=True) def test_eodag_download_ok(self, dag): """Calling eodag download with all args well formed succeed""" search_results_path = os.path.join( @@ -997,7 +999,7 @@ def test_eodag_download_ok(self, dag): mock.ANY, output_dir=output_dir, executor=mock.ANY ) - @mock.patch("eodag.cli.EODataAccessGateway", autospec=True) + @mock.patch("eodag.api.core.EODataAccessGateway", autospec=True) def test_eodag_download_ko(self, dag): """Calling eodag download with all args well formed fails""" search_results_path = os.path.join( @@ -1015,7 +1017,7 @@ def test_eodag_download_ko(self, dag): output, ) - @mock.patch("eodag.cli.EODataAccessGateway", autospec=True) + @mock.patch("eodag.api.core.EODataAccessGateway", autospec=True) def test_eodag_download_stac_items(self, dag): """Calling eodag download with --stac-item argument""" fake_result = SearchResult([mock.MagicMock() * 2], 2)