diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py index 35697e88fb..1c193261cd 100644 --- a/eodag/plugins/download/http.py +++ b/eodag/plugins/download/http.py @@ -73,6 +73,7 @@ DownloadError, MisconfiguredError, NotAvailableError, + QuotaExceededError, TimeOutError, ValidationError, ) @@ -223,6 +224,14 @@ def _order( except RequestException as e: self._check_auth_exception(e) msg = f"{product.properties['title']} could not be ordered" + if ( + e.response is not None + and e.response.status_code + and e.response.status_code == 429 + ): + raise QuotaExceededError( + f"Too many requests on provider {self.provider}, please check your quota!" + ) if e.response is not None and e.response.status_code == 400: raise ValidationError.from_error(e, msg) from e else: @@ -450,11 +459,12 @@ def _request( {k: v for k, v in status_dict.items() if v != NOT_AVAILABLE} ) - product.properties["eodag:order_status"] = status_dict.get( - "eodag:order_status" - ) + order_status = status_dict.get("eodag:order_status") + product.properties["eodag:order_status"] = order_status - status_message = status_dict.get("eodag:order_message") + status_message = status_dict.get( + "eodag:order_message", f"request status: {order_status}" + ) # handle status error errors: dict[str, Any] = status_config.get("error", {}) @@ -893,6 +903,15 @@ def _process_exception( self, e: Optional[RequestException], product: EOProduct, ordered_message: str ) -> None: self._check_auth_exception(e) + if ( + e + and e.response is not None + and e.response.status_code + and e.response.status_code == 429 + ): + raise QuotaExceededError( + f"Too many requests on provider {self.provider}, please check your quota!" + ) response_text = ( e.response.text.strip() if e is not None and e.response is not None else "" ) @@ -1362,6 +1381,14 @@ def download_asset(asset_stream: StreamResponse) -> None: def _handle_asset_exception(self, e: RequestException, asset: Asset) -> None: # check if error is identified as auth_error in provider conf + if ( + e.response is not None + and e.response.status_code + and e.response.status_code == 429 + ): + raise QuotaExceededError( + f"Too many requests on provider {self.provider}, please check your quota!" + ) auth_errors = getattr(self.config, "auth_error_code", [None]) if not isinstance(auth_errors, list): auth_errors = [auth_errors] diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index b6c38ae881..2a7f892d59 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -31,7 +31,7 @@ cast, get_args, ) -from urllib.error import URLError +from urllib.error import HTTPError, URLError from urllib.parse import ( parse_qs, parse_qsl, @@ -99,6 +99,7 @@ AuthenticationError, MisconfiguredError, PluginImplementationError, + QuotaExceededError, RequestError, TimeOutError, ValidationError, @@ -1414,6 +1415,21 @@ def get_provider_collections( else: return tuple(provider_collection) + def _raise_request_error( + self, err_msg: str, exception_message: Optional[str], url: str, e: Exception + ): + if exception_message: + logger.exception("%s %s" % (exception_message, err_msg)) + else: + logger.exception( + "Skipping error while requesting: %s (provider:%s, plugin:%s): %s", + url, + self.provider, + self.__class__.__name__, + err_msg, + ) + raise RequestError.from_error(e, exception_message) from e + def _request( self, prep: PreparedSearch, @@ -1495,19 +1511,27 @@ def _request( except socket.timeout: err = requests.exceptions.Timeout(request=requests.Request(url=url)) raise TimeOutError(err, timeout=timeout) - except (requests.RequestException, URLError) as err: - err_msg = err.readlines() if hasattr(err, "readlines") else "" - if exception_message: - logger.exception("%s %s" % (exception_message, err_msg)) - else: - logger.exception( - "Skipping error while requesting: %s (provider:%s, plugin:%s): %s", - url, - self.provider, - self.__class__.__name__, - err_msg, + except HTTPError as e: # raised by urlopen + if e.code and e.code == 429: + raise QuotaExceededError( + f"Too many requests on provider {self.provider}, please check your quota!" ) - raise RequestError.from_error(err, exception_message) from err + err_msg = e.msg + self._raise_request_error(err_msg, exception_message, url, e) + except URLError as e: + err_msg = str(e) + self._raise_request_error(err_msg, exception_message, url, e) + except requests.RequestException as err: + if ( + err.response + and err.response.status_code + and err.response.status_code == 429 + ): + raise QuotaExceededError( + f"Too many requests on provider {self.provider}, please check your quota!" + ) + err_msg = err.readlines() if hasattr(err, "readlines") else "" + self._raise_request_error(err_msg, exception_message, url, err) return response @@ -1591,7 +1615,15 @@ def do_search( response.raise_for_status() except requests.exceptions.Timeout as exc: raise TimeOutError(exc, timeout=HTTP_REQ_TIMEOUT) from exc - except requests.RequestException: + except requests.RequestException as e: + if ( + e.response + and e.response.status_code + and e.response.status_code == 429 + ): + logger.error( + f"Too many requests on provider {self.provider}, please check your quota!" + ) logger.exception( "Skipping error while searching for %s %s instance", self.provider, @@ -2037,6 +2069,10 @@ def _request( f"HTTP Error {response.status_code} returned.", response.text.strip(), ) + if response.status_code and response.status_code == 429: + raise QuotaExceededError( + f"Too many requests on provider {self.provider}, please check your quota!" + ) if exception_message: logger.exception(exception_message) else: diff --git a/eodag/utils/exceptions.py b/eodag/utils/exceptions.py index 6fec02b18f..e2b2a1dd01 100644 --- a/eodag/utils/exceptions.py +++ b/eodag/utils/exceptions.py @@ -140,3 +140,7 @@ def __init__( f"Request timeout {timeout_msg} for URL {url}" if url else str(exception) ) super().__init__(message) + + +class QuotaExceededError(RequestError): + """An error indicating that too many requests were sent to a provider""" diff --git a/tests/units/test_download_plugins.py b/tests/units/test_download_plugins.py index c0be5da65e..bfbe099c5b 100644 --- a/tests/units/test_download_plugins.py +++ b/tests/units/test_download_plugins.py @@ -36,6 +36,7 @@ DownloadError, MisconfiguredError, NoMatchingCollection, + QuotaExceededError, ValidationError, ) from tests import TEST_RESOURCES_PATH @@ -779,6 +780,28 @@ def test_plugins_download_http_assets_stream_zip_interrupt( self.assertEqual(self.product.location, "http://somewhere") self.assertEqual(self.product.remote_location, "http://somewhere") + @mock.patch("eodag.plugins.download.http.HTTPDownload._get_asset_sizes") + @mock.patch("eodag.plugins.download.http.requests.head", autospec=True) + @mock.patch("eodag.plugins.download.http.requests.get", autospec=True) + def test_plugins_download_http_assets_too_many_requests_error( + self, mock_requests_get, mock_requests_head, mock_asset_size + ): + """HTTPDownload.download() must handle a 429 (Too many requests) error""" + + plugin = self.get_download_plugin(self.product) + self.product.location = self.product.remote_location = "http://somewhere" + self.product.properties["id"] = "someproduct" + self.product.assets.clear() + self.product.assets.update({"foo": {"href": "http://somewhere/something"}}) + self.product.assets.update({"bar": {"href": "http://somewhere/anotherthing"}}) + res = MockResponse({"a": "a"}, 429) + mock_requests_get.side_effect = res + mock_requests_head.return_value.headers = CaseInsensitiveDict( + {"Content-Disposition": ""} + ) + with self.assertRaises(QuotaExceededError): + plugin.download(self.product, output_dir=self.output_dir) + def test_plugins_download_http_stream_dict_misconfigured(self): """HTTPDownload.stream_download() must raise an error if misconfigured""" diff --git a/tests/units/test_search_plugins.py b/tests/units/test_search_plugins.py index 6d48fbb095..eb23abd31f 100644 --- a/tests/units/test_search_plugins.py +++ b/tests/units/test_search_plugins.py @@ -46,6 +46,7 @@ from eodag.utils import deepcopy from eodag.utils.exceptions import ( PluginImplementationError, + QuotaExceededError, UnsupportedCollection, ValidationError, ) @@ -416,6 +417,19 @@ def test_plugins_search_querystringsearch_search_cop_dataspace_ko(self): # restore the original config self.sara_search_plugin.config = provider_search_plugin_config + @mock.patch("eodag.plugins.search.qssearch.requests.Session.get", autospec=True) + def test_plugins_search_querystringsearch_search_quota_exceeded( + self, mock__request + ): + """A query with a QueryStringSearch must handle a 429 response returned by the provider""" + response = MockResponse({}, status_code=429) + mock__request.side_effect = requests.exceptions.HTTPError(response=response) + prep = PreparedSearch(collection="S2_MSI_L1C", count=False) + with self.assertRaises(QuotaExceededError): + self.sara_search_plugin.query( + prep=prep, collection="S2_MSI_L1C", **{"eo:cloud_cover": 50} + ) + @mock.patch( "eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True ) @@ -1116,6 +1130,20 @@ def run(): run() + @mock.patch("eodag.plugins.search.qssearch.requests.post", autospec=True) + def test_plugins_search_postjsonsearch_search_quota_exceeded(self, mock__request): + """A query with a PostJsonSearch must handle a 429 response returned by the provider""" + + class MockResponseRequestsException(MockResponse): + def raise_for_status(self): + if self.status_code != 200: + raise requests.RequestException() + + response = MockResponseRequestsException({}, status_code=429) + mock__request.return_value = response + with self.assertRaises(QuotaExceededError): + self.awseos_search_plugin.query(collection="S2_MSI_L2A") + @mock.patch("eodag.plugins.search.qssearch.PostJsonSearch._request", autospec=True) def test_plugins_search_postjsonsearch_count_and_search_awseos(self, mock__request): """A query with a PostJsonSearch (here aws_eos) must return tuple with a list of EOProduct and a number of available products""" # noqa