refactor: improvements for error handling

eodag/plugins/download/http.py

@@ -73,6 +73,7 @@
     DownloadError,
     MisconfiguredError,
     NotAvailableError,
+    QuotaExceededError,
     TimeOutError,
     ValidationError,
 )
@@ -223,6 +224,14 @@ def _order(
                 except RequestException as e:
                     self._check_auth_exception(e)
                     msg = f"{product.properties['title']} could not be ordered"
+                    if (
+                        e.response is not None
+                        and e.response.status_code
+                        and e.response.status_code == 429
+                    ):
+                        raise QuotaExceededError(
+                            f"Too many requests on provider {self.provider}, please check your quota!"
+                        )
                     if e.response is not None and e.response.status_code == 400:
                         raise ValidationError.from_error(e, msg) from e
                     else:
@@ -450,11 +459,12 @@ def _request(
                 {k: v for k, v in status_dict.items() if v != NOT_AVAILABLE}
             )
 
-            product.properties["eodag:order_status"] = status_dict.get(
-                "eodag:order_status"
-            )
+            order_status = status_dict.get("eodag:order_status")
+            product.properties["eodag:order_status"] = order_status
 
-            status_message = status_dict.get("eodag:order_message")
+            status_message = status_dict.get(
+                "eodag:order_message", f"request status: {order_status}"
+            )
 
             # handle status error
             errors: dict[str, Any] = status_config.get("error", {})
@@ -893,6 +903,15 @@ def _process_exception(
         self, e: Optional[RequestException], product: EOProduct, ordered_message: str
     ) -> None:
         self._check_auth_exception(e)
+        if (
+            e
+            and e.response is not None
+            and e.response.status_code
+            and e.response.status_code == 429
+        ):
+            raise QuotaExceededError(
+                f"Too many requests on provider {self.provider}, please check your quota!"
+            )
         response_text = (
             e.response.text.strip() if e is not None and e.response is not None else ""
         )
@@ -1362,6 +1381,14 @@ def download_asset(asset_stream: StreamResponse) -> None:
 
     def _handle_asset_exception(self, e: RequestException, asset: Asset) -> None:
         # check if error is identified as auth_error in provider conf
+        if (
+            e.response is not None
+            and e.response.status_code
+            and e.response.status_code == 429
+        ):
+            raise QuotaExceededError(
+                f"Too many requests on provider {self.provider}, please check your quota!"
+            )
         auth_errors = getattr(self.config, "auth_error_code", [None])
         if not isinstance(auth_errors, list):
             auth_errors = [auth_errors]

eodag/plugins/search/qssearch.py

@@ -31,7 +31,7 @@
     cast,
     get_args,
 )
-from urllib.error import URLError
+from urllib.error import HTTPError, URLError
 from urllib.parse import (
     parse_qs,
     parse_qsl,
@@ -99,6 +99,7 @@
     AuthenticationError,
     MisconfiguredError,
     PluginImplementationError,
+    QuotaExceededError,
     RequestError,
     TimeOutError,
     ValidationError,
@@ -1414,6 +1415,21 @@ def get_provider_collections(
         else:
             return tuple(provider_collection)
 
+    def _raise_request_error(
+        self, err_msg: str, exception_message: Optional[str], url: str, e: Exception
+    ):
+        if exception_message:
+            logger.exception("%s %s" % (exception_message, err_msg))
+        else:
+            logger.exception(
+                "Skipping error while requesting: %s (provider:%s, plugin:%s): %s",
+                url,
+                self.provider,
+                self.__class__.__name__,
+                err_msg,
+            )
+        raise RequestError.from_error(e, exception_message) from e
+
     def _request(
         self,
         prep: PreparedSearch,
@@ -1495,19 +1511,27 @@ def _request(
         except socket.timeout:
             err = requests.exceptions.Timeout(request=requests.Request(url=url))
             raise TimeOutError(err, timeout=timeout)
-        except (requests.RequestException, URLError) as err:
-            err_msg = err.readlines() if hasattr(err, "readlines") else ""
-            if exception_message:
-                logger.exception("%s %s" % (exception_message, err_msg))
-            else:
-                logger.exception(
-                    "Skipping error while requesting: %s (provider:%s, plugin:%s): %s",
-                    url,
-                    self.provider,
-                    self.__class__.__name__,
-                    err_msg,
+        except HTTPError as e:  # raised by urlopen
+            if e.code and e.code == 429:
+                raise QuotaExceededError(
+                    f"Too many requests on provider {self.provider}, please check your quota!"
                 )
-            raise RequestError.from_error(err, exception_message) from err
+            err_msg = e.msg
+            self._raise_request_error(err_msg, exception_message, url, e)
+        except URLError as e:
+            err_msg = str(e)
+            self._raise_request_error(err_msg, exception_message, url, e)
+        except requests.RequestException as err:
+            if (
+                err.response
+                and err.response.status_code
+                and err.response.status_code == 429
+            ):
+                raise QuotaExceededError(
+                    f"Too many requests on provider {self.provider}, please check your quota!"
+                )
+            err_msg = err.readlines() if hasattr(err, "readlines") else ""
+            self._raise_request_error(err_msg, exception_message, url, err)
         return response
 
 
@@ -1591,7 +1615,15 @@ def do_search(
                     response.raise_for_status()
                 except requests.exceptions.Timeout as exc:
                     raise TimeOutError(exc, timeout=HTTP_REQ_TIMEOUT) from exc
-                except requests.RequestException:
+                except requests.RequestException as e:
+                    if (
+                        e.response
+                        and e.response.status_code
+                        and e.response.status_code == 429
+                    ):
+                        logger.error(
+                            f"Too many requests on provider {self.provider}, please check your quota!"
+                        )
                     logger.exception(
                         "Skipping error while searching for %s %s instance",
                         self.provider,
@@ -2037,6 +2069,10 @@ def _request(
                     f"HTTP Error {response.status_code} returned.",
                     response.text.strip(),
                 )
+            if response.status_code and response.status_code == 429:
+                raise QuotaExceededError(
+                    f"Too many requests on provider {self.provider}, please check your quota!"
+                )
             if exception_message:
                 logger.exception(exception_message)
             else:

eodag/utils/exceptions.py

@@ -140,3 +140,7 @@ def __init__(
             f"Request timeout {timeout_msg} for URL {url}" if url else str(exception)
         )
         super().__init__(message)
+
+
+class QuotaExceededError(RequestError):
+    """An error indicating that too many requests were sent to a provider"""

tests/units/test_download_plugins.py

@@ -36,6 +36,7 @@
     DownloadError,
     MisconfiguredError,
     NoMatchingCollection,
+    QuotaExceededError,
     ValidationError,
 )
 from tests import TEST_RESOURCES_PATH
@@ -779,6 +780,28 @@ def test_plugins_download_http_assets_stream_zip_interrupt(
         self.assertEqual(self.product.location, "http://somewhere")
         self.assertEqual(self.product.remote_location, "http://somewhere")
 
+    @mock.patch("eodag.plugins.download.http.HTTPDownload._get_asset_sizes")
+    @mock.patch("eodag.plugins.download.http.requests.head", autospec=True)
+    @mock.patch("eodag.plugins.download.http.requests.get", autospec=True)
+    def test_plugins_download_http_assets_too_many_requests_error(
+        self, mock_requests_get, mock_requests_head, mock_asset_size
+    ):
+        """HTTPDownload.download() must handle a 429 (Too many requests) error"""
+
+        plugin = self.get_download_plugin(self.product)
+        self.product.location = self.product.remote_location = "http://somewhere"
+        self.product.properties["id"] = "someproduct"
+        self.product.assets.clear()
+        self.product.assets.update({"foo": {"href": "http://somewhere/something"}})
+        self.product.assets.update({"bar": {"href": "http://somewhere/anotherthing"}})
+        res = MockResponse({"a": "a"}, 429)
+        mock_requests_get.side_effect = res
+        mock_requests_head.return_value.headers = CaseInsensitiveDict(
+            {"Content-Disposition": ""}
+        )
+        with self.assertRaises(QuotaExceededError):
+            plugin.download(self.product, output_dir=self.output_dir)
+
     def test_plugins_download_http_stream_dict_misconfigured(self):
         """HTTPDownload.stream_download() must raise an error if misconfigured"""
 

tests/units/test_search_plugins.py

@@ -46,6 +46,7 @@
 from eodag.utils import deepcopy
 from eodag.utils.exceptions import (
     PluginImplementationError,
+    QuotaExceededError,
     UnsupportedCollection,
     ValidationError,
 )
@@ -416,6 +417,19 @@ def test_plugins_search_querystringsearch_search_cop_dataspace_ko(self):
         # restore the original config
         self.sara_search_plugin.config = provider_search_plugin_config
 
+    @mock.patch("eodag.plugins.search.qssearch.requests.Session.get", autospec=True)
+    def test_plugins_search_querystringsearch_search_quota_exceeded(
+        self, mock__request
+    ):
+        """A query with a QueryStringSearch must handle a 429 response returned by the provider"""
+        response = MockResponse({}, status_code=429)
+        mock__request.side_effect = requests.exceptions.HTTPError(response=response)
+        prep = PreparedSearch(collection="S2_MSI_L1C", count=False)
+        with self.assertRaises(QuotaExceededError):
+            self.sara_search_plugin.query(
+                prep=prep, collection="S2_MSI_L1C", **{"eo:cloud_cover": 50}
+            )
+
     @mock.patch(
         "eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True
     )
@@ -1116,6 +1130,20 @@ def run():
 
         run()
 
+    @mock.patch("eodag.plugins.search.qssearch.requests.post", autospec=True)
+    def test_plugins_search_postjsonsearch_search_quota_exceeded(self, mock__request):
+        """A query with a PostJsonSearch must handle a 429 response returned by the provider"""
+
+        class MockResponseRequestsException(MockResponse):
+            def raise_for_status(self):
+                if self.status_code != 200:
+                    raise requests.RequestException()
+
+        response = MockResponseRequestsException({}, status_code=429)
+        mock__request.return_value = response
+        with self.assertRaises(QuotaExceededError):
+            self.awseos_search_plugin.query(collection="S2_MSI_L2A")
+
     @mock.patch("eodag.plugins.search.qssearch.PostJsonSearch._request", autospec=True)
     def test_plugins_search_postjsonsearch_count_and_search_awseos(self, mock__request):
         """A query with a PostJsonSearch (here aws_eos) must return tuple with a list of EOProduct and a number of available products"""  # noqa