[SDK] Add retry for subgraph requests that return bad indexer (#3661)

Author: flopez7

.changeset/wide-sites-pull.md

@@ -0,0 +1,6 @@
+---
+"@human-protocol/sdk": minor
+"@human-protocol/python-sdk": minor
+---
+
+Added new optional config for querying subgraph with retries when failuers are due to bad indexers errors in Python and Typescript SDK

packages/sdk/python/human-protocol-sdk/example.py

@@ -18,6 +18,7 @@
 from human_protocol_sdk.operator import OperatorUtils, OperatorFilter
 from human_protocol_sdk.agreement import agreement
 from human_protocol_sdk.staking.staking_utils import StakingUtils
+from human_protocol_sdk.utils import SubgraphOptions
 
 
 def get_escrow_statistics(statistics_client: StatisticsClient):
@@ -162,7 +163,8 @@ def get_escrows():
                 status=Status.Pending,
                 date_from=datetime.datetime(2023, 5, 8),
                 date_to=datetime.datetime(2023, 6, 8),
-            )
+            ),
+            SubgraphOptions(3, 1000),
         )
     )
 
@@ -232,12 +234,13 @@ def get_stakers_example():
             chain_id=ChainId.POLYGON_AMOY,
             order_by="lastDepositTimestamp",
             order_direction=OrderDirection.ASC,
-        )
+        ),
+        SubgraphOptions(3, 1000),
     )
     print("Filtered stakers:", len(stakers))
 
     if stakers:
-        staker = StakingUtils.get_staker(ChainId.LOCALHOST, stakers[0].address)
+        staker = StakingUtils.get_staker(ChainId.POLYGON_AMOY, stakers[0].address)
         print("Staker info:", staker.address)
     else:
         print("No stakers found.")

packages/sdk/python/human-protocol-sdk/human_protocol_sdk/escrow/escrow_utils.py

@@ -24,7 +24,6 @@
 ------
 """
 
-from datetime import datetime
 import logging
 from typing import List, Optional
 
@@ -38,7 +37,8 @@
     PayoutFilter,
 )
 from human_protocol_sdk.utils import (
-    get_data_from_subgraph,
+    SubgraphOptions,
+    custom_gql_fetch,
 )
 
 from human_protocol_sdk.escrow.escrow_client import EscrowClientError
@@ -219,10 +219,12 @@ class EscrowUtils:
     @staticmethod
     def get_escrows(
         filter: EscrowFilter,
+        options: Optional[SubgraphOptions] = None,
     ) -> List[EscrowData]:
         """Get an array of escrow addresses based on the specified filter parameters.
 
         :param filter: Object containing all the necessary parameters to filter
+        :param options: Optional config for subgraph requests
 
         :return: List of escrows
 
@@ -257,7 +259,7 @@ def get_escrows(
             else:
                 statuses = [filter.status.name]
 
-        escrows_data = get_data_from_subgraph(
+        escrows_data = custom_gql_fetch(
             network,
             query=get_escrows_query(filter),
             params={
@@ -283,6 +285,7 @@ def get_escrows(
                 "skip": filter.skip,
                 "orderDirection": filter.order_direction.value,
             },
+            options=options,
         )
 
         if (
@@ -334,11 +337,13 @@ def get_escrows(
     def get_escrow(
         chain_id: ChainId,
         escrow_address: str,
+        options: Optional[SubgraphOptions] = None,
     ) -> Optional[EscrowData]:
         """Returns the escrow for a given address.
 
         :param chain_id: Network in which the escrow has been deployed
         :param escrow_address: Address of the escrow
+        :param options: Optional config for subgraph requests
 
         :return: Escrow data
 
@@ -367,12 +372,13 @@ def get_escrow(
 
         network = NETWORKS[ChainId(chain_id)]
 
-        escrow_data = get_data_from_subgraph(
+        escrow_data = custom_gql_fetch(
             network,
             query=get_escrow_query(),
             params={
                 "escrowAddress": escrow_address.lower(),
             },
+            options=options,
         )
 
         if (
@@ -414,11 +420,15 @@ def get_escrow(
         )
 
     @staticmethod
-    def get_status_events(filter: StatusEventFilter) -> List[StatusEvent]:
+    def get_status_events(
+        filter: StatusEventFilter,
+        options: Optional[SubgraphOptions] = None,
+    ) -> List[StatusEvent]:
         """
         Retrieve status events for specified networks and statuses within a date range.
 
         :param filter: Object containing all the necessary parameters to filter status events.
+        :param options: Optional config for subgraph requests
 
         :return List[StatusEvent]: List of status events matching the query parameters.
 
@@ -435,7 +445,7 @@ def get_status_events(filter: StatusEventFilter) -> List[StatusEvent]:
 
         status_names = [status.name for status in filter.statuses]
 
-        data = get_data_from_subgraph(
+        data = custom_gql_fetch(
             network,
             get_status_query(filter.date_from, filter.date_to, filter.launcher),
             {
@@ -447,6 +457,7 @@ def get_status_events(filter: StatusEventFilter) -> List[StatusEvent]:
                 "skip": filter.skip,
                 "orderDirection": filter.order_direction.value,
             },
+            options=options,
         )
 
         if (
@@ -472,11 +483,15 @@ def get_status_events(filter: StatusEventFilter) -> List[StatusEvent]:
         return events_with_chain_id
 
     @staticmethod
-    def get_payouts(filter: PayoutFilter) -> List[Payout]:
+    def get_payouts(
+        filter: PayoutFilter,
+        options: Optional[SubgraphOptions] = None,
+    ) -> List[Payout]:
         """
         Fetch payouts from the subgraph based on the provided filter.
 
         :param filter: Object containing all the necessary parameters to filter payouts.
+        :param options: Optional config for subgraph requests
 
         :return List[Payout]: List of payouts matching the query parameters.
 
@@ -494,7 +509,7 @@ def get_payouts(filter: PayoutFilter) -> List[Payout]:
         if not network:
             raise EscrowClientError("Unsupported Chain ID")
 
-        data = get_data_from_subgraph(
+        data = custom_gql_fetch(
             network,
             get_payouts_query(filter),
             {
@@ -508,6 +523,7 @@ def get_payouts(filter: PayoutFilter) -> List[Payout]:
                 "skip": filter.skip,
                 "orderDirection": filter.order_direction.value,
             },
+            options=options,
         )
 
         if (
@@ -536,11 +552,13 @@ def get_payouts(filter: PayoutFilter) -> List[Payout]:
     @staticmethod
     def get_cancellation_refunds(
         filter: CancellationRefundFilter,
+        options: Optional[SubgraphOptions] = None,
     ) -> List[CancellationRefund]:
         """
         Fetch cancellation refunds from the subgraph based on the provided filter.
 
         :param filter: Object containing all the necessary parameters to filter cancellation refunds.
+        :param options: Optional config for subgraph requests
 
         :return List[CancellationRefund]: List of cancellation refunds matching the query parameters.
 
@@ -558,7 +576,7 @@ def get_cancellation_refunds(
         if not network:
             raise EscrowClientError("Unsupported Chain ID")
 
-        data = get_data_from_subgraph(
+        data = custom_gql_fetch(
             network,
             get_cancellation_refunds_query(filter),
             {
@@ -572,6 +590,7 @@ def get_cancellation_refunds(
                 "skip": filter.skip,
                 "orderDirection": filter.order_direction.value,
             },
+            options=options,
         )
 
         if (
@@ -601,13 +620,16 @@ def get_cancellation_refunds(
 
     @staticmethod
     def get_cancellation_refund(
-        chain_id: ChainId, escrow_address: str
+        chain_id: ChainId,
+        escrow_address: str,
+        options: Optional[SubgraphOptions] = None,
     ) -> CancellationRefund:
         """
         Returns the cancellation refund for a given escrow address.
 
         :param chain_id: Network in which the escrow has been deployed
         :param escrow_address: Address of the escrow
+        :param options: Optional config for subgraph requests
 
         :return: CancellationRefund data or None
 
@@ -635,12 +657,13 @@ def get_cancellation_refund(
         if not network:
             raise EscrowClientError("Unsupported Chain ID")
 
-        data = get_data_from_subgraph(
+        data = custom_gql_fetch(
             network,
             get_cancellation_refund_by_escrow_query(),
             {
                 "escrowAddress": escrow_address.lower(),
             },
+            options=options,
         )
 
         if (

packages/sdk/python/human-protocol-sdk/human_protocol_sdk/kvstore/kvstore_utils.py

@@ -29,7 +29,7 @@
 import requests
 
 from human_protocol_sdk.constants import NETWORKS, ChainId, KVStoreKeys
-from human_protocol_sdk.utils import get_data_from_subgraph
+from human_protocol_sdk.utils import SubgraphOptions, custom_gql_fetch
 
 from human_protocol_sdk.kvstore.kvstore_client import KVStoreClientError
 
@@ -57,11 +57,13 @@ class KVStoreUtils:
     def get_kvstore_data(
         chain_id: ChainId,
         address: str,
+        options: Optional[SubgraphOptions] = None,
     ) -> Optional[List[KVStoreData]]:
         """Returns the KVStore data for a given address.
 
         :param chain_id: Network in which the KVStore data has been deployed
         :param address: Address of the KVStore
+        :param options: Optional config for subgraph requests
 
         :return: List of KVStore data
 
@@ -88,12 +90,13 @@ def get_kvstore_data(
 
         network = NETWORKS[ChainId(chain_id)]
 
-        kvstore_data = get_data_from_subgraph(
+        kvstore_data = custom_gql_fetch(
             network,
             query=get_kvstore_by_address_query(),
             params={
                 "address": address.lower(),
             },
+            options=options,
         )
 
         if (
@@ -111,12 +114,18 @@ def get_kvstore_data(
         ]
 
     @staticmethod
-    def get(chain_id: ChainId, address: str, key: str) -> str:
+    def get(
+        chain_id: ChainId,
+        address: str,
+        key: str,
+        options: Optional[SubgraphOptions] = None,
+    ) -> str:
         """Gets the value of a key-value pair in the contract.
 
         :param chain_id: Network in which the KVStore data has been deployed
         :param address: The Ethereum address associated with the key-value pair
         :param key: The key of the key-value pair to get
+        :param options: Optional config for subgraph requests
 
         :return: The value of the key-value pair if it exists
 
@@ -142,13 +151,14 @@ def get(chain_id: ChainId, address: str, key: str) -> str:
 
         network = NETWORKS[ChainId(chain_id)]
 
-        kvstore_data = get_data_from_subgraph(
+        kvstore_data = custom_gql_fetch(
             network,
             query=get_kvstore_by_address_and_key_query(),
             params={
                 "address": address.lower(),
                 "key": key,
             },
+            options=options,
         )
 
         if (
@@ -163,13 +173,17 @@ def get(chain_id: ChainId, address: str, key: str) -> str:
 
     @staticmethod
     def get_file_url_and_verify_hash(
-        chain_id: ChainId, address: str, key: Optional[str] = "url"
+        chain_id: ChainId,
+        address: str,
+        key: Optional[str] = "url",
+        options: Optional[SubgraphOptions] = None,
     ) -> str:
         """Gets the URL value of the given entity, and verify its hash.
 
         :param chain_id: Network in which the KVStore data has been deployed
         :param address: Address from which to get the URL value.
         :param key: Configurable URL key. `url` by default.
+        :param options: Optional config for subgraph requests
 
         :return url: The URL value of the given address if exists, and the content is valid
 
@@ -189,8 +203,8 @@ def get_file_url_and_verify_hash(
         if not Web3.is_address(address):
             raise KVStoreClientError(f"Invalid address: {address}")
 
-        url = KVStoreUtils.get(chain_id, address, key)
-        hash = KVStoreUtils.get(chain_id, address, key + "_hash")
+        url = KVStoreUtils.get(chain_id, address, key, options=options)
+        hash = KVStoreUtils.get(chain_id, address, key + "_hash", options=options)
 
         if len(url) == 0:
             return url