diff --git a/ogr/abstract/git_service.py b/ogr/abstract/git_service.py index 1d7c92082..ed08e5856 100644 --- a/ogr/abstract/git_service.py +++ b/ogr/abstract/git_service.py @@ -141,3 +141,13 @@ def get_group(self, group_name: str): Get a group by name. """ raise NotImplementedError + + def get_rate_limit_remaining(self) -> Optional[int]: + """ + Get the remaining rate limit. + + Returns: + Number of remaining API requests, or None if rate limit information + is not available. + """ + raise NotImplementedError diff --git a/ogr/services/forgejo/service.py b/ogr/services/forgejo/service.py index 4039b11f2..ea1dc590b 100644 --- a/ogr/services/forgejo/service.py +++ b/ogr/services/forgejo/service.py @@ -87,3 +87,9 @@ def get_project_from_url(self, url: str) -> "ForgejoProject": repo = path_parts[1] return self.get_project(repo=repo, namespace=namespace) + + def get_rate_limit_remaining(self) -> Optional[int]: + """ + There is no way to check rate limit status from Forgejo API. + """ + return None diff --git a/ogr/services/github/service.py b/ogr/services/github/service.py index be6c91adf..64215ed18 100644 --- a/ogr/services/github/service.py +++ b/ogr/services/github/service.py @@ -239,3 +239,15 @@ def list_projects( ] return projects + + def get_rate_limit_remaining(self) -> Optional[int]: + rate_limit = self.github.get_rate_limit() + # Handle both old and new PyGithub API versions + # Old API in f42 and f43: rate_limit.resources.core.remaining + # New API in rawhide (f44): rate_limit.core.remaining (or rate_limit.remaining) + try: + # since PyGithub 2.7.0 + return rate_limit.resources.core.remaining + except AttributeError: + return rate_limit.core.remaining + return None diff --git a/ogr/services/gitlab/service.py b/ogr/services/gitlab/service.py index 23475711b..1824285f1 100644 --- a/ogr/services/gitlab/service.py +++ b/ogr/services/gitlab/service.py @@ -172,3 +172,29 @@ def list_projects( ) for project in projects_to_convert ] + + def get_rate_limit_remaining(self) -> Optional[int]: + # python-gitlab doesn't have get_rate_limit(), so we make a lightweight + # HEAD request to get rate limit headers from GitLab API + # GitLab returns rate limit in headers: ratelimit-remaining + # Use obey_rate_limit=False to prevent blocking if we've already hit the limit + try: + headers = self.gitlab_instance.http_head("/user", obey_rate_limit=False) + remaining = headers.get("ratelimit-remaining") + if remaining: + return int(remaining) + except gitlab.GitlabHttpError as e: + # If we get a 429, we've hit the rate limit + if e.response_code == 429: + logger.error( + f"Rate limit has been exceeded: {e}", + ) + return 0 + logger.error( + f"Could not get rate limit from GitLab: {e}", + ) + except Exception as e: + logger.error( + f"Could not get rate limit from GitLab: {e}", + ) + return None diff --git a/ogr/services/pagure/service.py b/ogr/services/pagure/service.py index b025c5a4d..9f201c764 100644 --- a/ogr/services/pagure/service.py +++ b/ogr/services/pagure/service.py @@ -403,3 +403,9 @@ def get_group(self, group_name: str) -> PagureGroup: """ url = self.get_api_url("group", group_name) return PagureGroup(group_name, self.call_api(url)) + + def get_rate_limit_remaining(self) -> Optional[int]: + """ + There is no way to check rate limit status from Pagure API. + """ + return None diff --git a/tests/integration/forgejo/test_service.py b/tests/integration/forgejo/test_service.py index 58e251fa4..57dfa821f 100644 --- a/tests/integration/forgejo/test_service.py +++ b/tests/integration/forgejo/test_service.py @@ -42,3 +42,8 @@ def test_project_create(service, kwargs_): # Try to fetch newly created project project = service.get_project(**kwargs_fetch) assert project.forgejo_repo + + +def test_get_rate_limit_remaining(service): + remaining = service.get_rate_limit_remaining() + assert remaining is None diff --git a/tests/integration/github/test_data/test_service/Service.test_get_rate_limit_remaining.yaml b/tests/integration/github/test_data/test_service/Service.test_get_rate_limit_remaining.yaml new file mode 100644 index 000000000..cde47b934 --- /dev/null +++ b/tests/integration/github/test_data/test_service/Service.test_get_rate_limit_remaining.yaml @@ -0,0 +1,142 @@ +_requre: + DataTypes: 1 + key_strategy: StorageKeysInspectSimple + version_storage_file: 3 +requests.sessions: + send: + GET: + https://api.github.com:443/rate_limit: + - metadata: + latency: 0.27362728118896484 + module_call_list: + - unittest.case + - requre.record_and_replace + - tests.integration.github.test_service + - ogr.abstract.exception + - ogr.services.github.service + - github.MainClass + - github.Requester + - requests.sessions + - requre.objects + - requre.cassette + - requests.sessions + - send + output: + __store_indicator: 2 + _content: + rate: + limit: 5000 + remaining: 5000 + reset: 1768315875 + used: 0 + resources: + actions_runner_registration: + limit: 10000 + remaining: 10000 + reset: 1768315875 + used: 0 + audit_log: + limit: 1750 + remaining: 1750 + reset: 1768315875 + used: 0 + audit_log_streaming: + limit: 15 + remaining: 15 + reset: 1768315875 + used: 0 + code_scanning_autofix: + limit: 10 + remaining: 10 + reset: 1768312335 + used: 0 + code_scanning_upload: + limit: 5000 + remaining: 5000 + reset: 1768315875 + used: 0 + code_search: + limit: 10 + remaining: 10 + reset: 1768312335 + used: 0 + core: + limit: 5000 + remaining: 5000 + reset: 1768315875 + used: 0 + dependency_sbom: + limit: 100 + remaining: 100 + reset: 1768312335 + used: 0 + dependency_snapshots: + limit: 100 + remaining: 100 + reset: 1768312335 + used: 0 + graphql: + limit: 5000 + remaining: 5000 + reset: 1768315875 + used: 0 + integration_manifest: + limit: 5000 + remaining: 5000 + reset: 1768315875 + used: 0 + scim: + limit: 15000 + remaining: 15000 + reset: 1768315875 + used: 0 + search: + limit: 30 + remaining: 30 + reset: 1768312335 + used: 0 + source_import: + limit: 100 + remaining: 100 + reset: 1768312335 + used: 0 + _next: null + elapsed: 0.2 + encoding: utf-8 + headers: + Access-Control-Allow-Origin: '*' + Access-Control-Expose-Headers: ETag, Link, Location, Retry-After, X-GitHub-OTP, + X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Used, X-RateLimit-Resource, + X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval, + X-GitHub-Media-Type, X-GitHub-SSO, X-GitHub-Request-Id, Deprecation, + Sunset + Cache-Control: no-cache + Content-Encoding: gzip + Content-Security-Policy: default-src 'self';script-src 'self' 'nonce-YqLDC0BS8d7iY8mKO7VtBbIne' + https://apps.fedoraproject.org; style-src 'self' 'nonce-YqLDC0BS8d7iY8mKO7VtBbIne'; + object-src 'none';base-uri 'self';img-src 'self' https:; + Content-Type: application/json; charset=utf-8 + Date: Fri, 01 Nov 2019 13-36-03 GMT + Referrer-Policy: origin-when-cross-origin, strict-origin-when-cross-origin + Server: github.com + Strict-Transport-Security: max-age=31536000; includeSubdomains; preload + Transfer-Encoding: chunked + Vary: Accept-Encoding, Accept, X-Requested-With + X-Accepted-OAuth-Scopes: '' + X-Content-Type-Options: nosniff + X-Frame-Options: deny + X-GitHub-Media-Type: github.v3; format=json + X-GitHub-Request-Id: 18FB:AA1A:99616C4:B8092CB:5CC15425 + X-OAuth-Scopes: public_repo + X-RateLimit-Limit: '5000' + X-RateLimit-Remaining: '4972' + X-RateLimit-Reset: '1572953901' + X-RateLimit-Resource: core + X-RateLimit-Used: '0' + X-XSS-Protection: '0' + github-authentication-token-expiration: 2026-02-12 23:00:00 UTC + x-github-api-version-selected: '2022-11-28' + raw: !!binary "" + raw_decoded: !!binary "" + reason: OK + status_code: 200 diff --git a/tests/integration/github/test_service.py b/tests/integration/github/test_service.py index fda040be5..82a67bfed 100644 --- a/tests/integration/github/test_service.py +++ b/tests/integration/github/test_service.py @@ -131,3 +131,7 @@ def test_wrong_auth_static_method(self): # Verify that the exception handler is applied to static methods with pytest.raises(GithubAPIException): GithubPullRequest.get(self.ogr_project, 1) + + def test_get_rate_limit_remaining(self): + remaining = self.service.get_rate_limit_remaining() + assert remaining >= 100 diff --git a/tests/integration/gitlab/test_data/test_service/Service.test_get_rate_limit_remaining.yaml b/tests/integration/gitlab/test_data/test_service/Service.test_get_rate_limit_remaining.yaml new file mode 100644 index 000000000..0c5148e7a --- /dev/null +++ b/tests/integration/gitlab/test_data/test_service/Service.test_get_rate_limit_remaining.yaml @@ -0,0 +1,166 @@ +_requre: + DataTypes: 1 + key_strategy: StorageKeysInspectSimple + version_storage_file: 3 +requests.sessions: + send: + GET: + https://gitlab.com/api/v4/user: + - metadata: + latency: 0.3923823833465576 + module_call_list: + - unittest.case + - requre.record_and_replace + - tests.integration.gitlab.test_service + - ogr.abstract.exception + - ogr.services.gitlab.service + - gitlab.client + - gitlab.exceptions + - gitlab.mixins + - gitlab.client + - gitlab._backends.requests_backend + - requests.sessions + - requre.objects + - requre.cassette + - requests.sessions + - send + output: + __store_indicator: 2 + _content: + avatar_url: https://gitlab.com/uploads/-/system/user/avatar/6307839/avatar.png + bio: '' + bot: false + can_create_group: true + can_create_project: true + color_scheme_id: 1 + commit_email: 6307839-packit-as-a-service-stg@users.noreply.gitlab.com + confirmed_at: '2022-11-11T09:16:52.861Z' + created_at: '2020-06-25T09:59:10.420Z' + current_sign_in_at: '2025-05-09T10:22:06.026Z' + discord: '' + email: stg@packit.dev + external: false + extra_shared_runners_minutes_limit: null + github: '' + id: 6307839 + identities: + - extern_uid: packit-stg + provider: group_saml + saml_provider_id: 3283 + - extern_uid: packit-stg + provider: group_saml + saml_provider_id: 2868 + job_title: '' + last_activity_on: '2026-01-13' + last_sign_in_at: '2025-02-03T13:40:46.152Z' + linkedin: '' + local_time: 2:28 PM + location: '' + locked: false + name: Packit Stage Service + organization: '' + preferred_language: en + private_profile: false + projects_limit: 100000 + pronouns: '' + public_email: '' + scim_identities: [] + shared_runners_minutes_limit: null + state: active + theme_id: 1 + twitter: '' + two_factor_enabled: true + username: packit-as-a-service-stg + web_url: https://gitlab.com/packit-as-a-service-stg + website_url: https://packit.dev + work_information: null + _next: null + elapsed: 0.2 + encoding: utf-8 + headers: + CF-Cache-Status: MISS + CF-Ray: 9bd594b73be0ee55-MXP + Cache-Control: max-age=0, private, must-revalidate + Connection: keep-alive + Content-Encoding: br + Content-Type: application/json + Date: Fri, 01 Nov 2019 13-36-03 GMT + ETag: W/"1e51b8e1c48787a433405211e9e0fe61" + Server: cloudflare + Set-Cookie: a='b'; + Strict-Transport-Security: max-age=31536000 + Transfer-Encoding: chunked + Vary: Origin, Accept-Encoding + content-security-policy: default-src 'none' + gitlab-lb: haproxy-main-52-lb-gprd + gitlab-sv: api-gke-us-east1-b + nel: '{"max_age": 0}' + ratelimit-limit: '2000' + ratelimit-name: throttle_authenticated_api + ratelimit-observed: '1' + ratelimit-remaining: '1999' + ratelimit-reset: '1768314540' + referrer-policy: strict-origin-when-cross-origin + x-content-type-options: nosniff + x-frame-options: SAMEORIGIN + x-gitlab-meta: '{"correlation_id":"9bd594b77613ee55-ATL","version":"1"}' + x-request-id: 9bd594b77613ee55-ATL + x-runtime: '0.079368' + raw: !!binary "" + raw_decoded: !!binary "" + reason: OK + status_code: 200 + HEAD: + https://gitlab.com/api/v4/user: + - metadata: + latency: 0.23740577697753906 + module_call_list: + - unittest.case + - requre.record_and_replace + - tests.integration.gitlab.test_service + - ogr.abstract.exception + - ogr.services.gitlab.service + - gitlab.client + - gitlab._backends.requests_backend + - requests.sessions + - requre.objects + - requre.cassette + - requests.sessions + - send + output: + __store_indicator: 1 + _content: '' + _next: null + elapsed: 0.2 + encoding: utf-8 + headers: + CF-Cache-Status: MISS + CF-Ray: 9bd594b91f54ee55-MXP + Cache-Control: max-age=0, private, must-revalidate + Connection: keep-alive + Content-Encoding: br + Content-Type: application/json + Date: Fri, 01 Nov 2019 13-36-03 GMT + ETag: W/"1e51b8e1c48787a433405211e9e0fe61" + Server: cloudflare + Strict-Transport-Security: max-age=31536000 + Vary: Origin, Accept-Encoding + content-security-policy: default-src 'none' + gitlab-lb: haproxy-main-39-lb-gprd + gitlab-sv: api-gke-us-east1-d + nel: '{"max_age": 0}' + ratelimit-limit: '2000' + ratelimit-name: throttle_authenticated_api + ratelimit-observed: '2' + ratelimit-remaining: '1998' + ratelimit-reset: '1768314540' + referrer-policy: strict-origin-when-cross-origin + x-content-type-options: nosniff + x-frame-options: SAMEORIGIN + x-gitlab-meta: '{"correlation_id":"9bd594b936aaee55-ATL","version":"1"}' + x-request-id: 9bd594b936aaee55-ATL + x-runtime: '0.051577' + raw: !!binary "" + raw_decoded: !!binary "" + reason: OK + status_code: 200 diff --git a/tests/integration/gitlab/test_service.py b/tests/integration/gitlab/test_service.py index 7db67152b..af17bdd50 100644 --- a/tests/integration/gitlab/test_service.py +++ b/tests/integration/gitlab/test_service.py @@ -143,3 +143,7 @@ def test_wrong_auth_static_method(self): # Verify that the exception handler is applied to static methods with pytest.raises(GitlabAPIException): GitlabPullRequest.get(self.project, 1) + + def test_get_rate_limit_remaining(self): + remaining = self.service.get_rate_limit_remaining() + assert remaining >= 100 diff --git a/tests/integration/pagure/test_service.py b/tests/integration/pagure/test_service.py index 5ee9f2c5a..cd2343139 100644 --- a/tests/integration/pagure/test_service.py +++ b/tests/integration/pagure/test_service.py @@ -94,3 +94,7 @@ def test_get_group(self): assert members assert len(members) > 1 assert "lbarczio" in members + + def test_get_rate_limit_remaining(self): + remaining = self.service.get_rate_limit_remaining() + assert remaining is None