Merge branch 'develop' into fix-broken-links

Author: jemeza-codegen

.circleci/config.yml

@@ -75,14 +75,14 @@ commands:
     steps:
       - restore_cache:
           keys:
-            - v1-verified-codemod-repos-{{ checksum "tests/verified_codemods/codemod_data/repo_commits.json" }}-{{.Environment.CIRCLE_NODE_INDEX}}-{{.Environment.CIRCLE_NODE_TOTAL}}
+            - v1-verified-codemod-repos-{{ checksum "tests/integration/verified_codemods/codemod_data/repo_commits.json" }}-{{.Environment.CIRCLE_NODE_INDEX}}-{{.Environment.CIRCLE_NODE_TOTAL}}
       - run:
           command: |
             uv run --frozen gs codemod clone-repos --verified-codemod-repos --token ${CODEGEN_BOT_GHE_TOKEN}
       - save_cache:
           paths:
             - $GITHUB_WORKSPACE
-          key: v1-verified-codemod-repos-{{ checksum "tests/verified_codemods/codemod_data/repo_commits.json" }}-{{.Environment.CIRCLE_NODE_INDEX}}-{{.Environment.CIRCLE_NODE_TOTAL}}
+          key: v1-verified-codemod-repos-{{ checksum "tests/integration/verified_codemods/codemod_data/repo_commits.json" }}-{{.Environment.CIRCLE_NODE_INDEX}}-{{.Environment.CIRCLE_NODE_TOTAL}}
   build-wheels:
     parameters:
       base_packages:
@@ -237,7 +237,7 @@ jobs:
     - fetch-verified-codemods
     - cache-verified-codemod-repos
     - run_ats:
-        default_tests: "tests/codemod/test_verified_codemods.py"
+        default_tests: "tests/integration/codemod/test_verified_codemods.py"
         codecov_flags: "smart-tests-codemod"
         collect_args: --cli-api-key ${PROD_CLI_API_KEY} --token ${CODEGEN_BOT_GHE_TOKEN}
         ats_collect_args: --cli-api-key=${PROD_CLI_API_KEY},--token=${CODEGEN_BOT_GHE_TOKEN},
@@ -264,7 +264,7 @@ jobs:
     - clone-repos:
         extra_repos: false
     - run_ats:
-        default_tests: "tests/codemod/test_codemods.py"
+        default_tests: "tests/integration/codemod/test_codemods.py"
         codecov_flags: "smart-tests-codemod-oss"
         collect_args: --size=<<parameters.size>> --sync-graph=<<parameters.sync_graph>> --token ${CODEGEN_BOT_GHE_TOKEN}
         ats_collect_args: --size=<<parameters.size>>,--sync-graph=<<parameters.sync_graph>>,--token=${CODEGEN_BOT_GHE_TOKEN},
@@ -297,7 +297,7 @@ jobs:
           if [ "<<parameters.extra_repos>>" = "true" ]; then
             EXTRA_REPOS_ARG="--extra-repos=true"
           fi
-          PYTEST_ARGS="${EXTRA_REPOS_ARG} --token ${CODEGEN_BOT_GHE_TOKEN} -o junit_suite_name=\"${CIRCLE_JOB}\" tests/codemod/test_parse.py"
+          PYTEST_ARGS="${EXTRA_REPOS_ARG} --token ${CODEGEN_BOT_GHE_TOKEN} -o junit_suite_name=\"${CIRCLE_JOB}\" tests/integration/codemod/test_parse.py"
           echo "Running tests with args: $PYTEST_ARGS"
           TESTS_TO_RUN=$(PYTEST_ARGS=${PYTEST_ARGS} ./.circleci/collect.sh)
           echo $TESTS_TO_RUN | circleci tests run --command "ulimit -s unlimited; xargs uv run --frozen pytest -n auto ${PYTEST_ARGS}"

.gitattributes

@@ -1,6 +1,6 @@
 **/expected_diff.txt filter=lfs diff=lfs merge=lfs -text
-tests/codemod/repos/extra/*.json filter=lfs diff=lfs merge=lfs -text
+tests/integration/codemod/repos/extra/*.json filter=lfs diff=lfs merge=lfs -text
 **/expected_diff.patch filter=lfs diff=lfs merge=lfs -text
-tests/codemod/repos/repos.json filter=lfs diff=lfs merge=lfs -text
-tests/verified_codemods/** filter=lfs diff=lfs merge=lfs -text
+tests/integration/codemod/repos/repos.json filter=lfs diff=lfs merge=lfs -text
+tests/integration/verified_codemods/** filter=lfs diff=lfs merge=lfs -text
 .github/disallowed-words.txt filter=lfs diff=lfs merge=lfs -text

.github/disallowed-words.txt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac0b7be00533f882fb174c498c27372fa78d3ec2d65c0e885475bf92b43b7303
-size 337
+oid sha256:c47fe11113256de71968b186bd459732c5f29547b9a57275f671502e9ebd8327
+size 328

.gitignore

@@ -63,5 +63,5 @@ uv-*.tar.gz
 graph-sitter-types/out/**
 graph-sitter-types/typings/**
 coverage.json
-tests/verified_codemods/codemod_data/repo_commits.json
+tests/integration/verified_codemods/codemod_data/repo_commits.json
 

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
     hooks:
       - id: biome-check
         additional_dependencies: ["@biomejs/[email protected]"]
-        exclude: (src/codemods/eval)|(tests/unit/codegen/sdk/skills)|(tests/unit/output)|(tests/verified_codemods)|(docs/samples)
+        exclude: (src/codemods/eval)|(tests/unit/codegen/sdk/skills)|(tests/unit/codegen/sdk/output)|(tests/integration/verified_codemods)|(docs/samples)
 
   - repo: https://github.com/MarcoGorelli/cython-lint
     rev: v0.16.6

CLA.md

@@ -196,6 +196,7 @@ pythonVersion = "3.12"
 enableExperimentalFeatures = true
 [tool.pytest.ini_options]
 # addopts = -v
+pythonpath = "."
 norecursedirs = "repos expected"
 # addopts = -v --cov=app --cov-report=term
 

CONTRIBUTING.md

@@ -25,8 +25,8 @@ class GithubClient:
     @classmethod
     def from_repo_config(cls, repo_config: RepoConfig) -> Self:
         gh_wrapper = cls()
-        gh_wrapper.read_client = gh_wrapper._create_client_for_repo(repo_config)
-        gh_wrapper._write_client = gh_wrapper._create_client_for_repo(repo_config)
+        gh_wrapper.read_client = gh_wrapper._create_client_for_repo(repo_config, github_scope=GithubScope.READ)
+        gh_wrapper._write_client = gh_wrapper._create_client_for_repo(repo_config, github_scope=GithubScope.WRITE)
         return gh_wrapper
 
     @classmethod
@@ -37,8 +37,8 @@ def from_token(cls, token: str | None = None) -> Self:
         gh_wrapper._write_client = Github(token, base_url=cls.base_url)
         return gh_wrapper
 
-    def _create_client_for_repo(self, repo_config: RepoConfig) -> Github:
-        token = get_token_for_repo_config(repo_config=repo_config, github_type=self.type)
+    def _create_client_for_repo(self, repo_config: RepoConfig, github_scope: GithubScope = GithubScope.READ) -> Github:
+        token = get_token_for_repo_config(repo_config=repo_config, github_type=self.type, github_scope=github_scope)
         return Github(token, base_url=self.base_url)
 
     def _get_client_for_scope(self, github_scope: GithubScope) -> Github:

pyproject.toml

@@ -1,14 +1,18 @@
 import logging
 
 from codegen.git.configs.config import config
-from codegen.git.schemas.github import GithubType
+from codegen.git.schemas.github import GithubScope, GithubType
 from codegen.git.schemas.repo_config import RepoConfig
 
 logger = logging.getLogger(__name__)
 
 
-def get_token_for_repo_config(repo_config: RepoConfig, github_type: GithubType = GithubType.GithubEnterprise) -> str:
-    # TODO: implement config such that we can retrieve tokens for different repos
+def get_token_for_repo_config(
+    repo_config: RepoConfig,
+    github_type: GithubType = GithubType.GithubEnterprise,
+    github_scope: GithubScope = GithubScope.READ,
+) -> str:
+    # TODO: implement config such that we can retrieve tokens for different repos + read/write scopes
     if github_type == GithubType.GithubEnterprise:
         return config.LOWSIDE_TOKEN
     elif github_type == GithubType.Github: