Feat/grep

.env.template

@@ -61,7 +61,8 @@ FIREBASE_SERVICE_ACCOUNT=
 KNOWLEDGE_GRAPH_URL=
 GITHUB_APP_ID=
 GITHUB_PRIVATE_KEY=
-GH_TOKEN_LIST=  # Comma-separated GitHub PAT tokens for github.com (e.g., ghp_token1,ghp_token2)
+# Comma-separated GitHub PAT tokens for github.com (e.g., ghp_token1,ghp_token2)
+GH_TOKEN_LIST=
 TRANSACTION_EMAILS_ENABLED=
 EMAIL_FROM_ADDRESS=
 RESEND_API_KEY=
@@ -86,9 +87,12 @@ PHOENIX_PROJECT_NAME=potpie-ai
 
 # Optional: Git provider configuration for self-hosted instances
 # Supported providers: github, gitbucket, gitlab, bitbucket, local
-CODE_PROVIDER=github  # Options: github, gitlab, gitbucket, local
-CODE_PROVIDER_BASE_URL=  # e.g., http://localhost:8080/api/v3 for GitBucket, /path/to/repo for local
-CODE_PROVIDER_TOKEN=  # PAT for self-hosted Git server (not needed for local)
+# Options: github, gitlab, gitbucket, local
+CODE_PROVIDER=github
+# e.g., http://localhost:8080/api/v3 for GitBucket, /path/to/repo for local
+CODE_PROVIDER_BASE_URL=
+# PAT for self-hosted Git server (not needed for local)
+CODE_PROVIDER_TOKEN=
 
 # For local provider:
 # CODE_PROVIDER=local

.gitignore

@@ -74,3 +74,4 @@ package-lock.json
 thoughts/
 .codex/
 worktrees/
+.repos/

README.md

@@ -197,44 +197,45 @@ Potpie provides a set of tools that agents can use to interact with the knowledg
       **`INFERENCE_MODEL`** and **`CHAT_MODEL`** correspond to the models that will be used for generating knowledge graph and for agent reasoning respectively. These model names should be in the format of `provider/model_name` format or as expected by Litellm. For more information, refer to the [Litellm documentation](https://docs.litellm.ai/docs/providers).
       <br>
 
-      #### GitHub Authentication Setup
+#### GitHub Authentication Setup
 
-      Potpie supports multiple authentication methods for accessing GitHub repositories:
+Potpie supports multiple authentication methods for accessing GitHub repositories:
 
-      ##### For GitHub.com Repositories:
+##### For GitHub.com Repositories:
 
-      **Option 1: GitHub App (Recommended for Production)**
-        - Create a GitHub App in your organization
-        - Set environment variables:
-          ```bash
-          GITHUB_APP_ID=your-app-id
-          GITHUB_PRIVATE_KEY=your-private-key
-          ```
+**Option 1: GitHub App (Recommended for Production)**
+  - Create a GitHub App in your organization
+  - Set environment variables:
+    ```bash
+    GITHUB_APP_ID=your-app-id
+    GITHUB_PRIVATE_KEY=your-private-key
+    ```
 
-      **Option 2: Personal Access Token (PAT) Pool**
-        - Create one or more GitHub PATs with `repo` scope
-        - Set environment variable (comma-separated for multiple tokens):
-          ```bash
-          GH_TOKEN_LIST=ghp_token1,ghp_token2,ghp_token3
-          ```
-        - Potpie will randomly select from the pool for load balancing
-        - **Rate Limit**: 5,000 requests/hour per token (authenticated)
+**Option 2: Personal Access Token (PAT) Pool**
+  - Create one or more GitHub PATs with `repo` scope
+  - Set environment variable (comma-separated for multiple tokens):
+    ```bash
+    GH_TOKEN_LIST=ghp_token1,ghp_token2,ghp_token3
+    ```
+  - Potpie will randomly select from the pool for load balancing
+  - **Rate Limit**: 5,000 requests/hour per token (authenticated)
 
-      **Option 3: Unauthenticated Access (Public Repos Only)**
-        - No configuration needed
-        - Automatically used as fallback for public repositories
-        - **Rate Limit**: 60 requests/hour per IP (very limited)
+**Option 3: Unauthenticated Access (Public Repos Only)**
+  - No configuration needed
+  - Automatically used as fallback for public repositories
+  - **Rate Limit**: 60 requests/hour per IP (very limited)
 
-      ##### For Self-Hosted Git Servers (GitBucket, GitLab, etc.):
+##### For Self-Hosted Git Servers (GitBucket, GitLab, etc.):
 
       Set the following environment variables:
       ```bash
-      CODE_PROVIDER=github  # or gitlab
+      # Options: github, gitlab, gitbucket
+      CODE_PROVIDER=github
       CODE_PROVIDER_BASE_URL=http://your-git-server.com/api/v3
       CODE_PROVIDER_TOKEN=your-token
       ```
 
-      **Important**: `GH_TOKEN_LIST` tokens are always used for GitHub.com, regardless of `CODE_PROVIDER_BASE_URL`.
+**Important**: `GH_TOKEN_LIST` tokens are always used for GitHub.com, regardless of `CODE_PROVIDER_BASE_URL`.
 
    -  Create a Virtual Environment using Python 3.10:
       ```

app/modules/code_provider/gitbucket/gitbucket_provider.py

@@ -1,5 +1,5 @@
 import logging
-from typing import List, Dict, Any, Optional
+from typing import Any, Dict, List, Optional, Set
 import chardet
 from github import Github
 from github.GithubException import GithubException
@@ -96,13 +96,32 @@ def _ensure_authenticated(self):
         if not self.client:
             raise RuntimeError("Provider not authenticated. Call authenticate() first.")
 
+    def _get_repo(self, repo_name: str):
+        """
+        Get repository object with normalized repo name conversion.
+
+        Converts normalized repo name (e.g., 'user/repo') back to GitBucket's
+        actual identifier format (e.g., 'root/repo') for API calls.
+
+        Args:
+            repo_name: Normalized repository name
+
+        Returns:
+            Repository object from PyGithub
+        """
+        from app.modules.parsing.utils.repo_name_normalizer import (
+            get_actual_repo_name_for_lookup,
+        )
+
+        actual_repo_name = get_actual_repo_name_for_lookup(repo_name, "gitbucket")
+        return self.client.get_repo(actual_repo_name)
+
     # ============ Repository Operations ============
 
     def get_repository(self, repo_name: str) -> Dict[str, Any]:
         """Get repository details."""
         self._ensure_authenticated()
 
-        # Convert normalized repo name back to GitBucket format for API calls
         from app.modules.parsing.utils.repo_name_normalizer import (
             get_actual_repo_name_for_lookup,
             normalize_repo_name,
@@ -114,7 +133,7 @@ def get_repository(self, repo_name: str) -> Dict[str, Any]:
             f"GitBucket: Attempting to get repository '{repo_name}' (actual: '{actual_repo_name}')"
         )
         try:
-            repo = self.client.get_repo(actual_repo_name)
+            repo = self._get_repo(repo_name)
             logger.info(
                 f"GitBucket: Successfully retrieved repository '{repo_name}' - ID: {repo.id}, Default branch: {repo.default_branch}"
             )
@@ -183,14 +202,7 @@ def get_file_content(
         """Get file content."""
         self._ensure_authenticated()
 
-        # Convert normalized repo name back to GitBucket format for API calls
-        from app.modules.parsing.utils.repo_name_normalizer import (
-            get_actual_repo_name_for_lookup,
-        )
-
-        actual_repo_name = get_actual_repo_name_for_lookup(repo_name, "gitbucket")
-
-        repo = self.client.get_repo(actual_repo_name)
+        repo = self._get_repo(repo_name)
         file_contents = repo.get_contents(file_path, ref=ref)
 
         # Decode content
@@ -223,23 +235,14 @@ def get_repository_structure(
         """Get repository structure recursively."""
         self._ensure_authenticated()
 
-        # Convert normalized repo name back to GitBucket format for API calls
-        from app.modules.parsing.utils.repo_name_normalizer import (
-            get_actual_repo_name_for_lookup,
-        )
-
-        actual_repo_name = get_actual_repo_name_for_lookup(repo_name, "gitbucket")
-
         try:
-            repo = self.client.get_repo(actual_repo_name)
+            repo = self._get_repo(repo_name)
         except GithubException as e:
-            logger.error(
-                f"GitBucket: Failed to get repository '{actual_repo_name}': {e}"
-            )
+            logger.error(f"GitBucket: Failed to get repository '{repo_name}': {e}")
             raise
         except Exception as e:
             logger.error(
-                f"GitBucket: Unexpected error getting repository '{actual_repo_name}': {e}"
+                f"GitBucket: Unexpected error getting repository '{repo_name}': {e}"
             )
             raise
 
@@ -449,14 +452,7 @@ def list_branches(self, repo_name: str) -> List[str]:
         """List branches."""
         self._ensure_authenticated()
 
-        # Convert normalized repo name back to GitBucket format for API calls
-        from app.modules.parsing.utils.repo_name_normalizer import (
-            get_actual_repo_name_for_lookup,
-        )
-
-        actual_repo_name = get_actual_repo_name_for_lookup(repo_name, "gitbucket")
-
-        repo = self.client.get_repo(actual_repo_name)
+        repo = self._get_repo(repo_name)
         branches = [branch.name for branch in repo.get_branches()]
 
         # Put default branch first
@@ -471,7 +467,6 @@ def get_branch(self, repo_name: str, branch_name: str) -> Dict[str, Any]:
         """Get branch details."""
         self._ensure_authenticated()
 
-        # Convert normalized repo name back to GitBucket format for API calls
         from app.modules.parsing.utils.repo_name_normalizer import (
             get_actual_repo_name_for_lookup,
         )
@@ -482,7 +477,7 @@ def get_branch(self, repo_name: str, branch_name: str) -> Dict[str, Any]:
             f"GitBucket: Getting branch '{branch_name}' for repository '{repo_name}' (actual: '{actual_repo_name}')"
         )
         try:
-            repo = self.client.get_repo(actual_repo_name)
+            repo = self._get_repo(repo_name)
             branch = repo.get_branch(branch_name)
 
             branch_data = {
@@ -521,7 +516,7 @@ def create_branch(
         self._ensure_authenticated()
 
         try:
-            repo = self.client.get_repo(repo_name)
+            repo = self._get_repo(repo_name)
 
             # Get base branch ref
             base_ref = repo.get_git_ref(f"heads/{base_branch}")
@@ -575,20 +570,24 @@ def compare_branches(
         self._ensure_authenticated()
 
         try:
-            repo = self.client.get_repo(repo_name)
+            repo = self._get_repo(repo_name)
 
             # Get commits on the head branch
             logging.info(f"[GITBUCKET] Getting commits for branch: {head_branch}")
             head_commits = repo.get_commits(sha=head_branch)
 
+            max_commits = 50  # Safety limit
+
             # Get commits on the base branch for comparison
-            base_commits = list(repo.get_commits(sha=base_branch))
-            base_commit_shas = {c.sha for c in base_commits}
+            base_commit_shas: Set[str] = set()
+            for idx, base_commit in enumerate(repo.get_commits(sha=base_branch)):
+                base_commit_shas.add(base_commit.sha)
+                if idx + 1 >= max_commits:
+                    break
 
             # Track files and their patches
             files_dict = {}
             commit_count = 0
-            max_commits = 50  # Safety limit
 
             # Iterate through head branch commits until we find common ancestor
             for commit in head_commits:
@@ -650,7 +649,7 @@ def list_pull_requests(
         """List pull requests."""
         self._ensure_authenticated()
 
-        repo = self.client.get_repo(repo_name)
+        repo = self._get_repo(repo_name)
         pulls = repo.get_pulls(state=state)[:limit]
 
         return [
@@ -674,7 +673,7 @@ def get_pull_request(
         """Get pull request details."""
         self._ensure_authenticated()
 
-        repo = self.client.get_repo(repo_name)
+        repo = self._get_repo(repo_name)
         pr = repo.get_pull(pr_number)
 
         result = {
@@ -719,7 +718,7 @@ def create_pull_request(
         self._ensure_authenticated()
 
         try:
-            repo = self.client.get_repo(repo_name)
+            repo = self._get_repo(repo_name)
 
             # Validate branches exist
             try:
@@ -783,7 +782,7 @@ def add_pull_request_comment(
         self._ensure_authenticated()
 
         try:
-            repo = self.client.get_repo(repo_name)
+            repo = self._get_repo(repo_name)
             pr = repo.get_pull(pr_number)
 
             if path and line:
@@ -819,7 +818,7 @@ def create_pull_request_review(
         self._ensure_authenticated()
 
         try:
-            repo = self.client.get_repo(repo_name)
+            repo = self._get_repo(repo_name)
             pr = repo.get_pull(pr_number)
 
             commits = list(pr.get_commits())
@@ -856,7 +855,7 @@ def list_issues(
         """List issues."""
         self._ensure_authenticated()
 
-        repo = self.client.get_repo(repo_name)
+        repo = self._get_repo(repo_name)
         issues = repo.get_issues(state=state)[:limit]
 
         return [
@@ -876,7 +875,7 @@ def get_issue(self, repo_name: str, issue_number: int) -> Dict[str, Any]:
         """Get issue details."""
         self._ensure_authenticated()
 
-        repo = self.client.get_repo(repo_name)
+        repo = self._get_repo(repo_name)
         issue = repo.get_issue(issue_number)
 
         return {
@@ -897,7 +896,7 @@ def create_issue(
         self._ensure_authenticated()
 
         try:
-            repo = self.client.get_repo(repo_name)
+            repo = self._get_repo(repo_name)
             issue = repo.create_issue(title=title, body=body, labels=labels or [])
 
             return {
@@ -928,7 +927,7 @@ def create_or_update_file(
         self._ensure_authenticated()
 
         try:
-            repo = self.client.get_repo(repo_name)
+            repo = self._get_repo(repo_name)
 
             # Check if file exists
             try:
@@ -1057,7 +1056,7 @@ def get_archive_link(self, repo_name: str, format_type: str, ref: str) -> str:
         )
 
         try:
-            repo = self.client.get_repo(actual_repo_name)
+            repo = self._get_repo(repo_name)
 
             # GitBucket uses a different URL format than GitHub API
             # The correct format is: http://hostname/owner/repo/archive/ref.format

app/modules/code_provider/local_repo/local_provider.py

@@ -299,8 +299,8 @@ def _traverse_directory(
             return []
 
         for entry in sorted(entries):
-            # Skip hidden files
-            if entry.startswith("."):
+            # Skip .git directory only
+            if entry == ".git":
                 continue
 
             entry_path = os.path.join(dir_path, entry)

app/modules/code_provider/local_repo/local_repo_service.py

@@ -225,8 +225,8 @@ async def _fetch_repo_structure_async(
             # Filter out files with excluded extensions, hidden files/folders, and gitignore matches
             filtered_contents = []
             for item in contents:
-                # Skip hidden files and directories (starting with .)
-                if item["name"].startswith(".") and item["name"] != ".gitignore":
+                # Skip .git directory only
+                if item["name"] == ".git":
                     continue
 
                 # Skip files with excluded extensions