Dev -> Stable 2.7.0

.github/workflows/benchmark.yml

@@ -0,0 +1,166 @@
+name: Performance Benchmarks
+
+on:
+  pull_request:
+    paths:
+      - 'bbot/**/*.py'
+      - 'pyproject.toml'
+      - '.github/workflows/benchmark.yml'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v5
+      with:
+        fetch-depth: 0  # Need full history for branch comparison
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.11"
+
+    - name: Install dependencies
+      run: |
+        pip install poetry
+        poetry install --with dev
+
+    - name: Install system dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libmagic1
+
+    # Generate benchmark comparison report using our branch-based script
+    - name: Generate benchmark comparison report
+      run: |
+        poetry run python bbot/scripts/benchmark_report.py \
+          --base ${{ github.base_ref }} \
+          --current ${{ github.head_ref }} \
+          --output benchmark_report.md \
+          --keep-results
+      continue-on-error: true
+
+    # Upload benchmark results as artifacts
+    - name: Upload benchmark results
+      uses: actions/upload-artifact@v4
+      with:
+        name: benchmark-results
+        path: |
+          benchmark_report.md
+          base_benchmark_results.json
+          current_benchmark_results.json
+        retention-days: 30
+
+    # Comment on PR with benchmark results
+    - name: Comment benchmark results on PR
+      uses: actions/github-script@v8
+      with:
+        script: |
+          const fs = require('fs');
+
+          try {
+            const report = fs.readFileSync('benchmark_report.md', 'utf8');
+
+            // Find existing benchmark comment (with pagination)
+            const comments = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              per_page: 100, // Get more comments per page
+            });
+
+            // Debug: log all comments to see what we're working with
+            console.log(`Found ${comments.data.length} comments on this PR`);
+            comments.data.forEach((comment, index) => {
+              console.log(`Comment ${index}: user=${comment.user.login}, body preview="${comment.body.substring(0, 100)}..."`);
+            });
+
+            const existingComments = comments.data.filter(comment => 
+              comment.body.toLowerCase().includes('performance benchmark') &&
+              comment.user.login === 'github-actions[bot]'
+            );
+
+            console.log(`Found ${existingComments.length} existing benchmark comments`);
+
+            if (existingComments.length > 0) {
+              // Sort comments by creation date to find the most recent
+              const sortedComments = existingComments.sort((a, b) => 
+                new Date(b.created_at) - new Date(a.created_at)
+              );
+
+              const mostRecentComment = sortedComments[0];
+              console.log(`Updating most recent benchmark comment: ${mostRecentComment.id} (created: ${mostRecentComment.created_at})`);
+
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: mostRecentComment.id,
+                body: report
+              });
+              console.log('Updated existing benchmark comment');
+
+              // Delete any older duplicate comments
+              if (existingComments.length > 1) {
+                console.log(`Deleting ${existingComments.length - 1} older duplicate comments`);
+                for (let i = 1; i < sortedComments.length; i++) {
+                  const commentToDelete = sortedComments[i];
+                  console.log(`Attempting to delete comment ${commentToDelete.id} (created: ${commentToDelete.created_at})`);
+
+                  try {
+                    await github.rest.issues.deleteComment({
+                      owner: context.repo.owner,
+                      repo: context.repo.repo,
+                      comment_id: commentToDelete.id
+                    });
+                    console.log(`Successfully deleted duplicate comment: ${commentToDelete.id}`);
+                  } catch (error) {
+                    console.error(`Failed to delete comment ${commentToDelete.id}: ${error.message}`);
+                    console.error(`Error details:`, error);
+                  }
+                }
+              }
+            } else {
+              // Create new comment
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: report
+              });
+              console.log('Created new benchmark comment');
+            }
+          } catch (error) {
+            console.error('Failed to post benchmark results:', error);
+
+            // Post a fallback comment
+            const fallbackMessage = [
+              '## Performance Benchmark Report',
+              '',
+              '> ⚠️ **Failed to generate detailed benchmark comparison**',
+              '> ',
+              '> The benchmark comparison failed to run. This might be because:',
+              '> - Benchmark tests don\'t exist on the base branch yet',
+              '> - Dependencies are missing', 
+              '> - Test execution failed',
+              '> ',
+              '> Please check the [workflow logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.',
+              '> ',
+              '> 📁 Benchmark artifacts may be available for download from the workflow run.'
+            ].join('\\n');
+
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: fallbackMessage
+            });
+          } 

.github/workflows/docs_updater.yml

@@ -14,7 +14,7 @@ jobs:
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
           ref: dev  # Checkout the dev branch
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.x"
       - name: Install dependencies

.github/workflows/tests.yml

@@ -21,7 +21,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
       - name: Set Python Version Environment Variable
@@ -59,7 +59,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.x"
       - name: Install dependencies
@@ -72,7 +72,7 @@ jobs:
         run: python -m build
       - name: Publish Pypi package
         if: github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev'
-        uses: pypa/gh-action-pypi-publish@release/v1.12
+        uses: pypa/gh-action-pypi-publish@release/v1.13
         with:
           password: ${{ secrets.PYPI_API_TOKEN }}
       - name: Get BBOT version
@@ -110,7 +110,7 @@ jobs:
       - uses: actions/checkout@v5
         with:
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: "3.11"
       - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV

.github/workflows/version_updater.yml

@@ -15,7 +15,7 @@ jobs:
           fetch-depth: 0
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.x'
       - name: Install dependencies
@@ -67,7 +67,7 @@ jobs:
           fetch-depth: 0
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.x'
       - name: Install dependencies

bbot/core/engine.py

@@ -636,7 +636,7 @@ async def finished_tasks(self, tasks, timeout=None):
         """
         if tasks:
             try:
-                done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED, timeout=timeout)
+                done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED, timeout=timeout)
                 return done
             except BaseException as e:
                 if isinstance(e, (TimeoutError, asyncio.exceptions.TimeoutError)):

bbot/core/helpers/bloom.py

@@ -1,6 +1,7 @@
 import os
 import mmh3
 import mmap
+import xxhash
 
 
 class BloomFilter:
@@ -55,14 +56,12 @@ def _hashes(self, item):
             if not isinstance(item, str):
                 item = str(item)
             item = item.encode("utf-8")
-        return [abs(hash(item)) % self.size, abs(mmh3.hash(item)) % self.size, abs(self._fnv1a_hash(item)) % self.size]
 
-    def _fnv1a_hash(self, data):
-        hash = 0x811C9DC5  # 2166136261
-        for byte in data:
-            hash ^= byte
-            hash = (hash * 0x01000193) % 2**32  # 16777619
-        return hash
+        return [
+            abs(hash(item)) % self.size,
+            abs(mmh3.hash(item)) % self.size,
+            abs(xxhash.xxh32(item).intdigest()) % self.size,
+        ]
 
     def close(self):
         """Explicitly close the memory-mapped file."""

bbot/core/helpers/dns/dns.py

@@ -38,7 +38,6 @@ class DNSHelper(EngineClient):
         _wildcard_cache (dict): Cache for wildcard detection results.
         _dns_cache (LRUCache): Cache for DNS resolution results, limited in size.
         resolver_file (Path): File containing system's current resolver nameservers.
-        filter_bad_ptrs (bool): Whether to filter out DNS names that appear to be auto-generated PTR records. Defaults to True.
 
     Args:
         parent_helper: The parent helper object with configuration details and utilities.

bbot/core/helpers/dns/engine.py

@@ -86,8 +86,6 @@ def __init__(self, socket_path, config={}, debug=False):
         self._debug = self.dns_config.get("debug", False)
         self._dns_cache = LRUCache(maxsize=10000)
 
-        self.filter_bad_ptrs = self.dns_config.get("filter_ptrs", True)
-
     async def resolve(self, query, **kwargs):
         """Resolve DNS names and IP addresses to their corresponding results.
 

bbot/core/helpers/files.py

@@ -9,7 +9,7 @@
 log = logging.getLogger("bbot.core.helpers.files")
 
 
-def tempfile(self, content, pipe=True):
+def tempfile(self, content, pipe=True, extension=None):
     """
     Creates a temporary file or named pipe and populates it with content.
 
@@ -29,7 +29,7 @@ def tempfile(self, content, pipe=True):
         >>> tempfile(["Another", "temp", "file"], pipe=False)
         '/home/user/.bbot/temp/someotherfile'
     """
-    filename = self.temp_filename()
+    filename = self.temp_filename(extension)
     rm_at_exit(filename)
     try:
         if type(content) not in (set, list, tuple):

bbot/core/helpers/git.py

@@ -0,0 +1,17 @@
+from pathlib import Path
+
+
+def sanitize_git_repo(repo_folder: Path):
+    # sanitizing the git config is infeasible since there are too many different ways to do evil things
+    # instead, we move it out of .git and into the repo folder, so we don't miss any secrets etc. inside
+    config_file = repo_folder / ".git" / "config"
+    if config_file.exists():
+        config_file.rename(repo_folder / "git_config_original")
+    # move the index file
+    index_file = repo_folder / ".git" / "index"
+    if index_file.exists():
+        index_file.rename(repo_folder / "git_index_original")
+    # move the hooks folder
+    hooks_folder = repo_folder / ".git" / "hooks"
+    if hooks_folder.exists():
+        hooks_folder.rename(repo_folder / "git_hooks_original")

bbot/core/helpers/misc.py

@@ -17,6 +17,7 @@
 from asyncio import create_task, gather, sleep, wait_for  # noqa
 from urllib.parse import urlparse, quote, unquote, urlunparse, urljoin  # noqa F401
 
+from .git import *  # noqa F401
 from .url import *  # noqa F401
 from ... import errors
 from . import regexes as bbot_regexes

bbot/core/helpers/ntlm.py

@@ -17,11 +17,9 @@ def __init__(self, pos_tup, raw):
         self.alloc = alloc
         self.offset = offset
         self.raw = raw[offset : offset + length]
-        self.utf16 = False
 
         if len(self.raw) >= 2 and self.raw[1] == "\0":
             self.string = self.raw.decode("utf-16")
-            self.utf16 = True
         else:
             self.string = self.raw
 

bbot/core/helpers/regex.py

@@ -65,7 +65,7 @@ def new_task(regex_name, r):
 
         while tasks:  # While there are tasks pending
             # Wait for the first task to complete
-            done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+            done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
 
             for task in done:
                 result = task.result()