From 4e490deb27f5b0eac61990ad168a8c08a48a13d5 Mon Sep 17 00:00:00 2001 From: Tobias Oberstein Date: Sat, 13 Dec 2025 21:28:24 +0100 Subject: [PATCH 1/9] add feature branch audit file --- .audit/oberstet_sync-autobahn-zlmdb.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .audit/oberstet_sync-autobahn-zlmdb.md diff --git a/.audit/oberstet_sync-autobahn-zlmdb.md b/.audit/oberstet_sync-autobahn-zlmdb.md new file mode 100644 index 000000000..efa137590 --- /dev/null +++ b/.audit/oberstet_sync-autobahn-zlmdb.md @@ -0,0 +1,8 @@ +- [ ] I did **not** use any AI-assistance tools to help create this pull request. +- [x] I **did** use AI-assistance tools to *help* create this pull request. +- [x] I have read, understood and followed the project's AI_POLICY.md when creating code, documentation etc. for this pull request. + +Submitted by: @oberstet +Date: 2025-12-13 +Related issue(s): #1821 +Branch: oberstet:sync-autobahn-zlmdb From 7ce4f0c4addcb848db03cff0463e41d2ad425d0e Mon Sep 17 00:00:00 2001 From: Tobias Oberstein Date: Sat, 13 Dec 2025 23:15:31 +0100 Subject: [PATCH 2/9] Add FlatBuffers/flatc bundling support and smoke tests Add shared flatc infrastructure from wamp-cicd: - Bump .cicd submodule to include new scripts/flatc/ files - Add src/autobahn/_flatc/ module with get_flatc_path() and run_flatc() - Add scripts/smoke_test.py and scripts/smoke_test_flatc.py - Add justfile recipes: test-smoke, test-wheel-install, test-sdist-install This syncs autobahn-python with zlmdb's FlatBuffers bundling approach, using shared code from wamp-cicd/scripts/flatc/ for DRY. Note: This work was completed with AI assistance (Claude Code). --- .cicd | 2 +- justfile | 174 ++++++++++++++++++++++++++++++++ scripts/smoke_test.py | 122 ++++++++++++++++++++++ scripts/smoke_test_flatc.py | 165 ++++++++++++++++++++++++++++++ src/autobahn/_flatc/__init__.py | 112 ++++++++++++++++++++ 5 files changed, 574 insertions(+), 1 deletion(-) create mode 100644 scripts/smoke_test.py create mode 100644 scripts/smoke_test_flatc.py create mode 100644 src/autobahn/_flatc/__init__.py diff --git a/.cicd b/.cicd index 2691c4b90..024df867f 160000 --- a/.cicd +++ b/.cicd @@ -1 +1 @@ -Subproject commit 2691c4b90fc0b25e8c687c33f22265f571156f2c +Subproject commit 024df867fe6154debfcba54cacdd2bc7a7ed691d diff --git a/justfile b/justfile index 9d68dea96..66744c136 100644 --- a/justfile +++ b/justfile @@ -1149,6 +1149,180 @@ test-serdes venv="": (install-tools venv) (install-dev venv) examples/serdes/tests/test_interrupt.py \ examples/serdes/tests/test_eventreceived.py +# ----------------------------------------------------------------------------- +# -- Smoke tests (package verification) +# ----------------------------------------------------------------------------- + +# Run smoke tests on an installed autobahn package (verifies FlatBuffers work) +# This is used by test-wheel-install and test-sdist-install after installation +test-smoke venv="": + #!/usr/bin/env bash + set -e + VENV_NAME="{{ venv }}" + if [ -z "${VENV_NAME}" ]; then + VENV_NAME=$(just --quiet _get-system-venv-name) + fi + VENV_PYTHON=$(just --quiet _get-venv-python "${VENV_NAME}") + VENV_PATH="{{ VENV_DIR }}/${VENV_NAME}" + + echo "Running smoke tests with Python: $(${VENV_PYTHON} --version)" + echo "Venv: ${VENV_PATH}" + echo "" + + # Run the smoke test Python script + ${VENV_PYTHON} "{{ PROJECT_DIR }}/scripts/smoke_test.py" + +# Test installing and verifying a built wheel (used in CI for artifact verification) +# Usage: just test-wheel-install /path/to/autobahn-*.whl +test-wheel-install wheel_path: + #!/usr/bin/env bash + set -e + WHEEL_PATH="{{ wheel_path }}" + + if [ ! -f "${WHEEL_PATH}" ]; then + echo "ERROR: Wheel file not found: ${WHEEL_PATH}" + exit 1 + fi + + WHEEL_NAME=$(basename "${WHEEL_PATH}") + echo "========================================================================" + echo " WHEEL INSTALL TEST" + echo "========================================================================" + echo "" + echo "Wheel: ${WHEEL_NAME}" + echo "" + + # Create ephemeral venv name based on wheel + EPHEMERAL_VENV="smoke-wheel-$$" + EPHEMERAL_PATH="{{ VENV_DIR }}/${EPHEMERAL_VENV}" + + # Extract Python version from wheel filename + # Wheel format: {name}-{version}-{python tag}-{abi tag}-{platform tag}.whl + # Python tag examples: cp312, cp311, pp311, py3 + PYTAG=$(echo "${WHEEL_NAME}" | sed -n 's/.*-\(cp[0-9]*\|pp[0-9]*\|py[0-9]*\)-.*/\1/p') + + if [[ "${PYTAG}" =~ ^cp([0-9])([0-9]+)$ ]]; then + # CPython wheel (e.g., cp312 -> 3.12) + MAJOR="${BASH_REMATCH[1]}" + MINOR="${BASH_REMATCH[2]}" + PYTHON_SPEC="cpython-${MAJOR}.${MINOR}" + echo "Detected CPython ${MAJOR}.${MINOR} wheel" + elif [[ "${PYTAG}" =~ ^pp([0-9])([0-9]+)$ ]]; then + # PyPy wheel (e.g., pp311 -> pypy-3.11) + MAJOR="${BASH_REMATCH[1]}" + MINOR="${BASH_REMATCH[2]}" + PYTHON_SPEC="pypy-${MAJOR}.${MINOR}" + echo "Detected PyPy ${MAJOR}.${MINOR} wheel" + elif [[ "${PYTAG}" =~ ^py([0-9])$ ]]; then + # Pure Python wheel (e.g., py3) - use system Python + SYSTEM_VERSION=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')") + PYTHON_SPEC="cpython-${SYSTEM_VERSION}" + echo "Pure Python wheel, using system Python ${SYSTEM_VERSION}" + else + # Fallback to system Python + SYSTEM_VERSION=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')") + PYTHON_SPEC="cpython-${SYSTEM_VERSION}" + echo "Could not detect Python version from wheel, using system Python ${SYSTEM_VERSION}" + fi + + echo "Creating ephemeral venv with ${PYTHON_SPEC}..." + + mkdir -p "{{ VENV_DIR }}" + uv venv --seed --python "${PYTHON_SPEC}" "${EPHEMERAL_PATH}" + + EPHEMERAL_PYTHON="${EPHEMERAL_PATH}/bin/python3" + + # Install the wheel + echo "" + echo "Installing wheel..." + ${EPHEMERAL_PYTHON} -m pip install "${WHEEL_PATH}" + + # Run smoke tests + echo "" + VENV_DIR="{{ VENV_DIR }}" just test-smoke "${EPHEMERAL_VENV}" + + # Cleanup + echo "" + echo "Cleaning up ephemeral venv..." + rm -rf "${EPHEMERAL_PATH}" + + echo "" + echo "========================================================================" + echo "WHEEL INSTALL TEST PASSED: ${WHEEL_NAME}" + echo "========================================================================" + +# Test installing and verifying a source distribution (used in CI for artifact verification) +# Usage: just test-sdist-install /path/to/autobahn-*.tar.gz +test-sdist-install sdist_path: + #!/usr/bin/env bash + set -e + SDIST_PATH="{{ sdist_path }}" + + if [ ! -f "${SDIST_PATH}" ]; then + echo "ERROR: Source distribution not found: ${SDIST_PATH}" + exit 1 + fi + + SDIST_NAME=$(basename "${SDIST_PATH}") + echo "========================================================================" + echo " SOURCE DISTRIBUTION INSTALL TEST" + echo "========================================================================" + echo "" + echo "Source dist: ${SDIST_NAME}" + echo "" + + # Check if cmake is available (required for flatc build) + if command -v cmake >/dev/null 2>&1; then + echo "cmake: $(cmake --version | head -1)" + else + echo "WARNING: cmake not found - flatc binary will not be built" + echo " Install cmake for full functionality" + fi + echo "" + + # Create ephemeral venv name + EPHEMERAL_VENV="smoke-sdist-$$" + EPHEMERAL_PATH="{{ VENV_DIR }}/${EPHEMERAL_VENV}" + + echo "Creating ephemeral venv: ${EPHEMERAL_VENV}..." + + # Detect system Python version and create venv + SYSTEM_VERSION=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')") + ENV_NAME="cpy$(echo ${SYSTEM_VERSION} | tr -d '.')" + PYTHON_SPEC="cpython-${SYSTEM_VERSION}" + + mkdir -p "{{ VENV_DIR }}" + uv venv --seed --python "${PYTHON_SPEC}" "${EPHEMERAL_PATH}" + + EPHEMERAL_PYTHON="${EPHEMERAL_PATH}/bin/python3" + + # Install build dependencies (required for --no-build-isolation) + echo "" + echo "Installing build dependencies..." + ${EPHEMERAL_PYTHON} -m pip install --no-cache-dir setuptools wheel hatchling + + # Install from source distribution + # Use --no-build-isolation to allow access to system cmake for building flatc + # Use --no-cache-dir to disable HTTP download cache + # Use --no-binary autobahn to force building from source (disable wheel cache) + echo "" + echo "Installing from source distribution..." + ${EPHEMERAL_PYTHON} -m pip install --no-build-isolation --no-cache-dir --no-binary autobahn "${SDIST_PATH}" + + # Run smoke tests + echo "" + VENV_DIR="{{ VENV_DIR }}" just test-smoke "${EPHEMERAL_VENV}" + + # Cleanup + echo "" + echo "Cleaning up ephemeral venv..." + rm -rf "${EPHEMERAL_PATH}" + + echo "" + echo "========================================================================" + echo "SOURCE DISTRIBUTION INSTALL TEST PASSED: ${SDIST_NAME}" + echo "========================================================================" + # ----------------------------------------------------------------------------- # -- Documentation # ----------------------------------------------------------------------------- diff --git a/scripts/smoke_test.py b/scripts/smoke_test.py new file mode 100644 index 000000000..064630e69 --- /dev/null +++ b/scripts/smoke_test.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# Copyright (c) typedef int GmbH, Germany, 2025. All rights reserved. +# +# Smoke tests for autobahn package verification. +# Used by CI to verify wheels and sdists actually work after building. + +""" +Smoke tests for autobahn package. + +This script verifies that an autobahn installation is functional by testing: +1. Import autobahn and check version +2. Import autobahn.websocket (WebSocket protocol) +3. Import autobahn.wamp (WAMP protocol) +4. Import autobahn.flatbuffers and check version (shared test) +5. Verify flatc binary is available and executable (shared test) +6. Verify reflection files are present (shared test) + +ALL TESTS ARE REQUIRED. Both wheel installs and sdist installs MUST +provide identical functionality including the flatc binary and +reflection.bfbs file. + +Note: FlatBuffers/flatc tests use shared functions from smoke_test_flatc.py + (source: wamp-cicd/scripts/flatc/smoke_test_flatc.py) +""" + +import sys + +# Import shared FlatBuffers test functions +from smoke_test_flatc import ( + test_import_flatbuffers, + test_flatc_binary, + test_reflection_files, +) + +# Package name for shared tests +PACKAGE_NAME = "autobahn" + + +def test_import_autobahn(): + """Test 1: Import autobahn and check version.""" + print("Test 1: Importing autobahn and checking version...") + try: + import autobahn + print(f" autobahn version: {autobahn.__version__}") + print(" PASS") + return True + except Exception as e: + print(f" FAIL: Could not import autobahn: {e}") + return False + + +def test_import_websocket(): + """Test 2: Import autobahn.websocket (WebSocket protocol).""" + print("Test 2: Importing autobahn.websocket...") + try: + from autobahn.websocket import protocol + print(" WebSocket protocol module loaded") + print(" PASS") + return True + except Exception as e: + print(f" FAIL: Could not import autobahn.websocket: {e}") + return False + + +def test_import_wamp(): + """Test 3: Import autobahn.wamp (WAMP protocol).""" + print("Test 3: Importing autobahn.wamp...") + try: + from autobahn.wamp import types + print(" WAMP types module loaded") + print(" PASS") + return True + except Exception as e: + print(f" FAIL: Could not import autobahn.wamp: {e}") + return False + + +def main(): + """Run all smoke tests.""" + print("=" * 72) + print(" SMOKE TESTS - Verifying autobahn installation") + print("=" * 72) + print() + print(f"Python: {sys.version}") + print() + + # All tests are REQUIRED - sdist MUST provide same functionality as wheels + # Tests 4-6 use shared functions from smoke_test_flatc.py + tests = [ + ("Test 1", test_import_autobahn), + ("Test 2", test_import_websocket), + ("Test 3", test_import_wamp), + ("Test 4", lambda: test_import_flatbuffers(PACKAGE_NAME)), + ("Test 5", lambda: test_flatc_binary(PACKAGE_NAME)), + ("Test 6", lambda: test_reflection_files(PACKAGE_NAME)), + ] + + failures = 0 + passed = 0 + + for name, test in tests: + result = test() + if result is True: + passed += 1 + else: + failures += 1 + print() + + total = len(tests) + print("=" * 72) + if failures == 0: + print(f"ALL SMOKE TESTS PASSED ({passed}/{total})") + print("=" * 72) + return 0 + else: + print(f"SMOKE TESTS FAILED ({passed} passed, {failures} failed)") + print("=" * 72) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/smoke_test_flatc.py b/scripts/smoke_test_flatc.py new file mode 100644 index 000000000..b2b7499da --- /dev/null +++ b/scripts/smoke_test_flatc.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +############################################################################### +# +# The MIT License (MIT) +# +# Copyright (c) typedef int GmbH +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +############################################################################### +""" +Shared smoke tests for FlatBuffers and flatc bundling verification. + +This module provides reusable test functions for verifying that FlatBuffers +runtime and flatc compiler are correctly bundled in a package. + +Note: This file is shared across WAMP ecosystem projects via wamp-cicd. + Source: wamp-cicd/scripts/flatc/smoke_test_flatc.py + Projects copy this file to: scripts/smoke_test_flatc.py + Then import and call from their main smoke_test.py + +Usage in project's smoke_test.py:: + + from smoke_test_flatc import ( + test_import_flatbuffers, + test_flatc_binary, + test_reflection_files, + ) + + # In main test list: + tests = [ + ..., + lambda: test_import_flatbuffers("zlmdb"), + lambda: test_flatc_binary("zlmdb"), + lambda: test_reflection_files("zlmdb"), + ] +""" + +import os +import subprocess +from pathlib import Path + + +def test_import_flatbuffers(package_name: str) -> bool: + """ + Test importing the vendored flatbuffers module and check version. + + :param package_name: Name of the package (e.g., "zlmdb", "autobahn") + :returns: True if test passed, False otherwise + """ + print(f"Test: Importing {package_name}.flatbuffers...") + try: + import importlib + flatbuffers = importlib.import_module(f"{package_name}.flatbuffers") + print(f" FlatBuffers version: {flatbuffers.__version__}") + print(" PASS") + return True + except Exception as e: + print(f" FAIL: Could not import {package_name}.flatbuffers: {e}") + return False + + +def test_flatc_binary(package_name: str) -> bool: + """ + Test that flatc binary is available and works. + + This is a REQUIRED test - both wheel and sdist installs MUST include flatc. + + :param package_name: Name of the package (e.g., "zlmdb", "autobahn") + :returns: True if test passed, False otherwise + """ + print("Test: Checking flatc binary...") + try: + import importlib + _flatc = importlib.import_module(f"{package_name}._flatc") + get_flatc_path = _flatc.get_flatc_path + + flatc_path = get_flatc_path() + print(f" flatc path: {flatc_path}") + + if not os.path.isfile(flatc_path): + print(" FAIL: flatc binary not found") + print(" This is a packaging bug - flatc MUST be included") + return False + + if not os.access(flatc_path, os.X_OK): + print(" FAIL: flatc exists but not executable") + return False + + # Try running flatc --version + result = subprocess.run( + [str(flatc_path), "--version"], + capture_output=True, + text=True, + timeout=10 + ) + version_output = result.stdout.strip() or result.stderr.strip() + print(f" flatc version: {version_output}") + + if "flatc" in version_output.lower(): + print(" PASS") + return True + else: + print(" FAIL: flatc --version returned unexpected output") + return False + except ImportError as e: + print(f" FAIL: {package_name}._flatc module not available: {e}") + return False + except Exception as e: + print(f" FAIL: Unexpected error: {e}") + return False + + +def test_reflection_files(package_name: str) -> bool: + """ + Test that FlatBuffers reflection files are present. + + This is a REQUIRED test - both wheel and sdist installs MUST include reflection files. + + :param package_name: Name of the package (e.g., "zlmdb", "autobahn") + :returns: True if test passed, False otherwise + """ + print("Test: Checking FlatBuffers reflection files...") + try: + import importlib + flatbuffers = importlib.import_module(f"{package_name}.flatbuffers") + + fbs_dir = Path(flatbuffers.__file__).parent + fbs_file = fbs_dir / "reflection.fbs" + bfbs_file = fbs_dir / "reflection.bfbs" + + # reflection.fbs MUST be present + if not fbs_file.exists(): + print(f" FAIL: reflection.fbs not found at {fbs_file}") + return False + print(f" reflection.fbs: {fbs_file.stat().st_size} bytes") + + # reflection.bfbs MUST be present (generated by flatc during build) + if not bfbs_file.exists(): + print(f" FAIL: reflection.bfbs not found at {bfbs_file}") + print(" This is a packaging bug - reflection.bfbs MUST be included") + return False + + print(f" reflection.bfbs: {bfbs_file.stat().st_size} bytes") + print(" PASS") + return True + except Exception as e: + print(f" FAIL: Reflection files check failed: {e}") + return False diff --git a/src/autobahn/_flatc/__init__.py b/src/autobahn/_flatc/__init__.py new file mode 100644 index 000000000..a0734f27e --- /dev/null +++ b/src/autobahn/_flatc/__init__.py @@ -0,0 +1,112 @@ +############################################################################### +# +# The MIT License (MIT) +# +# Copyright (c) typedef int GmbH +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +############################################################################### +""" +FlatBuffers compiler (flatc) bundled with the package. + +This module provides access to the flatc binary that is bundled with the package, +ensuring version compatibility with the vendored FlatBuffers runtime. + +Usage from command line:: + + flatc --version + flatc --python -o output/ schema.fbs + +Usage from Python:: + + from ._flatc import get_flatc_path, run_flatc + + # Get path to flatc binary + flatc_path = get_flatc_path() + + # Run flatc with arguments + returncode = run_flatc(["--version"]) + +Note: This file is shared across WAMP ecosystem projects via wamp-cicd. + Source: wamp-cicd/scripts/flatc/_flatc.py + Projects copy this file to: src//_flatc/__init__.py +""" + +import os +import subprocess +import sys +from pathlib import Path +from typing import List, Optional + + +def get_flatc_path() -> Path: + """ + Get the path to the bundled flatc executable. + + :returns: Path to the flatc binary + :raises FileNotFoundError: If flatc binary is not found + """ + exe_name = "flatc.exe" if os.name == "nt" else "flatc" + flatc_path = Path(__file__).parent / "bin" / exe_name + + if not flatc_path.exists(): + # Determine package name from module path for helpful error message + module_parts = __name__.split(".") + pkg_name = module_parts[0] if module_parts else "package" + raise FileNotFoundError( + f"flatc binary not found at {flatc_path}. " + f"This may indicate a corrupted installation. " + f"Try reinstalling: pip install --force-reinstall {pkg_name}" + ) + + return flatc_path + + +def run_flatc(args: List[str], cwd: Optional[str] = None) -> int: + """ + Run the bundled flatc with the given arguments. + + :param args: Command line arguments to pass to flatc + :param cwd: Working directory for flatc execution + :returns: Return code from flatc + """ + flatc_path = get_flatc_path() + return subprocess.call([str(flatc_path)] + args, cwd=cwd) + + +def main() -> None: + """ + Entry point for the flatc console script. + + Forwards all CLI arguments to the bundled flatc binary. + """ + try: + flatc_path = get_flatc_path() + except FileNotFoundError as e: + print(str(e), file=sys.stderr) + raise SystemExit(1) + + # Forward all CLI arguments to flatc + ret = subprocess.call([str(flatc_path)] + sys.argv[1:]) + raise SystemExit(ret) + + +if __name__ == "__main__": + main() From 42c4f2b29ff4398c184629cbbc69e3ddbe8a9ec1 Mon Sep 17 00:00:00 2001 From: Tobias Oberstein Date: Sun, 14 Dec 2025 01:40:05 +0100 Subject: [PATCH 3/9] Add complete FlatBuffers vendoring with bundled flatc compiler - hatch_build.py: Add _build_flatc(), _generate_reflection_bfbs(), and _update_flatbuffers_git_version() methods for wheel builds - pyproject.toml: Add flatc console script entry point - flatbuffers/__init__.py: Add version() function and _git_version import - flatbuffers/_git_version.py: Git version tracking file - flatbuffers/reflection.fbs: Copy reflection schema for runtime access - scripts/update_flatbuffers.sh: External script for updating vendored flatbuffers - justfile: Simplify update-flatbuffers recipe, add test-bundled-flatc - .gitignore: Exclude flatc binary and reflection.bfbs build artifacts The wheel now includes: - Bundled flatc compiler built from deps/flatbuffers - reflection.bfbs generated during build - Git version tracking for vendored runtime Tested: wheel install, sdist install, smoke tests all pass Note: This work was completed with AI assistance (Claude Code). --- .gitignore | 6 + hatch_build.py | 334 ++++++++++++++++++++++- justfile | 83 +++++- pyproject.toml | 1 + scripts/update_flatbuffers.sh | 121 ++++++++ src/autobahn/flatbuffers/__init__.py | 41 +++ src/autobahn/flatbuffers/_git_version.py | 24 ++ src/autobahn/flatbuffers/reflection.fbs | 156 +++++++++++ 8 files changed, 748 insertions(+), 18 deletions(-) create mode 100755 scripts/update_flatbuffers.sh create mode 100644 src/autobahn/flatbuffers/_git_version.py create mode 100644 src/autobahn/flatbuffers/reflection.fbs diff --git a/.gitignore b/.gitignore index 9856f54be..eeda4ad41 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,12 @@ node.key src/autobahn/nvx/_nvx_*.c src/_nvx_*.c +# Build artifact: flatc binary (compiled from deps/flatbuffers during wheel build) +src/autobahn/_flatc/bin/ + +# Build artifact: reflection.bfbs (generated from reflection.fbs during wheel build) +src/autobahn/flatbuffers/reflection.bfbs + .wheels get-pip.py docs/autoapi/ diff --git a/hatch_build.py b/hatch_build.py index 9995e9da3..2e093d174 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -1,23 +1,27 @@ """ -Hatchling custom build hook for CFFI extension modules. +Hatchling custom build hook for CFFI extension modules and flatc compiler. -This builds the NVX (Native Vector Extensions) for WebSocket frame masking -and UTF-8 validation using CFFI. +This builds: +1. The NVX (Native Vector Extensions) for WebSocket frame masking and UTF-8 validation +2. The FlatBuffers compiler (flatc) from deps/flatbuffers +3. The reflection.bfbs binary schema for runtime introspection See: https://hatch.pypa.io/latest/plugins/build-hook/custom/ """ +import importlib.util import os +import shutil +import subprocess import sys import sysconfig -import importlib.util from pathlib import Path from hatchling.builders.hooks.plugin.interface import BuildHookInterface class CFfiBuildHook(BuildHookInterface): - """Build hook for compiling CFFI extension modules.""" + """Build hook for compiling CFFI extension modules and flatc compiler.""" PLUGIN_NAME = "cffi" @@ -25,26 +29,36 @@ def initialize(self, version, build_data): """ Called before each build. - For wheel builds, compile the CFFI modules. + For wheel builds, compile the CFFI modules and flatc. For sdist builds, just ensure source files are included. """ + # Always capture flatbuffers git version (for both wheel and sdist) + self._update_flatbuffers_git_version() + if self.target_name != "wheel": # Only compile for wheel builds, sdist just includes source return + built_nvx = False + built_flatc = False + # Check if NVX build is disabled - if os.environ.get("AUTOBAHN_USE_NVX", "1") in ("0", "false"): + if os.environ.get("AUTOBAHN_USE_NVX", "1") not in ("0", "false"): + # Build CFFI modules (NVX) + built_nvx = self._build_cffi_modules(build_data) + else: print("AUTOBAHN_USE_NVX is disabled, skipping CFFI build") - return - # Build CFFI modules - built_extensions = self._build_cffi_modules(build_data) + # Build flatc compiler + built_flatc = self._build_flatc(build_data) + + # Generate reflection.bfbs using the built flatc + if built_flatc: + self._generate_reflection_bfbs(build_data) - # If we built extensions, mark this as a platform-specific wheel - if built_extensions: - # Setting infer_tag tells hatchling to use platform-specific wheel tag + # If we built any extensions, mark this as a platform-specific wheel + if built_nvx or built_flatc: build_data["infer_tag"] = True - # Mark as having binary extensions build_data["pure_python"] = False def _get_ext_suffix(self): @@ -128,3 +142,295 @@ def _build_cffi_modules(self, build_data): traceback.print_exc() return built_any + + def _build_flatc(self, build_data): + """Build the FlatBuffers compiler (flatc) from deps/flatbuffers. + + Returns True if flatc was successfully built. + """ + print("\n" + "=" * 70) + print("Building FlatBuffers compiler (flatc)") + print("=" * 70) + + flatbuffers_dir = Path(self.root) / "deps" / "flatbuffers" + build_dir = flatbuffers_dir / "build" + flatc_bin_dir = Path(self.root) / "src" / "autobahn" / "_flatc" / "bin" + + # Determine executable name based on platform + exe_name = "flatc.exe" if os.name == "nt" else "flatc" + + # Check if cmake is available + cmake_path = shutil.which("cmake") + if not cmake_path: + print("WARNING: cmake not found, skipping flatc build") + print(" -> Install cmake to enable flatc bundling") + return False + + # Check if flatbuffers source exists + if not flatbuffers_dir.exists(): + print(f"WARNING: {flatbuffers_dir} not found") + print(" -> Initialize git submodule: git submodule update --init") + return False + + # Clean and create build directory (remove any cached cmake config) + if build_dir.exists(): + shutil.rmtree(build_dir) + build_dir.mkdir(parents=True, exist_ok=True) + + # Step 1: Configure with cmake + print(" -> Configuring with cmake...") + cmake_args = [ + cmake_path, + "..", + "-DCMAKE_BUILD_TYPE=Release", + "-DFLATBUFFERS_BUILD_TESTS=OFF", + "-DFLATBUFFERS_BUILD_FLATLIB=OFF", + "-DFLATBUFFERS_BUILD_FLATHASH=OFF", + "-DFLATBUFFERS_BUILD_GRPCTEST=OFF", + "-DFLATBUFFERS_BUILD_SHAREDLIB=OFF", + ] + + # ==================================================================== + # Note on manylinux compatibility: + # ==================================================================== + # For manylinux-compatible Linux wheels, flatc must be built inside + # official PyPA manylinux containers (e.g., manylinux_2_28_x86_64). + # These containers have toolchains pre-configured for the correct + # glibc and ISA requirements. No special compiler flags needed. + # + # The wheels-docker.yml and wheels-arm64.yml workflows handle Linux + # builds using these containers. This hatch_build.py works correctly + # in those environments without any ISA-specific flags. + # + # macOS and Windows builds use native GitHub runners (wheels.yml). + # ==================================================================== + + result = subprocess.run( + cmake_args, + cwd=build_dir, + capture_output=True, + text=True, + ) + if result.returncode != 0: + print(f"ERROR: cmake configure failed:\n{result.stderr}") + return False + + # Step 2: Build flatc + print(" -> Building flatc...") + build_args = [ + cmake_path, + "--build", + ".", + "--config", + "Release", + "--target", + "flatc", + ] + + result = subprocess.run( + build_args, + cwd=build_dir, + capture_output=True, + text=True, + ) + if result.returncode != 0: + print(f"ERROR: cmake build failed:\n{result.stderr}") + return False + + # Step 3: Find and copy the built flatc + # flatc might be in different locations depending on platform/generator + possible_locations = [ + build_dir / exe_name, + build_dir / "Release" / exe_name, # Windows/MSVC + build_dir / "Debug" / exe_name, + ] + + flatc_src = None + for loc in possible_locations: + if loc.exists(): + flatc_src = loc + break + + if not flatc_src: + print(f"ERROR: Built flatc not found in {build_dir}") + for loc in possible_locations: + print(f" Checked: {loc}") + return False + + # Copy flatc to package bin directory + flatc_bin_dir.mkdir(parents=True, exist_ok=True) + flatc_dest = flatc_bin_dir / exe_name + shutil.copy2(flatc_src, flatc_dest) + + # Make executable on Unix + if os.name != "nt": + flatc_dest.chmod(0o755) + + print(f" -> Built flatc: {flatc_dest}") + + # Verify ISA level on Linux (check for x86_64_v2 instructions) + if sys.platform.startswith("linux"): + print(" -> Verifying ISA level...") + readelf_result = subprocess.run( + ["readelf", "-A", str(flatc_dest)], + capture_output=True, + text=True, + ) + if readelf_result.returncode == 0: + # Look for ISA info in output + for line in readelf_result.stdout.splitlines(): + if "ISA" in line or "x86" in line.lower(): + print(f" {line.strip()}") + # Also check file command for architecture info + file_result = subprocess.run( + ["file", str(flatc_dest)], + capture_output=True, + text=True, + ) + if file_result.returncode == 0: + print(f" {file_result.stdout.strip()}") + + # Add flatc to wheel + src_file = str(flatc_dest) + dest_path = f"autobahn/_flatc/bin/{exe_name}" + build_data["force_include"][src_file] = dest_path + print(f" -> Added to wheel: {dest_path}") + + # Store flatc path for later use (reflection.bfbs generation) + self._flatc_path = flatc_dest + return True + + def _generate_reflection_bfbs(self, build_data): + """Generate reflection.bfbs using the built flatc. + + This creates the binary FlatBuffers schema that allows runtime + schema introspection. + """ + print("\n" + "=" * 70) + print("Generating reflection.bfbs") + print("=" * 70) + + if not hasattr(self, "_flatc_path") or not self._flatc_path.exists(): + print("WARNING: flatc not available, skipping reflection.bfbs generation") + return False + + flatbuffers_dir = Path(self.root) / "deps" / "flatbuffers" + reflection_fbs = flatbuffers_dir / "reflection" / "reflection.fbs" + output_dir = Path(self.root) / "src" / "autobahn" / "flatbuffers" + + if not reflection_fbs.exists(): + print(f"WARNING: {reflection_fbs} not found") + return False + + # Generate reflection.bfbs + result = subprocess.run( + [ + str(self._flatc_path), + "--binary", + "--schema", + "--bfbs-comments", + "--bfbs-builtins", + "-o", + str(output_dir), + str(reflection_fbs), + ], + capture_output=True, + text=True, + ) + + if result.returncode != 0: + print(f"ERROR: flatc failed:\n{result.stderr}") + return False + + reflection_bfbs = output_dir / "reflection.bfbs" + if reflection_bfbs.exists(): + print(f" -> Generated: {reflection_bfbs}") + + # Add to wheel + src_file = str(reflection_bfbs) + dest_path = "autobahn/flatbuffers/reflection.bfbs" + build_data["force_include"][src_file] = dest_path + print(f" -> Added to wheel: {dest_path}") + return True + else: + print("WARNING: reflection.bfbs not generated") + return False + + def _update_flatbuffers_git_version(self): + """ + Capture the git describe version of deps/flatbuffers submodule. + + This writes the version to flatbuffers/_git_version.py so that + autobahn.flatbuffers.version() returns the exact git version at runtime. + """ + print("=" * 70) + print("Capturing FlatBuffers git version from deps/flatbuffers") + print("=" * 70) + + flatbuffers_dir = Path(self.root) / "deps" / "flatbuffers" + git_version_file = ( + Path(self.root) / "src" / "autobahn" / "flatbuffers" / "_git_version.py" + ) + + # Default version if git is not available or submodule not initialized + git_version = "unknown" + + if flatbuffers_dir.exists() and (flatbuffers_dir / ".git").exists(): + try: + result = subprocess.run( + ["git", "describe", "--tags", "--always"], + cwd=flatbuffers_dir, + capture_output=True, + text=True, + timeout=10, + ) + if result.returncode == 0: + git_version = result.stdout.strip() + print(f" -> Git version: {git_version}") + else: + print(f" -> git describe failed: {result.stderr}") + except FileNotFoundError: + print(" -> git command not found, using existing version") + # Keep existing version in file if git not available + return + except subprocess.TimeoutExpired: + print(" -> git describe timed out, using existing version") + return + except Exception as e: + print(f" -> Error getting git version: {e}") + return + else: + print(" -> deps/flatbuffers not found or not a git repo") + print(f" -> Using existing version in {git_version_file.name}") + return + + # Write the version file + content = """\ +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Git version from deps/flatbuffers submodule. +# This file is regenerated at build time by hatch_build.py. +# The version is captured via `git describe --tags` in the submodule. +# +# Format: "v25.9.23" (tagged release) or "v25.9.23-2-g95053e6a" (post-tag) +# +# If building from sdist without git, this will retain the version +# from when the sdist was created. + +__git_version__ = "{version}" +""".format(version=git_version) + + git_version_file.write_text(content) + print(f" -> Updated {git_version_file.name}") diff --git a/justfile b/justfile index 66744c136..409a6209b 100644 --- a/justfile +++ b/justfile @@ -1172,6 +1172,84 @@ test-smoke venv="": # Run the smoke test Python script ${VENV_PYTHON} "{{ PROJECT_DIR }}/scripts/smoke_test.py" +# Test bundled flatc compiler and Python API +test-bundled-flatc venv="": (install venv) + #!/usr/bin/env bash + set -e + VENV_NAME="{{ venv }}" + if [ -z "${VENV_NAME}" ]; then + VENV_NAME=$(just --quiet _get-system-venv-name) + fi + VENV_PATH="{{ VENV_DIR }}/${VENV_NAME}" + VENV_PYTHON=$(just --quiet _get-venv-python "${VENV_NAME}") + echo "==> Testing bundled flatc compiler in ${VENV_NAME}..." + echo "" + + # Test 1: flatc console script works + echo "Test 1: Verifying 'flatc --version' works via console script..." + FLATC_VERSION=$("${VENV_PATH}/bin/flatc" --version 2>&1) + if [ $? -eq 0 ]; then + echo " PASS: flatc console script works" + echo " Version: ${FLATC_VERSION}" + else + echo " FAIL: flatc console script failed" + exit 1 + fi + echo "" + + # Test 2: Python API get_flatc_path() works + echo "Test 2: Verifying autobahn._flatc.get_flatc_path() works..." + FLATC_PATH=$(${VENV_PYTHON} -c "from autobahn._flatc import get_flatc_path; print(get_flatc_path())") + if [ -x "${FLATC_PATH}" ]; then + echo " PASS: get_flatc_path() returns executable path" + echo " Path: ${FLATC_PATH}" + else + echo " FAIL: get_flatc_path() returned non-executable: ${FLATC_PATH}" + exit 1 + fi + echo "" + + # Test 3: Python API run_flatc() works + echo "Test 3: Verifying autobahn._flatc.run_flatc() works..." + RET=$(${VENV_PYTHON} -c "from autobahn._flatc import run_flatc; exit(run_flatc(['--version']))") + if [ $? -eq 0 ]; then + echo " PASS: run_flatc(['--version']) works" + else + echo " FAIL: run_flatc() failed" + exit 1 + fi + echo "" + + # Test 4: reflection.fbs is accessible + echo "Test 4: Verifying reflection.fbs is accessible at runtime..." + FBS_PATH=$(${VENV_PYTHON} -c 'import autobahn.flatbuffers; from pathlib import Path; p = Path(autobahn.flatbuffers.__file__).parent / "reflection.fbs"; print(p) if p.exists() else exit(1)') + if [ $? -eq 0 ]; then + FBS_SIZE=$(stat -c%s "${FBS_PATH}" 2>/dev/null || stat -f%z "${FBS_PATH}") + echo " PASS: reflection.fbs found at ${FBS_PATH}" + echo " Size: ${FBS_SIZE} bytes" + else + echo " FAIL: reflection.fbs not found" + exit 1 + fi + echo "" + + # Test 5: reflection.bfbs is accessible + echo "Test 5: Verifying reflection.bfbs is accessible at runtime..." + BFBS_PATH=$(${VENV_PYTHON} -c 'import autobahn.flatbuffers; from pathlib import Path; p = Path(autobahn.flatbuffers.__file__).parent / "reflection.bfbs"; print(p) if p.exists() else exit(1)') + if [ $? -eq 0 ]; then + BFBS_SIZE=$(stat -c%s "${BFBS_PATH}" 2>/dev/null || stat -f%z "${BFBS_PATH}") + echo " PASS: reflection.bfbs found at ${BFBS_PATH}" + echo " Size: ${BFBS_SIZE} bytes" + else + echo " FAIL: reflection.bfbs not found" + exit 1 + fi + echo "" + + echo "========================================================================" + echo "ALL BUNDLED FLATC TESTS PASSED" + echo "========================================================================" + # Test installing and verifying a built wheel (used in CI for artifact verification) # Usage: just test-wheel-install /path/to/autobahn-*.whl test-wheel-install wheel_path: @@ -1460,10 +1538,7 @@ bump-flatbuffers: # Update vendored flatbuffers Python runtime from git submodule update-flatbuffers: - echo "==> Updating vendored flatbuffers from submodule..." - rm -rf ./src/autobahn/flatbuffers - cp -R deps/flatbuffers/python/flatbuffers ./src/autobahn/flatbuffers - echo "✓ Flatbuffers vendor updated in src/autobahn/flatbuffers" + ./scripts/update_flatbuffers.sh # Build wheel only (usage: `just build cpy314`) build venv="": (install-build-tools venv) diff --git a/pyproject.toml b/pyproject.toml index 185dce654..c8fd15fb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -210,6 +210,7 @@ asyncio = [] [project.scripts] wamp = "autobahn.__main__:_main" +flatc = "autobahn._flatc:main" # Hatchling build configuration [tool.hatch.build.targets.wheel] diff --git a/scripts/update_flatbuffers.sh b/scripts/update_flatbuffers.sh new file mode 100755 index 000000000..30793ce82 --- /dev/null +++ b/scripts/update_flatbuffers.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +############################################################################### +# +# The MIT License (MIT) +# +# Copyright (c) typedef int GmbH +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +############################################################################### +# +# Update vendored flatbuffers Python runtime from deps/flatbuffers submodule. +# +# This script: +# 1. Copies the Python runtime from the submodule +# 2. Adds _git_version.py for version tracking +# 3. Copies reflection.fbs for runtime schema access +# 4. Patches __init__.py with version() function +# 5. Captures the git version from submodule +# +# Usage: ./scripts/update_flatbuffers.sh +# +############################################################################### + +set -e + +echo "==> Updating vendored flatbuffers from submodule..." + +# 1. Remove old vendored flatbuffers +rm -rf ./src/autobahn/flatbuffers + +# 2. Copy the Python runtime from submodule +cp -R deps/flatbuffers/python/flatbuffers ./src/autobahn/flatbuffers + +# 3. Add _git_version.py template (will be updated during build) +cat > ./src/autobahn/flatbuffers/_git_version.py << 'EOF' +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Git version from deps/flatbuffers submodule. +# This file is regenerated at build time by hatch_build.py. +# The version is captured via `git describe --tags` in the submodule. +# +# Format: "v25.9.23" (tagged release) or "v25.9.23-2-g95053e6a" (post-tag) +# +# If building from sdist without git, this will retain the version +# from when the sdist was created. + +__git_version__ = "unknown" +EOF + +# 4. Copy reflection.fbs for runtime schema access +cp deps/flatbuffers/reflection/reflection.fbs ./src/autobahn/flatbuffers/ + +# 5. Patch __init__.py to add version() function and _git_version import +cat >> ./src/autobahn/flatbuffers/__init__.py << 'EOF' + +# --- Autobahn additions for bundled flatc support --- +import re +from ._git_version import __git_version__ + + +def version() -> tuple[int, int, int, int | None, str | None]: + """ + Return the exact git version of the vendored FlatBuffers runtime. + + Handles: + + 1. "v25.9.23" -> (25, 9, 23, None, None) # Release (Named Tag, CalVer Year.Month.Day) + 2. "v25.9.23-71" -> (25, 9, 23, 71, None) # 71 commits ahead of the Release v25.9.23 + 3. "v25.9.23-71-g19b2300f" -> (25, 9, 23, 71, "19b2300f") # dito, with Git commit hash + """ + pattern = r"^v(\d+)\.(\d+)\.(\d+)(?:-(\d+))?(?:-g([0-9a-f]+))?$" + match = re.match(pattern, __git_version__) + if match: + major, minor, patch, commits, commit_hash = match.groups() + commits_int = int(commits) if commits else None + return (int(major), int(minor), int(patch), commits_int, commit_hash) + return (0, 0, 0, None, None) +EOF + +# 6. Capture current git version from submodule +if [ -d deps/flatbuffers/.git ]; then + GIT_VERSION=$(cd deps/flatbuffers && git describe --tags --always 2>/dev/null || echo "unknown") + sed -i "s/__git_version__ = \"unknown\"/__git_version__ = \"${GIT_VERSION}\"/" ./src/autobahn/flatbuffers/_git_version.py + echo " Git version captured: ${GIT_VERSION}" +fi + +echo "Flatbuffers vendor updated in src/autobahn/flatbuffers" +echo "" +echo "Files added/updated:" +echo " - _git_version.py (git version tracking)" +echo " - reflection.fbs (schema for reflection)" +echo " - __init__.py (patched with version() function)" diff --git a/src/autobahn/flatbuffers/__init__.py b/src/autobahn/flatbuffers/__init__.py index 55ef9377c..7da47918d 100644 --- a/src/autobahn/flatbuffers/__init__.py +++ b/src/autobahn/flatbuffers/__init__.py @@ -12,8 +12,49 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re + from . import util +from ._git_version import __git_version__ from ._version import __version__ from .builder import Builder from .compat import range_func as compat_range from .table import Table + + +def version() -> tuple[int, int, int, int | None, str | None]: + """ + Return the exact git version of the vendored FlatBuffers runtime. + + Handles: + + 1. "v25.9.23" -> (25, 9, 23, None, None) # Release (Named Tag, CalVer Year.Month.Day) + 2. "v25.9.23-71" -> (25, 9, 23, 71, None) # 71 commits ahead of the Release v25.9.23 + 3. "v25.9.23-71-g19b2300f" -> (25, 9, 23, 71, "19b2300f") # dito, with Git commit hash + """ + + # Pattern explanation: + # ^v : Start of string, literal 'v' + # (\d+)\.(\d+)\.(\d+) : Groups 1,2,3 - Major.Minor.Patch (Required) + # + # (?: ... )? : Non-capturing group (grouping only, not saved), optional '?' + # -(\d+) : Literal hyphen, Group 4 (Commits) + # + # (?: ... )? : Non-capturing group, optional '?' + # -g : Literal hyphen and 'g' (separator) + # ([0-9a-f]+) : Group 5 (Hash) + + pattern = r"^v(\d+)\.(\d+)\.(\d+)(?:-(\d+))?(?:-g([0-9a-f]+))?$" + + match = re.match(pattern, __git_version__) + + if match: + major, minor, patch, commits, commit_hash = match.groups() + + # Convert commits to int if present, else None + commits_int = int(commits) if commits else None + + return (int(major), int(minor), int(patch), commits_int, commit_hash) + + # Fallback if regex fails entirely (returns 0.0.0 to satisfy type hint) + return (0, 0, 0, None, None) diff --git a/src/autobahn/flatbuffers/_git_version.py b/src/autobahn/flatbuffers/_git_version.py new file mode 100644 index 000000000..535dc9e27 --- /dev/null +++ b/src/autobahn/flatbuffers/_git_version.py @@ -0,0 +1,24 @@ +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Git version from deps/flatbuffers submodule. +# This file is regenerated at build time by hatch_build.py. +# The version is captured via `git describe --tags` in the submodule. +# +# Format: "v25.9.23" (tagged release) or "v25.9.23-2-g95053e6a" (post-tag) +# +# If building from sdist without git, this will retain the version +# from when the sdist was created. + +__git_version__ = "v25.9.23" diff --git a/src/autobahn/flatbuffers/reflection.fbs b/src/autobahn/flatbuffers/reflection.fbs new file mode 100644 index 000000000..985b45160 --- /dev/null +++ b/src/autobahn/flatbuffers/reflection.fbs @@ -0,0 +1,156 @@ +// This schema defines objects that represent a parsed schema, like +// the binary version of a .fbs file. +// This could be used to operate on unknown FlatBuffers at runtime. +// It can even ... represent itself (!) + +namespace reflection; + +// These must correspond to the enum in idl.h. +enum BaseType : byte { + None, + UType, + Bool, + Byte, + UByte, + Short, + UShort, + Int, + UInt, + Long, + ULong, + Float, + Double, + String, + Vector, + Obj, // Used for tables & structs. + Union, + Array, + Vector64, + + // Add any new type above this value. + MaxBaseType +} + +table Type { + base_type:BaseType; + element:BaseType = None; // Only if base_type == Vector + // or base_type == Array. + index:int = -1; // If base_type == Object, index into "objects" below. + // If base_type == Union, UnionType, or integral derived + // from an enum, index into "enums" below. + // If base_type == Vector && element == Union or UnionType. + fixed_length:uint16 = 0; // Only if base_type == Array. + /// The size (octets) of the `base_type` field. + base_size:uint = 4; // 4 Is a common size due to offsets being that size. + /// The size (octets) of the `element` field, if present. + element_size:uint = 0; +} + +table KeyValue { + key:string (required, key); + value:string; +} + +table EnumVal { + name:string (required); + value:long (key); + object:Object (deprecated); + union_type:Type; + documentation:[string]; + attributes:[KeyValue]; +} + +table Enum { + name:string (required, key); + values:[EnumVal] (required); // In order of their values. + is_union:bool = false; + underlying_type:Type (required); + attributes:[KeyValue]; + documentation:[string]; + /// File that this Enum is declared in. + declaration_file: string; +} + +table Field { + name:string (required, key); + type:Type (required); + id:ushort; + offset:ushort; // Offset into the vtable for tables, or into the struct. + default_integer:long = 0; + default_real:double = 0.0; + deprecated:bool = false; + required:bool = false; + key:bool = false; + attributes:[KeyValue]; + documentation:[string]; + optional:bool = false; + /// Number of padding octets to always add after this field. Structs only. + padding:uint16 = 0; + /// If the field uses 64-bit offsets. + offset64:bool = false; +} + +table Object { // Used for both tables and structs. + name:string (required, key); + fields:[Field] (required); // Sorted. + is_struct:bool = false; + minalign:int; + bytesize:int; // For structs. + attributes:[KeyValue]; + documentation:[string]; + /// File that this Object is declared in. + declaration_file: string; +} + +table RPCCall { + name:string (required, key); + request:Object (required); // must be a table (not a struct) + response:Object (required); // must be a table (not a struct) + attributes:[KeyValue]; + documentation:[string]; +} + +table Service { + name:string (required, key); + calls:[RPCCall]; + attributes:[KeyValue]; + documentation:[string]; + /// File that this Service is declared in. + declaration_file: string; +} + +/// New schema language features that are not supported by old code generators. +enum AdvancedFeatures : ulong (bit_flags) { + AdvancedArrayFeatures, + AdvancedUnionFeatures, + OptionalScalars, + DefaultVectorsAndStrings, +} + +/// File specific information. +/// Symbols declared within a file may be recovered by iterating over all +/// symbols and examining the `declaration_file` field. +table SchemaFile { + /// Filename, relative to project root. + filename:string (required, key); + /// Names of included files, relative to project root. + included_filenames:[string]; +} + +table Schema { + objects:[Object] (required); // Sorted. + enums:[Enum] (required); // Sorted. + file_ident:string; + file_ext:string; + root_table:Object; + services:[Service]; // Sorted. + advanced_features:AdvancedFeatures; + /// All the files used in this compilation. Files are relative to where + /// flatc was invoked. + fbs_files:[SchemaFile]; // Sorted. +} + +root_type Schema; + +file_identifier "BFBS"; +file_extension "bfbs"; From 19ab6733138f77c21e922f5828ebe70d4e3fbb9d Mon Sep 17 00:00:00 2001 From: Tobias Oberstein Date: Sun, 14 Dec 2025 02:13:00 +0100 Subject: [PATCH 4/9] Fix manylinux ISA compatibility: use manylinux_2_28 instead of manylinux_2_34 The manylinux_2_34 container uses a toolchain that defaults to x86_64_v2 ISA level (SSE4.2, POPCNT), which causes auditwheel to fail with: ValueError: Cannot repair the wheel, because required library "flatc" could not be located. x86_64_v2 (ISA level) is not compatible with manylinux platform tag's (x86_64) requirement. The manylinux_2_28 container uses baseline x86_64 ISA which is compatible with all manylinux platform tags. This matches zlmdb which already uses manylinux_2_28 successfully. Note: This work was completed with AI assistance (Claude Code). --- .github/workflows/wheels-docker.yml | 35 ++++++++--------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/wheels-docker.yml b/.github/workflows/wheels-docker.yml index 385314ba3..166184c8c 100644 --- a/.github/workflows/wheels-docker.yml +++ b/.github/workflows/wheels-docker.yml @@ -45,22 +45,14 @@ jobs: fail-fast: false matrix: target: - # manylinux_2_34 (glibc 2.34+) - PEP 600 compliant for modern Linux + # manylinux_2_28 (glibc 2.28+) - PEP 600 compliant, wide compatibility + # IMPORTANT: manylinux_2_28 uses baseline x86_64 ISA level (no SSE4.2/POPCNT) + # which is required for auditwheel to repair binaries like flatc. + # manylinux_2_34 uses x86_64_v2 which causes auditwheel failures. + # Matches zlmdb which also uses manylinux_2_28. # see: https://github.com/pypa/manylinux - - name: "manylinux_2_34_x86_64" - base_image: "quay.io/pypa/manylinux_2_34_x86_64" - - # Future manylinux images can be added here: - # - name: "manylinux_2_34_aarch64" - # base_image: "quay.io/pypa/manylinux_2_34_aarch64" - - # Deactivated for now - focusing on standard manylinux wheels: - # - name: "debian12-amd64" - # base_image: "debian:12" - # - name: "rocky9-amd64" - # base_image: "rockylinux:9" - # - name: "ubuntu2404-amd64" - # base_image: "ubuntu:24.04" + - name: "manylinux_2_28_x86_64" + base_image: "quay.io/pypa/manylinux_2_28_x86_64" steps: - name: Checkout code @@ -197,22 +189,13 @@ jobs: command -v auditwheel >/dev/null || { echo "auditwheel missing, aborting"; exit 1; } mkdir -p wheelhouse - # Convert linux_x86_64 wheels to multi-platform tag (incl. manylinux_2_34_x86_64) using auditwheel + # Convert linux_x86_64 wheels to manylinux_2_28_x86_64 using auditwheel echo "" - echo "==> Converting wheels to multi-platform tag (incl. manylinux_2_34_x86_64) format..." + echo "==> Converting wheels to manylinux_2_28_x86_64 format..." for wheel in dist/*.whl; do if [[ "$wheel" == *"linux_x86_64"* ]]; then echo "Converting: $(basename $wheel)" - - # show autodetected set of platform tags auditwheel show "$wheel" - - # fix/rename wheel based on autodetected platform tags and store in wheelhouse/ - # -> auditwheel will encode all (!) supported tags into the wheel filename - # -> Python packaging ecosystem allows wheels to carry multiple platform tags - # -> pip will then pick the most specific tag that works for the current system - # -> so we do not need to manually set: --plat manylinux_2_34_x86_64 as in: - # auditwheel repair "$wheel" --plat manylinux_2_34_x86_64 -w wheelhouse/ auditwheel repair "$wheel" -w wheelhouse/ else echo "Copying non-linux wheel: $(basename $wheel)" From d36f7eae48e3f8ef0a351004d662eb830a4156c5 Mon Sep 17 00:00:00 2001 From: Tobias Oberstein Date: Sun, 14 Dec 2025 03:15:08 +0100 Subject: [PATCH 5/9] Increase ARM64 build timeout to 30 minutes CFFI compilation under QEMU ARM64 emulation is very slow. CPython builds can take 20+ minutes due to emulation overhead. This matches zlmdb which already uses 30 minute timeouts. Note: This work was completed with AI assistance (Claude Code). --- .github/workflows/wheels-arm64.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels-arm64.yml b/.github/workflows/wheels-arm64.yml index 6eb7f6af6..3c2b725fe 100644 --- a/.github/workflows/wheels-arm64.yml +++ b/.github/workflows/wheels-arm64.yml @@ -130,7 +130,9 @@ jobs: if: matrix.target.build_type == 'custom' uses: nick-fields/retry@v3 with: - timeout_minutes: 10 + # CFFI compilation under QEMU ARM64 emulation is very slow + # CPython builds can take 20+ minutes due to emulation overhead + timeout_minutes: 30 max_attempts: 3 retry_on: error warning_on_retry: true @@ -153,7 +155,9 @@ jobs: - name: Build ARM64 wheels with NVX extension (with retry for QEMU flakiness) uses: nick-fields/retry@v3 with: - timeout_minutes: 15 + # CFFI compilation under QEMU ARM64 emulation is very slow + # CPython builds can take 20+ minutes due to emulation overhead + timeout_minutes: 30 max_attempts: 3 retry_on: error warning_on_retry: true From 04a54a466ba72bf0fc8623ae51a5a3c9a05db437 Mon Sep 17 00:00:00 2001 From: Tobias Oberstein Date: Sun, 14 Dec 2025 13:10:04 +0100 Subject: [PATCH 6/9] Increase ARM64 build timeout to 60 minutes CFFI and flatc compilation under QEMU ARM64 emulation is very slow. Builds can take 45+ minutes due to emulation overhead. Note: This work was completed with AI assistance (Claude Code). --- .github/workflows/wheels-arm64.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/wheels-arm64.yml b/.github/workflows/wheels-arm64.yml index 3c2b725fe..4f27c7c68 100644 --- a/.github/workflows/wheels-arm64.yml +++ b/.github/workflows/wheels-arm64.yml @@ -130,9 +130,9 @@ jobs: if: matrix.target.build_type == 'custom' uses: nick-fields/retry@v3 with: - # CFFI compilation under QEMU ARM64 emulation is very slow - # CPython builds can take 20+ minutes due to emulation overhead - timeout_minutes: 30 + # CFFI and flatc compilation under QEMU ARM64 emulation is very slow + # Builds can take 45+ minutes due to emulation overhead + timeout_minutes: 60 max_attempts: 3 retry_on: error warning_on_retry: true @@ -155,9 +155,9 @@ jobs: - name: Build ARM64 wheels with NVX extension (with retry for QEMU flakiness) uses: nick-fields/retry@v3 with: - # CFFI compilation under QEMU ARM64 emulation is very slow - # CPython builds can take 20+ minutes due to emulation overhead - timeout_minutes: 30 + # CFFI and flatc compilation under QEMU ARM64 emulation is very slow + # Builds can take 45+ minutes due to emulation overhead + timeout_minutes: 60 max_attempts: 3 retry_on: error warning_on_retry: true From 8aa1e292df9249b1561b0e63c8b8843ac4a0aac5 Mon Sep 17 00:00:00 2001 From: Tobias Oberstein Date: Mon, 15 Dec 2025 00:58:24 +0100 Subject: [PATCH 7/9] Update release.yml to use manylinux_2_28 artifact names The wheels-docker workflow was changed to use manylinux_2_28 instead of manylinux_2_34 for ISA compatibility. The release workflow must use the matching artifact names. Note: This work was completed with AI assistance (Claude Code). --- .github/workflows/release.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d390455f8..47a2963e2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -137,7 +137,7 @@ jobs: // Query artifacts from wheels-docker workflow const wheelsDockerRunId = latestRuns['wheels-docker']?.id; - core.setOutput('artifact_manylinux_x86_64', await findArtifact(wheelsDockerRunId, 'artifacts-manylinux_2_34_x86_64')); + core.setOutput('artifact_manylinux_x86_64', await findArtifact(wheelsDockerRunId, 'artifacts-manylinux_2_28_x86_64')); // Query artifacts from wheels-arm64 workflow const wheelsArm64RunId = latestRuns['wheels-arm64']?.id; @@ -813,20 +813,20 @@ jobs: mode: strict keep-metadata: true # Keep CHECKSUMS for user verification targets: | - cpy311-linux-x86_64-manylinux_2_34 + cpy311-linux-x86_64-manylinux_2_28 cpy311-linux-aarch64-manylinux_2_28 cpy311-win-amd64 - cpy312-linux-x86_64-manylinux_2_34 + cpy312-linux-x86_64-manylinux_2_28 cpy312-win-amd64 cpy313-macos-arm64 - cpy313-linux-x86_64-manylinux_2_34 + cpy313-linux-x86_64-manylinux_2_28 cpy313-linux-aarch64-manylinux_2_28 cpy313-win-amd64 cpy314-macos-arm64 - cpy314-linux-x86_64-manylinux_2_34 + cpy314-linux-x86_64-manylinux_2_28 cpy314-win-amd64 pypy311-macos-arm64 - pypy311-linux-x86_64-manylinux_2_34 + pypy311-linux-x86_64-manylinux_2_28 pypy311-linux-aarch64-manylinux_2_17 pypy311-win-amd64 source @@ -1642,20 +1642,20 @@ jobs: mode: strict keep-metadata: true # Keep CHECKSUMS for user verification targets: | - cpy311-linux-x86_64-manylinux_2_34 + cpy311-linux-x86_64-manylinux_2_28 cpy311-linux-aarch64-manylinux_2_28 cpy311-win-amd64 - cpy312-linux-x86_64-manylinux_2_34 + cpy312-linux-x86_64-manylinux_2_28 cpy312-win-amd64 cpy313-macos-arm64 - cpy313-linux-x86_64-manylinux_2_34 + cpy313-linux-x86_64-manylinux_2_28 cpy313-linux-aarch64-manylinux_2_28 cpy313-win-amd64 cpy314-macos-arm64 - cpy314-linux-x86_64-manylinux_2_34 + cpy314-linux-x86_64-manylinux_2_28 cpy314-win-amd64 pypy311-macos-arm64 - pypy311-linux-x86_64-manylinux_2_34 + pypy311-linux-x86_64-manylinux_2_28 pypy311-linux-aarch64-manylinux_2_17 pypy311-win-amd64 source @@ -2262,20 +2262,20 @@ jobs: mode: strict # keep-metadata: false (default - removes CHECKSUMS for PyPI) targets: | - cpy311-linux-x86_64-manylinux_2_34 + cpy311-linux-x86_64-manylinux_2_28 cpy311-linux-aarch64-manylinux_2_28 cpy311-win-amd64 - cpy312-linux-x86_64-manylinux_2_34 + cpy312-linux-x86_64-manylinux_2_28 cpy312-win-amd64 cpy313-macos-arm64 - cpy313-linux-x86_64-manylinux_2_34 + cpy313-linux-x86_64-manylinux_2_28 cpy313-linux-aarch64-manylinux_2_28 cpy313-win-amd64 cpy314-macos-arm64 - cpy314-linux-x86_64-manylinux_2_34 + cpy314-linux-x86_64-manylinux_2_28 cpy314-win-amd64 pypy311-macos-arm64 - pypy311-linux-x86_64-manylinux_2_34 + pypy311-linux-x86_64-manylinux_2_28 pypy311-linux-aarch64-manylinux_2_17 pypy311-win-amd64 source From 3a1eadb2507d65391359893795d070a5f4f9794c Mon Sep 17 00:00:00 2001 From: Tobias Oberstein Date: Mon, 15 Dec 2025 01:24:49 +0100 Subject: [PATCH 8/9] Add workflows README, WAMP .bfbs generation, and version sync check 1. Add .github/workflows/README.md documenting the CI/CD architecture, artifact flow, and platform coverage (adapted from zlmdb) 2. Add _generate_wamp_bfbs() to hatch_build.py to compile wamp.bfbs from the WAMP FlatBuffers schemas during wheel build. This binary schema file enables runtime schema introspection for WAMP messages. 3. Add check_zlmdb_flatbuffers_version_in_sync() to autobahn/__init__.py This is the reciprocal of zlmdb.check_autobahn_flatbuffers_version_in_sync() and verifies both libraries use the same vendored FlatBuffers runtime for reliable data-in-transit/data-at-rest interoperability. 4. Add wamp.bfbs to .gitignore as it's a build artifact Note: This work was completed with AI assistance (Claude Code). --- .github/workflows/README.md | 260 ++++++++++++++++++++++++++++++++++++ .gitignore | 3 + hatch_build.py | 62 +++++++++ src/autobahn/__init__.py | 38 ++++++ 4 files changed, 363 insertions(+) create mode 100644 .github/workflows/README.md diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 000000000..702210129 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,260 @@ +# GitHub Actions Workflows + +This document describes the CI/CD workflow architecture for autobahn-python, +including artifact production and consumption flow. + +## Workflow Overview + +| Workflow | Purpose | Trigger | +|----------|---------|---------| +| `main.yml` | Code quality, tests, documentation | Push, PR | +| `wstest.yml` | WebSocket conformance testing | Push, PR | +| `wheels.yml` | macOS/Windows wheels + source dist | Push, PR, tags | +| `wheels-docker.yml` | Linux x86_64 manylinux wheels | Push, PR, tags | +| `wheels-arm64.yml` | Linux ARM64 manylinux wheels | Push, PR, tags | +| `release.yml` | Collect artifacts, publish releases | workflow_run | + +## Artifact Flow Diagram + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ ARTIFACT PRODUCERS │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ main.yml │ +│ └── documentation (docs HTML) │ +│ │ +│ wstest.yml │ +│ └── wstest-results (WebSocket conformance reports) │ +│ │ +│ wheels.yml (native GitHub runners) │ +│ ├── wheels-macos-arm64 (macOS ARM64 wheels) │ +│ ├── wheels-windows-x86_64 (Windows x64 wheels) │ +│ ├── linux-wheels-no-nvx (Linux pure Python wheels) │ +│ └── source-distribution (*.tar.gz sdist) │ +│ │ +│ wheels-docker.yml (manylinux_2_28_x86_64 container) │ +│ └── artifacts-manylinux_2_28_x86_64 (Linux x64 wheels with NVX+flatc) │ +│ │ +│ wheels-arm64.yml (manylinux_2_28_aarch64 container) │ +│ ├── artifacts-arm64-cpython-3.11-manylinux_2_28_aarch64 │ +│ ├── artifacts-arm64-cpython-3.13-manylinux_2_28_aarch64 │ +│ ├── artifacts-arm64-pypy-3.11-bookworm-manylinux_2_36_aarch64 │ +│ └── artifacts-arm64-pypy-3.11-trixie-manylinux_2_38_aarch64 │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ ARTIFACT CONSUMER │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ release.yml │ +│ ├── Downloads all artifacts from above workflows │ +│ ├── Creates GitHub Release (development or stable) │ +│ └── Publishes to PyPI (on tags only) │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +## Binary Wheel Features + +autobahn-python binary wheels include: + +1. **NVX Extension** - Native Rust-based acceleration for WebSocket/WAMP +2. **Bundled flatc** - FlatBuffers compiler for schema compilation +3. **reflection.bfbs** - Pre-compiled FlatBuffers reflection schema + +## wheels.yml Step Execution Order + +The `wheels.yml` workflow follows a deliberate 5-phase execution order +with filesystem sync points to ensure artifact integrity in CI environments. + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ PHASE 1: SETUP - Install toolchain (just, uv, rust) │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ ├── Checkout code │ +│ ├── Install Just (Linux/macOS) │ +│ ├── Install Just (Windows) │ +│ ├── Install uv (Linux/macOS) │ +│ ├── Install uv (Windows) │ +│ ├── Install Rust (for NVX extension) │ +│ ├── Verify toolchain installation │ +│ └── Setup uv cache │ +└─────────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ PHASE 2: BUILD - Build wheels/sdist for each platform │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ ├── Build binary wheels with NVX+flatc (macOS) │ +│ ├── Build binary wheels with NVX+flatc (Windows) │ +│ ├── Build pure Python wheels without NVX (Linux) │ +│ └── Build source distribution (Linux x86_64 only) │ +└─────────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ PHASE 3: VALIDATION - Validate artifacts (per-OS with FS sync points) │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ │ +│ │ --- macOS validation --- │ +│ ├── Force file system sync (post-build, pre-validation) - macOS │ +│ ├── Validate wheels integrity (macOS only) │ +│ ├── Generate SHA256 checksums (macOS only) │ +│ ├── Force file system sync (post-checksum) - macOS │ +│ │ │ +│ │ --- Windows validation --- │ +│ ├── Force file system sync (post-build, pre-validation) - Windows │ +│ ├── Validate wheels integrity (Windows only) │ +│ ├── Generate SHA256 checksums (Windows only) │ +│ ├── Force file system sync (post-checksum) - Windows │ +│ │ │ +│ │ --- Linux validation (source distribution) --- │ +│ ├── Force file system sync (post-build, pre-validation) - Linux │ +│ ├── Verify source distribution integrity (Linux x86_64 only) │ +│ ├── Verify source distribution installs and works (Linux x86_64 only) │ +│ ├── Generate SHA256 checksums (Linux x86_64 only) │ +│ └── Force file system sync (post-checksum) - Linux │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ PHASE 4: METADATA - Generate build metadata (after all validations) │ +│ This phase comes AFTER all OS validations/checksums so it can aggregate │ +│ results from VALIDATION.txt and CHECKSUMS.sha256 into build-info.txt │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ ├── Generate build metadata │ +│ ├── Force file system sync (post-metadata) - macOS │ +│ ├── Force file system sync (post-metadata) - Windows │ +│ └── Force file system sync (post-metadata) - Linux │ +└─────────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ PHASE 5: LIST & UPLOAD - List artifacts and upload │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ ├── List built artifacts (macOS) │ +│ ├── List built artifacts (Windows) │ +│ ├── List built artifacts (Linux - source distribution only) │ +│ ├── Upload wheel artifacts (macOS and Windows only) │ +│ └── Upload source distribution (Linux x86_64 only) │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +### Why Filesystem Sync Points? + +CI environments (especially macOS and Windows runners) may buffer filesystem +writes. Without explicit sync points, subsequent steps might read stale data +or incomplete files. The sync points ensure: + +1. **Post-build sync** - All build artifacts are fully written before validation +2. **Post-checksum sync** - Checksum files are complete before metadata generation +3. **Post-metadata sync** - All metadata is written before artifact upload + +### Smoke Test for Source Distribution + +The Linux runner performs a full smoke test of the source distribution: + +1. Creates an ephemeral venv with matching Python version +2. Installs build dependencies (cffi, setuptools, wheel, hatchling, maturin) +3. Installs sdist with `--no-build-isolation --no-cache-dir --no-binary autobahn` +4. Runs required smoke tests: + - Import autobahn and check version + - Import autobahn.flatbuffers and check version + - Verify flatc binary is available and executable + - Verify reflection files are present + +All tests are **required** - sdist installs MUST provide identical +functionality to wheel installs including the flatc binary. + +## Artifact Details + +### 1. Artifact Producers (Upload) + +| Workflow | Artifact Name | Contents | Platform | +|----------|--------------|----------|----------| +| **main.yml** | `documentation` | `docs/_build/html/` | N/A | +| **wstest.yml** | `wstest-results` | WebSocket conformance reports | N/A | +| **wheels.yml** | `wheels-macos-arm64` | macOS ARM64 wheels (cpy311-314, pypy311) | macOS arm64 | +| **wheels.yml** | `wheels-windows-x86_64` | Windows x64 wheels (cpy311-314, pypy311) | Windows x86_64 | +| **wheels.yml** | `linux-wheels-no-nvx` | Pure Python wheels (no NVX) | Linux x86_64 | +| **wheels.yml** | `source-distribution` | `*.tar.gz` sdist | Linux (build host) | +| **wheels-docker.yml** | `artifacts-manylinux_2_28_x86_64` | Linux x64 wheels (see below) | Linux x86_64 | +| **wheels-arm64.yml** | `artifacts-arm64-cpython-3.11-manylinux_2_28_aarch64` | CPython 3.11 wheel | Linux aarch64 | +| **wheels-arm64.yml** | `artifacts-arm64-cpython-3.13-manylinux_2_28_aarch64` | CPython 3.13 wheel | Linux aarch64 | +| **wheels-arm64.yml** | `artifacts-arm64-pypy-3.11-bookworm-manylinux_2_36_aarch64` | PyPy 3.11 wheel (Debian 12) | Linux aarch64 | +| **wheels-arm64.yml** | `artifacts-arm64-pypy-3.11-trixie-manylinux_2_38_aarch64` | PyPy 3.11 wheel (Debian 13) | Linux aarch64 | + +**wheels-docker.yml artifact contents** (`artifacts-manylinux_2_28_x86_64`): +- `cpy311-linux-x86_64-manylinux_2_28` +- `cpy312-linux-x86_64-manylinux_2_28` +- `cpy313-linux-x86_64-manylinux_2_28` +- `cpy314-linux-x86_64-manylinux_2_28` +- `pypy311-linux-x86_64-manylinux_2_28` + +### 2. Artifact Consumer (release.yml) + +The `release.yml` workflow downloads artifacts using the `wamp-cicd` verified +download action. It maps artifact names via the `check-workflows` job outputs: + +| Output Variable | Source Workflow | Artifact Pattern | +|-----------------|-----------------|------------------| +| `artifact_macos_wheels` | wheels.yml | `wheels-macos-arm64` | +| `artifact_windows_wheels` | wheels.yml | `wheels-windows-x86_64` | +| `artifact_source_dist` | wheels.yml | `source-distribution` | +| `artifact_linux_no_nvx` | wheels.yml | `linux-wheels-no-nvx` | +| `artifact_manylinux_x86_64` | wheels-docker.yml | `artifacts-manylinux_2_28_x86_64` | +| `artifact_arm64_cp311` | wheels-arm64.yml | `artifacts-arm64-cpython-3.11-manylinux_2_28_aarch64` | +| `artifact_arm64_cp313` | wheels-arm64.yml | `artifacts-arm64-cpython-3.13-manylinux_2_28_aarch64` | +| `artifact_arm64_pypy_bookworm` | wheels-arm64.yml | `artifacts-arm64-pypy-3.11-bookworm-manylinux_2_36_aarch64` | +| `artifact_arm64_pypy_trixie` | wheels-arm64.yml | `artifacts-arm64-pypy-3.11-trixie-manylinux_2_38_aarch64` | + +## Platform Coverage + +### Wheels Built + +| Platform | Architecture | Python Versions | Manylinux Tag | Workflow | +|----------|--------------|-----------------|---------------|----------| +| Linux | x86_64 | 3.11, 3.12, 3.13, 3.14, PyPy 3.11 | manylinux_2_28 | wheels-docker.yml | +| Linux | aarch64 | 3.11, 3.13 | manylinux_2_28 | wheels-arm64.yml | +| Linux | aarch64 | PyPy 3.11 | manylinux_2_36/2_38 | wheels-arm64.yml | +| macOS | arm64 | 3.11, 3.12, 3.13, 3.14, PyPy 3.11 | N/A | wheels.yml | +| Windows | x86_64 | 3.11, 3.12, 3.13, 3.14, PyPy 3.11 | N/A | wheels.yml | + +### Why Manylinux Containers? + +Linux wheels are built inside official PyPA manylinux containers to ensure: + +1. **ABI Compatibility** - Correct glibc symbol versions for target platforms +2. **ISA Compliance** - Baseline instruction set (no x86_64_v2+ on x86_64) +3. **auditwheel Success** - Clean wheel repair without ISA warnings +4. **Wide Compatibility** - Wheels work on older Linux distributions + +The `manylinux_2_28` tag targets glibc 2.28+ (RHEL 8, Ubuntu 20.04+, Debian 11+). + +**Important**: We use `manylinux_2_28` instead of `manylinux_2_34` because the +latter uses x86_64_v2 instruction set which causes auditwheel to fail when +bundling flatc binaries. + +## Release Process + +1. **On every push/PR**: All wheel workflows run and upload artifacts +2. **On workflow completion**: `release.yml` triggers via `workflow_run` +3. **Development releases**: Created automatically from feature branches +4. **Stable releases**: Created when a `v*` tag is pushed, also publishes to PyPI + +## Maintenance Notes + +When updating Python versions or manylinux tags: + +1. Update the matrix in the relevant workflow file +2. Update artifact name patterns in `release.yml` `check-workflows` job +3. Update this README to reflect changes +4. Test with a PR before merging + +--- + +*This documentation is maintained alongside the workflow files.* diff --git a/.gitignore b/.gitignore index eeda4ad41..4c71c784b 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,9 @@ src/autobahn/_flatc/bin/ # Build artifact: reflection.bfbs (generated from reflection.fbs during wheel build) src/autobahn/flatbuffers/reflection.bfbs +# Build artifact: wamp.bfbs (generated from wamp.fbs during wheel build) +src/autobahn/wamp/flatbuffers/wamp.bfbs + .wheels get-pip.py docs/autoapi/ diff --git a/hatch_build.py b/hatch_build.py index 2e093d174..aef4dd86a 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -55,6 +55,8 @@ def initialize(self, version, build_data): # Generate reflection.bfbs using the built flatc if built_flatc: self._generate_reflection_bfbs(build_data) + # Generate WAMP schema .bfbs files + self._generate_wamp_bfbs(build_data) # If we built any extensions, mark this as a platform-specific wheel if built_nvx or built_flatc: @@ -356,6 +358,66 @@ def _generate_reflection_bfbs(self, build_data): print("WARNING: reflection.bfbs not generated") return False + def _generate_wamp_bfbs(self, build_data): + """Generate .bfbs files for WAMP FlatBuffers schemas. + + This creates binary FlatBuffers schemas for the WAMP protocol schemas + located in src/autobahn/wamp/flatbuffers/. + """ + print("\n" + "=" * 70) + print("Generating WAMP schema .bfbs files") + print("=" * 70) + + if not hasattr(self, "_flatc_path") or not self._flatc_path.exists(): + print("WARNING: flatc not available, skipping WAMP .bfbs generation") + return False + + wamp_fbs_dir = Path(self.root) / "src" / "autobahn" / "wamp" / "flatbuffers" + + if not wamp_fbs_dir.exists(): + print(f"WARNING: {wamp_fbs_dir} not found") + return False + + # The main schema file that includes all others + wamp_fbs = wamp_fbs_dir / "wamp.fbs" + if not wamp_fbs.exists(): + print(f"WARNING: {wamp_fbs} not found") + return False + + # Generate wamp.bfbs (which includes all dependent schemas) + result = subprocess.run( + [ + str(self._flatc_path), + "--binary", + "--schema", + "--bfbs-comments", + "--bfbs-builtins", + "-o", + str(wamp_fbs_dir), + str(wamp_fbs), + ], + capture_output=True, + text=True, + ) + + if result.returncode != 0: + print(f"ERROR: flatc failed:\n{result.stderr}") + return False + + wamp_bfbs = wamp_fbs_dir / "wamp.bfbs" + if wamp_bfbs.exists(): + print(f" -> Generated: {wamp_bfbs}") + + # Add to wheel + src_file = str(wamp_bfbs) + dest_path = "autobahn/wamp/flatbuffers/wamp.bfbs" + build_data["force_include"][src_file] = dest_path + print(f" -> Added to wheel: {dest_path}") + return True + else: + print("WARNING: wamp.bfbs not generated") + return False + def _update_flatbuffers_git_version(self): """ Capture the git describe version of deps/flatbuffers submodule. diff --git a/src/autobahn/__init__.py b/src/autobahn/__init__.py index d59c6aa4e..1b657ab52 100644 --- a/src/autobahn/__init__.py +++ b/src/autobahn/__init__.py @@ -32,6 +32,44 @@ import txaio +from autobahn import flatbuffers + + +def check_zlmdb_flatbuffers_version_in_sync() -> tuple[int, int, int, int | None, str | None]: + """ + Check that autobahn and zlmdb have the same vendored flatbuffers version. + + This is important for applications like Crossbar.io that use both autobahn + (for data-in-transit) and zlmdb (for data-at-rest) with FlatBuffers + serialization. When sending a FlatBuffers database record as a WAMP + application payload, both libraries must use compatible FlatBuffers + runtimes to avoid subtle serialization issues. + + :returns: The flatbuffers version tuple (e.g. (25, 9, 23, 2, "95053e6a")) + if both are in sync. + :raises RuntimeError: If the versions differ. + :raises ImportError: If zlmdb is not installed. + + Example:: + + import autobahn + version = autobahn.check_zlmdb_flatbuffers_version_in_sync() + print(f"FlatBuffers version: {version}") + """ + import zlmdb.flatbuffers + + autobahn_version = flatbuffers.version() + zlmdb_version = zlmdb.flatbuffers.version() + + if autobahn_version != zlmdb_version: + raise RuntimeError( + f"FlatBuffers version mismatch: autobahn has {autobahn_version!r}, " + f"zlmdb has {zlmdb_version!r}. Both should be the same for " + f"reliable data-in-transit/data-at-rest interoperability." + ) + + return autobahn_version + # this is used in the unit tests (trial/pytest), and when already done here, there # is no risk and headaches with finding out if/where an import implies a framework if os.environ.get("USE_TWISTED", False) and os.environ.get("USE_ASYNCIO", False): From da48032beef1ee825badaf2dc4d186821b8c2225 Mon Sep 17 00:00:00 2001 From: Tobias Oberstein Date: Mon, 15 Dec 2025 01:36:11 +0100 Subject: [PATCH 9/9] Rename install-flatc to install-flatc-system with warning, add version check 1. Rename `install-flatc` to `install-flatc-system` with prominent warning explaining that users normally DON'T need this since autobahn bundles flatc in wheels. Adds interactive confirmation prompt. 2. Enhance smoke_test_flatc.py to verify flatc binary version matches the vendored FlatBuffers runtime version. This ensures consistency between the compiler and runtime library. Note: This work was completed with AI assistance (Claude Code). --- justfile | 47 +++++++++++++++++++++++++++++++++---- scripts/smoke_test_flatc.py | 34 +++++++++++++++++++++++---- 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/justfile b/justfile index 409a6209b..4948572b7 100644 --- a/justfile +++ b/justfile @@ -1798,14 +1798,49 @@ publish venv="" tag="": (publish-pypi venv tag) (publish-rtd tag) # -- FlatBuffers Schema Generation # ----------------------------------------------------------------------------- -# Install latest FlatBuffers compiler (flatc) to /usr/local/bin -install-flatc: +# Install FlatBuffers compiler (flatc) to /usr/local/bin (SYSTEM-WIDE) +# +# WARNING: You probably DON'T need this! +# +# autobahn-python bundles flatc in binary wheels and source distributions. +# After installing autobahn, you can use the bundled flatc via: +# +# flatc --version # If installed via pip/wheel +# python -m autobahn._flatc # Alternative invocation +# +# This recipe installs a SEPARATE system-wide flatc binary to /usr/local/bin. +# Only use this if you specifically need a system flatc that is independent +# of your Python environment. +# +install-flatc-system: #!/usr/bin/env bash set -e + + echo "======================================================================" + echo "WARNING: Installing SYSTEM-WIDE flatc to /usr/local/bin" + echo "======================================================================" + echo "" + echo "You probably DON'T need this!" + echo "" + echo "autobahn-python bundles flatc in binary wheels. After 'pip install autobahn':" + echo " - Run 'flatc --version' to use the bundled compiler" + echo " - The bundled flatc version matches the vendored FlatBuffers runtime" + echo "" + echo "This recipe installs a SEPARATE system flatc that may have a different" + echo "version than the bundled one, potentially causing compatibility issues." + echo "" + read -p "DO YOU REALLY WANT TO INSTALL SYSTEM-WIDE FLATC? [y/N] " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "Aborted." + exit 0 + fi + FLATC_VERSION="25.9.23" FLATC_URL="https://github.com/google/flatbuffers/releases/download/v${FLATC_VERSION}/Linux.flatc.binary.g++-13.zip" TEMP_DIR=$(mktemp -d) + echo "" echo "==> Installing FlatBuffers compiler v${FLATC_VERSION}..." echo " URL: ${FLATC_URL}" echo " Temp dir: ${TEMP_DIR}" @@ -1825,8 +1860,12 @@ install-flatc: # Verify installation echo "==> Verification:" - flatc --version - echo "✅ FlatBuffers compiler v${FLATC_VERSION} installed successfully!" + /usr/local/bin/flatc --version + echo "" + echo "✅ System-wide FlatBuffers compiler v${FLATC_VERSION} installed to /usr/local/bin/flatc" + echo "" + echo "NOTE: The bundled flatc in autobahn wheels may have a different path priority." + echo " Use '/usr/local/bin/flatc' explicitly if you need the system version." # Clean generated FlatBuffers files clean-fbs: diff --git a/scripts/smoke_test_flatc.py b/scripts/smoke_test_flatc.py index b2b7499da..f72ace370 100644 --- a/scripts/smoke_test_flatc.py +++ b/scripts/smoke_test_flatc.py @@ -113,12 +113,38 @@ def test_flatc_binary(package_name: str) -> bool: version_output = result.stdout.strip() or result.stderr.strip() print(f" flatc version: {version_output}") - if "flatc" in version_output.lower(): - print(" PASS") - return True - else: + if "flatc" not in version_output.lower(): print(" FAIL: flatc --version returned unexpected output") return False + + # Verify flatc version matches vendored FlatBuffers runtime version + flatbuffers = importlib.import_module(f"{package_name}.flatbuffers") + if hasattr(flatbuffers, "version"): + runtime_version = flatbuffers.version() + print(f" runtime version: {runtime_version}") + + # Extract version from flatc output (e.g., "flatc version 25.9.23") + import re + match = re.search(r"(\d+)\.(\d+)\.(\d+)", version_output) + if match: + flatc_major, flatc_minor, flatc_patch = map(int, match.groups()) + runtime_major, runtime_minor, runtime_patch = runtime_version[:3] + + if (flatc_major, flatc_minor, flatc_patch) == (runtime_major, runtime_minor, runtime_patch): + print(f" version match: flatc {flatc_major}.{flatc_minor}.{flatc_patch} == runtime {runtime_major}.{runtime_minor}.{runtime_patch}") + print(" PASS") + return True + else: + print(f" WARNING: version mismatch - flatc {flatc_major}.{flatc_minor}.{flatc_patch} != runtime {runtime_major}.{runtime_minor}.{runtime_patch}") + print(" PASS (with warning)") + return True # Still pass but warn - versions should match ideally + else: + print(" WARNING: could not parse flatc version for comparison") + print(" PASS") + return True + else: + print(" PASS (version check skipped - no version() function)") + return True except ImportError as e: print(f" FAIL: {package_name}._flatc module not available: {e}") return False