diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 27491b2..65aeb78 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -47,4 +47,4 @@ jobs: run: uv sync --all-extras - name: Run pytest - run: uv run pytest --cov=mitreattack + run: uv run --extra dev pytest -n 2 --cov=mitreattack diff --git a/AGENTS.md b/AGENTS.md index ac820fb..5768d02 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,9 @@ - Before committing, run `just lint`. - `just lint`: run pre-commit hooks across the repo. - `just test`: run the pytest suite. +- `just test-xdist`: run the pytest suite in parallel. - `just test-cov`: run tests with coverage for `mitreattack`. +- `just test-cov-xdist`: run tests with coverage in parallel. - `just build`: build distributions with `uv build`. - Without `just`, run the same tools through `uv run ...`. @@ -32,6 +34,10 @@ - Framework: `pytest` (with `pytest-cov` for coverage checks). - Place tests under `tests/` and name files/functions `test_*.py` / `test_*`. - Add or update tests for behavior changes, especially around STIX parsing and changelog/diff output paths. +- Tests that need real ATT&CK STIX data should use the shared STIX fixtures instead of downloading or + preparing bundles directly. +- Parallel runs warm the shared STIX cache before workers start; update `DEFAULT_ATTACK_STIX_PREP` in + `tests/conftest.py` if a new xdist-backed test needs another ATT&CK release. - Run `just test` locally before opening a PR; use `just test-cov` for larger changes. ## Commit & Pull Request Guidelines diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 3907374..968ab32 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -52,10 +52,17 @@ Run `just` with no arguments to see all available commands. Here are the most co ```bash just lint # Run pre-commit hooks (ruff format) on all files just test # Run tests +just test-xdist # Run tests in parallel just test-cov # Run tests with coverage report +just test-cov-xdist # Run tests with coverage in parallel just build # Build the package ``` +Tests that need real ATT&CK STIX data should use the shared STIX fixtures instead of downloading or +preparing bundles directly. Parallel test runs warm the shared STIX cache before workers start; if a +new xdist-backed test needs an additional ATT&CK release, update the cache warmup list in +`tests/conftest.py`. + ### Pull Requests When making a pull request, please make sure to: diff --git a/justfile b/justfile index 85544c9..eb80af3 100644 --- a/justfile +++ b/justfile @@ -35,6 +35,14 @@ ruff-format: test: uv run pytest +# Run tests in parallel +test-xdist workers="auto": + uv run --extra dev pytest -n {{ workers }} + +# Run tests with coverage in parallel +test-cov-xdist workers="auto": + uv run --extra dev pytest -n {{ workers }} --cov=mitreattack + # Run tests with coverage test-cov: uv run pytest --cov=mitreattack diff --git a/pyproject.toml b/pyproject.toml index d10e1b0..188b860 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ dev = [ "pytest>=8.4.2", "pytest-cov>=7.0.0", "pytest-dotenv>=0.5.2", + "pytest-xdist>=3.8.0", "python-semantic-release>=10.5.0", "responses>=0.25.8", "ruff>=0.14.2", @@ -128,3 +129,7 @@ version_files = [ "docs/conf.py:^release = ['\"](.*)['\"]", "mitreattack/__init__.py:^__version__ = ['\"](.*)['\"]", ] + +[[tool.uv.index]] +url = "https://pypi.org/simple" +default = true diff --git a/tests/changelog/conftest.py b/tests/changelog/conftest.py index 11344a4..d95e3e2 100644 --- a/tests/changelog/conftest.py +++ b/tests/changelog/conftest.py @@ -1594,10 +1594,10 @@ def golden_161_170_changelog_dir(): @pytest.fixture(scope="session") -def generated_161_170_diffstix(tmp_path_factory) -> DiffStix: +def generated_161_170_diffstix() -> DiffStix: """Create and cache a DiffStix instance for reuse across tests.""" versions_param = ["16.1", "17.0"] - result_paths = _download_attack_stix_data(versions_param, tmp_path_factory) + result_paths = _download_attack_stix_data(versions_param) return DiffStix( domains=["enterprise-attack", "mobile-attack", "ics-attack"], old=result_paths["16.1"], diff --git a/tests/conftest.py b/tests/conftest.py index 115ff82..324ab75 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ """Common fixtures and utilities for testing mitreattack-python.""" import os +from pathlib import Path import pytest from loguru import logger @@ -16,6 +17,18 @@ STIX_LOCATION_ENTERPRISE = os.getenv("STIX_LOCATION_ENTERPRISE") STIX_LOCATION_MOBILE = os.getenv("STIX_LOCATION_MOBILE") STIX_LOCATION_ICS = os.getenv("STIX_LOCATION_ICS") +TEST_CACHE_DIR = Path( + os.getenv("MITREATTACK_TEST_CACHE_DIR", Path(__file__).resolve().parent.parent / ".pytest_cache" / "attack-stix") +) +DEFAULT_ATTACK_STIX_PREP = (None, ["16.1", "17.0"]) + + +def pytest_sessionstart(session): + """Warm the ATT&CK STIX cache before xdist test sessions start.""" + config = session.config + if _should_prepare_attack_stix_cache(config): + for versions_param in DEFAULT_ATTACK_STIX_PREP: + _download_attack_stix_data(versions_param, config=config) def _parse_version_param(versions_param): @@ -81,7 +94,42 @@ def _get_stix_file_path(attack_stix_dir, domain, version_key="latest"): return f"{attack_stix_dir[first_version]}/{domain}-attack.json" -def _download_attack_stix_data(versions_param, tmp_path_factory): +def _get_release_dir(download_dir: Path, release: str) -> Path: + """Return the cache directory for a specific ATT&CK release.""" + return download_dir / f"v{release}" + + +def _is_xdist_worker(config=None) -> bool: + """Return whether the current process is an xdist worker.""" + if config is not None and getattr(config, "workerinput", None) is not None: + return True + return bool(os.getenv("PYTEST_XDIST_WORKER")) + + +def _should_prepare_attack_stix_cache(config=None) -> bool: + """Return whether the current pytest process should warm the shared STIX cache.""" + if _is_xdist_worker(config): + return False + + if config is None: + return False + + numprocesses = getattr(getattr(config, "option", None), "numprocesses", None) + return bool(numprocesses and int(numprocesses) > 0) + + +def _stix_bundles_present(download_dir: Path, releases: list[str]) -> bool: + """Check whether all cached STIX bundles needed for a run already exist.""" + domains = ["enterprise", "mobile", "ics"] + for release in releases: + release_dir = _get_release_dir(download_dir, release) + for domain in domains: + if not (release_dir / f"{domain}-attack.json").exists(): + return False + return True + + +def _download_attack_stix_data(versions_param, config=None): """Download ATT&CK STIX data and return paths. This is the core download logic shared by multiple fixtures. @@ -90,8 +138,8 @@ def _download_attack_stix_data(versions_param, tmp_path_factory): ---------- versions_param : None, str, list, or dict Version parameter to parse - tmp_path_factory : pytest.TempPathFactory - Pytest temp path factory + config : pytest.Config, optional + Active pytest config used to determine read-only cache behavior. Returns ------- @@ -99,32 +147,45 @@ def _download_attack_stix_data(versions_param, tmp_path_factory): Dictionary mapping version to download directory path """ versions, stix_version = _parse_version_param(versions_param) + requested_versions = versions or [LATEST_VERSION] - logger.debug(f"Downloading the ATT&CK STIX {stix_version} data for versions: {versions}") - download_dir = tmp_path_factory.mktemp("attack-releases") / f"stix-{stix_version}" + logger.debug(f"Preparing ATT&CK STIX {stix_version} data for versions: {requested_versions}") + download_dir = TEST_CACHE_DIR / f"stix-{stix_version}" + download_dir.mkdir(parents=True, exist_ok=True) - download_domains( - domains=["enterprise", "mobile", "ics"], - download_dir=download_dir, - all_versions=False, - stix_version=stix_version, - attack_versions=versions, - ) + if _stix_bundles_present(download_dir, requested_versions): + logger.debug(f"Reusing cached ATT&CK STIX bundles from {download_dir}") + else: + if _is_xdist_worker(config): + requested = ", ".join(requested_versions) + raise RuntimeError( + "ATT&CK STIX cache is missing required bundles for " + f"{requested}. xdist runs should warm this cache before workers start. " + "If you added a new ATT&CK version to xdist-backed tests, update DEFAULT_ATTACK_STIX_PREP." + ) + logger.debug(f"Downloading ATT&CK STIX bundles into cache at {download_dir}") + download_domains( + domains=["enterprise", "mobile", "ics"], + download_dir=download_dir, + all_versions=False, + stix_version=stix_version, + attack_versions=versions, + ) # Build return dictionary result_paths = {} if versions is None: - result_paths["latest"] = download_dir / f"v{LATEST_VERSION}" + result_paths["latest"] = _get_release_dir(download_dir, LATEST_VERSION) else: # Return paths for each requested version for version in versions: - result_paths[version] = download_dir / f"v{version}" + result_paths[version] = _get_release_dir(download_dir, version) return result_paths -@pytest.fixture(autouse=True, scope="session") -def attack_stix_dir(request, tmp_path_factory): +@pytest.fixture(scope="session") +def attack_stix_dir(request): """Download ATT&CK STIX data and return paths. Can be parametrized to download specific versions: @@ -151,7 +212,7 @@ def attack_stix_dir(request, tmp_path_factory): Directory paths for requested ATT&CK versions """ versions_param = getattr(request, "param", None) - result_paths = _download_attack_stix_data(versions_param, tmp_path_factory) + result_paths = _download_attack_stix_data(versions_param, config=request.config) yield result_paths diff --git a/uv.lock b/uv.lock index 1c851c0..8dda38f 100644 --- a/uv.lock +++ b/uv.lock @@ -429,6 +429,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "filelock" version = "3.24.3" @@ -674,6 +683,7 @@ dev = [ { name = "pytest" }, { name = "pytest-cov" }, { name = "pytest-dotenv" }, + { name = "pytest-xdist" }, { name = "python-semantic-release" }, { name = "responses" }, { name = "ruff" }, @@ -702,6 +712,7 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.4.2" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=7.0.0" }, { name = "pytest-dotenv", marker = "extra == 'dev'", specifier = ">=0.5.2" }, + { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.8.0" }, { name = "python-dateutil", specifier = ">=2.8.2" }, { name = "python-semantic-release", marker = "extra == 'dev'", specifier = ">=10.5.0" }, { name = "requests", specifier = ">=2.31.0" }, @@ -1210,6 +1221,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d0/da/9da67c67b3d0963160e3d2cbc7c38b6fae342670cc8e6d5936644b2cf944/pytest_dotenv-0.5.2-py3-none-any.whl", hash = "sha256:40a2cece120a213898afaa5407673f6bd924b1fa7eafce6bda0e8abffe2f710f", size = 3993, upload-time = "2020-06-16T12:38:01.139Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0"