diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9340e8d353..81a20eb2a9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1057,10 +1057,12 @@ jobs: with: username: ${{ vars.AIRE_DOCKERHUB_USERNAME }} password: ${{ secrets.AIRE_DOCKERHUB_ACCESS_TOKEN }} - - name: Pull mock base images + - name: Pull integration Docker images run: | docker pull nginx:1.29.4-alpine-slim docker pull alpine:3.23 + clickhouse_version="$(tr -d '[:space:]' < services/intake/.clickhouse-version)" + docker pull "clickhouse/clickhouse-server:${clickhouse_version}" shell: bash - name: Run integration tests run: make test-integration-ci diff --git a/Makefile b/Makefile index 3d263f30f5..674b704861 100644 --- a/Makefile +++ b/Makefile @@ -377,10 +377,11 @@ test: test-unit ## Run all Python unit tests (fast tests without infrastructure PYTEST_VERBOSITY := $(if $(filter true,$(CI)),-q,-v) PYTEST_WORKERS ?= auto PYTEST_MAX_WORKERS ?= 16 +PYTEST_MAX_WORKER_RESTART ?= 2 PYTEST_DIST ?= loadscope PYTEST_CMD = env PYTHONWARNINGS="ignore::UserWarning:pytest_only.version" $(UV) run --frozen \ pytest \ - -n $(PYTEST_WORKERS) --maxprocesses=$(PYTEST_MAX_WORKERS) --max-worker-restart=2 \ + -n $(PYTEST_WORKERS) --maxprocesses=$(PYTEST_MAX_WORKERS) --max-worker-restart=$(PYTEST_MAX_WORKER_RESTART) \ --dist $(PYTEST_DIST) --timeout=120 $(PYTEST_VERBOSITY) $(PYTEST_EXTRA) PYTEST_CI_OPTS := --cov=src --cov=packages \ @@ -403,6 +404,7 @@ PYTEST_CI_CMD = timeout --kill-after=60s $(PYTEST_CI_TIMEOUT)s $(PYTEST_CMD) # CI integration runs therefore use ``loadgroup`` while unit runs keep # ``loadscope``. test-integration test-integration-ci: PYTEST_DIST := loadgroup +test-integration-ci: PYTEST_MAX_WORKER_RESTART := 0 .PHONY: test-unit test-unit: ## Run Python unit tests across all packages and services diff --git a/plugins/nemo-evaluator/tests/integration/conftest.py b/plugins/nemo-evaluator/tests/integration/conftest.py index 35741ad639..2ae6eaf1c7 100644 --- a/plugins/nemo-evaluator/tests/integration/conftest.py +++ b/plugins/nemo-evaluator/tests/integration/conftest.py @@ -36,6 +36,7 @@ MOCK_PROVIDER_PREFIX_ENVVAR = "NMP_INFERENCE_GATEWAY_MOCK_PROVIDER_PREFIX" CLICKHOUSE_XDIST_GROUP = "nmp_intake_clickhouse" CLICKHOUSE_XDIST_FIXTURE = "_clickhouse" +CLICKHOUSE_TIMEOUT_SECONDS = 600 #: Base URL (and therefore port) for the agent-eval subprocess-backend platform. Distinct from #: other integration platforms so both can run in the same session without a port clash. @@ -87,6 +88,7 @@ def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: for item in items: if CLICKHOUSE_XDIST_FIXTURE in item.fixturenames: item.add_marker(pytest.mark.xdist_group(CLICKHOUSE_XDIST_GROUP)) + item.add_marker(pytest.mark.timeout(CLICKHOUSE_TIMEOUT_SECONDS)) @contextmanager diff --git a/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py b/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py index e5da9b3b92..13d6093700 100644 --- a/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py +++ b/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py @@ -91,9 +91,11 @@ def _wait_for_tcp(host: str, port: int, *, timeout: float) -> None: raise RuntimeError(f"{host}:{port} not reachable within {timeout}s") -def _wait_for_ready(base_url: str, *, timeout: float) -> None: +def _wait_for_ready(base_url: str, *, timeout: float, process: subprocess.Popen[bytes]) -> None: deadline = time.monotonic() + timeout while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError(f"platform at {base_url} exited early (code {process.returncode}) before ready") try: with urllib.request.urlopen(f"{base_url}/health/ready", timeout=2) as response: # noqa: S310 if response.status == 200: @@ -104,6 +106,12 @@ def _wait_for_ready(base_url: str, *, timeout: float) -> None: raise RuntimeError(f"platform at {base_url} not ready within {timeout}s") +def _port_in_use(host: str, port: int) -> bool: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(1) + return sock.connect_ex((host, port)) == 0 + + @pytest.fixture(scope="session") def _clickhouse(tmp_path_factory: pytest.TempPathFactory) -> Iterator[None]: if not _docker_available(): @@ -136,7 +144,11 @@ def platform_base_url(_clickhouse: None) -> Iterator[str]: # NMP_BASE_URL is client-side only, so without this the suite silently requires 8080 to be free # and cannot run alongside a local dev platform. Mirrors the sibling fixtures in conftest, which # each take their own port for the same reason. - port = urlsplit(BASE_URL).port or 8080 + split = urlsplit(BASE_URL) + host = split.hostname or "localhost" + port = split.port or 8080 + if _port_in_use(host, port): + raise RuntimeError(f"{host}:{port} is already in use; stop other platform instances before running this test") process = subprocess.Popen( ["uv", "run", "nemo", "services", "run", "--services", "auth,entities,intake", "--port", str(port)], cwd=REPO_ROOT, @@ -147,7 +159,7 @@ def platform_base_url(_clickhouse: None) -> Iterator[str]: }, ) try: - _wait_for_ready(BASE_URL, timeout=180) + _wait_for_ready(BASE_URL, timeout=180, process=process) yield BASE_URL finally: process.terminate() diff --git a/services/intake/tests/integration/test_local_clickhouse_provisioning.py b/services/intake/tests/integration/test_local_clickhouse_provisioning.py index 0d098394e4..51be9e23bf 100644 --- a/services/intake/tests/integration/test_local_clickhouse_provisioning.py +++ b/services/intake/tests/integration/test_local_clickhouse_provisioning.py @@ -38,6 +38,7 @@ def _docker_is_available() -> bool: @pytest.mark.integration # Serialize with evaluator integration tests that own the legacy fixed-port container. @pytest.mark.xdist_group("nmp_intake_clickhouse") +@pytest.mark.timeout(600) def test_data_directory_owned_container_uses_dynamic_loopback_port_and_is_reused( tmp_path: Path, ) -> None: