diff --git a/mergify_cli/ci/junit_processing/cli.py b/mergify_cli/ci/junit_processing/cli.py index efc50a2..44cf92a 100644 --- a/mergify_cli/ci/junit_processing/cli.py +++ b/mergify_cli/ci/junit_processing/cli.py @@ -1,6 +1,7 @@ from __future__ import annotations import sys +import typing import click import opentelemetry.trace @@ -10,6 +11,13 @@ from mergify_cli.ci.junit_processing import upload +if typing.TYPE_CHECKING: + from opentelemetry.sdk.trace import ReadableSpan + +SEPARATOR = "══════════════════════════════════════════" +SEPARATOR_LIGHT = "──────────────────────────────────────────" + + async def process_junit_files( *, api_url: str, @@ -20,30 +28,35 @@ async def process_junit_files( tests_target_branch: str, files: tuple[str, ...], ) -> None: - click.echo("🚀 CI Insights · Upload JUnit") - click.echo("────────────────────────────") - click.echo(f"📂 Discovered reports: {len(files)}") + # ── Header ── + click.echo(SEPARATOR) + click.echo(" 🚀 CI Insights") + click.echo("") + click.echo( + " Uploads JUnit test results to Mergify CI Insights and evaluates\n" + " quarantine status for failing tests. This step determines the\n" + " final CI status — quarantined failures are ignored.\n" + " Learn more: https://docs.mergify.com/ci-insights/quarantine", + ) + click.echo(SEPARATOR) try: - spans = await junit.files_to_spans( + run_id, spans = await junit.files_to_spans( files, test_language=test_language, test_framework=test_framework, ) except junit.InvalidJunitXMLError as e: - click.echo( - click.style( - f"❌ Error converting JUnit XML file to spans: {e.details}", - fg="red", - ), - err=True, + _print_early_exit_error( + f"Failed to parse JUnit XML: {e.details}", + "Check that your test framework is generating valid JUnit XML output.", ) sys.exit(1) if not spans: - click.echo( - click.style("❌ No spans found in the JUnit files", fg="red"), - err=True, + _print_early_exit_error( + "No spans found in the JUnit files", + "Check that the JUnit XML files are not empty.", ) sys.exit(1) @@ -54,9 +67,9 @@ async def process_junit_files( ] if not tests_cases: - click.echo( - click.style("❌ No test cases found in the JUnit files", fg="red"), - err=True, + _print_early_exit_error( + "No test cases found in the JUnit files", + "Check that your test step ran successfully before this step.", ) sys.exit(1) @@ -67,55 +80,63 @@ async def process_junit_files( if span.status.status_code == opentelemetry.trace.StatusCode.ERROR ], ) - nb_success_spans = len( - [ - span - for span in tests_cases - if span.status.status_code == opentelemetry.trace.StatusCode.OK - ], - ) - click.echo( - f"🧪 Parsed tests: {len(tests_cases)} (✅ passed: {nb_success_spans} | ❌ failed: {nb_failing_spans})", - ) + click.echo("") + click.echo(f" Run ID: {run_id}") # NOTE: Check quarantine before uploading in order to properly modify the # "cicd.test.quarantined" attribute for the required spans. quarantine_final_failure_message: str | None = None + quarantine_error: str | None = None try: - failing_tests_not_quarantined_count = ( - await quarantine.check_and_update_failing_spans( - api_url, - token, - repository, - tests_target_branch, - spans, - ) + result = await quarantine.check_and_update_failing_spans( + api_url, + token, + repository, + tests_target_branch, + spans, ) except quarantine.QuarantineFailedError as exc: - click.echo(click.style(exc.message, fg="red"), err=True) - click.echo( - click.style(quarantine.QUARANTINE_INFO_ERROR_MSG, fg="red"), - err=True, - ) + quarantine_error = exc.message quarantine_final_failure_message = ( - "Unable to determine quarantined failures due to above error" - ) - except Exception as exc: - msg = ( - f"❌ An unexpected error occurred when checking quarantined tests: {exc!s}" + f"Treating {nb_failing_spans}/{nb_failing_spans} failures as blocking" ) - click.echo(click.style(msg, fg="red"), err=True) - click.echo( - click.style(quarantine.QUARANTINE_INFO_ERROR_MSG, fg="red"), - err=True, + failing_spans = [ + span + for span in tests_cases + if span.status.status_code == opentelemetry.trace.StatusCode.ERROR + ] + result = quarantine.QuarantineResult( + failing_spans=failing_spans, + quarantined_spans=[], + non_quarantined_spans=failing_spans, + failing_tests_not_quarantined_count=len(failing_spans), ) + except Exception as exc: + quarantine_error = str(exc) quarantine_final_failure_message = ( - "Unable to determine quarantined failures due to above error" + f"Treating {nb_failing_spans}/{nb_failing_spans} failures as blocking" + ) + failing_spans = [ + span + for span in tests_cases + if span.status.status_code == opentelemetry.trace.StatusCode.ERROR + ] + result = quarantine.QuarantineResult( + failing_spans=failing_spans, + quarantined_spans=[], + non_quarantined_spans=failing_spans, + failing_tests_not_quarantined_count=len(failing_spans), ) else: - if failing_tests_not_quarantined_count > 0: - quarantine_final_failure_message = f"{failing_tests_not_quarantined_count} unquarantined failures detected ({nb_failing_spans - failing_tests_not_quarantined_count} quarantined)" + if result.failing_tests_not_quarantined_count > 0: + count = result.failing_tests_not_quarantined_count + total = len(result.failing_spans) + quarantined = total - count + quarantine_final_failure_message = ( + f"{quarantined}/{total} failures quarantined" + ) + upload_error: str | None = None try: upload.upload( api_url=api_url, @@ -124,23 +145,142 @@ async def process_junit_files( spans=spans, ) except Exception as e: - click.echo( - click.style(f"❌ Error uploading JUnit XML reports: {e}", fg="red"), - err=True, - ) + upload_error = str(e) + + reports_label = f"{len(files)} {'report' if len(files) == 1 else 'reports'}" + failures_label = ( + f"{nb_failing_spans} {'failure' if nb_failing_spans == 1 else 'failures'}" + ) + if upload_error is not None: + click.echo(f" ☁️ {reports_label} not uploaded") + else: + click.echo(f" ☁️ {reports_label} uploaded") + click.echo(f" 🧪 {len(tests_cases)} tests ({failures_label})") + + # ── Upload error ── + if upload_error is not None: + click.echo("\n ⚠️ Failed to upload test results") + click.echo(" Mergify CI Insights won't process these test results.") + click.echo(" Quarantine status and CI outcome are unaffected.") + click.echo("") + click.echo(" ┌ Details") + for line in upload_error.splitlines(): + click.echo(f" │ {line}") + click.echo(" └─") + # ── Quarantine ── + if _print_quarantine_section(result, error=quarantine_error): + pass + + # ── Verdict ── + nb_quarantined_failures = len(result.failing_spans) if result is not None else 0 + click.echo("") + click.echo(SEPARATOR) if quarantine_final_failure_message is None: - click.echo("\n🎉 Verdict") - click.echo( - f"• Status: ✅ OK — all {nb_failing_spans} failures are quarantined (ignored for CI status)", - ) + if nb_quarantined_failures == 0: + click.echo("✅ OK — all tests passed, no quarantine needed") + else: + click.echo( + f"✅ OK — {nb_quarantined_failures}/{nb_quarantined_failures}" + " failures quarantined, CI status unaffected", + ) quarantine_exit_error_code = 0 else: - click.echo("\n❌ Verdict") click.echo( - f"• Status: 🔴 FAIL — {quarantine_final_failure_message}", + f"❌ FAIL — {quarantine_final_failure_message}", ) quarantine_exit_error_code = 1 - click.echo(f"• Exit code: {quarantine_exit_error_code}") + click.echo(f" Exit code: {quarantine_exit_error_code}") + click.echo(SEPARATOR) sys.exit(quarantine_exit_error_code) + + +def _print_quarantine_section( + result: quarantine.QuarantineResult | None, + *, + error: str | None = None, +) -> bool: + """Print the quarantine section. Returns True if anything was printed.""" + if result is None and error is None: + return False + + if result is not None and not result.failing_spans and error is None: + return False + + click.echo("") + click.echo(SEPARATOR_LIGHT) + click.echo("") + click.echo("🛡️ Quarantine") + + if error is not None: + click.echo("") + click.echo(" ⚠️ Failed to check quarantine status") + click.echo(" Contact Mergify support if this error persists.") + click.echo("") + click.echo(" ┌ Details") + for line in error.splitlines(): + click.echo(f" │ {line}") + click.echo(" └─") + + if result is not None and result.quarantined_spans: + click.echo(f"\n 🔒 Quarantined ({len(result.quarantined_spans)}):") + for span in result.quarantined_spans: + click.echo(f" · {span.name}") + + if result is not None and result.non_quarantined_spans: + label = ( + "Could not verify quarantine status" + if error is not None + else "Unquarantined" + ) + click.echo(f"\n ❌ {label} ({len(result.non_quarantined_spans)}):") + for span in result.non_quarantined_spans: + _print_failure_block(span) + + return True + + +def _print_failure_block(span: ReadableSpan) -> None: + """Print a box-drawn failure block for a single failing test span.""" + click.echo(f"\n ┌ {span.name}") + + if span.attributes is None: + click.echo(" │") + click.echo(" │ (no error details in JUnit report)") + click.echo(" └─") + return + + exc_type = span.attributes.get("exception.type") + exc_message = span.attributes.get("exception.message") + exc_stacktrace = span.attributes.get("exception.stacktrace") + + if not exc_type and not exc_message and not exc_stacktrace: + click.echo(" │") + click.echo(" │ (no error details in JUnit report)") + click.echo(" └─") + return + + if exc_type or exc_message: + parts = [] + if exc_type: + parts.append(str(exc_type)) + if exc_message: + parts.append(str(exc_message)) + click.echo(" │") + click.echo(f" │ {': '.join(parts)}") + + if exc_stacktrace: + click.echo(" │") + for line in str(exc_stacktrace).splitlines(): + click.echo(f" │ {line}") + + click.echo(" └─") + + +def _print_early_exit_error(message: str, hint: str) -> None: + """Print an error section for early exits (invalid XML, no tests, etc.).""" + click.echo(f"❌ FAIL — {message}") + click.echo(f" {hint}") + click.echo(" Exit code: 1") + click.echo(SEPARATOR) diff --git a/mergify_cli/ci/junit_processing/junit.py b/mergify_cli/ci/junit_processing/junit.py index 8777714..380e932 100644 --- a/mergify_cli/ci/junit_processing/junit.py +++ b/mergify_cli/ci/junit_processing/junit.py @@ -18,7 +18,6 @@ from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator import opentelemetry.trace.span -from mergify_cli import console from mergify_cli.ci import detector @@ -37,11 +36,10 @@ async def files_to_spans( files: tuple[str, ...], test_language: str | None = None, test_framework: str | None = None, -) -> list[ReadableSpan]: +) -> tuple[str, list[ReadableSpan]]: spans = [] run_id = ID_GENERATOR.generate_span_id().to_bytes(8, "big").hex() - console.print(f"🛠️ MERGIFY_TEST_RUN_ID={run_id}") for filename in files: spans.extend( @@ -53,7 +51,7 @@ async def files_to_spans( ), ) - return spans + return run_id, spans async def junit_to_spans( diff --git a/mergify_cli/ci/junit_processing/quarantine.py b/mergify_cli/ci/junit_processing/quarantine.py index 368e9af..fc82748 100644 --- a/mergify_cli/ci/junit_processing/quarantine.py +++ b/mergify_cli/ci/junit_processing/quarantine.py @@ -3,7 +3,6 @@ import dataclasses import typing -import click import httpx import opentelemetry.trace import tenacity @@ -20,10 +19,12 @@ class QuarantineFailedError(Exception): message: str -QUARANTINE_INFO_ERROR_MSG = ( - "This error occurred because there are failed tests in your CI pipeline and will disappear once your CI passes successfully.\n\n" - "If you're unsure why this is happening or need assistance, please contact Mergify to report the issue." -) +@dataclasses.dataclass(frozen=True) +class QuarantineResult: + failing_spans: list[ReadableSpan] + quarantined_spans: list[ReadableSpan] + non_quarantined_spans: list[ReadableSpan] + failing_tests_not_quarantined_count: int async def check_and_update_failing_spans( @@ -32,18 +33,14 @@ async def check_and_update_failing_spans( repository: str, tests_target_branch: str, spans: list[ReadableSpan], -) -> int: +) -> QuarantineResult: """ Check all the `spans` with CI Insights Quarantine by: - - logging the failed and quarantined test - - logging the failed and non-quarantined test as error message - updating the `spans` of quarantined tests by setting the attribute `cicd.test.quarantined` to `true` - Returns the number of failing tests that are not quarantined. + Returns a QuarantineResult with the categorized spans. """ - click.echo("\n🛡️ Quarantine") - failing_spans = [ span for span in spans @@ -52,13 +49,15 @@ async def check_and_update_failing_spans( and span.attributes.get("test.scope") == "case" ] - failing_spans_name = [fspan.name for fspan in failing_spans] if not failing_spans: - click.echo( - "• No quarantine check required since no failed tests were detected", + return QuarantineResult( + failing_spans=[], + quarantined_spans=[], + non_quarantined_spans=[], + failing_tests_not_quarantined_count=0, ) - return 0 + failing_spans_name = [fspan.name for fspan in failing_spans] failing_tests_not_quarantined_count: int = 0 quarantined_tests_tuple = await fetch_quarantined_tests_from_failing_spans( api_url, @@ -85,32 +84,24 @@ async def check_and_update_failing_spans( ): failing_tests_not_quarantined_count += 1 - quarantined_tests_spans = [ + quarantined_spans = [ span for span in failing_spans if span.name in quarantined_tests_tuple.quarantined_tests_names ] - non_quarantined_tests_spans = [ + non_quarantined_spans = [ span for span in failing_spans if span.name in quarantined_tests_tuple.non_quarantined_tests_names ] - click.echo( - f"• Quarantined failures matched: {len(quarantined_tests_spans)}/{len(failing_spans)}", + return QuarantineResult( + failing_spans=failing_spans, + quarantined_spans=quarantined_spans, + non_quarantined_spans=non_quarantined_spans, + failing_tests_not_quarantined_count=failing_tests_not_quarantined_count, ) - if quarantined_tests_spans: - click.echo(" - 🔒 Quarantined:") - for qt_span in quarantined_tests_spans: - click.echo(f" · {qt_span.name}") - - if non_quarantined_tests_spans: - click.echo(" - ❌ Unquarantined:") - for nqt_span in non_quarantined_tests_spans: - click.echo(f" · {nqt_span.name}") - - return failing_tests_not_quarantined_count class QuarantinedTests(typing.NamedTuple): @@ -149,7 +140,7 @@ async def fetch_quarantined_tests_from_failing_spans( if response.status_code != 200: raise QuarantineFailedError( - message=f"HTTP error {response.status_code} while checking quarantined tests: {response.text}", + message=f"HTTP {response.status_code}: {response.text}", ) return QuarantinedTests( diff --git a/mergify_cli/ci/junit_processing/upload.py b/mergify_cli/ci/junit_processing/upload.py index 434eca4..9ff6e04 100644 --- a/mergify_cli/ci/junit_processing/upload.py +++ b/mergify_cli/ci/junit_processing/upload.py @@ -6,8 +6,6 @@ from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.sdk.trace import export -from mergify_cli import console - if typing.TYPE_CHECKING: from opentelemetry.sdk.trace import ReadableSpan @@ -61,15 +59,6 @@ def upload( repository: str, spans: list[ReadableSpan], ) -> None: - console.log("") - console.log("☁️ Upload") - console.log(f"• Owner/Repo: {repository}") - if spans: - try: - upload_spans(api_url, token, repository, spans) - except UploadError as e: - console.log(f"• ❌ Error uploading spans: {e}", style="red") - else: - console.log("• [green]🎉 File(s) uploaded[/]") - else: - console.log("• [orange]🟠 No tests were detected in the JUnit file(s)[/]") + if not spans: + return + upload_spans(api_url, token, repository, spans) diff --git a/mergify_cli/tests/ci/junit_example.xml b/mergify_cli/tests/ci/fixtures/junit_example.xml similarity index 100% rename from mergify_cli/tests/ci/junit_example.xml rename to mergify_cli/tests/ci/fixtures/junit_example.xml diff --git a/mergify_cli/tests/ci/report.xml b/mergify_cli/tests/ci/fixtures/report.xml similarity index 100% rename from mergify_cli/tests/ci/report.xml rename to mergify_cli/tests/ci/fixtures/report.xml diff --git a/mergify_cli/tests/ci/fixtures/report_all_pass.xml b/mergify_cli/tests/ci/fixtures/report_all_pass.xml new file mode 100644 index 0000000..8d5f687 --- /dev/null +++ b/mergify_cli/tests/ci/fixtures/report_all_pass.xml @@ -0,0 +1,8 @@ + + + + + + + diff --git a/mergify_cli/tests/ci/fixtures/report_invalid.xml b/mergify_cli/tests/ci/fixtures/report_invalid.xml new file mode 100644 index 0000000..6a1e363 --- /dev/null +++ b/mergify_cli/tests/ci/fixtures/report_invalid.xml @@ -0,0 +1 @@ +this is not valid xml <><> diff --git a/mergify_cli/tests/ci/fixtures/report_mixed.xml b/mergify_cli/tests/ci/fixtures/report_mixed.xml new file mode 100644 index 0000000..e60600a --- /dev/null +++ b/mergify_cli/tests/ci/fixtures/report_mixed.xml @@ -0,0 +1,15 @@ + + + + + + Connection timed out + + + Traceback: + File "test.py", line 10 +ValueError: invalid input + + + diff --git a/mergify_cli/tests/ci/fixtures/report_no_testcases.xml b/mergify_cli/tests/ci/fixtures/report_no_testcases.xml new file mode 100644 index 0000000..cbbb7c3 --- /dev/null +++ b/mergify_cli/tests/ci/fixtures/report_no_testcases.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/mergify_cli/tests/ci/junit_processing/test_check_failing_spans.py b/mergify_cli/tests/ci/junit_processing/test_check_failing_spans.py index 9a193e3..b9f63b8 100644 --- a/mergify_cli/tests/ci/junit_processing/test_check_failing_spans.py +++ b/mergify_cli/tests/ci/junit_processing/test_check_failing_spans.py @@ -68,14 +68,16 @@ async def test_no_failing_tests_quarantined( ), ] - failed_tests_quarantined_test_count = await check_and_update_failing_spans( + result = await check_and_update_failing_spans( API_MERGIFY_BASE_URL, "token", "foo/bar", "main", spans, ) - assert failed_tests_quarantined_test_count == 1 + assert result.failing_tests_not_quarantined_count == 1 + assert len(result.quarantined_spans) == 0 + assert len(result.non_quarantined_spans) == 1 assert spans[0].attributes is not None assert "cicd.test.quarantined" in spans[0].attributes assert spans[0].attributes["cicd.test.quarantined"] is False @@ -116,14 +118,16 @@ async def test_some_failing_tests_quarantined( ), ] - failed_tests_quarantined_count = await check_and_update_failing_spans( + result = await check_and_update_failing_spans( API_MERGIFY_BASE_URL, "token", "foo/bar", "main", spans, ) - assert failed_tests_quarantined_count == 1 + assert result.failing_tests_not_quarantined_count == 1 + assert len(result.quarantined_spans) == 1 + assert len(result.non_quarantined_spans) == 1 assert spans[0].attributes is not None assert spans[1].attributes is not None @@ -173,14 +177,16 @@ async def test_all_failing_tests_quarantined( ), ] - failed_tests_quarantined_count = await check_and_update_failing_spans( + result = await check_and_update_failing_spans( API_MERGIFY_BASE_URL, "token", "foo/bar", "main", spans, ) - assert failed_tests_quarantined_count == 0 + assert result.failing_tests_not_quarantined_count == 0 + assert len(result.quarantined_spans) == 3 + assert len(result.non_quarantined_spans) == 0 assert spans[0].attributes is not None assert spans[1].attributes is not None @@ -193,3 +199,25 @@ async def test_all_failing_tests_quarantined( assert spans[0].attributes["cicd.test.quarantined"] is True assert spans[1].attributes["cicd.test.quarantined"] is True assert spans[2].attributes["cicd.test.quarantined"] is True + + +async def test_no_failed_tests_result() -> None: + spans = [ + ReadableSpan( + name="test_me.py::test_ok", + status=Status(status_code=StatusCode.OK, description=""), + attributes={"test.scope": "case"}, + ), + ] + + result = await check_and_update_failing_spans( + API_MERGIFY_BASE_URL, + "token", + "foo/bar", + "main", + spans, + ) + assert result.failing_tests_not_quarantined_count == 0 + assert result.failing_spans == [] + assert result.quarantined_spans == [] + assert result.non_quarantined_spans == [] diff --git a/mergify_cli/tests/ci/junit_processing/test_cli.py b/mergify_cli/tests/ci/junit_processing/test_cli.py index 89a5b7b..700fc25 100644 --- a/mergify_cli/tests/ci/junit_processing/test_cli.py +++ b/mergify_cli/tests/ci/junit_processing/test_cli.py @@ -1,9 +1,15 @@ from __future__ import annotations +import dataclasses import pathlib from unittest import mock +import anys +import httpx +from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.sdk.trace import id_generator +from opentelemetry.trace import Status +from opentelemetry.trace import StatusCode import pytest from mergify_cli.ci.junit_processing import cli @@ -11,25 +17,73 @@ from mergify_cli.ci.junit_processing import upload -REPORT_XML = pathlib.Path(__file__).parent.parent / "report.xml" +FIXTURES_DIR = pathlib.Path(__file__).parent.parent / "fixtures" +REPORT_XML = FIXTURES_DIR / "report.xml" +REPORT_MIXED_XML = FIXTURES_DIR / "report_mixed.xml" +REPORT_INVALID_XML = FIXTURES_DIR / "report_invalid.xml" +REPORT_NO_TESTCASES_XML = FIXTURES_DIR / "report_no_testcases.xml" +REPORT_ALL_PASS_XML = FIXTURES_DIR / "report_all_pass.xml" +FAILING_SPAN = ReadableSpan( + name="mergify.tests.test_junit.test_failed", + status=Status(status_code=StatusCode.ERROR, description=""), + attributes={ + "test.scope": "case", + "exception.message": "assert 1 == 0", + "exception.stacktrace": ( + "def test_failed() -> None:\n" + " > assert 1 == 0\n" + " E assert 1 == 0\n" + "\n" + " mergify/tests/test_junit.py:6: AssertionError" + ), + }, +) + + +@dataclasses.dataclass +class ProcessResult: + exit_code: int + stdout: str + stderr: str + upload_mock: mock.MagicMock -async def test_process_junit_file_reporting( + +async def _run_process( + *, + files: tuple[str, ...] = (str(REPORT_XML),), + quarantine_result: quarantine.QuarantineResult | None = None, + quarantine_side_effect: Exception | None = None, + upload_side_effect: Exception | None = None, capsys: pytest.CaptureFixture[str], -) -> None: +) -> ProcessResult: + quarantine_mock = mock.AsyncMock() + if quarantine_side_effect is not None: + quarantine_mock.side_effect = quarantine_side_effect + elif quarantine_result is not None: + quarantine_mock.return_value = quarantine_result + + upload_mock = mock.Mock() + if upload_side_effect is not None: + upload_mock.side_effect = upload_side_effect + with ( mock.patch.object( quarantine, "check_and_update_failing_spans", - return_value=0, + quarantine_mock, ), - mock.patch.object(upload, "upload"), + mock.patch.object( + upload, + "upload", + upload_mock, + ) as mocked_upload, mock.patch.object( id_generator.RandomIdGenerator, "generate_span_id", return_value=12345678910, ), - pytest.raises(SystemExit), + pytest.raises(SystemExit) as exc_info, ): await cli.process_junit_files( api_url="https://api.mergify.com", @@ -38,20 +92,703 @@ async def test_process_junit_file_reporting( test_framework=None, test_language=None, tests_target_branch="main", - files=(str(REPORT_XML),), + files=files, ) captured = capsys.readouterr() - assert ( - captured.out - == """🚀 CI Insights · Upload JUnit -──────────────────────────── -📂 Discovered reports: 1 -🛠️ MERGIFY_TEST_RUN_ID=00000002dfdc1c3e -🧪 Parsed tests: 2 (✅ passed: 1 | ❌ failed: 1) - -🎉 Verdict -• Status: ✅ OK — all 1 failures are quarantined (ignored for CI status) -• Exit code: 0 -""" + assert isinstance(exc_info.value.code, int) + return ProcessResult( + exit_code=exc_info.value.code, + stdout=captured.out, + stderr=captured.err, + upload_mock=mocked_upload, + ) + + +# ── Happy paths ── + + +async def test_all_failures_quarantined( + capsys: pytest.CaptureFixture[str], +) -> None: + result = await _run_process( + quarantine_result=quarantine.QuarantineResult( + failing_spans=[FAILING_SPAN], + quarantined_spans=[FAILING_SPAN], + non_quarantined_spans=[], + failing_tests_not_quarantined_count=0, + ), + capsys=capsys, + ) + + assert result.exit_code == 0 + assert result.stdout == ( + "══════════════════════════════════════════\n" + " 🚀 CI Insights\n" + "\n" + " Uploads JUnit test results to Mergify CI Insights and evaluates\n" + " quarantine status for failing tests. This step determines the\n" + " final CI status — quarantined failures are ignored.\n" + " Learn more: https://docs.mergify.com/ci-insights/quarantine\n" + "══════════════════════════════════════════\n" + "\n" + " Run ID: 00000002dfdc1c3e\n" + " ☁️ 1 report uploaded\n" + " 🧪 2 tests (1 failure)\n" + "\n" + "──────────────────────────────────────────\n" + "\n" + "🛡️ Quarantine\n" + "\n" + " 🔒 Quarantined (1):\n" + " · mergify.tests.test_junit.test_failed\n" + "\n" + "══════════════════════════════════════════\n" + "✅ OK — 1/1 failures quarantined, CI status unaffected\n" + " Exit code: 0\n" + "══════════════════════════════════════════\n" + ) + + +async def test_no_failed_tests( + capsys: pytest.CaptureFixture[str], +) -> None: + result = await _run_process( + files=(str(REPORT_ALL_PASS_XML),), + quarantine_result=quarantine.QuarantineResult( + failing_spans=[], + quarantined_spans=[], + non_quarantined_spans=[], + failing_tests_not_quarantined_count=0, + ), + capsys=capsys, + ) + + assert result.exit_code == 0 + assert result.stdout == ( + "══════════════════════════════════════════\n" + " 🚀 CI Insights\n" + "\n" + " Uploads JUnit test results to Mergify CI Insights and evaluates\n" + " quarantine status for failing tests. This step determines the\n" + " final CI status — quarantined failures are ignored.\n" + " Learn more: https://docs.mergify.com/ci-insights/quarantine\n" + "══════════════════════════════════════════\n" + "\n" + " Run ID: 00000002dfdc1c3e\n" + " ☁️ 1 report uploaded\n" + " 🧪 2 tests (0 failures)\n" + "\n" + "══════════════════════════════════════════\n" + "✅ OK — all tests passed, no quarantine needed\n" + " Exit code: 0\n" + "══════════════════════════════════════════\n" + ) + + +async def test_multiple_report_files( + capsys: pytest.CaptureFixture[str], +) -> None: + result = await _run_process( + files=(str(REPORT_ALL_PASS_XML), str(REPORT_ALL_PASS_XML)), + quarantine_result=quarantine.QuarantineResult( + failing_spans=[], + quarantined_spans=[], + non_quarantined_spans=[], + failing_tests_not_quarantined_count=0, + ), + capsys=capsys, + ) + + assert result.exit_code == 0 + assert result.stdout == ( + "══════════════════════════════════════════\n" + " 🚀 CI Insights\n" + "\n" + " Uploads JUnit test results to Mergify CI Insights and evaluates\n" + " quarantine status for failing tests. This step determines the\n" + " final CI status — quarantined failures are ignored.\n" + " Learn more: https://docs.mergify.com/ci-insights/quarantine\n" + "══════════════════════════════════════════\n" + "\n" + " Run ID: 00000002dfdc1c3e\n" + " ☁️ 2 reports uploaded\n" + " 🧪 4 tests (0 failures)\n" + "\n" + "══════════════════════════════════════════\n" + "✅ OK — all tests passed, no quarantine needed\n" + " Exit code: 0\n" + "══════════════════════════════════════════\n" + ) + + +# ── Unquarantined failures ── + + +async def test_unquarantined_failure_with_stacktrace( + capsys: pytest.CaptureFixture[str], +) -> None: + result = await _run_process( + quarantine_result=quarantine.QuarantineResult( + failing_spans=[FAILING_SPAN], + quarantined_spans=[], + non_quarantined_spans=[FAILING_SPAN], + failing_tests_not_quarantined_count=1, + ), + capsys=capsys, + ) + + assert result.exit_code == 1 + assert result.stdout == ( + "══════════════════════════════════════════\n" + " 🚀 CI Insights\n" + "\n" + " Uploads JUnit test results to Mergify CI Insights and evaluates\n" + " quarantine status for failing tests. This step determines the\n" + " final CI status — quarantined failures are ignored.\n" + " Learn more: https://docs.mergify.com/ci-insights/quarantine\n" + "══════════════════════════════════════════\n" + "\n" + " Run ID: 00000002dfdc1c3e\n" + " ☁️ 1 report uploaded\n" + " 🧪 2 tests (1 failure)\n" + "\n" + "──────────────────────────────────────────\n" + "\n" + "🛡️ Quarantine\n" + "\n" + " ❌ Unquarantined (1):\n" + "\n" + " ┌ mergify.tests.test_junit.test_failed\n" + " │\n" + " │ assert 1 == 0\n" + " │\n" + " │ def test_failed() -> None:\n" + " │ > assert 1 == 0\n" + " │ E assert 1 == 0\n" + " │ \n" + " │ mergify/tests/test_junit.py:6: AssertionError\n" + " └─\n" + "\n" + "══════════════════════════════════════════\n" + "❌ FAIL — 0/1 failures quarantined\n" + " Exit code: 1\n" + "══════════════════════════════════════════\n" + ) + + +async def test_mixed_quarantined_and_unquarantined( + capsys: pytest.CaptureFixture[str], +) -> None: + quarantined_span = ReadableSpan( + name="tests.test_flaky", + status=Status(status_code=StatusCode.ERROR, description=""), + attributes={ + "test.scope": "case", + "exception.message": "flaky timeout", + "exception.stacktrace": "Connection timed out", + }, + ) + unquarantined_span = ReadableSpan( + name="tests.test_broken", + status=Status(status_code=StatusCode.ERROR, description=""), + attributes={ + "test.scope": "case", + "exception.type": "ValueError", + "exception.message": "invalid input", + "exception.stacktrace": ( + 'Traceback:\n File "test.py", line 10\nValueError: invalid input' + ), + }, + ) + + result = await _run_process( + files=(str(REPORT_MIXED_XML),), + quarantine_result=quarantine.QuarantineResult( + failing_spans=[quarantined_span, unquarantined_span], + quarantined_spans=[quarantined_span], + non_quarantined_spans=[unquarantined_span], + failing_tests_not_quarantined_count=1, + ), + capsys=capsys, + ) + + assert result.exit_code == 1 + assert result.stdout == ( + "══════════════════════════════════════════\n" + " 🚀 CI Insights\n" + "\n" + " Uploads JUnit test results to Mergify CI Insights and evaluates\n" + " quarantine status for failing tests. This step determines the\n" + " final CI status — quarantined failures are ignored.\n" + " Learn more: https://docs.mergify.com/ci-insights/quarantine\n" + "══════════════════════════════════════════\n" + "\n" + " Run ID: 00000002dfdc1c3e\n" + " ☁️ 1 report uploaded\n" + " 🧪 3 tests (2 failures)\n" + "\n" + "──────────────────────────────────────────\n" + "\n" + "🛡️ Quarantine\n" + "\n" + " 🔒 Quarantined (1):\n" + " · tests.test_flaky\n" + "\n" + " ❌ Unquarantined (1):\n" + "\n" + " ┌ tests.test_broken\n" + " │\n" + " │ ValueError: invalid input\n" + " │\n" + " │ Traceback:\n" + ' │ File "test.py", line 10\n' + " │ ValueError: invalid input\n" + " └─\n" + "\n" + "══════════════════════════════════════════\n" + "❌ FAIL — 1/2 failures quarantined\n" + " Exit code: 1\n" + "══════════════════════════════════════════\n" + ) + + +async def test_unquarantined_failure_no_exception_attributes( + capsys: pytest.CaptureFixture[str], +) -> None: + bare_span = ReadableSpan( + name="tests.test_bare", + status=Status(status_code=StatusCode.ERROR, description=""), + attributes={"test.scope": "case"}, + ) + + result = await _run_process( + quarantine_result=quarantine.QuarantineResult( + failing_spans=[bare_span], + quarantined_spans=[], + non_quarantined_spans=[bare_span], + failing_tests_not_quarantined_count=1, + ), + capsys=capsys, + ) + + assert result.exit_code == 1 + assert result.stdout == ( + "══════════════════════════════════════════\n" + " 🚀 CI Insights\n" + "\n" + " Uploads JUnit test results to Mergify CI Insights and evaluates\n" + " quarantine status for failing tests. This step determines the\n" + " final CI status — quarantined failures are ignored.\n" + " Learn more: https://docs.mergify.com/ci-insights/quarantine\n" + "══════════════════════════════════════════\n" + "\n" + " Run ID: 00000002dfdc1c3e\n" + " ☁️ 1 report uploaded\n" + " 🧪 2 tests (1 failure)\n" + "\n" + "──────────────────────────────────────────\n" + "\n" + "🛡️ Quarantine\n" + "\n" + " ❌ Unquarantined (1):\n" + "\n" + " ┌ tests.test_bare\n" + " │\n" + " │ (no error details in JUnit report)\n" + " └─\n" + "\n" + "══════════════════════════════════════════\n" + "❌ FAIL — 0/1 failures quarantined\n" + " Exit code: 1\n" + "══════════════════════════════════════════\n" + ) + + +async def test_unquarantined_failure_only_exception_type( + capsys: pytest.CaptureFixture[str], +) -> None: + span = ReadableSpan( + name="tests.test_type_only", + status=Status(status_code=StatusCode.ERROR, description=""), + attributes={ + "test.scope": "case", + "exception.type": "RuntimeError", + }, + ) + + result = await _run_process( + quarantine_result=quarantine.QuarantineResult( + failing_spans=[span], + quarantined_spans=[], + non_quarantined_spans=[span], + failing_tests_not_quarantined_count=1, + ), + capsys=capsys, + ) + + assert result.exit_code == 1 + assert result.stdout == ( + "══════════════════════════════════════════\n" + " 🚀 CI Insights\n" + "\n" + " Uploads JUnit test results to Mergify CI Insights and evaluates\n" + " quarantine status for failing tests. This step determines the\n" + " final CI status — quarantined failures are ignored.\n" + " Learn more: https://docs.mergify.com/ci-insights/quarantine\n" + "══════════════════════════════════════════\n" + "\n" + " Run ID: 00000002dfdc1c3e\n" + " ☁️ 1 report uploaded\n" + " 🧪 2 tests (1 failure)\n" + "\n" + "──────────────────────────────────────────\n" + "\n" + "🛡️ Quarantine\n" + "\n" + " ❌ Unquarantined (1):\n" + "\n" + " ┌ tests.test_type_only\n" + " │\n" + " │ RuntimeError\n" + " └─\n" + "\n" + "══════════════════════════════════════════\n" + "❌ FAIL — 0/1 failures quarantined\n" + " Exit code: 1\n" + "══════════════════════════════════════════\n" + ) + + +async def test_unquarantined_failure_with_none_attributes( + capsys: pytest.CaptureFixture[str], +) -> None: + span = ReadableSpan( + name="tests.test_none_attrs", + status=Status(status_code=StatusCode.ERROR, description=""), + attributes=None, + ) + + result = await _run_process( + quarantine_result=quarantine.QuarantineResult( + failing_spans=[span], + quarantined_spans=[], + non_quarantined_spans=[span], + failing_tests_not_quarantined_count=1, + ), + capsys=capsys, + ) + + assert result.exit_code == 1 + assert result.stdout == ( + "══════════════════════════════════════════\n" + " 🚀 CI Insights\n" + "\n" + " Uploads JUnit test results to Mergify CI Insights and evaluates\n" + " quarantine status for failing tests. This step determines the\n" + " final CI status — quarantined failures are ignored.\n" + " Learn more: https://docs.mergify.com/ci-insights/quarantine\n" + "══════════════════════════════════════════\n" + "\n" + " Run ID: 00000002dfdc1c3e\n" + " ☁️ 1 report uploaded\n" + " 🧪 2 tests (1 failure)\n" + "\n" + "──────────────────────────────────────────\n" + "\n" + "🛡️ Quarantine\n" + "\n" + " ❌ Unquarantined (1):\n" + "\n" + " ┌ tests.test_none_attrs\n" + " │\n" + " │ (no error details in JUnit report)\n" + " └─\n" + "\n" + "══════════════════════════════════════════\n" + "❌ FAIL — 0/1 failures quarantined\n" + " Exit code: 1\n" + "══════════════════════════════════════════\n" + ) + + +# ── Quarantine errors ── + + +async def test_quarantine_handled_error( + capsys: pytest.CaptureFixture[str], +) -> None: + result = await _run_process( + quarantine_side_effect=quarantine.QuarantineFailedError( + 'HTTP 422: {"detail": "No subscription"}', + ), + capsys=capsys, + ) + + assert result.exit_code == 1 + assert result.stdout == ( + "══════════════════════════════════════════\n" + " 🚀 CI Insights\n" + "\n" + " Uploads JUnit test results to Mergify CI Insights and evaluates\n" + " quarantine status for failing tests. This step determines the\n" + " final CI status — quarantined failures are ignored.\n" + " Learn more: https://docs.mergify.com/ci-insights/quarantine\n" + "══════════════════════════════════════════\n" + "\n" + " Run ID: 00000002dfdc1c3e\n" + " ☁️ 1 report uploaded\n" + " 🧪 2 tests (1 failure)\n" + "\n" + "──────────────────────────────────────────\n" + "\n" + "🛡️ Quarantine\n" + "\n" + " ⚠️ Failed to check quarantine status\n" + " Contact Mergify support if this error persists.\n" + "\n" + " ┌ Details\n" + ' │ HTTP 422: {"detail": "No subscription"}\n' + " └─\n" + "\n" + " ❌ Could not verify quarantine status (1):\n" + "\n" + " ┌ mergify.tests.test_junit.test_failed\n" + " │\n" + " │ assert 1 == 0\n" + " │\n" + " │ def test_failed() -> None:\n" + " │ > assert 1 == 0\n" + " │ E assert 1 == 0\n" + " │ \n" + " │ mergify/tests/test_junit.py:6: AssertionError\n" + " └─\n" + "\n" + "══════════════════════════════════════════\n" + "❌ FAIL — Treating 1/1 failures as blocking\n" + " Exit code: 1\n" + "══════════════════════════════════════════\n" + ) + + +async def test_quarantine_unhandled_error( + capsys: pytest.CaptureFixture[str], +) -> None: + result = await _run_process( + quarantine_side_effect=httpx.ConnectError("Connection refused"), + capsys=capsys, + ) + + assert result.exit_code == 1 + assert result.stdout == ( + "══════════════════════════════════════════\n" + " 🚀 CI Insights\n" + "\n" + " Uploads JUnit test results to Mergify CI Insights and evaluates\n" + " quarantine status for failing tests. This step determines the\n" + " final CI status — quarantined failures are ignored.\n" + " Learn more: https://docs.mergify.com/ci-insights/quarantine\n" + "══════════════════════════════════════════\n" + "\n" + " Run ID: 00000002dfdc1c3e\n" + " ☁️ 1 report uploaded\n" + " 🧪 2 tests (1 failure)\n" + "\n" + "──────────────────────────────────────────\n" + "\n" + "🛡️ Quarantine\n" + "\n" + " ⚠️ Failed to check quarantine status\n" + " Contact Mergify support if this error persists.\n" + "\n" + " ┌ Details\n" + " │ Connection refused\n" + " └─\n" + "\n" + " ❌ Could not verify quarantine status (1):\n" + "\n" + " ┌ mergify.tests.test_junit.test_failed\n" + " │\n" + " │ assert 1 == 0\n" + " │\n" + " │ def test_failed() -> None:\n" + " │ > assert 1 == 0\n" + " │ E assert 1 == 0\n" + " │ \n" + " │ mergify/tests/test_junit.py:6: AssertionError\n" + " └─\n" + "\n" + "══════════════════════════════════════════\n" + "❌ FAIL — Treating 1/1 failures as blocking\n" + " Exit code: 1\n" + "══════════════════════════════════════════\n" + ) + + +# ── Upload errors ── + + +async def test_upload_failure( + capsys: pytest.CaptureFixture[str], +) -> None: + result = await _run_process( + files=(str(REPORT_ALL_PASS_XML),), + quarantine_result=quarantine.QuarantineResult( + failing_spans=[], + quarantined_spans=[], + non_quarantined_spans=[], + failing_tests_not_quarantined_count=0, + ), + upload_side_effect=Exception("Connection refused"), + capsys=capsys, + ) + + assert result.exit_code == 0 + assert result.upload_mock.call_count == 1 + assert result.upload_mock.call_args.kwargs == { + "api_url": "https://api.mergify.com", + "token": "foobar", + "repository": "foo/bar", + "spans": anys.ANY_LIST, + } + assert result.stdout == ( + "══════════════════════════════════════════\n" + " 🚀 CI Insights\n" + "\n" + " Uploads JUnit test results to Mergify CI Insights and evaluates\n" + " quarantine status for failing tests. This step determines the\n" + " final CI status — quarantined failures are ignored.\n" + " Learn more: https://docs.mergify.com/ci-insights/quarantine\n" + "══════════════════════════════════════════\n" + "\n" + " Run ID: 00000002dfdc1c3e\n" + " ☁️ 1 report not uploaded\n" + " 🧪 2 tests (0 failures)\n" + "\n" + " ⚠️ Failed to upload test results\n" + " Mergify CI Insights won't process these test results.\n" + " Quarantine status and CI outcome are unaffected.\n" + "\n" + " ┌ Details\n" + " │ Connection refused\n" + " └─\n" + "\n" + "══════════════════════════════════════════\n" + "✅ OK — all tests passed, no quarantine needed\n" + " Exit code: 0\n" + "══════════════════════════════════════════\n" + ) + + +async def test_upload_failure_with_unquarantined_failures( + capsys: pytest.CaptureFixture[str], +) -> None: + result = await _run_process( + quarantine_result=quarantine.QuarantineResult( + failing_spans=[FAILING_SPAN], + quarantined_spans=[], + non_quarantined_spans=[FAILING_SPAN], + failing_tests_not_quarantined_count=1, + ), + upload_side_effect=Exception("Connection refused"), + capsys=capsys, + ) + + assert result.exit_code == 1 + assert result.stdout == ( + "══════════════════════════════════════════\n" + " 🚀 CI Insights\n" + "\n" + " Uploads JUnit test results to Mergify CI Insights and evaluates\n" + " quarantine status for failing tests. This step determines the\n" + " final CI status — quarantined failures are ignored.\n" + " Learn more: https://docs.mergify.com/ci-insights/quarantine\n" + "══════════════════════════════════════════\n" + "\n" + " Run ID: 00000002dfdc1c3e\n" + " ☁️ 1 report not uploaded\n" + " 🧪 2 tests (1 failure)\n" + "\n" + " ⚠️ Failed to upload test results\n" + " Mergify CI Insights won't process these test results.\n" + " Quarantine status and CI outcome are unaffected.\n" + "\n" + " ┌ Details\n" + " │ Connection refused\n" + " └─\n" + "\n" + "──────────────────────────────────────────\n" + "\n" + "🛡️ Quarantine\n" + "\n" + " ❌ Unquarantined (1):\n" + "\n" + " ┌ mergify.tests.test_junit.test_failed\n" + " │\n" + " │ assert 1 == 0\n" + " │\n" + " │ def test_failed() -> None:\n" + " │ > assert 1 == 0\n" + " │ E assert 1 == 0\n" + " │ \n" + " │ mergify/tests/test_junit.py:6: AssertionError\n" + " └─\n" + "\n" + "══════════════════════════════════════════\n" + "❌ FAIL — 0/1 failures quarantined\n" + " Exit code: 1\n" + "══════════════════════════════════════════\n" + ) + + +# ── Early exit cases ── + + +async def test_invalid_junit_xml( + capsys: pytest.CaptureFixture[str], +) -> None: + result = await _run_process( + files=(str(REPORT_INVALID_XML),), + capsys=capsys, + ) + + assert result.exit_code == 1 + assert result.stdout == ( + "══════════════════════════════════════════\n" + " 🚀 CI Insights\n" + "\n" + " Uploads JUnit test results to Mergify CI Insights and evaluates\n" + " quarantine status for failing tests. This step determines the\n" + " final CI status — quarantined failures are ignored.\n" + " Learn more: https://docs.mergify.com/ci-insights/quarantine\n" + "══════════════════════════════════════════\n" + "❌ FAIL — Failed to parse JUnit XML: syntax error: line 1, column 0\n" + " Check that your test framework is generating valid JUnit XML output.\n" + " Exit code: 1\n" + "══════════════════════════════════════════\n" + ) + + +async def test_no_test_cases_in_junit( + capsys: pytest.CaptureFixture[str], +) -> None: + result = await _run_process( + files=(str(REPORT_NO_TESTCASES_XML),), + capsys=capsys, + ) + + assert result.exit_code == 1 + assert result.stdout == ( + "══════════════════════════════════════════\n" + " 🚀 CI Insights\n" + "\n" + " Uploads JUnit test results to Mergify CI Insights and evaluates\n" + " quarantine status for failing tests. This step determines the\n" + " final CI status — quarantined failures are ignored.\n" + " Learn more: https://docs.mergify.com/ci-insights/quarantine\n" + "══════════════════════════════════════════\n" + "❌ FAIL — No test cases found in the JUnit files\n" + " Check that your test step ran successfully before this step.\n" + " Exit code: 1\n" + "══════════════════════════════════════════\n" ) diff --git a/mergify_cli/tests/ci/junit_processing/test_upload.py b/mergify_cli/tests/ci/junit_processing/test_upload.py index 5206416..5da28e1 100644 --- a/mergify_cli/tests/ci/junit_processing/test_upload.py +++ b/mergify_cli/tests/ci/junit_processing/test_upload.py @@ -1,7 +1,6 @@ from __future__ import annotations import pathlib -import re from opentelemetry.sdk.trace import ReadableSpan import opentelemetry.trace.span @@ -12,7 +11,8 @@ from mergify_cli.ci.junit_processing import upload -REPORT_XML = pathlib.Path(__file__).parent.parent / "report.xml" +FIXTURES_DIR = pathlib.Path(__file__).parent.parent / "fixtures" +REPORT_XML = FIXTURES_DIR / "report.xml" @responses.activate(assert_all_requests_are_fired=True) @@ -47,10 +47,9 @@ ) async def test_junit_upload( env: dict[str, str], - capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, ) -> None: - spans = await junit.files_to_spans(files=(str(REPORT_XML),)) + run_id, spans = await junit.files_to_spans(files=(str(REPORT_XML),)) for key, value in env.items(): monkeypatch.setenv(key, value) @@ -65,16 +64,7 @@ async def test_junit_upload( spans, ) - captured = capsys.readouterr() - matched = re.search( - r"^🛠️ MERGIFY_TEST_RUN_ID=(.+)", - captured.out, - re.MULTILINE, - ) - assert matched is not None - assert len(bytes.fromhex(matched.group(1))) == 8 - - assert "🎉 File(s) uploaded" in captured.out + assert len(bytes.fromhex(run_id)) == 8 @responses.activate(assert_all_requests_are_fired=True) @@ -104,25 +94,18 @@ def test_junit_upload_http_error() -> None: @responses.activate(assert_all_requests_are_fired=True) -async def test_junit_upload_http_error_console( - capsys: pytest.CaptureFixture[str], -) -> None: +async def test_junit_upload_http_error_raises() -> None: responses.post( "https://api.mergify.com/v1/repos/user/repo/ci/traces", status=422, json={"detail": "Not enabled on this repository"}, ) - spans = await junit.files_to_spans(files=(str(REPORT_XML),)) - upload.upload( - "https://api.mergify.com", - "token", - "user/repo", - spans, - ) - captured = capsys.readouterr() - assert ( - "• ❌ Error uploading spans: Failed to export span batch code: 422, reason:" - in captured.out - ) - assert "Not enabled on this repository" in captured.out + _run_id, spans = await junit.files_to_spans(files=(str(REPORT_XML),)) + with pytest.raises(upload.UploadError, match="422"): + upload.upload( + "https://api.mergify.com", + "token", + "user/repo", + spans, + ) diff --git a/mergify_cli/tests/ci/test_cli.py b/mergify_cli/tests/ci/test_cli.py index 844769d..e9ab57d 100644 --- a/mergify_cli/tests/ci/test_cli.py +++ b/mergify_cli/tests/ci/test_cli.py @@ -20,7 +20,8 @@ import respx -REPORT_XML = pathlib.Path(__file__).parent / "report.xml" +FIXTURES_DIR = pathlib.Path(__file__).parent / "fixtures" +REPORT_XML = FIXTURES_DIR / "report.xml" @pytest.mark.parametrize( @@ -69,7 +70,12 @@ def test_cli(env: dict[str, str], monkeypatch: pytest.MonkeyPatch) -> None: mock.patch.object( quarantine, "check_and_update_failing_spans", - return_value=0, + return_value=quarantine.QuarantineResult( + failing_spans=[], + quarantined_spans=[], + non_quarantined_spans=[], + failing_tests_not_quarantined_count=0, + ), ), ): result_process = runner.invoke( @@ -219,147 +225,6 @@ def test_tests_target_branch_environment_variable_processing( assert call_kwargs["tests_target_branch"] == expected_branch -def test_quarantine_unhandled_error(monkeypatch: pytest.MonkeyPatch) -> None: - for key, value in { - "GITHUB_EVENT_NAME": "push", - "GITHUB_ACTIONS": "true", - "MERGIFY_API_URL": "https://api.mergify.com", - "MERGIFY_TOKEN": "abc", - "GITHUB_REPOSITORY": "user/repo", - "GITHUB_SHA": "3af96aa24f1d32fcfbb7067793cacc6dc0c6b199", - "GITHUB_WORKFLOW": "JOB", - "GITHUB_BASE_REF": "main", - }.items(): - monkeypatch.setenv(key, value) - - runner = testing.CliRunner() - - with ( - mock.patch.object( - upload, - "upload", - mock.Mock(), - ), - mock.patch.object( - quarantine, - "check_and_update_failing_spans", - side_effect=Exception("API crashed"), - ) as mocked_quarantine, - ): - result = runner.invoke( - ci_cli.junit_process, - [str(REPORT_XML)], - ) - - assert result.exit_code == 1, (result.stdout, result.stderr) - assert ( - result.stderr - == """❌ An unexpected error occurred when checking quarantined tests: API crashed -This error occurred because there are failed tests in your CI pipeline and will disappear once your CI passes successfully. - -If you're unsure why this is happening or need assistance, please contact Mergify to report the issue. -""" - ) - assert ( - "FAIL — Unable to determine quarantined failures due to above error" - in result.stdout - ) - assert "MERGIFY_TEST_RUN_ID=" in result.stdout - assert mocked_quarantine.call_count == 1 - - -def test_quarantine_handled_error(monkeypatch: pytest.MonkeyPatch) -> None: - for key, value in { - "GITHUB_EVENT_NAME": "push", - "GITHUB_ACTIONS": "true", - "MERGIFY_API_URL": "https://api.mergify.com", - "MERGIFY_TOKEN": "abc", - "GITHUB_REPOSITORY": "user/repo", - "GITHUB_SHA": "3af96aa24f1d32fcfbb7067793cacc6dc0c6b199", - "GITHUB_WORKFLOW": "JOB", - "GITHUB_BASE_REF": "main", - }.items(): - monkeypatch.setenv(key, value) - - runner = testing.CliRunner() - - with ( - mock.patch.object( - upload, - "upload", - mock.Mock(), - ), - mock.patch.object( - quarantine, - "check_and_update_failing_spans", - side_effect=quarantine.QuarantineFailedError("It's not OK"), - ) as mocked_quarantine, - ): - result = runner.invoke( - ci_cli.junit_process, - [str(REPORT_XML)], - ) - assert result.exit_code == 1, (result.stdout, result.stderr) - assert ( - result.stderr - == """It's not OK -This error occurred because there are failed tests in your CI pipeline and will disappear once your CI passes successfully. - -If you're unsure why this is happening or need assistance, please contact Mergify to report the issue. -""" - ) - assert ( - "FAIL — Unable to determine quarantined failures due to above error" - in result.stdout - ) - assert "MERGIFY_TEST_RUN_ID=" in result.stdout - assert mocked_quarantine.call_count == 1 - - -def test_upload_error(monkeypatch: pytest.MonkeyPatch) -> None: - for key, value in { - "GITHUB_EVENT_NAME": "push", - "GITHUB_ACTIONS": "true", - "MERGIFY_API_URL": "https://api.mergify.com", - "MERGIFY_TOKEN": "abc", - "GITHUB_REPOSITORY": "user/repo", - "GITHUB_SHA": "3af96aa24f1d32fcfbb7067793cacc6dc0c6b199", - "GITHUB_WORKFLOW": "JOB", - "GITHUB_BASE_REF": "main", - }.items(): - monkeypatch.setenv(key, value) - - runner = testing.CliRunner() - - with ( - mock.patch.object( - upload, - "upload", - mock.Mock(), - ) as mocked_upload, - mock.patch.object( - quarantine, - "check_and_update_failing_spans", - return_value=0, - ), - ): - mocked_upload.side_effect = Exception("Upload failed") - result = runner.invoke( - ci_cli.junit_process, - [str(REPORT_XML)], - ) - assert result.exit_code == 0, (result.stdout, result.stderr) - assert result.stderr == "❌ Error uploading JUnit XML reports: Upload failed\n" - assert "MERGIFY_TEST_RUN_ID=" in result.stdout - assert mocked_upload.call_count == 1 - assert mocked_upload.call_args.kwargs == { - "api_url": "https://api.mergify.com", - "token": "abc", - "repository": "user/repo", - "spans": anys.ANY_LIST, - } - - def test_process_tests_target_branch_callback() -> None: """Test the _process_tests_target_branch callback function directly.""" context_mock = mock.MagicMock(spec=click.Context) diff --git a/mergify_cli/tests/ci/test_junit.py b/mergify_cli/tests/ci/test_junit.py index f50e00f..66f0096 100644 --- a/mergify_cli/tests/ci/test_junit.py +++ b/mergify_cli/tests/ci/test_junit.py @@ -48,7 +48,7 @@ async def test_parse( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setenv("_MERGIFY_TEST_JOB_NAME", "foobar") - filename = pathlib.Path(__file__).parent / "junit_example.xml" + filename = pathlib.Path(__file__).parent / "fixtures" / "junit_example.xml" run_id = (32312).to_bytes(8, "big").hex() spans = await junit.junit_to_spans( run_id, @@ -539,7 +539,7 @@ async def test_traceparent_injection( "00-80e1afed08e019fc1110464cfa66635c-7a085853722dc6d2-01", ) monkeypatch.setenv("_MERGIFY_TEST_JOB_NAME", "foobar") - filename = pathlib.Path(__file__).parent / "junit_example.xml" + filename = pathlib.Path(__file__).parent / "fixtures" / "junit_example.xml" run_id = (32312).to_bytes(8, "big").hex() spans = await junit.junit_to_spans( run_id,