diff --git a/.github/actions/generate_twister_report.py b/.github/actions/generate_twister_report.py new file mode 100644 index 000000000000..b364d558c521 --- /dev/null +++ b/.github/actions/generate_twister_report.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 +""" +generate_twister_report.py +────────────────────────── +Compare two Zephyr Twister JUnit XML files (previous and current CI run) and +produce a formatted .xlsx report. + +Usage: + python generate_twister_report.py \ + --prev previous_twister.xml \ + --curr merged_twister.xml \ + --out twister_comparison_report.xlsx \ + [--prev-date "Jun 14"] \ + [--curr-date "Jun 15"] + +The script is also called automatically by the 'Zephyr Twister Tests' GitHub +Actions workflow; the workflow passes the correct paths and date labels. +""" + +import argparse +import sys +from collections import defaultdict +from datetime import datetime + +from junitparser import JUnitXml +from openpyxl import Workbook +from openpyxl.styles import ( + Alignment, Border, Font, PatternFill, Side +) +from openpyxl.utils import get_column_letter + +# ── constants ───────────────────────────────────────────────────────────────── +PASS, FAIL, SKIP, ERROR = "PASS", "FAIL", "SKIP", "ERROR" + +C_DARK_BLUE = "1F3864" +C_MID_BLUE = "2E75B6" +C_LIGHT_BLUE= "D6E4F0" +C_WHITE = "FFFFFF" + +HEADER_FONT = Font(bold=True, color="FFFFFF", size=10) +TITLE_FONT = Font(bold=True, color="FFFFFF", size=13) +SUB_FONT = Font(bold=True, size=10) +NORM_FONT = Font(size=10) +CENTER_ALIGN= Alignment(horizontal="center", vertical="center") +CENTER_WRAP = Alignment(horizontal="center", vertical="top", wrap_text=True) + + +# ── helpers ─────────────────────────────────────────────────────────────────── +def hex_fill(h): + return PatternFill("solid", start_color=h, fgColor=h) + +def all_borders(): + s = Side(style="thin") + return Border(top=s, bottom=s, left=s, right=s) + +def set_col_widths(ws, widths: dict): + for col, w in widths.items(): + ws.column_dimensions[col].width = w + +def status_style(status): + if status == PASS: + return hex_fill("C6EFCE"), Font(color="375623", size=10) + if status in (FAIL, ERROR): + return hex_fill("FFC7CE"), Font(color="9C0006", size=10) + return hex_fill("FFEB9C"), Font(color="9C5700", size=10) # SKIP / other + +def change_style(change): + if change == "Fixed": + return hex_fill("C6EFCE"), Font(color="375623", bold=True, size=10) + if change == "Regressed": + return hex_fill("FFC7CE"), Font(color="9C0006", bold=True, size=10) + if change == "Still Failing": + return hex_fill("FFEB9C"), Font(color="9C5700", bold=True, size=10) + return hex_fill(C_WHITE), NORM_FONT + + +# ── XML parsing ─────────────────────────────────────────────────────────────── +def _case_status(case): + """Return PASS / FAIL / SKIP / ERROR for a junitparser TestCase.""" + from junitparser import Failure, Error, Skipped + results = list(case.iterchildren()) + for r in results: + if isinstance(r, Error): return ERROR + if isinstance(r, Failure): return FAIL + if isinstance(r, Skipped): return SKIP + return PASS + +def _case_message(case): + from junitparser import Failure, Error + for r in case.iterchildren(): + if isinstance(r, (Failure, Error)): + msg = getattr(r, "message", "") or "" + text= getattr(r, "text", "") or "" + combined = (msg + "\n" + text).strip() + return combined[:300] # cap to keep cells readable + return "" + +def parse_xml(path: str) -> dict: + """Return {(suite_name, test_name, classname): (status, message)}""" + xml = JUnitXml.fromfile(path) + results = {} + # JUnitXml may be a TestSuite or a TestSuites container + suites = xml if hasattr(xml, "__iter__") else [xml] + for suite in suites: + suite_name = suite.name or "" + for case in suite: + key = (suite_name, case.name or "", case.classname or "") + results[key] = (_case_status(case), _case_message(case)) + return results + + +# ── classification ──────────────────────────────────────────────────────────── +def classify(prev, curr): + prev_bad = prev in (FAIL, ERROR, SKIP) + curr_bad = curr in (FAIL, ERROR, SKIP) + if prev_bad and not curr_bad: return "Fixed" + if not prev_bad and curr_bad: return "Regressed" + if prev_bad and curr_bad: return "Still Failing" + return "Pass" + +def build_rows(prev_data: dict, curr_data: dict) -> list: + all_keys = set(prev_data) | set(curr_data) + rows = [] + for key in sorted(all_keys): + suite, test, classname = key + prev_status, _ = prev_data.get(key, (SKIP, "")) + curr_status, curr_msg = curr_data.get(key, (SKIP, "")) + rows.append({ + "suite": suite, + "test": test, + "platform": classname, + "prev": prev_status, + "curr": curr_status, + "status": classify(prev_status, curr_status), + "msg": curr_msg, + }) + return rows + + +# ── sheet builders ──────────────────────────────────────────────────────────── +def _title_row(ws, title, ncols): + ws.merge_cells(f"A1:{get_column_letter(ncols)}1") + ws["A1"].value = title + ws["A1"].font = TITLE_FONT + ws["A1"].fill = hex_fill(C_DARK_BLUE) + ws["A1"].alignment = CENTER_ALIGN + ws.row_dimensions[1].height = 22 + +def _header_row(ws, headers, hdr_color, row=2): + for i, h in enumerate(headers, 1): + c = ws.cell(row=row, column=i, value=h) + c.font = HEADER_FONT + c.fill = hex_fill(hdr_color) + c.alignment = CENTER_WRAP + c.border = all_borders() + ws.row_dimensions[row].height = 28 + +def build_full_comparison(wb, rows, prev_date, curr_date): + ws = wb.create_sheet("Full Comparison") + ws.freeze_panes = "A3" + + _title_row(ws, "Zephyr Twister Test Results – Full Comparison", 7) + headers = ["Suite", "Test Case", "Platform", + f"Status ({prev_date})", f"Status ({curr_date})", + "Change", "Failure Message"] + _header_row(ws, headers, C_MID_BLUE) + set_col_widths(ws, {"A":24,"B":34,"C":22,"D":14,"E":14,"F":16,"G":52}) + + for idx, r in enumerate(rows, 3): + bg = C_LIGHT_BLUE if idx % 2 == 0 else C_WHITE + for ci, val in enumerate( + [r["suite"], r["test"], r["platform"], r["prev"], r["curr"], r["status"], r["msg"]], 1 + ): + cell = ws.cell(row=idx, column=ci, value=val) + cell.border = all_borders() + cell.font = NORM_FONT + cell.alignment = Alignment(vertical="top", wrap_text=(ci == 7)) + if ci in (4, 5): + f, fn = status_style(val) + cell.fill = f; cell.font = fn; cell.alignment = CENTER_ALIGN + elif ci == 6: + f, fn = change_style(val) + cell.fill = f; cell.font = fn; cell.alignment = CENTER_ALIGN + else: + cell.fill = hex_fill(bg) + + # mini summary box + summary = [ + ("Total Tests", len(rows)), + ("Pass (curr)", sum(1 for r in rows if r["curr"] == PASS)), + ("Fail (curr)", sum(1 for r in rows if r["curr"] in (FAIL, ERROR))), + ("Skip (curr)", sum(1 for r in rows if r["curr"] == SKIP)), + ("Fixed", sum(1 for r in rows if r["status"] == "Fixed")), + ("Regressed", sum(1 for r in rows if r["status"] == "Regressed")), + ("Still Failing", sum(1 for r in rows if r["status"] == "Still Failing")), + ] + ws["I1"].value = "Summary" + ws["I1"].font = Font(bold=True, size=11, color=C_DARK_BLUE) + ws.merge_cells("I1:J1"); ws["I1"].alignment = CENTER_ALIGN + ws["I1"].fill = hex_fill("BDD7EE") + for i, (label, val) in enumerate(summary, 2): + ws.cell(row=i, column=9, value=label).font = SUB_FONT + c = ws.cell(row=i, column=10, value=val) + c.alignment = CENTER_ALIGN; c.font = NORM_FONT + ws.column_dimensions["I"].width = 18 + ws.column_dimensions["J"].width = 10 + + +def build_platform_summary(wb, rows, prev_date, curr_date): + ws = wb.create_sheet("Platform Summary") + _title_row(ws, "Platform Summary", 8) + headers = ["Platform", + f"Pass ({prev_date})", f"Fail ({prev_date})", f"Total ({prev_date})", + f"Pass ({curr_date})", f"Fail ({curr_date})", f"Total ({curr_date})", + "Δ Pass"] + _header_row(ws, headers, C_MID_BLUE) + set_col_widths(ws, {"A":22,"B":13,"C":13,"D":13,"E":13,"F":13,"G":13,"H":12}) + + plat = defaultdict(lambda: {"pp":0,"pf":0,"cp":0,"cf":0}) + for r in rows: + p = r["platform"] + plat[p]["pp"] += r["prev"] == PASS + plat[p]["pf"] += r["prev"] in (FAIL, ERROR, SKIP) + plat[p]["cp"] += r["curr"] == PASS + plat[p]["cf"] += r["curr"] in (FAIL, ERROR, SKIP) + + for idx, (platform, s) in enumerate(sorted(plat.items()), 3): + bg = C_LIGHT_BLUE if idx % 2 == 0 else C_WHITE + delta = s["cp"] - s["pp"] + vals = [platform, s["pp"], s["pf"], s["pp"]+s["pf"], + s["cp"], s["cf"], s["cp"]+s["cf"], delta] + for ci, val in enumerate(vals, 1): + cell = ws.cell(row=idx, column=ci, value=val) + cell.fill = hex_fill(bg); cell.border = all_borders() + cell.alignment = CENTER_ALIGN; cell.font = NORM_FONT + if ci == 8 and isinstance(val, int): + if val > 0: cell.font = Font(color="375623", bold=True, size=10) + elif val < 0: cell.font = Font(color="9C0006", bold=True, size=10) + + tr = len(plat) + 3 + ws.cell(row=tr, column=1, value="TOTAL").font = Font(bold=True, size=10) + for ci in range(2, 9): + col = get_column_letter(ci) + if ci <= 7: + ws.cell(row=tr, column=ci).value = f"=SUM({col}3:{col}{tr-1})" + else: + ws.cell(row=tr, column=ci).value = f"=E{tr}-B{tr}" + for ci in range(1, 9): + cell = ws.cell(row=tr, column=ci) + cell.fill = hex_fill("BDD7EE"); cell.font = Font(bold=True, size=10) + cell.border = all_borders(); cell.alignment = CENTER_ALIGN + + +def build_detail_sheet(wb, name, rows, title, hdr_color, row_color, prev_date, curr_date): + ws = wb.create_sheet(name) + _title_row(ws, title, 6) + headers = ["Suite", "Test Case", "Platform", + f"Status ({prev_date})", f"Status ({curr_date})", "Failure Message"] + _header_row(ws, headers, hdr_color) + set_col_widths(ws, {"A":24,"B":36,"C":22,"D":14,"E":14,"F":52}) + + if not rows: + ws.merge_cells("A3:F3") + ws["A3"] = "✅ No entries in this category." + ws["A3"].font = Font(italic=True, size=10) + ws["A3"].alignment = CENTER_ALIGN + return + + for idx, r in enumerate(rows, 3): + bg = row_color if idx % 2 != 0 else C_WHITE + for ci, val in enumerate( + [r["suite"], r["test"], r["platform"], r["prev"], r["curr"], r["msg"]], 1 + ): + cell = ws.cell(row=idx, column=ci, value=val) + cell.border = all_borders(); cell.font = NORM_FONT + cell.alignment = Alignment(vertical="top", wrap_text=(ci == 6)) + if ci in (4, 5): + f, fn = status_style(val); cell.fill = f; cell.font = fn + cell.alignment = CENTER_ALIGN + else: + cell.fill = hex_fill(bg) + + +# ── main ────────────────────────────────────────────────────────────────────── +def main(): + parser = argparse.ArgumentParser(description="Generate Twister comparison XLSX report.") + parser.add_argument("--prev", required=True, help="Previous run JUnit XML") + parser.add_argument("--curr", required=True, help="Current run JUnit XML") + parser.add_argument("--out", default="twister_comparison_report.xlsx") + parser.add_argument("--prev-date", default=None, help="Label for previous run e.g. 'Jun 14'") + parser.add_argument("--curr-date", default=None, help="Label for current run e.g. 'Jun 15'") + args = parser.parse_args() + + # auto-generate date labels from today if not supplied + today = datetime.utcnow() + curr_date = args.curr_date or today.strftime("%b %d") + prev_date = args.prev_date or "Prev Run" + + print(f"Parsing prev: {args.prev}") + prev_data = parse_xml(args.prev) + print(f"Parsing curr: {args.curr}") + curr_data = parse_xml(args.curr) + + rows = build_rows(prev_data, curr_data) + regressions = [r for r in rows if r["status"] == "Regressed"] + fixed = [r for r in rows if r["status"] == "Fixed"] + still_fail = [r for r in rows if r["status"] == "Still Failing"] + + print(f" Total : {len(rows)}") + print(f" Fixed : {len(fixed)} | Regressed : {len(regressions)} | Still Failing : {len(still_fail)}") + + wb = Workbook() + wb.remove(wb.active) + + build_full_comparison(wb, rows, prev_date, curr_date) + build_platform_summary(wb, rows, prev_date, curr_date) + build_detail_sheet(wb, "Regressions", regressions, + f"⚠️ Regressions ({prev_date} → {curr_date})", + "C00000", "FFE0E0", prev_date, curr_date) + build_detail_sheet(wb, "Fixed", fixed, + f"✅ Fixed ({prev_date} → {curr_date})", + "375623", "E2EFDA", prev_date, curr_date) + build_detail_sheet(wb, "Still Failing", still_fail, + f"🔴 Still Failing ({prev_date} → {curr_date})", + "7B3F00", "FFF2CC", prev_date, curr_date) + + wb.save(args.out) + print(f"Report saved → {args.out}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/zephyr-twister-tests.yml b/.github/workflows/zephyr-twister-tests.yml index 79a52448255a..68d4dc5a1ca6 100644 --- a/.github/workflows/zephyr-twister-tests.yml +++ b/.github/workflows/zephyr-twister-tests.yml @@ -139,3 +139,123 @@ jobs: with: name: zephyr-merged-twister-test-results path: merged_twister.xml + + #Generate comparison report xlxs format. + generate-comparison-report: + # Run even if tests or merging had failures — we still want a report. + if: always() && github.repository == 'qualcomm/cpullvm-toolchain' + name: Generate XLSX comparison report + needs: merge-twister-results + runs-on: ubuntu-24.04 + + steps: + # Sparse-checkout only the report script + # We only need one Python file from cpullvm-toolchain; sparse-checkout + # avoids pulling the entire repository history. + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + repository: qualcomm/cpullvm-toolchain + sparse-checkout: .github/scripts/generate_twister_report.py + + # Download THIS run's merged XML + # Fetches the artifact produced by the merge-twister-results job above + # and places it under current/merged_twister.xml. + - name: Download current merged XML + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: zephyr-merged-twister-test-results + path: current + + # Locate the most recent previously completed run + # Uses `gh run list` to find the last completed run of this workflow + # that is NOT the current run. The run ID is stored as a step output + # so the download and date-label steps can reference it. + - name: Find previous completed run ID + id: prev-run + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.run_id }} + run: | + PREV_RUN_ID=$(gh run list + --workflow zephyr-twister-tests.yml + --status completed + --limit 10 + --json databaseId,conclusion + --jq '[.[] | select(.databaseId != $ENV.RUN_ID)] | first | .databaseId' + --repo ${{ github.repository }}) + echo "Previous run ID: $PREV_RUN_ID" + echo "run_id=$PREV_RUN_ID" >> "$GITHUB_OUTPUT" + + # Download the previous run's merged XML + # `|| true` prevents the job from failing if the artifact has expired + # (artifacts are retained for 90 days by default). The next step handles + # the missing-file case by injecting an empty placeholder XML. + - name: Download previous merged XML + if: steps.prev-run.outputs.run_id != '' + env: + GH_TOKEN: ${{ github.token }} + run: | + gh run download ${{ steps.prev-run.outputs.run_id }} + --name zephyr-merged-twister-test-results + --dir previous + --repo ${{ github.repository }} || true + + # Inject empty placeholder XML when no previous run exists + # Handles two edge cases: (a) this is the very first ever run of the + # workflow, or (b) the previous artifact has already expired/been deleted. + # An empty is valid JUnit XML that the script treats as zero tests. + - name: Create empty previous XML if missing + run: | + if [ ! -f previous/merged_twister.xml ]; then + echo "No previous results found – creating empty placeholder." + echo '' > previous/merged_twister.xml + fi + + #Derive human-readable date labels for the report columns + # The XLSX report uses "MMM DD" column headers (e.g. "Jul 28" / "Jul 29"). + # Current date is taken from the runner clock; previous date is fetched + # from the GitHub API using the run ID resolved in Step 3. + - name: Compute date labels + id: dates + env: + GH_TOKEN: ${{ github.token }} + run: | + CURR_DATE=$(date -u +"%b %d") + if [ -n "${{ steps.prev-run.outputs.run_id }}" ]; then + PREV_DATE=$(gh run view ${{ steps.prev-run.outputs.run_id }} + --json createdAt --jq '.createdAt' + --repo ${{ github.repository }} + | xargs -I{} date -u -d {} +"%b %d" 2>/dev/null || echo "Prev Run") + else + PREV_DATE="Prev Run" + fi + echo "curr_date=$CURR_DATE" >> "$GITHUB_OUTPUT" + echo "prev_date=$PREV_DATE" >> "$GITHUB_OUTPUT" + + # Install Python dependencies for the report script + # openpyxl writes the .xlsx file; junitparser parses the merged XML files. + - name: Install Python dependencies + run: pip install junitparser openpyxl + + # Run the comparison report generator + # Passes both XMLs, date labels, and the output path to the script. + # Script lives at .github/scripts/generate_twister_report.py (checked out above). + - name: Generate XLSX report + run: | + python .github/scripts/generate_twister_report.py \ + --prev previous/merged_twister.xml \ + --curr current/merged_twister.xml \ + --out twister_comparison_report.xlsx \ + --prev-date "${{ steps.dates.outputs.prev_date }}" \ + --curr-date "${{ steps.dates.outputs.curr_date }}" + + # Upload the finished XLSX as a downloadable artifact + # `if: always()` ensures the report is uploaded even if some tests regressed. + # Downloadable from the Actions run summary page as 'zephyr-twister-comparison-report'. + - name: Upload XLSX report + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + if: always() + with: + name: zephyr-twister-comparison-report + path: twister_comparison_report.xlsx