diff --git a/notebooks/how_to/tests/custom_tests/implement_custom_tests.ipynb b/notebooks/how_to/tests/custom_tests/implement_custom_tests.ipynb index 8e4be82ac..c1a1c48f8 100644 --- a/notebooks/how_to/tests/custom_tests/implement_custom_tests.ipynb +++ b/notebooks/how_to/tests/custom_tests/implement_custom_tests.ipynb @@ -92,7 +92,7 @@ "\n", "### Key concepts\n", "\n", - "**Model documentation**: A structured and detailed record pertaining to a model, encompassing key components such as its underlying assumptions, methodologies, data sources, inputs, performance metrics, evaluations, limitations, and intended uses. It serves to ensure transparency, adherence to regulatory requirements, and a clear understanding of potential risks associated with the model’s application.\n", + "**Model documentation**: A structured and detailed record pertaining to a model, encompassing key components such as its underlying assumptions, methodologies, data sources, inputs, performance metrics, evaluations, limitations, and intended uses. It serves to ensure transparency, adherence to regulatory requirements, and a clear understanding of potential risks associated with the model\u2019s application.\n", "\n", "**Documentation template**: Functions as a test suite and lays out the structure of model documentation, segmented into various sections and sub-sections. Documentation templates define the structure of your model documentation, specifying the tests that should be run, and how the results should be displayed.\n", "\n", @@ -801,9 +801,14 @@ " \"My Cool Table\": table,\n", " \"Another Table\": table2,\n", " },\n", - " fig1,\n", - " fig2,\n", - " fig3,\n", + " {\n", + " # Figures support the same dict-of-titles convention as tables.\n", + " # These titles flow into the document media registry as\n", + " # \"Figure N. \" alongside table captions.\n", + " \"Random Line Plot\": fig1,\n", + " \"Random Bar Plot\": fig2,\n", + " \"Random Scatter Plot\": fig3,\n", + " },\n", " )\n", "\n", "\n", @@ -815,7 +820,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Notice how you can return the tables as a dictionary where the key is the title of the table and the value is the table itself. You could also just return the tables by themselves but this way you can give them a title to more easily identify them in the result.\n", + "Notice how you can return the tables as a dictionary where the key is the title of the table and the value is the table itself. The same convention works for **figures** \u2014 wrap them in a dict whose keys are the titles you want shown in the document media registry (e.g. *Figure 7. Random Line Plot*). You could also just return the figures by themselves but this way you can give them a title to more easily identify them in the result.\n", "\n", "![screenshot showing multiple tables and plots](./multiple-tables-plots-custom-metric.png)" ] @@ -990,7 +995,7 @@ "\n", "## Next steps\n", "\n", - "You can look at the results of this test suite right in the notebook where you ran the code, as you would expect. But there is a better way — use the ValidMind Platform to work with your model documentation.\n", + "You can look at the results of this test suite right in the notebook where you ran the code, as you would expect. But there is a better way \u2014 use the ValidMind Platform to work with your model documentation.\n", "\n", "<a id='toc8_1__'></a>\n", "\n", @@ -1023,7 +1028,7 @@ "\n", "## Upgrade ValidMind\n", "\n", - "<div class=\"alert alert-block alert-info\" style=\"background-color: #B5B5B510; color: black; border: 1px solid #083E44; border-left-width: 5px; box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.2);border-radius: 5px;\">After installing ValidMind, you’ll want to periodically make sure you are on the latest version to access any new features and other enhancements.</div>\n", + "<div class=\"alert alert-block alert-info\" style=\"background-color: #B5B5B510; color: black; border: 1px solid #083E44; border-left-width: 5px; box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.2);border-radius: 5px;\">After installing ValidMind, you\u2019ll want to periodically make sure you are on the latest version to access any new features and other enhancements.</div>\n", "\n", "Retrieve the information for the currently installed version of ValidMind:" ] @@ -1066,7 +1071,7 @@ "\n", "***\n", "\n", - "Copyright © 2023-2026 ValidMind Inc. All rights reserved.<br>\n", + "Copyright \u00a9 2023-2026 ValidMind Inc. All rights reserved.<br>\n", "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.<br>\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial</small>" ] diff --git a/tests/test_results.py b/tests/test_results.py index 87b74c3c0..035dadab9 100644 --- a/tests/test_results.py +++ b/tests/test_results.py @@ -501,6 +501,85 @@ def test_figure_interactive_toggle_matplotlib_unaffected(self): self.assertIn("data:image/png;base64", html) self.assertIn("vm-img-test_key", html) + def test_figure_title_serializes_as_caption(self): + """Figure.title should be serialized into metadata as `caption` so it + flows into the platform's document media registry (Figure N. <caption>). + """ + import json as _json + + plotly_fig = go.Figure(data=go.Scatter(x=[1, 2, 3], y=[4, 5, 6])) + + # With a title -> metadata.caption is set + titled = Figure( + key="k", figure=plotly_fig, ref_id="r1", title="My Cool Chart" + ) + payload = titled.serialize() + meta = _json.loads(payload["metadata"]) + self.assertEqual(meta["_ref_id"], "r1") + self.assertEqual(meta["caption"], "My Cool Chart") + + # Without a title -> no caption key (back-compat) + untitled = Figure(key="k", figure=plotly_fig, ref_id="r2") + meta_untitled = _json.loads(untitled.serialize()["metadata"]) + self.assertEqual(meta_untitled, {"_ref_id": "r2"}) + self.assertNotIn("caption", meta_untitled) + + def test_result_table_title_serializes_as_caption(self): + """ResultTable.title should be serialized into metadata under both + `title` (back-compat) and `caption` (consumed by the caption registry).""" + df = pd.DataFrame({"col1": [1, 2, 3]}) + + titled = ResultTable(data=df, title="Top Features") + payload = titled.serialize() + self.assertEqual(payload["metadata"]["title"], "Top Features") + self.assertEqual(payload["metadata"]["caption"], "Top Features") + + untitled = ResultTable(data=df) + self.assertNotIn("metadata", untitled.serialize()) + + def test_figure_output_handler_dict_assigns_titles(self): + """Returning ``{"My Chart Title": fig, ...}`` from a test should produce + Figure objects with the dict key applied as ``title``.""" + from validmind.tests.output import FigureOutputHandler + + png = b"\x89PNG\r\n\x1a\n" + b"\x00" * 16 + + result = TestResult(result_id="my.test", ref_id="ref-1") + handler = FigureOutputHandler() + + self.assertTrue(handler.can_handle({"A": png, "B": png})) + # dict of non-figures should NOT be claimed by FigureOutputHandler + self.assertFalse(handler.can_handle({"x": 1, "y": 2})) + + handler.process({"Chart A": png, "Chart B": png}, result) + + self.assertEqual(len(result.figures), 2) + self.assertEqual( + sorted(f.title for f in result.figures), ["Chart A", "Chart B"] + ) + # Each figure gets a unique key under the same result_id/ref_id + keys = [f.key for f in result.figures] + self.assertEqual(len(set(keys)), 2) + for f in result.figures: + self.assertTrue(f.key.startswith("my.test:")) + self.assertEqual(f.ref_id, "ref-1") + + def test_figure_output_handler_preserves_explicit_figure_title(self): + """If a user passes a pre-built Figure with an explicit title, the + dict-key wrapper must not overwrite it.""" + from validmind.tests.output import FigureOutputHandler + + png = b"\x89PNG\r\n\x1a\n" + b"\x00" * 16 + explicit = Figure( + key="explicit_key", figure=png, ref_id="ref-1", title="User Title" + ) + + result = TestResult(result_id="my.test", ref_id="ref-1") + FigureOutputHandler().process({"Dict Key Title": explicit}, result) + + self.assertEqual(len(result.figures), 1) + self.assertEqual(result.figures[0].title, "User Title") + if __name__ == "__main__": unittest.main() diff --git a/validmind/tests/decorator.py b/validmind/tests/decorator.py index c9aa6db38..c8db0b021 100644 --- a/validmind/tests/decorator.py +++ b/validmind/tests/decorator.py @@ -108,6 +108,24 @@ def test(func_or_id: Union[Callable[..., Any], str, None]) -> Callable[[F], F]: - Scalar: A single number (int or float) - Boolean: A single boolean value indicating whether the test passed or failed + Titled tables and figures + ------------------------- + Tables and figures may be titled by returning a dict whose keys are the + titles. The titles are picked up by the platform's caption registry and + rendered as e.g. ``Table 3. Top Features`` / ``Figure 7. Calibration Curve`` + in the model documentation:: + + @vm.test("my_test_id") + def my_test(dataset, model): + return ( + {"Top Features": features_df}, # titled table + {"Calibration Curve": calibration_fig}, # titled figure + ) + + Tables also accept ``ResultTable(data=df, title="…")`` and figures also + accept ``Figure(figure=fig, key=…, ref_id=…, title="…")`` for callers that + want to construct the objects explicitly. + The function may also include a docstring. This docstring will be used and logged as the metric's description. diff --git a/validmind/tests/output.py b/validmind/tests/output.py index 648d3b98d..d0f3835b8 100644 --- a/validmind/tests/output.py +++ b/validmind/tests/output.py @@ -54,7 +54,8 @@ def process(self, item: Any, result: TestResult) -> None: class FigureOutputHandler(OutputHandler): - def can_handle(self, item: Any) -> bool: + @staticmethod + def _is_figure_like(item: Any) -> bool: return ( isinstance(item, Figure) or is_matplotlib_figure(item) @@ -62,8 +63,23 @@ def can_handle(self, item: Any) -> bool: or is_png_image(item) ) - def process(self, item: Any, result: TestResult) -> None: + def can_handle(self, item: Any) -> bool: + if self._is_figure_like(item): + return True + # Allow `{"My Chart Title": fig, ...}` so figures can be titled the + # same way tables already are (the dict key becomes the figure title). + if ( + isinstance(item, dict) + and item + and all(self._is_figure_like(v) for v in item.values()) + ): + return True + return False + + def _add_one(self, item: Any, result: TestResult, title: str = None) -> None: if isinstance(item, Figure): + if title and not item.title: + item.title = title result.add_figure(item) else: random_id = str(uuid4())[:4] @@ -72,9 +88,17 @@ def process(self, item: Any, result: TestResult) -> None: key=f"{result.result_id}:{random_id}", figure=item, ref_id=result.ref_id, + title=title, ) ) + def process(self, item: Any, result: TestResult) -> None: + if isinstance(item, dict): + for title, fig in item.items(): + self._add_one(fig, result, title=title or None) + return + self._add_one(item, result) + class TableOutputHandler(OutputHandler): def can_handle(self, item: Any) -> bool: diff --git a/validmind/vm_models/figure.py b/validmind/vm_models/figure.py index b10717a8e..243cdde14 100644 --- a/validmind/vm_models/figure.py +++ b/validmind/vm_models/figure.py @@ -11,7 +11,7 @@ import os from dataclasses import dataclass from io import BytesIO -from typing import Union +from typing import Optional, Union import matplotlib import plotly.graph_objs as go @@ -38,10 +38,11 @@ def create_figure( figure: Union[matplotlib.figure.Figure, go.Figure, go.FigureWidget, bytes], key: str, ref_id: str, + title: Optional[str] = None, ) -> "Figure": """Create a VM Figure object from a raw figure object.""" if is_matplotlib_figure(figure) or is_plotly_figure(figure) or is_png_image(figure): - return Figure(key=key, figure=figure, ref_id=ref_id) + return Figure(key=key, figure=figure, ref_id=ref_id, title=title) raise ValueError(f"Unsupported figure type: {type(figure)}") @@ -50,11 +51,20 @@ def create_figure( class Figure: """ Figure objects track the schema supported by the ValidMind API. + + Attributes: + key: Unique identifier for the figure within a test run. + figure: The underlying figure object (matplotlib, plotly, or PNG bytes). + ref_id: ID used to link the figure to its parent test result. + title: Optional caption/title for the figure. When set, it is sent to + the platform as ``metadata.caption`` and rendered by the document + media registry as ``Figure N. <title>``. """ key: str figure: Union[matplotlib.figure.Figure, go.Figure, go.FigureWidget, bytes] ref_id: str # used to link figures to results + title: Optional[str] = None # caption/title used by the document media registry _type: str = "plot" # for now this is the only figure type @@ -109,10 +119,14 @@ def serialize(self): """ Serializes the Figure to a dictionary so it can be sent to the API. """ + metadata = {"_ref_id": self.ref_id} + if self.title: + metadata["caption"] = self.title + return { "type": self._type, "key": self.key, - "metadata": json.dumps({"_ref_id": self.ref_id}, allow_nan=False), + "metadata": json.dumps(metadata, allow_nan=False), } def _get_b64_url(self): diff --git a/validmind/vm_models/result/result.py b/validmind/vm_models/result/result.py index 90fe63a02..88ac7968c 100644 --- a/validmind/vm_models/result/result.py +++ b/validmind/vm_models/result/result.py @@ -90,6 +90,12 @@ def serialize(self) -> Dict[str, Any]: class ResultTable: """ A dataclass that holds the table summary of result. + + Attributes: + data: The table data as a list of dicts or a pandas DataFrame. + title: Optional caption/title for the table. When set, it is sent to + the platform as ``metadata.caption`` and rendered by the document + media registry as ``Table N. <title>``. """ data: Union[List[Any], pd.DataFrame] @@ -111,7 +117,10 @@ def serialize(self): } if self.title: - data["metadata"] = {"title": self.title} + # `title` is the user-facing attribute name; we serialize it as + # `caption` so it flows into the document media registry on the + # platform side (Table N. <caption>). + data["metadata"] = {"title": self.title, "caption": self.title} return data