Skip tests that runtimes don't support

wingo · wingo · commit dd87f08324c3 · 2025-09-24T16:54:55.000+02:00
Refactor to make runtime adapters declare the WASI versions they
support, and for test suites to declare the WASI version they are.  By
default, skip tests whose WASI version is not supported by the current
run-time.
diff --git a/adapters/pywasm.py b/adapters/pywasm.py
@@ -21,10 +21,14 @@ def get_version() -> str:
     return output[1]
 
 
+def get_wasi_versions() -> List[str]:
+    return ["wasm32-wasip1"]
+
 def compute_argv(test_path: str,
                  args: List[str],
                  env: Dict[str, str],
-                 dirs: List[Tuple[Path, str]]) -> List[str]:
+                 dirs: List[Tuple[Path, str]],
+                 wasi_version: str) -> List[str]:
     argv = [str(RUN_PYWASM)]
     for k, v in env.items():
         argv += ["--env", f"{k}={v}"]
diff --git a/adapters/wasm-micro-runtime.py b/adapters/wasm-micro-runtime.py
@@ -21,10 +21,15 @@ def get_version() -> str:
     return output[1]
 
 
+def get_wasi_versions() -> List[str]:
+    return ["wasm32-wasip1"]
+
+
 def compute_argv(test_path: str,
                  args: List[str],
                  env: Dict[str, str],
-                 dirs: List[Tuple[Path, str]]) -> List[str]:
+                 dirs: List[Tuple[Path, str]],
+                 wasi_version: str) -> List[str]:
     argv = [] + IWASM
     for k, v in env.items():
         argv += ["--env", f"{k}={v}"]
diff --git a/adapters/wasmedge.py b/adapters/wasmedge.py
@@ -21,10 +21,15 @@ def get_version() -> str:
     return output[1]
 
 
+def get_wasi_versions() -> List[str]:
+    return ["wasm32-wasip1"]
+
+
 def compute_argv(test_path: str,
                  args: List[str],
                  env: Dict[str, str],
-                 dirs: List[Tuple[Path, str]]) -> List[str]:
+                 dirs: List[Tuple[Path, str]],
+                 wasi_version: str) -> List[str]:
     argv = [] + WASMEDGE
     for k, v in env.items():
         argv += ["--env", f"{k}={v}"]
diff --git a/adapters/wasmtime.py b/adapters/wasmtime.py
@@ -21,10 +21,15 @@ def get_version() -> str:
     return output[1]
 
 
+def get_wasi_versions() -> List[str]:
+    return ["wasm32-wasip1", "wasm32-wasip3"]
+
+
 def compute_argv(test_path: str,
                  args: List[str],
                  env: Dict[str, str],
-                 dirs: List[Tuple[Path, str]]) -> List[str]:
+                 dirs: List[Tuple[Path, str]],
+                 wasi_version: str) -> List[str]:
     argv = [] + WASMTIME
     for k, v in env.items():
         argv += ["--env", f"{k}={v}"]
diff --git a/adapters/wazero.py b/adapters/wazero.py
@@ -23,10 +23,15 @@ def get_version() -> str:
     return version
 
 
+def get_wasi_versions() -> List[str]:
+    return ["wasm32-wasip1"]
+
+
 def compute_argv(test_path: str,
                  args: List[str],
                  env: Dict[str, str],
-                 dirs: List[Tuple[Path, str]]) -> List[str]:
+                 dirs: List[Tuple[Path, str]],
+                 wasi_version: str) -> List[str]:
     argv = WAZERO + ["run", "-hostlogging=filesystem"]
     for k, v in env.items():
         argv += [f"-env={k}={v}"]
diff --git a/adapters/wizard.py b/adapters/wizard.py
@@ -29,10 +29,15 @@ def get_version() -> str:
     return output[1]
 
 
+def get_wasi_versions() -> List[str]:
+    return ["wasm32-wasip1"]
+
+
 def compute_argv(test_path: str,
                  args: List[str],
                  env: Dict[str, str],
-                 dirs: List[Tuple[Path, str]]) -> List[str]:
+                 dirs: List[Tuple[Path, str]],
+                 wasi_version: str) -> List[str]:
     argv = [] + WIZARD
     for k, v in env.items():
         argv += [f"--env={k}={v}"]
diff --git a/run-tests b/run-tests
@@ -57,7 +57,7 @@ def find_runtime_adapters(root, verbose=False):
     for candidate in root.glob("*.py"):
         try:
             adapter = runtime_adapter.RuntimeAdapter(candidate)
-            print(f"  {candidate.name}: {adapter.get_version()}")
+            print(f"  {candidate.name}: {adapter.get_meta()}")
             adapters.append(adapter)
         except runtime_adapter.LegacyRuntimeAdapterError:
             print(f"  {candidate} is too old; update to new module format.")
@@ -85,9 +85,7 @@ if options.runtime_adapter:
 else:
     runtime_adapters = find_runtime_adapters(Path(__file__).parent / "adapters")
 
-exclude_filters = [JSONTestExcludeFilter(filt) for filt in options.exclude_filter]
-
 sys.exit(run_tests(runtime_adapters, test_suite,
                    color=not options.disable_colors,
                    json_log_file=options.json_output_location,
-                   exclude_filters=exclude_filters))
+                   exclude_filters=options.exclude_filter))
diff --git a/test-runner/tests/test_test_suite.py b/test-runner/tests/test_test_suite.py
@@ -2,7 +2,7 @@
 
 import wasi_test_runner.test_case as tc
 import wasi_test_runner.test_suite as ts
-from wasi_test_runner.runtime_adapter import RuntimeVersion
+from wasi_test_runner.runtime_adapter import RuntimeMeta
 
 
 def create_test_case(name: str, is_executed: bool, is_failed: bool) -> tc.TestCase:
@@ -18,8 +18,10 @@ def create_test_case(name: str, is_executed: bool, is_failed: bool) -> tc.TestCa
 
 def test_test_suite_should_return_correct_count() -> None:
     suite = ts.TestSuite(
-        "suite",
-        RuntimeVersion("test-runtime", "3.14"),
+        ts.TestSuiteMeta("suite",
+                         tc.WasiVersion.WASM32_WASIP1,
+                         RuntimeMeta("test-runtime", "3.14",
+                                     frozenset([tc.WasiVersion.WASM32_WASIP1]))),
         10.0,
         datetime.now(),
         [
diff --git a/test-runner/tests/test_test_suite_runner.py b/test-runner/tests/test_test_suite_runner.py
@@ -2,9 +2,10 @@
 from pathlib import Path
 from unittest.mock import ANY, MagicMock, Mock, patch, mock_open
 
+import wasi_test_runner.test_suite as ts
 import wasi_test_runner.test_case as tc
 import wasi_test_runner.test_suite_runner as tsr
-from wasi_test_runner.runtime_adapter import RuntimeVersion
+from wasi_test_runner.runtime_adapter import RuntimeMeta
 
 
 def get_mock_open() -> Mock:
@@ -26,6 +27,7 @@ def open_mock(filename: str, *_args: Any, **_kwargs: Any) -> Any:
 # pylint: disable-msg=too-many-locals
 @patch("builtins.open", get_mock_open())
 @patch("os.path.exists", Mock(return_value=True))
+@patch("pathlib.Path.exists", Mock(return_value=True))
 def test_runner_end_to_end() -> None:
     test_suite_dir = "my-path"
     test_suite_name = "test-suite"
@@ -52,7 +54,14 @@ def test_runner_end_to_end() -> None:
 
     runtime_name = "rt1"
     runtime_version_str = "4.2"
-    runtime_version = RuntimeVersion(runtime_name, runtime_version_str)
+    the_runtime_wasi_version = tc.WasiVersion.WASM32_WASIP1
+    runtime_wasi_versions = frozenset([the_runtime_wasi_version])
+    runtime_meta = RuntimeMeta(runtime_name, runtime_version_str,
+                               runtime_wasi_versions)
+
+    expected_test_suite_meta = ts.TestSuiteMeta(test_suite_name,
+                                                the_runtime_wasi_version,
+                                                runtime_meta)
 
     expected_argv = [runtime_name, "<test>"]
     expected_test_cases = [
@@ -64,7 +73,7 @@ def test_runner_end_to_end() -> None:
 
     runtime = Mock()
     runtime.get_name.return_value = runtime_name
-    runtime.get_version.return_value = runtime_version
+    runtime.get_meta.return_value = runtime_meta
     runtime.run_test.side_effect = outputs
     runtime.compute_argv.return_value = expected_argv
 
@@ -87,7 +96,7 @@ def test_runner_end_to_end() -> None:
                                               filters)     # type: ignore
 
     # Assert manifest was read correctly
-    assert suite.name == test_suite_name
+    assert suite.meta == expected_test_suite_meta
 
     # Assert test cases
     assert suite.test_count == 3
@@ -98,16 +107,16 @@ def test_runner_end_to_end() -> None:
     for test_path, config in zip(test_paths, expected_config):
         expected_dirs = [(Path(test_suite_dir) / d, d) for d in config.dirs]
         runtime.compute_argv.assert_any_call(
-            str(test_path), config.args, config.env, expected_dirs
+            str(test_path), config.args, config.env, expected_dirs,
+            "wasm32-wasip1"
         )
         runtime.run_test.assert_called_with(expected_argv)
 
     # Assert reporters calls
     for reporter in reporters:
         assert reporter.report_test.call_count == 3
         for test_case in expected_test_cases:
-            reporter.report_test.assert_any_call(test_suite_name,
-                                                 runtime_version,
+            reporter.report_test.assert_any_call(expected_test_suite_meta,
                                                  test_case)
 
     # Assert validators calls
@@ -120,12 +129,12 @@ def test_runner_end_to_end() -> None:
     for filt in filters:
         assert filt.should_skip.call_count == 3
         for test_case in expected_test_cases:
-            filt.should_skip.assert_any_call(runtime_version, suite.name,
+            filt.should_skip.assert_any_call(expected_test_suite_meta,
                                              test_case.name)
 
 
 @patch("os.path.exists", Mock(return_value=False))
 def test_runner_should_use_path_for_name_if_manifest_does_not_exist() -> None:
     suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], [], [])
 
-    assert suite.name == "my-path"
+    assert suite.meta.name == "my-path"
diff --git a/test-runner/wasi_test_runner/__main__.py b/test-runner/wasi_test_runner/__main__.py
@@ -6,7 +6,7 @@
 from .runtime_adapter import RuntimeAdapter
 from .harness import run_all_tests
 from .filters import TestFilter
-from .filters import JSONTestExcludeFilter
+from .filters import JSONTestExcludeFilter, UnsupportedWasiTestExcludeFilter
 from .reporters import TestReporter
 from .reporters.console import ConsoleTestReporter
 from .reporters.json import JSONTestReporter
@@ -56,7 +56,7 @@ def main() -> int:
 
     validators: List[Validator] = [exit_code_validator, stdout_validator]
 
-    filters: List[TestFilter] = []
+    filters: List[TestFilter] = [UnsupportedWasiTestExcludeFilter()]
     for filt in options.exclude_filter:
         filters.append(JSONTestExcludeFilter(filt))
 
diff --git a/test-runner/wasi_test_runner/filters.py b/test-runner/wasi_test_runner/filters.py
@@ -4,26 +4,35 @@
 
 import json
 
-from .runtime_adapter import RuntimeVersion
+from .test_suite import TestSuiteMeta
 
 
 class TestFilter(ABC):
     @abstractmethod
     def should_skip(
-        self, runtime: RuntimeVersion, test_suite_name: str, test_name: str
+        self, meta: TestSuiteMeta, test_name: str
     ) -> Union[Tuple[Literal[True], str], Tuple[Literal[False], Literal[None]]]:
         pass
 
 
+class UnsupportedWasiTestExcludeFilter(TestFilter):
+    def should_skip(
+        self, meta: TestSuiteMeta, test_name: str
+    ) -> Union[Tuple[Literal[True], str], Tuple[Literal[False], Literal[None]]]:
+        if meta.wasi_version not in meta.runtime.supported_wasi_versions:
+            return True, "WASI version unsupported by runtime"
+        return False, None
+
+
 class JSONTestExcludeFilter(TestFilter):
     def __init__(self, filename: str) -> None:
         with open(filename, encoding="utf-8") as file:
             self.filter_dict = json.load(file)
 
     def should_skip(
-        self, runtime: RuntimeVersion, test_suite_name: str, test_name: str
+        self, meta: TestSuiteMeta, test_name: str
     ) -> Union[Tuple[Literal[True], str], Tuple[Literal[False], Literal[None]]]:
-        test_suite_filter = self.filter_dict.get(test_suite_name)
+        test_suite_filter = self.filter_dict.get(meta.name)
         if test_suite_filter is None:
             return False, None
         why = test_suite_filter.get(test_name)
diff --git a/test-runner/wasi_test_runner/harness.py b/test-runner/wasi_test_runner/harness.py
@@ -1,7 +1,9 @@
 from typing import List
 from pathlib import Path
 
-from .filters import TestFilter, JSONTestExcludeFilter
+from .filters import (
+    TestFilter, JSONTestExcludeFilter, UnsupportedWasiTestExcludeFilter
+)
 from .reporters import TestReporter
 from .reporters.console import ConsoleTestReporter
 from .reporters.json import JSONTestReporter
@@ -19,9 +21,9 @@ def run_tests(runtimes: List[RuntimeAdapter],
     reporters: List[TestReporter] = [ConsoleTestReporter(color)]
     if json_log_file:
         reporters.append(JSONTestReporter(json_log_file))
-    filters: List[TestFilter] = []
+    filters: List[TestFilter] = [UnsupportedWasiTestExcludeFilter()]
     if exclude_filters is not None:
-        filters = [JSONTestExcludeFilter(str(filt)) for filt in exclude_filters]
+        filters += [JSONTestExcludeFilter(str(filt)) for filt in exclude_filters]
 
     return run_all_tests(runtimes, [str(p) for p in test_suite_paths],
                          validators, reporters, filters)
diff --git a/test-runner/wasi_test_runner/reporters/__init__.py b/test-runner/wasi_test_runner/reporters/__init__.py
@@ -1,13 +1,11 @@
 from abc import ABC
 
 from ..test_case import TestCase
-from ..test_suite import TestSuite
-from ..runtime_adapter import RuntimeVersion
+from ..test_suite import TestSuite, TestSuiteMeta
 
 
 class TestReporter(ABC):
-    def report_test(self, test_suite_name: str, runtime: RuntimeVersion,
-                    test: TestCase) -> None:
+    def report_test(self, meta: TestSuiteMeta, test: TestCase) -> None:
         pass
 
     def report_test_suite(self, test_suite: TestSuite) -> None:
diff --git a/test-runner/wasi_test_runner/reporters/console.py b/test-runner/wasi_test_runner/reporters/console.py
@@ -4,8 +4,8 @@
 
 from . import TestReporter
 from ..test_case import TestCase
-from ..test_suite import TestSuite
-from ..runtime_adapter import RuntimeVersion
+from ..test_suite import TestSuite, TestSuiteMeta
+from ..runtime_adapter import RuntimeMeta
 
 
 class ConsoleTestReporter(TestReporter):
@@ -18,15 +18,14 @@ def __init__(self, colored: bool = True, verbose: bool = False) -> None:
         super().__init__()
         init(autoreset=True)
         self._test_suites: List[TestSuite] = []
-        self._current_test_suite: Optional[str] = None
+        self._current_test_suite: Optional[TestSuiteMeta] = None
         self._colored = colored
         self._verbose = verbose
 
-    def report_test(self, test_suite_name: str, runtime: RuntimeVersion,
-                    test: TestCase) -> None:
+    def report_test(self, meta: TestSuiteMeta, test: TestCase) -> None:
         if self._current_test_suite is None:
-            print(f"Running test suite {test_suite_name} with {runtime}")
-            self._current_test_suite = test_suite_name
+            print(f"Running test suite {meta.name} with {meta.runtime}")
+            self._current_test_suite = meta
 
         if self._verbose:
             self._report_test_verbose(test)
@@ -63,16 +62,16 @@ def report_test_suite(self, test_suite: TestSuite) -> None:
     def finalize(self) -> None:
         print("===== Test results =====")
 
-        test_suites_by_runtime: Dict[RuntimeVersion, List[TestSuite]] = {}
+        test_suites_by_runtime: Dict[RuntimeMeta, List[TestSuite]] = {}
         for suite in self._test_suites:
-            if suite.runtime not in test_suites_by_runtime:
-                test_suites_by_runtime[suite.runtime] = []
-            test_suites_by_runtime[suite.runtime].append(suite)
+            if suite.meta.runtime not in test_suites_by_runtime:
+                test_suites_by_runtime[suite.meta.runtime] = []
+            test_suites_by_runtime[suite.meta.runtime].append(suite)
 
         for runtime, test_suites in test_suites_by_runtime.items():
             self._print_result_for_runtime(runtime, test_suites)
 
-    def _print_result_for_runtime(self, runtime: RuntimeVersion,
+    def _print_result_for_runtime(self, runtime: RuntimeMeta,
                                   suites: List[TestSuite]) -> None:
         total_skip = total_pass = total_fail = 0
 
diff --git a/test-runner/wasi_test_runner/reporters/json.py b/test-runner/wasi_test_runner/reporters/json.py
@@ -23,10 +23,10 @@ def finalize(self) -> None:
         for suite in self._test_suites:
             results.append(
                 {
-                    "name": suite.name,
+                    "name": suite.meta.name,
                     "runtime": {
-                        "name": suite.runtime.name,
-                        "version": suite.runtime.version
+                        "name": suite.meta.runtime.name,
+                        "version": suite.meta.runtime.version
                     },
                     "duration_s": suite.duration_s,
                     "failed": suite.fail_count,
diff --git a/test-runner/wasi_test_runner/runtime_adapter.py b/test-runner/wasi_test_runner/runtime_adapter.py
diff --git a/test-runner/wasi_test_runner/test_case.py b/test-runner/wasi_test_runner/test_case.py
diff --git a/test-runner/wasi_test_runner/test_suite.py b/test-runner/wasi_test_runner/test_suite.py
diff --git a/test-runner/wasi_test_runner/test_suite_runner.py b/test-runner/wasi_test_runner/test_suite_runner.py
