Skip tests that runtimes don't support

adapters/pywasm.py

@@ -21,10 +21,14 @@ def get_version() -> str:
     return output[1]
 
 
+def get_wasi_versions() -> List[str]:
+    return ["wasm32-wasip1"]
+
 def compute_argv(test_path: str,
                  args: List[str],
                  env: Dict[str, str],
-                 dirs: List[Tuple[Path, str]]) -> List[str]:
+                 dirs: List[Tuple[Path, str]],
+                 wasi_version: str) -> List[str]:
     argv = [str(RUN_PYWASM)]
     for k, v in env.items():
         argv += ["--env", f"{k}={v}"]

adapters/wasm-micro-runtime.py

@@ -21,10 +21,15 @@ def get_version() -> str:
     return output[1]
 
 
+def get_wasi_versions() -> List[str]:
+    return ["wasm32-wasip1"]
+
+
 def compute_argv(test_path: str,
                  args: List[str],
                  env: Dict[str, str],
-                 dirs: List[Tuple[Path, str]]) -> List[str]:
+                 dirs: List[Tuple[Path, str]],
+                 wasi_version: str) -> List[str]:
     argv = [] + IWASM
     for k, v in env.items():
         argv += ["--env", f"{k}={v}"]

adapters/wasmedge.py

@@ -21,10 +21,15 @@ def get_version() -> str:
     return output[1]
 
 
+def get_wasi_versions() -> List[str]:
+    return ["wasm32-wasip1"]
+
+
 def compute_argv(test_path: str,
                  args: List[str],
                  env: Dict[str, str],
-                 dirs: List[Tuple[Path, str]]) -> List[str]:
+                 dirs: List[Tuple[Path, str]],
+                 wasi_version: str) -> List[str]:
     argv = [] + WASMEDGE
     for k, v in env.items():
         argv += ["--env", f"{k}={v}"]

adapters/wasmtime.py

@@ -21,10 +21,15 @@ def get_version() -> str:
     return output[1]
 
 
+def get_wasi_versions() -> List[str]:
+    return ["wasm32-wasip1", "wasm32-wasip3"]
+
+
 def compute_argv(test_path: str,
                  args: List[str],
                  env: Dict[str, str],
-                 dirs: List[Tuple[Path, str]]) -> List[str]:
+                 dirs: List[Tuple[Path, str]],
+                 wasi_version: str) -> List[str]:
     argv = [] + WASMTIME
     for k, v in env.items():
         argv += ["--env", f"{k}={v}"]

adapters/wazero.py

@@ -23,10 +23,15 @@ def get_version() -> str:
     return version
 
 
+def get_wasi_versions() -> List[str]:
+    return ["wasm32-wasip1"]
+
+
 def compute_argv(test_path: str,
                  args: List[str],
                  env: Dict[str, str],
-                 dirs: List[Tuple[Path, str]]) -> List[str]:
+                 dirs: List[Tuple[Path, str]],
+                 wasi_version: str) -> List[str]:
     argv = WAZERO + ["run", "-hostlogging=filesystem"]
     for k, v in env.items():
         argv += [f"-env={k}={v}"]

adapters/wizard.py

@@ -29,10 +29,15 @@ def get_version() -> str:
     return output[1]
 
 
+def get_wasi_versions() -> List[str]:
+    return ["wasm32-wasip1"]
+
+
 def compute_argv(test_path: str,
                  args: List[str],
                  env: Dict[str, str],
-                 dirs: List[Tuple[Path, str]]) -> List[str]:
+                 dirs: List[Tuple[Path, str]],
+                 wasi_version: str) -> List[str]:
     argv = [] + WIZARD
     for k, v in env.items():
         argv += [f"--env={k}={v}"]

run-tests

@@ -57,7 +57,7 @@ def find_runtime_adapters(root, verbose=False):
     for candidate in root.glob("*.py"):
         try:
             adapter = runtime_adapter.RuntimeAdapter(candidate)
-            print(f"  {candidate.name}: {adapter.get_version()}")
+            print(f"  {candidate.name}: {adapter.get_meta()}")
             adapters.append(adapter)
         except runtime_adapter.LegacyRuntimeAdapterError:
             print(f"  {candidate} is too old; update to new module format.")
@@ -85,9 +85,7 @@ if options.runtime_adapter:
 else:
     runtime_adapters = find_runtime_adapters(Path(__file__).parent / "adapters")
 
-exclude_filters = [JSONTestExcludeFilter(filt) for filt in options.exclude_filter]
-
 sys.exit(run_tests(runtime_adapters, test_suite,
                    color=not options.disable_colors,
                    json_log_file=options.json_output_location,
-                   exclude_filters=exclude_filters))
+                   exclude_filters=options.exclude_filter))

test-runner/tests/test_test_suite.py

@@ -2,7 +2,7 @@
 
 import wasi_test_runner.test_case as tc
 import wasi_test_runner.test_suite as ts
-from wasi_test_runner.runtime_adapter import RuntimeVersion
+from wasi_test_runner.runtime_adapter import RuntimeMeta
 
 
 def create_test_case(name: str, is_executed: bool, is_failed: bool) -> tc.TestCase:
@@ -18,8 +18,10 @@ def create_test_case(name: str, is_executed: bool, is_failed: bool) -> tc.TestCa
 
 def test_test_suite_should_return_correct_count() -> None:
     suite = ts.TestSuite(
-        "suite",
-        RuntimeVersion("test-runtime", "3.14"),
+        ts.TestSuiteMeta("suite",
+                         tc.WasiVersion.WASM32_WASIP1,
+                         RuntimeMeta("test-runtime", "3.14",
+                                     frozenset([tc.WasiVersion.WASM32_WASIP1]))),
         10.0,
         datetime.now(),
         [

test-runner/tests/test_test_suite_runner.py

@@ -2,9 +2,10 @@
 from pathlib import Path
 from unittest.mock import ANY, MagicMock, Mock, patch, mock_open
 
+import wasi_test_runner.test_suite as ts
 import wasi_test_runner.test_case as tc
 import wasi_test_runner.test_suite_runner as tsr
-from wasi_test_runner.runtime_adapter import RuntimeVersion
+from wasi_test_runner.runtime_adapter import RuntimeMeta
 
 
 def get_mock_open() -> Mock:
@@ -26,6 +27,7 @@ def open_mock(filename: str, *_args: Any, **_kwargs: Any) -> Any:
 # pylint: disable-msg=too-many-locals
 @patch("builtins.open", get_mock_open())
 @patch("os.path.exists", Mock(return_value=True))
+@patch("pathlib.Path.exists", Mock(return_value=True))
 def test_runner_end_to_end() -> None:
     test_suite_dir = "my-path"
     test_suite_name = "test-suite"
@@ -52,7 +54,14 @@ def test_runner_end_to_end() -> None:
 
     runtime_name = "rt1"
     runtime_version_str = "4.2"
-    runtime_version = RuntimeVersion(runtime_name, runtime_version_str)
+    the_runtime_wasi_version = tc.WasiVersion.WASM32_WASIP1
+    runtime_wasi_versions = frozenset([the_runtime_wasi_version])
+    runtime_meta = RuntimeMeta(runtime_name, runtime_version_str,
+                               runtime_wasi_versions)
+
+    expected_test_suite_meta = ts.TestSuiteMeta(test_suite_name,
+                                                the_runtime_wasi_version,
+                                                runtime_meta)
 
     expected_argv = [runtime_name, "<test>"]
     expected_test_cases = [
@@ -64,7 +73,7 @@ def test_runner_end_to_end() -> None:
 
     runtime = Mock()
     runtime.get_name.return_value = runtime_name
-    runtime.get_version.return_value = runtime_version
+    runtime.get_meta.return_value = runtime_meta
     runtime.run_test.side_effect = outputs
     runtime.compute_argv.return_value = expected_argv
 
@@ -87,7 +96,7 @@ def test_runner_end_to_end() -> None:
                                               filters)     # type: ignore
 
     # Assert manifest was read correctly
-    assert suite.name == test_suite_name
+    assert suite.meta == expected_test_suite_meta
 
     # Assert test cases
     assert suite.test_count == 3
@@ -98,16 +107,16 @@ def test_runner_end_to_end() -> None:
     for test_path, config in zip(test_paths, expected_config):
         expected_dirs = [(Path(test_suite_dir) / d, d) for d in config.dirs]
         runtime.compute_argv.assert_any_call(
-            str(test_path), config.args, config.env, expected_dirs
+            str(test_path), config.args, config.env, expected_dirs,
+            "wasm32-wasip1"
         )
         runtime.run_test.assert_called_with(expected_argv)
 
     # Assert reporters calls
     for reporter in reporters:
         assert reporter.report_test.call_count == 3
         for test_case in expected_test_cases:
-            reporter.report_test.assert_any_call(test_suite_name,
-                                                 runtime_version,
+            reporter.report_test.assert_any_call(expected_test_suite_meta,
                                                  test_case)
 
     # Assert validators calls
@@ -120,12 +129,12 @@ def test_runner_end_to_end() -> None:
     for filt in filters:
         assert filt.should_skip.call_count == 3
         for test_case in expected_test_cases:
-            filt.should_skip.assert_any_call(runtime_version, suite.name,
+            filt.should_skip.assert_any_call(expected_test_suite_meta,
                                              test_case.name)
 
 
 @patch("os.path.exists", Mock(return_value=False))
 def test_runner_should_use_path_for_name_if_manifest_does_not_exist() -> None:
     suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], [], [])
 
-    assert suite.name == "my-path"
+    assert suite.meta.name == "my-path"

test-runner/wasi_test_runner/__main__.py

@@ -6,7 +6,7 @@
 from .runtime_adapter import RuntimeAdapter
 from .harness import run_all_tests
 from .filters import TestFilter
-from .filters import JSONTestExcludeFilter
+from .filters import JSONTestExcludeFilter, UnsupportedWasiTestExcludeFilter
 from .reporters import TestReporter
 from .reporters.console import ConsoleTestReporter
 from .reporters.json import JSONTestReporter
@@ -56,7 +56,7 @@ def main() -> int:
 
     validators: List[Validator] = [exit_code_validator, stdout_validator]
 
-    filters: List[TestFilter] = []
+    filters: List[TestFilter] = [UnsupportedWasiTestExcludeFilter()]
     for filt in options.exclude_filter:
         filters.append(JSONTestExcludeFilter(filt))
 

test-runner/wasi_test_runner/filters.py

@@ -4,26 +4,35 @@
 
 import json
 
-from .runtime_adapter import RuntimeVersion
+from .test_suite import TestSuiteMeta
 
 
 class TestFilter(ABC):
     @abstractmethod
     def should_skip(
-        self, runtime: RuntimeVersion, test_suite_name: str, test_name: str
+        self, meta: TestSuiteMeta, test_name: str
     ) -> Union[Tuple[Literal[True], str], Tuple[Literal[False], Literal[None]]]:
         pass
 
 
+class UnsupportedWasiTestExcludeFilter(TestFilter):
+    def should_skip(
+        self, meta: TestSuiteMeta, test_name: str
+    ) -> Union[Tuple[Literal[True], str], Tuple[Literal[False], Literal[None]]]:
+        if meta.wasi_version not in meta.runtime.supported_wasi_versions:
+            return True, "WASI version unsupported by runtime"
+        return False, None
+
+
 class JSONTestExcludeFilter(TestFilter):
     def __init__(self, filename: str) -> None:
         with open(filename, encoding="utf-8") as file:
             self.filter_dict = json.load(file)
 
     def should_skip(
-        self, runtime: RuntimeVersion, test_suite_name: str, test_name: str
+        self, meta: TestSuiteMeta, test_name: str
     ) -> Union[Tuple[Literal[True], str], Tuple[Literal[False], Literal[None]]]:
-        test_suite_filter = self.filter_dict.get(test_suite_name)
+        test_suite_filter = self.filter_dict.get(meta.name)
         if test_suite_filter is None:
             return False, None
         why = test_suite_filter.get(test_name)

test-runner/wasi_test_runner/harness.py

@@ -1,7 +1,9 @@
 from typing import List
 from pathlib import Path
 
-from .filters import TestFilter, JSONTestExcludeFilter
+from .filters import (
+    TestFilter, JSONTestExcludeFilter, UnsupportedWasiTestExcludeFilter
+)
 from .reporters import TestReporter
 from .reporters.console import ConsoleTestReporter
 from .reporters.json import JSONTestReporter
@@ -19,9 +21,9 @@ def run_tests(runtimes: List[RuntimeAdapter],
     reporters: List[TestReporter] = [ConsoleTestReporter(color)]
     if json_log_file:
         reporters.append(JSONTestReporter(json_log_file))
-    filters: List[TestFilter] = []
+    filters: List[TestFilter] = [UnsupportedWasiTestExcludeFilter()]
     if exclude_filters is not None:
-        filters = [JSONTestExcludeFilter(str(filt)) for filt in exclude_filters]
+        filters += [JSONTestExcludeFilter(str(filt)) for filt in exclude_filters]
 
     return run_all_tests(runtimes, [str(p) for p in test_suite_paths],
                          validators, reporters, filters)

test-runner/wasi_test_runner/reporters/__init__.py

@@ -1,13 +1,11 @@
 from abc import ABC
 
 from ..test_case import TestCase
-from ..test_suite import TestSuite
-from ..runtime_adapter import RuntimeVersion
+from ..test_suite import TestSuite, TestSuiteMeta
 
 
 class TestReporter(ABC):
-    def report_test(self, test_suite_name: str, runtime: RuntimeVersion,
-                    test: TestCase) -> None:
+    def report_test(self, meta: TestSuiteMeta, test: TestCase) -> None:
         pass
 
     def report_test_suite(self, test_suite: TestSuite) -> None:

test-runner/wasi_test_runner/reporters/console.py

@@ -4,8 +4,8 @@
 
 from . import TestReporter
 from ..test_case import TestCase
-from ..test_suite import TestSuite
-from ..runtime_adapter import RuntimeVersion
+from ..test_suite import TestSuite, TestSuiteMeta
+from ..runtime_adapter import RuntimeMeta
 
 
 class ConsoleTestReporter(TestReporter):
@@ -18,15 +18,14 @@ def __init__(self, colored: bool = True, verbose: bool = False) -> None:
         super().__init__()
         init(autoreset=True)
         self._test_suites: List[TestSuite] = []
-        self._current_test_suite: Optional[str] = None
+        self._current_test_suite: Optional[TestSuiteMeta] = None
         self._colored = colored
         self._verbose = verbose
 
-    def report_test(self, test_suite_name: str, runtime: RuntimeVersion,
-                    test: TestCase) -> None:
+    def report_test(self, meta: TestSuiteMeta, test: TestCase) -> None:
         if self._current_test_suite is None:
-            print(f"Running test suite {test_suite_name} with {runtime}")
-            self._current_test_suite = test_suite_name
+            print(f"Running test suite {meta.name} with {meta.runtime}")
+            self._current_test_suite = meta
 
         if self._verbose:
             self._report_test_verbose(test)
@@ -63,16 +62,16 @@ def report_test_suite(self, test_suite: TestSuite) -> None:
     def finalize(self) -> None:
         print("===== Test results =====")
 
-        test_suites_by_runtime: Dict[RuntimeVersion, List[TestSuite]] = {}
+        test_suites_by_runtime: Dict[RuntimeMeta, List[TestSuite]] = {}
         for suite in self._test_suites:
-            if suite.runtime not in test_suites_by_runtime:
-                test_suites_by_runtime[suite.runtime] = []
-            test_suites_by_runtime[suite.runtime].append(suite)
+            if suite.meta.runtime not in test_suites_by_runtime:
+                test_suites_by_runtime[suite.meta.runtime] = []
+            test_suites_by_runtime[suite.meta.runtime].append(suite)
 
         for runtime, test_suites in test_suites_by_runtime.items():
             self._print_result_for_runtime(runtime, test_suites)
 
-    def _print_result_for_runtime(self, runtime: RuntimeVersion,
+    def _print_result_for_runtime(self, runtime: RuntimeMeta,
                                   suites: List[TestSuite]) -> None:
         total_skip = total_pass = total_fail = 0
 

test-runner/wasi_test_runner/reporters/json.py

@@ -23,10 +23,10 @@ def finalize(self) -> None:
         for suite in self._test_suites:
             results.append(
                 {
-                    "name": suite.name,
+                    "name": suite.meta.name,
                     "runtime": {
-                        "name": suite.runtime.name,
-                        "version": suite.runtime.version
+                        "name": suite.meta.runtime.name,
+                        "version": suite.meta.runtime.version
                     },
                     "duration_s": suite.duration_s,
                     "failed": suite.fail_count,