Improve test runner ergonomics

Add `./run-tests` executable that doesn't require arguments,
automatically detecting available runtimes and tests.

Simplify `README.md` accordingly.

Console test case reporter more concise and useful:
 - Single line to indicate total results
 - One copy-pastable command line per failure
 - Requires refactors to record command lines, remove cwd change

Allow testing multiple engines in one run.
 - Required changing `TEST_RUNTIME_EXE` environment variable
   convention to one env var per executable
 - Requires refactor to collect which runtime ran a testcase

Author: wingo

README.md

@@ -44,32 +44,31 @@ executor is quite simple; see the [specification] document for the details and t
 5. Execute the test suites from this repository:
 
    ```bash
-   python3 test-runner/wasi_test_runner.py                                                  \
-      -t ./tests/assemblyscript/testsuite/ `# path to folders containing .wasm test files` \
-         ./tests/c/testsuite/wasm32-wasip1                                                 \
-         ./tests/rust/testsuite/                                                           \
-      -r adapters/wasmtime.py # path to a runtime adapter
+   ./run-tests
    ```
 
-Optionally you can specify test cases to skip with the `--exclude-filter` option.
+By default, the test runner will detect available WASI runtimes from
+those available in [adapters/](adapters/), and will run tests on all
+available runtimes.  Pass `--runtime` to instead use a specific runtime.
+
+```
+./run-tests --runtime adapters/wasmtime.py
+```
+
+Running tests will invoke the WASI runtime's binary in a subprocess:
+`wasmtime` for `adapters/wasmtime.py`, `iwasm` for
+`adapters/wasm-micro-runtime.py`, and so on.  These binaries can be
+overridden by setting corresponding environment variables (`WASMTIME`,
+`IWASM`, etc):
 
-```bash
-python3 test-runner/wasi_test_runner.py                                                  \
-    -t ./tests/assemblyscript/testsuite/ `# path to folders containing .wasm test files` \
-       ./tests/c/testsuite/wasm32-wasip1                                                 \
-       ./tests/rust/testsuite/                                                           \
-    --exclude-filter examples/skip.json                                                  \
-    -r adapters/wasmtime.py # path to a runtime adapter
+```
+WASMTIME="wasmtime --wasm-features all" ./run-tests
 ```
 
-The default executable in the adapter used for test execution can be
-overridden using `TEST_RUNTIME_EXE` variable. This only works with adapters defined in
-[adapters/](adapters/), and might not work with 3rd party adapters.
+Optionally you can specify test cases to skip with the `--exclude-filter` option.
 
 ```bash
-TEST_RUNTIME_EXE="wasmtime --wasm-features all" python3 test-runner/wasi_test_runner.py                                                  \
-    -t ./tests/assemblyscript/testsuite/ \
-    -r adapters/wasmtime.py
+./run-tests --exclude-filter examples/skip.json                                                  \
 ```
 
 ## Contributing

adapters/wasm-micro-runtime.py

@@ -5,7 +5,7 @@
 import shlex
 
 # shlex.split() splits according to shell quoting rules
-IWASM = shlex.split(os.getenv("TEST_RUNTIME_EXE", "iwasm"))
+IWASM = shlex.split(os.getenv("IWASM", "iwasm"))
 
 parser = argparse.ArgumentParser()
 parser.add_argument("--version", action="store_true")
@@ -23,7 +23,7 @@
 TEST_FILE = args.test_file
 PROG_ARGS = args.arg
 ENV_ARGS = [f"--env={i}" for i in args.env]
-DIR_ARGS = [f"--dir={i}" for i in args.dir]
+DIR_ARGS = [f"--map-dir={i}" for i in args.dir]
 
 r = subprocess.run(IWASM + ENV_ARGS + DIR_ARGS + [TEST_FILE] + PROG_ARGS)
 sys.exit(r.returncode)

adapters/wasmtime.py

@@ -5,7 +5,7 @@
 import shlex
 
 # shlex.split() splits according to shell quoting rules
-WASMTIME = shlex.split(os.getenv("TEST_RUNTIME_EXE", "wasmtime"))
+WASMTIME = shlex.split(os.getenv("WASMTIME", "wasmtime"))
 
 parser = argparse.ArgumentParser()
 parser.add_argument("--version", action="store_true")

adapters/wizard.py

@@ -5,7 +5,7 @@
 import shlex
 
 # shlex.split() splits according to shell quoting rules
-WIZARD = shlex.split(os.getenv("TEST_RUNTIME_EXE", "wizeng.x86-64-linux"))
+WIZARD = shlex.split(os.getenv("WIZARD", "wizeng.x86-64-linux"))
 
 parser = argparse.ArgumentParser()
 parser.add_argument("--version", action="store_true")

run-tests

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+
+import argparse
+import subprocess
+import sys
+from typing import List
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent / "test-runner"))
+
+from wasi_test_runner.harness import run_tests
+from wasi_test_runner.runtime_adapter import RuntimeAdapter
+
+parser = argparse.ArgumentParser(
+    description="WASI test runner"
+)
+
+parser.add_argument(
+    "-f",
+    "--exclude-filter",
+    action="append",
+    default=[],
+    help="Path to JSON file indicating tests to exclude.",
+)
+parser.add_argument(
+    "-r", "--runtime-adapter", help="Path to a runtime adapter."
+)
+parser.add_argument(
+    "--json-output-location",
+    help="JSON test result destination. If not specified, JSON output won't be generated.",
+)
+parser.add_argument(
+    "--disable-colors",
+    action="store_true",
+    default=False,
+    help="Disables color for console output reporter.",
+)
+
+def find_test_dirs(root):
+    test_dirs = []
+    for root, dirs, files in root.walk(on_error=print):
+        if "manifest.json" in files:
+            test_dirs.append(root)
+    return test_dirs
+
+def find_runtime_adapters(root, verbose=False):
+    print(f"Detecting WASI runtime availability:")
+    adapters = []
+    for candidate in root.glob("*.py"):
+        adapter = RuntimeAdapter(candidate)
+        try:
+            print(f"  {candidate.name}: {adapter.get_version()}")
+            adapters.append(adapter)
+        except subprocess.CalledProcessError as e:
+            print(f"  {candidate.name}: unavailable; pass `--runtime {candidate}` to debug.")
+    print("")
+    if len(adapters) == 0:
+        print("Error: No WASI runtimes found")
+        sys.exit(1)
+    return adapters
+
+options = parser.parse_args()
+test_suite = find_test_dirs(Path(__file__).parent / "tests")
+if options.runtime_adapter:
+    runtime_adapters = [RuntimeAdapter(options.runtime_adapter)]
+    # Ensure it works.
+    try:
+        runtime_adapters[0].get_version()
+    except subprocess.CalledProcessError as e:
+        print(f"Error: failed to load {options.runtime_adapter}:")
+        print(f"  Failed command line: {' '.join(e.cmd)}")
+        if e.stdout.strip() != "":
+            print(f"  stdout:\n{e.stdout}")
+        if e.stderr.strip() != "":
+            print(f"  stderr:\n{e.stderr}")
+        sys.exit(1)
+else:
+    runtime_adapters = find_runtime_adapters(Path(__file__).parent / "adapters")
+
+exclude_filters = [JSONTestExcludeFilter(filt) for filt in options.exclude_filter]
+
+sys.exit(run_tests(runtime_adapters, test_suite,
+                   color=not options.disable_colors,
+                   json_log_file=options.json_output_location,
+                   exclude_filters=exclude_filters))

test-runner/tests/test_test_suite.py

@@ -4,10 +4,13 @@
 import wasi_test_runner.test_suite as ts
 
 
-def create_test_case(name: str, is_executed: bool, is_failed: bool) -> tc.TestCase:
+def create_test_case(name: str, runtime: str,
+                     is_executed: bool, is_failed: bool) -> tc.TestCase:
     failures = [tc.Failure("a", "b")] if is_failed else []
     return tc.TestCase(
         name,
+        runtime,
+        [runtime, name],
         tc.Config(),
         tc.Result(tc.Output(0, "", ""), is_executed, failures),
         1.0,
@@ -20,10 +23,10 @@ def test_test_suite_should_return_correct_count() -> None:
         10.0,
         datetime.now(),
         [
-            create_test_case("t1", True, True),
-            create_test_case("t2", True, False),
-            create_test_case("t3", False, True),
-            create_test_case("t4", False, False),
+            create_test_case("t1", "rt1", True, True),
+            create_test_case("t2", "rt1", True, False),
+            create_test_case("t3", "rt1", False, True),
+            create_test_case("t4", "rt1", False, False),
         ],
     )
 

test-runner/tests/test_test_suite_runner.py

@@ -45,14 +45,20 @@ def test_runner_end_to_end() -> None:
         tc.Config(stdout="output", env={"x": "1"}),
     ]
 
+    runtime_name = "rt1"
+    runtime_version_str = "4.2"
+
     expected_test_cases = [
-        tc.TestCase(test_name, config, result, ANY)
+        tc.TestCase(test_name, runtime_name, [runtime_name, test_name],
+                    config, result, ANY)
         for config, test_name, result in zip(
             expected_config, ["test1", "test2", "test3"], expected_results
         )
     ]
 
     runtime = Mock()
+    runtime.get_name.return_value = runtime_name
+    runtime.get_version.return_value = (runtime_name, runtime_version_str)
     runtime.run_test.side_effect = outputs
 
     validators = [
@@ -67,7 +73,9 @@ def test_runner_end_to_end() -> None:
     filters = [filt]
 
     with patch("glob.glob", return_value=test_paths):
-        suite = tsr.run_tests_from_test_suite("my-path", runtime, validators, reporters, filters)  # type: ignore
+        suite = tsr.run_tests_from_test_suite("my-path", [runtime],
+                                              validators, reporters,
+                                              filters)  # type: ignore
 
     # Assert manifest was read correctly
     assert suite.name == "test-suite"
@@ -99,7 +107,8 @@ def test_runner_end_to_end() -> None:
     for filt in filters:
         assert filt.should_skip.call_count == 3
         for test_case in expected_test_cases:
-            filt.should_skip.assert_any_call(suite.name, test_case.name)
+            filt.should_skip.assert_any_call(runtime, suite.name,
+                                             test_case.name)
 
 
 @patch("os.path.exists", Mock(return_value=False))

test-runner/wasi_test_runner/__main__.py

@@ -60,7 +60,7 @@ def main() -> int:
         filters.append(JSONTestExcludeFilter(filt))
 
     return run_all_tests(
-        RuntimeAdapter(options.runtime_adapter),
+        [RuntimeAdapter(options.runtime_adapter)],
         options.test_suite,
         validators,
         reporters,

test-runner/wasi_test_runner/filters.py

@@ -8,7 +8,7 @@
 class TestFilter(ABC):
     @abstractmethod
     def should_skip(
-        self, test_suite_name: str, test_name: str
+        self, runtime, test_suite_name: str, test_name: str
     ) -> Union[Tuple[Literal[True], str], Tuple[Literal[False], Literal[None]]]:
         pass
 
@@ -19,7 +19,7 @@ def __init__(self, filename: str) -> None:
             self.filter_dict = json.load(file)
 
     def should_skip(
-        self, test_suite_name: str, test_name: str
+        self, runtime, test_suite_name: str, test_name: str
     ) -> Union[Tuple[Literal[True], str], Tuple[Literal[False], Literal[None]]]:
         test_suite_filter = self.filter_dict.get(test_suite_name)
         if test_suite_filter is None:

test-runner/wasi_test_runner/harness.py

@@ -1,14 +1,30 @@
 from typing import List
+from pathlib import Path
 
 from .filters import TestFilter
 from .reporters import TestReporter
+from .reporters.console import ConsoleTestReporter
+from .reporters.json import JSONTestReporter
 from .test_suite_runner import run_tests_from_test_suite
 from .runtime_adapter import RuntimeAdapter
-from .validators import Validator
+from .validators import exit_code_validator, stdout_validator, Validator
 
+def run_tests(runtimes: List[RuntimeAdapter],
+              test_suite_paths: List[Path],
+              exclude_filters: List[Path] | None = None,
+              color: bool = True,
+              json_log_file: str | None = None):
+    validators = [exit_code_validator, stdout_validator]
+    reporters = [ConsoleTestReporter(color)]
+    if json_log_file:
+        reporters.append(JSONTestReporter(json_log_file))
+    filters = [JSONTestExcludeFilter(filt) for filt in exclude_filters]
+
+    return run_all_tests(runtimes, [str(p) for p in test_suite_paths],
+                         validators, reporters, filters)
 
 def run_all_tests(
-    runtime: RuntimeAdapter,
+    runtimes: List[RuntimeAdapter],
     test_suite_paths: List[str],
     validators: List[Validator],
     reporters: List[TestReporter],
@@ -17,15 +33,16 @@ def run_all_tests(
     ret = 0
 
     for test_suite_path in test_suite_paths:
-        test_suite = run_tests_from_test_suite(
-            test_suite_path, runtime, validators, reporters, filters,
-        )
-        for reporter in reporters:
-            reporter.report_test_suite(test_suite)
-        if test_suite.fail_count > 0:
-            ret = 1
+        for runtime in runtimes:
+            test_suite = run_tests_from_test_suite(
+                test_suite_path, runtime, validators, reporters, filters,
+            )
+            for reporter in reporters:
+                reporter.report_test_suite(test_suite)
+            if test_suite.fail_count > 0:
+                ret = 1
 
     for reporter in reporters:
-        reporter.finalize(runtime.get_version())
+        reporter.finalize()
 
     return ret