Add support for test config overrides

It can be useful to override the config for certain tests. The main use
case for this is to override environment variables which control test
assertions (e.g. NO_DANGLING_FILESYSTEM, ERRNO_MODE_WINDOWS). When
running the tests in CI, it would be nicer to support this without
manually modifying the JSON test config files.

Author: zoraaver

README.md

@@ -62,6 +62,21 @@ python3 test-runner/wasi_test_runner.py
     -r adapters/wasmtime.py # path to a runtime adapter
 ```
 
+You can also optionally override test configs with the `--config-override`
+option:
+
+```bash
+python3 test-runner/wasi_test_runner.py                                                  \
+    -t ./tests/assemblyscript/testsuite/ # path to folders containing .wasm test files   \
+       ./tests/c/testsuite/                                                              \
+       ./tests/rust/testsuite/                                                           \
+    --config-override examples/config_override.json                                      \
+    -r adapters/wasmtime.py # path to a runtime adapter
+```
+
+This can be useful for passing additional environment variables to certain
+tests without modifying the test config files.
+
 The default executable in the adapter used for test execution can be
 overridden using `TEST_RUNTIME_EXE` variable. This only works with adapters defined in
 [adapters/](adapters/), and might not work with 3rd party adapters.

examples/config_override.json

@@ -0,0 +1,16 @@
+{
+    "WASI C tests": {
+        "stat-dev-ino": {
+            "dirs": [
+                "fs-tests.dir"
+            ],
+            "env": {
+                "TEST_VAR": "test"
+            },
+            "args": [
+                "fs-tests.dir",
+                "test_arg"
+            ]
+        }
+    }
+}

test-runner/tests/test_test_suite_runner.py

@@ -3,6 +3,7 @@
 
 import wasi_test_runner.test_case as tc
 import wasi_test_runner.test_suite_runner as tsr
+from wasi_test_runner.override import StubConfigOverride
 
 
 def get_mock_open() -> Mock:
@@ -67,7 +68,9 @@ def test_runner_end_to_end() -> None:
     filters = [filt]
 
     with patch("glob.glob", return_value=test_paths):
-        suite = tsr.run_tests_from_test_suite("my-path", runtime, validators, reporters, filters)  # type: ignore
+        suite = tsr.run_tests_from_test_suite(
+            "my-path", runtime, validators, reporters, filters, StubConfigOverride()  # type: ignore
+        )
 
     # Assert manifest was read correctly
     assert suite.name == "test-suite"
@@ -104,6 +107,8 @@ def test_runner_end_to_end() -> None:
 
 @patch("os.path.exists", Mock(return_value=False))
 def test_runner_should_use_path_for_name_if_manifest_does_not_exist() -> None:
-    suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], [], [])
+    suite = tsr.run_tests_from_test_suite(
+        "my-path", Mock(), [], [], [], StubConfigOverride()
+    )
 
     assert suite.name == "my-path"

test-runner/wasi_test_runner/__main__.py

@@ -11,6 +11,7 @@
 from .reporters.console import ConsoleTestReporter
 from .reporters.json import JSONTestReporter
 from .validators import exit_code_validator, stdout_validator, Validator
+from .override import ConfigOverride, JSONConfigOverride, StubConfigOverride
 
 
 def main() -> int:
@@ -46,6 +47,11 @@ def main() -> int:
         default=False,
         help="Disables color for console output reporter.",
     )
+    parser.add_argument(
+        "--config-override",
+        required=False,
+        help="Location of JSON file containing overrides for the config used for each test",
+    )
 
     options = parser.parse_args()
 
@@ -59,12 +65,17 @@ def main() -> int:
     for filt in options.exclude_filter:
         filters.append(JSONTestExcludeFilter(filt))
 
+    override: ConfigOverride = StubConfigOverride()
+    if options.config_override:
+        override = JSONConfigOverride(options.config_override)
+
     return run_all_tests(
         RuntimeAdapter(options.runtime_adapter),
         options.test_suite,
         validators,
         reporters,
         filters,
+        override,
     )
 
 

test-runner/wasi_test_runner/harness.py

@@ -5,6 +5,7 @@
 from .test_suite_runner import run_tests_from_test_suite
 from .runtime_adapter import RuntimeAdapter
 from .validators import Validator
+from .override import ConfigOverride
 
 
 def run_all_tests(
@@ -13,12 +14,13 @@ def run_all_tests(
     validators: List[Validator],
     reporters: List[TestReporter],
     filters: List[TestFilter],
+    override: ConfigOverride,
 ) -> int:
     ret = 0
 
     for test_suite_path in test_suite_paths:
         test_suite = run_tests_from_test_suite(
-            test_suite_path, runtime, validators, reporters, filters,
+            test_suite_path, runtime, validators, reporters, filters, override
         )
         for reporter in reporters:
             reporter.report_test_suite(test_suite)

test-runner/wasi_test_runner/override.py

@@ -0,0 +1,45 @@
+import json
+from typing import Any, Dict, Optional
+from .test_case import (
+    Config,
+)
+from abc import ABC
+from abc import abstractmethod
+
+
+class ConfigOverride(ABC):
+    @abstractmethod
+    def get_test_override(
+        self, test_suite_name: str, test_name: str
+    ) -> Optional[Config]:
+        pass
+
+
+class JSONConfigOverride(ConfigOverride):
+    overrides_dict: Dict[str, Dict[str, Dict[str, Any]]]
+
+    def __init__(self, overrides_path: str) -> None:
+        with open(overrides_path, encoding="utf-8") as file:
+            self.overrides_dict = json.load(file)
+
+    def get_test_override(
+        self, test_suite_name: str, test_name: str
+    ) -> Optional[Config]:
+        test_suite_overrides = self.overrides_dict.get(test_suite_name)
+
+        if test_suite_overrides is None:
+            return None
+
+        test_override = test_suite_overrides.get(test_name)
+
+        if test_override is None:
+            return None
+
+        return Config.from_dict(test_override)
+
+
+class StubConfigOverride(ConfigOverride):
+    def get_test_override(
+        self, test_suite_name: str, test_name: str
+    ) -> Optional[Config]:
+        return None

test-runner/wasi_test_runner/test_case.py

@@ -37,10 +37,19 @@ class Config(NamedTuple):
 
     @classmethod
     def from_file(cls: Type[T], config_file: str) -> T:
-        default = cls()
-
         with open(config_file, encoding="utf-8") as file:
             dict_config = json.load(file)
+            return cls.from_dict(dict_config)
+
+    @classmethod
+    def _validate_dict(cls: Type[T], dict_config: Dict[str, Any]) -> None:
+        for field_name in dict_config:
+            if field_name not in cls._fields:
+                logging.warning("Unknown field in the config file: %s", field_name)
+
+    @classmethod
+    def from_dict(cls: Type[T], dict_config: Dict[str, Any]) -> T:
+        default = cls()
 
         cls._validate_dict(dict_config)
 
@@ -53,12 +62,6 @@ def from_file(cls: Type[T], config_file: str) -> T:
             stdout=dict_config.get("stdout", default.stdout),
         )
 
-    @classmethod
-    def _validate_dict(cls: Type[T], dict_config: Dict[str, Any]) -> None:
-        for field_name in dict_config:
-            if field_name not in cls._fields:
-                logging.warning("Unknown field in the config file: %s", field_name)
-
 
 class TestCase(NamedTuple):
     name: str

test-runner/wasi_test_runner/test_suite_runner.py

@@ -6,7 +6,7 @@
 import time
 
 from datetime import datetime
-from typing import List, cast
+from typing import List, Optional, cast
 
 from .filters import TestFilter
 from .runtime_adapter import RuntimeAdapter
@@ -19,6 +19,7 @@
 from .reporters import TestReporter
 from .test_suite import TestSuite
 from .validators import Validator
+from .override import ConfigOverride
 
 
 def run_tests_from_test_suite(
@@ -27,6 +28,7 @@ def run_tests_from_test_suite(
     validators: List[Validator],
     reporters: List[TestReporter],
     filters: List[TestFilter],
+    config_override: ConfigOverride,
 ) -> TestSuite:
     test_cases: List[TestCase] = []
     test_start = datetime.now()
@@ -45,7 +47,12 @@ def run_tests_from_test_suite(
                 test_case = _skip_single_test(runtime, validators, test_path)
                 break
         else:
-            test_case = _execute_single_test(runtime, validators, test_path)
+            test_config_override = config_override.get_test_override(
+                test_suite_name, test_name
+            )
+            test_case = _execute_single_test(
+                runtime, validators, test_path, test_config_override
+            )
         test_cases.append(test_case)
         for reporter in reporters:
             reporter.report_test(test_case)
@@ -73,9 +80,12 @@ def _skip_single_test(
 
 
 def _execute_single_test(
-    runtime: RuntimeAdapter, validators: List[Validator], test_path: str
+    runtime: RuntimeAdapter,
+    validators: List[Validator],
+    test_path: str,
+    config_override: Optional[Config],
 ) -> TestCase:
-    config = _read_test_config(test_path)
+    config = config_override or _read_test_config(test_path)
     test_start = time.time()
     test_output = runtime.run_test(test_path, config.args, config.env, config.dirs)
     elapsed = time.time() - test_start