Enhance e2e testing by adding stdout and stderr checks, and normalizing input lists

build_defs/e2e.build_defs

@@ -1,3 +1,15 @@
+def _shell_quote(value:str) -> str:
+    return "'" + value.replace("'", "'\"'\"'") + "'"
+
+
+def _normalise_list(value):
+    if value is None:
+        return []
+    if isinstance(value, str):
+        return [value]
+    return value
+
+
 def please_tool(name:str, version=CONFIG.E2E.PLEASE_VERSION, visibility:list):
 
     version_dep = None
@@ -60,33 +72,50 @@ def please_repo_e2e_test(
         name: str,
         plz_command: str,
         repo: str,
-        data: dict={},
-        deps: list=[],
-        tools: dict={},
+        data: dict = {},
+        deps: list = [],
+        tools: dict = {},
         expected_failure: bool = False,
         expected_output: dict = {},
         expect_output_contains: dict = {},
         expect_output_doesnt_contain: dict = {},
+        expect_stdout_contains=None,
+        expect_stderr_contains=None,
         labels: list = [],
 ):
     plz_command = plz_command.replace("plz ", "$TOOLS_PLEASE ")
-    if expected_failure:
-        plz_command += "; [ ! $? -eq 0 ]"
+
+    expect_stdout_contains = _normalise_list(expect_stdout_contains)
+    expect_stderr_contains = _normalise_list(expect_stderr_contains)
 
     test_cmd = [
         "mv $DATA_BASE_CONFIG $DATA_REPO",
         "cd $DATA_REPO",
-        plz_command,
+        "STDOUT_LOG=plz_stdout.log",
+        "STDERR_LOG=plz_stderr.log",
+        f"( {plz_command} ) > >(tee $STDOUT_LOG) 2> >(tee $STDERR_LOG >&2)",
+        "STATUS=$?",
     ]
 
+    if expected_failure:
+        test_cmd.append('[ ! "$STATUS" -eq 0 ]')
+    else:
+        test_cmd.append('[ "$STATUS" -eq 0 ]')
+
     if expected_output:
         test_cmd += [f"$TOOLS_CONTENT_CHECKER '{o}' '{c}'" for o, c in expected_output.items()]
 
     if expect_output_contains:
         test_cmd += [f'_STR="$(cat {o})" _SUBSTR="{c}" && if [ "${_STR##*$_SUBSTR*}" ]; then echo "$_STR"; exit 1; fi' for o, c in expect_output_contains.items()]
 
     if expect_output_doesnt_contain:
-        test_cmd += [f'_STR="$(cat {o})" _SUBSTR="{c}" && if [ -z "${_STR##*$_SUBSTR*}" ]; then echo "$_STR"; exit 1; fi' for o, c in expect_output_contains.items()]
+        test_cmd += [f'_STR="$(cat {o})" _SUBSTR="{c}" && if [ -z "${_STR##*$_SUBSTR*}" ]; then echo "$_STR"; exit 1; fi' for o, c in expect_output_doesnt_contain.items()]
+
+    if expect_stdout_contains:
+        test_cmd += [f'grep -F -- {_shell_quote(substr)} "$STDOUT_LOG" >/dev/null || (cat "$STDOUT_LOG"; exit 1)' for substr in expect_stdout_contains]
+
+    if expect_stderr_contains:
+        test_cmd += [f'grep -F -- {_shell_quote(substr)} "$STDERR_LOG" >/dev/null || (cat "$STDERR_LOG"; exit 1)' for substr in expect_stderr_contains]
 
     test_cmd = ' && '.join(test_cmd)
 
@@ -121,6 +150,8 @@ def plugin_e2e_test(
     expected_output:dict={},
     expect_output_contains:dict={},
     expect_output_doesnt_contain:dict={},
+    expect_stdout_contains=None,
+    expect_stderr_contains=None,
     expected_failure:bool=False,
     deps:list=[],
 ):
@@ -155,6 +186,8 @@ def plugin_e2e_test(
                                      the given string by the time the test terminates.
       expect_output_doesnt_contain (dict): A mapping of file names to strings. The test will fail if any of the files
                                            contains the given string by the time the test terminates.
+      expect_stdout_contains (list | str | None): Strings that must appear in stdout generated by the Please command.
+      expect_stderr_contains (list | str | None): Strings that must appear in stderr generated by the Please command.
       expected_failure (bool): if True, the usual test logic is inverted: the test passes if test_cmd exits with an
                                error and fails if test_cmd exits successfully.
       deps (list): Additional dependencies for this target.
@@ -176,6 +209,8 @@ def plugin_e2e_test(
         expected_output = expected_output,
         expect_output_contains = expect_output_contains,
         expect_output_doesnt_contain = expect_output_doesnt_contain,
+        expect_stdout_contains = expect_stdout_contains,
+        expect_stderr_contains = expect_stderr_contains,
         expected_failure = expected_failure,
         deps = deps,
     )