Auto merge of #143002 - Enselic:tests-ui-run-fail-exit-vs-signal, r=<try>

tests: Require `run-fail` ui tests to have an exit code (`SIGABRT` not ok)

Normally a `run-fail` ui test shall not be terminated by a signal like `SIGABRT`. So begin having that as a hard requirement.

Some of our current tests do terminate by a signal however. Introduce and use `run-fail-without-exit-code` for those tests.

This adds further (cc #142304, #142886) protection against the regression in #123733 since that bug also manifested as `SIGABRT` in `tests/ui/panics/panic-main.rs` (shown as `Aborted (core dumped)` in the logs attached to that issue, and I have also been able to reproduce this locally).

### TODO
- [ ] **Q:** what about on Windows?
    **A:** we'll treat any exit code outside of 1 - 127 as "crashed", and we'll do the same on unix.
- [x] test all permutations of actual vs expected
    **Done:** See #143002 (comment).

### Zulip discussion

See https://rust-lang.zulipchat.com/#narrow/channel/122651-general/topic/compiletest.3A.20terminate.20by.20signal.20vs.20exit.20with.20error/with/525611235

try-job: x86_64-msvc-1
try-job: x86_64-msvc-2

Author: rust-bors[bot]

src/doc/rustc-dev-guide/src/tests/directives.md

@@ -75,8 +75,9 @@ expectations](ui.md#controlling-passfail-expectations).
 | `check-fail`                | Building (no codegen) should fail           | `ui`, `crashes`                           | N/A             |
 | `build-pass`                | Building should pass                        | `ui`, `crashes`, `codegen`, `incremental` | N/A             |
 | `build-fail`                | Building should fail                        | `ui`, `crashes`                           | N/A             |
-| `run-pass`                  | Running the test binary should pass         | `ui`, `crashes`, `incremental`            | N/A             |
-| `run-fail`                  | Running the test binary should fail         | `ui`, `crashes`                           | N/A             |
+| `run-pass`                  | Program must exit with code `0`             | `ui`, `crashes`, `incremental`            | N/A             |
+| `run-fail`                  | Program must exit with code `1..=127`       | `ui`, `crashes`                           | N/A             |
+| `run-crash`                 | Program must crash                          | `ui`,                                     | N/A             |
 | `ignore-pass`               | Ignore `--pass` flag                        | `ui`, `crashes`, `codegen`, `incremental` | N/A             |
 | `dont-check-failure-status` | Don't check exact failure status (i.e. `1`) | `ui`, `incremental`                       | N/A             |
 | `failure-status`            | Check                                       | `ui`, `crashes`                           | Any `u16`       |

src/doc/rustc-dev-guide/src/tests/ui.md

@@ -443,7 +443,7 @@ even run the resulting program. Just add one of the following
   - `//@ build-pass` — compilation and linking should succeed but do
     not run the resulting binary.
   - `//@ run-pass` — compilation should succeed and running the resulting
-    binary should also succeed.
+    binary should make it exit with code 0 which indicates success.
 - Fail directives:
   - `//@ check-fail` — compilation should fail (the codegen phase is skipped).
     This is the default for UI tests.
@@ -452,7 +452,10 @@ even run the resulting program. Just add one of the following
     without the codegen phase, then a second time the full compile should
     fail.
   - `//@ run-fail` — compilation should succeed, but running the resulting
-    binary should fail.
+    binary should make it exit with a code in the `1..=127` which indicates
+    regular failure.
+  - `//@ run-crash` — compilation should succeed, but running the resulting
+    binary should fail with a crash (e.g. `SIGABRT` or `SIGSEGV`).
 
 For `run-pass` and `run-fail` tests, by default the output of the program itself
 is not checked.

src/tools/compiletest/src/common.rs

@@ -110,11 +110,23 @@ string_enum! {
     }
 }
 
+#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
+pub enum RunFailMode {
+    /// Running the program must make it exit with a regular failure exit code
+    /// in the range `1..=127`. If the program is terminated by e.g. a signal
+    /// the test will fail.
+    ExitWithFailure,
+    /// Running the program must result in a crash, e.g. by `SIGABRT` or
+    /// `SIGSEGV` on Unix or on Windows by having an appropriate NTSTATUS high
+    /// bit in the exit code.
+    Crash,
+}
+
 #[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
 pub enum FailMode {
     Check,
     Build,
-    Run,
+    Run(RunFailMode),
 }
 
 string_enum! {

src/tools/compiletest/src/directive-list.rs

@@ -241,6 +241,7 @@ const KNOWN_DIRECTIVE_NAMES: &[&str] = &[
     "regex-error-pattern",
     "remap-src-base",
     "revisions",
+    "run-crash",
     "run-fail",
     "run-flags",
     "run-pass",

src/tools/compiletest/src/header.rs

@@ -9,7 +9,7 @@ use camino::{Utf8Path, Utf8PathBuf};
 use semver::Version;
 use tracing::*;
 
-use crate::common::{Config, Debugger, FailMode, Mode, PassMode};
+use crate::common::{Config, Debugger, FailMode, Mode, PassMode, RunFailMode};
 use crate::debuggers::{extract_cdb_version, extract_gdb_version};
 use crate::errors::ErrorKind;
 use crate::executor::{CollectedTestDesc, ShouldPanic};
@@ -655,7 +655,10 @@ impl TestProps {
             Some(FailMode::Build)
         } else if config.parse_name_directive(ln, "run-fail") {
             check_ui("run");
-            Some(FailMode::Run)
+            Some(FailMode::Run(RunFailMode::ExitWithFailure))
+        } else if config.parse_name_directive(ln, "run-crash") {
+            check_ui("run");
+            Some(FailMode::Run(RunFailMode::Crash))
         } else {
             None
         };

src/tools/compiletest/src/runtest.rs

@@ -17,10 +17,10 @@ use tracing::*;
 
 use crate::common::{
     Assembly, Codegen, CodegenUnits, CompareMode, Config, CoverageMap, CoverageRun, Crashes,
-    DebugInfo, Debugger, FailMode, Incremental, MirOpt, PassMode, Pretty, RunMake, Rustdoc,
-    RustdocJs, RustdocJson, TestPaths, UI_EXTENSIONS, UI_FIXED, UI_RUN_STDERR, UI_RUN_STDOUT,
-    UI_STDERR, UI_STDOUT, UI_SVG, UI_WINDOWS_SVG, Ui, expected_output_path, incremental_dir,
-    output_base_dir, output_base_name, output_testname_unique,
+    DebugInfo, Debugger, FailMode, Incremental, MirOpt, PassMode, Pretty, RunFailMode, RunMake,
+    Rustdoc, RustdocJs, RustdocJson, TestPaths, UI_EXTENSIONS, UI_FIXED, UI_RUN_STDERR,
+    UI_RUN_STDOUT, UI_STDERR, UI_STDOUT, UI_SVG, UI_WINDOWS_SVG, Ui, expected_output_path,
+    incremental_dir, output_base_dir, output_base_name, output_testname_unique,
 };
 use crate::compute_diff::{DiffLine, make_diff, write_diff, write_filtered_diff};
 use crate::errors::{Error, ErrorKind, load_errors};
@@ -277,7 +277,11 @@ impl<'test> TestCx<'test> {
 
     fn should_run(&self, pm: Option<PassMode>) -> WillExecute {
         let test_should_run = match self.config.mode {
-            Ui if pm == Some(PassMode::Run) || self.props.fail_mode == Some(FailMode::Run) => true,
+            Ui if pm == Some(PassMode::Run)
+                || matches!(self.props.fail_mode, Some(FailMode::Run(_))) =>
+            {
+                true
+            }
             MirOpt if pm == Some(PassMode::Run) => true,
             Ui | MirOpt => false,
             mode => panic!("unimplemented for mode {:?}", mode),
@@ -296,6 +300,13 @@ impl<'test> TestCx<'test> {
         }
     }
 
+    fn should_exit_with_failure(&self) -> bool {
+        match self.config.mode {
+            Ui => self.props.fail_mode == Some(FailMode::Run(RunFailMode::ExitWithFailure)),
+            mode => panic!("unimplemented for mode {:?}", mode),
+        }
+    }
+
     fn should_compile_successfully(&self, pm: Option<PassMode>) -> bool {
         match self.config.mode {
             RustdocJs => true,

src/tools/compiletest/src/runtest/ui.rs

@@ -140,14 +140,32 @@ impl TestCx<'_> {
                     &proc_res,
                 );
             }
+            let code = proc_res.status.code();
+            let exit_with_success = proc_res.status.success();
+            // A normal failure exit code is between 1 and 127. If we don't have
+            // an exit code or if it is outside that range (and not 0) we
+            // consider it a crash.
+            let exit_with_failure = code.is_some_and(|c| c >= 1 && c <= 127);
             if self.should_run_successfully(pm) {
-                if !proc_res.status.success() {
-                    self.fatal_proc_rec("test run failed!", &proc_res);
+                if !exit_with_success {
+                    self.fatal_proc_rec(&format!("test run failed! code={code:?}"), &proc_res);
+                }
+            } else if self.should_exit_with_failure() {
+                if !exit_with_failure {
+                    self.fatal_proc_rec(
+                        &format!("test did not exit with failure! code={code:?}"),
+                        &proc_res,
+                    );
+                }
+            } else {
+                // If we get here it means we should not run successfully and we
+                // should not exit with failure. So we should crash. And if we
+                // did exit with success or did exit with failure it means we
+                // did NOT crash.
+                if exit_with_success || exit_with_failure {
+                    self.fatal_proc_rec(&format!("test did not crash! code={code:?}"), &proc_res);
                 }
-            } else if proc_res.status.success() {
-                self.fatal_proc_rec("test run succeeded!", &proc_res);
             }
-
             self.get_output(&proc_res)
         } else {
             self.get_output(&proc_res)

tests/ui/contracts/contract-attributes-generics.rs

@@ -5,9 +5,9 @@
 //@ [unchk_pass] run-pass
 //@ [chk_pass] run-pass
 //
-//@ [chk_fail_pre] run-fail
-//@ [chk_fail_post] run-fail
-//@ [chk_const_fail] run-fail
+//@ [chk_fail_pre] run-crash
+//@ [chk_fail_post] run-crash
+//@ [chk_const_fail] run-crash
 //
 //@ [unchk_pass] compile-flags: -Zcontract-checks=no
 //

tests/ui/contracts/contract-attributes-nest.rs

@@ -5,8 +5,8 @@
 //@ [unchk_fail_post] run-pass
 //@ [chk_pass] run-pass
 //
-//@ [chk_fail_pre] run-fail
-//@ [chk_fail_post] run-fail
+//@ [chk_fail_pre] run-crash
+//@ [chk_fail_post] run-crash
 //
 //@ [unchk_pass] compile-flags: -Zcontract-checks=no
 //@ [unchk_fail_pre] compile-flags: -Zcontract-checks=no

tests/ui/contracts/contract-attributes-tail.rs

@@ -5,8 +5,8 @@
 //@ [unchk_fail_post] run-pass
 //@ [chk_pass] run-pass
 //
-//@ [chk_fail_pre] run-fail
-//@ [chk_fail_post] run-fail
+//@ [chk_fail_pre] run-crash
+//@ [chk_fail_post] run-crash
 //
 //@ [unchk_pass] compile-flags: -Zcontract-checks=no
 //@ [unchk_fail_pre] compile-flags: -Zcontract-checks=no

tests/ui/contracts/contract-captures-via-closure-copy.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ compile-flags: -Zcontract-checks=yes
 
 #![feature(contracts)]

tests/ui/contracts/contract-const-fn.rs

@@ -8,8 +8,8 @@
 //
 //@ [all_pass] run-pass
 //
-//@ [runtime_fail_pre] run-fail
-//@ [runtime_fail_post] run-fail
+//@ [runtime_fail_pre] run-crash
+//@ [runtime_fail_post] run-crash
 //
 //@ [all_pass] compile-flags: -Zcontract-checks=yes
 //@ [runtime_fail_pre] compile-flags: -Zcontract-checks=yes

tests/ui/contracts/contracts-ensures-early-fn-exit.rs

@@ -2,9 +2,9 @@
 //
 //@ [unchk_pass] run-pass
 //@ [chk_pass] run-pass
-//@ [chk_fail_try] run-fail
-//@ [chk_fail_ret] run-fail
-//@ [chk_fail_yeet] run-fail
+//@ [chk_fail_try] run-crash
+//@ [chk_fail_ret] run-crash
+//@ [chk_fail_yeet] run-crash
 //
 //@ [unchk_pass] compile-flags: -Zcontract-checks=no
 //@ [chk_pass] compile-flags: -Zcontract-checks=yes

tests/ui/contracts/internal_machinery/contract-ast-extensions-nest.rs

@@ -5,8 +5,8 @@
 //@ [unchk_fail_post] run-pass
 //@ [chk_pass] run-pass
 //
-//@ [chk_fail_pre] run-fail
-//@ [chk_fail_post] run-fail
+//@ [chk_fail_pre] run-crash
+//@ [chk_fail_post] run-crash
 //
 //@ [unchk_pass] compile-flags: -Zcontract-checks=no
 //@ [unchk_fail_pre] compile-flags: -Zcontract-checks=no

tests/ui/contracts/internal_machinery/contract-ast-extensions-tail.rs

@@ -5,8 +5,8 @@
 //@ [unchk_fail_post] run-pass
 //@ [chk_pass] run-pass
 //
-//@ [chk_fail_pre] run-fail
-//@ [chk_fail_post] run-fail
+//@ [chk_fail_pre] run-crash
+//@ [chk_fail_post] run-crash
 //
 //@ [unchk_pass] compile-flags: -Zcontract-checks=no
 //@ [unchk_fail_pre] compile-flags: -Zcontract-checks=no

tests/ui/contracts/internal_machinery/contract-intrinsics.rs

@@ -3,8 +3,8 @@
 //@ [default] run-pass
 //@ [unchk_pass] run-pass
 //@ [chk_pass] run-pass
-//@ [chk_fail_requires] run-fail
-//@ [chk_fail_ensures] run-fail
+//@ [chk_fail_requires] run-crash
+//@ [chk_fail_ensures] run-crash
 //
 //@ [unchk_pass] compile-flags: -Zcontract-checks=no
 //@ [chk_pass] compile-flags: -Zcontract-checks=yes

tests/ui/contracts/internal_machinery/contract-lang-items.rs

@@ -4,7 +4,7 @@
 //@ [unchk_fail_post] run-pass
 //@ [chk_pass] run-pass
 //
-//@ [chk_fail_post] run-fail
+//@ [chk_fail_post] run-crash
 //
 //@ [unchk_pass] compile-flags: -Zcontract-checks=no
 //@ [unchk_fail_post] compile-flags: -Zcontract-checks=no

tests/ui/extern/extern-types-field-offset.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ check-run-results
 //@ exec-env:RUST_BACKTRACE=0
 //@ normalize-stderr: "(core/src/panicking\.rs):[0-9]+:[0-9]+" -> "$1:$$LINE:$$COL"

tests/ui/mir/alignment/borrow_misaligned_field_projection.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ ignore-i686-pc-windows-msvc: #112480
 //@ compile-flags: -C debug-assertions
 //@ error-pattern: misaligned pointer dereference: address must be a multiple of 0x4 but is

tests/ui/mir/alignment/misaligned_borrow.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ ignore-i686-pc-windows-msvc: #112480
 //@ compile-flags: -C debug-assertions
 //@ error-pattern: misaligned pointer dereference: address must be a multiple of 0x4 but is

tests/ui/mir/alignment/misaligned_lhs.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ ignore-i686-pc-windows-msvc: #112480
 //@ compile-flags: -C debug-assertions
 //@ error-pattern: misaligned pointer dereference: address must be a multiple of 0x4 but is

tests/ui/mir/alignment/misaligned_mut_borrow.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ ignore-i686-pc-windows-msvc: #112480
 //@ compile-flags: -C debug-assertions
 //@ error-pattern: misaligned pointer dereference: address must be a multiple of 0x4 but is

tests/ui/mir/alignment/misaligned_rhs.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ ignore-i686-pc-windows-msvc: #112480
 //@ compile-flags: -C debug-assertions
 //@ error-pattern: misaligned pointer dereference: address must be a multiple of 0x4 but is

tests/ui/mir/alignment/two_pointers.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ ignore-i686-pc-windows-msvc: #112480
 //@ compile-flags: -C debug-assertions
 //@ error-pattern: misaligned pointer dereference: address must be a multiple of 0x4 but is

tests/ui/mir/null/borrowed_mut_null.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ compile-flags: -C debug-assertions
 //@ error-pattern: null pointer dereference occurred
 

tests/ui/mir/null/borrowed_null.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ compile-flags: -C debug-assertions
 //@ error-pattern: null pointer dereference occurred
 

tests/ui/mir/null/borrowed_null_zst.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ compile-flags: -C debug-assertions
 //@ error-pattern: null pointer dereference occurred
 

tests/ui/mir/null/null_lhs.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ compile-flags: -C debug-assertions
 //@ error-pattern: null pointer dereference occurred
 

tests/ui/mir/null/null_rhs.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ compile-flags: -C debug-assertions
 //@ error-pattern: null pointer dereference occurred
 

tests/ui/mir/null/two_pointers.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ compile-flags: -C debug-assertions
 //@ error-pattern: null pointer dereference occurred
 

tests/ui/panics/panic-in-cleanup.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ exec-env:RUST_BACKTRACE=0
 //@ check-run-results
 //@ error-pattern: panic in a destructor during cleanup

tests/ui/panics/panic-in-ffi.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ exec-env:RUST_BACKTRACE=0
 //@ check-run-results
 //@ error-pattern: panic in a function that cannot unwind

tests/ui/panics/panic-in-message-fmt.rs

@@ -1,6 +1,6 @@
 // Checks what happens when formatting the panic message panics.
 
-//@ run-fail
+//@ run-crash
 //@ exec-env:RUST_BACKTRACE=0
 //@ check-run-results
 //@ error-pattern: panicked while processing panic

tests/ui/precondition-checks/alignment.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ compile-flags: -Copt-level=3 -Cdebug-assertions=no -Zub-checks=yes
 //@ error-pattern: unsafe precondition(s) violated: Alignment::new_unchecked requires
 

tests/ui/precondition-checks/ascii-char-digit_unchecked.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ compile-flags: -Copt-level=3 -Cdebug-assertions=no -Zub-checks=yes
 //@ error-pattern: unsafe precondition(s) violated: `ascii::Char::digit_unchecked` input cannot exceed 9
 

tests/ui/precondition-checks/assert_unchecked.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ compile-flags: -Copt-level=3 -Cdebug-assertions=no -Zub-checks=yes
 //@ error-pattern: unsafe precondition(s) violated: hint::assert_unchecked must never be called when the condition is false
 

tests/ui/precondition-checks/char-from_u32_unchecked.rs

@@ -1,4 +1,4 @@
-//@ run-fail
+//@ run-crash
 //@ compile-flags: -Copt-level=3 -Cdebug-assertions=no -Zub-checks=yes
 //@ error-pattern: unsafe precondition(s) violated: invalid value for `char`