Rollup merge of rust-lang#148878 - folkertdev:tail-call-unsupported-abi, r=WaffleLapkin

error when ABI does not support guaranteed tail calls

Some ABIs cannot support guaranteed tail calls. There isn't really an exhaustive list, so this is a best effort. Conveniently, we already disallow calling most of these directly anyway. The only exception that I was able to trigger an LLVM assertion with so far was `cmse-nonsecure-entry`.

For that calling convention, LLVM specifically notes that  (guaranteed) tail calls cannot be supported:

https://github.com/llvm/llvm-project/blob/28dbbba6c3a4e026e085c48cc022cb97b5d8bc6d/llvm/lib/Target/ARM/ARMISelLowering.cpp#L2331-L2335

---

I have some doubts about the implementation here though. I think it would be nicer to use `CanonAbi`, and move the `become` ABI check into `rustc_hir_typeck`, similar to `check_call_abi`:

https://github.com/rust-lang/rust/blob/d6deffe2debecc66501e50f9573214139ab4d678/compiler/rustc_hir_typeck/src/callee.rs#L157-L194

Both the check for whether an ABI is callable and whether it supports guaranteed tail calls can then be methods (containing exhaustive matches) on `CanonAbi`. I'm however not sure

- if the ABI checks are deliberately only performed when constructing MIR
- what assumptions can be made about the `call` expression in [`check_expr_become`](https://github.com/rust-lang/rust/blob/d6deffe2debecc66501e50f9573214139ab4d678/compiler/rustc_hir_typeck/src/expr.rs#L1126-L1150), it looks like currently the check that the "argument" to `become` is a function call also only occurs later during MIR construction

Are there issues with validating the ABI earlier in `rustc_hir_typeck` that I'm overlooking? I believe that we should already know the call's ABI and whether it is c-variadic at that point.

cc ```@workingjubilee``` for `CanonAbi`, ```@davidtwco``` for cmse
r? ```@WaffleLapkin```

Author: Zalathar

compiler/rustc_abi/src/extern_abi.rs

@@ -276,6 +276,51 @@ impl ExternAbi {
             _ => CVariadicStatus::NotSupported,
         }
     }
+
+    /// Returns whether the ABI supports guaranteed tail calls.
+    #[cfg(feature = "nightly")]
+    pub fn supports_guaranteed_tail_call(self) -> bool {
+        match self {
+            Self::CmseNonSecureCall | Self::CmseNonSecureEntry => {
+                // See https://godbolt.org/z/9jhdeqErv. The CMSE calling conventions clear registers
+                // before returning, and hence cannot guarantee a tail call.
+                false
+            }
+            Self::AvrInterrupt
+            | Self::AvrNonBlockingInterrupt
+            | Self::Msp430Interrupt
+            | Self::RiscvInterruptM
+            | Self::RiscvInterruptS
+            | Self::X86Interrupt => {
+                // See https://godbolt.org/z/Edfjnxxcq. Interrupts cannot be called directly.
+                false
+            }
+            Self::GpuKernel | Self::PtxKernel => {
+                // See https://godbolt.org/z/jq5TE5jK1.
+                false
+            }
+            Self::Custom => {
+                // This ABI does not support calls at all (except via assembly).
+                false
+            }
+            Self::C { .. }
+            | Self::System { .. }
+            | Self::Rust
+            | Self::RustCall
+            | Self::RustCold
+            | Self::RustInvalid
+            | Self::Unadjusted
+            | Self::EfiApi
+            | Self::Aapcs { .. }
+            | Self::Cdecl { .. }
+            | Self::Stdcall { .. }
+            | Self::Fastcall { .. }
+            | Self::Thiscall { .. }
+            | Self::Vectorcall { .. }
+            | Self::SysV64 { .. }
+            | Self::Win64 { .. } => true,
+        }
+    }
 }
 
 pub fn all_names() -> Vec<&'static str> {

compiler/rustc_mir_build/src/check_tail_calls.rs

@@ -135,6 +135,10 @@ impl<'tcx> TailCallCkVisitor<'_, 'tcx> {
             self.report_abi_mismatch(expr.span, caller_sig.abi, callee_sig.abi);
         }
 
+        if !callee_sig.abi.supports_guaranteed_tail_call() {
+            self.report_unsupported_abi(expr.span, callee_sig.abi);
+        }
+
         // FIXME(explicit_tail_calls): this currently fails for cases where opaques are used.
         // e.g.
         // ```
@@ -358,6 +362,16 @@ impl<'tcx> TailCallCkVisitor<'_, 'tcx> {
         self.found_errors = Err(err);
     }
 
+    fn report_unsupported_abi(&mut self, sp: Span, callee_abi: ExternAbi) {
+        let err = self
+            .tcx
+            .dcx()
+            .struct_span_err(sp, "ABI does not support guaranteed tail calls")
+            .with_note(format!("`become` is not supported for `extern {callee_abi}` functions"))
+            .emit();
+        self.found_errors = Err(err);
+    }
+
     fn report_signature_mismatch(
         &mut self,
         sp: Span,

tests/ui/explicit-tail-calls/unsupported-abi/cmse-nonsecure-call.rs

@@ -0,0 +1,38 @@
+//@ add-minicore
+//@ ignore-backends: gcc
+//@ compile-flags: --target thumbv8m.main-none-eabi --crate-type lib
+//@ needs-llvm-components: arm
+#![expect(incomplete_features)]
+#![feature(no_core, explicit_tail_calls, abi_cmse_nonsecure_call)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+unsafe extern "C" {
+    safe fn magic() -> extern "cmse-nonsecure-call" fn(u32, u32) -> u32;
+}
+
+// The `cmse-nonsecure-call` ABI can only occur on function pointers:
+//
+// - a `cmse-nonsecure-call` definition throws an error
+// - a `cmse-nonsecure-call` become in a definition with any other ABI is an ABI mismatch
+#[no_mangle]
+extern "cmse-nonsecure-call" fn become_nonsecure_call_1(x: u32, y: u32) -> u32 {
+    //~^ ERROR the `"cmse-nonsecure-call"` ABI is only allowed on function pointers
+    unsafe {
+        let f = magic();
+        become f(1, 2)
+        //~^ ERROR ABI does not support guaranteed tail calls
+    }
+}
+
+#[no_mangle]
+extern "C" fn become_nonsecure_call_2(x: u32, y: u32) -> u32 {
+    unsafe {
+        let f = magic();
+        become f(1, 2)
+        //~^ ERROR mismatched function ABIs
+        //~| ERROR ABI does not support guaranteed tail calls
+    }
+}

tests/ui/explicit-tail-calls/unsupported-abi/cmse-nonsecure-call.stderr

@@ -0,0 +1,34 @@
+error[E0781]: the `"cmse-nonsecure-call"` ABI is only allowed on function pointers
+  --> $DIR/cmse-nonsecure-call.rs:21:1
+   |
+LL | extern "cmse-nonsecure-call" fn become_nonsecure_call_1(x: u32, y: u32) -> u32 {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: ABI does not support guaranteed tail calls
+  --> $DIR/cmse-nonsecure-call.rs:25:9
+   |
+LL |         become f(1, 2)
+   |         ^^^^^^^^^^^^^^
+   |
+   = note: `become` is not supported for `extern "cmse-nonsecure-call"` functions
+
+error: mismatched function ABIs
+  --> $DIR/cmse-nonsecure-call.rs:34:9
+   |
+LL |         become f(1, 2)
+   |         ^^^^^^^^^^^^^^
+   |
+   = note: `become` requires caller and callee to have the same ABI
+   = note: caller ABI is `"C"`, while callee ABI is `"cmse-nonsecure-call"`
+
+error: ABI does not support guaranteed tail calls
+  --> $DIR/cmse-nonsecure-call.rs:34:9
+   |
+LL |         become f(1, 2)
+   |         ^^^^^^^^^^^^^^
+   |
+   = note: `become` is not supported for `extern "cmse-nonsecure-call"` functions
+
+error: aborting due to 4 previous errors
+
+For more information about this error, try `rustc --explain E0781`.

tests/ui/explicit-tail-calls/unsupported-abi/cmse-nonsecure-entry.rs

@@ -0,0 +1,22 @@
+//@ add-minicore
+//@ ignore-backends: gcc
+//@ compile-flags: --target thumbv8m.main-none-eabi --crate-type lib
+//@ needs-llvm-components: arm
+#![expect(incomplete_features)]
+#![feature(no_core, explicit_tail_calls, cmse_nonsecure_entry)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+#[inline(never)]
+extern "cmse-nonsecure-entry" fn entry(c: bool, x: u32, y: u32) -> u32 {
+    if c { x } else { y }
+}
+
+// A `cmse-nonsecure-entry` clears registers before returning, so a tail call cannot be guaranteed.
+#[unsafe(no_mangle)]
+extern "cmse-nonsecure-entry" fn become_nonsecure_entry(c: bool, x: u32, y: u32) -> u32 {
+    become entry(c, x, y)
+    //~^ ERROR ABI does not support guaranteed tail calls
+}

tests/ui/explicit-tail-calls/unsupported-abi/cmse-nonsecure-entry.stderr

@@ -0,0 +1,10 @@
+error: ABI does not support guaranteed tail calls
+  --> $DIR/cmse-nonsecure-entry.rs:20:5
+   |
+LL |     become entry(c, x, y)
+   |     ^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: `become` is not supported for `extern "cmse-nonsecure-entry"` functions
+
+error: aborting due to 1 previous error
+