error when ABI does not support guaranteed tail calls

Author: folkertdev

compiler/rustc_abi/src/extern_abi.rs

@@ -276,6 +276,36 @@ impl ExternAbi {
             _ => CVariadicStatus::NotSupported,
         }
     }
+
+    /// Returns whether the ABI supports guaranteed tail calls.
+    #[cfg(feature = "nightly")]
+    pub fn supports_guaranteed_tail_call(self) -> bool {
+        match self {
+            Self::CmseNonSecureCall | Self::CmseNonSecureEntry => {
+                // See https://godbolt.org/z/9jhdeqErv. The CMSE calling conventions clear registers
+                // before returning, and hence cannot guarantee a tail call.
+                false
+            }
+            Self::AvrInterrupt
+            | Self::AvrNonBlockingInterrupt
+            | Self::Msp430Interrupt
+            | Self::RiscvInterruptM
+            | Self::RiscvInterruptS
+            | Self::X86Interrupt => {
+                // See https://godbolt.org/z/Edfjnxxcq. Interrupts cannot be called directly.
+                false
+            }
+            Self::GpuKernel | Self::PtxKernel => {
+                // See https://godbolt.org/z/jq5TE5jK1.
+                false
+            }
+            Self::Custom => {
+                // This ABI does not support calls at all (except via assembly).
+                false
+            }
+            _ => true,
+        }
+    }
 }
 
 pub fn all_names() -> Vec<&'static str> {

compiler/rustc_mir_build/src/check_tail_calls.rs

@@ -135,6 +135,10 @@ impl<'tcx> TailCallCkVisitor<'_, 'tcx> {
             self.report_abi_mismatch(expr.span, caller_sig.abi, callee_sig.abi);
         }
 
+        if !callee_sig.abi.supports_guaranteed_tail_call() {
+            self.report_unsupported_abi(expr.span, callee_sig.abi);
+        }
+
         // FIXME(explicit_tail_calls): this currently fails for cases where opaques are used.
         // e.g.
         // ```
@@ -358,6 +362,16 @@ impl<'tcx> TailCallCkVisitor<'_, 'tcx> {
         self.found_errors = Err(err);
     }
 
+    fn report_unsupported_abi(&mut self, sp: Span, callee_abi: ExternAbi) {
+        let err = self
+            .tcx
+            .dcx()
+            .struct_span_err(sp, "ABI does not support guaranteed tail calls")
+            .with_note(format!("`become` is not supported for `extern {callee_abi}` functions"))
+            .emit();
+        self.found_errors = Err(err);
+    }
+
     fn report_signature_mismatch(
         &mut self,
         sp: Span,

tests/ui/explicit-tail-calls/unsupported-abi/cmse-nonsecure-call.rs

@@ -0,0 +1,37 @@
+//@ add-minicore
+//@ compile-flags: --target thumbv8m.main-none-eabi --crate-type lib
+//@ needs-llvm-components: arm
+#![expect(incomplete_features)]
+#![feature(no_core, explicit_tail_calls, abi_cmse_nonsecure_call)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+unsafe extern "C" {
+    safe fn magic() -> extern "cmse-nonsecure-call" fn(u32, u32) -> u32;
+}
+
+// The `cmse-nonsecure-call` ABI can only occur on function pointers:
+//
+// - a `cmse-nonsecure-call` definition throws an error
+// - a `cmse-nonsecure-call` become in a definition with any other ABI is an ABI mismatch
+#[no_mangle]
+extern "cmse-nonsecure-call" fn become_nonsecure_call_1(x: u32, y: u32) -> u32 {
+    //~^ ERROR the `"cmse-nonsecure-call"` ABI is only allowed on function pointers
+    unsafe {
+        let f = magic();
+        become f(1, 2)
+        //~^ ERROR ABI does not support guaranteed tail calls
+    }
+}
+
+#[no_mangle]
+extern "C" fn become_nonsecure_call_2(x: u32, y: u32) -> u32 {
+    unsafe {
+        let f = magic();
+        become f(1, 2)
+        //~^ ERROR mismatched function ABIs
+        //~| ERROR ABI does not support guaranteed tail calls
+    }
+}

tests/ui/explicit-tail-calls/unsupported-abi/cmse-nonsecure-call.stderr

@@ -0,0 +1,34 @@
+error[E0781]: the `"cmse-nonsecure-call"` ABI is only allowed on function pointers
+  --> $DIR/cmse-nonsecure-call.rs:20:1
+   |
+LL | extern "cmse-nonsecure-call" fn become_nonsecure_call_1(x: u32, y: u32) -> u32 {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: ABI does not support guaranteed tail calls
+  --> $DIR/cmse-nonsecure-call.rs:24:9
+   |
+LL |         become f(1, 2)
+   |         ^^^^^^^^^^^^^^
+   |
+   = note: `become` is not supported for `extern "cmse-nonsecure-call"` functions
+
+error: mismatched function ABIs
+  --> $DIR/cmse-nonsecure-call.rs:33:9
+   |
+LL |         become f(1, 2)
+   |         ^^^^^^^^^^^^^^
+   |
+   = note: `become` requires caller and callee to have the same ABI
+   = note: caller ABI is `"C"`, while callee ABI is `"cmse-nonsecure-call"`
+
+error: ABI does not support guaranteed tail calls
+  --> $DIR/cmse-nonsecure-call.rs:33:9
+   |
+LL |         become f(1, 2)
+   |         ^^^^^^^^^^^^^^
+   |
+   = note: `become` is not supported for `extern "cmse-nonsecure-call"` functions
+
+error: aborting due to 4 previous errors
+
+For more information about this error, try `rustc --explain E0781`.

tests/ui/explicit-tail-calls/unsupported-abi/cmse-nonsecure-entry.rs

@@ -0,0 +1,21 @@
+//@ add-minicore
+//@ compile-flags: --target thumbv8m.main-none-eabi --crate-type lib
+//@ needs-llvm-components: arm
+#![expect(incomplete_features)]
+#![feature(no_core, explicit_tail_calls, cmse_nonsecure_entry)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+#[inline(never)]
+extern "cmse-nonsecure-entry" fn entry(c: bool, x: u32, y: u32) -> u32 {
+    if c { x } else { y }
+}
+
+// A `cmse-nonsecure-entry` clears registers before returning, so a tail call cannot be guaranteed.
+#[unsafe(no_mangle)]
+extern "cmse-nonsecure-entry" fn become_nonsecure_entry(c: bool, x: u32, y: u32) -> u32 {
+    become entry(c, x, y)
+    //~^ ERROR ABI does not support guaranteed tail calls
+}

tests/ui/explicit-tail-calls/unsupported-abi/cmse-nonsecure-entry.stderr

@@ -0,0 +1,10 @@
+error: ABI does not support guaranteed tail calls
+  --> $DIR/cmse-nonsecure-entry.rs:19:5
+   |
+LL |     become entry(c, x, y)
+   |     ^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: `become` is not supported for `extern "cmse-nonsecure-entry"` functions
+
+error: aborting due to 1 previous error
+

tests/ui/explicit-tail-calls/unsupported-abi/gpu-kernel.amdgpu.stderr

@@ -0,0 +1,22 @@
+error: functions with the "gpu-kernel" ABI cannot be called
+  --> $DIR/gpu-kernel.rs:21:12
+   |
+LL |     become f2(ptr)
+   |            ^^^^^^^
+   |
+note: an `extern "gpu-kernel"` function can only be called using inline assembly
+  --> $DIR/gpu-kernel.rs:21:12
+   |
+LL |     become f2(ptr)
+   |            ^^^^^^^
+
+error: ABI does not support guaranteed tail calls
+  --> $DIR/gpu-kernel.rs:21:5
+   |
+LL |     become f2(ptr)
+   |     ^^^^^^^^^^^^^^
+   |
+   = note: `become` is not supported for `extern "gpu-kernel"` functions
+
+error: aborting due to 2 previous errors
+

tests/ui/explicit-tail-calls/unsupported-abi/gpu-kernel.nvptx.stderr

@@ -0,0 +1,22 @@
+error: functions with the "gpu-kernel" ABI cannot be called
+  --> $DIR/gpu-kernel.rs:21:12
+   |
+LL |     become f2(ptr)
+   |            ^^^^^^^
+   |
+note: an `extern "gpu-kernel"` function can only be called using inline assembly
+  --> $DIR/gpu-kernel.rs:21:12
+   |
+LL |     become f2(ptr)
+   |            ^^^^^^^
+
+error: ABI does not support guaranteed tail calls
+  --> $DIR/gpu-kernel.rs:21:5
+   |
+LL |     become f2(ptr)
+   |     ^^^^^^^^^^^^^^
+   |
+   = note: `become` is not supported for `extern "gpu-kernel"` functions
+
+error: aborting due to 2 previous errors
+

tests/ui/explicit-tail-calls/unsupported-abi/gpu-kernel.rs

@@ -0,0 +1,24 @@
+// Checks that the gpu-kernel calling convention correctly translates to LLVM calling conventions.
+
+//@ add-minicore
+//@ revisions: amdgpu nvptx
+//@ [amdgpu] compile-flags: --crate-type=rlib --target=amdgcn-amd-amdhsa -Ctarget-cpu=gfx900
+//@ [amdgpu] needs-llvm-components: amdgpu
+//@ [nvptx] compile-flags: --crate-type=rlib --target=nvptx64-nvidia-cuda
+//@ [nvptx] needs-llvm-components: nvptx
+#![expect(incomplete_features)]
+#![feature(no_core, explicit_tail_calls, abi_gpu_kernel)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+#[inline(never)]
+extern "gpu-kernel" fn f2(ptr: *mut i32) {}
+
+#[no_mangle]
+extern "gpu-kernel" fn f1(ptr: *mut i32) {
+    become f2(ptr)
+    //~^ ERROR ABI does not support guaranteed tail calls
+    //~| ERROR functions with the "gpu-kernel" ABI cannot be called
+}

tests/ui/explicit-tail-calls/unsupported-abi/x86-interrupt.rs

@@ -0,0 +1,19 @@
+//@ add-minicore
+//@ compile-flags: --target x86_64-unknown-linux-gnu --crate-type lib
+//@ needs-llvm-components: x86
+#![expect(incomplete_features)]
+#![feature(no_core, explicit_tail_calls, abi_x86_interrupt)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+#[inline(never)]
+extern "x86-interrupt" fn f2(_sf: *mut u8) {}
+
+#[no_mangle]
+extern "x86-interrupt" fn f1(sf: *mut u8) {
+    become f2(sf)
+    //~^ ERROR ABI does not support guaranteed tail calls
+    //~| ERROR functions with the "x86-interrupt" ABI cannot be called
+}