Reject unsupported, unstable ABIs during AST lowering

Cargo.lock

@@ -3773,6 +3773,7 @@ dependencies = [
  "rustc_abi",
  "rustc_arena",
  "rustc_ast",
+ "rustc_ast_lowering",
  "rustc_attr_data_structures",
  "rustc_data_structures",
  "rustc_errors",

compiler/rustc_ast_lowering/src/stability.rs

@@ -1,6 +1,7 @@
 use std::fmt;
 
 use rustc_abi::ExternAbi;
+use rustc_errors::{E0570, struct_span_code_err};
 use rustc_feature::Features;
 use rustc_session::Session;
 use rustc_session::parse::feature_err;
@@ -19,10 +20,10 @@ pub(crate) fn extern_abi_enabled(
     features: &rustc_feature::Features,
     span: Span,
     abi: ExternAbi,
-) -> Result<(), UnstableAbi> {
+) -> Result<ExternAbi, UnstableAbi> {
     extern_abi_stability(abi).or_else(|unstable @ UnstableAbi { feature, .. }| {
         if features.enabled(feature) || span.allows_unstable(feature) {
-            Ok(())
+            Ok(abi)
         } else {
             Err(unstable)
         }
@@ -31,13 +32,28 @@ pub(crate) fn extern_abi_enabled(
 
 #[allow(rustc::untranslatable_diagnostic)]
 pub(crate) fn gate_unstable_abi(sess: &Session, features: &Features, span: Span, abi: ExternAbi) {
-    match extern_abi_enabled(features, span, abi) {
-        Ok(_) => (),
-        Err(unstable_abi) => {
-            let explain = unstable_abi.to_string();
-            feature_err(sess, unstable_abi.feature, span, explain).emit();
-        }
+    let Err(unstable) = extern_abi_stability(abi) else { return };
+    // what are we doing here? this is mixing target support with stability?
+    // well, unfortunately we allowed some ABIs to be used via fn pointers and such on stable,
+    // so we can't simply error any time someone uses certain ABIs as we want to let the FCW ride.
+    // however, for a number of *unstable* ABIs, we can simply fix them because they're unstable!
+    // otherwise it's the same idea as checking during lowering at all: because `extern "ABI"` has to
+    // be visible during lowering of some crate, we can easily nail use of certain ABIs before we
+    // get to e.g. attempting to do invalid codegen for the target.
+    if !sess.target.is_abi_supported(unstable.abi) {
+        struct_span_code_err!(
+            sess.dcx(),
+            span,
+            E0570,
+            "`{abi}` is not a supported ABI for the current target",
+        )
+        .emit();
+    }
+    if features.enabled(unstable.feature) || span.allows_unstable(unstable.feature) {
+        return;
     }
+    let explain = unstable.to_string();
+    feature_err(sess, unstable.feature, span, explain).emit();
 }
 
 pub struct UnstableAbi {
@@ -65,7 +81,7 @@ impl fmt::Display for UnstableAbi {
     }
 }
 
-pub fn extern_abi_stability(abi: ExternAbi) -> Result<(), UnstableAbi> {
+pub fn extern_abi_stability(abi: ExternAbi) -> Result<ExternAbi, UnstableAbi> {
     match abi {
         // stable ABIs
         ExternAbi::Rust
@@ -78,7 +94,7 @@ pub fn extern_abi_stability(abi: ExternAbi) -> Result<(), UnstableAbi> {
         | ExternAbi::Win64 { .. }
         | ExternAbi::SysV64 { .. }
         | ExternAbi::System { .. }
-        | ExternAbi::EfiApi => Ok(()),
+        | ExternAbi::EfiApi => Ok(abi),
         ExternAbi::Unadjusted => {
             Err(UnstableAbi { abi, feature: sym::abi_unadjusted, explain: GateReason::ImplDetail })
         }

compiler/rustc_hir_analysis/Cargo.toml

@@ -13,6 +13,7 @@ itertools = "0.12"
 rustc_abi = { path = "../rustc_abi" }
 rustc_arena = { path = "../rustc_arena" }
 rustc_ast = { path = "../rustc_ast" }
+rustc_ast_lowering = { path = "../rustc_ast_lowering" }
 rustc_attr_data_structures = { path = "../rustc_attr_data_structures" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }

compiler/rustc_hir_analysis/src/check/check.rs

@@ -2,6 +2,7 @@ use std::cell::LazyCell;
 use std::ops::ControlFlow;
 
 use rustc_abi::FieldIdx;
+use rustc_ast_lowering::stability::extern_abi_stability;
 use rustc_attr_data_structures::ReprAttr::ReprPacked;
 use rustc_data_structures::unord::{UnordMap, UnordSet};
 use rustc_errors::MultiSpan;
@@ -35,25 +36,33 @@ use {rustc_attr_data_structures as attrs, rustc_hir as hir};
 use super::compare_impl_item::check_type_bounds;
 use super::*;
 
-pub fn check_abi(tcx: TyCtxt<'_>, span: Span, abi: ExternAbi) {
-    if !tcx.sess.target.is_abi_supported(abi) {
-        struct_span_code_err!(
-            tcx.dcx(),
-            span,
-            E0570,
-            "`{abi}` is not a supported ABI for the current target",
-        )
-        .emit();
+/// check if a stable ABI passes e.g. target support
+///
+/// we are gating unstable target-unsupported ABIs in rustc_ast_lowering which is more foolproof.
+/// but until we finish the FCW associated with check_abi_fn_ptr we can't simply consolidate them.
+pub fn gate_stable_abi(tcx: TyCtxt<'_>, span: Span, abi: ExternAbi) {
+    if let Ok(abi) = extern_abi_stability(abi) {
+        if !tcx.sess.target.is_abi_supported(abi) {
+            struct_span_code_err!(
+                tcx.dcx(),
+                span,
+                E0570,
+                "`{abi}` is not a supported ABI for the current target",
+            )
+            .emit();
+        }
     }
 }
 
-pub fn check_abi_fn_ptr(tcx: TyCtxt<'_>, hir_id: hir::HirId, span: Span, abi: ExternAbi) {
-    if !tcx.sess.target.is_abi_supported(abi) {
-        tcx.node_span_lint(UNSUPPORTED_FN_PTR_CALLING_CONVENTIONS, hir_id, span, |lint| {
-            lint.primary_message(format!(
-                "the calling convention {abi} is not supported on this target"
-            ));
-        });
+pub fn gate_stable_fn_ptr_abi(tcx: TyCtxt<'_>, hir_id: hir::HirId, span: Span, abi: ExternAbi) {
+    if let Ok(abi) = extern_abi_stability(abi) {
+        if !tcx.sess.target.is_abi_supported(abi) {
+            tcx.node_span_lint(UNSUPPORTED_FN_PTR_CALLING_CONVENTIONS, hir_id, span, |lint| {
+                lint.primary_message(format!(
+                    "the calling convention {abi} is not supported on this target"
+                ));
+            });
+        }
     }
 }
 
@@ -779,7 +788,7 @@ pub(crate) fn check_item_type(tcx: TyCtxt<'_>, def_id: LocalDefId) {
             let hir::ItemKind::ForeignMod { abi, items } = it.kind else {
                 return;
             };
-            check_abi(tcx, it.span, abi);
+            gate_stable_abi(tcx, it.span, abi);
 
             for item in items {
                 let def_id = item.id.owner_id.def_id;

compiler/rustc_hir_analysis/src/check/mod.rs

@@ -72,7 +72,7 @@ pub mod wfcheck;
 
 use std::num::NonZero;
 
-pub use check::{check_abi, check_abi_fn_ptr};
+pub use check::{gate_stable_abi, gate_stable_fn_ptr_abi};
 use rustc_abi::{ExternAbi, VariantIdx};
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
 use rustc_errors::{Diag, ErrorGuaranteed, pluralize, struct_span_code_err};

compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs

@@ -50,7 +50,7 @@ use rustc_trait_selection::traits::wf::object_region_bounds;
 use rustc_trait_selection::traits::{self, ObligationCtxt};
 use tracing::{debug, instrument};
 
-use crate::check::check_abi_fn_ptr;
+use crate::check::gate_stable_fn_ptr_abi;
 use crate::errors::{AmbiguousLifetimeBound, BadReturnTypeNotation};
 use crate::hir_ty_lowering::errors::{GenericsArgsErrExtend, prohibit_assoc_item_constraint};
 use crate::hir_ty_lowering::generics::{check_generic_arg_count, lower_generic_args};
@@ -2739,7 +2739,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         if let hir::Node::Ty(hir::Ty { kind: hir::TyKind::BareFn(bare_fn_ty), span, .. }) =
             tcx.hir_node(hir_id)
         {
-            check_abi_fn_ptr(tcx, hir_id, *span, bare_fn_ty.abi);
+            gate_stable_fn_ptr_abi(tcx, hir_id, *span, bare_fn_ty.abi);
         }
 
         // reject function types that violate cmse ABI requirements

compiler/rustc_hir_typeck/src/lib.rs

@@ -48,7 +48,7 @@ use rustc_errors::{Applicability, ErrorGuaranteed, pluralize, struct_span_code_e
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::{HirId, HirIdMap, Node};
-use rustc_hir_analysis::check::check_abi;
+use rustc_hir_analysis::check::gate_stable_abi;
 use rustc_hir_analysis::hir_ty_lowering::HirTyLowerer;
 use rustc_infer::traits::{ObligationCauseCode, ObligationInspector, WellFormedLoc};
 use rustc_middle::query::Providers;
@@ -149,7 +149,7 @@ fn typeck_with_inspect<'tcx>(
             tcx.fn_sig(def_id).instantiate_identity()
         };
 
-        check_abi(tcx, span, fn_sig.abi());
+        gate_stable_abi(tcx, span, fn_sig.abi());
 
         // Compute the function signature from point of view of inside the fn.
         let mut fn_sig = tcx.liberate_late_bound_regions(def_id.to_def_id(), fn_sig);

tests/rustdoc-json/fn_pointer/abi.rs

@@ -1,4 +1,4 @@
-#![feature(abi_vectorcall)]
+#![feature(rust_cold_cc)]
 
 //@ is "$.index[?(@.name=='AbiRust')].inner.type_alias.type.function_pointer.header.abi" \"Rust\"
 pub type AbiRust = fn();
@@ -15,8 +15,5 @@ pub type AbiCUnwind = extern "C-unwind" fn();
 //@ is "$.index[?(@.name=='AbiSystemUnwind')].inner.type_alias.type.function_pointer.header.abi" '{"System": {"unwind": true}}'
 pub type AbiSystemUnwind = extern "system-unwind" fn();
 
-//@ is "$.index[?(@.name=='AbiVecorcall')].inner.type_alias.type.function_pointer.header.abi.Other" '"\"vectorcall\""'
-pub type AbiVecorcall = extern "vectorcall" fn();
-
-//@ is "$.index[?(@.name=='AbiVecorcallUnwind')].inner.type_alias.type.function_pointer.header.abi.Other" '"\"vectorcall-unwind\""'
-pub type AbiVecorcallUnwind = extern "vectorcall-unwind" fn();
+//@ is "$.index[?(@.name=='AbiRustCold')].inner.type_alias.type.function_pointer.header.abi.Other" '"\"rust-cold\""'
+pub type AbiRustCold = extern "rust-cold" fn();

tests/rustdoc-json/fns/abi.rs

@@ -1,4 +1,4 @@
-#![feature(abi_vectorcall)]
+#![feature(rust_cold_cc)]
 
 //@ is "$.index[?(@.name=='abi_rust')].inner.function.header.abi" \"Rust\"
 pub fn abi_rust() {}
@@ -15,8 +15,5 @@ pub extern "C-unwind" fn abi_c_unwind() {}
 //@ is "$.index[?(@.name=='abi_system_unwind')].inner.function.header.abi" '{"System": {"unwind": true}}'
 pub extern "system-unwind" fn abi_system_unwind() {}
 
-//@ is "$.index[?(@.name=='abi_vectorcall')].inner.function.header.abi.Other" '"\"vectorcall\""'
-pub extern "vectorcall" fn abi_vectorcall() {}
-
-//@ is "$.index[?(@.name=='abi_vectorcall_unwind')].inner.function.header.abi.Other" '"\"vectorcall-unwind\""'
-pub extern "vectorcall-unwind" fn abi_vectorcall_unwind() {}
+//@ is "$.index[?(@.name=='abi_rust_cold')].inner.function.header.abi.Other" '"\"rust-cold\""'
+pub extern "rust-cold" fn abi_rust_cold() {}

tests/rustdoc-json/methods/abi.rs

@@ -1,5 +1,4 @@
-#![feature(abi_vectorcall)]
-
+#![feature(rust_cold_cc)]
 //@ has "$.index[?(@.name=='Foo')]"
 pub struct Foo;
 
@@ -19,11 +18,8 @@ impl Foo {
     //@ is "$.index[?(@.name=='abi_system_unwind')].inner.function.header.abi" '{"System": {"unwind": true}}'
     pub extern "system-unwind" fn abi_system_unwind() {}
 
-    //@ is "$.index[?(@.name=='abi_vectorcall')].inner.function.header.abi.Other" '"\"vectorcall\""'
-    pub extern "vectorcall" fn abi_vectorcall() {}
-
-    //@ is "$.index[?(@.name=='abi_vectorcall_unwind')].inner.function.header.abi.Other" '"\"vectorcall-unwind\""'
-    pub extern "vectorcall-unwind" fn abi_vectorcall_unwind() {}
+    //@ is "$.index[?(@.name=='abi_rust_cold')].inner.function.header.abi.Other" '"\"rust-cold\""'
+    pub extern "rust-cold" fn abi_rust_cold() {}
 }
 
 pub trait Bar {
@@ -42,9 +38,6 @@ pub trait Bar {
     //@ is "$.index[?(@.name=='trait_abi_system_unwind')].inner.function.header.abi" '{"System": {"unwind": true}}'
     extern "system-unwind" fn trait_abi_system_unwind() {}
 
-    //@ is "$.index[?(@.name=='trait_abi_vectorcall')].inner.function.header.abi.Other" '"\"vectorcall\""'
-    extern "vectorcall" fn trait_abi_vectorcall() {}
-
-    //@ is "$.index[?(@.name=='trait_abi_vectorcall_unwind')].inner.function.header.abi.Other" '"\"vectorcall-unwind\""'
-    extern "vectorcall-unwind" fn trait_abi_vectorcall_unwind() {}
+    //@ is "$.index[?(@.name=='trait_abi_rust_cold')].inner.function.header.abi.Other" '"\"rust-cold\""'
+    extern "rust-cold" fn trait_abi_rust_cold() {}
 }

tests/rustdoc-json/vectorcall.rs

@@ -0,0 +1,27 @@
+#![feature(abi_vectorcall)]
+//@ only-x86_64
+
+//@ is "$.index[?(@.name=='AbiVectorcall')].inner.type_alias.type.function_pointer.header.abi.Other" '"\"vectorcall\""'
+pub type AbiVectorcall = extern "vectorcall" fn();
+
+//@ is "$.index[?(@.name=='AbiVectorcallUnwind')].inner.type_alias.type.function_pointer.header.abi.Other" '"\"vectorcall-unwind\""'
+pub type AbiVectorcallUnwind = extern "vectorcall-unwind" fn();
+
+//@ has "$.index[?(@.name=='Foo')]"
+pub struct Foo;
+
+impl Foo {
+    //@ is "$.index[?(@.name=='abi_vectorcall')].inner.function.header.abi.Other" '"\"vectorcall\""'
+    pub extern "vectorcall" fn abi_vectorcall() {}
+
+    //@ is "$.index[?(@.name=='abi_vectorcall_unwind')].inner.function.header.abi.Other" '"\"vectorcall-unwind\""'
+    pub extern "vectorcall-unwind" fn abi_vectorcall_unwind() {}
+}
+
+pub trait Bar {
+    //@ is "$.index[?(@.name=='trait_abi_vectorcall')].inner.function.header.abi.Other" '"\"vectorcall\""'
+    extern "vectorcall" fn trait_abi_vectorcall() {}
+
+    //@ is "$.index[?(@.name=='trait_abi_vectorcall_unwind')].inner.function.header.abi.Other" '"\"vectorcall-unwind\""'
+    extern "vectorcall-unwind" fn trait_abi_vectorcall_unwind() {}
+}

tests/ui/abi/unsupported-abi-transmute.rs

@@ -0,0 +1,7 @@
+#![feature(abi_gpu_kernel)]
+// Check we error before unsupported ABIs reach codegen stages.
+
+fn main() {
+    let a = unsafe { core::mem::transmute::<usize, extern "gpu-kernel" fn(i32)>(4) }(2);
+    //~^ ERROR E0570
+}

tests/ui/abi/unsupported-abi-transmute.stderr

@@ -0,0 +1,9 @@
+error[E0570]: `"gpu-kernel"` is not a supported ABI for the current target
+  --> $DIR/unsupported-abi-transmute.rs:5:59
+   |
+LL |     let a = unsafe { core::mem::transmute::<usize, extern "gpu-kernel" fn(i32)>(4) }(2);
+   |                                                           ^^^^^^^^^^^^
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0570`.

tests/ui/abi/unsupported-extern-abi-in-type-impl.rs

@@ -0,0 +1,9 @@
+//@ compile-flags: --crate-type=lib
+//@ edition: 2018
+#![feature(abi_gpu_kernel)]
+struct Test;
+
+impl Test {
+    pub extern "gpu-kernel" fn test(val: &str) {}
+    //~^ ERROR [E0570]
+}

tests/ui/abi/unsupported-extern-abi-in-type-impl.stderr

@@ -0,0 +1,9 @@
+error[E0570]: `"gpu-kernel"` is not a supported ABI for the current target
+  --> $DIR/unsupported-extern-abi-in-type-impl.rs:7:16
+   |
+LL |     pub extern "gpu-kernel" fn test(val: &str) {}
+   |                ^^^^^^^^^^^^
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0570`.