compiler: Reject impossible reprs during enum layout

Author: workingjubilee

Diff for: compiler/rustc_abi/src/layout.rs

@@ -54,6 +54,9 @@ pub enum LayoutCalculatorError<F> {
 
     /// A union had no fields.
     EmptyUnion,
+
+    /// The fields or variants have irreconcilable reprs
+    ReprConflict,
 }
 
 impl<F> LayoutCalculatorError<F> {
@@ -64,6 +67,7 @@ impl<F> LayoutCalculatorError<F> {
             }
             LayoutCalculatorError::SizeOverflow => LayoutCalculatorError::SizeOverflow,
             LayoutCalculatorError::EmptyUnion => LayoutCalculatorError::EmptyUnion,
+            LayoutCalculatorError::ReprConflict => LayoutCalculatorError::ReprConflict,
         }
     }
 
@@ -77,6 +81,7 @@ impl<F> LayoutCalculatorError<F> {
             }
             LayoutCalculatorError::SizeOverflow => "size overflow",
             LayoutCalculatorError::EmptyUnion => "type is a union with no fields",
+            LayoutCalculatorError::ReprConflict => "type has an invalid repr",
         })
     }
 }
@@ -514,6 +519,10 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
         }
 
         let dl = self.cx.data_layout();
+        // bail if the enum has an incoherent repr that cannot be computed
+        if repr.packed() {
+            return Err(LayoutCalculatorError::ReprConflict);
+        }
 
         let calculate_niche_filling_layout = || -> Option<TmpLayout<FieldIdx, VariantIdx>> {
             if dont_niche_optimize_enum {

Diff for: compiler/rustc_ty_utils/src/layout.rs

@@ -116,6 +116,11 @@ fn map_error<'tcx>(
             cx.tcx().dcx().delayed_bug(format!("computed layout of empty union: {ty:?}"));
             LayoutError::Unknown(ty)
         }
+        LayoutCalculatorError::ReprConflict => {
+            // packed enums are the only known trigger of this, but others might arise
+            cx.tcx().dcx().delayed_bug(format!("computed impossible repr (packed enum?): {ty:?}"));
+            LayoutError::Unknown(ty)
+        }
     };
     error(cx, err)
 }

Diff for: tests/ui/layout/thaw-transmute-invalid-enum.rs

@@ -1,10 +1,14 @@
-//@ known-bug: rust-lang/rust#126966
+#![crate_type = "lib"]
+
 mod assert {
     use std::mem::{Assume, TransmuteFrom};
+    //~^ ERROR: use of unstable library feature 'transmutability'
+    //~| ERROR: use of unstable library feature 'transmutability'
 
     pub fn is_transmutable<Src, Dst>()
     where
         Dst: TransmuteFrom<Src>,
+        //~^ ERROR: use of unstable library feature 'transmutability'
     {
     }
 }
@@ -15,15 +19,18 @@ enum Ox00 {
 }
 
 #[repr(C, packed(2))]
+//~^ ERROR: attribute should be applied to a struct
 enum OxFF {
     V = 0xFF,
 }
 
 fn test() {
     union Superset {
         a: Ox00,
+        //~^ ERROR: field must implement `Copy`
         b: OxFF,
     }
 
     assert::is_transmutable::<Superset, Subset>();
+    //~^ ERROR: cannot find type `Subset`
 }

Diff for: tests/ui/layout/thaw-transmute-invalid-enum.stderr

@@ -0,0 +1,68 @@
+error[E0412]: cannot find type `Subset` in this scope
+  --> $DIR/thaw-transmute-invalid-enum.rs:34:41
+   |
+LL |     assert::is_transmutable::<Superset, Subset>();
+   |                                         ^^^^^^ not found in this scope
+   |
+help: you might be missing a type parameter
+   |
+LL | fn test<Subset>() {
+   |        ++++++++
+
+error[E0517]: attribute should be applied to a struct or union
+  --> $DIR/thaw-transmute-invalid-enum.rs:21:11
+   |
+LL |   #[repr(C, packed(2))]
+   |             ^^^^^^^^^
+LL |
+LL | / enum OxFF {
+LL | |     V = 0xFF,
+LL | | }
+   | |_- not a struct or union
+
+error[E0658]: use of unstable library feature 'transmutability'
+  --> $DIR/thaw-transmute-invalid-enum.rs:4:20
+   |
+LL |     use std::mem::{Assume, TransmuteFrom};
+   |                    ^^^^^^
+   |
+   = note: see issue #99571 <https://github.com/rust-lang/rust/issues/99571> for more information
+   = help: add `#![feature(transmutability)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error[E0658]: use of unstable library feature 'transmutability'
+  --> $DIR/thaw-transmute-invalid-enum.rs:4:28
+   |
+LL |     use std::mem::{Assume, TransmuteFrom};
+   |                            ^^^^^^^^^^^^^
+   |
+   = note: see issue #99571 <https://github.com/rust-lang/rust/issues/99571> for more information
+   = help: add `#![feature(transmutability)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error[E0658]: use of unstable library feature 'transmutability'
+  --> $DIR/thaw-transmute-invalid-enum.rs:10:14
+   |
+LL |         Dst: TransmuteFrom<Src>,
+   |              ^^^^^^^^^^^^^^^^^^
+   |
+   = note: see issue #99571 <https://github.com/rust-lang/rust/issues/99571> for more information
+   = help: add `#![feature(transmutability)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error[E0740]: field must implement `Copy` or be wrapped in `ManuallyDrop<...>` to be used in a union
+  --> $DIR/thaw-transmute-invalid-enum.rs:29:9
+   |
+LL |         a: Ox00,
+   |         ^^^^^^^
+   |
+   = note: union fields must not have drop side-effects, which is currently enforced via either `Copy` or `ManuallyDrop<...>`
+help: wrap the field type in `ManuallyDrop<...>`
+   |
+LL |         a: std::mem::ManuallyDrop<Ox00>,
+   |            +++++++++++++++++++++++    +
+
+error: aborting due to 6 previous errors
+
+Some errors have detailed explanations: E0412, E0517, E0658, E0740.
+For more information about an error, try `rustc --explain E0412`.

Diff for: tests/ui/layout/thaw-validate-invalid-enum.rs

@@ -1,7 +1,6 @@
-//@ known-bug: rust-lang/rust#128870
 //@ compile-flags: -Zvalidate-mir
 
-#[repr(packed)]
+#[repr(packed)] //~ ERROR: attribute should be applied to a struct
 #[repr(u32)]
 enum E {
     A,
@@ -12,7 +11,7 @@ enum E {
 fn main() {
     union InvalidTag {
         int: u32,
-        e: E,
+        e: E, //~ ERROR: field must implement `Copy`
     }
     let _invalid_tag = InvalidTag { int: 4 };
 }

Diff for: tests/ui/layout/thaw-validate-invalid-enum.stderr

@@ -0,0 +1,29 @@
+error[E0517]: attribute should be applied to a struct or union
+  --> $DIR/thaw-validate-invalid-enum.rs:3:8
+   |
+LL |   #[repr(packed)]
+   |          ^^^^^^
+LL |   #[repr(u32)]
+LL | / enum E {
+LL | |     A,
+LL | |     B,
+LL | |     C,
+LL | | }
+   | |_- not a struct or union
+
+error[E0740]: field must implement `Copy` or be wrapped in `ManuallyDrop<...>` to be used in a union
+  --> $DIR/thaw-validate-invalid-enum.rs:14:9
+   |
+LL |         e: E,
+   |         ^^^^
+   |
+   = note: union fields must not have drop side-effects, which is currently enforced via either `Copy` or `ManuallyDrop<...>`
+help: wrap the field type in `ManuallyDrop<...>`
+   |
+LL |         e: std::mem::ManuallyDrop<E>,
+   |            +++++++++++++++++++++++ +
+
+error: aborting due to 2 previous errors
+
+Some errors have detailed explanations: E0517, E0740.
+For more information about an error, try `rustc --explain E0517`.