1+ use crate :: layout:: ty:: ParamEnv ;
12use hir:: def_id:: DefId ;
23use rustc_hir as hir;
34use rustc_index:: bit_set:: BitSet ;
@@ -13,9 +14,12 @@ use rustc_session::{DataTypeKind, FieldInfo, FieldKind, SizeKind, VariantInfo};
1314use rustc_span:: sym;
1415use rustc_span:: symbol:: Symbol ;
1516use rustc_target:: abi:: * ;
17+ use rustc_type_ir:: DynKind ;
1618
19+ use std:: cmp;
1720use std:: fmt:: Debug ;
1821use std:: iter;
22+ use std:: ops:: ControlFlow ;
1923
2024use crate :: errors:: {
2125 MultipleArrayFieldsSimdType , NonPrimitiveSimdType , OversizedSimdType , ZeroLengthSimdType ,
@@ -123,7 +127,7 @@ fn layout_of_uncached<'tcx>(
123127 } ;
124128 debug_assert ! ( !ty. has_non_region_infer( ) ) ;
125129
126- Ok ( match * ty. kind ( ) {
130+ let layout = match * ty. kind ( ) {
127131 // Basic scalars.
128132 ty:: Bool => tcx. mk_layout ( LayoutS :: scalar (
129133 cx,
@@ -201,10 +205,32 @@ fn layout_of_uncached<'tcx>(
201205 return Ok ( tcx. mk_layout ( LayoutS :: scalar ( cx, data_ptr) ) ) ;
202206 }
203207
204- let Abi :: Scalar ( metadata) = metadata_layout. abi else {
208+ let Abi :: Scalar ( mut metadata) = metadata_layout. abi else {
205209 return Err ( error ( cx, LayoutError :: Unknown ( pointee) ) ) ;
206210 } ;
207211
212+ if !ty. is_unsafe_ptr ( ) && metadata_ty == tcx. types . usize {
213+ let tail = tcx. struct_tail_erasing_lifetimes ( pointee, param_env) ;
214+ // // eprintln!("usize-meta {:?} {}", pointee, pointee_zst);
215+ match tail. kind ( ) {
216+ ty:: Slice ( element) => match ty_is_non_zst ( * element, param_env, tcx) {
217+ NonZst :: True => {
218+ metadata. valid_range_mut ( ) . end =
219+ dl. ptr_sized_integer ( ) . signed_max ( ) as u128
220+ }
221+ NonZst :: Unknown => return Err ( error ( cx, LayoutError :: Unknown ( ty) ) ) ,
222+ _ => { }
223+ } ,
224+ ty:: Str => {
225+ metadata. valid_range_mut ( ) . end =
226+ dl. ptr_sized_integer ( ) . signed_max ( ) as u128 ;
227+ }
228+ _ => {
229+ eprint ! ( "unexpected tail {:?}" , tail) ;
230+ }
231+ }
232+ }
233+
208234 metadata
209235 } else {
210236 let unsized_part = tcx. struct_tail_erasing_lifetimes ( pointee, param_env) ;
@@ -213,7 +239,28 @@ fn layout_of_uncached<'tcx>(
213239 ty:: Foreign ( ..) => {
214240 return Ok ( tcx. mk_layout ( LayoutS :: scalar ( cx, data_ptr) ) ) ;
215241 }
216- ty:: Slice ( _) | ty:: Str => scalar_unit ( Int ( dl. ptr_sized_integer ( ) , false ) ) ,
242+ ty:: Slice ( element) => {
243+ let mut metadata = scalar_unit ( Int ( dl. ptr_sized_integer ( ) , false ) ) ;
244+ if !ty. is_unsafe_ptr ( ) {
245+ match ty_is_non_zst ( * element, param_env, tcx) {
246+ NonZst :: True => {
247+ metadata. valid_range_mut ( ) . end =
248+ dl. ptr_sized_integer ( ) . signed_max ( ) as u128
249+ }
250+ NonZst :: Unknown => return Err ( error ( cx, LayoutError :: Unknown ( ty) ) ) ,
251+ _ => { }
252+ }
253+ }
254+ metadata
255+ }
256+ ty:: Str => {
257+ let mut metadata = scalar_unit ( Int ( dl. ptr_sized_integer ( ) , false ) ) ;
258+ if !ty. is_unsafe_ptr ( ) {
259+ metadata. valid_range_mut ( ) . end =
260+ dl. ptr_sized_integer ( ) . signed_max ( ) as u128 ;
261+ }
262+ metadata
263+ }
217264 ty:: Dynamic ( ..) => {
218265 let mut vtable = scalar_unit ( Pointer ( AddressSpace :: DATA ) ) ;
219266 vtable. valid_range_mut ( ) . start = 1 ;
@@ -606,7 +653,148 @@ fn layout_of_uncached<'tcx>(
606653 ty:: Placeholder ( ..) | ty:: Param ( _) => {
607654 return Err ( error ( cx, LayoutError :: Unknown ( ty) ) ) ;
608655 }
609- } )
656+ } ;
657+
658+ #[ cfg( debug_assertions) ]
659+ if layout. is_sized ( ) && !layout. abi . is_uninhabited ( ) {
660+ match ( ty_is_non_zst ( ty, param_env, tcx) , layout. is_zst ( ) ) {
661+ ( NonZst :: Unknown , _) => {
662+ bug ! ( "ZSTness should not be unknown at this point {:?} {:?}" , ty, layout)
663+ }
664+ ( NonZst :: False | NonZst :: Uninhabited , false ) => {
665+ bug ! ( "{:?} is not a ZST but ty_is_non_zst() thinks it is" , ty)
666+ }
667+ ( NonZst :: True , true ) => bug ! ( "{:?} is a ZST but ty_is_non_zst() thinks it isn't" , ty) ,
668+ _ => { }
669+ }
670+ }
671+
672+ Ok ( layout)
673+ }
674+
675+ fn ty_is_non_zst < ' tcx > ( ty : Ty < ' tcx > , param_env : ParamEnv < ' tcx > , tcx : TyCtxt < ' tcx > ) -> NonZst {
676+ fn fold_fields < ' tcx > (
677+ mut it : impl Iterator < Item = Ty < ' tcx > > ,
678+ param_env : ParamEnv < ' tcx > ,
679+ tcx : TyCtxt < ' tcx > ,
680+ ) -> NonZst {
681+ let ( ControlFlow :: Break ( res) | ControlFlow :: Continue ( res) ) =
682+ it. try_fold ( NonZst :: False , |acc, ty| {
683+ if acc == NonZst :: True {
684+ return ControlFlow :: Break ( acc) ;
685+ }
686+
687+ ControlFlow :: Continue ( cmp:: max ( acc, ty_is_non_zst ( ty, param_env, tcx) ) )
688+ } ) ;
689+
690+ res
691+ }
692+
693+ match ty. kind ( ) {
694+ ty:: Infer ( ty:: IntVar ( _) | ty:: FloatVar ( _) )
695+ | ty:: Uint ( _)
696+ | ty:: Int ( _)
697+ | ty:: Bool
698+ | ty:: Float ( _)
699+ | ty:: FnPtr ( _)
700+ | ty:: RawPtr ( ..)
701+ | ty:: Dynamic ( _, _, DynKind :: DynStar )
702+ | ty:: Char
703+ | ty:: Ref ( ..) => NonZst :: True ,
704+
705+ ty:: Closure ( _, args) => fold_fields ( args. as_closure ( ) . upvar_tys ( ) . iter ( ) , param_env, tcx) ,
706+ ty:: Coroutine ( _, _) => NonZst :: True ,
707+ ty:: CoroutineClosure ( _, args) => {
708+ fold_fields ( args. as_coroutine_closure ( ) . upvar_tys ( ) . iter ( ) , param_env, tcx)
709+ }
710+ ty:: Array ( ty, len) => {
711+ let len = if len. has_projections ( ) {
712+ tcx. normalize_erasing_regions ( param_env, * len)
713+ } else {
714+ * len
715+ } ;
716+
717+ if let Some ( len) = len. try_to_target_usize ( tcx) {
718+ if len == 0 {
719+ return NonZst :: False ;
720+ }
721+ let element_zst = ty_is_non_zst ( * ty, param_env, tcx) ;
722+ if element_zst != NonZst :: Unknown {
723+ return element_zst;
724+ }
725+ }
726+ NonZst :: Unknown
727+ }
728+ ty:: Tuple ( tys) => fold_fields ( tys. iter ( ) , param_env, tcx) ,
729+ ty:: Adt ( def, args) => {
730+ if ty. is_enum ( ) {
731+ if def. variants ( ) . len ( ) == 0 {
732+ return NonZst :: Uninhabited ;
733+ }
734+ // An enum is !ZST if
735+ // * it has a repr and at least one non-uninhabited variant
736+ // * it has at least one variant with a !ZST payload
737+ // * it has multiple variants that are not uninhabited
738+
739+ let min_empty_variants = if def. repr ( ) . inhibit_enum_layout_opt ( ) { 1 } else { 2 } ;
740+
741+ // first check without recursing
742+ let simple_variants = def. variants ( ) . iter ( ) . filter ( |v| v. fields . len ( ) == 0 ) . count ( ) ;
743+ if simple_variants >= min_empty_variants {
744+ return NonZst :: True ;
745+ }
746+
747+ let mut inhabited_zst_variants = 0 ;
748+ let mut unknown = false ;
749+
750+ for variant in def. variants ( ) . iter ( ) . filter ( |v| v. fields . len ( ) != 0 ) {
751+ let variant_sized =
752+ fold_fields ( variant. fields . iter ( ) . map ( |f| f. ty ( tcx, args) ) , param_env, tcx) ;
753+
754+ match variant_sized {
755+ // enum E { A(!, u32) } counts as !ZST for our purposes
756+ NonZst :: True => return NonZst :: True ,
757+ NonZst :: False => inhabited_zst_variants += 1 ,
758+ NonZst :: Unknown => unknown = true ,
759+ NonZst :: Uninhabited => { }
760+ }
761+ }
762+
763+ if simple_variants + inhabited_zst_variants >= min_empty_variants {
764+ return NonZst :: True ;
765+ }
766+ if unknown {
767+ return NonZst :: Unknown ;
768+ }
769+ if simple_variants + inhabited_zst_variants == 0 {
770+ return NonZst :: Uninhabited ;
771+ }
772+
773+ NonZst :: False
774+ } else {
775+ fold_fields ( def. all_fields ( ) . map ( |f| f. ty ( tcx, args) ) , param_env, tcx)
776+ }
777+ }
778+ ty:: FnDef ( ..) => NonZst :: False ,
779+ ty:: Never => NonZst :: Uninhabited ,
780+ ty:: Param ( ..) => NonZst :: Unknown ,
781+ ty:: Str => NonZst :: True ,
782+ // treat unsized types as potentially-ZST
783+ ty:: Dynamic ( ..) | ty:: Slice ( ..) => NonZst :: False ,
784+ ty:: Alias ( ..) => match tcx. try_normalize_erasing_regions ( param_env, ty) {
785+ Ok ( ty) if !matches ! ( ty. kind( ) , ty:: Alias ( ..) ) => ty_is_non_zst ( ty, param_env, tcx) ,
786+ _ => NonZst :: Unknown ,
787+ } ,
788+ _ => bug ! ( "is_non_zst not implemented for this kind {:?}" , ty) ,
789+ }
790+ }
791+
792+ #[ derive( Clone , Copy , PartialEq , Eq , Debug , PartialOrd , Ord ) ]
793+ pub enum NonZst {
794+ False ,
795+ Uninhabited ,
796+ Unknown ,
797+ True ,
610798}
611799
612800/// Overlap eligibility and variant assignment for each CoroutineSavedLocal.
0 commit comments