improve TagEncoding::Niche docs and sanity check

compiler/rustc_abi/src/lib.rs

@@ -1215,6 +1215,15 @@ impl Scalar {
             Scalar::Union { .. } => true,
         }
     }
+
+    /// Returns `true` if this is a signed integer scalar
+    #[inline]
+    pub fn is_signed(&self) -> bool {
+        match self.primitive() {
+            Primitive::Int(_, signed) => signed,
+            _ => false,
+        }
+    }
 }
 
 // NOTE: This struct is generic over the FieldIdx for rust-analyzer usage.
@@ -1401,10 +1410,7 @@ impl BackendRepr {
     #[inline]
     pub fn is_signed(&self) -> bool {
         match self {
-            BackendRepr::Scalar(scal) => match scal.primitive() {
-                Primitive::Int(_, signed) => signed,
-                _ => false,
-            },
+            BackendRepr::Scalar(scal) => scal.is_signed(),
             _ => panic!("`is_signed` on non-scalar ABI {self:?}"),
         }
     }
@@ -1528,14 +1534,22 @@ pub enum TagEncoding<VariantIdx: Idx> {
     /// The variant `untagged_variant` contains a niche at an arbitrary
     /// offset (field `tag_field` of the enum), which for a variant with
     /// discriminant `d` is set to
-    /// `(d - niche_variants.start).wrapping_add(niche_start)`.
+    /// `(d - niche_variants.start).wrapping_add(niche_start)`
+    /// (this is wrapping arithmetic using the type of the niche field).
     ///
     /// For example, `Option<(usize, &T)>`  is represented such that
     /// `None` has a null pointer for the second tuple field, and
     /// `Some` is the identity function (with a non-null reference).
+    ///
+    /// Other variants that are not `untagged_variant` and that are outside the `niche_variants`
+    /// range cannot be represented; they must be uninhabited.
     Niche {
         untagged_variant: VariantIdx,
+        /// This range *may* contain `untagged_variant`; that is then just a "dead value" and
+        /// not used to encode anything.
         niche_variants: RangeInclusive<VariantIdx>,
+        /// This is inbounds of the type of the niche field
+        /// (not sign-extended, i.e., all bits beyond the niche field size are 0).
         niche_start: u128,
     },
 }

compiler/rustc_ty_utils/src/layout/invariant.rs

@@ -1,11 +1,11 @@
 use std::assert_matches::assert_matches;
 
-use rustc_abi::{BackendRepr, FieldsShape, Scalar, Size, Variants};
+use rustc_abi::{BackendRepr, FieldsShape, Scalar, Size, TagEncoding, Variants};
 use rustc_middle::bug;
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutCx, TyAndLayout};
 
 /// Enforce some basic invariants on layouts.
-pub(super) fn partially_check_layout<'tcx>(cx: &LayoutCx<'tcx>, layout: &TyAndLayout<'tcx>) {
+pub(super) fn layout_sanity_check<'tcx>(cx: &LayoutCx<'tcx>, layout: &TyAndLayout<'tcx>) {
     let tcx = cx.tcx();
 
     // Type-level uninhabitedness should always imply ABI uninhabitedness.
@@ -241,7 +241,17 @@ pub(super) fn partially_check_layout<'tcx>(cx: &LayoutCx<'tcx>, layout: &TyAndLa
 
     check_layout_abi(cx, layout);
 
-    if let Variants::Multiple { variants, .. } = &layout.variants {
+    if let Variants::Multiple { variants, tag, tag_encoding, .. } = &layout.variants {
+        if let TagEncoding::Niche { niche_start, untagged_variant, niche_variants } = tag_encoding {
+            let niche_size = tag.size(cx);
+            assert!(*niche_start <= niche_size.unsigned_int_max());
+            for (idx, variant) in variants.iter_enumerated() {
+                // Ensure all inhabited variants are accounted for.
+                if !variant.is_uninhabited() {
+                    assert!(idx == *untagged_variant || niche_variants.contains(&idx));
+                }
+            }
+        }
         for variant in variants.iter() {
             // No nested "multiple".
             assert_matches!(variant.variants, Variants::Single { .. });

src/tools/miri/tests/fail/enum-untagged-variant-invalid-encoding.rs

@@ -0,0 +1,27 @@
+// Validity makes this fail at the wrong place.
+//@compile-flags: -Zmiri-disable-validation
+use std::mem;
+
+// This enum has untagged variant idx 1, with niche_variants being 0..=2
+// and niche_start being 2.
+// That means the untagged variants is in the niche variant range!
+// However, using the corresponding value (2+1 = 3) is not a valid encoding of this variant.
+#[derive(Copy, Clone, PartialEq)]
+enum Foo {
+    Var1,
+    Var2(bool),
+    Var3,
+}
+
+fn main() {
+    unsafe {
+        assert!(Foo::Var2(false) == mem::transmute(0u8));
+        assert!(Foo::Var2(true) == mem::transmute(1u8));
+        assert!(Foo::Var1 == mem::transmute(2u8));
+        assert!(Foo::Var3 == mem::transmute(4u8));
+
+        let invalid: Foo = mem::transmute(3u8);
+        assert!(matches!(invalid, Foo::Var2(_)));
+        //~^ ERROR: invalid tag
+    }
+}

src/tools/miri/tests/fail/enum-untagged-variant-invalid-encoding.stderr

@@ -0,0 +1,15 @@
+error: Undefined Behavior: enum value has invalid tag: 0x03
+  --> tests/fail/enum-untagged-variant-invalid-encoding.rs:LL:CC
+   |
+LL |         assert!(matches!(invalid, Foo::Var2(_)));
+   |                          ^^^^^^^ enum value has invalid tag: 0x03
+   |
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+   = note: BACKTRACE:
+   = note: inside `main` at tests/fail/enum-untagged-variant-invalid-encoding.rs:LL:CC
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+