Simplify slice::Iter::next enough that it inlines

library/core/src/slice/iter.rs

@@ -11,7 +11,7 @@ use crate::iter::{
 use crate::marker::PhantomData;
 use crate::mem::{self, SizedTypeProperties};
 use crate::num::NonZero;
-use crate::ptr::{NonNull, without_provenance, without_provenance_mut};
+use crate::ptr::{NonNull, null, without_provenance, without_provenance_mut};
 use crate::{cmp, fmt};
 
 #[stable(feature = "boxed_slice_into_iter", since = "1.80.0")]

library/core/src/slice/iter/macros.rs

@@ -154,16 +154,26 @@ macro_rules! iterator {
 
             #[inline]
             fn next(&mut self) -> Option<$elem> {
-                // could be implemented with slices, but this avoids bounds checks
+                // intentionally not using the helpers because this is
+                // one of the most mono'd things in the library.
 
-                // SAFETY: The call to `next_unchecked` is
+                let ptr = self.ptr;
-                // safe since we check if the iterator is empty first.
+                let end_or_len = self.end_or_len;
+                // SAFETY: Type invariants.
                 unsafe {
-                    if is_empty!(self) {
+                    if T::IS_ZST {
-                        None
+                        let byte_end = end_or_len as *const u8;
+                        if byte_end == null() {
+                            return None;
+                        }
+                        self.end_or_len = byte_end.wrapping_sub(1) as _;
                     } else {
-                        Some(self.next_unchecked())
+                        if ptr == crate::intrinsics::transmute::<*const T, NonNull<T>>(end_or_len) {
+                            return None;
+                        }
+                        self.ptr = ptr.add(1);
                     }
+                    crate::intrinsics::transmute::<NonNull<T>, Option<$elem>>(ptr)
                 }
             }
 

tests/codegen/slice-iter-nonnull.rs

@@ -14,11 +14,11 @@
 // CHECK-LABEL: @slice_iter_next(
 #[no_mangle]
 pub fn slice_iter_next<'a>(it: &mut std::slice::Iter<'a, u32>) -> Option<&'a u32> {
-    // CHECK: %[[ENDP:.+]] = getelementptr inbounds{{( nuw)?}} i8, ptr %it, {{i32 4|i64 8}}
+    // CHECK: %[[START:.+]] = load ptr, ptr %it,
-    // CHECK: %[[END:.+]] = load ptr, ptr %[[ENDP]]
     // CHECK-SAME: !nonnull
     // CHECK-SAME: !noundef
-    // CHECK: %[[START:.+]] = load ptr, ptr %it,
+    // CHECK: %[[ENDP:.+]] = getelementptr inbounds{{( nuw)?}} i8, ptr %it, {{i32 4|i64 8}}
+    // CHECK: %[[END:.+]] = load ptr, ptr %[[ENDP]]
     // CHECK-SAME: !nonnull
     // CHECK-SAME: !noundef
     // CHECK: icmp eq ptr %[[START]], %[[END]]

tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir

@@ -4,28 +4,30 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
     debug slice => _1;
     debug f => _2;
     let mut _0: ();
-    let mut _11: std::slice::Iter<'_, T>;
+    let mut _11: std::ptr::NonNull<T>;
-    let mut _12: std::iter::Enumerate<std::slice::Iter<'_, T>>;
+    let mut _12: *const T;
-    let mut _13: std::iter::Enumerate<std::slice::Iter<'_, T>>;
+    let mut _13: usize;
-    let mut _21: std::option::Option<(usize, &T)>;
+    let mut _33: std::option::Option<(usize, &T)>;
-    let mut _24: &impl Fn(usize, &T);
+    let mut _36: &impl Fn(usize, &T);
-    let mut _25: (usize, &T);
+    let mut _37: (usize, &T);
-    let _26: ();
+    let _38: ();
     scope 1 {
-        debug iter => _13;
+        debug (((iter: Enumerate<std::slice::Iter<'_, T>>).0: std::slice::Iter<'_, T>).0: std::ptr::NonNull<T>) => _11;
-        let _22: usize;
+        debug (((iter: Enumerate<std::slice::Iter<'_, T>>).0: std::slice::Iter<'_, T>).1: *const T) => _12;
-        let _23: &T;
+        debug (((iter: Enumerate<std::slice::Iter<'_, T>>).0: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>;
+        debug ((iter: Enumerate<std::slice::Iter<'_, T>>).1: usize) => _13;
+        let _34: usize;
+        let _35: &T;
         scope 2 {
-            debug i => _22;
+            debug i => _34;
-            debug x => _23;
+            debug x => _35;
         }
         scope 19 (inlined <Enumerate<std::slice::Iter<'_, T>> as Iterator>::next) {
-            let mut _14: &mut std::slice::Iter<'_, T>;
+            let mut _27: std::option::Option<&T>;
-            let mut _15: std::option::Option<&T>;
+            let mut _31: (usize, bool);
-            let mut _19: (usize, bool);
+            let mut _32: (usize, &T);
-            let mut _20: (usize, &T);
             scope 20 {
-                let _18: usize;
+                let _30: usize;
                 scope 25 {
                 }
             }
@@ -40,11 +42,58 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
                 }
             }
             scope 26 (inlined <Option<&T> as Try>::branch) {
-                let mut _16: isize;
+                let mut _28: isize;
-                let _17: &T;
+                let _29: &T;
                 scope 27 {
                 }
             }
+            scope 29 (inlined <std::slice::Iter<'_, T> as Iterator>::next) {
+                let _14: std::ptr::NonNull<T>;
+                let _16: std::ptr::NonNull<T>;
+                let mut _19: bool;
+                let mut _22: std::ptr::NonNull<T>;
+                let mut _24: bool;
+                let mut _25: *const u8;
+                let mut _26: *const T;
+                scope 30 {
+                    let _15: *const T;
+                    scope 31 {
+                        let _23: *const u8;
+                        scope 32 {
+                            scope 33 (inlined null::<u8>) {
+                                scope 34 (inlined without_provenance::<()>) {
+                                    scope 35 (inlined without_provenance_mut::<()>) {
+                                    }
+                                }
+                                scope 36 (inlined std::ptr::from_raw_parts::<u8, ()>) {
+                                }
+                            }
+                            scope 37 (inlined std::ptr::const_ptr::<impl *const u8>::wrapping_sub) {
+                                scope 38 (inlined core::num::<impl isize>::wrapping_neg) {
+                                    scope 39 (inlined core::num::<impl isize>::wrapping_sub) {
+                                    }
+                                }
+                                scope 40 (inlined std::ptr::const_ptr::<impl *const u8>::wrapping_offset) {
+                                }
+                            }
+                        }
+                        scope 41 (inlined <NonNull<T> as PartialEq>::eq) {
+                            let mut _17: *mut T;
+                            let mut _18: *mut T;
+                            scope 42 (inlined NonNull::<T>::as_ptr) {
+                            }
+                            scope 43 (inlined NonNull::<T>::as_ptr) {
+                            }
+                        }
+                        scope 44 (inlined NonNull::<T>::add) {
+                            let mut _20: *const T;
+                            let mut _21: *const T;
+                            scope 45 (inlined NonNull::<T>::as_ptr) {
+                            }
+                        }
+                    }
+                }
+            }
         }
     }
     scope 3 (inlined core::slice::<impl [T]>::iter) {
@@ -89,9 +138,7 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
     }
 
     bb0: {
-        StorageLive(_11);
         StorageLive(_3);
-        StorageLive(_6);
         StorageLive(_4);
         _3 = PtrMetadata(copy _1);
         _4 = &raw const (*_1);
@@ -120,86 +167,162 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
     }
 
     bb3: {
-        StorageLive(_10);
         _10 = copy _9;
-        _11 = std::slice::Iter::<'_, T> { ptr: copy _6, end_or_len: move _10, _marker: const ZeroSized: PhantomData<&T> };
-        StorageDead(_10);
         StorageDead(_9);
         StorageDead(_4);
-        StorageDead(_6);
         StorageDead(_3);
-        _12 = Enumerate::<std::slice::Iter<'_, T>> { iter: copy _11, count: const 0_usize };
+        StorageLive(_11);
-        StorageDead(_11);
+        StorageLive(_12);
         StorageLive(_13);
-        _13 = copy _12;
+        _11 = copy _6;
+        _12 = copy _10;
+        _13 = const 0_usize;
         goto -> bb4;
     }
 
     bb4: {
-        StorageLive(_21);
+        StorageLive(_33);
-        StorageLive(_18);
+        StorageLive(_30);
-        StorageLive(_19);
+        StorageLive(_31);
-        StorageLive(_15);
+        StorageLive(_27);
         StorageLive(_14);
-        _14 = &mut (_13.0: std::slice::Iter<'_, T>);
+        StorageLive(_15);
-        _15 = <std::slice::Iter<'_, T> as Iterator>::next(move _14) -> [return: bb5, unwind unreachable];
+        StorageLive(_23);
+        StorageLive(_16);
+        _14 = copy _11;
+        _15 = copy _12;
+        switchInt(const <T as std::mem::SizedTypeProperties>::IS_ZST) -> [0: bb5, otherwise: bb8];
     }
 
     bb5: {
-        StorageDead(_14);
+        StorageLive(_19);
-        StorageLive(_16);
+        _16 = copy _15 as std::ptr::NonNull<T> (Transmute);
-        _16 = discriminant(_15);
+        StorageLive(_17);
-        switchInt(move _16) -> [0: bb6, 1: bb8, otherwise: bb11];
+        _17 = copy _14 as *mut T (Transmute);
+        StorageLive(_18);
+        _18 = copy _16 as *mut T (Transmute);
+        _19 = Eq(move _17, move _18);
+        StorageDead(_18);
+        StorageDead(_17);
+        switchInt(move _19) -> [0: bb6, otherwise: bb7];
     }
 
     bb6: {
-        StorageDead(_16);
-        StorageDead(_15);
         StorageDead(_19);
-        StorageDead(_18);
+        StorageLive(_22);
+        StorageLive(_21);
+        StorageLive(_20);
+        _20 = copy _14 as *const T (Transmute);
+        _21 = Offset(move _20, const 1_usize);
+        StorageDead(_20);
+        _22 = NonNull::<T> { pointer: move _21 };
         StorageDead(_21);
-        StorageDead(_13);
+        _11 = move _22;
-        drop(_2) -> [return: bb7, unwind unreachable];
+        StorageDead(_22);
+        goto -> bb11;
     }
 
     bb7: {
-        return;
+        StorageDead(_19);
+        StorageDead(_16);
+        StorageDead(_23);
+        StorageDead(_15);
+        StorageDead(_14);
+        StorageLive(_28);
+        goto -> bb17;
     }
 
     bb8: {
-        _17 = move ((_15 as Some).0: &T);
+        _23 = copy _15 as *const u8 (PtrToPtr);
-        StorageDead(_16);
+        StorageLive(_24);
-        StorageDead(_15);
+        _24 = Eq(copy _23, const {0x0 as *const u8});
-        _18 = copy (_13.1: usize);
+        switchInt(move _24) -> [0: bb9, otherwise: bb16];
-        _19 = AddWithOverflow(copy (_13.1: usize), const 1_usize);
-        assert(!move (_19.1: bool), "attempt to compute `{} + {}`, which would overflow", copy (_13.1: usize), const 1_usize) -> [success: bb9, unwind unreachable];
     }
 
     bb9: {
-        (_13.1: usize) = move (_19.0: usize);
+        StorageDead(_24);
-        StorageLive(_20);
-        _20 = (copy _18, copy _17);
-        _21 = Option::<(usize, &T)>::Some(move _20);
-        StorageDead(_20);
-        StorageDead(_19);
-        StorageDead(_18);
-        _22 = copy (((_21 as Some).0: (usize, &T)).0: usize);
-        _23 = copy (((_21 as Some).0: (usize, &T)).1: &T);
-        StorageLive(_24);
-        _24 = &_2;
         StorageLive(_25);
-        _25 = (copy _22, copy _23);
+        _25 = arith_offset::<u8>(move _23, const -1_isize) -> [return: bb10, unwind unreachable];
-        _26 = <impl Fn(usize, &T) as Fn<(usize, &T)>>::call(move _24, move _25) -> [return: bb10, unwind unreachable];
     }
 
     bb10: {
+        _26 = move _25 as *const T (PtrToPtr);
         StorageDead(_25);
-        StorageDead(_24);
+        _12 = copy _26;
-        StorageDead(_21);
+        goto -> bb11;
-        goto -> bb4;
     }
 
     bb11: {
+        _27 = copy _14 as std::option::Option<&T> (Transmute);
+        StorageDead(_16);
+        StorageDead(_23);
+        StorageDead(_15);
+        StorageDead(_14);
+        StorageLive(_28);
+        _28 = discriminant(_27);
+        switchInt(move _28) -> [0: bb17, 1: bb12, otherwise: bb15];
+    }
+
+    bb12: {
+        _29 = move ((_27 as Some).0: &T);
+        StorageDead(_28);
+        StorageDead(_27);
+        _30 = copy _13;
+        _31 = AddWithOverflow(copy _13, const 1_usize);
+        assert(!move (_31.1: bool), "attempt to compute `{} + {}`, which would overflow", copy _13, const 1_usize) -> [success: bb13, unwind unreachable];
+    }
+
+    bb13: {
+        _13 = move (_31.0: usize);
+        StorageLive(_32);
+        _32 = (copy _30, copy _29);
+        _33 = Option::<(usize, &T)>::Some(move _32);
+        StorageDead(_32);
+        StorageDead(_31);
+        StorageDead(_30);
+        _34 = copy (((_33 as Some).0: (usize, &T)).0: usize);
+        _35 = copy (((_33 as Some).0: (usize, &T)).1: &T);
+        StorageLive(_36);
+        _36 = &_2;
+        StorageLive(_37);
+        _37 = (copy _34, copy _35);
+        _38 = <impl Fn(usize, &T) as Fn<(usize, &T)>>::call(move _36, move _37) -> [return: bb14, unwind unreachable];
+    }
+
+    bb14: {
+        StorageDead(_37);
+        StorageDead(_36);
+        StorageDead(_33);
+        goto -> bb4;
+    }
+
+    bb15: {
         unreachable;
     }
+
+    bb16: {
+        StorageDead(_24);
+        StorageDead(_16);
+        StorageDead(_23);
+        StorageDead(_15);
+        StorageDead(_14);
+        StorageLive(_28);
+        goto -> bb17;
+    }
+
+    bb17: {
+        StorageDead(_28);
+        StorageDead(_27);
+        StorageDead(_31);
+        StorageDead(_30);
+        StorageDead(_33);
+        StorageDead(_11);
+        StorageDead(_12);
+        StorageDead(_13);
+        drop(_2) -> [return: bb18, unwind unreachable];
+    }
+
+    bb18: {
+        return;
+    }
 }

tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir

@@ -4,19 +4,67 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
     debug slice => _1;
     debug f => _2;
     let mut _0: ();
-    let mut _11: std::slice::Iter<'_, T>;
+    let mut _11: std::ptr::NonNull<T>;
-    let mut _12: std::slice::Iter<'_, T>;
+    let mut _12: *const T;
-    let mut _13: &mut std::slice::Iter<'_, T>;
+    let mut _26: std::option::Option<&T>;
-    let mut _14: std::option::Option<&T>;
+    let mut _27: isize;
-    let mut _15: isize;
+    let mut _29: &impl Fn(&T);
-    let mut _17: &impl Fn(&T);
+    let mut _30: (&T,);
-    let mut _18: (&T,);
+    let _31: ();
-    let _19: ();
     scope 1 {
-        debug iter => _12;
+        debug ((iter: std::slice::Iter<'_, T>).0: std::ptr::NonNull<T>) => _11;
-        let _16: &T;
+        debug ((iter: std::slice::Iter<'_, T>).1: *const T) => _12;
+        debug ((iter: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>;
+        let _28: &T;
         scope 2 {
-            debug x => _16;
+            debug x => _28;
+        }
+        scope 17 (inlined <std::slice::Iter<'_, T> as Iterator>::next) {
+            let _13: std::ptr::NonNull<T>;
+            let _15: std::ptr::NonNull<T>;
+            let mut _18: bool;
+            let mut _21: std::ptr::NonNull<T>;
+            let mut _23: bool;
+            let mut _24: *const u8;
+            let mut _25: *const T;
+            scope 18 {
+                let _14: *const T;
+                scope 19 {
+                    let _22: *const u8;
+                    scope 20 {
+                        scope 21 (inlined null::<u8>) {
+                            scope 22 (inlined without_provenance::<()>) {
+                                scope 23 (inlined without_provenance_mut::<()>) {
+                                }
+                            }
+                            scope 24 (inlined std::ptr::from_raw_parts::<u8, ()>) {
+                            }
+                        }
+                        scope 25 (inlined std::ptr::const_ptr::<impl *const u8>::wrapping_sub) {
+                            scope 26 (inlined core::num::<impl isize>::wrapping_neg) {
+                                scope 27 (inlined core::num::<impl isize>::wrapping_sub) {
+                                }
+                            }
+                            scope 28 (inlined std::ptr::const_ptr::<impl *const u8>::wrapping_offset) {
+                            }
+                        }
+                    }
+                    scope 29 (inlined <NonNull<T> as PartialEq>::eq) {
+                        let mut _16: *mut T;
+                        let mut _17: *mut T;
+                        scope 30 (inlined NonNull::<T>::as_ptr) {
+                        }
+                        scope 31 (inlined NonNull::<T>::as_ptr) {
+                        }
+                    }
+                    scope 32 (inlined NonNull::<T>::add) {
+                        let mut _19: *const T;
+                        let mut _20: *const T;
+                        scope 33 (inlined NonNull::<T>::as_ptr) {
+                        }
+                    }
+                }
+            }
         }
     }
     scope 3 (inlined core::slice::<impl [T]>::iter) {
@@ -58,7 +106,6 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
 
     bb0: {
         StorageLive(_3);
-        StorageLive(_6);
         StorageLive(_4);
         _3 = PtrMetadata(copy _1);
         _4 = &raw const (*_1);
@@ -87,57 +134,132 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
     }
 
     bb3: {
-        StorageLive(_10);
         _10 = copy _9;
-        _11 = std::slice::Iter::<'_, T> { ptr: copy _6, end_or_len: move _10, _marker: const ZeroSized: PhantomData<&T> };
-        StorageDead(_10);
         StorageDead(_9);
         StorageDead(_4);
-        StorageDead(_6);
         StorageDead(_3);
+        StorageLive(_11);
         StorageLive(_12);
-        _12 = copy _11;
+        _11 = copy _6;
+        _12 = copy _10;
         goto -> bb4;
     }
 
     bb4: {
+        StorageLive(_26);
+        StorageLive(_13);
         StorageLive(_14);
-        _13 = &mut _12;
+        StorageLive(_22);
-        _14 = <std::slice::Iter<'_, T> as Iterator>::next(move _13) -> [return: bb5, unwind unreachable];
+        StorageLive(_15);
+        _13 = copy _11;
+        _14 = copy _12;
+        switchInt(const <T as std::mem::SizedTypeProperties>::IS_ZST) -> [0: bb5, otherwise: bb8];
     }
 
     bb5: {
-        _15 = discriminant(_14);
+        StorageLive(_18);
-        switchInt(move _15) -> [0: bb6, 1: bb8, otherwise: bb10];
+        _15 = copy _14 as std::ptr::NonNull<T> (Transmute);
+        StorageLive(_16);
+        _16 = copy _13 as *mut T (Transmute);
+        StorageLive(_17);
+        _17 = copy _15 as *mut T (Transmute);
+        _18 = Eq(move _16, move _17);
+        StorageDead(_17);
+        StorageDead(_16);
+        switchInt(move _18) -> [0: bb6, otherwise: bb7];
     }
 
     bb6: {
-        StorageDead(_14);
+        StorageDead(_18);
-        StorageDead(_12);
+        StorageLive(_21);
-        drop(_2) -> [return: bb7, unwind unreachable];
+        StorageLive(_20);
+        StorageLive(_19);
+        _19 = copy _13 as *const T (Transmute);
+        _20 = Offset(move _19, const 1_usize);
+        StorageDead(_19);
+        _21 = NonNull::<T> { pointer: move _20 };
+        StorageDead(_20);
+        _11 = move _21;
+        StorageDead(_21);
+        goto -> bb11;
     }
 
     bb7: {
-        return;
+        StorageDead(_18);
+        StorageDead(_15);
+        StorageDead(_22);
+        StorageDead(_14);
+        StorageDead(_13);
+        goto -> bb16;
     }
 
     bb8: {
-        _16 = copy ((_14 as Some).0: &T);
+        _22 = copy _14 as *const u8 (PtrToPtr);
-        StorageLive(_17);
+        StorageLive(_23);
-        _17 = &_2;
+        _23 = Eq(copy _22, const {0x0 as *const u8});
-        StorageLive(_18);
+        switchInt(move _23) -> [0: bb9, otherwise: bb15];
-        _18 = (copy _16,);
-        _19 = <impl Fn(&T) as Fn<(&T,)>>::call(move _17, move _18) -> [return: bb9, unwind unreachable];
     }
 
     bb9: {
-        StorageDead(_18);
+        StorageDead(_23);
-        StorageDead(_17);
+        StorageLive(_24);
-        StorageDead(_14);
+        _24 = arith_offset::<u8>(move _22, const -1_isize) -> [return: bb10, unwind unreachable];
-        goto -> bb4;
     }
 
     bb10: {
+        _25 = move _24 as *const T (PtrToPtr);
+        StorageDead(_24);
+        _12 = copy _25;
+        goto -> bb11;
+    }
+
+    bb11: {
+        _26 = copy _13 as std::option::Option<&T> (Transmute);
+        StorageDead(_15);
+        StorageDead(_22);
+        StorageDead(_14);
+        StorageDead(_13);
+        _27 = discriminant(_26);
+        switchInt(move _27) -> [0: bb16, 1: bb12, otherwise: bb14];
+    }
+
+    bb12: {
+        _28 = copy ((_26 as Some).0: &T);
+        StorageLive(_29);
+        _29 = &_2;
+        StorageLive(_30);
+        _30 = (copy _28,);
+        _31 = <impl Fn(&T) as Fn<(&T,)>>::call(move _29, move _30) -> [return: bb13, unwind unreachable];
+    }
+
+    bb13: {
+        StorageDead(_30);
+        StorageDead(_29);
+        StorageDead(_26);
+        goto -> bb4;
+    }
+
+    bb14: {
         unreachable;
     }
+
+    bb15: {
+        StorageDead(_23);
+        StorageDead(_15);
+        StorageDead(_22);
+        StorageDead(_14);
+        StorageDead(_13);
+        goto -> bb16;
+    }
+
+    bb16: {
+        StorageDead(_26);
+        StorageDead(_11);
+        StorageDead(_12);
+        drop(_2) -> [return: bb17, unwind unreachable];
+    }
+
+    bb17: {
+        return;
+    }
 }

tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir

@@ -4,19 +4,67 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
     debug slice => _1;
     debug f => _2;
     let mut _0: ();
-    let mut _11: std::slice::Iter<'_, T>;
+    let mut _11: std::ptr::NonNull<T>;
-    let mut _12: std::slice::Iter<'_, T>;
+    let mut _12: *const T;
-    let mut _13: &mut std::slice::Iter<'_, T>;
+    let mut _26: std::option::Option<&T>;
-    let mut _14: std::option::Option<&T>;
+    let mut _27: isize;
-    let mut _15: isize;
+    let mut _29: &impl Fn(&T);
-    let mut _17: &impl Fn(&T);
+    let mut _30: (&T,);
-    let mut _18: (&T,);
+    let _31: ();
-    let _19: ();
     scope 1 {
-        debug iter => _12;
+        debug ((iter: std::slice::Iter<'_, T>).0: std::ptr::NonNull<T>) => _11;
-        let _16: &T;
+        debug ((iter: std::slice::Iter<'_, T>).1: *const T) => _12;
+        debug ((iter: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>;
+        let _28: &T;
         scope 2 {
-            debug x => _16;
+            debug x => _28;
+        }
+        scope 17 (inlined <std::slice::Iter<'_, T> as Iterator>::next) {
+            let _13: std::ptr::NonNull<T>;
+            let _15: std::ptr::NonNull<T>;
+            let mut _18: bool;
+            let mut _21: std::ptr::NonNull<T>;
+            let mut _23: bool;
+            let mut _24: *const u8;
+            let mut _25: *const T;
+            scope 18 {
+                let _14: *const T;
+                scope 19 {
+                    let _22: *const u8;
+                    scope 20 {
+                        scope 21 (inlined null::<u8>) {
+                            scope 22 (inlined without_provenance::<()>) {
+                                scope 23 (inlined without_provenance_mut::<()>) {
+                                }
+                            }
+                            scope 24 (inlined std::ptr::from_raw_parts::<u8, ()>) {
+                            }
+                        }
+                        scope 25 (inlined std::ptr::const_ptr::<impl *const u8>::wrapping_sub) {
+                            scope 26 (inlined core::num::<impl isize>::wrapping_neg) {
+                                scope 27 (inlined core::num::<impl isize>::wrapping_sub) {
+                                }
+                            }
+                            scope 28 (inlined std::ptr::const_ptr::<impl *const u8>::wrapping_offset) {
+                            }
+                        }
+                    }
+                    scope 29 (inlined <NonNull<T> as PartialEq>::eq) {
+                        let mut _16: *mut T;
+                        let mut _17: *mut T;
+                        scope 30 (inlined NonNull::<T>::as_ptr) {
+                        }
+                        scope 31 (inlined NonNull::<T>::as_ptr) {
+                        }
+                    }
+                    scope 32 (inlined NonNull::<T>::add) {
+                        let mut _19: *const T;
+                        let mut _20: *const T;
+                        scope 33 (inlined NonNull::<T>::as_ptr) {
+                        }
+                    }
+                }
+            }
         }
     }
     scope 3 (inlined core::slice::<impl [T]>::iter) {
@@ -58,7 +106,6 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
 
     bb0: {
         StorageLive(_3);
-        StorageLive(_6);
         StorageLive(_4);
         _3 = PtrMetadata(copy _1);
         _4 = &raw const (*_1);
@@ -87,65 +134,140 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
     }
 
     bb3: {
-        StorageLive(_10);
         _10 = copy _9;
-        _11 = std::slice::Iter::<'_, T> { ptr: copy _6, end_or_len: move _10, _marker: const ZeroSized: PhantomData<&T> };
-        StorageDead(_10);
         StorageDead(_9);
         StorageDead(_4);
-        StorageDead(_6);
         StorageDead(_3);
+        StorageLive(_11);
         StorageLive(_12);
-        _12 = copy _11;
+        _11 = copy _6;
+        _12 = copy _10;
         goto -> bb4;
     }
 
     bb4: {
+        StorageLive(_26);
+        StorageLive(_13);
         StorageLive(_14);
-        _13 = &mut _12;
+        StorageLive(_22);
-        _14 = <std::slice::Iter<'_, T> as Iterator>::next(move _13) -> [return: bb5, unwind: bb11];
+        StorageLive(_15);
+        _13 = copy _11;
+        _14 = copy _12;
+        switchInt(const <T as std::mem::SizedTypeProperties>::IS_ZST) -> [0: bb5, otherwise: bb8];
     }
 
     bb5: {
-        _15 = discriminant(_14);
+        StorageLive(_18);
-        switchInt(move _15) -> [0: bb6, 1: bb8, otherwise: bb10];
+        _15 = copy _14 as std::ptr::NonNull<T> (Transmute);
+        StorageLive(_16);
+        _16 = copy _13 as *mut T (Transmute);
+        StorageLive(_17);
+        _17 = copy _15 as *mut T (Transmute);
+        _18 = Eq(move _16, move _17);
+        StorageDead(_17);
+        StorageDead(_16);
+        switchInt(move _18) -> [0: bb6, otherwise: bb7];
     }
 
     bb6: {
-        StorageDead(_14);
+        StorageDead(_18);
-        StorageDead(_12);
+        StorageLive(_21);
-        drop(_2) -> [return: bb7, unwind continue];
+        StorageLive(_20);
+        StorageLive(_19);
+        _19 = copy _13 as *const T (Transmute);
+        _20 = Offset(move _19, const 1_usize);
+        StorageDead(_19);
+        _21 = NonNull::<T> { pointer: move _20 };
+        StorageDead(_20);
+        _11 = move _21;
+        StorageDead(_21);
+        goto -> bb11;
     }
 
     bb7: {
-        return;
+        StorageDead(_18);
+        StorageDead(_15);
+        StorageDead(_22);
+        StorageDead(_14);
+        StorageDead(_13);
+        goto -> bb18;
     }
 
     bb8: {
-        _16 = copy ((_14 as Some).0: &T);
+        _22 = copy _14 as *const u8 (PtrToPtr);
-        StorageLive(_17);
+        StorageLive(_23);
-        _17 = &_2;
+        _23 = Eq(copy _22, const {0x0 as *const u8});
-        StorageLive(_18);
+        switchInt(move _23) -> [0: bb9, otherwise: bb17];
-        _18 = (copy _16,);
-        _19 = <impl Fn(&T) as Fn<(&T,)>>::call(move _17, move _18) -> [return: bb9, unwind: bb11];
     }
 
     bb9: {
-        StorageDead(_18);
+        StorageDead(_23);
-        StorageDead(_17);
+        StorageLive(_24);
-        StorageDead(_14);
+        _24 = arith_offset::<u8>(move _22, const -1_isize) -> [return: bb10, unwind unreachable];
-        goto -> bb4;
     }
 
     bb10: {
+        _25 = move _24 as *const T (PtrToPtr);
+        StorageDead(_24);
+        _12 = copy _25;
+        goto -> bb11;
+    }
+
+    bb11: {
+        _26 = copy _13 as std::option::Option<&T> (Transmute);
+        StorageDead(_15);
+        StorageDead(_22);
+        StorageDead(_14);
+        StorageDead(_13);
+        _27 = discriminant(_26);
+        switchInt(move _27) -> [0: bb18, 1: bb12, otherwise: bb16];
+    }
+
+    bb12: {
+        _28 = copy ((_26 as Some).0: &T);
+        StorageLive(_29);
+        _29 = &_2;
+        StorageLive(_30);
+        _30 = (copy _28,);
+        _31 = <impl Fn(&T) as Fn<(&T,)>>::call(move _29, move _30) -> [return: bb13, unwind: bb14];
+    }
+
+    bb13: {
+        StorageDead(_30);
+        StorageDead(_29);
+        StorageDead(_26);
+        goto -> bb4;
+    }
+
+    bb14 (cleanup): {
+        drop(_2) -> [return: bb15, unwind terminate(cleanup)];
+    }
+
+    bb15 (cleanup): {
+        resume;
+    }
+
+    bb16: {
         unreachable;
     }
 
-    bb11 (cleanup): {
+    bb17: {
-        drop(_2) -> [return: bb12, unwind terminate(cleanup)];
+        StorageDead(_23);
+        StorageDead(_15);
+        StorageDead(_22);
+        StorageDead(_14);
+        StorageDead(_13);
+        goto -> bb18;
     }
 
-    bb12 (cleanup): {
+    bb18: {
-        resume;
+        StorageDead(_26);
+        StorageDead(_11);
+        StorageDead(_12);
+        drop(_2) -> [return: bb19, unwind continue];
+    }
+
+    bb19: {
+        return;
     }
 }

tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-abort.mir

@@ -3,12 +3,134 @@
 fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> {
     debug it => _1;
     let mut _0: std::option::Option<&T>;
+    scope 1 (inlined <std::slice::Iter<'_, T> as Iterator>::next) {
+        let _2: std::ptr::NonNull<T>;
+        let _4: std::ptr::NonNull<T>;
+        let mut _7: bool;
+        let mut _10: std::ptr::NonNull<T>;
+        let mut _12: bool;
+        let mut _13: *const u8;
+        let mut _14: *const T;
+        scope 2 {
+            let _3: *const T;
+            scope 3 {
+                let _11: *const u8;
+                scope 4 {
+                    scope 5 (inlined null::<u8>) {
+                        scope 6 (inlined without_provenance::<()>) {
+                            scope 7 (inlined without_provenance_mut::<()>) {
+                            }
+                        }
+                        scope 8 (inlined std::ptr::from_raw_parts::<u8, ()>) {
+                        }
+                    }
+                    scope 9 (inlined std::ptr::const_ptr::<impl *const u8>::wrapping_sub) {
+                        scope 10 (inlined core::num::<impl isize>::wrapping_neg) {
+                            scope 11 (inlined core::num::<impl isize>::wrapping_sub) {
+                            }
+                        }
+                        scope 12 (inlined std::ptr::const_ptr::<impl *const u8>::wrapping_offset) {
+                        }
+                    }
+                }
+                scope 13 (inlined <NonNull<T> as PartialEq>::eq) {
+                    let mut _5: *mut T;
+                    let mut _6: *mut T;
+                    scope 14 (inlined NonNull::<T>::as_ptr) {
+                    }
+                    scope 15 (inlined NonNull::<T>::as_ptr) {
+                    }
+                }
+                scope 16 (inlined NonNull::<T>::add) {
+                    let mut _8: *const T;
+                    let mut _9: *const T;
+                    scope 17 (inlined NonNull::<T>::as_ptr) {
+                    }
+                }
+            }
+        }
+    }
 
     bb0: {
-        _0 = <std::slice::Iter<'_, T> as Iterator>::next(move _1) -> [return: bb1, unwind unreachable];
+        StorageLive(_2);
+        StorageLive(_3);
+        StorageLive(_11);
+        StorageLive(_4);
+        _2 = copy ((*_1).0: std::ptr::NonNull<T>);
+        _3 = copy ((*_1).1: *const T);
+        switchInt(const <T as std::mem::SizedTypeProperties>::IS_ZST) -> [0: bb1, otherwise: bb4];
     }
 
     bb1: {
+        StorageLive(_7);
+        _4 = copy _3 as std::ptr::NonNull<T> (Transmute);
+        StorageLive(_5);
+        _5 = copy _2 as *mut T (Transmute);
+        StorageLive(_6);
+        _6 = copy _4 as *mut T (Transmute);
+        _7 = Eq(move _5, move _6);
+        StorageDead(_6);
+        StorageDead(_5);
+        switchInt(move _7) -> [0: bb2, otherwise: bb3];
+    }
+
+    bb2: {
+        StorageDead(_7);
+        StorageLive(_10);
+        StorageLive(_9);
+        StorageLive(_8);
+        _8 = copy _2 as *const T (Transmute);
+        _9 = Offset(move _8, const 1_usize);
+        StorageDead(_8);
+        _10 = NonNull::<T> { pointer: move _9 };
+        StorageDead(_9);
+        ((*_1).0: std::ptr::NonNull<T>) = move _10;
+        StorageDead(_10);
+        goto -> bb7;
+    }
+
+    bb3: {
+        _0 = const {transmute(0x0000000000000000): Option<&T>};
+        StorageDead(_7);
+        goto -> bb9;
+    }
+
+    bb4: {
+        _11 = copy _3 as *const u8 (PtrToPtr);
+        StorageLive(_12);
+        _12 = Eq(copy _11, const {0x0 as *const u8});
+        switchInt(move _12) -> [0: bb5, otherwise: bb8];
+    }
+
+    bb5: {
+        StorageDead(_12);
+        StorageLive(_13);
+        _13 = arith_offset::<u8>(move _11, const -1_isize) -> [return: bb6, unwind unreachable];
+    }
+
+    bb6: {
+        _14 = move _13 as *const T (PtrToPtr);
+        StorageDead(_13);
+        ((*_1).1: *const T) = copy _14;
+        goto -> bb7;
+    }
+
+    bb7: {
+        _0 = copy _2 as std::option::Option<&T> (Transmute);
+        goto -> bb9;
+    }
+
+    bb8: {
+        _0 = const {transmute(0x0000000000000000): Option<&T>};
+        StorageDead(_12);
+        goto -> bb9;
+    }
+
+    bb9: {
+        StorageDead(_4);
+        StorageDead(_11);
+        StorageDead(_3);
+        StorageDead(_2);
         return;
     }
 }

tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-unwind.mir

@@ -3,12 +3,134 @@
 fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> {
     debug it => _1;
     let mut _0: std::option::Option<&T>;
+    scope 1 (inlined <std::slice::Iter<'_, T> as Iterator>::next) {
+        let _2: std::ptr::NonNull<T>;
+        let _4: std::ptr::NonNull<T>;
+        let mut _7: bool;
+        let mut _10: std::ptr::NonNull<T>;
+        let mut _12: bool;
+        let mut _13: *const u8;
+        let mut _14: *const T;
+        scope 2 {
+            let _3: *const T;
+            scope 3 {
+                let _11: *const u8;
+                scope 4 {
+                    scope 5 (inlined null::<u8>) {
+                        scope 6 (inlined without_provenance::<()>) {
+                            scope 7 (inlined without_provenance_mut::<()>) {
+                            }
+                        }
+                        scope 8 (inlined std::ptr::from_raw_parts::<u8, ()>) {
+                        }
+                    }
+                    scope 9 (inlined std::ptr::const_ptr::<impl *const u8>::wrapping_sub) {
+                        scope 10 (inlined core::num::<impl isize>::wrapping_neg) {
+                            scope 11 (inlined core::num::<impl isize>::wrapping_sub) {
+                            }
+                        }
+                        scope 12 (inlined std::ptr::const_ptr::<impl *const u8>::wrapping_offset) {
+                        }
+                    }
+                }
+                scope 13 (inlined <NonNull<T> as PartialEq>::eq) {
+                    let mut _5: *mut T;
+                    let mut _6: *mut T;
+                    scope 14 (inlined NonNull::<T>::as_ptr) {
+                    }
+                    scope 15 (inlined NonNull::<T>::as_ptr) {
+                    }
+                }
+                scope 16 (inlined NonNull::<T>::add) {
+                    let mut _8: *const T;
+                    let mut _9: *const T;
+                    scope 17 (inlined NonNull::<T>::as_ptr) {
+                    }
+                }
+            }
+        }
+    }
 
     bb0: {
-        _0 = <std::slice::Iter<'_, T> as Iterator>::next(move _1) -> [return: bb1, unwind continue];
+        StorageLive(_2);
+        StorageLive(_3);
+        StorageLive(_11);
+        StorageLive(_4);
+        _2 = copy ((*_1).0: std::ptr::NonNull<T>);
+        _3 = copy ((*_1).1: *const T);
+        switchInt(const <T as std::mem::SizedTypeProperties>::IS_ZST) -> [0: bb1, otherwise: bb4];
     }
 
     bb1: {
+        StorageLive(_7);
+        _4 = copy _3 as std::ptr::NonNull<T> (Transmute);
+        StorageLive(_5);
+        _5 = copy _2 as *mut T (Transmute);
+        StorageLive(_6);
+        _6 = copy _4 as *mut T (Transmute);
+        _7 = Eq(move _5, move _6);
+        StorageDead(_6);
+        StorageDead(_5);
+        switchInt(move _7) -> [0: bb2, otherwise: bb3];
+    }
+
+    bb2: {
+        StorageDead(_7);
+        StorageLive(_10);
+        StorageLive(_9);
+        StorageLive(_8);
+        _8 = copy _2 as *const T (Transmute);
+        _9 = Offset(move _8, const 1_usize);
+        StorageDead(_8);
+        _10 = NonNull::<T> { pointer: move _9 };
+        StorageDead(_9);
+        ((*_1).0: std::ptr::NonNull<T>) = move _10;
+        StorageDead(_10);
+        goto -> bb7;
+    }
+
+    bb3: {
+        _0 = const {transmute(0x0000000000000000): Option<&T>};
+        StorageDead(_7);
+        goto -> bb9;
+    }
+
+    bb4: {
+        _11 = copy _3 as *const u8 (PtrToPtr);
+        StorageLive(_12);
+        _12 = Eq(copy _11, const {0x0 as *const u8});
+        switchInt(move _12) -> [0: bb5, otherwise: bb8];
+    }
+
+    bb5: {
+        StorageDead(_12);
+        StorageLive(_13);
+        _13 = arith_offset::<u8>(move _11, const -1_isize) -> [return: bb6, unwind unreachable];
+    }
+
+    bb6: {
+        _14 = move _13 as *const T (PtrToPtr);
+        StorageDead(_13);
+        ((*_1).1: *const T) = copy _14;
+        goto -> bb7;
+    }
+
+    bb7: {
+        _0 = copy _2 as std::option::Option<&T> (Transmute);
+        goto -> bb9;
+    }
+
+    bb8: {
+        _0 = const {transmute(0x0000000000000000): Option<&T>};
+        StorageDead(_12);
+        goto -> bb9;
+    }
+
+    bb9: {
+        StorageDead(_4);
+        StorageDead(_11);
+        StorageDead(_3);
+        StorageDead(_2);
         return;
     }
 }