Allow GVN to produce places and not just locals.

compiler/rustc_mir_transform/src/gvn.rs

@@ -980,27 +980,15 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             }
         }
 
-        let tcx = self.tcx;
+        if let Some(place) = self.try_as_place(copy_from_local_value, location) {
-        let mut projection = SmallVec::<[PlaceElem<'tcx>; 1]>::new();
+            if rvalue.ty(self.local_decls, self.tcx) == place.ty(self.local_decls, self.tcx).ty {
-        loop {
+                self.reused_locals.insert(place.local);
-            if let Some(local) = self.try_as_local(copy_from_local_value, location) {
-                projection.reverse();
-                let place = Place { local, projection: tcx.mk_place_elems(projection.as_slice()) };
-                if rvalue.ty(self.local_decls, tcx) == place.ty(self.local_decls, tcx).ty {
-                    self.reused_locals.insert(local);
                 *rvalue = Rvalue::Use(Operand::Copy(place));
-                    return Some(copy_from_value);
+                return Some(copy_from_local_value);
-                }
-                return None;
-            } else if let Value::Projection(pointer, proj) = *self.get(copy_from_local_value)
-                && let Some(proj) = self.try_as_place_elem(proj, location)
-            {
-                projection.push(proj);
-                copy_from_local_value = pointer;
-            } else {
-                return None;
             }
         }
+
+        None
     }
 
     fn simplify_aggregate(
@@ -1672,14 +1660,14 @@ fn op_to_prop_const<'tcx>(
 }
 
 impl<'tcx> VnState<'_, 'tcx> {
-    /// If either [`Self::try_as_constant`] as [`Self::try_as_local`] succeeds,
+    /// If either [`Self::try_as_constant`] as [`Self::try_as_place`] succeeds,
     /// returns that result as an [`Operand`].
     fn try_as_operand(&mut self, index: VnIndex, location: Location) -> Option<Operand<'tcx>> {
         if let Some(const_) = self.try_as_constant(index) {
             Some(Operand::Constant(Box::new(const_)))
-        } else if let Some(local) = self.try_as_local(index, location) {
+        } else if let Some(place) = self.try_as_place(index, location) {
-            self.reused_locals.insert(local);
+            self.reused_locals.insert(place.local);
-            Some(Operand::Copy(local.into()))
+            Some(Operand::Copy(place))
         } else {
             None
         }
@@ -1712,6 +1700,29 @@ impl<'tcx> VnState<'_, 'tcx> {
         Some(ConstOperand { span: DUMMY_SP, user_ty: None, const_ })
     }
 
+    /// Construct a place which holds the same value as `index` and for which all locals strictly
+    /// dominate `loc`. If you used this place, add its base local to `reused_locals` to remove
+    /// storage statements.
+    #[instrument(level = "trace", skip(self), ret)]
+    fn try_as_place(&mut self, mut index: VnIndex, loc: Location) -> Option<Place<'tcx>> {
+        let mut projection = SmallVec::<[PlaceElem<'tcx>; 1]>::new();
+        loop {
+            if let Some(local) = self.try_as_local(index, loc) {
+                projection.reverse();
+                let place =
+                    Place { local, projection: self.tcx.mk_place_elems(projection.as_slice()) };
+                return Some(place);
+            } else if let Value::Projection(pointer, proj) = *self.get(index)
+                && let Some(proj) = self.try_as_place_elem(proj, loc)
+            {
+                projection.push(proj);
+                index = pointer;
+            } else {
+                return None;
+            }
+        }
+    }
+
     /// If there is a local which is assigned `index`, and its assignment strictly dominates `loc`,
     /// return it. If you used this local, add it to `reused_locals` to remove storage statements.
     fn try_as_local(&mut self, index: VnIndex, loc: Location) -> Option<Local> {
@@ -1762,11 +1773,12 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, 'tcx> {
             if let Some(value) = value {
                 if let Some(const_) = self.try_as_constant(value) {
                     *rvalue = Rvalue::Use(Operand::Constant(Box::new(const_)));
-                } else if let Some(local) = self.try_as_local(value, location)
+                } else if let Some(place) = self.try_as_place(value, location)
-                    && *rvalue != Rvalue::Use(Operand::Move(local.into()))
+                    && *rvalue != Rvalue::Use(Operand::Move(place))
+                    && *rvalue != Rvalue::Use(Operand::Copy(place))
                 {
-                    *rvalue = Rvalue::Use(Operand::Copy(local.into()));
+                    *rvalue = Rvalue::Use(Operand::Copy(place));
-                    self.reused_locals.insert(local);
+                    self.reused_locals.insert(place.local);
                 }
             }
         }

tests/mir-opt/const_prop/address_of_pair.fn0.GVN.diff

@@ -31,15 +31,13 @@
           (*_3) = const true;
           _4 = const ();
           StorageDead(_4);
--         StorageLive(_5);
+          StorageLive(_5);
-+         nop;
           StorageLive(_6);
           _6 = copy (_2.1: bool);
           _5 = Not(move _6);
           StorageDead(_6);
           _0 = copy _5;
--         StorageDead(_5);
+          StorageDead(_5);
-+         nop;
           StorageDead(_3);
           StorageDead(_2);
           return;

tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-abort.diff

@@ -69,8 +69,7 @@
   
       bb0: {
           StorageLive(_1);
--         StorageLive(_2);
+          StorageLive(_2);
-+         nop;
           StorageLive(_3);
           StorageLive(_11);
           StorageLive(_12);
@@ -119,8 +118,7 @@
           StorageDead(_11);
           _2 = &_3;
           _1 = copy _2;
--         StorageDead(_2);
+          StorageDead(_2);
-+         nop;
           StorageLive(_4);
 -         _9 = deref_copy _3;
 +         _9 = copy _3;
@@ -141,7 +139,7 @@
           StorageLive(_8);
           _8 = copy _5;
 -         _7 = copy _8 as *mut () (PtrToPtr);
-+         _7 = copy _5 as *mut () (PtrToPtr);
++         _7 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *mut () (Transmute);
           StorageDead(_8);
           StorageDead(_7);
 -         StorageDead(_5);

tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-unwind.diff

@@ -35,8 +35,7 @@
   
       bb0: {
           StorageLive(_1);
--         StorageLive(_2);
+          StorageLive(_2);
-+         nop;
           StorageLive(_3);
           _3 = Box::<()>::new(const ()) -> [return: bb1, unwind continue];
       }
@@ -44,8 +43,7 @@
       bb1: {
           _2 = &_3;
           _1 = copy _2;
--         StorageDead(_2);
+          StorageDead(_2);
-+         nop;
           StorageLive(_4);
 -         _9 = deref_copy _3;
 +         _9 = copy _3;
@@ -66,7 +64,7 @@
           StorageLive(_8);
           _8 = copy _5;
 -         _7 = copy _8 as *mut () (PtrToPtr);
-+         _7 = copy _5 as *mut () (PtrToPtr);
++         _7 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *mut () (Transmute);
           StorageDead(_8);
           StorageDead(_7);
 -         StorageDead(_5);

tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-abort.diff

@@ -69,8 +69,7 @@
   
       bb0: {
           StorageLive(_1);
--         StorageLive(_2);
+          StorageLive(_2);
-+         nop;
           StorageLive(_3);
           StorageLive(_11);
           StorageLive(_12);
@@ -119,8 +118,7 @@
           StorageDead(_11);
           _2 = &_3;
           _1 = copy _2;
--         StorageDead(_2);
+          StorageDead(_2);
-+         nop;
           StorageLive(_4);
 -         _9 = deref_copy _3;
 +         _9 = copy _3;
@@ -141,7 +139,7 @@
           StorageLive(_8);
           _8 = copy _5;
 -         _7 = copy _8 as *mut () (PtrToPtr);
-+         _7 = copy _5 as *mut () (PtrToPtr);
++         _7 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *mut () (Transmute);
           StorageDead(_8);
           StorageDead(_7);
 -         StorageDead(_5);

tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-unwind.diff

@@ -35,8 +35,7 @@
   
       bb0: {
           StorageLive(_1);
--         StorageLive(_2);
+          StorageLive(_2);
-+         nop;
           StorageLive(_3);
           _3 = Box::<()>::new(const ()) -> [return: bb1, unwind continue];
       }
@@ -44,8 +43,7 @@
       bb1: {
           _2 = &_3;
           _1 = copy _2;
--         StorageDead(_2);
+          StorageDead(_2);
-+         nop;
           StorageLive(_4);
 -         _9 = deref_copy _3;
 +         _9 = copy _3;
@@ -66,7 +64,7 @@
           StorageLive(_8);
           _8 = copy _5;
 -         _7 = copy _8 as *mut () (PtrToPtr);
-+         _7 = copy _5 as *mut () (PtrToPtr);
++         _7 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *mut () (Transmute);
           StorageDead(_8);
           StorageDead(_7);
 -         StorageDead(_5);

tests/mir-opt/gvn.remove_casts_must_change_both_sides.GVN.panic-abort.diff

@@ -9,7 +9,8 @@
       bb0: {
           _3 = copy (*_1) as *const u8 (PtrToPtr);
           _4 = copy _2 as *const u8 (PtrToPtr);
-          _0 = Eq(copy _3, copy _4);
+-         _0 = Eq(copy _3, copy _4);
++         _0 = Eq(copy (*_1), copy _2);
           return;
       }
   }

tests/mir-opt/gvn.remove_casts_must_change_both_sides.GVN.panic-unwind.diff

@@ -9,7 +9,8 @@
       bb0: {
           _3 = copy (*_1) as *const u8 (PtrToPtr);
           _4 = copy _2 as *const u8 (PtrToPtr);
-          _0 = Eq(copy _3, copy _4);
+-         _0 = Eq(copy _3, copy _4);
++         _0 = Eq(copy (*_1), copy _2);
           return;
       }
   }

tests/mir-opt/gvn.rs

@@ -1048,7 +1048,7 @@ fn remove_casts_must_change_both_sides(mut_a: &*mut u8, mut_b: *mut u8) -> bool
             let a = *mut_a as *const u8;
             // CHECK: [[B:_.+]] = copy _2 as *const u8 (PtrToPtr);
             let b = mut_b as *const u8;
-            // CHECK: _0 = Eq(copy [[A]], copy [[B]]);
+            // CHECK: _0 = Eq(copy (*_1), copy _2);
             RET = a == b;
             Return()
         }

tests/mir-opt/gvn_copy_constant_projection.compare_constant_index.GVN.panic-abort.diff

@@ -0,0 +1,58 @@
+- // MIR for `compare_constant_index` before GVN
++ // MIR for `compare_constant_index` after GVN
+  
+  fn compare_constant_index(_1: [i32; 1], _2: [i32; 1]) -> std::cmp::Ordering {
+      debug x => _1;
+      debug y => _2;
+      let mut _0: std::cmp::Ordering;
+      let _3: &i32;
+      let _4: usize;
+      let mut _5: bool;
+      let _6: &i32;
+      let _7: usize;
+      let mut _8: bool;
+      scope 1 (inlined std::cmp::impls::<impl Ord for i32>::cmp) {
+          let mut _9: i32;
+          let mut _10: i32;
+      }
+  
+      bb0: {
+-         StorageLive(_4);
++         nop;
+          _4 = const 0_usize;
+-         _5 = Lt(copy _4, const 1_usize);
+-         assert(move _5, "index out of bounds: the length is {} but the index is {}", const 1_usize, copy _4) -> [success: bb1, unwind unreachable];
++         _5 = const true;
++         assert(const true, "index out of bounds: the length is {} but the index is {}", const 1_usize, const 0_usize) -> [success: bb1, unwind unreachable];
+      }
+  
+      bb1: {
+-         _3 = &_1[_4];
++         _3 = &_1[0 of 1];
+          StorageLive(_7);
+          _7 = const 0_usize;
+-         _8 = Lt(copy _7, const 1_usize);
+-         assert(move _8, "index out of bounds: the length is {} but the index is {}", const 1_usize, copy _7) -> [success: bb2, unwind unreachable];
++         _8 = const true;
++         assert(const true, "index out of bounds: the length is {} but the index is {}", const 1_usize, const 0_usize) -> [success: bb2, unwind unreachable];
+      }
+  
+      bb2: {
+-         _6 = &_2[_7];
++         _6 = &_2[0 of 1];
+          StorageLive(_9);
+-         _9 = copy (*_3);
++         _9 = copy _1[0 of 1];
+          StorageLive(_10);
+-         _10 = copy (*_6);
++         _10 = copy _2[0 of 1];
+          _0 = Cmp(move _9, move _10);
+          StorageDead(_10);
+          StorageDead(_9);
+          StorageDead(_7);
+-         StorageDead(_4);
++         nop;
+          return;
+      }
+  }
+  

tests/mir-opt/gvn_copy_constant_projection.compare_constant_index.GVN.panic-unwind.diff

@@ -0,0 +1,58 @@
+- // MIR for `compare_constant_index` before GVN
++ // MIR for `compare_constant_index` after GVN
+  
+  fn compare_constant_index(_1: [i32; 1], _2: [i32; 1]) -> std::cmp::Ordering {
+      debug x => _1;
+      debug y => _2;
+      let mut _0: std::cmp::Ordering;
+      let _3: &i32;
+      let _4: usize;
+      let mut _5: bool;
+      let _6: &i32;
+      let _7: usize;
+      let mut _8: bool;
+      scope 1 (inlined std::cmp::impls::<impl Ord for i32>::cmp) {
+          let mut _9: i32;
+          let mut _10: i32;
+      }
+  
+      bb0: {
+-         StorageLive(_4);
++         nop;
+          _4 = const 0_usize;
+-         _5 = Lt(copy _4, const 1_usize);
+-         assert(move _5, "index out of bounds: the length is {} but the index is {}", const 1_usize, copy _4) -> [success: bb1, unwind continue];
++         _5 = const true;
++         assert(const true, "index out of bounds: the length is {} but the index is {}", const 1_usize, const 0_usize) -> [success: bb1, unwind continue];
+      }
+  
+      bb1: {
+-         _3 = &_1[_4];
++         _3 = &_1[0 of 1];
+          StorageLive(_7);
+          _7 = const 0_usize;
+-         _8 = Lt(copy _7, const 1_usize);
+-         assert(move _8, "index out of bounds: the length is {} but the index is {}", const 1_usize, copy _7) -> [success: bb2, unwind continue];
++         _8 = const true;
++         assert(const true, "index out of bounds: the length is {} but the index is {}", const 1_usize, const 0_usize) -> [success: bb2, unwind continue];
+      }
+  
+      bb2: {
+-         _6 = &_2[_7];
++         _6 = &_2[0 of 1];
+          StorageLive(_9);
+-         _9 = copy (*_3);
++         _9 = copy _1[0 of 1];
+          StorageLive(_10);
+-         _10 = copy (*_6);
++         _10 = copy _2[0 of 1];
+          _0 = Cmp(move _9, move _10);
+          StorageDead(_10);
+          StorageDead(_9);
+          StorageDead(_7);
+-         StorageDead(_4);
++         nop;
+          return;
+      }
+  }
+  

tests/mir-opt/gvn_copy_constant_projection.rs

@@ -0,0 +1,18 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+
+use std::cmp::Ordering;
+fn compare_constant_index(x: [i32; 1], y: [i32; 1]) -> Ordering {
+    // CHECK-LABEL: fn compare_constant_index(
+    // CHECK-NOT: (*{{_.*}});
+    // CHECK: [[lhs:_.*]] = copy _1[0 of 1];
+    // CHECK-NOT: (*{{_.*}});
+    // CHECK: [[rhs:_.*]] = copy _2[0 of 1];
+    // CHECK: _0 = Cmp(move [[lhs]], move [[rhs]]);
+    Ord::cmp(&x[0], &y[0])
+}
+
+fn main() {
+    compare_constant_index([1], [2]);
+}
+
+// EMIT_MIR gvn_copy_constant_projection.compare_constant_index.GVN.diff

tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff

@@ -229,7 +229,6 @@
 +         StorageDead(_24);
 +         StorageLive(_45);
 +         StorageLive(_46);
-+         StorageLive(_49);
 +         StorageLive(_51);
 +         StorageLive(_42);
 +         StorageLive(_43);
@@ -243,9 +242,11 @@
 +         _47 = Pin::<&mut std::future::Ready<()>> { __pointer: copy _45 };
 +         StorageDead(_47);
 +         _44 = &mut ((*_45).0: std::option::Option<()>);
++         StorageLive(_49);
 +         _49 = Option::<()>::None;
 +         _43 = copy ((*_45).0: std::option::Option<()>);
 +         ((*_45).0: std::option::Option<()>) = copy _49;
++         StorageDead(_49);
 +         StorageDead(_44);
 +         StorageLive(_50);
 +         _50 = discriminant(_43);
@@ -322,7 +323,6 @@
 +         _18 = Poll::<()>::Ready(move _42);
 +         StorageDead(_42);
 +         StorageDead(_51);
-+         StorageDead(_49);
 +         StorageDead(_46);
 +         StorageDead(_45);
 +         StorageDead(_22);

tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff

@@ -246,7 +246,6 @@
 +         StorageDead(_24);
 +         StorageLive(_47);
 +         StorageLive(_48);
-+         StorageLive(_51);
 +         StorageLive(_53);
 +         StorageLive(_44);
 +         StorageLive(_45);
@@ -260,9 +259,11 @@
 +         _49 = Pin::<&mut std::future::Ready<()>> { __pointer: copy _47 };
 +         StorageDead(_49);
 +         _46 = &mut ((*_47).0: std::option::Option<()>);
++         StorageLive(_51);
 +         _51 = Option::<()>::None;
 +         _45 = copy ((*_47).0: std::option::Option<()>);
 +         ((*_47).0: std::option::Option<()>) = copy _51;
++         StorageDead(_51);
 +         StorageDead(_46);
 +         StorageLive(_52);
 +         _52 = discriminant(_45);
@@ -363,7 +364,6 @@
 +         _18 = Poll::<()>::Ready(move _44);
 +         StorageDead(_44);
 +         StorageDead(_53);
-+         StorageDead(_51);
 +         StorageDead(_48);
 +         StorageDead(_47);
 +         StorageDead(_22);

tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir

@@ -40,6 +40,7 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool {
         StorageLive(_12);
         StorageLive(_11);
         StorageLive(_5);
+        StorageLive(_6);
         StorageLive(_7);
         StorageLive(_3);
         _3 = copy ((*_1).0: char);
@@ -65,6 +66,7 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool {
         _11 = Option::<std::cmp::Ordering>::Some(move _10);
         StorageDead(_10);
         StorageDead(_7);
+        StorageDead(_6);
         StorageDead(_5);
         goto -> bb3;
     }
@@ -72,6 +74,7 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool {
     bb2: {
         _11 = copy _6;
         StorageDead(_7);
+        StorageDead(_6);
         StorageDead(_5);
         goto -> bb3;
     }

tests/mir-opt/pre-codegen/tuple_ord.demo_ge_partial.PreCodegen.after.mir

@@ -44,8 +44,10 @@ fn demo_ge_partial(_1: &(f32, f32), _2: &(f32, f32)) -> bool {
         StorageDead(_5);
         StorageDead(_4);
         StorageDead(_3);
+        StorageLive(_8);
         _8 = copy ((_7 as Break).0: bool);
         _0 = copy _8;
+        StorageDead(_8);
         goto -> bb3;
     }
 

tests/mir-opt/pre-codegen/tuple_ord.demo_le_total.PreCodegen.after.mir

@@ -44,8 +44,10 @@ fn demo_le_total(_1: &(u16, i16), _2: &(u16, i16)) -> bool {
         StorageDead(_5);
         StorageDead(_4);
         StorageDead(_3);
+        StorageLive(_8);
         _8 = copy ((_7 as Break).0: bool);
         _0 = copy _8;
+        StorageDead(_8);
         goto -> bb3;
     }