Generalize the Assume intrinsic statement to a general Intrinsic statement

compiler/rustc_borrowck/src/dataflow.rs

@@ -391,8 +391,7 @@ impl<'tcx> rustc_mir_dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> {
             | mir::StatementKind::Retag { .. }
             | mir::StatementKind::AscribeUserType(..)
             | mir::StatementKind::Coverage(..)
-            | mir::StatementKind::CopyNonOverlapping(..)
+            | mir::StatementKind::Intrinsic(..)
-            | mir::StatementKind::Assume(..)
             | mir::StatementKind::Nop => {}
         }
     }

compiler/rustc_borrowck/src/invalidation.rs

@@ -1,6 +1,6 @@
 use rustc_data_structures::graph::dominators::Dominators;
 use rustc_middle::mir::visit::Visitor;
-use rustc_middle::mir::{BasicBlock, Body, Location, Place, Rvalue};
+use rustc_middle::mir::{self, BasicBlock, Body, Location, NonDivergingIntrinsic, Place, Rvalue};
 use rustc_middle::mir::{BorrowKind, Mutability, Operand};
 use rustc_middle::mir::{InlineAsmOperand, Terminator, TerminatorKind};
 use rustc_middle::mir::{Statement, StatementKind};
@@ -63,11 +63,14 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
             StatementKind::FakeRead(box (_, _)) => {
                 // Only relevant for initialized/liveness/safety checks.
             }
-            StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
+            StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op)) => {
+                self.consume_operand(location, op);
+            }
+            StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
                 ref src,
                 ref dst,
                 ref count,
-            }) => {
+            })) => {
                 self.consume_operand(location, src);
                 self.consume_operand(location, dst);
                 self.consume_operand(location, count);
@@ -76,8 +79,6 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
             StatementKind::AscribeUserType(..)
             // Doesn't have any language semantics
             | StatementKind::Coverage(..)
-            // Takes a `bool` argument, and has no return value, thus being irrelevant for borrowck
-            | StatementKind::Assume(..)
             // Does not actually affect borrowck
             | StatementKind::StorageLive(..) => {}
             StatementKind::StorageDead(local) => {

compiler/rustc_borrowck/src/lib.rs

@@ -26,8 +26,8 @@ use rustc_index::bit_set::ChunkedBitSet;
 use rustc_index::vec::IndexVec;
 use rustc_infer::infer::{DefiningAnchor, InferCtxt, TyCtxtInferExt};
 use rustc_middle::mir::{
-    traversal, Body, ClearCrossCrate, Local, Location, Mutability, Operand, Place, PlaceElem,
+    traversal, Body, ClearCrossCrate, Local, Location, Mutability, NonDivergingIntrinsic, Operand,
-    PlaceRef, VarDebugInfoContents,
+    Place, PlaceElem, PlaceRef, VarDebugInfoContents,
 };
 use rustc_middle::mir::{AggregateKind, BasicBlock, BorrowCheckResult, BorrowKind};
 use rustc_middle::mir::{Field, ProjectionElem, Promoted, Rvalue, Statement, StatementKind};
@@ -591,10 +591,10 @@ impl<'cx, 'tcx> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtx
                     flow_state,
                 );
             }
-            StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
+            StatementKind::Intrinsic(box ref kind) => match kind {
-                ..
+                // Takes a `bool` argument, and has no return value, thus being irrelevant for borrowck
-            }) => {
+                NonDivergingIntrinsic::Assume(..) => {},
-                span_bug!(
+                NonDivergingIntrinsic::CopyNonOverlapping(..) => span_bug!(
                     span,
                     "Unexpected CopyNonOverlapping, should only appear after lower_intrinsics",
                 )
@@ -603,8 +603,6 @@ impl<'cx, 'tcx> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtx
             StatementKind::AscribeUserType(..)
             // Doesn't have any language semantics
             | StatementKind::Coverage(..)
-            // Takes a `bool` argument, and has no return value, thus being irrelevant for borrowck
-            | StatementKind::Assume(..)
             // Does not actually affect borrowck
             | StatementKind::StorageLive(..) => {}
             StatementKind::StorageDead(local) => {

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -1302,14 +1302,14 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                     );
                 }
             }
-            StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
+            StatementKind::Intrinsic(box ref kind) => match kind {
-                ..
+                NonDivergingIntrinsic::Assume(..) => {}
-            }) => span_bug!(
+                NonDivergingIntrinsic::CopyNonOverlapping(..) => span_bug!(
                     stmt.source_info.span,
-                "Unexpected StatementKind::CopyNonOverlapping, should only appear after lowering_intrinsics",
+                    "Unexpected NonDivergingIntrinsic::CopyNonOverlapping, should only appear after lowering_intrinsics",
                 ),
+            },
             StatementKind::FakeRead(..)
-            | StatementKind::Assume(..)
             | StatementKind::StorageLive(..)
             | StatementKind::StorageDead(..)
             | StatementKind::Retag { .. }

compiler/rustc_codegen_cranelift/src/base.rs

@@ -791,25 +791,34 @@ fn codegen_stmt<'tcx>(
         | StatementKind::Nop
         | StatementKind::FakeRead(..)
         | StatementKind::Retag { .. }
-        // We ignore `assume` intrinsics, they are only useful for optimizations
-        | StatementKind::Assume(..)
         | StatementKind::AscribeUserType(..) => {}
 
         StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
-        StatementKind::CopyNonOverlapping(inner) => {
+        StatementKind::Intrinsic(ref intrinsic) => match &**intrinsic {
-            let dst = codegen_operand(fx, &inner.dst);
+            // We ignore `assume` intrinsics, they are only useful for optimizations
+            NonDivergingIntrinsic::Assume(_) => {}
+            NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
+                src,
+                dst,
+                count,
+            }) => {
+                let dst = codegen_operand(fx, dst);
                 let pointee = dst
                     .layout()
                     .pointee_info_at(fx, rustc_target::abi::Size::ZERO)
                     .expect("Expected pointer");
                 let dst = dst.load_scalar(fx);
-            let src = codegen_operand(fx, &inner.src).load_scalar(fx);
+                let src = codegen_operand(fx, src).load_scalar(fx);
-            let count = codegen_operand(fx, &inner.count).load_scalar(fx);
+                let count = codegen_operand(fx, count).load_scalar(fx);
                 let elem_size: u64 = pointee.size.bytes();
-            let bytes =
+                let bytes = if elem_size != 1 {
-                if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
+                    fx.bcx.ins().imul_imm(count, elem_size as i64)
+                } else {
+                    count
+                };
                 fx.bcx.call_memcpy(fx.target_config, dst, src, bytes);
             }
+        },
     }
 }
 

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -536,11 +536,12 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
                         {
                             return None;
                         }
-                        StatementKind::CopyNonOverlapping(_) => {
+                        StatementKind::Intrinsic(ref intrinsic) => match **intrinsic {
-                            return None;
+                            NonDivergingIntrinsic::CopyNonOverlapping(..) => return None,
-                        } // conservative handling
+                            NonDivergingIntrinsic::Assume(..) => {}
+                        },
+                        // conservative handling
                         StatementKind::Assign(_)
-                        | StatementKind::Assume(_)
                         | StatementKind::FakeRead(_)
                         | StatementKind::SetDiscriminant { .. }
                         | StatementKind::Deinit(_)

compiler/rustc_codegen_ssa/src/mir/statement.rs

@@ -1,4 +1,5 @@
 use rustc_middle::mir;
+use rustc_middle::mir::NonDivergingIntrinsic;
 
 use super::FunctionCx;
 use super::LocalRef;
@@ -73,11 +74,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 self.codegen_coverage(&mut bx, coverage.clone(), statement.source_info.scope);
                 bx
             }
-            mir::StatementKind::CopyNonOverlapping(box mir::CopyNonOverlapping {
+            mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(ref op)) => {
-                ref src,
+                let op_val = self.codegen_operand(&mut bx, op);
-                ref dst,
+                bx.assume(op_val.immediate());
-                ref count,
+                bx
-            }) => {
+            }
+            mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(
+                mir::CopyNonOverlapping { ref count, ref src, ref dst },
+            )) => {
                 let dst_val = self.codegen_operand(&mut bx, dst);
                 let src_val = self.codegen_operand(&mut bx, src);
                 let count = self.codegen_operand(&mut bx, count).immediate();
@@ -93,11 +97,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 bx.memcpy(dst, align, src, align, bytes, crate::MemFlags::empty());
                 bx
             }
-            mir::StatementKind::Assume(box ref op) => {
-                let op_val = self.codegen_operand(&mut bx, op);
-                bx.assume(op_val.immediate());
-                bx
-            }
             mir::StatementKind::FakeRead(..)
             | mir::StatementKind::Retag { .. }
             | mir::StatementKind::AscribeUserType(..)

compiler/rustc_const_eval/src/interpret/step.rs

@@ -2,8 +2,8 @@
 //!
 //! The main entry point is the `step` method.
 
-use rustc_middle::mir;
 use rustc_middle::mir::interpret::{InterpResult, Scalar};
+use rustc_middle::mir::{self, NonDivergingIntrinsic};
 use rustc_middle::ty::layout::LayoutOf;
 
 use super::{InterpCx, Machine};
@@ -114,22 +114,23 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 M::retag(self, *kind, &dest)?;
             }
 
-            // Call CopyNonOverlapping
+            Intrinsic(box NonDivergingIntrinsic::Assume(op)) => {
-            CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping { src, dst, count }) => {
-                let src = self.eval_operand(src, None)?;
-                let dst = self.eval_operand(dst, None)?;
-                let count = self.eval_operand(count, None)?;
-                self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true)?;
-            }
-
-            // Call Assume
-            Assume(box op) => {
                 let op = self.eval_operand(op, None)?;
                 let cond = self.read_scalar(&op)?.to_bool()?;
                 if !cond {
                     throw_ub_format!("`assume` called with `false`");
                 }
             }
+            Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
+                ref count,
+                ref src,
+                ref dst,
+            })) => {
+                let src = self.eval_operand(src, None)?;
+                let dst = self.eval_operand(dst, None)?;
+                let count = self.eval_operand(count, None)?;
+                self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true)?;
+            }
 
             // Statements we do not track.
             AscribeUserType(..) => {}

compiler/rustc_const_eval/src/transform/check_consts/check.rs

@@ -678,8 +678,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
             | StatementKind::Retag { .. }
             | StatementKind::AscribeUserType(..)
             | StatementKind::Coverage(..)
-            | StatementKind::CopyNonOverlapping(..)
+            | StatementKind::Intrinsic(..)
-            | StatementKind::Assume(..)
             | StatementKind::Nop => {}
         }
     }

compiler/rustc_const_eval/src/transform/validate.rs

@@ -7,9 +7,10 @@ use rustc_middle::mir::interpret::Scalar;
 use rustc_middle::mir::visit::NonUseContext::VarDebugInfo;
 use rustc_middle::mir::visit::{PlaceContext, Visitor};
 use rustc_middle::mir::{
-    traversal, AggregateKind, BasicBlock, BinOp, Body, BorrowKind, CastKind, Local, Location,
+    traversal, AggregateKind, BasicBlock, BinOp, Body, BorrowKind, CastKind, CopyNonOverlapping,
-    MirPass, MirPhase, Operand, Place, PlaceElem, PlaceRef, ProjectionElem, RuntimePhase, Rvalue,
+    Local, Location, MirPass, MirPhase, NonDivergingIntrinsic, Operand, Place, PlaceElem, PlaceRef,
-    SourceScope, Statement, StatementKind, Terminator, TerminatorKind, UnOp, START_BLOCK,
+    ProjectionElem, RuntimePhase, Rvalue, SourceScope, Statement, StatementKind, Terminator,
+    TerminatorKind, UnOp, START_BLOCK,
 };
 use rustc_middle::ty::fold::BottomUpFolder;
 use rustc_middle::ty::subst::Subst;
@@ -636,7 +637,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                     );
                 }
             }
-            StatementKind::Assume(box ref op) => {
+            StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op)) => {
                 let ty = op.ty(&self.body.local_decls, self.tcx);
                 if !ty.is_bool() {
                     self.fail(
@@ -645,11 +646,9 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                     );
                 }
             }
-            StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
+            StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(
-                ref src,
+                CopyNonOverlapping { src, dst, count },
-                ref dst,
+            )) => {
-                ref count,
-            }) => {
                 let src_ty = src.ty(&self.body.local_decls, self.tcx);
                 let op_src_ty = if let Some(src_deref) = src_ty.builtin_deref(true) {
                     src_deref.ty

compiler/rustc_middle/src/mir/mod.rs

@@ -1370,14 +1370,7 @@ impl Debug for Statement<'_> {
                 write!(fmt, "Coverage::{:?} for {:?}", kind, rgn)
             }
             Coverage(box ref coverage) => write!(fmt, "Coverage::{:?}", coverage.kind),
-            CopyNonOverlapping(box crate::mir::CopyNonOverlapping {
+            Intrinsic(box ref intrinsic) => write!(fmt, "{intrinsic}"),
-                ref src,
-                ref dst,
-                ref count,
-            }) => {
-                write!(fmt, "copy_nonoverlapping(src={:?}, dst={:?}, count={:?})", src, dst, count)
-            }
-            Assume(box ref cond) => write!(fmt, "assume({:?})", cond),
             Nop => write!(fmt, "nop"),
         }
     }

compiler/rustc_middle/src/mir/spanview.rs

@@ -249,8 +249,7 @@ pub fn statement_kind_name(statement: &Statement<'_>) -> &'static str {
         Retag(..) => "Retag",
         AscribeUserType(..) => "AscribeUserType",
         Coverage(..) => "Coverage",
-        CopyNonOverlapping(..) => "CopyNonOverlapping",
+        Intrinsic(..) => "Intrinsic",
-        Assume(..) => "Assume",
         Nop => "Nop",
     }
 }

compiler/rustc_middle/src/mir/syntax.rs

@@ -327,6 +327,34 @@ pub enum StatementKind<'tcx> {
     /// executed.
     Coverage(Box<Coverage>),
 
+    /// Denotes a call to an intrinsic that does not require an unwind path. This avoids
+    /// adding a new block and a terminator for simple intrinsics.
+    Intrinsic(Box<NonDivergingIntrinsic<'tcx>>),
+
+    /// No-op. Useful for deleting instructions without affecting statement indices.
+    Nop,
+}
+
+#[derive(
+    Clone,
+    TyEncodable,
+    TyDecodable,
+    Debug,
+    PartialEq,
+    Hash,
+    HashStable,
+    TypeFoldable,
+    TypeVisitable
+)]
+pub enum NonDivergingIntrinsic<'tcx> {
+    /// Denotes a call to the intrinsic function `assume`.
+    ///
+    /// The operand must be a boolean. Optimizers may use the value of the boolean to backtrack its
+    /// computation to infer information about other variables. So if the boolean came from a
+    /// `x < y` operation, subsequent operations on `x` and `y` could elide various bound checks.
+    /// If the argument is `false`, this operation is equivalent to `TerminatorKind::Unreachable`.
+    Assume(Operand<'tcx>),
+
     /// Denotes a call to the intrinsic function `copy_nonoverlapping`.
     ///
     /// First, all three operands are evaluated. `src` and `dest` must each be a reference, pointer,
@@ -340,18 +368,18 @@ pub enum StatementKind<'tcx> {
     ///
     /// **Needs clarification**: Is this typed or not, ie is there a typed load and store involved?
     /// I vaguely remember Ralf saying somewhere that he thought it should not be.
-    CopyNonOverlapping(Box<CopyNonOverlapping<'tcx>>),
+    CopyNonOverlapping(CopyNonOverlapping<'tcx>),
+}
 
-    /// Denotes a call to the intrinsic function `assume`.
+impl std::fmt::Display for NonDivergingIntrinsic<'_> {
-    ///
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-    /// The operand must be a boolean. Optimizers may use the value of the boolean to backtrack its
+        match self {
-    /// computation to infer information about other variables. So if the boolean came from a
+            Self::Assume(op) => write!(f, "assume({op:?})"),
-    /// `x < y` operation, subsequent operations on `x` and `y` could elide various bound checks.
+            Self::CopyNonOverlapping(CopyNonOverlapping { src, dst, count }) => {
-    /// If the argument is `false`, this operation is equivalent to `TerminatorKind::Unreachable`.
+                write!(f, "copy_nonoverlapping(dst = {dst:?}, src = {src:?}, count = {count:?})")
-    Assume(Box<Operand<'tcx>>),
+            }
-
+        }
-    /// No-op. Useful for deleting instructions without affecting statement indices.
+    }
-    Nop,
 }
 
 /// Describes what kind of retag is to be performed.

compiler/rustc_middle/src/mir/visit.rs

@@ -425,17 +425,15 @@ macro_rules! make_mir_visitor {
                             location
                         )
                     }
-                    StatementKind::CopyNonOverlapping(box crate::mir::CopyNonOverlapping{
+                    StatementKind::Intrinsic(box ref $($mutability)? intrinsic) => {
-                        src,
+                        match intrinsic {
-                        dst,
+                            NonDivergingIntrinsic::Assume(op) => self.visit_operand(op, location),
-                        count,
+                            NonDivergingIntrinsic::CopyNonOverlapping(CopyNonOverlapping { src, dst, count }) => {
-                    }) => {
                                 self.visit_operand(src, location);
                                 self.visit_operand(dst, location);
-                      self.visit_operand(count, location)
+                                self.visit_operand(count, location);
+                            }
                         }
-                    StatementKind::Assume(box ref $($mutability)? val) => {
-                        self.visit_operand(val, location)
                     }
                     StatementKind::Nop => {}
                 }

compiler/rustc_mir_dataflow/src/impls/liveness.rs

@@ -270,8 +270,7 @@ impl<'a, 'tcx> Analysis<'tcx> for MaybeTransitiveLiveLocals<'a> {
             | StatementKind::Retag(..)
             | StatementKind::AscribeUserType(..)
             | StatementKind::Coverage(..)
-            | StatementKind::CopyNonOverlapping(..)
+            | StatementKind::Intrinsic(..)
-            | StatementKind::Assume(..)
             | StatementKind::Nop => None,
         };
         if let Some(destination) = destination {

compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs

@@ -142,8 +142,7 @@ impl<'mir, 'tcx> crate::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'mir, 'tc
             | StatementKind::FakeRead(..)
             | StatementKind::Nop
             | StatementKind::Retag(..)
-            | StatementKind::CopyNonOverlapping(..)
+            | StatementKind::Intrinsic(..)
-            | StatementKind::Assume(..)
             | StatementKind::StorageLive(..) => {}
         }
     }

compiler/rustc_mir_dataflow/src/move_paths/builder.rs

@@ -330,8 +330,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
             StatementKind::Retag { .. }
             | StatementKind::AscribeUserType(..)
             | StatementKind::Coverage(..)
-            | StatementKind::CopyNonOverlapping(..)
+            | StatementKind::Intrinsic(..)
-            | StatementKind::Assume(..)
             | StatementKind::Nop => {}
         }
     }

compiler/rustc_mir_transform/src/check_unsafety.rs

@@ -106,9 +106,7 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> {
             }
 
             // Move to above list once mir construction uses it.
-            StatementKind::Assume(..) => unreachable!(),
+            StatementKind::Intrinsic(..) => unreachable!(),
-
-            StatementKind::CopyNonOverlapping(..) => unreachable!(),
         }
         self.super_statement(statement, location);
     }

compiler/rustc_mir_transform/src/coverage/spans.rs

@@ -825,8 +825,7 @@ pub(super) fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span>
 
         // Retain spans from all other statements
         StatementKind::FakeRead(box (_, _)) // Not including `ForGuardBinding`
-        | StatementKind::CopyNonOverlapping(..)
+        | StatementKind::Intrinsic(..)
-        | StatementKind::Assume(..)
         | StatementKind::Assign(_)
         | StatementKind::SetDiscriminant { .. }
         | StatementKind::Deinit(..)

compiler/rustc_mir_transform/src/dead_store_elimination.rs

@@ -52,8 +52,7 @@ pub fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, borrowed: &BitS
                 | StatementKind::StorageLive(_)
                 | StatementKind::StorageDead(_)
                 | StatementKind::Coverage(_)
-                | StatementKind::CopyNonOverlapping(_)
+                | StatementKind::Intrinsic(_)
-                | StatementKind::Assume(_)
                 | StatementKind::Nop => (),
 
                 StatementKind::FakeRead(_) | StatementKind::AscribeUserType(_, _) => {

compiler/rustc_mir_transform/src/dest_prop.rs

@@ -537,8 +537,7 @@ impl<'a> Conflicts<'a> {
             | StatementKind::FakeRead(..)
             | StatementKind::AscribeUserType(..)
             | StatementKind::Coverage(..)
-            | StatementKind::CopyNonOverlapping(..)
+            | StatementKind::Intrinsic(..)
-            | StatementKind::Assume(..)
             | StatementKind::Nop => {}
         }
     }

compiler/rustc_mir_transform/src/generator.rs

@@ -1452,8 +1452,7 @@ impl<'tcx> Visitor<'tcx> for EnsureGeneratorFieldAssignmentsNeverAlias<'_> {
             | StatementKind::Retag(..)
             | StatementKind::AscribeUserType(..)
             | StatementKind::Coverage(..)
-            | StatementKind::CopyNonOverlapping(..)
+            | StatementKind::Intrinsic(..)
-            | StatementKind::Assume(..)
             | StatementKind::Nop => {}
         }
     }

compiler/rustc_mir_transform/src/lower_intrinsics.rs

@@ -46,12 +46,14 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                         let mut args = args.drain(..);
                         block.statements.push(Statement {
                             source_info: terminator.source_info,
-                            kind: StatementKind::CopyNonOverlapping(Box::new(
+                            kind: StatementKind::Intrinsic(Box::new(
+                                NonDivergingIntrinsic::CopyNonOverlapping(
                                     rustc_middle::mir::CopyNonOverlapping {
                                         src: args.next().unwrap(),
                                         dst: args.next().unwrap(),
                                         count: args.next().unwrap(),
                                     },
+                                ),
                             )),
                         });
                         assert_eq!(
@@ -67,9 +69,15 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                         let mut args = args.drain(..);
                         block.statements.push(Statement {
                             source_info: terminator.source_info,
-                            kind: StatementKind::Assume(Box::new(args.next().unwrap())),
+                            kind: StatementKind::Intrinsic(Box::new(
+                                NonDivergingIntrinsic::Assume(args.next().unwrap()),
+                            )),
                         });
-                        assert_eq!(args.next(), None, "Extra argument for assume intrinsic");
+                        assert_eq!(
+                            args.next(),
+                            None,
+                            "Extra argument for copy_non_overlapping intrinsic"
+                        );
                         drop(args);
                         terminator.kind = TerminatorKind::Goto { target };
                     }

compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs

@@ -51,8 +51,7 @@ impl RemoveNoopLandingPads {
                 StatementKind::Assign { .. }
                 | StatementKind::SetDiscriminant { .. }
                 | StatementKind::Deinit(..)
-                | StatementKind::CopyNonOverlapping(..)
+                | StatementKind::Intrinsic(..)
-                | StatementKind::Assume(..)
                 | StatementKind::Retag { .. } => {
                     return false;
                 }

compiler/rustc_mir_transform/src/separate_const_switch.rs

@@ -249,8 +249,7 @@ fn is_likely_const<'tcx>(mut tracked_place: Place<'tcx>, block: &BasicBlockData<
             | StatementKind::AscribeUserType(_, _)
             | StatementKind::Coverage(_)
             | StatementKind::StorageDead(_)
-            | StatementKind::CopyNonOverlapping(_)
+            | StatementKind::Intrinsic(_)
-            | StatementKind::Assume(_)
             | StatementKind::Nop => {}
         }
     }
@@ -318,8 +317,7 @@ fn find_determining_place<'tcx>(
             | StatementKind::Retag(_, _)
             | StatementKind::AscribeUserType(_, _)
             | StatementKind::Coverage(_)
-            | StatementKind::CopyNonOverlapping(_)
+            | StatementKind::Intrinsic(_)
-            | StatementKind::Assume(_)
             | StatementKind::Nop => {}
 
             // If the discriminant is set, it is always set

compiler/rustc_mir_transform/src/simplify.rs

@@ -499,8 +499,7 @@ impl UsedLocals {
 impl<'tcx> Visitor<'tcx> for UsedLocals {
     fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) {
         match statement.kind {
-            StatementKind::CopyNonOverlapping(..)
+            StatementKind::Intrinsic(..)
-            | StatementKind::Assume(..)
             | StatementKind::Retag(..)
             | StatementKind::Coverage(..)
             | StatementKind::FakeRead(..)

src/test/mir-opt/lower_intrinsics.f_copy_nonoverlapping.LowerIntrinsics.diff

@@ -51,7 +51,7 @@
 -                                          // mir::Constant
 -                                          // + span: $DIR/lower_intrinsics.rs:90:9: 90:28
 -                                          // + literal: Const { ty: unsafe extern "rust-intrinsic" fn(*const i32, *mut i32, usize) {copy_nonoverlapping::<i32>}, val: Value(<ZST>) }
-+         copy_nonoverlapping(src=move _4, dst=move _8, count=const 0_usize); // scope 3 at $DIR/lower_intrinsics.rs:+4:9: +4:95
++         copy_nonoverlapping(dst = move _8, src = move _4, count = const 0_usize); // scope 3 at $DIR/lower_intrinsics.rs:+4:9: +4:95
 +         goto -> bb1;                     // scope 3 at $DIR/lower_intrinsics.rs:+4:9: +4:95
       }
   

src/tools/clippy/clippy_utils/src/qualify_min_const_fn.rs

@@ -7,7 +7,7 @@ use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 use rustc_middle::mir::{
     Body, CastKind, NullOp, Operand, Place, ProjectionElem, Rvalue, Statement, StatementKind, Terminator,
-    TerminatorKind,
+    TerminatorKind, NonDivergingIntrinsic
 };
 use rustc_middle::ty::subst::GenericArgKind;
 use rustc_middle::ty::{self, adjustment::PointerCast, Ty, TyCtxt};
@@ -211,15 +211,19 @@ fn check_statement<'tcx>(
         StatementKind::SetDiscriminant { place, .. } | StatementKind::Deinit(place) => {
             check_place(tcx, **place, span, body)
         },
-        StatementKind::Assume(box op) => {
+
+        StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op)) => {
             check_operand(tcx, op, span, body)
         },
 
-        StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping { dst, src, count }) => {
+        StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(
+            rustc_middle::mir::CopyNonOverlapping { dst, src, count },
+        )) => {
             check_operand(tcx, dst, span, body)?;
             check_operand(tcx, src, span, body)?;
             check_operand(tcx, count, span, body)
         },
+
         // These are all NOPs
         StatementKind::StorageLive(_)
         | StatementKind::StorageDead(_)