mv compiler to compiler/

compiler/rustc_mir_build/src/build/block.rs

@@ -0,0 +1,246 @@
+use crate::build::matches::ArmHasGuard;
+use crate::build::ForGuard::OutsideGuard;
+use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder};
+use crate::thir::*;
+use rustc_hir as hir;
+use rustc_middle::mir::*;
+use rustc_session::lint::builtin::UNSAFE_OP_IN_UNSAFE_FN;
+use rustc_session::lint::Level;
+use rustc_span::Span;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    crate fn ast_block(
+        &mut self,
+        destination: Place<'tcx>,
+        block: BasicBlock,
+        ast_block: &'tcx hir::Block<'tcx>,
+        source_info: SourceInfo,
+    ) -> BlockAnd<()> {
+        let Block {
+            region_scope,
+            opt_destruction_scope,
+            span,
+            stmts,
+            expr,
+            targeted_by_break,
+            safety_mode,
+        } = self.hir.mirror(ast_block);
+        self.in_opt_scope(opt_destruction_scope.map(|de| (de, source_info)), move |this| {
+            this.in_scope((region_scope, source_info), LintLevel::Inherited, move |this| {
+                if targeted_by_break {
+                    // This is a `break`-able block
+                    let exit_block = this.cfg.start_new_block();
+                    let block_exit =
+                        this.in_breakable_scope(None, exit_block, destination, |this| {
+                            this.ast_block_stmts(destination, block, span, stmts, expr, safety_mode)
+                        });
+                    this.cfg.goto(unpack!(block_exit), source_info, exit_block);
+                    exit_block.unit()
+                } else {
+                    this.ast_block_stmts(destination, block, span, stmts, expr, safety_mode)
+                }
+            })
+        })
+    }
+
+    fn ast_block_stmts(
+        &mut self,
+        destination: Place<'tcx>,
+        mut block: BasicBlock,
+        span: Span,
+        stmts: Vec<StmtRef<'tcx>>,
+        expr: Option<ExprRef<'tcx>>,
+        safety_mode: BlockSafety,
+    ) -> BlockAnd<()> {
+        let this = self;
+
+        // This convoluted structure is to avoid using recursion as we walk down a list
+        // of statements. Basically, the structure we get back is something like:
+        //
+        //    let x = <init> in {
+        //       expr1;
+        //       let y = <init> in {
+        //           expr2;
+        //           expr3;
+        //           ...
+        //       }
+        //    }
+        //
+        // The let bindings are valid till the end of block so all we have to do is to pop all
+        // the let-scopes at the end.
+        //
+        // First we build all the statements in the block.
+        let mut let_scope_stack = Vec::with_capacity(8);
+        let outer_source_scope = this.source_scope;
+        let outer_push_unsafe_count = this.push_unsafe_count;
+        let outer_unpushed_unsafe = this.unpushed_unsafe;
+        this.update_source_scope_for_safety_mode(span, safety_mode);
+
+        let source_info = this.source_info(span);
+        for stmt in stmts {
+            let Stmt { kind, opt_destruction_scope } = this.hir.mirror(stmt);
+            match kind {
+                StmtKind::Expr { scope, expr } => {
+                    this.block_context.push(BlockFrame::Statement { ignores_expr_result: true });
+                    unpack!(
+                        block = this.in_opt_scope(
+                            opt_destruction_scope.map(|de| (de, source_info)),
+                            |this| {
+                                let si = (scope, source_info);
+                                this.in_scope(si, LintLevel::Inherited, |this| {
+                                    let expr = this.hir.mirror(expr);
+                                    this.stmt_expr(block, expr, Some(scope))
+                                })
+                            }
+                        )
+                    );
+                }
+                StmtKind::Let { remainder_scope, init_scope, pattern, initializer, lint_level } => {
+                    let ignores_expr_result =
+                        if let PatKind::Wild = *pattern.kind { true } else { false };
+                    this.block_context.push(BlockFrame::Statement { ignores_expr_result });
+
+                    // Enter the remainder scope, i.e., the bindings' destruction scope.
+                    this.push_scope((remainder_scope, source_info));
+                    let_scope_stack.push(remainder_scope);
+
+                    // Declare the bindings, which may create a source scope.
+                    let remainder_span =
+                        remainder_scope.span(this.hir.tcx(), &this.hir.region_scope_tree);
+
+                    let visibility_scope =
+                        Some(this.new_source_scope(remainder_span, LintLevel::Inherited, None));
+
+                    // Evaluate the initializer, if present.
+                    if let Some(init) = initializer {
+                        let initializer_span = init.span();
+
+                        unpack!(
+                            block = this.in_opt_scope(
+                                opt_destruction_scope.map(|de| (de, source_info)),
+                                |this| {
+                                    let scope = (init_scope, source_info);
+                                    this.in_scope(scope, lint_level, |this| {
+                                        this.declare_bindings(
+                                            visibility_scope,
+                                            remainder_span,
+                                            &pattern,
+                                            ArmHasGuard(false),
+                                            Some((None, initializer_span)),
+                                        );
+                                        this.expr_into_pattern(block, pattern, init)
+                                    })
+                                }
+                            )
+                        );
+                    } else {
+                        let scope = (init_scope, source_info);
+                        unpack!(this.in_scope(scope, lint_level, |this| {
+                            this.declare_bindings(
+                                visibility_scope,
+                                remainder_span,
+                                &pattern,
+                                ArmHasGuard(false),
+                                None,
+                            );
+                            block.unit()
+                        }));
+
+                        debug!("ast_block_stmts: pattern={:?}", pattern);
+                        this.visit_primary_bindings(
+                            &pattern,
+                            UserTypeProjections::none(),
+                            &mut |this, _, _, _, node, span, _, _| {
+                                this.storage_live_binding(block, node, span, OutsideGuard, true);
+                                this.schedule_drop_for_binding(node, span, OutsideGuard);
+                            },
+                        )
+                    }
+
+                    // Enter the visibility scope, after evaluating the initializer.
+                    if let Some(source_scope) = visibility_scope {
+                        this.source_scope = source_scope;
+                    }
+                }
+            }
+
+            let popped = this.block_context.pop();
+            assert!(popped.map_or(false, |bf| bf.is_statement()));
+        }
+
+        // Then, the block may have an optional trailing expression which is a “return” value
+        // of the block, which is stored into `destination`.
+        let tcx = this.hir.tcx();
+        let destination_ty = destination.ty(&this.local_decls, tcx).ty;
+        if let Some(expr) = expr {
+            let tail_result_is_ignored =
+                destination_ty.is_unit() || this.block_context.currently_ignores_tail_results();
+            let span = match expr {
+                ExprRef::Thir(expr) => expr.span,
+                ExprRef::Mirror(ref expr) => expr.span,
+            };
+            this.block_context.push(BlockFrame::TailExpr { tail_result_is_ignored, span });
+
+            unpack!(block = this.into(destination, block, expr));
+            let popped = this.block_context.pop();
+
+            assert!(popped.map_or(false, |bf| bf.is_tail_expr()));
+        } else {
+            // If a block has no trailing expression, then it is given an implicit return type.
+            // This return type is usually `()`, unless the block is diverging, in which case the
+            // return type is `!`. For the unit type, we need to actually return the unit, but in
+            // the case of `!`, no return value is required, as the block will never return.
+            if destination_ty.is_unit() {
+                // We only want to assign an implicit `()` as the return value of the block if the
+                // block does not diverge. (Otherwise, we may try to assign a unit to a `!`-type.)
+                this.cfg.push_assign_unit(block, source_info, destination, this.hir.tcx());
+            }
+        }
+        // Finally, we pop all the let scopes before exiting out from the scope of block
+        // itself.
+        for scope in let_scope_stack.into_iter().rev() {
+            unpack!(block = this.pop_scope((scope, source_info), block));
+        }
+        // Restore the original source scope.
+        this.source_scope = outer_source_scope;
+        this.push_unsafe_count = outer_push_unsafe_count;
+        this.unpushed_unsafe = outer_unpushed_unsafe;
+        block.unit()
+    }
+
+    /// If we are changing the safety mode, create a new source scope
+    fn update_source_scope_for_safety_mode(&mut self, span: Span, safety_mode: BlockSafety) {
+        debug!("update_source_scope_for({:?}, {:?})", span, safety_mode);
+        let new_unsafety = match safety_mode {
+            BlockSafety::Safe => None,
+            BlockSafety::ExplicitUnsafe(hir_id) => {
+                assert_eq!(self.push_unsafe_count, 0);
+                match self.unpushed_unsafe {
+                    Safety::Safe => {}
+                    // no longer treat `unsafe fn`s as `unsafe` contexts (see RFC #2585)
+                    Safety::FnUnsafe
+                        if self.hir.tcx().lint_level_at_node(UNSAFE_OP_IN_UNSAFE_FN, hir_id).0
+                            != Level::Allow => {}
+                    _ => return,
+                }
+                self.unpushed_unsafe = Safety::ExplicitUnsafe(hir_id);
+                Some(Safety::ExplicitUnsafe(hir_id))
+            }
+            BlockSafety::PushUnsafe => {
+                self.push_unsafe_count += 1;
+                Some(Safety::BuiltinUnsafe)
+            }
+            BlockSafety::PopUnsafe => {
+                self.push_unsafe_count = self
+                    .push_unsafe_count
+                    .checked_sub(1)
+                    .unwrap_or_else(|| span_bug!(span, "unsafe count underflow"));
+                if self.push_unsafe_count == 0 { Some(self.unpushed_unsafe) } else { None }
+            }
+        };
+
+        if let Some(unsafety) = new_unsafety {
+            self.source_scope = self.new_source_scope(span, LintLevel::Inherited, Some(unsafety));
+        }
+    }
+}

compiler/rustc_mir_build/src/build/cfg.rs

@@ -0,0 +1,108 @@
+//! Routines for manipulating the control-flow graph.
+
+use crate::build::CFG;
+use rustc_middle::mir::*;
+use rustc_middle::ty::{self, TyCtxt};
+
+impl<'tcx> CFG<'tcx> {
+    crate fn block_data(&self, blk: BasicBlock) -> &BasicBlockData<'tcx> {
+        &self.basic_blocks[blk]
+    }
+
+    crate fn block_data_mut(&mut self, blk: BasicBlock) -> &mut BasicBlockData<'tcx> {
+        &mut self.basic_blocks[blk]
+    }
+
+    // llvm.org/PR32488 makes this function use an excess of stack space. Mark
+    // it as #[inline(never)] to keep rustc's stack use in check.
+    #[inline(never)]
+    crate fn start_new_block(&mut self) -> BasicBlock {
+        self.basic_blocks.push(BasicBlockData::new(None))
+    }
+
+    crate fn start_new_cleanup_block(&mut self) -> BasicBlock {
+        let bb = self.start_new_block();
+        self.block_data_mut(bb).is_cleanup = true;
+        bb
+    }
+
+    crate fn push(&mut self, block: BasicBlock, statement: Statement<'tcx>) {
+        debug!("push({:?}, {:?})", block, statement);
+        self.block_data_mut(block).statements.push(statement);
+    }
+
+    crate fn push_assign(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        place: Place<'tcx>,
+        rvalue: Rvalue<'tcx>,
+    ) {
+        self.push(
+            block,
+            Statement { source_info, kind: StatementKind::Assign(box (place, rvalue)) },
+        );
+    }
+
+    crate fn push_assign_constant(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        temp: Place<'tcx>,
+        constant: Constant<'tcx>,
+    ) {
+        self.push_assign(block, source_info, temp, Rvalue::Use(Operand::Constant(box constant)));
+    }
+
+    crate fn push_assign_unit(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        place: Place<'tcx>,
+        tcx: TyCtxt<'tcx>,
+    ) {
+        self.push_assign(
+            block,
+            source_info,
+            place,
+            Rvalue::Use(Operand::Constant(box Constant {
+                span: source_info.span,
+                user_ty: None,
+                literal: ty::Const::zero_sized(tcx, tcx.types.unit),
+            })),
+        );
+    }
+
+    crate fn push_fake_read(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        cause: FakeReadCause,
+        place: Place<'tcx>,
+    ) {
+        let kind = StatementKind::FakeRead(cause, box place);
+        let stmt = Statement { source_info, kind };
+        self.push(block, stmt);
+    }
+
+    crate fn terminate(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        kind: TerminatorKind<'tcx>,
+    ) {
+        debug!("terminating block {:?} <- {:?}", block, kind);
+        debug_assert!(
+            self.block_data(block).terminator.is_none(),
+            "terminate: block {:?}={:?} already has a terminator set",
+            block,
+            self.block_data(block)
+        );
+        self.block_data_mut(block).terminator = Some(Terminator { source_info, kind });
+    }
+
+    /// In the `origin` block, push a `goto -> target` terminator.
+    crate fn goto(&mut self, origin: BasicBlock, source_info: SourceInfo, target: BasicBlock) {
+        self.terminate(origin, source_info, TerminatorKind::Goto { target })
+    }
+}

compiler/rustc_mir_build/src/build/expr/as_constant.rs

@@ -0,0 +1,39 @@
+//! See docs in build/expr/mod.rs
+
+use crate::build::Builder;
+use crate::thir::*;
+use rustc_middle::mir::*;
+use rustc_middle::ty::CanonicalUserTypeAnnotation;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Compile `expr`, yielding a compile-time constant. Assumes that
+    /// `expr` is a valid compile-time constant!
+    crate fn as_constant<M>(&mut self, expr: M) -> Constant<'tcx>
+    where
+        M: Mirror<'tcx, Output = Expr<'tcx>>,
+    {
+        let expr = self.hir.mirror(expr);
+        self.expr_as_constant(expr)
+    }
+
+    fn expr_as_constant(&mut self, expr: Expr<'tcx>) -> Constant<'tcx> {
+        let this = self;
+        let Expr { ty, temp_lifetime: _, span, kind } = expr;
+        match kind {
+            ExprKind::Scope { region_scope: _, lint_level: _, value } => this.as_constant(value),
+            ExprKind::Literal { literal, user_ty } => {
+                let user_ty = user_ty.map(|user_ty| {
+                    this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
+                        span,
+                        user_ty,
+                        inferred_ty: ty,
+                    })
+                });
+                assert_eq!(literal.ty, ty);
+                Constant { span, user_ty, literal }
+            }
+            ExprKind::StaticRef { literal, .. } => Constant { span, user_ty: None, literal },
+            _ => span_bug!(span, "expression is not a valid constant {:?}", kind),
+        }
+    }
+}

compiler/rustc_mir_build/src/build/expr/as_operand.rs

@@ -0,0 +1,198 @@
+//! See docs in build/expr/mod.rs
+
+use crate::build::expr::category::Category;
+use crate::build::{BlockAnd, BlockAndExtension, Builder};
+use crate::thir::*;
+use rustc_middle::middle::region;
+use rustc_middle::mir::*;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Returns an operand suitable for use until the end of the current
+    /// scope expression.
+    ///
+    /// The operand returned from this function will *not be valid*
+    /// after the current enclosing `ExprKind::Scope` has ended, so
+    /// please do *not* return it from functions to avoid bad
+    /// miscompiles.
+    crate fn as_local_operand<M>(&mut self, block: BasicBlock, expr: M) -> BlockAnd<Operand<'tcx>>
+    where
+        M: Mirror<'tcx, Output = Expr<'tcx>>,
+    {
+        let local_scope = self.local_scope();
+        self.as_operand(block, local_scope, expr)
+    }
+
+    /// Returns an operand suitable for use until the end of the current scope expression and
+    /// suitable also to be passed as function arguments.
+    ///
+    /// The operand returned from this function will *not be valid* after an ExprKind::Scope is
+    /// passed, so please do *not* return it from functions to avoid bad miscompiles.  Returns an
+    /// operand suitable for use as a call argument. This is almost always equivalent to
+    /// `as_operand`, except for the particular case of passing values of (potentially) unsized
+    /// types "by value" (see details below).
+    ///
+    /// The operand returned from this function will *not be valid*
+    /// after the current enclosing `ExprKind::Scope` has ended, so
+    /// please do *not* return it from functions to avoid bad
+    /// miscompiles.
+    ///
+    /// # Parameters of unsized types
+    ///
+    /// We tweak the handling of parameters of unsized type slightly to avoid the need to create a
+    /// local variable of unsized type. For example, consider this program:
+    ///
+    /// ```rust
+    /// fn foo(p: dyn Debug) { ... }
+    ///
+    /// fn bar(box_p: Box<dyn Debug>) { foo(*p); }
+    /// ```
+    ///
+    /// Ordinarily, for sized types, we would compile the call `foo(*p)` like so:
+    ///
+    /// ```rust
+    /// let tmp0 = *box_p; // tmp0 would be the operand returned by this function call
+    /// foo(tmp0)
+    /// ```
+    ///
+    /// But because the parameter to `foo` is of the unsized type `dyn Debug`, and because it is
+    /// being moved the deref of a box, we compile it slightly differently. The temporary `tmp0`
+    /// that we create *stores the entire box*, and the parameter to the call itself will be
+    /// `*tmp0`:
+    ///
+    /// ```rust
+    /// let tmp0 = box_p; call foo(*tmp0)
+    /// ```
+    ///
+    /// This way, the temporary `tmp0` that we create has type `Box<dyn Debug>`, which is sized.
+    /// The value passed to the call (`*tmp0`) still has the `dyn Debug` type -- but the way that
+    /// calls are compiled means that this parameter will be passed "by reference", meaning that we
+    /// will actually provide a pointer to the interior of the box, and not move the `dyn Debug`
+    /// value to the stack.
+    ///
+    /// See #68034 for more details.
+    crate fn as_local_call_operand<M>(
+        &mut self,
+        block: BasicBlock,
+        expr: M,
+    ) -> BlockAnd<Operand<'tcx>>
+    where
+        M: Mirror<'tcx, Output = Expr<'tcx>>,
+    {
+        let local_scope = self.local_scope();
+        self.as_call_operand(block, local_scope, expr)
+    }
+
+    /// Compile `expr` into a value that can be used as an operand.
+    /// If `expr` is a place like `x`, this will introduce a
+    /// temporary `tmp = x`, so that we capture the value of `x` at
+    /// this time.
+    ///
+    /// The operand is known to be live until the end of `scope`.
+    crate fn as_operand<M>(
+        &mut self,
+        block: BasicBlock,
+        scope: Option<region::Scope>,
+        expr: M,
+    ) -> BlockAnd<Operand<'tcx>>
+    where
+        M: Mirror<'tcx, Output = Expr<'tcx>>,
+    {
+        let expr = self.hir.mirror(expr);
+        self.expr_as_operand(block, scope, expr)
+    }
+
+    /// Like `as_local_call_operand`, except that the argument will
+    /// not be valid once `scope` ends.
+    fn as_call_operand<M>(
+        &mut self,
+        block: BasicBlock,
+        scope: Option<region::Scope>,
+        expr: M,
+    ) -> BlockAnd<Operand<'tcx>>
+    where
+        M: Mirror<'tcx, Output = Expr<'tcx>>,
+    {
+        let expr = self.hir.mirror(expr);
+        self.expr_as_call_operand(block, scope, expr)
+    }
+
+    fn expr_as_operand(
+        &mut self,
+        mut block: BasicBlock,
+        scope: Option<region::Scope>,
+        expr: Expr<'tcx>,
+    ) -> BlockAnd<Operand<'tcx>> {
+        debug!("expr_as_operand(block={:?}, expr={:?})", block, expr);
+        let this = self;
+
+        if let ExprKind::Scope { region_scope, lint_level, value } = expr.kind {
+            let source_info = this.source_info(expr.span);
+            let region_scope = (region_scope, source_info);
+            return this
+                .in_scope(region_scope, lint_level, |this| this.as_operand(block, scope, value));
+        }
+
+        let category = Category::of(&expr.kind).unwrap();
+        debug!("expr_as_operand: category={:?} for={:?}", category, expr.kind);
+        match category {
+            Category::Constant => {
+                let constant = this.as_constant(expr);
+                block.and(Operand::Constant(box constant))
+            }
+            Category::Place | Category::Rvalue(..) => {
+                let operand = unpack!(block = this.as_temp(block, scope, expr, Mutability::Mut));
+                block.and(Operand::Move(Place::from(operand)))
+            }
+        }
+    }
+
+    fn expr_as_call_operand(
+        &mut self,
+        mut block: BasicBlock,
+        scope: Option<region::Scope>,
+        expr: Expr<'tcx>,
+    ) -> BlockAnd<Operand<'tcx>> {
+        debug!("expr_as_call_operand(block={:?}, expr={:?})", block, expr);
+        let this = self;
+
+        if let ExprKind::Scope { region_scope, lint_level, value } = expr.kind {
+            let source_info = this.source_info(expr.span);
+            let region_scope = (region_scope, source_info);
+            return this.in_scope(region_scope, lint_level, |this| {
+                this.as_call_operand(block, scope, value)
+            });
+        }
+
+        let tcx = this.hir.tcx();
+
+        if tcx.features().unsized_locals {
+            let ty = expr.ty;
+            let span = expr.span;
+            let param_env = this.hir.param_env;
+
+            if !ty.is_sized(tcx.at(span), param_env) {
+                // !sized means !copy, so this is an unsized move
+                assert!(!ty.is_copy_modulo_regions(tcx.at(span), param_env));
+
+                // As described above, detect the case where we are passing a value of unsized
+                // type, and that value is coming from the deref of a box.
+                if let ExprKind::Deref { ref arg } = expr.kind {
+                    let arg = this.hir.mirror(arg.clone());
+
+                    // Generate let tmp0 = arg0
+                    let operand = unpack!(block = this.as_temp(block, scope, arg, Mutability::Mut));
+
+                    // Return the operand *tmp0 to be used as the call argument
+                    let place = Place {
+                        local: operand,
+                        projection: tcx.intern_place_elems(&[PlaceElem::Deref]),
+                    };
+
+                    return block.and(Operand::Move(place));
+                }
+            }
+        }
+
+        this.expr_as_operand(block, scope, expr)
+    }
+}

compiler/rustc_mir_build/src/build/expr/as_place.rs

@@ -0,0 +1,439 @@
+//! See docs in build/expr/mod.rs
+
+use crate::build::expr::category::Category;
+use crate::build::ForGuard::{OutsideGuard, RefWithinGuard};
+use crate::build::{BlockAnd, BlockAndExtension, Builder};
+use crate::thir::*;
+use rustc_middle::middle::region;
+use rustc_middle::mir::AssertKind::BoundsCheck;
+use rustc_middle::mir::*;
+use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, Ty, TyCtxt, Variance};
+use rustc_span::Span;
+
+use rustc_index::vec::Idx;
+
+/// `PlaceBuilder` is used to create places during MIR construction. It allows you to "build up" a
+/// place by pushing more and more projections onto the end, and then convert the final set into a
+/// place using the `into_place` method.
+///
+/// This is used internally when building a place for an expression like `a.b.c`. The fields `b`
+/// and `c` can be progressively pushed onto the place builder that is created when converting `a`.
+#[derive(Clone)]
+struct PlaceBuilder<'tcx> {
+    local: Local,
+    projection: Vec<PlaceElem<'tcx>>,
+}
+
+impl<'tcx> PlaceBuilder<'tcx> {
+    fn into_place(self, tcx: TyCtxt<'tcx>) -> Place<'tcx> {
+        Place { local: self.local, projection: tcx.intern_place_elems(&self.projection) }
+    }
+
+    fn field(self, f: Field, ty: Ty<'tcx>) -> Self {
+        self.project(PlaceElem::Field(f, ty))
+    }
+
+    fn deref(self) -> Self {
+        self.project(PlaceElem::Deref)
+    }
+
+    fn index(self, index: Local) -> Self {
+        self.project(PlaceElem::Index(index))
+    }
+
+    fn project(mut self, elem: PlaceElem<'tcx>) -> Self {
+        self.projection.push(elem);
+        self
+    }
+}
+
+impl<'tcx> From<Local> for PlaceBuilder<'tcx> {
+    fn from(local: Local) -> Self {
+        Self { local, projection: Vec::new() }
+    }
+}
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Compile `expr`, yielding a place that we can move from etc.
+    ///
+    /// WARNING: Any user code might:
+    /// * Invalidate any slice bounds checks performed.
+    /// * Change the address that this `Place` refers to.
+    /// * Modify the memory that this place refers to.
+    /// * Invalidate the memory that this place refers to, this will be caught
+    ///   by borrow checking.
+    ///
+    /// Extra care is needed if any user code is allowed to run between calling
+    /// this method and using it, as is the case for `match` and index
+    /// expressions.
+    crate fn as_place<M>(&mut self, mut block: BasicBlock, expr: M) -> BlockAnd<Place<'tcx>>
+    where
+        M: Mirror<'tcx, Output = Expr<'tcx>>,
+    {
+        let place_builder = unpack!(block = self.as_place_builder(block, expr));
+        block.and(place_builder.into_place(self.hir.tcx()))
+    }
+
+    /// This is used when constructing a compound `Place`, so that we can avoid creating
+    /// intermediate `Place` values until we know the full set of projections.
+    fn as_place_builder<M>(&mut self, block: BasicBlock, expr: M) -> BlockAnd<PlaceBuilder<'tcx>>
+    where
+        M: Mirror<'tcx, Output = Expr<'tcx>>,
+    {
+        let expr = self.hir.mirror(expr);
+        self.expr_as_place(block, expr, Mutability::Mut, None)
+    }
+
+    /// Compile `expr`, yielding a place that we can move from etc.
+    /// Mutability note: The caller of this method promises only to read from the resulting
+    /// place. The place itself may or may not be mutable:
+    /// * If this expr is a place expr like a.b, then we will return that place.
+    /// * Otherwise, a temporary is created: in that event, it will be an immutable temporary.
+    crate fn as_read_only_place<M>(
+        &mut self,
+        mut block: BasicBlock,
+        expr: M,
+    ) -> BlockAnd<Place<'tcx>>
+    where
+        M: Mirror<'tcx, Output = Expr<'tcx>>,
+    {
+        let place_builder = unpack!(block = self.as_read_only_place_builder(block, expr));
+        block.and(place_builder.into_place(self.hir.tcx()))
+    }
+
+    /// This is used when constructing a compound `Place`, so that we can avoid creating
+    /// intermediate `Place` values until we know the full set of projections.
+    /// Mutability note: The caller of this method promises only to read from the resulting
+    /// place. The place itself may or may not be mutable:
+    /// * If this expr is a place expr like a.b, then we will return that place.
+    /// * Otherwise, a temporary is created: in that event, it will be an immutable temporary.
+    fn as_read_only_place_builder<M>(
+        &mut self,
+        block: BasicBlock,
+        expr: M,
+    ) -> BlockAnd<PlaceBuilder<'tcx>>
+    where
+        M: Mirror<'tcx, Output = Expr<'tcx>>,
+    {
+        let expr = self.hir.mirror(expr);
+        self.expr_as_place(block, expr, Mutability::Not, None)
+    }
+
+    fn expr_as_place(
+        &mut self,
+        mut block: BasicBlock,
+        expr: Expr<'tcx>,
+        mutability: Mutability,
+        fake_borrow_temps: Option<&mut Vec<Local>>,
+    ) -> BlockAnd<PlaceBuilder<'tcx>> {
+        debug!("expr_as_place(block={:?}, expr={:?}, mutability={:?})", block, expr, mutability);
+
+        let this = self;
+        let expr_span = expr.span;
+        let source_info = this.source_info(expr_span);
+        match expr.kind {
+            ExprKind::Scope { region_scope, lint_level, value } => {
+                this.in_scope((region_scope, source_info), lint_level, |this| {
+                    let value = this.hir.mirror(value);
+                    this.expr_as_place(block, value, mutability, fake_borrow_temps)
+                })
+            }
+            ExprKind::Field { lhs, name } => {
+                let lhs = this.hir.mirror(lhs);
+                let place_builder =
+                    unpack!(block = this.expr_as_place(block, lhs, mutability, fake_borrow_temps,));
+                block.and(place_builder.field(name, expr.ty))
+            }
+            ExprKind::Deref { arg } => {
+                let arg = this.hir.mirror(arg);
+                let place_builder =
+                    unpack!(block = this.expr_as_place(block, arg, mutability, fake_borrow_temps,));
+                block.and(place_builder.deref())
+            }
+            ExprKind::Index { lhs, index } => this.lower_index_expression(
+                block,
+                lhs,
+                index,
+                mutability,
+                fake_borrow_temps,
+                expr.temp_lifetime,
+                expr_span,
+                source_info,
+            ),
+            ExprKind::SelfRef => block.and(PlaceBuilder::from(Local::new(1))),
+            ExprKind::VarRef { id } => {
+                let place_builder = if this.is_bound_var_in_guard(id) {
+                    let index = this.var_local_id(id, RefWithinGuard);
+                    PlaceBuilder::from(index).deref()
+                } else {
+                    let index = this.var_local_id(id, OutsideGuard);
+                    PlaceBuilder::from(index)
+                };
+                block.and(place_builder)
+            }
+
+            ExprKind::PlaceTypeAscription { source, user_ty } => {
+                let source = this.hir.mirror(source);
+                let place_builder = unpack!(
+                    block = this.expr_as_place(block, source, mutability, fake_borrow_temps,)
+                );
+                if let Some(user_ty) = user_ty {
+                    let annotation_index =
+                        this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
+                            span: source_info.span,
+                            user_ty,
+                            inferred_ty: expr.ty,
+                        });
+
+                    let place = place_builder.clone().into_place(this.hir.tcx());
+                    this.cfg.push(
+                        block,
+                        Statement {
+                            source_info,
+                            kind: StatementKind::AscribeUserType(
+                                box (
+                                    place,
+                                    UserTypeProjection { base: annotation_index, projs: vec![] },
+                                ),
+                                Variance::Invariant,
+                            ),
+                        },
+                    );
+                }
+                block.and(place_builder)
+            }
+            ExprKind::ValueTypeAscription { source, user_ty } => {
+                let source = this.hir.mirror(source);
+                let temp =
+                    unpack!(block = this.as_temp(block, source.temp_lifetime, source, mutability));
+                if let Some(user_ty) = user_ty {
+                    let annotation_index =
+                        this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
+                            span: source_info.span,
+                            user_ty,
+                            inferred_ty: expr.ty,
+                        });
+                    this.cfg.push(
+                        block,
+                        Statement {
+                            source_info,
+                            kind: StatementKind::AscribeUserType(
+                                box (
+                                    Place::from(temp),
+                                    UserTypeProjection { base: annotation_index, projs: vec![] },
+                                ),
+                                Variance::Invariant,
+                            ),
+                        },
+                    );
+                }
+                block.and(PlaceBuilder::from(temp))
+            }
+
+            ExprKind::Array { .. }
+            | ExprKind::Tuple { .. }
+            | ExprKind::Adt { .. }
+            | ExprKind::Closure { .. }
+            | ExprKind::Unary { .. }
+            | ExprKind::Binary { .. }
+            | ExprKind::LogicalOp { .. }
+            | ExprKind::Box { .. }
+            | ExprKind::Cast { .. }
+            | ExprKind::Use { .. }
+            | ExprKind::NeverToAny { .. }
+            | ExprKind::Pointer { .. }
+            | ExprKind::Repeat { .. }
+            | ExprKind::Borrow { .. }
+            | ExprKind::AddressOf { .. }
+            | ExprKind::Match { .. }
+            | ExprKind::Loop { .. }
+            | ExprKind::Block { .. }
+            | ExprKind::Assign { .. }
+            | ExprKind::AssignOp { .. }
+            | ExprKind::Break { .. }
+            | ExprKind::Continue { .. }
+            | ExprKind::Return { .. }
+            | ExprKind::Literal { .. }
+            | ExprKind::StaticRef { .. }
+            | ExprKind::InlineAsm { .. }
+            | ExprKind::LlvmInlineAsm { .. }
+            | ExprKind::Yield { .. }
+            | ExprKind::ThreadLocalRef(_)
+            | ExprKind::Call { .. } => {
+                // these are not places, so we need to make a temporary.
+                debug_assert!(match Category::of(&expr.kind) {
+                    Some(Category::Place) => false,
+                    _ => true,
+                });
+                let temp =
+                    unpack!(block = this.as_temp(block, expr.temp_lifetime, expr, mutability));
+                block.and(PlaceBuilder::from(temp))
+            }
+        }
+    }
+
+    /// Lower an index expression
+    ///
+    /// This has two complications;
+    ///
+    /// * We need to do a bounds check.
+    /// * We need to ensure that the bounds check can't be invalidated using an
+    ///   expression like `x[1][{x = y; 2}]`. We use fake borrows here to ensure
+    ///   that this is the case.
+    fn lower_index_expression(
+        &mut self,
+        mut block: BasicBlock,
+        base: ExprRef<'tcx>,
+        index: ExprRef<'tcx>,
+        mutability: Mutability,
+        fake_borrow_temps: Option<&mut Vec<Local>>,
+        temp_lifetime: Option<region::Scope>,
+        expr_span: Span,
+        source_info: SourceInfo,
+    ) -> BlockAnd<PlaceBuilder<'tcx>> {
+        let lhs = self.hir.mirror(base);
+
+        let base_fake_borrow_temps = &mut Vec::new();
+        let is_outermost_index = fake_borrow_temps.is_none();
+        let fake_borrow_temps = fake_borrow_temps.unwrap_or(base_fake_borrow_temps);
+
+        let base_place =
+            unpack!(block = self.expr_as_place(block, lhs, mutability, Some(fake_borrow_temps),));
+
+        // Making this a *fresh* temporary means we do not have to worry about
+        // the index changing later: Nothing will ever change this temporary.
+        // The "retagging" transformation (for Stacked Borrows) relies on this.
+        let idx = unpack!(block = self.as_temp(block, temp_lifetime, index, Mutability::Not,));
+
+        block = self.bounds_check(
+            block,
+            base_place.clone().into_place(self.hir.tcx()),
+            idx,
+            expr_span,
+            source_info,
+        );
+
+        if is_outermost_index {
+            self.read_fake_borrows(block, fake_borrow_temps, source_info)
+        } else {
+            self.add_fake_borrows_of_base(
+                &base_place,
+                block,
+                fake_borrow_temps,
+                expr_span,
+                source_info,
+            );
+        }
+
+        block.and(base_place.index(idx))
+    }
+
+    fn bounds_check(
+        &mut self,
+        block: BasicBlock,
+        slice: Place<'tcx>,
+        index: Local,
+        expr_span: Span,
+        source_info: SourceInfo,
+    ) -> BasicBlock {
+        let usize_ty = self.hir.usize_ty();
+        let bool_ty = self.hir.bool_ty();
+        // bounds check:
+        let len = self.temp(usize_ty, expr_span);
+        let lt = self.temp(bool_ty, expr_span);
+
+        // len = len(slice)
+        self.cfg.push_assign(block, source_info, len, Rvalue::Len(slice));
+        // lt = idx < len
+        self.cfg.push_assign(
+            block,
+            source_info,
+            lt,
+            Rvalue::BinaryOp(BinOp::Lt, Operand::Copy(Place::from(index)), Operand::Copy(len)),
+        );
+        let msg = BoundsCheck { len: Operand::Move(len), index: Operand::Copy(Place::from(index)) };
+        // assert!(lt, "...")
+        self.assert(block, Operand::Move(lt), true, msg, expr_span)
+    }
+
+    fn add_fake_borrows_of_base(
+        &mut self,
+        base_place: &PlaceBuilder<'tcx>,
+        block: BasicBlock,
+        fake_borrow_temps: &mut Vec<Local>,
+        expr_span: Span,
+        source_info: SourceInfo,
+    ) {
+        let tcx = self.hir.tcx();
+        let place_ty =
+            Place::ty_from(base_place.local, &base_place.projection, &self.local_decls, tcx);
+        if let ty::Slice(_) = place_ty.ty.kind {
+            // We need to create fake borrows to ensure that the bounds
+            // check that we just did stays valid. Since we can't assign to
+            // unsized values, we only need to ensure that none of the
+            // pointers in the base place are modified.
+            for (idx, elem) in base_place.projection.iter().enumerate().rev() {
+                match elem {
+                    ProjectionElem::Deref => {
+                        let fake_borrow_deref_ty = Place::ty_from(
+                            base_place.local,
+                            &base_place.projection[..idx],
+                            &self.local_decls,
+                            tcx,
+                        )
+                        .ty;
+                        let fake_borrow_ty =
+                            tcx.mk_imm_ref(tcx.lifetimes.re_erased, fake_borrow_deref_ty);
+                        let fake_borrow_temp =
+                            self.local_decls.push(LocalDecl::new(fake_borrow_ty, expr_span));
+                        let projection = tcx.intern_place_elems(&base_place.projection[..idx]);
+                        self.cfg.push_assign(
+                            block,
+                            source_info,
+                            fake_borrow_temp.into(),
+                            Rvalue::Ref(
+                                tcx.lifetimes.re_erased,
+                                BorrowKind::Shallow,
+                                Place { local: base_place.local, projection },
+                            ),
+                        );
+                        fake_borrow_temps.push(fake_borrow_temp);
+                    }
+                    ProjectionElem::Index(_) => {
+                        let index_ty = Place::ty_from(
+                            base_place.local,
+                            &base_place.projection[..idx],
+                            &self.local_decls,
+                            tcx,
+                        );
+                        match index_ty.ty.kind {
+                            // The previous index expression has already
+                            // done any index expressions needed here.
+                            ty::Slice(_) => break,
+                            ty::Array(..) => (),
+                            _ => bug!("unexpected index base"),
+                        }
+                    }
+                    ProjectionElem::Field(..)
+                    | ProjectionElem::Downcast(..)
+                    | ProjectionElem::ConstantIndex { .. }
+                    | ProjectionElem::Subslice { .. } => (),
+                }
+            }
+        }
+    }
+
+    fn read_fake_borrows(
+        &mut self,
+        bb: BasicBlock,
+        fake_borrow_temps: &mut Vec<Local>,
+        source_info: SourceInfo,
+    ) {
+        // All indexes have been evaluated now, read all of the
+        // fake borrows so that they are live across those index
+        // expressions.
+        for temp in fake_borrow_temps {
+            self.cfg.push_fake_read(bb, source_info, FakeReadCause::ForIndex, Place::from(*temp));
+        }
+    }
+}

compiler/rustc_mir_build/src/build/expr/as_rvalue.rs

@@ -0,0 +1,468 @@
+//! See docs in `build/expr/mod.rs`.
+
+use rustc_index::vec::Idx;
+
+use crate::build::expr::category::{Category, RvalueFunc};
+use crate::build::{BlockAnd, BlockAndExtension, Builder};
+use crate::thir::*;
+use rustc_middle::middle::region;
+use rustc_middle::mir::AssertKind;
+use rustc_middle::mir::*;
+use rustc_middle::ty::{self, Ty, UpvarSubsts};
+use rustc_span::Span;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Returns an rvalue suitable for use until the end of the current
+    /// scope expression.
+    ///
+    /// The operand returned from this function will *not be valid* after
+    /// an ExprKind::Scope is passed, so please do *not* return it from
+    /// functions to avoid bad miscompiles.
+    crate fn as_local_rvalue<M>(&mut self, block: BasicBlock, expr: M) -> BlockAnd<Rvalue<'tcx>>
+    where
+        M: Mirror<'tcx, Output = Expr<'tcx>>,
+    {
+        let local_scope = self.local_scope();
+        self.as_rvalue(block, local_scope, expr)
+    }
+
+    /// Compile `expr`, yielding an rvalue.
+    fn as_rvalue<M>(
+        &mut self,
+        block: BasicBlock,
+        scope: Option<region::Scope>,
+        expr: M,
+    ) -> BlockAnd<Rvalue<'tcx>>
+    where
+        M: Mirror<'tcx, Output = Expr<'tcx>>,
+    {
+        let expr = self.hir.mirror(expr);
+        self.expr_as_rvalue(block, scope, expr)
+    }
+
+    fn expr_as_rvalue(
+        &mut self,
+        mut block: BasicBlock,
+        scope: Option<region::Scope>,
+        expr: Expr<'tcx>,
+    ) -> BlockAnd<Rvalue<'tcx>> {
+        debug!("expr_as_rvalue(block={:?}, scope={:?}, expr={:?})", block, scope, expr);
+
+        let this = self;
+        let expr_span = expr.span;
+        let source_info = this.source_info(expr_span);
+
+        match expr.kind {
+            ExprKind::ThreadLocalRef(did) => block.and(Rvalue::ThreadLocalRef(did)),
+            ExprKind::Scope { region_scope, lint_level, value } => {
+                let region_scope = (region_scope, source_info);
+                this.in_scope(region_scope, lint_level, |this| this.as_rvalue(block, scope, value))
+            }
+            ExprKind::Repeat { value, count } => {
+                let value_operand = unpack!(block = this.as_operand(block, scope, value));
+                block.and(Rvalue::Repeat(value_operand, count))
+            }
+            ExprKind::Binary { op, lhs, rhs } => {
+                let lhs = unpack!(block = this.as_operand(block, scope, lhs));
+                let rhs = unpack!(block = this.as_operand(block, scope, rhs));
+                this.build_binary_op(block, op, expr_span, expr.ty, lhs, rhs)
+            }
+            ExprKind::Unary { op, arg } => {
+                let arg = unpack!(block = this.as_operand(block, scope, arg));
+                // Check for -MIN on signed integers
+                if this.hir.check_overflow() && op == UnOp::Neg && expr.ty.is_signed() {
+                    let bool_ty = this.hir.bool_ty();
+
+                    let minval = this.minval_literal(expr_span, expr.ty);
+                    let is_min = this.temp(bool_ty, expr_span);
+
+                    this.cfg.push_assign(
+                        block,
+                        source_info,
+                        is_min,
+                        Rvalue::BinaryOp(BinOp::Eq, arg.to_copy(), minval),
+                    );
+
+                    block = this.assert(
+                        block,
+                        Operand::Move(is_min),
+                        false,
+                        AssertKind::OverflowNeg(arg.to_copy()),
+                        expr_span,
+                    );
+                }
+                block.and(Rvalue::UnaryOp(op, arg))
+            }
+            ExprKind::Box { value } => {
+                let value = this.hir.mirror(value);
+                // The `Box<T>` temporary created here is not a part of the HIR,
+                // and therefore is not considered during generator OIBIT
+                // determination. See the comment about `box` at `yield_in_scope`.
+                let result = this.local_decls.push(LocalDecl::new(expr.ty, expr_span).internal());
+                this.cfg.push(
+                    block,
+                    Statement { source_info, kind: StatementKind::StorageLive(result) },
+                );
+                if let Some(scope) = scope {
+                    // schedule a shallow free of that memory, lest we unwind:
+                    this.schedule_drop_storage_and_value(expr_span, scope, result);
+                }
+
+                // malloc some memory of suitable type (thus far, uninitialized):
+                let box_ = Rvalue::NullaryOp(NullOp::Box, value.ty);
+                this.cfg.push_assign(block, source_info, Place::from(result), box_);
+
+                // initialize the box contents:
+                unpack!(
+                    block =
+                        this.into(this.hir.tcx().mk_place_deref(Place::from(result)), block, value)
+                );
+                block.and(Rvalue::Use(Operand::Move(Place::from(result))))
+            }
+            ExprKind::Cast { source } => {
+                let source = unpack!(block = this.as_operand(block, scope, source));
+                block.and(Rvalue::Cast(CastKind::Misc, source, expr.ty))
+            }
+            ExprKind::Pointer { cast, source } => {
+                let source = unpack!(block = this.as_operand(block, scope, source));
+                block.and(Rvalue::Cast(CastKind::Pointer(cast), source, expr.ty))
+            }
+            ExprKind::Array { fields } => {
+                // (*) We would (maybe) be closer to codegen if we
+                // handled this and other aggregate cases via
+                // `into()`, not `as_rvalue` -- in that case, instead
+                // of generating
+                //
+                //     let tmp1 = ...1;
+                //     let tmp2 = ...2;
+                //     dest = Rvalue::Aggregate(Foo, [tmp1, tmp2])
+                //
+                // we could just generate
+                //
+                //     dest.f = ...1;
+                //     dest.g = ...2;
+                //
+                // The problem is that then we would need to:
+                //
+                // (a) have a more complex mechanism for handling
+                //     partial cleanup;
+                // (b) distinguish the case where the type `Foo` has a
+                //     destructor, in which case creating an instance
+                //     as a whole "arms" the destructor, and you can't
+                //     write individual fields; and,
+                // (c) handle the case where the type Foo has no
+                //     fields. We don't want `let x: ();` to compile
+                //     to the same MIR as `let x = ();`.
+
+                // first process the set of fields
+                let el_ty = expr.ty.sequence_element_type(this.hir.tcx());
+                let fields: Vec<_> = fields
+                    .into_iter()
+                    .map(|f| unpack!(block = this.as_operand(block, scope, f)))
+                    .collect();
+
+                block.and(Rvalue::Aggregate(box AggregateKind::Array(el_ty), fields))
+            }
+            ExprKind::Tuple { fields } => {
+                // see (*) above
+                // first process the set of fields
+                let fields: Vec<_> = fields
+                    .into_iter()
+                    .map(|f| unpack!(block = this.as_operand(block, scope, f)))
+                    .collect();
+
+                block.and(Rvalue::Aggregate(box AggregateKind::Tuple, fields))
+            }
+            ExprKind::Closure { closure_id, substs, upvars, movability } => {
+                // see (*) above
+                let operands: Vec<_> = upvars
+                    .into_iter()
+                    .map(|upvar| {
+                        let upvar = this.hir.mirror(upvar);
+                        match Category::of(&upvar.kind) {
+                            // Use as_place to avoid creating a temporary when
+                            // moving a variable into a closure, so that
+                            // borrowck knows which variables to mark as being
+                            // used as mut. This is OK here because the upvar
+                            // expressions have no side effects and act on
+                            // disjoint places.
+                            // This occurs when capturing by copy/move, while
+                            // by reference captures use as_operand
+                            Some(Category::Place) => {
+                                let place = unpack!(block = this.as_place(block, upvar));
+                                this.consume_by_copy_or_move(place)
+                            }
+                            _ => {
+                                // Turn mutable borrow captures into unique
+                                // borrow captures when capturing an immutable
+                                // variable. This is sound because the mutation
+                                // that caused the capture will cause an error.
+                                match upvar.kind {
+                                    ExprKind::Borrow {
+                                        borrow_kind:
+                                            BorrowKind::Mut { allow_two_phase_borrow: false },
+                                        arg,
+                                    } => unpack!(
+                                        block = this.limit_capture_mutability(
+                                            upvar.span, upvar.ty, scope, block, arg,
+                                        )
+                                    ),
+                                    _ => unpack!(block = this.as_operand(block, scope, upvar)),
+                                }
+                            }
+                        }
+                    })
+                    .collect();
+                let result = match substs {
+                    UpvarSubsts::Generator(substs) => {
+                        // We implicitly set the discriminant to 0. See
+                        // librustc_mir/transform/deaggregator.rs for details.
+                        let movability = movability.unwrap();
+                        box AggregateKind::Generator(closure_id, substs, movability)
+                    }
+                    UpvarSubsts::Closure(substs) => box AggregateKind::Closure(closure_id, substs),
+                };
+                block.and(Rvalue::Aggregate(result, operands))
+            }
+            ExprKind::Assign { .. } | ExprKind::AssignOp { .. } => {
+                block = unpack!(this.stmt_expr(block, expr, None));
+                block.and(Rvalue::Use(Operand::Constant(box Constant {
+                    span: expr_span,
+                    user_ty: None,
+                    literal: ty::Const::zero_sized(this.hir.tcx(), this.hir.tcx().types.unit),
+                })))
+            }
+            ExprKind::Yield { .. }
+            | ExprKind::Literal { .. }
+            | ExprKind::StaticRef { .. }
+            | ExprKind::Block { .. }
+            | ExprKind::Match { .. }
+            | ExprKind::NeverToAny { .. }
+            | ExprKind::Use { .. }
+            | ExprKind::Borrow { .. }
+            | ExprKind::AddressOf { .. }
+            | ExprKind::Adt { .. }
+            | ExprKind::Loop { .. }
+            | ExprKind::LogicalOp { .. }
+            | ExprKind::Call { .. }
+            | ExprKind::Field { .. }
+            | ExprKind::Deref { .. }
+            | ExprKind::Index { .. }
+            | ExprKind::VarRef { .. }
+            | ExprKind::SelfRef
+            | ExprKind::Break { .. }
+            | ExprKind::Continue { .. }
+            | ExprKind::Return { .. }
+            | ExprKind::InlineAsm { .. }
+            | ExprKind::LlvmInlineAsm { .. }
+            | ExprKind::PlaceTypeAscription { .. }
+            | ExprKind::ValueTypeAscription { .. } => {
+                // these do not have corresponding `Rvalue` variants,
+                // so make an operand and then return that
+                debug_assert!(match Category::of(&expr.kind) {
+                    Some(Category::Rvalue(RvalueFunc::AsRvalue)) => false,
+                    _ => true,
+                });
+                let operand = unpack!(block = this.as_operand(block, scope, expr));
+                block.and(Rvalue::Use(operand))
+            }
+        }
+    }
+
+    crate fn build_binary_op(
+        &mut self,
+        mut block: BasicBlock,
+        op: BinOp,
+        span: Span,
+        ty: Ty<'tcx>,
+        lhs: Operand<'tcx>,
+        rhs: Operand<'tcx>,
+    ) -> BlockAnd<Rvalue<'tcx>> {
+        let source_info = self.source_info(span);
+        let bool_ty = self.hir.bool_ty();
+        if self.hir.check_overflow() && op.is_checkable() && ty.is_integral() {
+            let result_tup = self.hir.tcx().intern_tup(&[ty, bool_ty]);
+            let result_value = self.temp(result_tup, span);
+
+            self.cfg.push_assign(
+                block,
+                source_info,
+                result_value,
+                Rvalue::CheckedBinaryOp(op, lhs.to_copy(), rhs.to_copy()),
+            );
+            let val_fld = Field::new(0);
+            let of_fld = Field::new(1);
+
+            let tcx = self.hir.tcx();
+            let val = tcx.mk_place_field(result_value, val_fld, ty);
+            let of = tcx.mk_place_field(result_value, of_fld, bool_ty);
+
+            let err = AssertKind::Overflow(op, lhs, rhs);
+
+            block = self.assert(block, Operand::Move(of), false, err, span);
+
+            block.and(Rvalue::Use(Operand::Move(val)))
+        } else {
+            if ty.is_integral() && (op == BinOp::Div || op == BinOp::Rem) {
+                // Checking division and remainder is more complex, since we 1. always check
+                // and 2. there are two possible failure cases, divide-by-zero and overflow.
+
+                let zero_err = if op == BinOp::Div {
+                    AssertKind::DivisionByZero(lhs.to_copy())
+                } else {
+                    AssertKind::RemainderByZero(lhs.to_copy())
+                };
+                let overflow_err = AssertKind::Overflow(op, lhs.to_copy(), rhs.to_copy());
+
+                // Check for / 0
+                let is_zero = self.temp(bool_ty, span);
+                let zero = self.zero_literal(span, ty);
+                self.cfg.push_assign(
+                    block,
+                    source_info,
+                    is_zero,
+                    Rvalue::BinaryOp(BinOp::Eq, rhs.to_copy(), zero),
+                );
+
+                block = self.assert(block, Operand::Move(is_zero), false, zero_err, span);
+
+                // We only need to check for the overflow in one case:
+                // MIN / -1, and only for signed values.
+                if ty.is_signed() {
+                    let neg_1 = self.neg_1_literal(span, ty);
+                    let min = self.minval_literal(span, ty);
+
+                    let is_neg_1 = self.temp(bool_ty, span);
+                    let is_min = self.temp(bool_ty, span);
+                    let of = self.temp(bool_ty, span);
+
+                    // this does (rhs == -1) & (lhs == MIN). It could short-circuit instead
+
+                    self.cfg.push_assign(
+                        block,
+                        source_info,
+                        is_neg_1,
+                        Rvalue::BinaryOp(BinOp::Eq, rhs.to_copy(), neg_1),
+                    );
+                    self.cfg.push_assign(
+                        block,
+                        source_info,
+                        is_min,
+                        Rvalue::BinaryOp(BinOp::Eq, lhs.to_copy(), min),
+                    );
+
+                    let is_neg_1 = Operand::Move(is_neg_1);
+                    let is_min = Operand::Move(is_min);
+                    self.cfg.push_assign(
+                        block,
+                        source_info,
+                        of,
+                        Rvalue::BinaryOp(BinOp::BitAnd, is_neg_1, is_min),
+                    );
+
+                    block = self.assert(block, Operand::Move(of), false, overflow_err, span);
+                }
+            }
+
+            block.and(Rvalue::BinaryOp(op, lhs, rhs))
+        }
+    }
+
+    fn limit_capture_mutability(
+        &mut self,
+        upvar_span: Span,
+        upvar_ty: Ty<'tcx>,
+        temp_lifetime: Option<region::Scope>,
+        mut block: BasicBlock,
+        arg: ExprRef<'tcx>,
+    ) -> BlockAnd<Operand<'tcx>> {
+        let this = self;
+
+        let source_info = this.source_info(upvar_span);
+        let temp = this.local_decls.push(LocalDecl::new(upvar_ty, upvar_span));
+
+        this.cfg.push(block, Statement { source_info, kind: StatementKind::StorageLive(temp) });
+
+        let arg_place = unpack!(block = this.as_place(block, arg));
+
+        let mutability = match arg_place.as_ref() {
+            PlaceRef { local, projection: &[] } => this.local_decls[local].mutability,
+            PlaceRef { local, projection: &[ProjectionElem::Deref] } => {
+                debug_assert!(
+                    this.local_decls[local].is_ref_for_guard(),
+                    "Unexpected capture place",
+                );
+                this.local_decls[local].mutability
+            }
+            PlaceRef {
+                local,
+                projection: &[ref proj_base @ .., ProjectionElem::Field(upvar_index, _)],
+            }
+            | PlaceRef {
+                local,
+                projection:
+                    &[ref proj_base @ .., ProjectionElem::Field(upvar_index, _), ProjectionElem::Deref],
+            } => {
+                let place = PlaceRef { local, projection: proj_base };
+
+                // Not projected from the implicit `self` in a closure.
+                debug_assert!(
+                    match place.local_or_deref_local() {
+                        Some(local) => local == Local::new(1),
+                        None => false,
+                    },
+                    "Unexpected capture place"
+                );
+                // Not in a closure
+                debug_assert!(
+                    this.upvar_mutbls.len() > upvar_index.index(),
+                    "Unexpected capture place"
+                );
+                this.upvar_mutbls[upvar_index.index()]
+            }
+            _ => bug!("Unexpected capture place"),
+        };
+
+        let borrow_kind = match mutability {
+            Mutability::Not => BorrowKind::Unique,
+            Mutability::Mut => BorrowKind::Mut { allow_two_phase_borrow: false },
+        };
+
+        this.cfg.push_assign(
+            block,
+            source_info,
+            Place::from(temp),
+            Rvalue::Ref(this.hir.tcx().lifetimes.re_erased, borrow_kind, arg_place),
+        );
+
+        // In constants, temp_lifetime is None. We should not need to drop
+        // anything because no values with a destructor can be created in
+        // a constant at this time, even if the type may need dropping.
+        if let Some(temp_lifetime) = temp_lifetime {
+            this.schedule_drop_storage_and_value(upvar_span, temp_lifetime, temp);
+        }
+
+        block.and(Operand::Move(Place::from(temp)))
+    }
+
+    // Helper to get a `-1` value of the appropriate type
+    fn neg_1_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> {
+        let param_ty = ty::ParamEnv::empty().and(ty);
+        let bits = self.hir.tcx().layout_of(param_ty).unwrap().size.bits();
+        let n = (!0u128) >> (128 - bits);
+        let literal = ty::Const::from_bits(self.hir.tcx(), n, param_ty);
+
+        self.literal_operand(span, literal)
+    }
+
+    // Helper to get the minimum value of the appropriate type
+    fn minval_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> {
+        assert!(ty.is_signed());
+        let param_ty = ty::ParamEnv::empty().and(ty);
+        let bits = self.hir.tcx().layout_of(param_ty).unwrap().size.bits();
+        let n = 1 << (bits - 1);
+        let literal = ty::Const::from_bits(self.hir.tcx(), n, param_ty);
+
+        self.literal_operand(span, literal)
+    }
+}

compiler/rustc_mir_build/src/build/expr/as_temp.rs

@@ -0,0 +1,122 @@
+//! See docs in build/expr/mod.rs
+
+use crate::build::scope::DropKind;
+use crate::build::{BlockAnd, BlockAndExtension, Builder};
+use crate::thir::*;
+use rustc_data_structures::stack::ensure_sufficient_stack;
+use rustc_hir as hir;
+use rustc_middle::middle::region;
+use rustc_middle::mir::*;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Compile `expr` into a fresh temporary. This is used when building
+    /// up rvalues so as to freeze the value that will be consumed.
+    crate fn as_temp<M>(
+        &mut self,
+        block: BasicBlock,
+        temp_lifetime: Option<region::Scope>,
+        expr: M,
+        mutability: Mutability,
+    ) -> BlockAnd<Local>
+    where
+        M: Mirror<'tcx, Output = Expr<'tcx>>,
+    {
+        let expr = self.hir.mirror(expr);
+        //
+        // this is the only place in mir building that we need to truly need to worry about
+        // infinite recursion. Everything else does recurse, too, but it always gets broken up
+        // at some point by inserting an intermediate temporary
+        ensure_sufficient_stack(|| self.expr_as_temp(block, temp_lifetime, expr, mutability))
+    }
+
+    fn expr_as_temp(
+        &mut self,
+        mut block: BasicBlock,
+        temp_lifetime: Option<region::Scope>,
+        expr: Expr<'tcx>,
+        mutability: Mutability,
+    ) -> BlockAnd<Local> {
+        debug!(
+            "expr_as_temp(block={:?}, temp_lifetime={:?}, expr={:?}, mutability={:?})",
+            block, temp_lifetime, expr, mutability
+        );
+        let this = self;
+
+        let expr_span = expr.span;
+        let source_info = this.source_info(expr_span);
+        if let ExprKind::Scope { region_scope, lint_level, value } = expr.kind {
+            return this.in_scope((region_scope, source_info), lint_level, |this| {
+                this.as_temp(block, temp_lifetime, value, mutability)
+            });
+        }
+
+        let expr_ty = expr.ty;
+        let temp = {
+            let mut local_decl = LocalDecl::new(expr_ty, expr_span);
+            if mutability == Mutability::Not {
+                local_decl = local_decl.immutable();
+            }
+
+            debug!("creating temp {:?} with block_context: {:?}", local_decl, this.block_context);
+            // Find out whether this temp is being created within the
+            // tail expression of a block whose result is ignored.
+            if let Some(tail_info) = this.block_context.currently_in_block_tail() {
+                local_decl = local_decl.block_tail(tail_info);
+            }
+            match expr.kind {
+                ExprKind::StaticRef { def_id, .. } => {
+                    assert!(!this.hir.tcx().is_thread_local_static(def_id));
+                    local_decl.internal = true;
+                    local_decl.local_info =
+                        Some(box LocalInfo::StaticRef { def_id, is_thread_local: false });
+                }
+                ExprKind::ThreadLocalRef(def_id) => {
+                    assert!(this.hir.tcx().is_thread_local_static(def_id));
+                    local_decl.internal = true;
+                    local_decl.local_info =
+                        Some(box LocalInfo::StaticRef { def_id, is_thread_local: true });
+                }
+                _ => {}
+            }
+            this.local_decls.push(local_decl)
+        };
+        let temp_place = Place::from(temp);
+
+        match expr.kind {
+            // Don't bother with StorageLive and Dead for these temporaries,
+            // they are never assigned.
+            ExprKind::Break { .. } | ExprKind::Continue { .. } | ExprKind::Return { .. } => (),
+            ExprKind::Block { body: hir::Block { expr: None, targeted_by_break: false, .. } }
+                if expr_ty.is_never() => {}
+            _ => {
+                this.cfg
+                    .push(block, Statement { source_info, kind: StatementKind::StorageLive(temp) });
+
+                // In constants, `temp_lifetime` is `None` for temporaries that
+                // live for the `'static` lifetime. Thus we do not drop these
+                // temporaries and simply leak them.
+                // This is equivalent to what `let x = &foo();` does in
+                // functions. The temporary is lifted to their surrounding
+                // scope. In a function that means the temporary lives until
+                // just before the function returns. In constants that means it
+                // outlives the constant's initialization value computation.
+                // Anything outliving a constant must have the `'static`
+                // lifetime and live forever.
+                // Anything with a shorter lifetime (e.g the `&foo()` in
+                // `bar(&foo())` or anything within a block will keep the
+                // regular drops just like runtime code.
+                if let Some(temp_lifetime) = temp_lifetime {
+                    this.schedule_drop(expr_span, temp_lifetime, temp, DropKind::Storage);
+                }
+            }
+        }
+
+        unpack!(block = this.into(temp_place, block, expr));
+
+        if let Some(temp_lifetime) = temp_lifetime {
+            this.schedule_drop(expr_span, temp_lifetime, temp, DropKind::Value);
+        }
+
+        block.and(temp)
+    }
+}

compiler/rustc_mir_build/src/build/expr/category.rs

@@ -0,0 +1,85 @@
+use crate::thir::*;
+
+#[derive(Debug, PartialEq)]
+crate enum Category {
+    // An assignable memory location like `x`, `x.f`, `foo()[3]`, that
+    // sort of thing. Something that could appear on the LHS of an `=`
+    // sign.
+    Place,
+
+    // A literal like `23` or `"foo"`. Does not include constant
+    // expressions like `3 + 5`.
+    Constant,
+
+    // Something that generates a new value at runtime, like `x + y`
+    // or `foo()`.
+    Rvalue(RvalueFunc),
+}
+
+// Rvalues fall into different "styles" that will determine which fn
+// is best suited to generate them.
+#[derive(Debug, PartialEq)]
+crate enum RvalueFunc {
+    // Best generated by `into`. This is generally exprs that
+    // cause branching, like `match`, but also includes calls.
+    Into,
+
+    // Best generated by `as_rvalue`. This is usually the case.
+    AsRvalue,
+}
+
+/// Determines the category for a given expression. Note that scope
+/// and paren expressions have no category.
+impl Category {
+    crate fn of(ek: &ExprKind<'_>) -> Option<Category> {
+        match *ek {
+            ExprKind::Scope { .. } => None,
+
+            ExprKind::Field { .. }
+            | ExprKind::Deref { .. }
+            | ExprKind::Index { .. }
+            | ExprKind::SelfRef
+            | ExprKind::VarRef { .. }
+            | ExprKind::PlaceTypeAscription { .. }
+            | ExprKind::ValueTypeAscription { .. } => Some(Category::Place),
+
+            ExprKind::LogicalOp { .. }
+            | ExprKind::Match { .. }
+            | ExprKind::NeverToAny { .. }
+            | ExprKind::Use { .. }
+            | ExprKind::Adt { .. }
+            | ExprKind::Borrow { .. }
+            | ExprKind::AddressOf { .. }
+            | ExprKind::Yield { .. }
+            | ExprKind::Call { .. }
+            | ExprKind::InlineAsm { .. } => Some(Category::Rvalue(RvalueFunc::Into)),
+
+            ExprKind::Array { .. }
+            | ExprKind::Tuple { .. }
+            | ExprKind::Closure { .. }
+            | ExprKind::Unary { .. }
+            | ExprKind::Binary { .. }
+            | ExprKind::Box { .. }
+            | ExprKind::Cast { .. }
+            | ExprKind::Pointer { .. }
+            | ExprKind::Repeat { .. }
+            | ExprKind::Assign { .. }
+            | ExprKind::AssignOp { .. }
+            | ExprKind::ThreadLocalRef(_)
+            | ExprKind::LlvmInlineAsm { .. } => Some(Category::Rvalue(RvalueFunc::AsRvalue)),
+
+            ExprKind::Literal { .. } | ExprKind::StaticRef { .. } => Some(Category::Constant),
+
+            ExprKind::Loop { .. }
+            | ExprKind::Block { .. }
+            | ExprKind::Break { .. }
+            | ExprKind::Continue { .. }
+            | ExprKind::Return { .. } =>
+            // FIXME(#27840) these probably want their own
+            // category, like "nonterminating"
+            {
+                Some(Category::Rvalue(RvalueFunc::Into))
+            }
+        }
+    }
+}

compiler/rustc_mir_build/src/build/expr/into.rs

@@ -0,0 +1,487 @@
+//! See docs in build/expr/mod.rs
+
+use crate::build::expr::category::{Category, RvalueFunc};
+use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder};
+use crate::thir::*;
+use rustc_ast::InlineAsmOptions;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::stack::ensure_sufficient_stack;
+use rustc_hir as hir;
+use rustc_middle::mir::*;
+use rustc_middle::ty::{self, CanonicalUserTypeAnnotation};
+use rustc_span::symbol::sym;
+
+use rustc_target::spec::abi::Abi;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Compile `expr`, storing the result into `destination`, which
+    /// is assumed to be uninitialized.
+    crate fn into_expr(
+        &mut self,
+        destination: Place<'tcx>,
+        mut block: BasicBlock,
+        expr: Expr<'tcx>,
+    ) -> BlockAnd<()> {
+        debug!("into_expr(destination={:?}, block={:?}, expr={:?})", destination, block, expr);
+
+        // since we frequently have to reference `self` from within a
+        // closure, where `self` would be shadowed, it's easier to
+        // just use the name `this` uniformly
+        let this = self;
+        let expr_span = expr.span;
+        let source_info = this.source_info(expr_span);
+
+        let expr_is_block_or_scope = match expr.kind {
+            ExprKind::Block { .. } => true,
+            ExprKind::Scope { .. } => true,
+            _ => false,
+        };
+
+        if !expr_is_block_or_scope {
+            this.block_context.push(BlockFrame::SubExpr);
+        }
+
+        let block_and = match expr.kind {
+            ExprKind::Scope { region_scope, lint_level, value } => {
+                let region_scope = (region_scope, source_info);
+                ensure_sufficient_stack(|| {
+                    this.in_scope(region_scope, lint_level, |this| {
+                        this.into(destination, block, value)
+                    })
+                })
+            }
+            ExprKind::Block { body: ast_block } => {
+                this.ast_block(destination, block, ast_block, source_info)
+            }
+            ExprKind::Match { scrutinee, arms } => {
+                this.match_expr(destination, expr_span, block, scrutinee, arms)
+            }
+            ExprKind::NeverToAny { source } => {
+                let source = this.hir.mirror(source);
+                let is_call = match source.kind {
+                    ExprKind::Call { .. } | ExprKind::InlineAsm { .. } => true,
+                    _ => false,
+                };
+
+                // (#66975) Source could be a const of type `!`, so has to
+                // exist in the generated MIR.
+                unpack!(block = this.as_temp(block, this.local_scope(), source, Mutability::Mut,));
+
+                // This is an optimization. If the expression was a call then we already have an
+                // unreachable block. Don't bother to terminate it and create a new one.
+                if is_call {
+                    block.unit()
+                } else {
+                    this.cfg.terminate(block, source_info, TerminatorKind::Unreachable);
+                    let end_block = this.cfg.start_new_block();
+                    end_block.unit()
+                }
+            }
+            ExprKind::LogicalOp { op, lhs, rhs } => {
+                // And:
+                //
+                // [block: If(lhs)] -true-> [else_block: If(rhs)] -true-> [true_block]
+                //        |                          | (false)
+                //        +----------false-----------+------------------> [false_block]
+                //
+                // Or:
+                //
+                // [block: If(lhs)] -false-> [else_block: If(rhs)] -true-> [true_block]
+                //        | (true)                   | (false)
+                //  [true_block]               [false_block]
+
+                let (true_block, false_block, mut else_block, join_block) = (
+                    this.cfg.start_new_block(),
+                    this.cfg.start_new_block(),
+                    this.cfg.start_new_block(),
+                    this.cfg.start_new_block(),
+                );
+
+                let lhs = unpack!(block = this.as_local_operand(block, lhs));
+                let blocks = match op {
+                    LogicalOp::And => (else_block, false_block),
+                    LogicalOp::Or => (true_block, else_block),
+                };
+                let term = TerminatorKind::if_(this.hir.tcx(), lhs, blocks.0, blocks.1);
+                this.cfg.terminate(block, source_info, term);
+
+                let rhs = unpack!(else_block = this.as_local_operand(else_block, rhs));
+                let term = TerminatorKind::if_(this.hir.tcx(), rhs, true_block, false_block);
+                this.cfg.terminate(else_block, source_info, term);
+
+                this.cfg.push_assign_constant(
+                    true_block,
+                    source_info,
+                    destination,
+                    Constant { span: expr_span, user_ty: None, literal: this.hir.true_literal() },
+                );
+
+                this.cfg.push_assign_constant(
+                    false_block,
+                    source_info,
+                    destination,
+                    Constant { span: expr_span, user_ty: None, literal: this.hir.false_literal() },
+                );
+
+                // Link up both branches:
+                this.cfg.goto(true_block, source_info, join_block);
+                this.cfg.goto(false_block, source_info, join_block);
+                join_block.unit()
+            }
+            ExprKind::Loop { body } => {
+                // [block]
+                //    |
+                //   [loop_block] -> [body_block] -/eval. body/-> [body_block_end]
+                //    |        ^                                         |
+                // false link  |                                         |
+                //    |        +-----------------------------------------+
+                //    +-> [diverge_cleanup]
+                // The false link is required to make sure borrowck considers unwinds through the
+                // body, even when the exact code in the body cannot unwind
+
+                let loop_block = this.cfg.start_new_block();
+                let exit_block = this.cfg.start_new_block();
+
+                // Start the loop.
+                this.cfg.goto(block, source_info, loop_block);
+
+                this.in_breakable_scope(Some(loop_block), exit_block, destination, move |this| {
+                    // conduct the test, if necessary
+                    let body_block = this.cfg.start_new_block();
+                    let diverge_cleanup = this.diverge_cleanup();
+                    this.cfg.terminate(
+                        loop_block,
+                        source_info,
+                        TerminatorKind::FalseUnwind {
+                            real_target: body_block,
+                            unwind: Some(diverge_cleanup),
+                        },
+                    );
+
+                    // The “return” value of the loop body must always be an unit. We therefore
+                    // introduce a unit temporary as the destination for the loop body.
+                    let tmp = this.get_unit_temp();
+                    // Execute the body, branching back to the test.
+                    let body_block_end = unpack!(this.into(tmp, body_block, body));
+                    this.cfg.goto(body_block_end, source_info, loop_block);
+                });
+                exit_block.unit()
+            }
+            ExprKind::Call { ty, fun, args, from_hir_call, fn_span } => {
+                let intrinsic = match ty.kind {
+                    ty::FnDef(def_id, _) => {
+                        let f = ty.fn_sig(this.hir.tcx());
+                        if f.abi() == Abi::RustIntrinsic || f.abi() == Abi::PlatformIntrinsic {
+                            Some(this.hir.tcx().item_name(def_id))
+                        } else {
+                            None
+                        }
+                    }
+                    _ => None,
+                };
+                let fun = unpack!(block = this.as_local_operand(block, fun));
+                if let Some(sym::move_val_init) = intrinsic {
+                    // `move_val_init` has "magic" semantics - the second argument is
+                    // always evaluated "directly" into the first one.
+
+                    let mut args = args.into_iter();
+                    let ptr = args.next().expect("0 arguments to `move_val_init`");
+                    let val = args.next().expect("1 argument to `move_val_init`");
+                    assert!(args.next().is_none(), ">2 arguments to `move_val_init`");
+
+                    let ptr = this.hir.mirror(ptr);
+                    let ptr_ty = ptr.ty;
+                    // Create an *internal* temp for the pointer, so that unsafety
+                    // checking won't complain about the raw pointer assignment.
+                    let ptr_temp = this
+                        .local_decls
+                        .push(LocalDecl::with_source_info(ptr_ty, source_info).internal());
+                    let ptr_temp = Place::from(ptr_temp);
+                    let block = unpack!(this.into(ptr_temp, block, ptr));
+                    this.into(this.hir.tcx().mk_place_deref(ptr_temp), block, val)
+                } else {
+                    let args: Vec<_> = args
+                        .into_iter()
+                        .map(|arg| unpack!(block = this.as_local_call_operand(block, arg)))
+                        .collect();
+
+                    let success = this.cfg.start_new_block();
+                    let cleanup = this.diverge_cleanup();
+
+                    this.record_operands_moved(&args);
+
+                    debug!("into_expr: fn_span={:?}", fn_span);
+
+                    this.cfg.terminate(
+                        block,
+                        source_info,
+                        TerminatorKind::Call {
+                            func: fun,
+                            args,
+                            cleanup: Some(cleanup),
+                            // FIXME(varkor): replace this with an uninhabitedness-based check.
+                            // This requires getting access to the current module to call
+                            // `tcx.is_ty_uninhabited_from`, which is currently tricky to do.
+                            destination: if expr.ty.is_never() {
+                                None
+                            } else {
+                                Some((destination, success))
+                            },
+                            from_hir_call,
+                            fn_span,
+                        },
+                    );
+                    success.unit()
+                }
+            }
+            ExprKind::Use { source } => this.into(destination, block, source),
+            ExprKind::Borrow { arg, borrow_kind } => {
+                // We don't do this in `as_rvalue` because we use `as_place`
+                // for borrow expressions, so we cannot create an `RValue` that
+                // remains valid across user code. `as_rvalue` is usually called
+                // by this method anyway, so this shouldn't cause too many
+                // unnecessary temporaries.
+                let arg_place = match borrow_kind {
+                    BorrowKind::Shared => unpack!(block = this.as_read_only_place(block, arg)),
+                    _ => unpack!(block = this.as_place(block, arg)),
+                };
+                let borrow =
+                    Rvalue::Ref(this.hir.tcx().lifetimes.re_erased, borrow_kind, arg_place);
+                this.cfg.push_assign(block, source_info, destination, borrow);
+                block.unit()
+            }
+            ExprKind::AddressOf { mutability, arg } => {
+                let place = match mutability {
+                    hir::Mutability::Not => this.as_read_only_place(block, arg),
+                    hir::Mutability::Mut => this.as_place(block, arg),
+                };
+                let address_of = Rvalue::AddressOf(mutability, unpack!(block = place));
+                this.cfg.push_assign(block, source_info, destination, address_of);
+                block.unit()
+            }
+            ExprKind::Adt { adt_def, variant_index, substs, user_ty, fields, base } => {
+                // See the notes for `ExprKind::Array` in `as_rvalue` and for
+                // `ExprKind::Borrow` above.
+                let is_union = adt_def.is_union();
+                let active_field_index = if is_union { Some(fields[0].name.index()) } else { None };
+
+                let scope = this.local_scope();
+
+                // first process the set of fields that were provided
+                // (evaluating them in order given by user)
+                let fields_map: FxHashMap<_, _> = fields
+                    .into_iter()
+                    .map(|f| (f.name, unpack!(block = this.as_operand(block, scope, f.expr))))
+                    .collect();
+
+                let field_names = this.hir.all_fields(adt_def, variant_index);
+
+                let fields = if let Some(FruInfo { base, field_types }) = base {
+                    let base = unpack!(block = this.as_place(block, base));
+
+                    // MIR does not natively support FRU, so for each
+                    // base-supplied field, generate an operand that
+                    // reads it from the base.
+                    field_names
+                        .into_iter()
+                        .zip(field_types.into_iter())
+                        .map(|(n, ty)| match fields_map.get(&n) {
+                            Some(v) => v.clone(),
+                            None => this.consume_by_copy_or_move(
+                                this.hir.tcx().mk_place_field(base, n, ty),
+                            ),
+                        })
+                        .collect()
+                } else {
+                    field_names.iter().filter_map(|n| fields_map.get(n).cloned()).collect()
+                };
+
+                let inferred_ty = expr.ty;
+                let user_ty = user_ty.map(|ty| {
+                    this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
+                        span: source_info.span,
+                        user_ty: ty,
+                        inferred_ty,
+                    })
+                });
+                let adt = box AggregateKind::Adt(
+                    adt_def,
+                    variant_index,
+                    substs,
+                    user_ty,
+                    active_field_index,
+                );
+                this.cfg.push_assign(
+                    block,
+                    source_info,
+                    destination,
+                    Rvalue::Aggregate(adt, fields),
+                );
+                block.unit()
+            }
+            ExprKind::InlineAsm { template, operands, options, line_spans } => {
+                use crate::thir;
+                use rustc_middle::mir;
+                let operands = operands
+                    .into_iter()
+                    .map(|op| match op {
+                        thir::InlineAsmOperand::In { reg, expr } => mir::InlineAsmOperand::In {
+                            reg,
+                            value: unpack!(block = this.as_local_operand(block, expr)),
+                        },
+                        thir::InlineAsmOperand::Out { reg, late, expr } => {
+                            mir::InlineAsmOperand::Out {
+                                reg,
+                                late,
+                                place: expr.map(|expr| unpack!(block = this.as_place(block, expr))),
+                            }
+                        }
+                        thir::InlineAsmOperand::InOut { reg, late, expr } => {
+                            let place = unpack!(block = this.as_place(block, expr));
+                            mir::InlineAsmOperand::InOut {
+                                reg,
+                                late,
+                                // This works because asm operands must be Copy
+                                in_value: Operand::Copy(place),
+                                out_place: Some(place),
+                            }
+                        }
+                        thir::InlineAsmOperand::SplitInOut { reg, late, in_expr, out_expr } => {
+                            mir::InlineAsmOperand::InOut {
+                                reg,
+                                late,
+                                in_value: unpack!(block = this.as_local_operand(block, in_expr)),
+                                out_place: out_expr.map(|out_expr| {
+                                    unpack!(block = this.as_place(block, out_expr))
+                                }),
+                            }
+                        }
+                        thir::InlineAsmOperand::Const { expr } => mir::InlineAsmOperand::Const {
+                            value: unpack!(block = this.as_local_operand(block, expr)),
+                        },
+                        thir::InlineAsmOperand::SymFn { expr } => {
+                            mir::InlineAsmOperand::SymFn { value: box this.as_constant(expr) }
+                        }
+                        thir::InlineAsmOperand::SymStatic { def_id } => {
+                            mir::InlineAsmOperand::SymStatic { def_id }
+                        }
+                    })
+                    .collect();
+
+                let destination = this.cfg.start_new_block();
+
+                this.cfg.terminate(
+                    block,
+                    source_info,
+                    TerminatorKind::InlineAsm {
+                        template,
+                        operands,
+                        options,
+                        line_spans,
+                        destination: if options.contains(InlineAsmOptions::NORETURN) {
+                            None
+                        } else {
+                            Some(destination)
+                        },
+                    },
+                );
+                destination.unit()
+            }
+
+            // These cases don't actually need a destination
+            ExprKind::Assign { .. }
+            | ExprKind::AssignOp { .. }
+            | ExprKind::LlvmInlineAsm { .. } => {
+                unpack!(block = this.stmt_expr(block, expr, None));
+                this.cfg.push_assign_unit(block, source_info, destination, this.hir.tcx());
+                block.unit()
+            }
+
+            ExprKind::Continue { .. } | ExprKind::Break { .. } | ExprKind::Return { .. } => {
+                unpack!(block = this.stmt_expr(block, expr, None));
+                // No assign, as these have type `!`.
+                block.unit()
+            }
+
+            // Avoid creating a temporary
+            ExprKind::VarRef { .. }
+            | ExprKind::SelfRef
+            | ExprKind::PlaceTypeAscription { .. }
+            | ExprKind::ValueTypeAscription { .. } => {
+                debug_assert!(Category::of(&expr.kind) == Some(Category::Place));
+
+                let place = unpack!(block = this.as_place(block, expr));
+                let rvalue = Rvalue::Use(this.consume_by_copy_or_move(place));
+                this.cfg.push_assign(block, source_info, destination, rvalue);
+                block.unit()
+            }
+            ExprKind::Index { .. } | ExprKind::Deref { .. } | ExprKind::Field { .. } => {
+                debug_assert!(Category::of(&expr.kind) == Some(Category::Place));
+
+                // Create a "fake" temporary variable so that we check that the
+                // value is Sized. Usually, this is caught in type checking, but
+                // in the case of box expr there is no such check.
+                if !destination.projection.is_empty() {
+                    this.local_decls.push(LocalDecl::new(expr.ty, expr.span));
+                }
+
+                debug_assert!(Category::of(&expr.kind) == Some(Category::Place));
+
+                let place = unpack!(block = this.as_place(block, expr));
+                let rvalue = Rvalue::Use(this.consume_by_copy_or_move(place));
+                this.cfg.push_assign(block, source_info, destination, rvalue);
+                block.unit()
+            }
+
+            ExprKind::Yield { value } => {
+                let scope = this.local_scope();
+                let value = unpack!(block = this.as_operand(block, scope, value));
+                let resume = this.cfg.start_new_block();
+                let cleanup = this.generator_drop_cleanup();
+                this.cfg.terminate(
+                    block,
+                    source_info,
+                    TerminatorKind::Yield { value, resume, resume_arg: destination, drop: cleanup },
+                );
+                resume.unit()
+            }
+
+            // these are the cases that are more naturally handled by some other mode
+            ExprKind::Unary { .. }
+            | ExprKind::Binary { .. }
+            | ExprKind::Box { .. }
+            | ExprKind::Cast { .. }
+            | ExprKind::Pointer { .. }
+            | ExprKind::Repeat { .. }
+            | ExprKind::Array { .. }
+            | ExprKind::Tuple { .. }
+            | ExprKind::Closure { .. }
+            | ExprKind::Literal { .. }
+            | ExprKind::ThreadLocalRef(_)
+            | ExprKind::StaticRef { .. } => {
+                debug_assert!(match Category::of(&expr.kind).unwrap() {
+                    // should be handled above
+                    Category::Rvalue(RvalueFunc::Into) => false,
+
+                    // must be handled above or else we get an
+                    // infinite loop in the builder; see
+                    // e.g., `ExprKind::VarRef` above
+                    Category::Place => false,
+
+                    _ => true,
+                });
+
+                let rvalue = unpack!(block = this.as_local_rvalue(block, expr));
+                this.cfg.push_assign(block, source_info, destination, rvalue);
+                block.unit()
+            }
+        };
+
+        if !expr_is_block_or_scope {
+            let popped = this.block_context.pop();
+            assert!(popped.is_some());
+        }
+
+        block_and
+    }
+}

compiler/rustc_mir_build/src/build/expr/mod.rs

@@ -0,0 +1,70 @@
+//! Builds MIR from expressions. As a caller into this module, you
+//! have many options, but the first thing you have to decide is
+//! whether you are evaluating this expression for its *value*, its
+//! *location*, or as a *constant*.
+//!
+//! Typically, you want the value: e.g., if you are doing `expr_a +
+//! expr_b`, you want the values of those expressions. In that case,
+//! you want one of the following functions. Note that if the expr has
+//! a type that is not `Copy`, then using any of these functions will
+//! "move" the value out of its current home (if any).
+//!
+//! - `into` -- writes the value into a specific location, which
+//!   should be uninitialized
+//! - `as_operand` -- evaluates the value and yields an `Operand`,
+//!   suitable for use as an argument to an `Rvalue`
+//! - `as_temp` -- evaluates into a temporary; this is similar to `as_operand`
+//!   except it always returns a fresh place, even for constants
+//! - `as_rvalue` -- yields an `Rvalue`, suitable for use in an assignment;
+//!   as of this writing, never needed outside of the `expr` module itself
+//!
+//! Sometimes though want the expression's *location*. An example
+//! would be during a match statement, or the operand of the `&`
+//! operator. In that case, you want `as_place`. This will create a
+//! temporary if necessary.
+//!
+//! Finally, if it's a constant you seek, then call
+//! `as_constant`. This creates a `Constant<H>`, but naturally it can
+//! only be used on constant expressions and hence is needed only in
+//! very limited contexts.
+//!
+//! ### Implementation notes
+//!
+//! For any given kind of expression, there is generally one way that
+//! can be lowered most naturally. This is specified by the
+//! `Category::of` function in the `category` module. For example, a
+//! struct expression (or other expression that creates a new value)
+//! is typically easiest to write in terms of `as_rvalue` or `into`,
+//! whereas a reference to a field is easiest to write in terms of
+//! `as_place`. (The exception to this is scope and paren
+//! expressions, which have no category.)
+//!
+//! Therefore, the various functions above make use of one another in
+//! a descending fashion. For any given expression, you should pick
+//! the most suitable spot to implement it, and then just let the
+//! other fns cycle around. The handoff works like this:
+//!
+//! - `into(place)` -> fallback is to create a rvalue with `as_rvalue` and assign it to `place`
+//! - `as_rvalue` -> fallback is to create an Operand with `as_operand` and use `Rvalue::use`
+//! - `as_operand` -> either invokes `as_constant` or `as_temp`
+//! - `as_constant` -> (no fallback)
+//! - `as_temp` -> creates a temporary and either calls `as_place` or `into`
+//! - `as_place` -> for rvalues, falls back to `as_temp` and returns that
+//!
+//! As you can see, there is a cycle where `into` can (in theory) fallback to `as_temp`
+//! which can fallback to `into`. So if one of the `ExprKind` variants is not, in fact,
+//! implemented in the category where it is supposed to be, there will be a problem.
+//!
+//! Of those fallbacks, the most interesting one is `into`, because
+//! it discriminates based on the category of the expression. This is
+//! basically the point where the "by value" operations are bridged
+//! over to the "by reference" mode (`as_place`).
+
+mod as_constant;
+mod as_operand;
+mod as_place;
+mod as_rvalue;
+mod as_temp;
+mod category;
+mod into;
+mod stmt;

compiler/rustc_mir_build/src/build/expr/stmt.rs

@@ -0,0 +1,175 @@
+use crate::build::scope::BreakableTarget;
+use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder};
+use crate::thir::*;
+use rustc_middle::middle::region;
+use rustc_middle::mir::*;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Builds a block of MIR statements to evaluate the THIR `expr`.
+    /// If the original expression was an AST statement,
+    /// (e.g., `some().code(&here());`) then `opt_stmt_span` is the
+    /// span of that statement (including its semicolon, if any).
+    /// The scope is used if a statement temporary must be dropped.
+    crate fn stmt_expr(
+        &mut self,
+        mut block: BasicBlock,
+        expr: Expr<'tcx>,
+        statement_scope: Option<region::Scope>,
+    ) -> BlockAnd<()> {
+        let this = self;
+        let expr_span = expr.span;
+        let source_info = this.source_info(expr.span);
+        // Handle a number of expressions that don't need a destination at all. This
+        // avoids needing a mountain of temporary `()` variables.
+        let expr2 = expr.clone();
+        match expr.kind {
+            ExprKind::Scope { region_scope, lint_level, value } => {
+                let value = this.hir.mirror(value);
+                this.in_scope((region_scope, source_info), lint_level, |this| {
+                    this.stmt_expr(block, value, statement_scope)
+                })
+            }
+            ExprKind::Assign { lhs, rhs } => {
+                let lhs = this.hir.mirror(lhs);
+                let rhs = this.hir.mirror(rhs);
+                let lhs_span = lhs.span;
+
+                // Note: we evaluate assignments right-to-left. This
+                // is better for borrowck interaction with overloaded
+                // operators like x[j] = x[i].
+
+                debug!("stmt_expr Assign block_context.push(SubExpr) : {:?}", expr2);
+                this.block_context.push(BlockFrame::SubExpr);
+
+                // Generate better code for things that don't need to be
+                // dropped.
+                if this.hir.needs_drop(lhs.ty) {
+                    let rhs = unpack!(block = this.as_local_operand(block, rhs));
+                    let lhs = unpack!(block = this.as_place(block, lhs));
+                    unpack!(block = this.build_drop_and_replace(block, lhs_span, lhs, rhs));
+                } else {
+                    let rhs = unpack!(block = this.as_local_rvalue(block, rhs));
+                    let lhs = unpack!(block = this.as_place(block, lhs));
+                    this.cfg.push_assign(block, source_info, lhs, rhs);
+                }
+
+                this.block_context.pop();
+                block.unit()
+            }
+            ExprKind::AssignOp { op, lhs, rhs } => {
+                // FIXME(#28160) there is an interesting semantics
+                // question raised here -- should we "freeze" the
+                // value of the lhs here?  I'm inclined to think not,
+                // since it seems closer to the semantics of the
+                // overloaded version, which takes `&mut self`.  This
+                // only affects weird things like `x += {x += 1; x}`
+                // -- is that equal to `x + (x + 1)` or `2*(x+1)`?
+
+                let lhs = this.hir.mirror(lhs);
+                let lhs_ty = lhs.ty;
+
+                debug!("stmt_expr AssignOp block_context.push(SubExpr) : {:?}", expr2);
+                this.block_context.push(BlockFrame::SubExpr);
+
+                // As above, RTL.
+                let rhs = unpack!(block = this.as_local_operand(block, rhs));
+                let lhs = unpack!(block = this.as_place(block, lhs));
+
+                // we don't have to drop prior contents or anything
+                // because AssignOp is only legal for Copy types
+                // (overloaded ops should be desugared into a call).
+                let result = unpack!(
+                    block =
+                        this.build_binary_op(block, op, expr_span, lhs_ty, Operand::Copy(lhs), rhs)
+                );
+                this.cfg.push_assign(block, source_info, lhs, result);
+
+                this.block_context.pop();
+                block.unit()
+            }
+            ExprKind::Continue { label } => {
+                this.break_scope(block, None, BreakableTarget::Continue(label), source_info)
+            }
+            ExprKind::Break { label, value } => {
+                this.break_scope(block, value, BreakableTarget::Break(label), source_info)
+            }
+            ExprKind::Return { value } => {
+                this.break_scope(block, value, BreakableTarget::Return, source_info)
+            }
+            ExprKind::LlvmInlineAsm { asm, outputs, inputs } => {
+                debug!("stmt_expr LlvmInlineAsm block_context.push(SubExpr) : {:?}", expr2);
+                this.block_context.push(BlockFrame::SubExpr);
+                let outputs = outputs
+                    .into_iter()
+                    .map(|output| unpack!(block = this.as_place(block, output)))
+                    .collect::<Vec<_>>()
+                    .into_boxed_slice();
+                let inputs = inputs
+                    .into_iter()
+                    .map(|input| {
+                        (input.span(), unpack!(block = this.as_local_operand(block, input)))
+                    })
+                    .collect::<Vec<_>>()
+                    .into_boxed_slice();
+                this.cfg.push(
+                    block,
+                    Statement {
+                        source_info,
+                        kind: StatementKind::LlvmInlineAsm(box LlvmInlineAsm {
+                            asm: asm.clone(),
+                            outputs,
+                            inputs,
+                        }),
+                    },
+                );
+                this.block_context.pop();
+                block.unit()
+            }
+            _ => {
+                assert!(
+                    statement_scope.is_some(),
+                    "Should not be calling `stmt_expr` on a general expression \
+                     without a statement scope",
+                );
+
+                // Issue #54382: When creating temp for the value of
+                // expression like:
+                //
+                // `{ side_effects(); { let l = stuff(); the_value } }`
+                //
+                // it is usually better to focus on `the_value` rather
+                // than the entirety of block(s) surrounding it.
+                let adjusted_span = (|| {
+                    if let ExprKind::Block { body } = expr.kind {
+                        if let Some(tail_expr) = &body.expr {
+                            let mut expr = tail_expr;
+                            while let rustc_hir::ExprKind::Block(subblock, _label) = &expr.kind {
+                                if let Some(subtail_expr) = &subblock.expr {
+                                    expr = subtail_expr
+                                } else {
+                                    break;
+                                }
+                            }
+                            this.block_context.push(BlockFrame::TailExpr {
+                                tail_result_is_ignored: true,
+                                span: expr.span,
+                            });
+                            return Some(expr.span);
+                        }
+                    }
+                    None
+                })();
+
+                let temp =
+                    unpack!(block = this.as_temp(block, statement_scope, expr, Mutability::Not));
+
+                if let Some(span) = adjusted_span {
+                    this.local_decls[temp].source_info.span = span;
+                    this.block_context.pop();
+                }
+
+                block.unit()
+            }
+        }
+    }
+}

compiler/rustc_mir_build/src/build/into.rs

@@ -0,0 +1,55 @@
+//! In general, there are a number of things for which it's convenient
+//! to just call `builder.into` and have it emit its result into a
+//! given location. This is basically for expressions or things that can be
+//! wrapped up as expressions (e.g., blocks). To make this ergonomic, we use this
+//! latter `EvalInto` trait.
+
+use crate::build::{BlockAnd, Builder};
+use crate::thir::*;
+use rustc_middle::mir::*;
+
+pub(in crate::build) trait EvalInto<'tcx> {
+    fn eval_into(
+        self,
+        builder: &mut Builder<'_, 'tcx>,
+        destination: Place<'tcx>,
+        block: BasicBlock,
+    ) -> BlockAnd<()>;
+}
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    crate fn into<E>(
+        &mut self,
+        destination: Place<'tcx>,
+        block: BasicBlock,
+        expr: E,
+    ) -> BlockAnd<()>
+    where
+        E: EvalInto<'tcx>,
+    {
+        expr.eval_into(self, destination, block)
+    }
+}
+
+impl<'tcx> EvalInto<'tcx> for ExprRef<'tcx> {
+    fn eval_into(
+        self,
+        builder: &mut Builder<'_, 'tcx>,
+        destination: Place<'tcx>,
+        block: BasicBlock,
+    ) -> BlockAnd<()> {
+        let expr = builder.hir.mirror(self);
+        builder.into_expr(destination, block, expr)
+    }
+}
+
+impl<'tcx> EvalInto<'tcx> for Expr<'tcx> {
+    fn eval_into(
+        self,
+        builder: &mut Builder<'_, 'tcx>,
+        destination: Place<'tcx>,
+        block: BasicBlock,
+    ) -> BlockAnd<()> {
+        builder.into_expr(destination, block, self)
+    }
+}

compiler/rustc_mir_build/src/build/matches/simplify.rs

@@ -0,0 +1,258 @@
+//! Simplifying Candidates
+//!
+//! *Simplifying* a match pair `place @ pattern` means breaking it down
+//! into bindings or other, simpler match pairs. For example:
+//!
+//! - `place @ (P1, P2)` can be simplified to `[place.0 @ P1, place.1 @ P2]`
+//! - `place @ x` can be simplified to `[]` by binding `x` to `place`
+//!
+//! The `simplify_candidate` routine just repeatedly applies these
+//! sort of simplifications until there is nothing left to
+//! simplify. Match pairs cannot be simplified if they require some
+//! sort of test: for example, testing which variant an enum is, or
+//! testing a value against a constant.
+
+use crate::build::matches::{Ascription, Binding, Candidate, MatchPair};
+use crate::build::Builder;
+use crate::thir::{self, *};
+use rustc_attr::{SignedInt, UnsignedInt};
+use rustc_hir::RangeEnd;
+use rustc_middle::mir::interpret::truncate;
+use rustc_middle::mir::Place;
+use rustc_middle::ty;
+use rustc_middle::ty::layout::IntegerExt;
+use rustc_target::abi::{Integer, Size};
+
+use std::mem;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Simplify a candidate so that all match pairs require a test.
+    ///
+    /// This method will also split a candidate where the only match-pair is an
+    /// or-pattern into multiple candidates. This is so that
+    ///
+    /// match x {
+    ///     0 | 1 => { ... },
+    ///     2 | 3 => { ... },
+    /// }
+    ///
+    /// only generates a single switch. If this happens this method returns
+    /// `true`.
+    pub(super) fn simplify_candidate<'pat>(
+        &mut self,
+        candidate: &mut Candidate<'pat, 'tcx>,
+    ) -> bool {
+        // repeatedly simplify match pairs until fixed point is reached
+        loop {
+            let match_pairs = mem::take(&mut candidate.match_pairs);
+
+            if let [MatchPair { pattern: Pat { kind: box PatKind::Or { pats }, .. }, place }] =
+                *match_pairs
+            {
+                candidate.subcandidates = self.create_or_subcandidates(candidate, place, pats);
+                return true;
+            }
+
+            let mut changed = false;
+            for match_pair in match_pairs {
+                match self.simplify_match_pair(match_pair, candidate) {
+                    Ok(()) => {
+                        changed = true;
+                    }
+                    Err(match_pair) => {
+                        candidate.match_pairs.push(match_pair);
+                    }
+                }
+            }
+            if !changed {
+                // Move or-patterns to the end, because they can result in us
+                // creating additional candidates, so we want to test them as
+                // late as possible.
+                candidate
+                    .match_pairs
+                    .sort_by_key(|pair| matches!(*pair.pattern.kind, PatKind::Or { .. }));
+                return false; // if we were not able to simplify any, done.
+            }
+        }
+    }
+
+    /// Given `candidate` that has a single or-pattern for its match-pairs,
+    /// creates a fresh candidate for each of its input subpatterns passed via
+    /// `pats`.
+    fn create_or_subcandidates<'pat>(
+        &mut self,
+        candidate: &Candidate<'pat, 'tcx>,
+        place: Place<'tcx>,
+        pats: &'pat [Pat<'tcx>],
+    ) -> Vec<Candidate<'pat, 'tcx>> {
+        pats.iter()
+            .map(|pat| {
+                let mut candidate = Candidate::new(place, pat, candidate.has_guard);
+                self.simplify_candidate(&mut candidate);
+                candidate
+            })
+            .collect()
+    }
+
+    /// Tries to simplify `match_pair`, returning `Ok(())` if
+    /// successful. If successful, new match pairs and bindings will
+    /// have been pushed into the candidate. If no simplification is
+    /// possible, `Err` is returned and no changes are made to
+    /// candidate.
+    fn simplify_match_pair<'pat>(
+        &mut self,
+        match_pair: MatchPair<'pat, 'tcx>,
+        candidate: &mut Candidate<'pat, 'tcx>,
+    ) -> Result<(), MatchPair<'pat, 'tcx>> {
+        let tcx = self.hir.tcx();
+        match *match_pair.pattern.kind {
+            PatKind::AscribeUserType {
+                ref subpattern,
+                ascription: thir::pattern::Ascription { variance, user_ty, user_ty_span },
+            } => {
+                // Apply the type ascription to the value at `match_pair.place`, which is the
+                // value being matched, taking the variance field into account.
+                candidate.ascriptions.push(Ascription {
+                    span: user_ty_span,
+                    user_ty,
+                    source: match_pair.place,
+                    variance,
+                });
+
+                candidate.match_pairs.push(MatchPair::new(match_pair.place, subpattern));
+
+                Ok(())
+            }
+
+            PatKind::Wild => {
+                // nothing left to do
+                Ok(())
+            }
+
+            PatKind::Binding { name, mutability, mode, var, ty, ref subpattern, is_primary: _ } => {
+                candidate.bindings.push(Binding {
+                    name,
+                    mutability,
+                    span: match_pair.pattern.span,
+                    source: match_pair.place,
+                    var_id: var,
+                    var_ty: ty,
+                    binding_mode: mode,
+                });
+
+                if let Some(subpattern) = subpattern.as_ref() {
+                    // this is the `x @ P` case; have to keep matching against `P` now
+                    candidate.match_pairs.push(MatchPair::new(match_pair.place, subpattern));
+                }
+
+                Ok(())
+            }
+
+            PatKind::Constant { .. } => {
+                // FIXME normalize patterns when possible
+                Err(match_pair)
+            }
+
+            PatKind::Range(PatRange { lo, hi, end }) => {
+                let (range, bias) = match lo.ty.kind {
+                    ty::Char => {
+                        (Some(('\u{0000}' as u128, '\u{10FFFF}' as u128, Size::from_bits(32))), 0)
+                    }
+                    ty::Int(ity) => {
+                        let size = Integer::from_attr(&tcx, SignedInt(ity)).size();
+                        let max = truncate(u128::MAX, size);
+                        let bias = 1u128 << (size.bits() - 1);
+                        (Some((0, max, size)), bias)
+                    }
+                    ty::Uint(uty) => {
+                        let size = Integer::from_attr(&tcx, UnsignedInt(uty)).size();
+                        let max = truncate(u128::MAX, size);
+                        (Some((0, max, size)), 0)
+                    }
+                    _ => (None, 0),
+                };
+                if let Some((min, max, sz)) = range {
+                    if let (Some(lo), Some(hi)) = (lo.val.try_to_bits(sz), hi.val.try_to_bits(sz)) {
+                        // We want to compare ranges numerically, but the order of the bitwise
+                        // representation of signed integers does not match their numeric order.
+                        // Thus, to correct the ordering, we need to shift the range of signed
+                        // integers to correct the comparison. This is achieved by XORing with a
+                        // bias (see pattern/_match.rs for another pertinent example of this
+                        // pattern).
+                        let (lo, hi) = (lo ^ bias, hi ^ bias);
+                        if lo <= min && (hi > max || hi == max && end == RangeEnd::Included) {
+                            // Irrefutable pattern match.
+                            return Ok(());
+                        }
+                    }
+                }
+                Err(match_pair)
+            }
+
+            PatKind::Slice { ref prefix, ref slice, ref suffix } => {
+                if prefix.is_empty() && slice.is_some() && suffix.is_empty() {
+                    // irrefutable
+                    self.prefix_slice_suffix(
+                        &mut candidate.match_pairs,
+                        &match_pair.place,
+                        prefix,
+                        slice.as_ref(),
+                        suffix,
+                    );
+                    Ok(())
+                } else {
+                    Err(match_pair)
+                }
+            }
+
+            PatKind::Variant { adt_def, substs, variant_index, ref subpatterns } => {
+                let irrefutable = adt_def.variants.iter_enumerated().all(|(i, v)| {
+                    i == variant_index || {
+                        self.hir.tcx().features().exhaustive_patterns
+                            && !v
+                                .uninhabited_from(
+                                    self.hir.tcx(),
+                                    substs,
+                                    adt_def.adt_kind(),
+                                    self.hir.param_env,
+                                )
+                                .is_empty()
+                    }
+                }) && (adt_def.did.is_local()
+                    || !adt_def.is_variant_list_non_exhaustive());
+                if irrefutable {
+                    let place = tcx.mk_place_downcast(match_pair.place, adt_def, variant_index);
+                    candidate.match_pairs.extend(self.field_match_pairs(place, subpatterns));
+                    Ok(())
+                } else {
+                    Err(match_pair)
+                }
+            }
+
+            PatKind::Array { ref prefix, ref slice, ref suffix } => {
+                self.prefix_slice_suffix(
+                    &mut candidate.match_pairs,
+                    &match_pair.place,
+                    prefix,
+                    slice.as_ref(),
+                    suffix,
+                );
+                Ok(())
+            }
+
+            PatKind::Leaf { ref subpatterns } => {
+                // tuple struct, match subpats (if any)
+                candidate.match_pairs.extend(self.field_match_pairs(match_pair.place, subpatterns));
+                Ok(())
+            }
+
+            PatKind::Deref { ref subpattern } => {
+                let place = tcx.mk_place_deref(match_pair.place);
+                candidate.match_pairs.push(MatchPair::new(place, subpattern));
+                Ok(())
+            }
+
+            PatKind::Or { .. } => Err(match_pair),
+        }
+    }
+}

compiler/rustc_mir_build/src/build/matches/test.rs

@@ -0,0 +1,810 @@
+// Testing candidates
+//
+// After candidates have been simplified, the only match pairs that
+// remain are those that require some sort of test. The functions here
+// identify what tests are needed, perform the tests, and then filter
+// the candidates based on the result.
+
+use crate::build::matches::{Candidate, MatchPair, Test, TestKind};
+use crate::build::Builder;
+use crate::thir::pattern::compare_const_vals;
+use crate::thir::*;
+use rustc_data_structures::fx::FxIndexMap;
+use rustc_hir::{LangItem, RangeEnd};
+use rustc_index::bit_set::BitSet;
+use rustc_middle::mir::*;
+use rustc_middle::ty::util::IntTypeExt;
+use rustc_middle::ty::{self, adjustment::PointerCast, Ty};
+use rustc_span::symbol::sym;
+use rustc_target::abi::VariantIdx;
+
+use std::cmp::Ordering;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Identifies what test is needed to decide if `match_pair` is applicable.
+    ///
+    /// It is a bug to call this with a simplifiable pattern.
+    pub(super) fn test<'pat>(&mut self, match_pair: &MatchPair<'pat, 'tcx>) -> Test<'tcx> {
+        match *match_pair.pattern.kind {
+            PatKind::Variant { ref adt_def, substs: _, variant_index: _, subpatterns: _ } => Test {
+                span: match_pair.pattern.span,
+                kind: TestKind::Switch {
+                    adt_def,
+                    variants: BitSet::new_empty(adt_def.variants.len()),
+                },
+            },
+
+            PatKind::Constant { .. } if is_switch_ty(match_pair.pattern.ty) => {
+                // For integers, we use a `SwitchInt` match, which allows
+                // us to handle more cases.
+                Test {
+                    span: match_pair.pattern.span,
+                    kind: TestKind::SwitchInt {
+                        switch_ty: match_pair.pattern.ty,
+
+                        // these maps are empty to start; cases are
+                        // added below in add_cases_to_switch
+                        options: Default::default(),
+                    },
+                }
+            }
+
+            PatKind::Constant { value } => Test {
+                span: match_pair.pattern.span,
+                kind: TestKind::Eq { value, ty: match_pair.pattern.ty.clone() },
+            },
+
+            PatKind::Range(range) => {
+                assert_eq!(range.lo.ty, match_pair.pattern.ty);
+                assert_eq!(range.hi.ty, match_pair.pattern.ty);
+                Test { span: match_pair.pattern.span, kind: TestKind::Range(range) }
+            }
+
+            PatKind::Slice { ref prefix, ref slice, ref suffix } => {
+                let len = prefix.len() + suffix.len();
+                let op = if slice.is_some() { BinOp::Ge } else { BinOp::Eq };
+                Test { span: match_pair.pattern.span, kind: TestKind::Len { len: len as u64, op } }
+            }
+
+            PatKind::Or { .. } => bug!("or-patterns should have already been handled"),
+
+            PatKind::AscribeUserType { .. }
+            | PatKind::Array { .. }
+            | PatKind::Wild
+            | PatKind::Binding { .. }
+            | PatKind::Leaf { .. }
+            | PatKind::Deref { .. } => self.error_simplifyable(match_pair),
+        }
+    }
+
+    pub(super) fn add_cases_to_switch<'pat>(
+        &mut self,
+        test_place: &Place<'tcx>,
+        candidate: &Candidate<'pat, 'tcx>,
+        switch_ty: Ty<'tcx>,
+        options: &mut FxIndexMap<&'tcx ty::Const<'tcx>, u128>,
+    ) -> bool {
+        let match_pair = match candidate.match_pairs.iter().find(|mp| mp.place == *test_place) {
+            Some(match_pair) => match_pair,
+            _ => {
+                return false;
+            }
+        };
+
+        match *match_pair.pattern.kind {
+            PatKind::Constant { value } => {
+                options.entry(value).or_insert_with(|| {
+                    value.eval_bits(self.hir.tcx(), self.hir.param_env, switch_ty)
+                });
+                true
+            }
+            PatKind::Variant { .. } => {
+                panic!("you should have called add_variants_to_switch instead!");
+            }
+            PatKind::Range(range) => {
+                // Check that none of the switch values are in the range.
+                self.values_not_contained_in_range(range, options).unwrap_or(false)
+            }
+            PatKind::Slice { .. }
+            | PatKind::Array { .. }
+            | PatKind::Wild
+            | PatKind::Or { .. }
+            | PatKind::Binding { .. }
+            | PatKind::AscribeUserType { .. }
+            | PatKind::Leaf { .. }
+            | PatKind::Deref { .. } => {
+                // don't know how to add these patterns to a switch
+                false
+            }
+        }
+    }
+
+    pub(super) fn add_variants_to_switch<'pat>(
+        &mut self,
+        test_place: &Place<'tcx>,
+        candidate: &Candidate<'pat, 'tcx>,
+        variants: &mut BitSet<VariantIdx>,
+    ) -> bool {
+        let match_pair = match candidate.match_pairs.iter().find(|mp| mp.place == *test_place) {
+            Some(match_pair) => match_pair,
+            _ => {
+                return false;
+            }
+        };
+
+        match *match_pair.pattern.kind {
+            PatKind::Variant { adt_def: _, variant_index, .. } => {
+                // We have a pattern testing for variant `variant_index`
+                // set the corresponding index to true
+                variants.insert(variant_index);
+                true
+            }
+            _ => {
+                // don't know how to add these patterns to a switch
+                false
+            }
+        }
+    }
+
+    pub(super) fn perform_test(
+        &mut self,
+        block: BasicBlock,
+        place: Place<'tcx>,
+        test: &Test<'tcx>,
+        make_target_blocks: impl FnOnce(&mut Self) -> Vec<BasicBlock>,
+    ) {
+        debug!(
+            "perform_test({:?}, {:?}: {:?}, {:?})",
+            block,
+            place,
+            place.ty(&self.local_decls, self.hir.tcx()),
+            test
+        );
+
+        let source_info = self.source_info(test.span);
+        match test.kind {
+            TestKind::Switch { adt_def, ref variants } => {
+                let target_blocks = make_target_blocks(self);
+                // Variants is a BitVec of indexes into adt_def.variants.
+                let num_enum_variants = adt_def.variants.len();
+                let used_variants = variants.count();
+                debug_assert_eq!(target_blocks.len(), num_enum_variants + 1);
+                let otherwise_block = *target_blocks.last().unwrap();
+                let mut targets = Vec::with_capacity(used_variants + 1);
+                let mut values = Vec::with_capacity(used_variants);
+                let tcx = self.hir.tcx();
+                for (idx, discr) in adt_def.discriminants(tcx) {
+                    if variants.contains(idx) {
+                        debug_assert_ne!(
+                            target_blocks[idx.index()],
+                            otherwise_block,
+                            "no canididates for tested discriminant: {:?}",
+                            discr,
+                        );
+                        values.push(discr.val);
+                        targets.push(target_blocks[idx.index()]);
+                    } else {
+                        debug_assert_eq!(
+                            target_blocks[idx.index()],
+                            otherwise_block,
+                            "found canididates for untested discriminant: {:?}",
+                            discr,
+                        );
+                    }
+                }
+                targets.push(otherwise_block);
+                debug!(
+                    "num_enum_variants: {}, tested variants: {:?}, variants: {:?}",
+                    num_enum_variants, values, variants
+                );
+                let discr_ty = adt_def.repr.discr_type().to_ty(tcx);
+                let discr = self.temp(discr_ty, test.span);
+                self.cfg.push_assign(block, source_info, discr, Rvalue::Discriminant(place));
+                assert_eq!(values.len() + 1, targets.len());
+                self.cfg.terminate(
+                    block,
+                    source_info,
+                    TerminatorKind::SwitchInt {
+                        discr: Operand::Move(discr),
+                        switch_ty: discr_ty,
+                        values: From::from(values),
+                        targets,
+                    },
+                );
+            }
+
+            TestKind::SwitchInt { switch_ty, ref options } => {
+                let target_blocks = make_target_blocks(self);
+                let terminator = if switch_ty.kind == ty::Bool {
+                    assert!(!options.is_empty() && options.len() <= 2);
+                    if let [first_bb, second_bb] = *target_blocks {
+                        let (true_bb, false_bb) = match options[0] {
+                            1 => (first_bb, second_bb),
+                            0 => (second_bb, first_bb),
+                            v => span_bug!(test.span, "expected boolean value but got {:?}", v),
+                        };
+                        TerminatorKind::if_(self.hir.tcx(), Operand::Copy(place), true_bb, false_bb)
+                    } else {
+                        bug!("`TestKind::SwitchInt` on `bool` should have two targets")
+                    }
+                } else {
+                    // The switch may be inexhaustive so we have a catch all block
+                    debug_assert_eq!(options.len() + 1, target_blocks.len());
+                    TerminatorKind::SwitchInt {
+                        discr: Operand::Copy(place),
+                        switch_ty,
+                        values: options.values().copied().collect(),
+                        targets: target_blocks,
+                    }
+                };
+                self.cfg.terminate(block, source_info, terminator);
+            }
+
+            TestKind::Eq { value, ty } => {
+                if !ty.is_scalar() {
+                    // Use `PartialEq::eq` instead of `BinOp::Eq`
+                    // (the binop can only handle primitives)
+                    self.non_scalar_compare(
+                        block,
+                        make_target_blocks,
+                        source_info,
+                        value,
+                        place,
+                        ty,
+                    );
+                } else {
+                    if let [success, fail] = *make_target_blocks(self) {
+                        assert_eq!(value.ty, ty);
+                        let expect = self.literal_operand(test.span, value);
+                        let val = Operand::Copy(place);
+                        self.compare(block, success, fail, source_info, BinOp::Eq, expect, val);
+                    } else {
+                        bug!("`TestKind::Eq` should have two target blocks");
+                    }
+                }
+            }
+
+            TestKind::Range(PatRange { ref lo, ref hi, ref end }) => {
+                let lower_bound_success = self.cfg.start_new_block();
+                let target_blocks = make_target_blocks(self);
+
+                // Test `val` by computing `lo <= val && val <= hi`, using primitive comparisons.
+                let lo = self.literal_operand(test.span, lo);
+                let hi = self.literal_operand(test.span, hi);
+                let val = Operand::Copy(place);
+
+                if let [success, fail] = *target_blocks {
+                    self.compare(
+                        block,
+                        lower_bound_success,
+                        fail,
+                        source_info,
+                        BinOp::Le,
+                        lo,
+                        val.clone(),
+                    );
+                    let op = match *end {
+                        RangeEnd::Included => BinOp::Le,
+                        RangeEnd::Excluded => BinOp::Lt,
+                    };
+                    self.compare(lower_bound_success, success, fail, source_info, op, val, hi);
+                } else {
+                    bug!("`TestKind::Range` should have two target blocks");
+                }
+            }
+
+            TestKind::Len { len, op } => {
+                let target_blocks = make_target_blocks(self);
+
+                let usize_ty = self.hir.usize_ty();
+                let actual = self.temp(usize_ty, test.span);
+
+                // actual = len(place)
+                self.cfg.push_assign(block, source_info, actual, Rvalue::Len(place));
+
+                // expected = <N>
+                let expected = self.push_usize(block, source_info, len);
+
+                if let [true_bb, false_bb] = *target_blocks {
+                    // result = actual == expected OR result = actual < expected
+                    // branch based on result
+                    self.compare(
+                        block,
+                        true_bb,
+                        false_bb,
+                        source_info,
+                        op,
+                        Operand::Move(actual),
+                        Operand::Move(expected),
+                    );
+                } else {
+                    bug!("`TestKind::Len` should have two target blocks");
+                }
+            }
+        }
+    }
+
+    /// Compare using the provided built-in comparison operator
+    fn compare(
+        &mut self,
+        block: BasicBlock,
+        success_block: BasicBlock,
+        fail_block: BasicBlock,
+        source_info: SourceInfo,
+        op: BinOp,
+        left: Operand<'tcx>,
+        right: Operand<'tcx>,
+    ) {
+        let bool_ty = self.hir.bool_ty();
+        let result = self.temp(bool_ty, source_info.span);
+
+        // result = op(left, right)
+        self.cfg.push_assign(block, source_info, result, Rvalue::BinaryOp(op, left, right));
+
+        // branch based on result
+        self.cfg.terminate(
+            block,
+            source_info,
+            TerminatorKind::if_(self.hir.tcx(), Operand::Move(result), success_block, fail_block),
+        );
+    }
+
+    /// Compare two `&T` values using `<T as std::compare::PartialEq>::eq`
+    fn non_scalar_compare(
+        &mut self,
+        block: BasicBlock,
+        make_target_blocks: impl FnOnce(&mut Self) -> Vec<BasicBlock>,
+        source_info: SourceInfo,
+        value: &'tcx ty::Const<'tcx>,
+        place: Place<'tcx>,
+        mut ty: Ty<'tcx>,
+    ) {
+        let mut expect = self.literal_operand(source_info.span, value);
+        let mut val = Operand::Copy(place);
+
+        // If we're using `b"..."` as a pattern, we need to insert an
+        // unsizing coercion, as the byte string has the type `&[u8; N]`.
+        //
+        // We want to do this even when the scrutinee is a reference to an
+        // array, so we can call `<[u8]>::eq` rather than having to find an
+        // `<[u8; N]>::eq`.
+        let unsize = |ty: Ty<'tcx>| match ty.kind {
+            ty::Ref(region, rty, _) => match rty.kind {
+                ty::Array(inner_ty, n) => Some((region, inner_ty, n)),
+                _ => None,
+            },
+            _ => None,
+        };
+        let opt_ref_ty = unsize(ty);
+        let opt_ref_test_ty = unsize(value.ty);
+        match (opt_ref_ty, opt_ref_test_ty) {
+            // nothing to do, neither is an array
+            (None, None) => {}
+            (Some((region, elem_ty, _)), _) | (None, Some((region, elem_ty, _))) => {
+                let tcx = self.hir.tcx();
+                // make both a slice
+                ty = tcx.mk_imm_ref(region, tcx.mk_slice(elem_ty));
+                if opt_ref_ty.is_some() {
+                    let temp = self.temp(ty, source_info.span);
+                    self.cfg.push_assign(
+                        block,
+                        source_info,
+                        temp,
+                        Rvalue::Cast(CastKind::Pointer(PointerCast::Unsize), val, ty),
+                    );
+                    val = Operand::Move(temp);
+                }
+                if opt_ref_test_ty.is_some() {
+                    let slice = self.temp(ty, source_info.span);
+                    self.cfg.push_assign(
+                        block,
+                        source_info,
+                        slice,
+                        Rvalue::Cast(CastKind::Pointer(PointerCast::Unsize), expect, ty),
+                    );
+                    expect = Operand::Move(slice);
+                }
+            }
+        }
+
+        let deref_ty = match ty.kind {
+            ty::Ref(_, deref_ty, _) => deref_ty,
+            _ => bug!("non_scalar_compare called on non-reference type: {}", ty),
+        };
+
+        let eq_def_id = self.hir.tcx().require_lang_item(LangItem::PartialEq, None);
+        let method = self.hir.trait_method(eq_def_id, sym::eq, deref_ty, &[deref_ty.into()]);
+
+        let bool_ty = self.hir.bool_ty();
+        let eq_result = self.temp(bool_ty, source_info.span);
+        let eq_block = self.cfg.start_new_block();
+        let cleanup = self.diverge_cleanup();
+        self.cfg.terminate(
+            block,
+            source_info,
+            TerminatorKind::Call {
+                func: Operand::Constant(box Constant {
+                    span: source_info.span,
+
+                    // FIXME(#54571): This constant comes from user input (a
+                    // constant in a pattern).  Are there forms where users can add
+                    // type annotations here?  For example, an associated constant?
+                    // Need to experiment.
+                    user_ty: None,
+
+                    literal: method,
+                }),
+                args: vec![val, expect],
+                destination: Some((eq_result, eq_block)),
+                cleanup: Some(cleanup),
+                from_hir_call: false,
+                fn_span: source_info.span,
+            },
+        );
+
+        if let [success_block, fail_block] = *make_target_blocks(self) {
+            // check the result
+            self.cfg.terminate(
+                eq_block,
+                source_info,
+                TerminatorKind::if_(
+                    self.hir.tcx(),
+                    Operand::Move(eq_result),
+                    success_block,
+                    fail_block,
+                ),
+            );
+        } else {
+            bug!("`TestKind::Eq` should have two target blocks")
+        }
+    }
+
+    /// Given that we are performing `test` against `test_place`, this job
+    /// sorts out what the status of `candidate` will be after the test. See
+    /// `test_candidates` for the usage of this function. The returned index is
+    /// the index that this candidate should be placed in the
+    /// `target_candidates` vec. The candidate may be modified to update its
+    /// `match_pairs`.
+    ///
+    /// So, for example, if this candidate is `x @ Some(P0)` and the `Test` is
+    /// a variant test, then we would modify the candidate to be `(x as
+    /// Option).0 @ P0` and return the index corresponding to the variant
+    /// `Some`.
+    ///
+    /// However, in some cases, the test may just not be relevant to candidate.
+    /// For example, suppose we are testing whether `foo.x == 22`, but in one
+    /// match arm we have `Foo { x: _, ... }`... in that case, the test for
+    /// what value `x` has has no particular relevance to this candidate. In
+    /// such cases, this function just returns None without doing anything.
+    /// This is used by the overall `match_candidates` algorithm to structure
+    /// the match as a whole. See `match_candidates` for more details.
+    ///
+    /// FIXME(#29623). In some cases, we have some tricky choices to make.  for
+    /// example, if we are testing that `x == 22`, but the candidate is `x @
+    /// 13..55`, what should we do? In the event that the test is true, we know
+    /// that the candidate applies, but in the event of false, we don't know
+    /// that it *doesn't* apply. For now, we return false, indicate that the
+    /// test does not apply to this candidate, but it might be we can get
+    /// tighter match code if we do something a bit different.
+    pub(super) fn sort_candidate<'pat>(
+        &mut self,
+        test_place: &Place<'tcx>,
+        test: &Test<'tcx>,
+        candidate: &mut Candidate<'pat, 'tcx>,
+    ) -> Option<usize> {
+        // Find the match_pair for this place (if any). At present,
+        // afaik, there can be at most one. (In the future, if we
+        // adopted a more general `@` operator, there might be more
+        // than one, but it'd be very unusual to have two sides that
+        // both require tests; you'd expect one side to be simplified
+        // away.)
+        let (match_pair_index, match_pair) =
+            candidate.match_pairs.iter().enumerate().find(|&(_, mp)| mp.place == *test_place)?;
+
+        match (&test.kind, &*match_pair.pattern.kind) {
+            // If we are performing a variant switch, then this
+            // informs variant patterns, but nothing else.
+            (
+                &TestKind::Switch { adt_def: tested_adt_def, .. },
+                &PatKind::Variant { adt_def, variant_index, ref subpatterns, .. },
+            ) => {
+                assert_eq!(adt_def, tested_adt_def);
+                self.candidate_after_variant_switch(
+                    match_pair_index,
+                    adt_def,
+                    variant_index,
+                    subpatterns,
+                    candidate,
+                );
+                Some(variant_index.as_usize())
+            }
+
+            (&TestKind::Switch { .. }, _) => None,
+
+            // If we are performing a switch over integers, then this informs integer
+            // equality, but nothing else.
+            //
+            // FIXME(#29623) we could use PatKind::Range to rule
+            // things out here, in some cases.
+            (
+                &TestKind::SwitchInt { switch_ty: _, ref options },
+                &PatKind::Constant { ref value },
+            ) if is_switch_ty(match_pair.pattern.ty) => {
+                let index = options.get_index_of(value).unwrap();
+                self.candidate_without_match_pair(match_pair_index, candidate);
+                Some(index)
+            }
+
+            (&TestKind::SwitchInt { switch_ty: _, ref options }, &PatKind::Range(range)) => {
+                let not_contained =
+                    self.values_not_contained_in_range(range, options).unwrap_or(false);
+
+                if not_contained {
+                    // No switch values are contained in the pattern range,
+                    // so the pattern can be matched only if this test fails.
+                    let otherwise = options.len();
+                    Some(otherwise)
+                } else {
+                    None
+                }
+            }
+
+            (&TestKind::SwitchInt { .. }, _) => None,
+
+            (
+                &TestKind::Len { len: test_len, op: BinOp::Eq },
+                &PatKind::Slice { ref prefix, ref slice, ref suffix },
+            ) => {
+                let pat_len = (prefix.len() + suffix.len()) as u64;
+                match (test_len.cmp(&pat_len), slice) {
+                    (Ordering::Equal, &None) => {
+                        // on true, min_len = len = $actual_length,
+                        // on false, len != $actual_length
+                        self.candidate_after_slice_test(
+                            match_pair_index,
+                            candidate,
+                            prefix,
+                            slice.as_ref(),
+                            suffix,
+                        );
+                        Some(0)
+                    }
+                    (Ordering::Less, _) => {
+                        // test_len < pat_len. If $actual_len = test_len,
+                        // then $actual_len < pat_len and we don't have
+                        // enough elements.
+                        Some(1)
+                    }
+                    (Ordering::Equal | Ordering::Greater, &Some(_)) => {
+                        // This can match both if $actual_len = test_len >= pat_len,
+                        // and if $actual_len > test_len. We can't advance.
+                        None
+                    }
+                    (Ordering::Greater, &None) => {
+                        // test_len != pat_len, so if $actual_len = test_len, then
+                        // $actual_len != pat_len.
+                        Some(1)
+                    }
+                }
+            }
+
+            (
+                &TestKind::Len { len: test_len, op: BinOp::Ge },
+                &PatKind::Slice { ref prefix, ref slice, ref suffix },
+            ) => {
+                // the test is `$actual_len >= test_len`
+                let pat_len = (prefix.len() + suffix.len()) as u64;
+                match (test_len.cmp(&pat_len), slice) {
+                    (Ordering::Equal, &Some(_)) => {
+                        // $actual_len >= test_len = pat_len,
+                        // so we can match.
+                        self.candidate_after_slice_test(
+                            match_pair_index,
+                            candidate,
+                            prefix,
+                            slice.as_ref(),
+                            suffix,
+                        );
+                        Some(0)
+                    }
+                    (Ordering::Less, _) | (Ordering::Equal, &None) => {
+                        // test_len <= pat_len. If $actual_len < test_len,
+                        // then it is also < pat_len, so the test passing is
+                        // necessary (but insufficient).
+                        Some(0)
+                    }
+                    (Ordering::Greater, &None) => {
+                        // test_len > pat_len. If $actual_len >= test_len > pat_len,
+                        // then we know we won't have a match.
+                        Some(1)
+                    }
+                    (Ordering::Greater, &Some(_)) => {
+                        // test_len < pat_len, and is therefore less
+                        // strict. This can still go both ways.
+                        None
+                    }
+                }
+            }
+
+            (&TestKind::Range(test), &PatKind::Range(pat)) => {
+                if test == pat {
+                    self.candidate_without_match_pair(match_pair_index, candidate);
+                    return Some(0);
+                }
+
+                let no_overlap = (|| {
+                    use rustc_hir::RangeEnd::*;
+                    use std::cmp::Ordering::*;
+
+                    let tcx = self.hir.tcx();
+
+                    let test_ty = test.lo.ty;
+                    let lo = compare_const_vals(tcx, test.lo, pat.hi, self.hir.param_env, test_ty)?;
+                    let hi = compare_const_vals(tcx, test.hi, pat.lo, self.hir.param_env, test_ty)?;
+
+                    match (test.end, pat.end, lo, hi) {
+                        // pat < test
+                        (_, _, Greater, _) |
+                        (_, Excluded, Equal, _) |
+                        // pat > test
+                        (_, _, _, Less) |
+                        (Excluded, _, _, Equal) => Some(true),
+                        _ => Some(false),
+                    }
+                })();
+
+                if let Some(true) = no_overlap {
+                    // Testing range does not overlap with pattern range,
+                    // so the pattern can be matched only if this test fails.
+                    Some(1)
+                } else {
+                    None
+                }
+            }
+
+            (&TestKind::Range(range), &PatKind::Constant { value }) => {
+                if let Some(false) = self.const_range_contains(range, value) {
+                    // `value` is not contained in the testing range,
+                    // so `value` can be matched only if this test fails.
+                    Some(1)
+                } else {
+                    None
+                }
+            }
+
+            (&TestKind::Range { .. }, _) => None,
+
+            (&TestKind::Eq { .. } | &TestKind::Len { .. }, _) => {
+                // These are all binary tests.
+                //
+                // FIXME(#29623) we can be more clever here
+                let pattern_test = self.test(&match_pair);
+                if pattern_test.kind == test.kind {
+                    self.candidate_without_match_pair(match_pair_index, candidate);
+                    Some(0)
+                } else {
+                    None
+                }
+            }
+        }
+    }
+
+    fn candidate_without_match_pair(
+        &mut self,
+        match_pair_index: usize,
+        candidate: &mut Candidate<'_, 'tcx>,
+    ) {
+        candidate.match_pairs.remove(match_pair_index);
+    }
+
+    fn candidate_after_slice_test<'pat>(
+        &mut self,
+        match_pair_index: usize,
+        candidate: &mut Candidate<'pat, 'tcx>,
+        prefix: &'pat [Pat<'tcx>],
+        opt_slice: Option<&'pat Pat<'tcx>>,
+        suffix: &'pat [Pat<'tcx>],
+    ) {
+        let removed_place = candidate.match_pairs.remove(match_pair_index).place;
+        self.prefix_slice_suffix(
+            &mut candidate.match_pairs,
+            &removed_place,
+            prefix,
+            opt_slice,
+            suffix,
+        );
+    }
+
+    fn candidate_after_variant_switch<'pat>(
+        &mut self,
+        match_pair_index: usize,
+        adt_def: &'tcx ty::AdtDef,
+        variant_index: VariantIdx,
+        subpatterns: &'pat [FieldPat<'tcx>],
+        candidate: &mut Candidate<'pat, 'tcx>,
+    ) {
+        let match_pair = candidate.match_pairs.remove(match_pair_index);
+        let tcx = self.hir.tcx();
+
+        // So, if we have a match-pattern like `x @ Enum::Variant(P1, P2)`,
+        // we want to create a set of derived match-patterns like
+        // `(x as Variant).0 @ P1` and `(x as Variant).1 @ P1`.
+        let elem = ProjectionElem::Downcast(
+            Some(adt_def.variants[variant_index].ident.name),
+            variant_index,
+        );
+        let downcast_place = tcx.mk_place_elem(match_pair.place, elem); // `(x as Variant)`
+        let consequent_match_pairs = subpatterns.iter().map(|subpattern| {
+            // e.g., `(x as Variant).0`
+            let place = tcx.mk_place_field(downcast_place, subpattern.field, subpattern.pattern.ty);
+            // e.g., `(x as Variant).0 @ P1`
+            MatchPair::new(place, &subpattern.pattern)
+        });
+
+        candidate.match_pairs.extend(consequent_match_pairs);
+    }
+
+    fn error_simplifyable<'pat>(&mut self, match_pair: &MatchPair<'pat, 'tcx>) -> ! {
+        span_bug!(match_pair.pattern.span, "simplifyable pattern found: {:?}", match_pair.pattern)
+    }
+
+    fn const_range_contains(
+        &self,
+        range: PatRange<'tcx>,
+        value: &'tcx ty::Const<'tcx>,
+    ) -> Option<bool> {
+        use std::cmp::Ordering::*;
+
+        let tcx = self.hir.tcx();
+
+        let a = compare_const_vals(tcx, range.lo, value, self.hir.param_env, range.lo.ty)?;
+        let b = compare_const_vals(tcx, value, range.hi, self.hir.param_env, range.lo.ty)?;
+
+        match (b, range.end) {
+            (Less, _) | (Equal, RangeEnd::Included) if a != Greater => Some(true),
+            _ => Some(false),
+        }
+    }
+
+    fn values_not_contained_in_range(
+        &self,
+        range: PatRange<'tcx>,
+        options: &FxIndexMap<&'tcx ty::Const<'tcx>, u128>,
+    ) -> Option<bool> {
+        for &val in options.keys() {
+            if self.const_range_contains(range, val)? {
+                return Some(false);
+            }
+        }
+
+        Some(true)
+    }
+}
+
+impl Test<'_> {
+    pub(super) fn targets(&self) -> usize {
+        match self.kind {
+            TestKind::Eq { .. } | TestKind::Range(_) | TestKind::Len { .. } => 2,
+            TestKind::Switch { adt_def, .. } => {
+                // While the switch that we generate doesn't test for all
+                // variants, we have a target for each variant and the
+                // otherwise case, and we make sure that all of the cases not
+                // specified have the same block.
+                adt_def.variants.len() + 1
+            }
+            TestKind::SwitchInt { switch_ty, ref options, .. } => {
+                if switch_ty.is_bool() {
+                    // `bool` is special cased in `perform_test` to always
+                    // branch to two blocks.
+                    2
+                } else {
+                    options.len() + 1
+                }
+            }
+        }
+    }
+}
+
+fn is_switch_ty(ty: Ty<'_>) -> bool {
+    ty.is_integral() || ty.is_char() || ty.is_bool()
+}

compiler/rustc_mir_build/src/build/matches/util.rs

@@ -0,0 +1,100 @@
+use crate::build::matches::MatchPair;
+use crate::build::Builder;
+use crate::thir::*;
+use rustc_middle::mir::*;
+use rustc_middle::ty;
+use smallvec::SmallVec;
+use std::convert::TryInto;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    crate fn field_match_pairs<'pat>(
+        &mut self,
+        place: Place<'tcx>,
+        subpatterns: &'pat [FieldPat<'tcx>],
+    ) -> Vec<MatchPair<'pat, 'tcx>> {
+        subpatterns
+            .iter()
+            .map(|fieldpat| {
+                let place =
+                    self.hir.tcx().mk_place_field(place, fieldpat.field, fieldpat.pattern.ty);
+                MatchPair::new(place, &fieldpat.pattern)
+            })
+            .collect()
+    }
+
+    crate fn prefix_slice_suffix<'pat>(
+        &mut self,
+        match_pairs: &mut SmallVec<[MatchPair<'pat, 'tcx>; 1]>,
+        place: &Place<'tcx>,
+        prefix: &'pat [Pat<'tcx>],
+        opt_slice: Option<&'pat Pat<'tcx>>,
+        suffix: &'pat [Pat<'tcx>],
+    ) {
+        let tcx = self.hir.tcx();
+        let (min_length, exact_size) = match place.ty(&self.local_decls, tcx).ty.kind {
+            ty::Array(_, length) => {
+                (length.eval_usize(tcx, self.hir.param_env).try_into().unwrap(), true)
+            }
+            _ => ((prefix.len() + suffix.len()).try_into().unwrap(), false),
+        };
+
+        match_pairs.extend(prefix.iter().enumerate().map(|(idx, subpattern)| {
+            let elem =
+                ProjectionElem::ConstantIndex { offset: idx as u64, min_length, from_end: false };
+            let place = tcx.mk_place_elem(*place, elem);
+            MatchPair::new(place, subpattern)
+        }));
+
+        if let Some(subslice_pat) = opt_slice {
+            let suffix_len = suffix.len() as u64;
+            let subslice = tcx.mk_place_elem(
+                *place,
+                ProjectionElem::Subslice {
+                    from: prefix.len() as u64,
+                    to: if exact_size { min_length - suffix_len } else { suffix_len },
+                    from_end: !exact_size,
+                },
+            );
+            match_pairs.push(MatchPair::new(subslice, subslice_pat));
+        }
+
+        match_pairs.extend(suffix.iter().rev().enumerate().map(|(idx, subpattern)| {
+            let end_offset = (idx + 1) as u64;
+            let elem = ProjectionElem::ConstantIndex {
+                offset: if exact_size { min_length - end_offset } else { end_offset },
+                min_length,
+                from_end: !exact_size,
+            };
+            let place = tcx.mk_place_elem(*place, elem);
+            MatchPair::new(place, subpattern)
+        }));
+    }
+
+    /// Creates a false edge to `imaginary_target` and a real edge to
+    /// real_target. If `imaginary_target` is none, or is the same as the real
+    /// target, a Goto is generated instead to simplify the generated MIR.
+    crate fn false_edges(
+        &mut self,
+        from_block: BasicBlock,
+        real_target: BasicBlock,
+        imaginary_target: Option<BasicBlock>,
+        source_info: SourceInfo,
+    ) {
+        match imaginary_target {
+            Some(target) if target != real_target => {
+                self.cfg.terminate(
+                    from_block,
+                    source_info,
+                    TerminatorKind::FalseEdge { real_target, imaginary_target: target },
+                );
+            }
+            _ => self.cfg.goto(from_block, source_info, real_target),
+        }
+    }
+}
+
+impl<'pat, 'tcx> MatchPair<'pat, 'tcx> {
+    crate fn new(place: Place<'tcx>, pattern: &'pat Pat<'tcx>) -> MatchPair<'pat, 'tcx> {
+        MatchPair { place, pattern }
+    }
+}

compiler/rustc_mir_build/src/build/misc.rs

@@ -0,0 +1,75 @@
+//! Miscellaneous builder routines that are not specific to building any particular
+//! kind of thing.
+
+use crate::build::Builder;
+
+use rustc_middle::ty::{self, Ty};
+
+use rustc_middle::mir::*;
+use rustc_span::{Span, DUMMY_SP};
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Adds a new temporary value of type `ty` storing the result of
+    /// evaluating `expr`.
+    ///
+    /// N.B., **No cleanup is scheduled for this temporary.** You should
+    /// call `schedule_drop` once the temporary is initialized.
+    crate fn temp(&mut self, ty: Ty<'tcx>, span: Span) -> Place<'tcx> {
+        // Mark this local as internal to avoid temporaries with types not present in the
+        // user's code resulting in ICEs from the generator transform.
+        let temp = self.local_decls.push(LocalDecl::new(ty, span).internal());
+        let place = Place::from(temp);
+        debug!("temp: created temp {:?} with type {:?}", place, self.local_decls[temp].ty);
+        place
+    }
+
+    /// Convenience function for creating a literal operand, one
+    /// without any user type annotation.
+    crate fn literal_operand(
+        &mut self,
+        span: Span,
+        literal: &'tcx ty::Const<'tcx>,
+    ) -> Operand<'tcx> {
+        let constant = box Constant { span, user_ty: None, literal };
+        Operand::Constant(constant)
+    }
+
+    // Returns a zero literal operand for the appropriate type, works for
+    // bool, char and integers.
+    crate fn zero_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> {
+        let literal = ty::Const::from_bits(self.hir.tcx(), 0, ty::ParamEnv::empty().and(ty));
+
+        self.literal_operand(span, literal)
+    }
+
+    crate fn push_usize(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        value: u64,
+    ) -> Place<'tcx> {
+        let usize_ty = self.hir.usize_ty();
+        let temp = self.temp(usize_ty, source_info.span);
+        self.cfg.push_assign_constant(
+            block,
+            source_info,
+            temp,
+            Constant {
+                span: source_info.span,
+                user_ty: None,
+                literal: self.hir.usize_literal(value),
+            },
+        );
+        temp
+    }
+
+    crate fn consume_by_copy_or_move(&self, place: Place<'tcx>) -> Operand<'tcx> {
+        let tcx = self.hir.tcx();
+        let ty = place.ty(&self.local_decls, tcx).ty;
+        if !self.hir.type_is_copy_modulo_regions(ty, DUMMY_SP) {
+            Operand::Move(place)
+        } else {
+            Operand::Copy(place)
+        }
+    }
+}

compiler/rustc_mir_build/src/lib.rs

@@ -0,0 +1,29 @@
+//! Construction of MIR from HIR.
+//!
+//! This crate also contains the match exhaustiveness and usefulness checking.
+
+#![feature(box_patterns)]
+#![feature(box_syntax)]
+#![feature(const_fn)]
+#![feature(const_panic)]
+#![feature(crate_visibility_modifier)]
+#![feature(bool_to_option)]
+#![feature(or_patterns)]
+#![recursion_limit = "256"]
+
+#[macro_use]
+extern crate tracing;
+#[macro_use]
+extern crate rustc_middle;
+
+mod build;
+mod lints;
+mod thir;
+
+use rustc_middle::ty::query::Providers;
+
+pub fn provide(providers: &mut Providers) {
+    providers.check_match = thir::pattern::check_match;
+    providers.lit_to_const = thir::constant::lit_to_const;
+    providers.mir_built = build::mir_built;
+}

compiler/rustc_mir_build/src/lints.rs

@@ -0,0 +1,161 @@
+use rustc_data_structures::graph::iterate::{
+    ControlFlow, NodeStatus, TriColorDepthFirstSearch, TriColorVisitor,
+};
+use rustc_hir::def_id::LocalDefId;
+use rustc_hir::intravisit::FnKind;
+use rustc_middle::hir::map::blocks::FnLikeNode;
+use rustc_middle::mir::{BasicBlock, Body, Operand, TerminatorKind};
+use rustc_middle::ty::subst::{GenericArg, InternalSubsts};
+use rustc_middle::ty::{self, AssocItem, AssocItemContainer, Instance, TyCtxt};
+use rustc_session::lint::builtin::UNCONDITIONAL_RECURSION;
+use rustc_span::Span;
+
+crate fn check<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, def_id: LocalDefId) {
+    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+
+    if let Some(fn_like_node) = FnLikeNode::from_node(tcx.hir().get(hir_id)) {
+        if let FnKind::Closure(_) = fn_like_node.kind() {
+            // closures can't recur, so they don't matter.
+            return;
+        }
+
+        // If this is trait/impl method, extract the trait's substs.
+        let trait_substs = match tcx.opt_associated_item(def_id.to_def_id()) {
+            Some(AssocItem {
+                container: AssocItemContainer::TraitContainer(trait_def_id), ..
+            }) => {
+                let trait_substs_count = tcx.generics_of(*trait_def_id).count();
+                &InternalSubsts::identity_for_item(tcx, def_id.to_def_id())[..trait_substs_count]
+            }
+            _ => &[],
+        };
+
+        let mut vis = Search { tcx, body, def_id, reachable_recursive_calls: vec![], trait_substs };
+        if let Some(NonRecursive) = TriColorDepthFirstSearch::new(&body).run_from_start(&mut vis) {
+            return;
+        }
+
+        vis.reachable_recursive_calls.sort();
+
+        let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+        let sp = tcx.sess.source_map().guess_head_span(tcx.hir().span_with_body(hir_id));
+        tcx.struct_span_lint_hir(UNCONDITIONAL_RECURSION, hir_id, sp, |lint| {
+            let mut db = lint.build("function cannot return without recursing");
+            db.span_label(sp, "cannot return without recursing");
+            // offer some help to the programmer.
+            for call_span in vis.reachable_recursive_calls {
+                db.span_label(call_span, "recursive call site");
+            }
+            db.help("a `loop` may express intention better if this is on purpose");
+            db.emit();
+        });
+    }
+}
+
+struct NonRecursive;
+
+struct Search<'mir, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    body: &'mir Body<'tcx>,
+    def_id: LocalDefId,
+    trait_substs: &'tcx [GenericArg<'tcx>],
+
+    reachable_recursive_calls: Vec<Span>,
+}
+
+impl<'mir, 'tcx> Search<'mir, 'tcx> {
+    /// Returns `true` if `func` refers to the function we are searching in.
+    fn is_recursive_call(&self, func: &Operand<'tcx>) -> bool {
+        let Search { tcx, body, def_id, trait_substs, .. } = *self;
+        let param_env = tcx.param_env(def_id);
+
+        let func_ty = func.ty(body, tcx);
+        if let ty::FnDef(fn_def_id, substs) = func_ty.kind {
+            let (call_fn_id, call_substs) =
+                if let Ok(Some(instance)) = Instance::resolve(tcx, param_env, fn_def_id, substs) {
+                    (instance.def_id(), instance.substs)
+                } else {
+                    (fn_def_id, substs)
+                };
+
+            // FIXME(#57965): Make this work across function boundaries
+
+            // If this is a trait fn, the substs on the trait have to match, or we might be
+            // calling into an entirely different method (for example, a call from the default
+            // method in the trait to `<A as Trait<B>>::method`, where `A` and/or `B` are
+            // specific types).
+            return call_fn_id == def_id.to_def_id()
+                && &call_substs[..trait_substs.len()] == trait_substs;
+        }
+
+        false
+    }
+}
+
+impl<'mir, 'tcx> TriColorVisitor<&'mir Body<'tcx>> for Search<'mir, 'tcx> {
+    type BreakVal = NonRecursive;
+
+    fn node_examined(
+        &mut self,
+        bb: BasicBlock,
+        prior_status: Option<NodeStatus>,
+    ) -> ControlFlow<Self::BreakVal> {
+        // Back-edge in the CFG (loop).
+        if let Some(NodeStatus::Visited) = prior_status {
+            return ControlFlow::Break(NonRecursive);
+        }
+
+        match self.body[bb].terminator().kind {
+            // These terminators return control flow to the caller.
+            TerminatorKind::Abort
+            | TerminatorKind::GeneratorDrop
+            | TerminatorKind::Resume
+            | TerminatorKind::Return
+            | TerminatorKind::Unreachable
+            | TerminatorKind::Yield { .. } => ControlFlow::Break(NonRecursive),
+
+            // A diverging InlineAsm is treated as non-recursing
+            TerminatorKind::InlineAsm { destination, .. } => {
+                if destination.is_some() {
+                    ControlFlow::Continue
+                } else {
+                    ControlFlow::Break(NonRecursive)
+                }
+            }
+
+            // These do not.
+            TerminatorKind::Assert { .. }
+            | TerminatorKind::Call { .. }
+            | TerminatorKind::Drop { .. }
+            | TerminatorKind::DropAndReplace { .. }
+            | TerminatorKind::FalseEdge { .. }
+            | TerminatorKind::FalseUnwind { .. }
+            | TerminatorKind::Goto { .. }
+            | TerminatorKind::SwitchInt { .. } => ControlFlow::Continue,
+        }
+    }
+
+    fn node_settled(&mut self, bb: BasicBlock) -> ControlFlow<Self::BreakVal> {
+        // When we examine a node for the last time, remember it if it is a recursive call.
+        let terminator = self.body[bb].terminator();
+        if let TerminatorKind::Call { func, .. } = &terminator.kind {
+            if self.is_recursive_call(func) {
+                self.reachable_recursive_calls.push(terminator.source_info.span);
+            }
+        }
+
+        ControlFlow::Continue
+    }
+
+    fn ignore_edge(&mut self, bb: BasicBlock, target: BasicBlock) -> bool {
+        // Don't traverse successors of recursive calls or false CFG edges.
+        match self.body[bb].terminator().kind {
+            TerminatorKind::Call { ref func, .. } => self.is_recursive_call(func),
+
+            TerminatorKind::FalseUnwind { unwind: Some(imaginary_target), .. }
+            | TerminatorKind::FalseEdge { imaginary_target, .. } => imaginary_target == target,
+
+            _ => false,
+        }
+    }
+}

compiler/rustc_mir_build/src/thir/constant.rs

@@ -0,0 +1,84 @@
+use rustc_ast as ast;
+use rustc_middle::mir::interpret::{
+    truncate, Allocation, ConstValue, LitToConstError, LitToConstInput, Scalar,
+};
+use rustc_middle::ty::{self, ParamEnv, TyCtxt, TyS};
+use rustc_span::symbol::Symbol;
+use rustc_target::abi::Size;
+
+crate fn lit_to_const<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    lit_input: LitToConstInput<'tcx>,
+) -> Result<&'tcx ty::Const<'tcx>, LitToConstError> {
+    let LitToConstInput { lit, ty, neg } = lit_input;
+
+    let trunc = |n| {
+        let param_ty = ParamEnv::reveal_all().and(ty);
+        let width = tcx.layout_of(param_ty).map_err(|_| LitToConstError::Reported)?.size;
+        trace!("trunc {} with size {} and shift {}", n, width.bits(), 128 - width.bits());
+        let result = truncate(n, width);
+        trace!("trunc result: {}", result);
+        Ok(ConstValue::Scalar(Scalar::from_uint(result, width)))
+    };
+
+    let lit = match (lit, &ty.kind) {
+        (ast::LitKind::Str(s, _), ty::Ref(_, TyS { kind: ty::Str, .. }, _)) => {
+            let s = s.as_str();
+            let allocation = Allocation::from_byte_aligned_bytes(s.as_bytes());
+            let allocation = tcx.intern_const_alloc(allocation);
+            ConstValue::Slice { data: allocation, start: 0, end: s.len() }
+        }
+        (ast::LitKind::ByteStr(data), ty::Ref(_, TyS { kind: ty::Slice(_), .. }, _)) => {
+            let allocation = Allocation::from_byte_aligned_bytes(data as &Vec<u8>);
+            let allocation = tcx.intern_const_alloc(allocation);
+            ConstValue::Slice { data: allocation, start: 0, end: data.len() }
+        }
+        (ast::LitKind::ByteStr(data), ty::Ref(_, TyS { kind: ty::Array(_, _), .. }, _)) => {
+            let id = tcx.allocate_bytes(data);
+            ConstValue::Scalar(Scalar::Ptr(id.into()))
+        }
+        (ast::LitKind::Byte(n), ty::Uint(ast::UintTy::U8)) => {
+            ConstValue::Scalar(Scalar::from_uint(*n, Size::from_bytes(1)))
+        }
+        (ast::LitKind::Int(n, _), ty::Uint(_)) | (ast::LitKind::Int(n, _), ty::Int(_)) => {
+            trunc(if neg { (*n as i128).overflowing_neg().0 as u128 } else { *n })?
+        }
+        (ast::LitKind::Float(n, _), ty::Float(fty)) => {
+            parse_float(*n, *fty, neg).map_err(|_| LitToConstError::UnparseableFloat)?
+        }
+        (ast::LitKind::Bool(b), ty::Bool) => ConstValue::Scalar(Scalar::from_bool(*b)),
+        (ast::LitKind::Char(c), ty::Char) => ConstValue::Scalar(Scalar::from_char(*c)),
+        (ast::LitKind::Err(_), _) => return Err(LitToConstError::Reported),
+        _ => return Err(LitToConstError::TypeError),
+    };
+    Ok(ty::Const::from_value(tcx, lit, ty))
+}
+
+fn parse_float<'tcx>(num: Symbol, fty: ast::FloatTy, neg: bool) -> Result<ConstValue<'tcx>, ()> {
+    let num = num.as_str();
+    use rustc_apfloat::ieee::{Double, Single};
+    let scalar = match fty {
+        ast::FloatTy::F32 => {
+            num.parse::<f32>().map_err(|_| ())?;
+            let mut f = num.parse::<Single>().unwrap_or_else(|e| {
+                panic!("apfloat::ieee::Single failed to parse `{}`: {:?}", num, e)
+            });
+            if neg {
+                f = -f;
+            }
+            Scalar::from_f32(f)
+        }
+        ast::FloatTy::F64 => {
+            num.parse::<f64>().map_err(|_| ())?;
+            let mut f = num.parse::<Double>().unwrap_or_else(|e| {
+                panic!("apfloat::ieee::Double failed to parse `{}`: {:?}", num, e)
+            });
+            if neg {
+                f = -f;
+            }
+            Scalar::from_f64(f)
+        }
+    };
+
+    Ok(ConstValue::Scalar(scalar))
+}

compiler/rustc_mir_build/src/thir/cx/block.rs

@@ -0,0 +1,117 @@
+use crate::thir::cx::to_ref::ToRef;
+use crate::thir::cx::Cx;
+use crate::thir::{self, *};
+
+use rustc_hir as hir;
+use rustc_middle::middle::region;
+use rustc_middle::ty;
+
+use rustc_index::vec::Idx;
+
+impl<'tcx> Mirror<'tcx> for &'tcx hir::Block<'tcx> {
+    type Output = Block<'tcx>;
+
+    fn make_mirror(self, cx: &mut Cx<'_, 'tcx>) -> Block<'tcx> {
+        // We have to eagerly lower the "spine" of the statements
+        // in order to get the lexical scoping correctly.
+        let stmts = mirror_stmts(cx, self.hir_id.local_id, &*self.stmts);
+        let opt_destruction_scope =
+            cx.region_scope_tree.opt_destruction_scope(self.hir_id.local_id);
+        Block {
+            targeted_by_break: self.targeted_by_break,
+            region_scope: region::Scope { id: self.hir_id.local_id, data: region::ScopeData::Node },
+            opt_destruction_scope,
+            span: self.span,
+            stmts,
+            expr: self.expr.to_ref(),
+            safety_mode: match self.rules {
+                hir::BlockCheckMode::DefaultBlock => BlockSafety::Safe,
+                hir::BlockCheckMode::UnsafeBlock(..) => BlockSafety::ExplicitUnsafe(self.hir_id),
+                hir::BlockCheckMode::PushUnsafeBlock(..) => BlockSafety::PushUnsafe,
+                hir::BlockCheckMode::PopUnsafeBlock(..) => BlockSafety::PopUnsafe,
+            },
+        }
+    }
+}
+
+fn mirror_stmts<'a, 'tcx>(
+    cx: &mut Cx<'a, 'tcx>,
+    block_id: hir::ItemLocalId,
+    stmts: &'tcx [hir::Stmt<'tcx>],
+) -> Vec<StmtRef<'tcx>> {
+    let mut result = vec![];
+    for (index, stmt) in stmts.iter().enumerate() {
+        let hir_id = stmt.hir_id;
+        let opt_dxn_ext = cx.region_scope_tree.opt_destruction_scope(hir_id.local_id);
+        match stmt.kind {
+            hir::StmtKind::Expr(ref expr) | hir::StmtKind::Semi(ref expr) => {
+                result.push(StmtRef::Mirror(Box::new(Stmt {
+                    kind: StmtKind::Expr {
+                        scope: region::Scope { id: hir_id.local_id, data: region::ScopeData::Node },
+                        expr: expr.to_ref(),
+                    },
+                    opt_destruction_scope: opt_dxn_ext,
+                })))
+            }
+            hir::StmtKind::Item(..) => {
+                // ignore for purposes of the MIR
+            }
+            hir::StmtKind::Local(ref local) => {
+                let remainder_scope = region::Scope {
+                    id: block_id,
+                    data: region::ScopeData::Remainder(region::FirstStatementIndex::new(index)),
+                };
+
+                let mut pattern = cx.pattern_from_hir(&local.pat);
+
+                if let Some(ty) = &local.ty {
+                    if let Some(&user_ty) = cx.typeck_results.user_provided_types().get(ty.hir_id) {
+                        debug!("mirror_stmts: user_ty={:?}", user_ty);
+                        pattern = Pat {
+                            ty: pattern.ty,
+                            span: pattern.span,
+                            kind: Box::new(PatKind::AscribeUserType {
+                                ascription: thir::pattern::Ascription {
+                                    user_ty: PatTyProj::from_user_type(user_ty),
+                                    user_ty_span: ty.span,
+                                    variance: ty::Variance::Covariant,
+                                },
+                                subpattern: pattern,
+                            }),
+                        };
+                    }
+                }
+
+                result.push(StmtRef::Mirror(Box::new(Stmt {
+                    kind: StmtKind::Let {
+                        remainder_scope,
+                        init_scope: region::Scope {
+                            id: hir_id.local_id,
+                            data: region::ScopeData::Node,
+                        },
+                        pattern,
+                        initializer: local.init.to_ref(),
+                        lint_level: LintLevel::Explicit(local.hir_id),
+                    },
+                    opt_destruction_scope: opt_dxn_ext,
+                })));
+            }
+        }
+    }
+    result
+}
+
+crate fn to_expr_ref<'a, 'tcx>(
+    cx: &mut Cx<'a, 'tcx>,
+    block: &'tcx hir::Block<'tcx>,
+) -> ExprRef<'tcx> {
+    let block_ty = cx.typeck_results().node_type(block.hir_id);
+    let temp_lifetime = cx.region_scope_tree.temporary_scope(block.hir_id.local_id);
+    let expr = Expr {
+        ty: block_ty,
+        temp_lifetime,
+        span: block.span,
+        kind: ExprKind::Block { body: block },
+    };
+    expr.to_ref()
+}

compiler/rustc_mir_build/src/thir/cx/mod.rs

@@ -0,0 +1,218 @@
+//! This module contains the functionality to convert from the wacky tcx data
+//! structures into the THIR. The `builder` is generally ignorant of the tcx,
+//! etc., and instead goes through the `Cx` for most of its work.
+
+use crate::thir::util::UserAnnotatedTyHelpers;
+use crate::thir::*;
+
+use rustc_ast as ast;
+use rustc_hir as hir;
+use rustc_hir::def_id::{DefId, LocalDefId};
+use rustc_hir::Node;
+use rustc_index::vec::Idx;
+use rustc_infer::infer::InferCtxt;
+use rustc_middle::middle::region;
+use rustc_middle::mir::interpret::{LitToConstError, LitToConstInput};
+use rustc_middle::ty::subst::Subst;
+use rustc_middle::ty::subst::{GenericArg, InternalSubsts};
+use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_span::symbol::{sym, Symbol};
+use rustc_target::abi::VariantIdx;
+use rustc_trait_selection::infer::InferCtxtExt;
+
+#[derive(Clone)]
+crate struct Cx<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    infcx: &'a InferCtxt<'a, 'tcx>,
+
+    crate root_lint_level: hir::HirId,
+    crate param_env: ty::ParamEnv<'tcx>,
+
+    /// Identity `InternalSubsts` for use with const-evaluation.
+    crate identity_substs: &'tcx InternalSubsts<'tcx>,
+
+    crate region_scope_tree: &'tcx region::ScopeTree,
+    crate typeck_results: &'a ty::TypeckResults<'tcx>,
+
+    /// This is `Constness::Const` if we are compiling a `static`,
+    /// `const`, or the body of a `const fn`.
+    constness: hir::Constness,
+
+    /// The `DefId` of the owner of this body.
+    body_owner: DefId,
+
+    /// What kind of body is being compiled.
+    crate body_owner_kind: hir::BodyOwnerKind,
+
+    /// Whether this constant/function needs overflow checks.
+    check_overflow: bool,
+}
+
+impl<'a, 'tcx> Cx<'a, 'tcx> {
+    crate fn new(
+        infcx: &'a InferCtxt<'a, 'tcx>,
+        def: ty::WithOptConstParam<LocalDefId>,
+        src_id: hir::HirId,
+    ) -> Cx<'a, 'tcx> {
+        let tcx = infcx.tcx;
+        let typeck_results = tcx.typeck_opt_const_arg(def);
+        let body_owner_kind = tcx.hir().body_owner_kind(src_id);
+
+        let constness = match body_owner_kind {
+            hir::BodyOwnerKind::Const | hir::BodyOwnerKind::Static(_) => hir::Constness::Const,
+            hir::BodyOwnerKind::Closure | hir::BodyOwnerKind::Fn => hir::Constness::NotConst,
+        };
+
+        let attrs = tcx.hir().attrs(src_id);
+
+        // Some functions always have overflow checks enabled,
+        // however, they may not get codegen'd, depending on
+        // the settings for the crate they are codegened in.
+        let mut check_overflow = tcx.sess.contains_name(attrs, sym::rustc_inherit_overflow_checks);
+
+        // Respect -C overflow-checks.
+        check_overflow |= tcx.sess.overflow_checks();
+
+        // Constants always need overflow checks.
+        check_overflow |= constness == hir::Constness::Const;
+
+        Cx {
+            tcx,
+            infcx,
+            root_lint_level: src_id,
+            param_env: tcx.param_env(def.did),
+            identity_substs: InternalSubsts::identity_for_item(tcx, def.did.to_def_id()),
+            region_scope_tree: tcx.region_scope_tree(def.did),
+            typeck_results,
+            constness,
+            body_owner: def.did.to_def_id(),
+            body_owner_kind,
+            check_overflow,
+        }
+    }
+}
+
+impl<'a, 'tcx> Cx<'a, 'tcx> {
+    /// Normalizes `ast` into the appropriate "mirror" type.
+    crate fn mirror<M: Mirror<'tcx>>(&mut self, ast: M) -> M::Output {
+        ast.make_mirror(self)
+    }
+
+    crate fn usize_ty(&mut self) -> Ty<'tcx> {
+        self.tcx.types.usize
+    }
+
+    crate fn usize_literal(&mut self, value: u64) -> &'tcx ty::Const<'tcx> {
+        ty::Const::from_usize(self.tcx, value)
+    }
+
+    crate fn bool_ty(&mut self) -> Ty<'tcx> {
+        self.tcx.types.bool
+    }
+
+    crate fn unit_ty(&mut self) -> Ty<'tcx> {
+        self.tcx.mk_unit()
+    }
+
+    crate fn true_literal(&mut self) -> &'tcx ty::Const<'tcx> {
+        ty::Const::from_bool(self.tcx, true)
+    }
+
+    crate fn false_literal(&mut self) -> &'tcx ty::Const<'tcx> {
+        ty::Const::from_bool(self.tcx, false)
+    }
+
+    crate fn const_eval_literal(
+        &mut self,
+        lit: &'tcx ast::LitKind,
+        ty: Ty<'tcx>,
+        sp: Span,
+        neg: bool,
+    ) -> &'tcx ty::Const<'tcx> {
+        trace!("const_eval_literal: {:#?}, {:?}, {:?}, {:?}", lit, ty, sp, neg);
+
+        match self.tcx.at(sp).lit_to_const(LitToConstInput { lit, ty, neg }) {
+            Ok(c) => c,
+            Err(LitToConstError::UnparseableFloat) => {
+                // FIXME(#31407) this is only necessary because float parsing is buggy
+                self.tcx.sess.span_err(sp, "could not evaluate float literal (see issue #31407)");
+                // create a dummy value and continue compiling
+                Const::from_bits(self.tcx, 0, self.param_env.and(ty))
+            }
+            Err(LitToConstError::Reported) => {
+                // create a dummy value and continue compiling
+                Const::from_bits(self.tcx, 0, self.param_env.and(ty))
+            }
+            Err(LitToConstError::TypeError) => bug!("const_eval_literal: had type error"),
+        }
+    }
+
+    crate fn pattern_from_hir(&mut self, p: &hir::Pat<'_>) -> Pat<'tcx> {
+        let p = match self.tcx.hir().get(p.hir_id) {
+            Node::Pat(p) | Node::Binding(p) => p,
+            node => bug!("pattern became {:?}", node),
+        };
+        Pat::from_hir(self.tcx, self.param_env, self.typeck_results(), p)
+    }
+
+    crate fn trait_method(
+        &mut self,
+        trait_def_id: DefId,
+        method_name: Symbol,
+        self_ty: Ty<'tcx>,
+        params: &[GenericArg<'tcx>],
+    ) -> &'tcx ty::Const<'tcx> {
+        let substs = self.tcx.mk_substs_trait(self_ty, params);
+
+        // The unhygienic comparison here is acceptable because this is only
+        // used on known traits.
+        let item = self
+            .tcx
+            .associated_items(trait_def_id)
+            .filter_by_name_unhygienic(method_name)
+            .find(|item| item.kind == ty::AssocKind::Fn)
+            .expect("trait method not found");
+
+        let method_ty = self.tcx.type_of(item.def_id);
+        let method_ty = method_ty.subst(self.tcx, substs);
+        ty::Const::zero_sized(self.tcx, method_ty)
+    }
+
+    crate fn all_fields(&mut self, adt_def: &ty::AdtDef, variant_index: VariantIdx) -> Vec<Field> {
+        (0..adt_def.variants[variant_index].fields.len()).map(Field::new).collect()
+    }
+
+    crate fn needs_drop(&mut self, ty: Ty<'tcx>) -> bool {
+        ty.needs_drop(self.tcx, self.param_env)
+    }
+
+    crate fn tcx(&self) -> TyCtxt<'tcx> {
+        self.tcx
+    }
+
+    crate fn typeck_results(&self) -> &'a ty::TypeckResults<'tcx> {
+        self.typeck_results
+    }
+
+    crate fn check_overflow(&self) -> bool {
+        self.check_overflow
+    }
+
+    crate fn type_is_copy_modulo_regions(&self, ty: Ty<'tcx>, span: Span) -> bool {
+        self.infcx.type_is_copy_modulo_regions(self.param_env, ty, span)
+    }
+}
+
+impl<'tcx> UserAnnotatedTyHelpers<'tcx> for Cx<'_, 'tcx> {
+    fn tcx(&self) -> TyCtxt<'tcx> {
+        self.tcx()
+    }
+
+    fn typeck_results(&self) -> &ty::TypeckResults<'tcx> {
+        self.typeck_results()
+    }
+}
+
+mod block;
+mod expr;
+mod to_ref;

compiler/rustc_mir_build/src/thir/cx/to_ref.rs

@@ -0,0 +1,65 @@
+use crate::thir::*;
+
+use rustc_hir as hir;
+
+crate trait ToRef {
+    type Output;
+    fn to_ref(self) -> Self::Output;
+}
+
+impl<'tcx> ToRef for &'tcx hir::Expr<'tcx> {
+    type Output = ExprRef<'tcx>;
+
+    fn to_ref(self) -> ExprRef<'tcx> {
+        ExprRef::Thir(self)
+    }
+}
+
+impl<'tcx> ToRef for &'tcx &'tcx hir::Expr<'tcx> {
+    type Output = ExprRef<'tcx>;
+
+    fn to_ref(self) -> ExprRef<'tcx> {
+        ExprRef::Thir(&**self)
+    }
+}
+
+impl<'tcx> ToRef for Expr<'tcx> {
+    type Output = ExprRef<'tcx>;
+
+    fn to_ref(self) -> ExprRef<'tcx> {
+        ExprRef::Mirror(Box::new(self))
+    }
+}
+
+impl<'tcx, T, U> ToRef for &'tcx Option<T>
+where
+    &'tcx T: ToRef<Output = U>,
+{
+    type Output = Option<U>;
+
+    fn to_ref(self) -> Option<U> {
+        self.as_ref().map(|expr| expr.to_ref())
+    }
+}
+
+impl<'tcx, T, U> ToRef for &'tcx Vec<T>
+where
+    &'tcx T: ToRef<Output = U>,
+{
+    type Output = Vec<U>;
+
+    fn to_ref(self) -> Vec<U> {
+        self.iter().map(|expr| expr.to_ref()).collect()
+    }
+}
+
+impl<'tcx, T, U> ToRef for &'tcx [T]
+where
+    &'tcx T: ToRef<Output = U>,
+{
+    type Output = Vec<U>;
+
+    fn to_ref(self) -> Vec<U> {
+        self.iter().map(|expr| expr.to_ref()).collect()
+    }
+}

compiler/rustc_mir_build/src/thir/mod.rs

@@ -0,0 +1,448 @@
+//! The MIR is built from some typed high-level IR
+//! (THIR). This section defines the THIR along with a trait for
+//! accessing it. The intention is to allow MIR construction to be
+//! unit-tested and separated from the Rust source and compiler data
+//! structures.
+
+use self::cx::Cx;
+use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
+use rustc_hir as hir;
+use rustc_hir::def_id::DefId;
+use rustc_middle::infer::canonical::Canonical;
+use rustc_middle::middle::region;
+use rustc_middle::mir::{BinOp, BorrowKind, Field, UnOp};
+use rustc_middle::ty::adjustment::PointerCast;
+use rustc_middle::ty::subst::SubstsRef;
+use rustc_middle::ty::{AdtDef, Const, Ty, UpvarSubsts, UserType};
+use rustc_span::Span;
+use rustc_target::abi::VariantIdx;
+use rustc_target::asm::InlineAsmRegOrRegClass;
+
+crate mod constant;
+crate mod cx;
+
+crate mod pattern;
+crate use self::pattern::PatTyProj;
+crate use self::pattern::{BindingMode, FieldPat, Pat, PatKind, PatRange};
+
+mod util;
+
+#[derive(Copy, Clone, Debug)]
+crate enum LintLevel {
+    Inherited,
+    Explicit(hir::HirId),
+}
+
+#[derive(Clone, Debug)]
+crate struct Block<'tcx> {
+    crate targeted_by_break: bool,
+    crate region_scope: region::Scope,
+    crate opt_destruction_scope: Option<region::Scope>,
+    crate span: Span,
+    crate stmts: Vec<StmtRef<'tcx>>,
+    crate expr: Option<ExprRef<'tcx>>,
+    crate safety_mode: BlockSafety,
+}
+
+#[derive(Copy, Clone, Debug)]
+crate enum BlockSafety {
+    Safe,
+    ExplicitUnsafe(hir::HirId),
+    PushUnsafe,
+    PopUnsafe,
+}
+
+#[derive(Clone, Debug)]
+crate enum StmtRef<'tcx> {
+    Mirror(Box<Stmt<'tcx>>),
+}
+
+#[derive(Clone, Debug)]
+crate struct Stmt<'tcx> {
+    crate kind: StmtKind<'tcx>,
+    crate opt_destruction_scope: Option<region::Scope>,
+}
+
+#[derive(Clone, Debug)]
+crate enum StmtKind<'tcx> {
+    Expr {
+        /// scope for this statement; may be used as lifetime of temporaries
+        scope: region::Scope,
+
+        /// expression being evaluated in this statement
+        expr: ExprRef<'tcx>,
+    },
+
+    Let {
+        /// scope for variables bound in this let; covers this and
+        /// remaining statements in block
+        remainder_scope: region::Scope,
+
+        /// scope for the initialization itself; might be used as
+        /// lifetime of temporaries
+        init_scope: region::Scope,
+
+        /// `let <PAT> = ...`
+        ///
+        /// if a type is included, it is added as an ascription pattern
+        pattern: Pat<'tcx>,
+
+        /// let pat: ty = <INIT> ...
+        initializer: Option<ExprRef<'tcx>>,
+
+        /// the lint level for this let-statement
+        lint_level: LintLevel,
+    },
+}
+
+// `Expr` is used a lot. Make sure it doesn't unintentionally get bigger.
+#[cfg(target_arch = "x86_64")]
+rustc_data_structures::static_assert_size!(Expr<'_>, 168);
+
+/// The Thir trait implementor lowers their expressions (`&'tcx H::Expr`)
+/// into instances of this `Expr` enum. This lowering can be done
+/// basically as lazily or as eagerly as desired: every recursive
+/// reference to an expression in this enum is an `ExprRef<'tcx>`, which
+/// may in turn be another instance of this enum (boxed), or else an
+/// unlowered `&'tcx H::Expr`. Note that instances of `Expr` are very
+/// short-lived. They are created by `Thir::to_expr`, analyzed and
+/// converted into MIR, and then discarded.
+///
+/// If you compare `Expr` to the full compiler AST, you will see it is
+/// a good bit simpler. In fact, a number of the more straight-forward
+/// MIR simplifications are already done in the impl of `Thir`. For
+/// example, method calls and overloaded operators are absent: they are
+/// expected to be converted into `Expr::Call` instances.
+#[derive(Clone, Debug)]
+crate struct Expr<'tcx> {
+    /// type of this expression
+    crate ty: Ty<'tcx>,
+
+    /// lifetime of this expression if it should be spilled into a
+    /// temporary; should be None only if in a constant context
+    crate temp_lifetime: Option<region::Scope>,
+
+    /// span of the expression in the source
+    crate span: Span,
+
+    /// kind of expression
+    crate kind: ExprKind<'tcx>,
+}
+
+#[derive(Clone, Debug)]
+crate enum ExprKind<'tcx> {
+    Scope {
+        region_scope: region::Scope,
+        lint_level: LintLevel,
+        value: ExprRef<'tcx>,
+    },
+    Box {
+        value: ExprRef<'tcx>,
+    },
+    Call {
+        ty: Ty<'tcx>,
+        fun: ExprRef<'tcx>,
+        args: Vec<ExprRef<'tcx>>,
+        // Whether this is from a call in HIR, rather than from an overloaded
+        // operator. True for overloaded function call.
+        from_hir_call: bool,
+        /// This `Span` is the span of the function, without the dot and receiver
+        /// (e.g. `foo(a, b)` in `x.foo(a, b)`
+        fn_span: Span,
+    },
+    Deref {
+        arg: ExprRef<'tcx>,
+    }, // NOT overloaded!
+    Binary {
+        op: BinOp,
+        lhs: ExprRef<'tcx>,
+        rhs: ExprRef<'tcx>,
+    }, // NOT overloaded!
+    LogicalOp {
+        op: LogicalOp,
+        lhs: ExprRef<'tcx>,
+        rhs: ExprRef<'tcx>,
+    }, // NOT overloaded!
+    // LogicalOp is distinct from BinaryOp because of lazy evaluation of the operands.
+    Unary {
+        op: UnOp,
+        arg: ExprRef<'tcx>,
+    }, // NOT overloaded!
+    Cast {
+        source: ExprRef<'tcx>,
+    },
+    Use {
+        source: ExprRef<'tcx>,
+    }, // Use a lexpr to get a vexpr.
+    NeverToAny {
+        source: ExprRef<'tcx>,
+    },
+    Pointer {
+        cast: PointerCast,
+        source: ExprRef<'tcx>,
+    },
+    Loop {
+        body: ExprRef<'tcx>,
+    },
+    Match {
+        scrutinee: ExprRef<'tcx>,
+        arms: Vec<Arm<'tcx>>,
+    },
+    Block {
+        body: &'tcx hir::Block<'tcx>,
+    },
+    Assign {
+        lhs: ExprRef<'tcx>,
+        rhs: ExprRef<'tcx>,
+    },
+    AssignOp {
+        op: BinOp,
+        lhs: ExprRef<'tcx>,
+        rhs: ExprRef<'tcx>,
+    },
+    Field {
+        lhs: ExprRef<'tcx>,
+        name: Field,
+    },
+    Index {
+        lhs: ExprRef<'tcx>,
+        index: ExprRef<'tcx>,
+    },
+    VarRef {
+        id: hir::HirId,
+    },
+    /// first argument, used for self in a closure
+    SelfRef,
+    Borrow {
+        borrow_kind: BorrowKind,
+        arg: ExprRef<'tcx>,
+    },
+    /// A `&raw [const|mut] $place_expr` raw borrow resulting in type `*[const|mut] T`.
+    AddressOf {
+        mutability: hir::Mutability,
+        arg: ExprRef<'tcx>,
+    },
+    Break {
+        label: region::Scope,
+        value: Option<ExprRef<'tcx>>,
+    },
+    Continue {
+        label: region::Scope,
+    },
+    Return {
+        value: Option<ExprRef<'tcx>>,
+    },
+    Repeat {
+        value: ExprRef<'tcx>,
+        count: &'tcx Const<'tcx>,
+    },
+    Array {
+        fields: Vec<ExprRef<'tcx>>,
+    },
+    Tuple {
+        fields: Vec<ExprRef<'tcx>>,
+    },
+    Adt {
+        adt_def: &'tcx AdtDef,
+        variant_index: VariantIdx,
+        substs: SubstsRef<'tcx>,
+
+        /// Optional user-given substs: for something like `let x =
+        /// Bar::<T> { ... }`.
+        user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
+
+        fields: Vec<FieldExprRef<'tcx>>,
+        base: Option<FruInfo<'tcx>>,
+    },
+    PlaceTypeAscription {
+        source: ExprRef<'tcx>,
+        /// Type that the user gave to this expression
+        user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
+    },
+    ValueTypeAscription {
+        source: ExprRef<'tcx>,
+        /// Type that the user gave to this expression
+        user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
+    },
+    Closure {
+        closure_id: DefId,
+        substs: UpvarSubsts<'tcx>,
+        upvars: Vec<ExprRef<'tcx>>,
+        movability: Option<hir::Movability>,
+    },
+    Literal {
+        literal: &'tcx Const<'tcx>,
+        user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
+    },
+    /// A literal containing the address of a `static`.
+    ///
+    /// This is only distinguished from `Literal` so that we can register some
+    /// info for diagnostics.
+    StaticRef {
+        literal: &'tcx Const<'tcx>,
+        def_id: DefId,
+    },
+    InlineAsm {
+        template: &'tcx [InlineAsmTemplatePiece],
+        operands: Vec<InlineAsmOperand<'tcx>>,
+        options: InlineAsmOptions,
+        line_spans: &'tcx [Span],
+    },
+    /// An expression taking a reference to a thread local.
+    ThreadLocalRef(DefId),
+    LlvmInlineAsm {
+        asm: &'tcx hir::LlvmInlineAsmInner,
+        outputs: Vec<ExprRef<'tcx>>,
+        inputs: Vec<ExprRef<'tcx>>,
+    },
+    Yield {
+        value: ExprRef<'tcx>,
+    },
+}
+
+#[derive(Clone, Debug)]
+crate enum ExprRef<'tcx> {
+    Thir(&'tcx hir::Expr<'tcx>),
+    Mirror(Box<Expr<'tcx>>),
+}
+
+#[derive(Clone, Debug)]
+crate struct FieldExprRef<'tcx> {
+    crate name: Field,
+    crate expr: ExprRef<'tcx>,
+}
+
+#[derive(Clone, Debug)]
+crate struct FruInfo<'tcx> {
+    crate base: ExprRef<'tcx>,
+    crate field_types: Vec<Ty<'tcx>>,
+}
+
+#[derive(Clone, Debug)]
+crate struct Arm<'tcx> {
+    crate pattern: Pat<'tcx>,
+    crate guard: Option<Guard<'tcx>>,
+    crate body: ExprRef<'tcx>,
+    crate lint_level: LintLevel,
+    crate scope: region::Scope,
+    crate span: Span,
+}
+
+#[derive(Clone, Debug)]
+crate enum Guard<'tcx> {
+    If(ExprRef<'tcx>),
+}
+
+#[derive(Copy, Clone, Debug)]
+crate enum LogicalOp {
+    And,
+    Or,
+}
+
+impl<'tcx> ExprRef<'tcx> {
+    crate fn span(&self) -> Span {
+        match self {
+            ExprRef::Thir(expr) => expr.span,
+            ExprRef::Mirror(expr) => expr.span,
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+crate enum InlineAsmOperand<'tcx> {
+    In {
+        reg: InlineAsmRegOrRegClass,
+        expr: ExprRef<'tcx>,
+    },
+    Out {
+        reg: InlineAsmRegOrRegClass,
+        late: bool,
+        expr: Option<ExprRef<'tcx>>,
+    },
+    InOut {
+        reg: InlineAsmRegOrRegClass,
+        late: bool,
+        expr: ExprRef<'tcx>,
+    },
+    SplitInOut {
+        reg: InlineAsmRegOrRegClass,
+        late: bool,
+        in_expr: ExprRef<'tcx>,
+        out_expr: Option<ExprRef<'tcx>>,
+    },
+    Const {
+        expr: ExprRef<'tcx>,
+    },
+    SymFn {
+        expr: ExprRef<'tcx>,
+    },
+    SymStatic {
+        def_id: DefId,
+    },
+}
+
+///////////////////////////////////////////////////////////////////////////
+// The Mirror trait
+
+/// "Mirroring" is the process of converting from a HIR type into one
+/// of the THIR types defined in this file. This is basically a "on
+/// the fly" desugaring step that hides a lot of the messiness in the
+/// tcx. For example, the mirror of a `&'tcx hir::Expr` is an
+/// `Expr<'tcx>`.
+///
+/// Mirroring is gradual: when you mirror an outer expression like `e1
+/// + e2`, the references to the inner expressions `e1` and `e2` are
+/// `ExprRef<'tcx>` instances, and they may or may not be eagerly
+/// mirrored. This allows a single AST node from the compiler to
+/// expand into one or more Thir nodes, which lets the Thir nodes be
+/// simpler.
+crate trait Mirror<'tcx> {
+    type Output;
+
+    fn make_mirror(self, cx: &mut Cx<'_, 'tcx>) -> Self::Output;
+}
+
+impl<'tcx> Mirror<'tcx> for Expr<'tcx> {
+    type Output = Expr<'tcx>;
+
+    fn make_mirror(self, _: &mut Cx<'_, 'tcx>) -> Expr<'tcx> {
+        self
+    }
+}
+
+impl<'tcx> Mirror<'tcx> for ExprRef<'tcx> {
+    type Output = Expr<'tcx>;
+
+    fn make_mirror(self, hir: &mut Cx<'_, 'tcx>) -> Expr<'tcx> {
+        match self {
+            ExprRef::Thir(h) => h.make_mirror(hir),
+            ExprRef::Mirror(m) => *m,
+        }
+    }
+}
+
+impl<'tcx> Mirror<'tcx> for Stmt<'tcx> {
+    type Output = Stmt<'tcx>;
+
+    fn make_mirror(self, _: &mut Cx<'_, 'tcx>) -> Stmt<'tcx> {
+        self
+    }
+}
+
+impl<'tcx> Mirror<'tcx> for StmtRef<'tcx> {
+    type Output = Stmt<'tcx>;
+
+    fn make_mirror(self, _: &mut Cx<'_, 'tcx>) -> Stmt<'tcx> {
+        match self {
+            StmtRef::Mirror(m) => *m,
+        }
+    }
+}
+
+impl<'tcx> Mirror<'tcx> for Block<'tcx> {
+    type Output = Block<'tcx>;
+
+    fn make_mirror(self, _: &mut Cx<'_, 'tcx>) -> Block<'tcx> {
+        self
+    }
+}

compiler/rustc_mir_build/src/thir/pattern/check_match.rs

@@ -0,0 +1,818 @@
+use super::_match::Usefulness::*;
+use super::_match::WitnessPreference::*;
+use super::_match::{expand_pattern, is_useful, MatchCheckCtxt, Matrix, PatStack};
+use super::{PatCtxt, PatKind, PatternError};
+
+use rustc_arena::TypedArena;
+use rustc_ast::Mutability;
+use rustc_errors::{error_code, struct_span_err, Applicability, DiagnosticBuilder};
+use rustc_hir as hir;
+use rustc_hir::def::*;
+use rustc_hir::def_id::DefId;
+use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor};
+use rustc_hir::{HirId, Pat};
+use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_session::config::nightly_options;
+use rustc_session::lint::builtin::BINDINGS_WITH_VARIANT_NAME;
+use rustc_session::lint::builtin::{IRREFUTABLE_LET_PATTERNS, UNREACHABLE_PATTERNS};
+use rustc_session::parse::feature_err;
+use rustc_session::Session;
+use rustc_span::{sym, Span};
+use std::slice;
+
+crate fn check_match(tcx: TyCtxt<'_>, def_id: DefId) {
+    let body_id = match def_id.as_local() {
+        None => return,
+        Some(id) => tcx.hir().body_owned_by(tcx.hir().local_def_id_to_hir_id(id)),
+    };
+
+    let mut visitor = MatchVisitor {
+        tcx,
+        typeck_results: tcx.typeck_body(body_id),
+        param_env: tcx.param_env(def_id),
+        pattern_arena: TypedArena::default(),
+    };
+    visitor.visit_body(tcx.hir().body(body_id));
+}
+
+fn create_e0004(sess: &Session, sp: Span, error_message: String) -> DiagnosticBuilder<'_> {
+    struct_span_err!(sess, sp, E0004, "{}", &error_message)
+}
+
+struct MatchVisitor<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    typeck_results: &'a ty::TypeckResults<'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
+    pattern_arena: TypedArena<super::Pat<'tcx>>,
+}
+
+impl<'tcx> Visitor<'tcx> for MatchVisitor<'_, 'tcx> {
+    type Map = intravisit::ErasedMap<'tcx>;
+
+    fn nested_visit_map(&mut self) -> NestedVisitorMap<Self::Map> {
+        NestedVisitorMap::None
+    }
+
+    fn visit_expr(&mut self, ex: &'tcx hir::Expr<'tcx>) {
+        intravisit::walk_expr(self, ex);
+
+        if let hir::ExprKind::Match(ref scrut, ref arms, source) = ex.kind {
+            self.check_match(scrut, arms, source);
+        }
+    }
+
+    fn visit_local(&mut self, loc: &'tcx hir::Local<'tcx>) {
+        intravisit::walk_local(self, loc);
+
+        let (msg, sp) = match loc.source {
+            hir::LocalSource::Normal => ("local binding", Some(loc.span)),
+            hir::LocalSource::ForLoopDesugar => ("`for` loop binding", None),
+            hir::LocalSource::AsyncFn => ("async fn binding", None),
+            hir::LocalSource::AwaitDesugar => ("`await` future binding", None),
+        };
+        self.check_irrefutable(&loc.pat, msg, sp);
+        self.check_patterns(false, &loc.pat);
+    }
+
+    fn visit_param(&mut self, param: &'tcx hir::Param<'tcx>) {
+        intravisit::walk_param(self, param);
+        self.check_irrefutable(&param.pat, "function argument", None);
+        self.check_patterns(false, &param.pat);
+    }
+}
+
+impl PatCtxt<'_, '_> {
+    fn report_inlining_errors(&self, pat_span: Span) {
+        for error in &self.errors {
+            match *error {
+                PatternError::StaticInPattern(span) => {
+                    self.span_e0158(span, "statics cannot be referenced in patterns")
+                }
+                PatternError::AssocConstInPattern(span) => {
+                    self.span_e0158(span, "associated consts cannot be referenced in patterns")
+                }
+                PatternError::ConstParamInPattern(span) => {
+                    self.span_e0158(span, "const parameters cannot be referenced in patterns")
+                }
+                PatternError::FloatBug => {
+                    // FIXME(#31407) this is only necessary because float parsing is buggy
+                    ::rustc_middle::mir::interpret::struct_error(
+                        self.tcx.at(pat_span),
+                        "could not evaluate float literal (see issue #31407)",
+                    )
+                    .emit();
+                }
+                PatternError::NonConstPath(span) => {
+                    ::rustc_middle::mir::interpret::struct_error(
+                        self.tcx.at(span),
+                        "runtime values cannot be referenced in patterns",
+                    )
+                    .emit();
+                }
+            }
+        }
+    }
+
+    fn span_e0158(&self, span: Span, text: &str) {
+        struct_span_err!(self.tcx.sess, span, E0158, "{}", text).emit();
+    }
+}
+
+impl<'tcx> MatchVisitor<'_, 'tcx> {
+    fn check_patterns(&mut self, has_guard: bool, pat: &Pat<'_>) {
+        if !self.tcx.features().move_ref_pattern {
+            check_legality_of_move_bindings(self, has_guard, pat);
+        }
+        pat.walk_always(|pat| check_borrow_conflicts_in_at_patterns(self, pat));
+        if !self.tcx.features().bindings_after_at {
+            check_legality_of_bindings_in_at_patterns(self, pat);
+        }
+        check_for_bindings_named_same_as_variants(self, pat);
+    }
+
+    fn lower_pattern<'p>(
+        &self,
+        cx: &mut MatchCheckCtxt<'p, 'tcx>,
+        pat: &'tcx hir::Pat<'tcx>,
+        have_errors: &mut bool,
+    ) -> (&'p super::Pat<'tcx>, Ty<'tcx>) {
+        let mut patcx = PatCtxt::new(self.tcx, self.param_env, self.typeck_results);
+        patcx.include_lint_checks();
+        let pattern = patcx.lower_pattern(pat);
+        let pattern_ty = pattern.ty;
+        let pattern: &_ = cx.pattern_arena.alloc(expand_pattern(cx, pattern));
+        if !patcx.errors.is_empty() {
+            *have_errors = true;
+            patcx.report_inlining_errors(pat.span);
+        }
+        (pattern, pattern_ty)
+    }
+
+    fn new_cx(&self, hir_id: HirId) -> MatchCheckCtxt<'_, 'tcx> {
+        MatchCheckCtxt {
+            tcx: self.tcx,
+            param_env: self.param_env,
+            module: self.tcx.parent_module(hir_id).to_def_id(),
+            pattern_arena: &self.pattern_arena,
+        }
+    }
+
+    fn check_match(
+        &mut self,
+        scrut: &hir::Expr<'_>,
+        arms: &'tcx [hir::Arm<'tcx>],
+        source: hir::MatchSource,
+    ) {
+        for arm in arms {
+            // Check the arm for some things unrelated to exhaustiveness.
+            self.check_patterns(arm.guard.is_some(), &arm.pat);
+        }
+
+        let mut cx = self.new_cx(scrut.hir_id);
+
+        let mut have_errors = false;
+
+        let inlined_arms: Vec<_> = arms
+            .iter()
+            .map(|hir::Arm { pat, guard, .. }| {
+                (self.lower_pattern(&mut cx, pat, &mut have_errors).0, pat.hir_id, guard.is_some())
+            })
+            .collect();
+
+        // Bail out early if inlining failed.
+        if have_errors {
+            return;
+        }
+
+        // Fourth, check for unreachable arms.
+        let matrix = check_arms(&mut cx, &inlined_arms, source);
+
+        // Fifth, check if the match is exhaustive.
+        // Note: An empty match isn't the same as an empty matrix for diagnostics purposes,
+        // since an empty matrix can occur when there are arms, if those arms all have guards.
+        let scrut_ty = self.typeck_results.expr_ty_adjusted(scrut);
+        let is_empty_match = inlined_arms.is_empty();
+        check_exhaustive(&mut cx, scrut_ty, scrut.span, &matrix, scrut.hir_id, is_empty_match);
+    }
+
+    fn check_irrefutable(&self, pat: &'tcx Pat<'tcx>, origin: &str, sp: Option<Span>) {
+        let mut cx = self.new_cx(pat.hir_id);
+
+        let (pattern, pattern_ty) = self.lower_pattern(&mut cx, pat, &mut false);
+        let pats: Matrix<'_, '_> = vec![PatStack::from_pattern(pattern)].into_iter().collect();
+
+        let witnesses = match check_not_useful(&mut cx, pattern_ty, &pats, pat.hir_id) {
+            Ok(_) => return,
+            Err(err) => err,
+        };
+
+        let joined_patterns = joined_uncovered_patterns(&witnesses);
+        let mut err = struct_span_err!(
+            self.tcx.sess,
+            pat.span,
+            E0005,
+            "refutable pattern in {}: {} not covered",
+            origin,
+            joined_patterns
+        );
+        let suggest_if_let = match &pat.kind {
+            hir::PatKind::Path(hir::QPath::Resolved(None, path))
+                if path.segments.len() == 1 && path.segments[0].args.is_none() =>
+            {
+                const_not_var(&mut err, cx.tcx, pat, path);
+                false
+            }
+            _ => {
+                err.span_label(pat.span, pattern_not_covered_label(&witnesses, &joined_patterns));
+                true
+            }
+        };
+
+        if let (Some(span), true) = (sp, suggest_if_let) {
+            err.note(
+                "`let` bindings require an \"irrefutable pattern\", like a `struct` or \
+                 an `enum` with only one variant",
+            );
+            if let Ok(snippet) = self.tcx.sess.source_map().span_to_snippet(span) {
+                err.span_suggestion(
+                    span,
+                    "you might want to use `if let` to ignore the variant that isn't matched",
+                    format!("if {} {{ /* */ }}", &snippet[..snippet.len() - 1]),
+                    Applicability::HasPlaceholders,
+                );
+            }
+            err.note(
+                "for more information, visit \
+                 https://doc.rust-lang.org/book/ch18-02-refutability.html",
+            );
+        }
+
+        adt_defined_here(&cx, &mut err, pattern_ty, &witnesses);
+        err.note(&format!("the matched value is of type `{}`", pattern_ty));
+        err.emit();
+    }
+}
+
+/// A path pattern was interpreted as a constant, not a new variable.
+/// This caused an irrefutable match failure in e.g. `let`.
+fn const_not_var(
+    err: &mut DiagnosticBuilder<'_>,
+    tcx: TyCtxt<'_>,
+    pat: &Pat<'_>,
+    path: &hir::Path<'_>,
+) {
+    let descr = path.res.descr();
+    err.span_label(
+        pat.span,
+        format!("interpreted as {} {} pattern, not a new variable", path.res.article(), descr,),
+    );
+
+    err.span_suggestion(
+        pat.span,
+        "introduce a variable instead",
+        format!("{}_var", path.segments[0].ident).to_lowercase(),
+        // Cannot use `MachineApplicable` as it's not really *always* correct
+        // because there may be such an identifier in scope or the user maybe
+        // really wanted to match against the constant. This is quite unlikely however.
+        Applicability::MaybeIncorrect,
+    );
+
+    if let Some(span) = tcx.hir().res_span(path.res) {
+        err.span_label(span, format!("{} defined here", descr));
+    }
+}
+
+fn check_for_bindings_named_same_as_variants(cx: &MatchVisitor<'_, '_>, pat: &Pat<'_>) {
+    pat.walk_always(|p| {
+        if let hir::PatKind::Binding(_, _, ident, None) = p.kind {
+            if let Some(ty::BindByValue(hir::Mutability::Not)) =
+                cx.typeck_results.extract_binding_mode(cx.tcx.sess, p.hir_id, p.span)
+            {
+                let pat_ty = cx.typeck_results.pat_ty(p).peel_refs();
+                if let ty::Adt(edef, _) = pat_ty.kind {
+                    if edef.is_enum()
+                        && edef.variants.iter().any(|variant| {
+                            variant.ident == ident && variant.ctor_kind == CtorKind::Const
+                        })
+                    {
+                        cx.tcx.struct_span_lint_hir(
+                            BINDINGS_WITH_VARIANT_NAME,
+                            p.hir_id,
+                            p.span,
+                            |lint| {
+                                let ty_path = cx.tcx.def_path_str(edef.did);
+                                lint.build(&format!(
+                                    "pattern binding `{}` is named the same as one \
+                                                of the variants of the type `{}`",
+                                    ident, ty_path
+                                ))
+                                .code(error_code!(E0170))
+                                .span_suggestion(
+                                    p.span,
+                                    "to match on the variant, qualify the path",
+                                    format!("{}::{}", ty_path, ident),
+                                    Applicability::MachineApplicable,
+                                )
+                                .emit();
+                            },
+                        )
+                    }
+                }
+            }
+        }
+    });
+}
+
+/// Checks for common cases of "catchall" patterns that may not be intended as such.
+fn pat_is_catchall(pat: &super::Pat<'_>) -> bool {
+    use super::PatKind::*;
+    match &*pat.kind {
+        Binding { subpattern: None, .. } => true,
+        Binding { subpattern: Some(s), .. } | Deref { subpattern: s } => pat_is_catchall(s),
+        Leaf { subpatterns: s } => s.iter().all(|p| pat_is_catchall(&p.pattern)),
+        _ => false,
+    }
+}
+
+fn unreachable_pattern(tcx: TyCtxt<'_>, span: Span, id: HirId, catchall: Option<Span>) {
+    tcx.struct_span_lint_hir(UNREACHABLE_PATTERNS, id, span, |lint| {
+        let mut err = lint.build("unreachable pattern");
+        if let Some(catchall) = catchall {
+            // We had a catchall pattern, hint at that.
+            err.span_label(span, "unreachable pattern");
+            err.span_label(catchall, "matches any value");
+        }
+        err.emit();
+    });
+}
+
+fn irrefutable_let_pattern(tcx: TyCtxt<'_>, span: Span, id: HirId, source: hir::MatchSource) {
+    tcx.struct_span_lint_hir(IRREFUTABLE_LET_PATTERNS, id, span, |lint| {
+        let msg = match source {
+            hir::MatchSource::IfLetDesugar { .. } => "irrefutable if-let pattern",
+            hir::MatchSource::WhileLetDesugar => "irrefutable while-let pattern",
+            _ => bug!(),
+        };
+        lint.build(msg).emit()
+    });
+}
+
+/// Check for unreachable patterns.
+fn check_arms<'p, 'tcx>(
+    cx: &mut MatchCheckCtxt<'p, 'tcx>,
+    arms: &[(&'p super::Pat<'tcx>, HirId, bool)],
+    source: hir::MatchSource,
+) -> Matrix<'p, 'tcx> {
+    let mut seen = Matrix::empty();
+    let mut catchall = None;
+    for (arm_index, (pat, id, has_guard)) in arms.iter().copied().enumerate() {
+        let v = PatStack::from_pattern(pat);
+        match is_useful(cx, &seen, &v, LeaveOutWitness, id, has_guard, true) {
+            NotUseful => {
+                match source {
+                    hir::MatchSource::IfDesugar { .. } | hir::MatchSource::WhileDesugar => bug!(),
+
+                    hir::MatchSource::IfLetDesugar { .. } | hir::MatchSource::WhileLetDesugar => {
+                        // Check which arm we're on.
+                        match arm_index {
+                            // The arm with the user-specified pattern.
+                            0 => unreachable_pattern(cx.tcx, pat.span, id, None),
+                            // The arm with the wildcard pattern.
+                            1 => irrefutable_let_pattern(cx.tcx, pat.span, id, source),
+                            _ => bug!(),
+                        }
+                    }
+
+                    hir::MatchSource::ForLoopDesugar | hir::MatchSource::Normal => {
+                        unreachable_pattern(cx.tcx, pat.span, id, catchall);
+                    }
+
+                    // Unreachable patterns in try and await expressions occur when one of
+                    // the arms are an uninhabited type. Which is OK.
+                    hir::MatchSource::AwaitDesugar | hir::MatchSource::TryDesugar => {}
+                }
+            }
+            Useful(unreachable_subpatterns) => {
+                for span in unreachable_subpatterns {
+                    unreachable_pattern(cx.tcx, span, id, None);
+                }
+            }
+            UsefulWithWitness(_) => bug!(),
+        }
+        if !has_guard {
+            seen.push(v);
+            if catchall.is_none() && pat_is_catchall(pat) {
+                catchall = Some(pat.span);
+            }
+        }
+    }
+    seen
+}
+
+fn check_not_useful<'p, 'tcx>(
+    cx: &mut MatchCheckCtxt<'p, 'tcx>,
+    ty: Ty<'tcx>,
+    matrix: &Matrix<'p, 'tcx>,
+    hir_id: HirId,
+) -> Result<(), Vec<super::Pat<'tcx>>> {
+    let wild_pattern = cx.pattern_arena.alloc(super::Pat::wildcard_from_ty(ty));
+    let v = PatStack::from_pattern(wild_pattern);
+
+    // false is given for `is_under_guard` argument due to the wildcard
+    // pattern not having a guard
+    match is_useful(cx, matrix, &v, ConstructWitness, hir_id, false, true) {
+        NotUseful => Ok(()), // This is good, wildcard pattern isn't reachable.
+        UsefulWithWitness(pats) => Err(if pats.is_empty() {
+            bug!("Exhaustiveness check returned no witnesses")
+        } else {
+            pats.into_iter().map(|w| w.single_pattern()).collect()
+        }),
+        Useful(_) => bug!(),
+    }
+}
+
+fn check_exhaustive<'p, 'tcx>(
+    cx: &mut MatchCheckCtxt<'p, 'tcx>,
+    scrut_ty: Ty<'tcx>,
+    sp: Span,
+    matrix: &Matrix<'p, 'tcx>,
+    hir_id: HirId,
+    is_empty_match: bool,
+) {
+    // In the absence of the `exhaustive_patterns` feature, empty matches are not detected by
+    // `is_useful` to exhaustively match uninhabited types, so we manually check here.
+    if is_empty_match && !cx.tcx.features().exhaustive_patterns {
+        let scrutinee_is_visibly_uninhabited = match scrut_ty.kind {
+            ty::Never => true,
+            ty::Adt(def, _) => {
+                def.is_enum()
+                    && def.variants.is_empty()
+                    && !cx.is_foreign_non_exhaustive_enum(scrut_ty)
+            }
+            _ => false,
+        };
+        if scrutinee_is_visibly_uninhabited {
+            // If the type *is* uninhabited, an empty match is vacuously exhaustive.
+            return;
+        }
+    }
+
+    let witnesses = match check_not_useful(cx, scrut_ty, matrix, hir_id) {
+        Ok(_) => return,
+        Err(err) => err,
+    };
+
+    let non_empty_enum = match scrut_ty.kind {
+        ty::Adt(def, _) => def.is_enum() && !def.variants.is_empty(),
+        _ => false,
+    };
+    // In the case of an empty match, replace the '`_` not covered' diagnostic with something more
+    // informative.
+    let mut err;
+    if is_empty_match && !non_empty_enum {
+        err = create_e0004(
+            cx.tcx.sess,
+            sp,
+            format!("non-exhaustive patterns: type `{}` is non-empty", scrut_ty),
+        );
+    } else {
+        let joined_patterns = joined_uncovered_patterns(&witnesses);
+        err = create_e0004(
+            cx.tcx.sess,
+            sp,
+            format!("non-exhaustive patterns: {} not covered", joined_patterns),
+        );
+        err.span_label(sp, pattern_not_covered_label(&witnesses, &joined_patterns));
+    };
+
+    adt_defined_here(cx, &mut err, scrut_ty, &witnesses);
+    err.help(
+        "ensure that all possible cases are being handled, \
+              possibly by adding wildcards or more match arms",
+    );
+    err.note(&format!("the matched value is of type `{}`", scrut_ty));
+    if (scrut_ty == cx.tcx.types.usize || scrut_ty == cx.tcx.types.isize)
+        && !is_empty_match
+        && witnesses.len() == 1
+        && witnesses[0].is_wildcard()
+    {
+        err.note(&format!(
+            "`{}` does not have a fixed maximum value, \
+                so a wildcard `_` is necessary to match exhaustively",
+            scrut_ty,
+        ));
+        if nightly_options::is_nightly_build() {
+            err.help(&format!(
+                "add `#![feature(precise_pointer_size_matching)]` \
+                    to the crate attributes to enable precise `{}` matching",
+                scrut_ty,
+            ));
+        }
+    }
+    err.emit();
+}
+
+fn joined_uncovered_patterns(witnesses: &[super::Pat<'_>]) -> String {
+    const LIMIT: usize = 3;
+    match witnesses {
+        [] => bug!(),
+        [witness] => format!("`{}`", witness),
+        [head @ .., tail] if head.len() < LIMIT => {
+            let head: Vec<_> = head.iter().map(<_>::to_string).collect();
+            format!("`{}` and `{}`", head.join("`, `"), tail)
+        }
+        _ => {
+            let (head, tail) = witnesses.split_at(LIMIT);
+            let head: Vec<_> = head.iter().map(<_>::to_string).collect();
+            format!("`{}` and {} more", head.join("`, `"), tail.len())
+        }
+    }
+}
+
+fn pattern_not_covered_label(witnesses: &[super::Pat<'_>], joined_patterns: &str) -> String {
+    format!("pattern{} {} not covered", rustc_errors::pluralize!(witnesses.len()), joined_patterns)
+}
+
+/// Point at the definition of non-covered `enum` variants.
+fn adt_defined_here(
+    cx: &MatchCheckCtxt<'_, '_>,
+    err: &mut DiagnosticBuilder<'_>,
+    ty: Ty<'_>,
+    witnesses: &[super::Pat<'_>],
+) {
+    let ty = ty.peel_refs();
+    if let ty::Adt(def, _) = ty.kind {
+        if let Some(sp) = cx.tcx.hir().span_if_local(def.did) {
+            err.span_label(sp, format!("`{}` defined here", ty));
+        }
+
+        if witnesses.len() < 4 {
+            for sp in maybe_point_at_variant(ty, &witnesses) {
+                err.span_label(sp, "not covered");
+            }
+        }
+    }
+}
+
+fn maybe_point_at_variant(ty: Ty<'_>, patterns: &[super::Pat<'_>]) -> Vec<Span> {
+    let mut covered = vec![];
+    if let ty::Adt(def, _) = ty.kind {
+        // Don't point at variants that have already been covered due to other patterns to avoid
+        // visual clutter.
+        for pattern in patterns {
+            use PatKind::{AscribeUserType, Deref, Leaf, Or, Variant};
+            match &*pattern.kind {
+                AscribeUserType { subpattern, .. } | Deref { subpattern } => {
+                    covered.extend(maybe_point_at_variant(ty, slice::from_ref(&subpattern)));
+                }
+                Variant { adt_def, variant_index, subpatterns, .. } if adt_def.did == def.did => {
+                    let sp = def.variants[*variant_index].ident.span;
+                    if covered.contains(&sp) {
+                        continue;
+                    }
+                    covered.push(sp);
+
+                    let pats = subpatterns
+                        .iter()
+                        .map(|field_pattern| field_pattern.pattern.clone())
+                        .collect::<Box<[_]>>();
+                    covered.extend(maybe_point_at_variant(ty, &pats));
+                }
+                Leaf { subpatterns } => {
+                    let pats = subpatterns
+                        .iter()
+                        .map(|field_pattern| field_pattern.pattern.clone())
+                        .collect::<Box<[_]>>();
+                    covered.extend(maybe_point_at_variant(ty, &pats));
+                }
+                Or { pats } => {
+                    let pats = pats.iter().cloned().collect::<Box<[_]>>();
+                    covered.extend(maybe_point_at_variant(ty, &pats));
+                }
+                _ => {}
+            }
+        }
+    }
+    covered
+}
+
+/// Check if a by-value binding is by-value. That is, check if the binding's type is not `Copy`.
+fn is_binding_by_move(cx: &MatchVisitor<'_, '_>, hir_id: HirId, span: Span) -> bool {
+    !cx.typeck_results.node_type(hir_id).is_copy_modulo_regions(cx.tcx.at(span), cx.param_env)
+}
+
+/// Check the legality of legality of by-move bindings.
+fn check_legality_of_move_bindings(cx: &mut MatchVisitor<'_, '_>, has_guard: bool, pat: &Pat<'_>) {
+    let sess = cx.tcx.sess;
+    let typeck_results = cx.typeck_results;
+
+    // Find all by-ref spans.
+    let mut by_ref_spans = Vec::new();
+    pat.each_binding(|_, hir_id, span, _| {
+        if let Some(ty::BindByReference(_)) =
+            typeck_results.extract_binding_mode(sess, hir_id, span)
+        {
+            by_ref_spans.push(span);
+        }
+    });
+
+    // Find bad by-move spans:
+    let by_move_spans = &mut Vec::new();
+    let mut check_move = |p: &Pat<'_>, sub: Option<&Pat<'_>>| {
+        // Check legality of moving out of the enum.
+        //
+        // `x @ Foo(..)` is legal, but `x @ Foo(y)` isn't.
+        if sub.map_or(false, |p| p.contains_bindings()) {
+            struct_span_err!(sess, p.span, E0007, "cannot bind by-move with sub-bindings")
+                .span_label(p.span, "binds an already bound by-move value by moving it")
+                .emit();
+        } else if !has_guard && !by_ref_spans.is_empty() {
+            by_move_spans.push(p.span);
+        }
+    };
+    pat.walk_always(|p| {
+        if let hir::PatKind::Binding(.., sub) = &p.kind {
+            if let Some(ty::BindByValue(_)) =
+                typeck_results.extract_binding_mode(sess, p.hir_id, p.span)
+            {
+                if is_binding_by_move(cx, p.hir_id, p.span) {
+                    check_move(p, sub.as_deref());
+                }
+            }
+        }
+    });
+
+    // Found some bad by-move spans, error!
+    if !by_move_spans.is_empty() {
+        let mut err = feature_err(
+            &sess.parse_sess,
+            sym::move_ref_pattern,
+            by_move_spans.clone(),
+            "binding by-move and by-ref in the same pattern is unstable",
+        );
+        for span in by_ref_spans.iter() {
+            err.span_label(*span, "by-ref pattern here");
+        }
+        for span in by_move_spans.iter() {
+            err.span_label(*span, "by-move pattern here");
+        }
+        err.emit();
+    }
+}
+
+/// Check that there are no borrow or move conflicts in `binding @ subpat` patterns.
+///
+/// For example, this would reject:
+/// - `ref x @ Some(ref mut y)`,
+/// - `ref mut x @ Some(ref y)`,
+/// - `ref mut x @ Some(ref mut y)`,
+/// - `ref mut? x @ Some(y)`, and
+/// - `x @ Some(ref mut? y)`.
+///
+/// This analysis is *not* subsumed by NLL.
+fn check_borrow_conflicts_in_at_patterns(cx: &MatchVisitor<'_, '_>, pat: &Pat<'_>) {
+    // Extract `sub` in `binding @ sub`.
+    let (name, sub) = match &pat.kind {
+        hir::PatKind::Binding(.., name, Some(sub)) => (*name, sub),
+        _ => return,
+    };
+    let binding_span = pat.span.with_hi(name.span.hi());
+
+    let typeck_results = cx.typeck_results;
+    let sess = cx.tcx.sess;
+
+    // Get the binding move, extract the mutability if by-ref.
+    let mut_outer = match typeck_results.extract_binding_mode(sess, pat.hir_id, pat.span) {
+        Some(ty::BindByValue(_)) if is_binding_by_move(cx, pat.hir_id, pat.span) => {
+            // We have `x @ pat` where `x` is by-move. Reject all borrows in `pat`.
+            let mut conflicts_ref = Vec::new();
+            sub.each_binding(|_, hir_id, span, _| {
+                match typeck_results.extract_binding_mode(sess, hir_id, span) {
+                    Some(ty::BindByValue(_)) | None => {}
+                    Some(ty::BindByReference(_)) => conflicts_ref.push(span),
+                }
+            });
+            if !conflicts_ref.is_empty() {
+                let occurs_because = format!(
+                    "move occurs because `{}` has type `{}` which does not implement the `Copy` trait",
+                    name,
+                    typeck_results.node_type(pat.hir_id),
+                );
+                sess.struct_span_err(pat.span, "borrow of moved value")
+                    .span_label(binding_span, format!("value moved into `{}` here", name))
+                    .span_label(binding_span, occurs_because)
+                    .span_labels(conflicts_ref, "value borrowed here after move")
+                    .emit();
+            }
+            return;
+        }
+        Some(ty::BindByValue(_)) | None => return,
+        Some(ty::BindByReference(m)) => m,
+    };
+
+    // We now have `ref $mut_outer binding @ sub` (semantically).
+    // Recurse into each binding in `sub` and find mutability or move conflicts.
+    let mut conflicts_move = Vec::new();
+    let mut conflicts_mut_mut = Vec::new();
+    let mut conflicts_mut_ref = Vec::new();
+    sub.each_binding(|_, hir_id, span, name| {
+        match typeck_results.extract_binding_mode(sess, hir_id, span) {
+            Some(ty::BindByReference(mut_inner)) => match (mut_outer, mut_inner) {
+                (Mutability::Not, Mutability::Not) => {} // Both sides are `ref`.
+                (Mutability::Mut, Mutability::Mut) => conflicts_mut_mut.push((span, name)), // 2x `ref mut`.
+                _ => conflicts_mut_ref.push((span, name)), // `ref` + `ref mut` in either direction.
+            },
+            Some(ty::BindByValue(_)) if is_binding_by_move(cx, hir_id, span) => {
+                conflicts_move.push((span, name)) // `ref mut?` + by-move conflict.
+            }
+            Some(ty::BindByValue(_)) | None => {} // `ref mut?` + by-copy is fine.
+        }
+    });
+
+    // Report errors if any.
+    if !conflicts_mut_mut.is_empty() {
+        // Report mutability conflicts for e.g. `ref mut x @ Some(ref mut y)`.
+        let mut err = sess
+            .struct_span_err(pat.span, "cannot borrow value as mutable more than once at a time");
+        err.span_label(binding_span, format!("first mutable borrow, by `{}`, occurs here", name));
+        for (span, name) in conflicts_mut_mut {
+            err.span_label(span, format!("another mutable borrow, by `{}`, occurs here", name));
+        }
+        for (span, name) in conflicts_mut_ref {
+            err.span_label(span, format!("also borrowed as immutable, by `{}`, here", name));
+        }
+        for (span, name) in conflicts_move {
+            err.span_label(span, format!("also moved into `{}` here", name));
+        }
+        err.emit();
+    } else if !conflicts_mut_ref.is_empty() {
+        // Report mutability conflicts for e.g. `ref x @ Some(ref mut y)` or the converse.
+        let (primary, also) = match mut_outer {
+            Mutability::Mut => ("mutable", "immutable"),
+            Mutability::Not => ("immutable", "mutable"),
+        };
+        let msg =
+            format!("cannot borrow value as {} because it is also borrowed as {}", also, primary);
+        let mut err = sess.struct_span_err(pat.span, &msg);
+        err.span_label(binding_span, format!("{} borrow, by `{}`, occurs here", primary, name));
+        for (span, name) in conflicts_mut_ref {
+            err.span_label(span, format!("{} borrow, by `{}`, occurs here", also, name));
+        }
+        for (span, name) in conflicts_move {
+            err.span_label(span, format!("also moved into `{}` here", name));
+        }
+        err.emit();
+    } else if !conflicts_move.is_empty() {
+        // Report by-ref and by-move conflicts, e.g. `ref x @ y`.
+        let mut err =
+            sess.struct_span_err(pat.span, "cannot move out of value because it is borrowed");
+        err.span_label(binding_span, format!("value borrowed, by `{}`, here", name));
+        for (span, name) in conflicts_move {
+            err.span_label(span, format!("value moved into `{}` here", name));
+        }
+        err.emit();
+    }
+}
+
+/// Forbids bindings in `@` patterns. This used to be is necessary for memory safety,
+/// because of the way rvalues were handled in the borrow check. (See issue #14587.)
+fn check_legality_of_bindings_in_at_patterns(cx: &MatchVisitor<'_, '_>, pat: &Pat<'_>) {
+    AtBindingPatternVisitor { cx, bindings_allowed: true }.visit_pat(pat);
+
+    struct AtBindingPatternVisitor<'a, 'b, 'tcx> {
+        cx: &'a MatchVisitor<'b, 'tcx>,
+        bindings_allowed: bool,
+    }
+
+    impl<'v> Visitor<'v> for AtBindingPatternVisitor<'_, '_, '_> {
+        type Map = intravisit::ErasedMap<'v>;
+
+        fn nested_visit_map(&mut self) -> NestedVisitorMap<Self::Map> {
+            NestedVisitorMap::None
+        }
+
+        fn visit_pat(&mut self, pat: &Pat<'_>) {
+            match pat.kind {
+                hir::PatKind::Binding(.., ref subpat) => {
+                    if !self.bindings_allowed {
+                        feature_err(
+                            &self.cx.tcx.sess.parse_sess,
+                            sym::bindings_after_at,
+                            pat.span,
+                            "pattern bindings after an `@` are unstable",
+                        )
+                        .emit();
+                    }
+
+                    if subpat.is_some() {
+                        let bindings_were_allowed = self.bindings_allowed;
+                        self.bindings_allowed = false;
+                        intravisit::walk_pat(self, pat);
+                        self.bindings_allowed = bindings_were_allowed;
+                    }
+                }
+                _ => intravisit::walk_pat(self, pat),
+            }
+        }
+    }
+}

compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs

@@ -0,0 +1,306 @@
+use rustc_hir as hir;
+use rustc_index::vec::Idx;
+use rustc_infer::infer::{InferCtxt, TyCtxtInferExt};
+use rustc_middle::mir::Field;
+use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_session::lint;
+use rustc_span::Span;
+use rustc_trait_selection::traits::predicate_for_trait_def;
+use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt;
+use rustc_trait_selection::traits::{self, ObligationCause, PredicateObligation};
+
+use std::cell::Cell;
+
+use super::{FieldPat, Pat, PatCtxt, PatKind};
+
+impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
+    /// Converts an evaluated constant to a pattern (if possible).
+    /// This means aggregate values (like structs and enums) are converted
+    /// to a pattern that matches the value (as if you'd compared via structural equality).
+    pub(super) fn const_to_pat(
+        &self,
+        cv: &'tcx ty::Const<'tcx>,
+        id: hir::HirId,
+        span: Span,
+        mir_structural_match_violation: bool,
+    ) -> Pat<'tcx> {
+        debug!("const_to_pat: cv={:#?} id={:?}", cv, id);
+        debug!("const_to_pat: cv.ty={:?} span={:?}", cv.ty, span);
+
+        self.tcx.infer_ctxt().enter(|infcx| {
+            let mut convert = ConstToPat::new(self, id, span, infcx);
+            convert.to_pat(cv, mir_structural_match_violation)
+        })
+    }
+}
+
+struct ConstToPat<'a, 'tcx> {
+    id: hir::HirId,
+    span: Span,
+    param_env: ty::ParamEnv<'tcx>,
+
+    // This tracks if we signal some hard error for a given const value, so that
+    // we will not subsequently issue an irrelevant lint for the same const
+    // value.
+    saw_const_match_error: Cell<bool>,
+
+    // inference context used for checking `T: Structural` bounds.
+    infcx: InferCtxt<'a, 'tcx>,
+
+    include_lint_checks: bool,
+}
+
+impl<'a, 'tcx> ConstToPat<'a, 'tcx> {
+    fn new(
+        pat_ctxt: &PatCtxt<'_, 'tcx>,
+        id: hir::HirId,
+        span: Span,
+        infcx: InferCtxt<'a, 'tcx>,
+    ) -> Self {
+        ConstToPat {
+            id,
+            span,
+            infcx,
+            param_env: pat_ctxt.param_env,
+            include_lint_checks: pat_ctxt.include_lint_checks,
+            saw_const_match_error: Cell::new(false),
+        }
+    }
+
+    fn tcx(&self) -> TyCtxt<'tcx> {
+        self.infcx.tcx
+    }
+
+    fn search_for_structural_match_violation(
+        &self,
+        ty: Ty<'tcx>,
+    ) -> Option<traits::NonStructuralMatchTy<'tcx>> {
+        traits::search_for_structural_match_violation(self.id, self.span, self.tcx(), ty)
+    }
+
+    fn type_marked_structural(&self, ty: Ty<'tcx>) -> bool {
+        ty.is_structural_eq_shallow(self.infcx.tcx)
+    }
+
+    fn to_pat(
+        &mut self,
+        cv: &'tcx ty::Const<'tcx>,
+        mir_structural_match_violation: bool,
+    ) -> Pat<'tcx> {
+        // This method is just a wrapper handling a validity check; the heavy lifting is
+        // performed by the recursive `recur` method, which is not meant to be
+        // invoked except by this method.
+        //
+        // once indirect_structural_match is a full fledged error, this
+        // level of indirection can be eliminated
+
+        let inlined_const_as_pat = self.recur(cv);
+
+        if self.include_lint_checks && !self.saw_const_match_error.get() {
+            // If we were able to successfully convert the const to some pat,
+            // double-check that all types in the const implement `Structural`.
+
+            let structural = self.search_for_structural_match_violation(cv.ty);
+            debug!(
+                "search_for_structural_match_violation cv.ty: {:?} returned: {:?}",
+                cv.ty, structural
+            );
+
+            // This can occur because const qualification treats all associated constants as
+            // opaque, whereas `search_for_structural_match_violation` tries to monomorphize them
+            // before it runs.
+            //
+            // FIXME(#73448): Find a way to bring const qualification into parity with
+            // `search_for_structural_match_violation`.
+            if structural.is_none() && mir_structural_match_violation {
+                warn!("MIR const-checker found novel structural match violation. See #73448.");
+                return inlined_const_as_pat;
+            }
+
+            if let Some(non_sm_ty) = structural {
+                let msg = match non_sm_ty {
+                    traits::NonStructuralMatchTy::Adt(adt_def) => {
+                        let path = self.tcx().def_path_str(adt_def.did);
+                        format!(
+                            "to use a constant of type `{}` in a pattern, \
+                             `{}` must be annotated with `#[derive(PartialEq, Eq)]`",
+                            path, path,
+                        )
+                    }
+                    traits::NonStructuralMatchTy::Dynamic => {
+                        "trait objects cannot be used in patterns".to_string()
+                    }
+                    traits::NonStructuralMatchTy::Opaque => {
+                        "opaque types cannot be used in patterns".to_string()
+                    }
+                    traits::NonStructuralMatchTy::Generator => {
+                        "generators cannot be used in patterns".to_string()
+                    }
+                    traits::NonStructuralMatchTy::Closure => {
+                        "closures cannot be used in patterns".to_string()
+                    }
+                    traits::NonStructuralMatchTy::Param => {
+                        bug!("use of a constant whose type is a parameter inside a pattern")
+                    }
+                    traits::NonStructuralMatchTy::Projection => {
+                        bug!("use of a constant whose type is a projection inside a pattern")
+                    }
+                    traits::NonStructuralMatchTy::Foreign => {
+                        bug!("use of a value of a foreign type inside a pattern")
+                    }
+                };
+
+                // double-check there even *is* a semantic `PartialEq` to dispatch to.
+                //
+                // (If there isn't, then we can safely issue a hard
+                // error, because that's never worked, due to compiler
+                // using `PartialEq::eq` in this scenario in the past.)
+                //
+                // Note: To fix rust-lang/rust#65466, one could lift this check
+                // *before* any structural-match checking, and unconditionally error
+                // if `PartialEq` is not implemented. However, that breaks stable
+                // code at the moment, because types like `for <'a> fn(&'a ())` do
+                // not *yet* implement `PartialEq`. So for now we leave this here.
+                let ty_is_partial_eq: bool = {
+                    let partial_eq_trait_id =
+                        self.tcx().require_lang_item(hir::LangItem::PartialEq, Some(self.span));
+                    let obligation: PredicateObligation<'_> = predicate_for_trait_def(
+                        self.tcx(),
+                        self.param_env,
+                        ObligationCause::misc(self.span, self.id),
+                        partial_eq_trait_id,
+                        0,
+                        cv.ty,
+                        &[],
+                    );
+                    // FIXME: should this call a `predicate_must_hold` variant instead?
+                    self.infcx.predicate_may_hold(&obligation)
+                };
+
+                if !ty_is_partial_eq {
+                    // span_fatal avoids ICE from resolution of non-existent method (rare case).
+                    self.tcx().sess.span_fatal(self.span, &msg);
+                } else if mir_structural_match_violation {
+                    self.tcx().struct_span_lint_hir(
+                        lint::builtin::INDIRECT_STRUCTURAL_MATCH,
+                        self.id,
+                        self.span,
+                        |lint| lint.build(&msg).emit(),
+                    );
+                } else {
+                    debug!(
+                        "`search_for_structural_match_violation` found one, but `CustomEq` was \
+                          not in the qualifs for that `const`"
+                    );
+                }
+            }
+        }
+
+        inlined_const_as_pat
+    }
+
+    // Recursive helper for `to_pat`; invoke that (instead of calling this directly).
+    fn recur(&self, cv: &'tcx ty::Const<'tcx>) -> Pat<'tcx> {
+        let id = self.id;
+        let span = self.span;
+        let tcx = self.tcx();
+        let param_env = self.param_env;
+
+        let field_pats = |vals: &[&'tcx ty::Const<'tcx>]| {
+            vals.iter()
+                .enumerate()
+                .map(|(idx, val)| {
+                    let field = Field::new(idx);
+                    FieldPat { field, pattern: self.recur(val) }
+                })
+                .collect()
+        };
+
+        let kind = match cv.ty.kind {
+            ty::Float(_) => {
+                tcx.struct_span_lint_hir(
+                    lint::builtin::ILLEGAL_FLOATING_POINT_LITERAL_PATTERN,
+                    id,
+                    span,
+                    |lint| lint.build("floating-point types cannot be used in patterns").emit(),
+                );
+                PatKind::Constant { value: cv }
+            }
+            ty::Adt(adt_def, _) if adt_def.is_union() => {
+                // Matching on union fields is unsafe, we can't hide it in constants
+                self.saw_const_match_error.set(true);
+                tcx.sess.span_err(span, "cannot use unions in constant patterns");
+                PatKind::Wild
+            }
+            // keep old code until future-compat upgraded to errors.
+            ty::Adt(adt_def, _) if !self.type_marked_structural(cv.ty) => {
+                debug!("adt_def {:?} has !type_marked_structural for cv.ty: {:?}", adt_def, cv.ty);
+                let path = tcx.def_path_str(adt_def.did);
+                let msg = format!(
+                    "to use a constant of type `{}` in a pattern, \
+                     `{}` must be annotated with `#[derive(PartialEq, Eq)]`",
+                    path, path,
+                );
+                self.saw_const_match_error.set(true);
+                tcx.sess.span_err(span, &msg);
+                PatKind::Wild
+            }
+            // keep old code until future-compat upgraded to errors.
+            ty::Ref(_, adt_ty @ ty::TyS { kind: ty::Adt(_, _), .. }, _)
+                if !self.type_marked_structural(adt_ty) =>
+            {
+                let adt_def =
+                    if let ty::Adt(adt_def, _) = adt_ty.kind { adt_def } else { unreachable!() };
+
+                debug!(
+                    "adt_def {:?} has !type_marked_structural for adt_ty: {:?}",
+                    adt_def, adt_ty
+                );
+
+                // HACK(estebank): Side-step ICE #53708, but anything other than erroring here
+                // would be wrong. Returnging `PatKind::Wild` is not technically correct.
+                let path = tcx.def_path_str(adt_def.did);
+                let msg = format!(
+                    "to use a constant of type `{}` in a pattern, \
+                     `{}` must be annotated with `#[derive(PartialEq, Eq)]`",
+                    path, path,
+                );
+                self.saw_const_match_error.set(true);
+                tcx.sess.span_err(span, &msg);
+                PatKind::Wild
+            }
+            ty::Adt(adt_def, substs) if adt_def.is_enum() => {
+                let destructured = tcx.destructure_const(param_env.and(cv));
+                PatKind::Variant {
+                    adt_def,
+                    substs,
+                    variant_index: destructured
+                        .variant
+                        .expect("destructed const of adt without variant id"),
+                    subpatterns: field_pats(destructured.fields),
+                }
+            }
+            ty::Adt(_, _) => {
+                let destructured = tcx.destructure_const(param_env.and(cv));
+                PatKind::Leaf { subpatterns: field_pats(destructured.fields) }
+            }
+            ty::Tuple(_) => {
+                let destructured = tcx.destructure_const(param_env.and(cv));
+                PatKind::Leaf { subpatterns: field_pats(destructured.fields) }
+            }
+            ty::Array(..) => PatKind::Array {
+                prefix: tcx
+                    .destructure_const(param_env.and(cv))
+                    .fields
+                    .iter()
+                    .map(|val| self.recur(val))
+                    .collect(),
+                slice: None,
+                suffix: Vec::new(),
+            },
+            _ => PatKind::Constant { value: cv },
+        };
+
+        Pat { span, ty: cv.ty, kind: Box::new(kind) }
+    }
+}

compiler/rustc_mir_build/src/thir/util.rs

@@ -0,0 +1,31 @@
+use rustc_hir as hir;
+use rustc_middle::ty::{self, CanonicalUserType, TyCtxt, UserType};
+
+crate trait UserAnnotatedTyHelpers<'tcx> {
+    fn tcx(&self) -> TyCtxt<'tcx>;
+
+    fn typeck_results(&self) -> &ty::TypeckResults<'tcx>;
+
+    /// Looks up the type associated with this hir-id and applies the
+    /// user-given substitutions; the hir-id must map to a suitable
+    /// type.
+    fn user_substs_applied_to_ty_of_hir_id(
+        &self,
+        hir_id: hir::HirId,
+    ) -> Option<CanonicalUserType<'tcx>> {
+        let user_provided_types = self.typeck_results().user_provided_types();
+        let mut user_ty = *user_provided_types.get(hir_id)?;
+        debug!("user_subts_applied_to_ty_of_hir_id: user_ty={:?}", user_ty);
+        let ty = self.typeck_results().node_type(hir_id);
+        match ty.kind {
+            ty::Adt(adt_def, ..) => {
+                if let UserType::TypeOf(ref mut did, _) = &mut user_ty.value {
+                    *did = adt_def.did;
+                }
+                Some(user_ty)
+            }
+            ty::FnDef(..) => Some(user_ty),
+            _ => bug!("ty: {:?} should not have user provided type {:?} recorded ", ty, user_ty),
+        }
+    }
+}