Rollup merge of #103464 - JakobDegen:mir-parsing, r=oli-obk

Add support for custom mir

This implements rust-lang/compiler-team#564 . Details about the design, motivation, etc. can be found in there.

r? ```@oli-obk```

compiler/rustc_borrowck/src/lib.rs

@@ -18,6 +18,7 @@ extern crate tracing;
 
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::graph::dominators::Dominators;
+use rustc_data_structures::vec_map::VecMap;
 use rustc_errors::{Diagnostic, DiagnosticBuilder, ErrorGuaranteed};
 use rustc_hir as hir;
 use rustc_hir::def_id::LocalDefId;
@@ -129,6 +130,19 @@ fn mir_borrowck<'tcx>(
 ) -> &'tcx BorrowCheckResult<'tcx> {
     let (input_body, promoted) = tcx.mir_promoted(def);
     debug!("run query mir_borrowck: {}", tcx.def_path_str(def.did.to_def_id()));
+
+    if input_body.borrow().should_skip() {
+        debug!("Skipping borrowck because of injected body");
+        // Let's make up a borrowck result! Fun times!
+        let result = BorrowCheckResult {
+            concrete_opaque_types: VecMap::new(),
+            closure_requirements: None,
+            used_mut_upvars: SmallVec::new(),
+            tainted_by_errors: None,
+        };
+        return tcx.arena.alloc(result);
+    }
+
     let hir_owner = tcx.hir().local_def_id_to_hir_id(def.did).owner;
 
     let infcx =

compiler/rustc_feature/src/active.rs

@@ -152,6 +152,8 @@ declare_features! (
     (active, anonymous_lifetime_in_impl_trait, "1.63.0", None, None),
     /// Allows identifying the `compiler_builtins` crate.
     (active, compiler_builtins, "1.13.0", None, None),
+    /// Allows writing custom MIR
+    (active, custom_mir, "1.65.0", None, None),
     /// Outputs useful `assert!` messages
     (active, generic_assert, "1.63.0", None, None),
     /// Allows using the `rust-intrinsic`'s "ABI".

compiler/rustc_feature/src/builtin_attrs.rs

@@ -810,6 +810,10 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
     rustc_attr!(TEST, rustc_polymorphize_error, Normal, template!(Word), WarnFollowing),
     rustc_attr!(TEST, rustc_def_path, Normal, template!(Word), WarnFollowing),
     rustc_attr!(TEST, rustc_mir, Normal, template!(List: "arg1, arg2, ..."), DuplicatesOk),
+    gated!(
+        custom_mir, Normal, template!(List: r#"dialect = "...", phase = "...""#),
+        ErrorFollowing, "the `#[custom_mir]` attribute is just used for the Rust test suite",
+    ),
     rustc_attr!(TEST, rustc_dump_program_clauses, Normal, template!(Word), WarnFollowing),
     rustc_attr!(TEST, rustc_dump_env_program_clauses, Normal, template!(Word), WarnFollowing),
     rustc_attr!(TEST, rustc_object_lifetime_default, Normal, template!(Word), WarnFollowing),

compiler/rustc_middle/src/mir/mod.rs

@@ -138,6 +138,48 @@ impl MirPhase {
             }
         }
     }
+
+    /// Parses an `MirPhase` from a pair of strings. Panics if this isn't possible for any reason.
+    pub fn parse(dialect: String, phase: Option<String>) -> Self {
+        match &*dialect.to_ascii_lowercase() {
+            "built" => {
+                assert!(phase.is_none(), "Cannot specify a phase for `Built` MIR");
+                MirPhase::Built
+            }
+            "analysis" => Self::Analysis(AnalysisPhase::parse(phase)),
+            "runtime" => Self::Runtime(RuntimePhase::parse(phase)),
+            _ => panic!("Unknown MIR dialect {}", dialect),
+        }
+    }
+}
+
+impl AnalysisPhase {
+    pub fn parse(phase: Option<String>) -> Self {
+        let Some(phase) = phase else {
+            return Self::Initial;
+        };
+
+        match &*phase.to_ascii_lowercase() {
+            "initial" => Self::Initial,
+            "post_cleanup" | "post-cleanup" | "postcleanup" => Self::PostCleanup,
+            _ => panic!("Unknown analysis phase {}", phase),
+        }
+    }
+}
+
+impl RuntimePhase {
+    pub fn parse(phase: Option<String>) -> Self {
+        let Some(phase) = phase else {
+            return Self::Initial;
+        };
+
+        match &*phase.to_ascii_lowercase() {
+            "initial" => Self::Initial,
+            "post_cleanup" | "post-cleanup" | "postcleanup" => Self::PostCleanup,
+            "optimized" => Self::Optimized,
+            _ => panic!("Unknown runtime phase {}", phase),
+        }
+    }
 }
 
 impl Display for MirPhase {
@@ -293,6 +335,13 @@ pub struct Body<'tcx> {
     /// potentially allow things like `[u8; std::mem::size_of::<T>() * 0]` due to this.
     pub is_polymorphic: bool,
 
+    /// The phase at which this MIR should be "injected" into the compilation process.
+    ///
+    /// Everything that comes before this `MirPhase` should be skipped.
+    ///
+    /// This is only `Some` if the function that this body comes from was annotated with `rustc_custom_mir`.
+    pub injection_phase: Option<MirPhase>,
+
     pub tainted_by_errors: Option<ErrorGuaranteed>,
 }
 
@@ -339,6 +388,7 @@ impl<'tcx> Body<'tcx> {
             span,
             required_consts: Vec::new(),
             is_polymorphic: false,
+            injection_phase: None,
             tainted_by_errors,
         };
         body.is_polymorphic = body.has_non_region_param();
@@ -366,6 +416,7 @@ impl<'tcx> Body<'tcx> {
             required_consts: Vec::new(),
             var_debug_info: Vec::new(),
             is_polymorphic: false,
+            injection_phase: None,
             tainted_by_errors: None,
         };
         body.is_polymorphic = body.has_non_region_param();
@@ -508,6 +559,14 @@ impl<'tcx> Body<'tcx> {
     pub fn generator_kind(&self) -> Option<GeneratorKind> {
         self.generator.as_ref().map(|generator| generator.generator_kind)
     }
+
+    #[inline]
+    pub fn should_skip(&self) -> bool {
+        let Some(injection_phase) = self.injection_phase else {
+            return false;
+        };
+        injection_phase > self.phase
+    }
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, Debug, TyEncodable, TyDecodable, HashStable)]

compiler/rustc_mir_build/src/build/custom/mod.rs

@@ -0,0 +1,155 @@
+//! Provides the implementation of the `custom_mir` attribute.
+//!
+//! Up until MIR building, this attribute has absolutely no effect. The `mir!` macro is a normal
+//! decl macro that expands like any other, and the code goes through parsing, name resolution and
+//! type checking like all other code. In MIR building we finally detect whether this attribute is
+//! present, and if so we branch off into this module, which implements the attribute by
+//! implementing a custom lowering from THIR to MIR.
+//!
+//! The result of this lowering is returned "normally" from the `mir_built` query, with the only
+//! notable difference being that the `injected` field in the body is set. Various components of the
+//! MIR pipeline, like borrowck and the pass manager will then consult this field (via
+//! `body.should_skip()`) to skip the parts of the MIR pipeline that precede the MIR phase the user
+//! specified.
+//!
+//! This file defines the general framework for the custom parsing. The parsing for all the
+//! "top-level" constructs can be found in the `parse` submodule, while the parsing for statements,
+//! terminators, and everything below can be found in the `parse::instruction` submodule.
+//!
+
+use rustc_ast::Attribute;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_hir::def_id::DefId;
+use rustc_index::vec::IndexVec;
+use rustc_middle::{
+    mir::*,
+    thir::*,
+    ty::{Ty, TyCtxt},
+};
+use rustc_span::Span;
+
+mod parse;
+
+pub(super) fn build_custom_mir<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    did: DefId,
+    thir: &Thir<'tcx>,
+    expr: ExprId,
+    params: &IndexVec<ParamId, Param<'tcx>>,
+    return_ty: Ty<'tcx>,
+    return_ty_span: Span,
+    span: Span,
+    attr: &Attribute,
+) -> Body<'tcx> {
+    let mut body = Body {
+        basic_blocks: BasicBlocks::new(IndexVec::new()),
+        source: MirSource::item(did),
+        phase: MirPhase::Built,
+        source_scopes: IndexVec::new(),
+        generator: None,
+        local_decls: LocalDecls::new(),
+        user_type_annotations: IndexVec::new(),
+        arg_count: params.len(),
+        spread_arg: None,
+        var_debug_info: Vec::new(),
+        span,
+        required_consts: Vec::new(),
+        is_polymorphic: false,
+        tainted_by_errors: None,
+        injection_phase: None,
+        pass_count: 1,
+    };
+
+    body.local_decls.push(LocalDecl::new(return_ty, return_ty_span));
+    body.basic_blocks_mut().push(BasicBlockData::new(None));
+    body.source_scopes.push(SourceScopeData {
+        span,
+        parent_scope: None,
+        inlined: None,
+        inlined_parent_scope: None,
+        local_data: ClearCrossCrate::Clear,
+    });
+    body.injection_phase = Some(parse_attribute(attr));
+
+    let mut pctxt = ParseCtxt {
+        tcx,
+        thir,
+        source_info: SourceInfo { span, scope: OUTERMOST_SOURCE_SCOPE },
+        body: &mut body,
+        local_map: FxHashMap::default(),
+        block_map: FxHashMap::default(),
+    };
+
+    let res = (|| {
+        pctxt.parse_args(&params)?;
+        pctxt.parse_body(expr)
+    })();
+    if let Err(err) = res {
+        tcx.sess.diagnostic().span_fatal(
+            err.span,
+            format!("Could not parse {}, found: {:?}", err.expected, err.item_description),
+        )
+    }
+
+    body
+}
+
+fn parse_attribute(attr: &Attribute) -> MirPhase {
+    let meta_items = attr.meta_item_list().unwrap();
+    let mut dialect: Option<String> = None;
+    let mut phase: Option<String> = None;
+
+    for nested in meta_items {
+        let name = nested.name_or_empty();
+        let value = nested.value_str().unwrap().as_str().to_string();
+        match name.as_str() {
+            "dialect" => {
+                assert!(dialect.is_none());
+                dialect = Some(value);
+            }
+            "phase" => {
+                assert!(phase.is_none());
+                phase = Some(value);
+            }
+            other => {
+                panic!("Unexpected key {}", other);
+            }
+        }
+    }
+
+    let Some(dialect) = dialect else {
+        assert!(phase.is_none());
+        return MirPhase::Built;
+    };
+
+    MirPhase::parse(dialect, phase)
+}
+
+struct ParseCtxt<'tcx, 'body> {
+    tcx: TyCtxt<'tcx>,
+    thir: &'body Thir<'tcx>,
+    source_info: SourceInfo,
+
+    body: &'body mut Body<'tcx>,
+    local_map: FxHashMap<LocalVarId, Local>,
+    block_map: FxHashMap<LocalVarId, BasicBlock>,
+}
+
+struct ParseError {
+    span: Span,
+    item_description: String,
+    expected: String,
+}
+
+impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
+    fn expr_error(&self, expr: ExprId, expected: &'static str) -> ParseError {
+        let expr = &self.thir[expr];
+        ParseError {
+            span: expr.span,
+            item_description: format!("{:?}", expr.kind),
+            expected: expected.to_string(),
+        }
+    }
+}
+
+type PResult<T> = Result<T, ParseError>;

compiler/rustc_mir_build/src/build/custom/parse.rs

@@ -0,0 +1,245 @@
+use rustc_index::vec::IndexVec;
+use rustc_middle::{mir::*, thir::*, ty::Ty};
+use rustc_span::Span;
+
+use super::{PResult, ParseCtxt, ParseError};
+
+mod instruction;
+
+/// Helper macro for parsing custom MIR.
+///
+/// Example usage looks something like:
+/// ```rust,ignore (incomplete example)
+/// parse_by_kind!(
+///     self, // : &ParseCtxt
+///     expr_id, // what you're matching against
+///     "assignment", // the thing you're trying to parse
+///     @call("mir_assign", args) => { args[0] }, // match invocations of the `mir_assign` special function
+///     ExprKind::Assign { lhs, .. } => { lhs }, // match thir assignment expressions
+///     // no need for fallthrough case - reasonable error is automatically generated
+/// )
+/// ```
+macro_rules! parse_by_kind {
+    (
+        $self:ident,
+        $expr_id:expr,
+        $expected:literal,
+        $(
+            @call($name:literal, $args:ident) => $call_expr:expr,
+        )*
+        $(
+            $pat:pat => $expr:expr,
+        )*
+    ) => {{
+        let expr_id = $self.preparse($expr_id);
+        let expr = &$self.thir[expr_id];
+        match &expr.kind {
+            $(
+                ExprKind::Call { ty, fun: _, args: $args, .. } if {
+                    match ty.kind() {
+                        ty::FnDef(did, _) => {
+                            $self.tcx.is_diagnostic_item(rustc_span::Symbol::intern($name), *did)
+                        }
+                        _ => false,
+                    }
+                } => $call_expr,
+            )*
+            $(
+                $pat => $expr,
+            )*
+            #[allow(unreachable_patterns)]
+            _ => return Err($self.expr_error(expr_id, $expected))
+        }
+    }};
+}
+pub(crate) use parse_by_kind;
+
+impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
+    /// Expressions should only ever be matched on after preparsing them. This removes extra scopes
+    /// we don't care about.
+    fn preparse(&self, expr_id: ExprId) -> ExprId {
+        let expr = &self.thir[expr_id];
+        match expr.kind {
+            ExprKind::Scope { value, .. } => self.preparse(value),
+            _ => expr_id,
+        }
+    }
+
+    fn statement_as_expr(&self, stmt_id: StmtId) -> PResult<ExprId> {
+        match &self.thir[stmt_id].kind {
+            StmtKind::Expr { expr, .. } => Ok(*expr),
+            kind @ StmtKind::Let { pattern, .. } => {
+                return Err(ParseError {
+                    span: pattern.span,
+                    item_description: format!("{:?}", kind),
+                    expected: "expression".to_string(),
+                });
+            }
+        }
+    }
+
+    pub fn parse_args(&mut self, params: &IndexVec<ParamId, Param<'tcx>>) -> PResult<()> {
+        for param in params.iter() {
+            let (var, span) = {
+                let pat = param.pat.as_ref().unwrap();
+                match &pat.kind {
+                    PatKind::Binding { var, .. } => (*var, pat.span),
+                    _ => {
+                        return Err(ParseError {
+                            span: pat.span,
+                            item_description: format!("{:?}", pat.kind),
+                            expected: "local".to_string(),
+                        });
+                    }
+                }
+            };
+            let decl = LocalDecl::new(param.ty, span);
+            let local = self.body.local_decls.push(decl);
+            self.local_map.insert(var, local);
+        }
+
+        Ok(())
+    }
+
+    /// Bodies are of the form:
+    ///
+    /// ```text
+    /// {
+    ///     let bb1: BasicBlock;
+    ///     let bb2: BasicBlock;
+    ///     {
+    ///         let RET: _;
+    ///         let local1;
+    ///         let local2;
+    ///
+    ///         {
+    ///             { // entry block
+    ///                 statement1;
+    ///                 terminator1
+    ///             };
+    ///
+    ///             bb1 = {
+    ///                 statement2;
+    ///                 terminator2
+    ///             };
+    ///
+    ///             bb2 = {
+    ///                 statement3;
+    ///                 terminator3
+    ///             }
+    ///
+    ///             RET
+    ///         }
+    ///     }
+    /// }
+    /// ```
+    ///
+    /// This allows us to easily parse the basic blocks declarations, local declarations, and
+    /// basic block definitions in order.
+    pub fn parse_body(&mut self, expr_id: ExprId) -> PResult<()> {
+        let body = parse_by_kind!(self, expr_id, "whole body",
+            ExprKind::Block { block } => self.thir[*block].expr.unwrap(),
+        );
+        let (block_decls, rest) = parse_by_kind!(self, body, "body with block decls",
+            ExprKind::Block { block } => {
+                let block = &self.thir[*block];
+                (&block.stmts, block.expr.unwrap())
+            },
+        );
+        self.parse_block_decls(block_decls.iter().copied())?;
+
+        let (local_decls, rest) = parse_by_kind!(self, rest, "body with local decls",
+            ExprKind::Block { block } => {
+                let block = &self.thir[*block];
+                (&block.stmts, block.expr.unwrap())
+            },
+        );
+        self.parse_local_decls(local_decls.iter().copied())?;
+
+        let block_defs = parse_by_kind!(self, rest, "body with block defs",
+            ExprKind::Block { block } => &self.thir[*block].stmts,
+        );
+        for (i, block_def) in block_defs.iter().enumerate() {
+            let block = self.parse_block_def(self.statement_as_expr(*block_def)?)?;
+            self.body.basic_blocks_mut()[BasicBlock::from_usize(i)] = block;
+        }
+
+        Ok(())
+    }
+
+    fn parse_block_decls(&mut self, stmts: impl Iterator<Item = StmtId>) -> PResult<()> {
+        for stmt in stmts {
+            let (var, _, _) = self.parse_let_statement(stmt)?;
+            let data = BasicBlockData::new(None);
+            let block = self.body.basic_blocks_mut().push(data);
+            self.block_map.insert(var, block);
+        }
+
+        Ok(())
+    }
+
+    fn parse_local_decls(&mut self, mut stmts: impl Iterator<Item = StmtId>) -> PResult<()> {
+        let (ret_var, ..) = self.parse_let_statement(stmts.next().unwrap())?;
+        self.local_map.insert(ret_var, Local::from_u32(0));
+
+        for stmt in stmts {
+            let (var, ty, span) = self.parse_let_statement(stmt)?;
+            let decl = LocalDecl::new(ty, span);
+            let local = self.body.local_decls.push(decl);
+            self.local_map.insert(var, local);
+        }
+
+        Ok(())
+    }
+
+    fn parse_let_statement(&mut self, stmt_id: StmtId) -> PResult<(LocalVarId, Ty<'tcx>, Span)> {
+        let pattern = match &self.thir[stmt_id].kind {
+            StmtKind::Let { pattern, .. } => pattern,
+            StmtKind::Expr { expr, .. } => {
+                return Err(self.expr_error(*expr, "let statement"));
+            }
+        };
+
+        self.parse_var(pattern)
+    }
+
+    fn parse_var(&mut self, mut pat: &Pat<'tcx>) -> PResult<(LocalVarId, Ty<'tcx>, Span)> {
+        // Make sure we throw out any `AscribeUserType` we find
+        loop {
+            match &pat.kind {
+                PatKind::Binding { var, ty, .. } => break Ok((*var, *ty, pat.span)),
+                PatKind::AscribeUserType { subpattern, .. } => {
+                    pat = subpattern;
+                }
+                _ => {
+                    break Err(ParseError {
+                        span: pat.span,
+                        item_description: format!("{:?}", pat.kind),
+                        expected: "local".to_string(),
+                    });
+                }
+            }
+        }
+    }
+
+    fn parse_block_def(&self, expr_id: ExprId) -> PResult<BasicBlockData<'tcx>> {
+        let block = parse_by_kind!(self, expr_id, "basic block",
+            ExprKind::Block { block } => &self.thir[*block],
+        );
+
+        let mut data = BasicBlockData::new(None);
+        for stmt_id in &*block.stmts {
+            let stmt = self.statement_as_expr(*stmt_id)?;
+            let statement = self.parse_statement(stmt)?;
+            data.statements.push(Statement { source_info: self.source_info, kind: statement });
+        }
+
+        let Some(trailing) = block.expr else {
+            return Err(self.expr_error(expr_id, "terminator"))
+        };
+        let terminator = self.parse_terminator(trailing)?;
+        data.terminator = Some(Terminator { source_info: self.source_info, kind: terminator });
+
+        Ok(data)
+    }
+}

compiler/rustc_mir_build/src/build/custom/parse/instruction.rs

@@ -0,0 +1,72 @@
+use rustc_middle::{mir::*, thir::*, ty};
+
+use super::{parse_by_kind, PResult, ParseCtxt};
+
+impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
+    pub fn parse_statement(&self, expr_id: ExprId) -> PResult<StatementKind<'tcx>> {
+        parse_by_kind!(self, expr_id, "statement",
+            @call("mir_retag", args) => {
+                Ok(StatementKind::Retag(RetagKind::Default, Box::new(self.parse_place(args[0])?)))
+            },
+            @call("mir_retag_raw", args) => {
+                Ok(StatementKind::Retag(RetagKind::Raw, Box::new(self.parse_place(args[0])?)))
+            },
+            ExprKind::Assign { lhs, rhs } => {
+                let lhs = self.parse_place(*lhs)?;
+                let rhs = self.parse_rvalue(*rhs)?;
+                Ok(StatementKind::Assign(Box::new((lhs, rhs))))
+            },
+        )
+    }
+
+    pub fn parse_terminator(&self, expr_id: ExprId) -> PResult<TerminatorKind<'tcx>> {
+        parse_by_kind!(self, expr_id, "terminator",
+            @call("mir_return", _args) => {
+                Ok(TerminatorKind::Return)
+            },
+            @call("mir_goto", args) => {
+                Ok(TerminatorKind::Goto { target: self.parse_block(args[0])? } )
+            },
+        )
+    }
+
+    fn parse_rvalue(&self, expr_id: ExprId) -> PResult<Rvalue<'tcx>> {
+        parse_by_kind!(self, expr_id, "rvalue",
+            ExprKind::Borrow { borrow_kind, arg } => Ok(
+                Rvalue::Ref(self.tcx.lifetimes.re_erased, *borrow_kind, self.parse_place(*arg)?)
+            ),
+            ExprKind::AddressOf { mutability, arg } => Ok(
+                Rvalue::AddressOf(*mutability, self.parse_place(*arg)?)
+            ),
+            _ => self.parse_operand(expr_id).map(Rvalue::Use),
+        )
+    }
+
+    fn parse_operand(&self, expr_id: ExprId) -> PResult<Operand<'tcx>> {
+        parse_by_kind!(self, expr_id, "operand",
+            @call("mir_move", args) => self.parse_place(args[0]).map(Operand::Move),
+            _ => self.parse_place(expr_id).map(Operand::Copy),
+        )
+    }
+
+    fn parse_place(&self, expr_id: ExprId) -> PResult<Place<'tcx>> {
+        parse_by_kind!(self, expr_id, "place",
+            ExprKind::Deref { arg } => Ok(
+                self.parse_place(*arg)?.project_deeper(&[PlaceElem::Deref], self.tcx)
+            ),
+            _ => self.parse_local(expr_id).map(Place::from),
+        )
+    }
+
+    fn parse_local(&self, expr_id: ExprId) -> PResult<Local> {
+        parse_by_kind!(self, expr_id, "local",
+            ExprKind::VarRef { id } => Ok(self.local_map[id]),
+        )
+    }
+
+    fn parse_block(&self, expr_id: ExprId) -> PResult<BasicBlock> {
+        parse_by_kind!(self, expr_id, "basic block",
+            ExprKind::VarRef { id } => Ok(self.block_map[id]),
+        )
+    }
+}

compiler/rustc_mir_build/src/build/mod.rs

@@ -481,6 +481,22 @@ fn construct_fn<'tcx>(
         (None, fn_sig.output())
     };
 
+    if let Some(custom_mir_attr) =
+        tcx.hir().attrs(fn_id).iter().find(|attr| attr.name_or_empty() == sym::custom_mir)
+    {
+        return custom::build_custom_mir(
+            tcx,
+            fn_def.did.to_def_id(),
+            thir,
+            expr,
+            arguments,
+            return_ty,
+            return_ty_span,
+            span,
+            custom_mir_attr,
+        );
+    }
+
     let infcx = tcx.infer_ctxt().build();
     let mut builder = Builder::new(
         thir,
@@ -1033,6 +1049,7 @@ pub(crate) fn parse_float_into_scalar(
 
 mod block;
 mod cfg;
+mod custom;
 mod expr;
 mod matches;
 mod misc;

compiler/rustc_mir_build/src/thir/cx/expr.rs

@@ -51,11 +51,17 @@ impl<'tcx> Cx<'tcx> {
         trace!(?expr.ty);
 
         // Now apply adjustments, if any.
-        for adjustment in self.typeck_results.expr_adjustments(hir_expr) {
+        if self.apply_adjustments {
-            trace!(?expr, ?adjustment);
+            for adjustment in self.typeck_results.expr_adjustments(hir_expr) {
-            let span = expr.span;
+                trace!(?expr, ?adjustment);
-            expr =
+                let span = expr.span;
-                self.apply_adjustment(hir_expr, expr, adjustment, adjustment_span.unwrap_or(span));
+                expr = self.apply_adjustment(
+                    hir_expr,
+                    expr,
+                    adjustment,
+                    adjustment_span.unwrap_or(span),
+                );
+            }
         }
 
         trace!(?expr.ty, "after adjustments");

compiler/rustc_mir_build/src/thir/cx/mod.rs

@@ -80,6 +80,9 @@ struct Cx<'tcx> {
     /// for the receiver.
     adjustment_span: Option<(HirId, Span)>,
 
+    /// False to indicate that adjustments should not be applied. Only used for `custom_mir`
+    apply_adjustments: bool,
+
     /// The `DefId` of the owner of this body.
     body_owner: DefId,
 }
@@ -87,6 +90,8 @@ struct Cx<'tcx> {
 impl<'tcx> Cx<'tcx> {
     fn new(tcx: TyCtxt<'tcx>, def: ty::WithOptConstParam<LocalDefId>) -> Cx<'tcx> {
         let typeck_results = tcx.typeck_opt_const_arg(def);
+        let did = def.did;
+        let hir = tcx.hir();
         Cx {
             tcx,
             thir: Thir::new(),
@@ -94,8 +99,12 @@ impl<'tcx> Cx<'tcx> {
             region_scope_tree: tcx.region_scope_tree(def.did),
             typeck_results,
             rvalue_scopes: &typeck_results.rvalue_scopes,
-            body_owner: def.did.to_def_id(),
+            body_owner: did.to_def_id(),
             adjustment_span: None,
+            apply_adjustments: hir
+                .attrs(hir.local_def_id_to_hir_id(did))
+                .iter()
+                .all(|attr| attr.name_or_empty() != rustc_span::sym::custom_mir),
         }
     }
 

compiler/rustc_mir_transform/src/check_unsafety.rs

@@ -471,6 +471,14 @@ fn unsafety_check_result<'tcx>(
     // `mir_built` force this.
     let body = &tcx.mir_built(def).borrow();
 
+    if body.should_skip() {
+        return tcx.arena.alloc(UnsafetyCheckResult {
+            violations: Vec::new(),
+            used_unsafe_blocks: FxHashSet::default(),
+            unused_unsafes: Some(Vec::new()),
+        });
+    }
+
     let param_env = tcx.param_env(def.did);
 
     let mut checker = UnsafetyChecker::new(body, def.did, tcx, param_env);

compiler/rustc_mir_transform/src/pass_manager.rs

@@ -96,45 +96,48 @@ fn run_passes_inner<'tcx>(
     phase_change: Option<MirPhase>,
     validate_each: bool,
 ) {
-    let validate = validate_each & tcx.sess.opts.unstable_opts.validate_mir;
+    let validate = validate_each & tcx.sess.opts.unstable_opts.validate_mir & !body.should_skip();
     let overridden_passes = &tcx.sess.opts.unstable_opts.mir_enable_passes;
     trace!(?overridden_passes);
 
-    for pass in passes {
+    if !body.should_skip() {
-        let name = pass.name();
+        for pass in passes {
+            let name = pass.name();
 
-        let overridden =
+            let overridden = overridden_passes.iter().rev().find(|(s, _)| s == &*name).map(
-            overridden_passes.iter().rev().find(|(s, _)| s == &*name).map(|(_name, polarity)| {
+                |(_name, polarity)| {
-                trace!(
+                    trace!(
-                    pass = %name,
+                        pass = %name,
-                    "{} as requested by flag",
+                        "{} as requested by flag",
-                    if *polarity { "Running" } else { "Not running" },
+                        if *polarity { "Running" } else { "Not running" },
-                );
+                    );
-                *polarity
+                    *polarity
-            });
+                },
-        if !overridden.unwrap_or_else(|| pass.is_enabled(&tcx.sess)) {
+            );
-            continue;
+            if !overridden.unwrap_or_else(|| pass.is_enabled(&tcx.sess)) {
+                continue;
+            }
+
+            let dump_enabled = pass.is_mir_dump_enabled();
+
+            if dump_enabled {
+                dump_mir_for_pass(tcx, body, &name, false);
+            }
+            if validate {
+                validate_body(tcx, body, format!("before pass {}", name));
+            }
+
+            pass.run_pass(tcx, body);
+
+            if dump_enabled {
+                dump_mir_for_pass(tcx, body, &name, true);
+            }
+            if validate {
+                validate_body(tcx, body, format!("after pass {}", name));
+            }
+
+            body.pass_count += 1;
         }
-
-        let dump_enabled = pass.is_mir_dump_enabled();
-
-        if dump_enabled {
-            dump_mir_for_pass(tcx, body, &name, false);
-        }
-        if validate {
-            validate_body(tcx, body, format!("before pass {}", name));
-        }
-
-        pass.run_pass(tcx, body);
-
-        if dump_enabled {
-            dump_mir_for_pass(tcx, body, &name, true);
-        }
-        if validate {
-            validate_body(tcx, body, format!("after pass {}", name));
-        }
-
-        body.pass_count += 1;
     }
 
     if let Some(new_phase) = phase_change {

compiler/rustc_span/src/symbol.rs

@@ -584,6 +584,7 @@ symbols! {
         custom_attribute,
         custom_derive,
         custom_inner_attributes,
+        custom_mir,
         custom_test_frameworks,
         d,
         d32,

library/core/src/intrinsics.rs

@@ -59,6 +59,9 @@ use crate::marker::DiscriminantKind;
 use crate::marker::Tuple;
 use crate::mem;
 
+#[cfg(not(bootstrap))]
+pub mod mir;
+
 // These imports are used for simplifying intra-doc links
 #[allow(unused_imports)]
 #[cfg(all(target_has_atomic = "8", target_has_atomic = "32", target_has_atomic = "ptr"))]

library/core/src/intrinsics/mir.rs

@@ -0,0 +1,123 @@
+//! Rustc internal tooling for hand-writing MIR.
+//!
+//! If for some reasons you are not writing rustc tests and have found yourself considering using
+//! this feature, turn back. This is *exceptionally* unstable. There is no attempt at all to make
+//! anything work besides those things which the rustc test suite happened to need. If you make a
+//! typo you'll probably ICE. Really, this is not the solution to your problems. Consider instead
+//! supporting the [stable MIR project group](https://github.com/rust-lang/project-stable-mir).
+//!
+//! The documentation for this module describes how to use this feature. If you are interested in
+//! hacking on the implementation, most of that documentation lives at
+//! `rustc_mir_building/src/build/custom/mod.rs`.
+//!
+//! Typical usage will look like this:
+//!
+//! ```rust
+//! #![feature(core_intrinsics, custom_mir)]
+//!
+//! extern crate core;
+//! use core::intrinsics::mir::*;
+//!
+//! #[custom_mir(dialect = "built")]
+//! pub fn simple(x: i32) -> i32 {
+//!     mir!(
+//!         let temp1: i32;
+//!         let temp2: _;
+//!
+//!         {
+//!             temp1 = x;
+//!             Goto(exit)
+//!         }
+//!
+//!         exit = {
+//!             temp2 = Move(temp1);
+//!             RET = temp2;
+//!             Return()
+//!         }
+//!     )
+//! }
+//! ```
+//!
+//! Hopefully most of this is fairly self-explanatory. Expanding on some notable details:
+//!
+//!  - The `custom_mir` attribute tells the compiler to treat the function as being custom MIR. This
+//!    attribute only works on functions - there is no way to insert custom MIR into the middle of
+//!    another function.
+//!  - The `dialect` and `phase` parameters indicate which version of MIR you are inserting here.
+//!    This will normally be the phase that corresponds to the thing you are trying to test. The
+//!    phase can be omitted for dialects that have just one.
+//!  - You should define your function signature like you normally would. Externally, this function
+//!    can be called like any other function.
+//!  - Type inference works - you don't have to spell out the type of all of your locals.
+//!
+//! For now, all statements and terminators are parsed from nested invocations of the special
+//! functions provided in this module. We additionally want to (but do not yet) support more
+//! "normal" Rust syntax in places where it makes sense. Also, most kinds of instructions are not
+//! supported yet.
+//!
+
+#![unstable(
+    feature = "custom_mir",
+    reason = "MIR is an implementation detail and extremely unstable",
+    issue = "none"
+)]
+#![allow(unused_variables, non_snake_case, missing_debug_implementations)]
+
+/// Type representing basic blocks.
+///
+/// All terminators will have this type as a return type. It helps achieve some type safety.
+pub struct BasicBlock;
+
+macro_rules! define {
+    ($name:literal, $($sig:tt)*) => {
+        #[rustc_diagnostic_item = $name]
+        pub $($sig)* { panic!() }
+    }
+}
+
+define!("mir_return", fn Return() -> BasicBlock);
+define!("mir_goto", fn Goto(destination: BasicBlock) -> BasicBlock);
+define!("mir_retag", fn Retag<T>(place: T));
+define!("mir_retag_raw", fn RetagRaw<T>(place: T));
+define!("mir_move", fn Move<T>(place: T) -> T);
+
+/// Convenience macro for generating custom MIR.
+///
+/// See the module documentation for syntax details. This macro is not magic - it only transforms
+/// your MIR into something that is easier to parse in the compiler.
+#[rustc_macro_transparency = "transparent"]
+pub macro mir {
+    (
+        $(let $local_decl:ident $(: $local_decl_ty:ty)? ;)*
+
+        $entry_block:block
+
+        $(
+            $block_name:ident = $block:block
+        )*
+    ) => {{
+        // First, we declare all basic blocks.
+        $(
+            let $block_name: ::core::intrinsics::mir::BasicBlock;
+        )*
+
+        {
+            // Now all locals
+            #[allow(non_snake_case)]
+            let RET;
+            $(
+                let $local_decl $(: $local_decl_ty)? ;
+            )*
+
+            {
+                // Finally, the contents of the basic blocks
+                $entry_block;
+                $(
+                    $block;
+                )*
+
+                RET
+            }
+        }
+    }}
+}

src/test/mir-opt/building/custom/references.immut_ref.built.after.mir

@@ -0,0 +1,14 @@
+// MIR for `immut_ref` after built
+
+fn immut_ref(_1: &i32) -> &i32 {
+    let mut _0: &i32;                    // return place in scope 0 at $DIR/references.rs:+0:30: +0:34
+    let mut _2: *const i32;              // in scope 0 at $SRC_DIR/core/src/intrinsics/mir.rs:LL:COL
+
+    bb0: {
+        _2 = &raw const (*_1);           // scope 0 at $DIR/references.rs:+0:1: +0:34
+        Retag([raw] _2);                 // scope 0 at $DIR/references.rs:+0:1: +0:34
+        _0 = &(*_2);                     // scope 0 at $DIR/references.rs:+0:1: +0:34
+        Retag(_0);                       // scope 0 at $DIR/references.rs:+0:1: +0:34
+        return;                          // scope 0 at $DIR/references.rs:+0:1: +0:34
+    }
+}

src/test/mir-opt/building/custom/references.mut_ref.built.after.mir

@@ -0,0 +1,14 @@
+// MIR for `mut_ref` after built
+
+fn mut_ref(_1: &mut i32) -> &mut i32 {
+    let mut _0: &mut i32;                // return place in scope 0 at $DIR/references.rs:+0:32: +0:40
+    let mut _2: *mut i32;                // in scope 0 at $SRC_DIR/core/src/intrinsics/mir.rs:LL:COL
+
+    bb0: {
+        _2 = &raw mut (*_1);             // scope 0 at $DIR/references.rs:+0:1: +0:40
+        Retag([raw] _2);                 // scope 0 at $DIR/references.rs:+0:1: +0:40
+        _0 = &mut (*_2);                 // scope 0 at $DIR/references.rs:+0:1: +0:40
+        Retag(_0);                       // scope 0 at $DIR/references.rs:+0:1: +0:40
+        return;                          // scope 0 at $DIR/references.rs:+0:1: +0:40
+    }
+}

src/test/mir-opt/building/custom/references.rs

@@ -0,0 +1,43 @@
+#![feature(custom_mir, core_intrinsics)]
+
+extern crate core;
+use core::intrinsics::mir::*;
+use core::ptr::{addr_of, addr_of_mut};
+
+// EMIT_MIR references.mut_ref.built.after.mir
+#[custom_mir(dialect = "runtime", phase = "optimized")]
+pub fn mut_ref(x: &mut i32) -> &mut i32 {
+    mir!(
+        let t: *mut i32;
+
+        {
+            t = addr_of_mut!(*x);
+            RetagRaw(t);
+            RET = &mut *t;
+            Retag(RET);
+            Return()
+        }
+    )
+}
+
+// EMIT_MIR references.immut_ref.built.after.mir
+#[custom_mir(dialect = "runtime", phase = "optimized")]
+pub fn immut_ref(x: &i32) -> &i32 {
+    mir!(
+        let t: *const i32;
+
+        {
+            t = addr_of!(*x);
+            RetagRaw(t);
+            RET = & *t;
+            Retag(RET);
+            Return()
+        }
+    )
+}
+
+fn main() {
+    let mut x = 5;
+    assert_eq!(*mut_ref(&mut x), 5);
+    assert_eq!(*immut_ref(&x), 5);
+}

src/test/mir-opt/building/custom/simple_assign.rs

@@ -0,0 +1,37 @@
+#![feature(custom_mir, core_intrinsics)]
+
+extern crate core;
+use core::intrinsics::mir::*;
+
+// EMIT_MIR simple_assign.simple.built.after.mir
+#[custom_mir(dialect = "built")]
+pub fn simple(x: i32) -> i32 {
+    mir!(
+        let temp1: i32;
+        let temp2: _;
+
+        {
+            temp1 = x;
+            Goto(exit)
+        }
+
+        exit = {
+            temp2 = Move(temp1);
+            RET = temp2;
+            Return()
+        }
+    )
+}
+
+// EMIT_MIR simple_assign.simple_ref.built.after.mir
+#[custom_mir(dialect = "built")]
+pub fn simple_ref(x: &mut i32) -> &mut i32 {
+    mir!({
+        RET = Move(x);
+        Return()
+    })
+}
+
+fn main() {
+    assert_eq!(5, simple(5));
+}

src/test/mir-opt/building/custom/simple_assign.simple.built.after.mir

@@ -0,0 +1,18 @@
+// MIR for `simple` after built
+
+fn simple(_1: i32) -> i32 {
+    let mut _0: i32;                     // return place in scope 0 at $DIR/simple_assign.rs:+0:26: +0:29
+    let mut _2: i32;                     // in scope 0 at $SRC_DIR/core/src/intrinsics/mir.rs:LL:COL
+    let mut _3: i32;                     // in scope 0 at $SRC_DIR/core/src/intrinsics/mir.rs:LL:COL
+
+    bb0: {
+        _2 = _1;                         // scope 0 at $DIR/simple_assign.rs:+0:1: +0:29
+        goto -> bb1;                     // scope 0 at $DIR/simple_assign.rs:+0:1: +0:29
+    }
+
+    bb1: {
+        _3 = move _2;                    // scope 0 at $DIR/simple_assign.rs:+0:1: +0:29
+        _0 = _3;                         // scope 0 at $DIR/simple_assign.rs:+0:1: +0:29
+        return;                          // scope 0 at $DIR/simple_assign.rs:+0:1: +0:29
+    }
+}

src/test/mir-opt/building/custom/simple_assign.simple_ref.built.after.mir

@@ -0,0 +1,10 @@
+// MIR for `simple_ref` after built
+
+fn simple_ref(_1: &mut i32) -> &mut i32 {
+    let mut _0: &mut i32;                // return place in scope 0 at $DIR/simple_assign.rs:+0:35: +0:43
+
+    bb0: {
+        _0 = move _1;                    // scope 0 at $DIR/simple_assign.rs:+0:1: +0:43
+        return;                          // scope 0 at $DIR/simple_assign.rs:+0:1: +0:43
+    }
+}

src/test/ui/feature-gates/feature-gate-custom_mir.rs

@@ -0,0 +1,12 @@
+#![feature(core_intrinsics)]
+
+extern crate core;
+
+#[custom_mir(dialect = "built")] //~ ERROR the `#[custom_mir]` attribute is just used for the Rust test suite
+pub fn foo(_x: i32) -> i32 {
+    0
+}
+
+fn main() {
+    assert_eq!(2, foo(2));
+}

src/test/ui/feature-gates/feature-gate-custom_mir.stderr

@@ -0,0 +1,11 @@
+error[E0658]: the `#[custom_mir]` attribute is just used for the Rust test suite
+  --> $DIR/feature-gate-custom_mir.rs:5:1
+   |
+LL | #[custom_mir(dialect = "built")]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = help: add `#![feature(custom_mir)]` to the crate attributes to enable
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0658`.