Auto merge of #117050 - c410-f3r:here-we-go-again, r=petrochenkov

[`RFC 3086`] Attempt to try to resolve blocking concerns

Implements what is described at https://github.com/rust-lang/rust/issues/83527#issuecomment-1744822345 to hopefully make some progress.

It is unknown if such approach is or isn't desired due to the lack of further feedback, as such, it is probably best to nominate this PR to the official entities.

`@rustbot` labels +I-compiler-nominated

compiler/rustc_expand/src/expand.rs

@@ -41,6 +41,7 @@ use std::path::PathBuf;
 use std::rc::Rc;
 use std::{iter, mem};
 
+#[cfg(bootstrap)]
 macro_rules! ast_fragments {
     (
         $($Kind:ident($AstTy:ty) {
@@ -165,6 +166,131 @@ macro_rules! ast_fragments {
     }
 }
 
+#[cfg(not(bootstrap))]
+macro_rules! ast_fragments {
+    (
+        $($Kind:ident($AstTy:ty) {
+            $kind_name:expr;
+            $(one fn $mut_visit_ast:ident; fn $visit_ast:ident;)?
+            $(many fn $flat_map_ast_elt:ident; fn $visit_ast_elt:ident($($args:tt)*);)?
+            fn $make_ast:ident;
+        })*
+    ) => {
+        /// A fragment of AST that can be produced by a single macro expansion.
+        /// Can also serve as an input and intermediate result for macro expansion operations.
+        pub enum AstFragment {
+            OptExpr(Option<P<ast::Expr>>),
+            MethodReceiverExpr(P<ast::Expr>),
+            $($Kind($AstTy),)*
+        }
+
+        /// "Discriminant" of an AST fragment.
+        #[derive(Copy, Clone, PartialEq, Eq)]
+        pub enum AstFragmentKind {
+            OptExpr,
+            MethodReceiverExpr,
+            $($Kind,)*
+        }
+
+        impl AstFragmentKind {
+            pub fn name(self) -> &'static str {
+                match self {
+                    AstFragmentKind::OptExpr => "expression",
+                    AstFragmentKind::MethodReceiverExpr => "expression",
+                    $(AstFragmentKind::$Kind => $kind_name,)*
+                }
+            }
+
+            fn make_from<'a>(self, result: Box<dyn MacResult + 'a>) -> Option<AstFragment> {
+                match self {
+                    AstFragmentKind::OptExpr =>
+                        result.make_expr().map(Some).map(AstFragment::OptExpr),
+                    AstFragmentKind::MethodReceiverExpr =>
+                        result.make_expr().map(AstFragment::MethodReceiverExpr),
+                    $(AstFragmentKind::$Kind => result.$make_ast().map(AstFragment::$Kind),)*
+                }
+            }
+        }
+
+        impl AstFragment {
+            pub fn add_placeholders(&mut self, placeholders: &[NodeId]) {
+                if placeholders.is_empty() {
+                    return;
+                }
+                match self {
+                    $($(AstFragment::$Kind(ast) => ast.extend(placeholders.iter().flat_map(|id| {
+                        ${ignore($flat_map_ast_elt)}
+                        placeholder(AstFragmentKind::$Kind, *id, None).$make_ast()
+                    })),)?)*
+                    _ => panic!("unexpected AST fragment kind")
+                }
+            }
+
+            pub fn make_opt_expr(self) -> Option<P<ast::Expr>> {
+                match self {
+                    AstFragment::OptExpr(expr) => expr,
+                    _ => panic!("AstFragment::make_* called on the wrong kind of fragment"),
+                }
+            }
+
+            pub fn make_method_receiver_expr(self) -> P<ast::Expr> {
+                match self {
+                    AstFragment::MethodReceiverExpr(expr) => expr,
+                    _ => panic!("AstFragment::make_* called on the wrong kind of fragment"),
+                }
+            }
+
+            $(pub fn $make_ast(self) -> $AstTy {
+                match self {
+                    AstFragment::$Kind(ast) => ast,
+                    _ => panic!("AstFragment::make_* called on the wrong kind of fragment"),
+                }
+            })*
+
+            fn make_ast<T: InvocationCollectorNode>(self) -> T::OutputTy {
+                T::fragment_to_output(self)
+            }
+
+            pub fn mut_visit_with<F: MutVisitor>(&mut self, vis: &mut F) {
+                match self {
+                    AstFragment::OptExpr(opt_expr) => {
+                        visit_clobber(opt_expr, |opt_expr| {
+                            if let Some(expr) = opt_expr {
+                                vis.filter_map_expr(expr)
+                            } else {
+                                None
+                            }
+                        });
+                    }
+                    AstFragment::MethodReceiverExpr(expr) => vis.visit_method_receiver_expr(expr),
+                    $($(AstFragment::$Kind(ast) => vis.$mut_visit_ast(ast),)?)*
+                    $($(AstFragment::$Kind(ast) =>
+                        ast.flat_map_in_place(|ast| vis.$flat_map_ast_elt(ast)),)?)*
+                }
+            }
+
+            pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) {
+                match self {
+                    AstFragment::OptExpr(Some(expr)) => visitor.visit_expr(expr),
+                    AstFragment::OptExpr(None) => {}
+                    AstFragment::MethodReceiverExpr(expr) => visitor.visit_method_receiver_expr(expr),
+                    $($(AstFragment::$Kind(ast) => visitor.$visit_ast(ast),)?)*
+                    $($(AstFragment::$Kind(ast) => for ast_elt in &ast[..] {
+                        visitor.$visit_ast_elt(ast_elt, $($args)*);
+                    })?)*
+                }
+            }
+        }
+
+        impl<'a> MacResult for crate::mbe::macro_rules::ParserAnyMacro<'a> {
+            $(fn $make_ast(self: Box<crate::mbe::macro_rules::ParserAnyMacro<'a>>)
+                           -> Option<$AstTy> {
+                Some(self.make(AstFragmentKind::$Kind).$make_ast())
+            })*
+        }
+    }
+}
+
 ast_fragments! {
     Expr(P<ast::Expr>) { "expression"; one fn visit_expr; fn visit_expr; fn make_expr; }
     Pat(P<ast::Pat>) { "pattern"; one fn visit_pat; fn visit_pat; fn make_pat; }

compiler/rustc_expand/src/mbe/metavar_expr.rs

@@ -10,9 +10,8 @@ use rustc_span::Span;
 /// A meta-variable expression, for expansions based on properties of meta-variables.
 #[derive(Debug, Clone, PartialEq, Encodable, Decodable)]
 pub(crate) enum MetaVarExpr {
-    /// The number of repetitions of an identifier, optionally limited to a number
-    /// of outer-most repetition depths. If the depth limit is `None` then the depth is unlimited.
-    Count(Ident, Option<usize>),
+    /// The number of repetitions of an identifier.
+    Count(Ident, usize),
 
     /// Ignore a meta-variable for repetition without expansion.
     Ignore(Ident),
@@ -43,7 +42,10 @@ impl MetaVarExpr {
         let mut iter = args.trees();
         let rslt = match ident.as_str() {
             "count" => parse_count(&mut iter, sess, ident.span)?,
-            "ignore" => MetaVarExpr::Ignore(parse_ident(&mut iter, sess, ident.span)?),
+            "ignore" => {
+                eat_dollar(&mut iter, sess, ident.span)?;
+                MetaVarExpr::Ignore(parse_ident(&mut iter, sess, ident.span)?)
+            }
             "index" => MetaVarExpr::Index(parse_depth(&mut iter, sess, ident.span)?),
             "length" => MetaVarExpr::Length(parse_depth(&mut iter, sess, ident.span)?),
             _ => {
@@ -92,6 +94,7 @@ fn parse_count<'sess>(
     sess: &'sess ParseSess,
     span: Span,
 ) -> PResult<'sess, MetaVarExpr> {
+    eat_dollar(iter, sess, span)?;
     let ident = parse_ident(iter, sess, span)?;
     let depth = if try_eat_comma(iter) {
         if iter.look_ahead(0).is_none() {
@@ -100,9 +103,9 @@ fn parse_count<'sess>(
                 "`count` followed by a comma must have an associated index indicating its depth",
             ));
         }
-        Some(parse_depth(iter, sess, span)?)
+        parse_depth(iter, sess, span)?
     } else {
-        None
+        0
     };
     Ok(MetaVarExpr::Count(ident, depth))
 }
@@ -166,3 +169,20 @@ fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool {
     }
     false
 }
+
+/// Expects that the next item is a dollar sign.
+fn eat_dollar<'sess>(
+    iter: &mut RefTokenTreeCursor<'_>,
+    sess: &'sess ParseSess,
+    span: Span,
+) -> PResult<'sess, ()> {
+    if let Some(TokenTree::Token(token::Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0)
+    {
+        let _ = iter.next();
+        return Ok(());
+    }
+    Err(sess.span_diagnostic.struct_span_err(
+        span,
+        "meta-variables within meta-variable expressions must be referenced using a dollar sign",
+    ))
+}

compiler/rustc_expand/src/mbe/transcribe.rs

@@ -440,7 +440,7 @@ fn lockstep_iter_size(
 ///   declared inside a single repetition and the index `1` implies two nested repetitions.
 fn count_repetitions<'a>(
     cx: &ExtCtxt<'a>,
-    depth_opt: Option<usize>,
+    depth_user: usize,
     mut matched: &NamedMatch,
     repeats: &[(usize, usize)],
     sp: &DelimSpan,
@@ -449,37 +449,45 @@ fn count_repetitions<'a>(
     // (or at the top-level of `matched` if no depth is given).
     fn count<'a>(
         cx: &ExtCtxt<'a>,
-        declared_lhs_depth: usize,
-        depth_opt: Option<usize>,
+        depth_curr: usize,
+        depth_max: usize,
         matched: &NamedMatch,
         sp: &DelimSpan,
     ) -> PResult<'a, usize> {
         match matched {
-            MatchedTokenTree(_) | MatchedNonterminal(_) => {
-                if declared_lhs_depth == 0 {
-                    return Err(cx.create_err(CountRepetitionMisplaced { span: sp.entire() }));
-                }
-                match depth_opt {
-                    None => Ok(1),
-                    Some(_) => Err(out_of_bounds_err(cx, declared_lhs_depth, sp.entire(), "count")),
-                }
-            }
+            MatchedTokenTree(_) | MatchedNonterminal(_) => Ok(1),
             MatchedSeq(named_matches) => {
-                let new_declared_lhs_depth = declared_lhs_depth + 1;
-                match depth_opt {
-                    None => named_matches
+                if depth_curr == depth_max {
+                    Ok(named_matches.len())
+                } else {
+                    named_matches
                         .iter()
-                        .map(|elem| count(cx, new_declared_lhs_depth, None, elem, sp))
-                        .sum(),
-                    Some(0) => Ok(named_matches.len()),
-                    Some(depth) => named_matches
-                        .iter()
-                        .map(|elem| count(cx, new_declared_lhs_depth, Some(depth - 1), elem, sp))
-                        .sum(),
+                        .map(|elem| count(cx, depth_curr + 1, depth_max, elem, sp))
+                        .sum()
                 }
             }
         }
     }
+
+    /// Maximum depth
+    fn depth(counter: usize, matched: &NamedMatch) -> usize {
+        match matched {
+            MatchedTokenTree(_) | MatchedNonterminal(_) => counter,
+            MatchedSeq(named_matches) => {
+                let rslt = counter + 1;
+                if let Some(elem) = named_matches.first() { depth(rslt, elem) } else { rslt }
+            }
+        }
+    }
+
+    let depth_max = depth(0, matched)
+        .checked_sub(1)
+        .and_then(|el| el.checked_sub(repeats.len()))
+        .unwrap_or_default();
+    if depth_user > depth_max {
+        return Err(out_of_bounds_err(cx, depth_max + 1, sp.entire(), "count"));
+    }
+
     // `repeats` records all of the nested levels at which we are currently
     // matching meta-variables. The meta-var-expr `count($x)` only counts
     // matches that occur in this "subtree" of the `NamedMatch` where we
@@ -491,7 +499,12 @@ fn count_repetitions<'a>(
             matched = &ads[idx];
         }
     }
-    count(cx, 0, depth_opt, matched, sp)
+
+    if let MatchedTokenTree(_) | MatchedNonterminal(_) = matched {
+        return Err(cx.create_err(CountRepetitionMisplaced { span: sp.entire() }));
+    }
+
+    count(cx, depth_user, depth_max, matched, sp)
 }
 
 /// Returns a `NamedMatch` item declared on the LHS given an arbitrary [Ident]
@@ -523,7 +536,7 @@ fn out_of_bounds_err<'a>(
         )
     } else {
         format!(
-            "depth parameter on meta-variable expression `{ty}` \
+            "depth parameter of meta-variable expression `{ty}` \
              must be less than {max}"
         )
     };
@@ -545,9 +558,9 @@ fn transcribe_metavar_expr<'a>(
         span
     };
     match *expr {
-        MetaVarExpr::Count(original_ident, depth_opt) => {
+        MetaVarExpr::Count(original_ident, depth) => {
             let matched = matched_from_ident(cx, original_ident, interp)?;
-            let count = count_repetitions(cx, depth_opt, matched, repeats, sp)?;
+            let count = count_repetitions(cx, depth, matched, repeats, sp)?;
             let tt = TokenTree::token_alone(
                 TokenKind::lit(token::Integer, sym::integer(count), None),
                 visited_span(),