bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Introduce TtParser.

Browse source 
It currently has no state, just the three methods `parse_tt`,
`parse_tt_inner`, and `bb_items_ambiguity_error`.

This commit is large but trivial, and mostly consists of changes to the
indentation of those methods. Subsequent commits will do more.
This commit is contained in:
Nicholas Nethercote 2022-03-19 07:47:22 +11:00
parent 1bfe40d11c
commit d21b4f30c1
2 changed files with 325 additions and 306 deletions
Download patch file Download diff file Expand all files Collapse all files

77
compiler/rustc_expand/src/mbe/macro_parser.rs
View file

@ -492,6 +492,9 @@ fn token_name_eq(t1: &Token, t2: &Token) -> bool {
} }
} }
pub struct TtParser;
impl TtParser {
/// Process the matcher positions of `cur_items` until it is empty. In the process, this will /// Process the matcher positions of `cur_items` until it is empty. In the process, this will
/// produce more items in `next_items` and `bb_items`. /// produce more items in `next_items` and `bb_items`.
/// ///
@ -500,8 +503,8 @@ fn token_name_eq(t1: &Token, t2: &Token) -> bool {
/// ///
/// # Parameters /// # Parameters
/// ///
/// - `cur_items`: the set of current items to be processed. This should be empty by the end of a /// - `cur_items`: the set of current items to be processed. This should be empty by the end of
/// successful execution of this function. /// a successful execution of this function.
/// - `next_items`: the set of newly generated items. These are used to replenish `cur_items` in /// - `next_items`: the set of newly generated items. These are used to replenish `cur_items` in
/// the function `parse`. /// the function `parse`.
/// - `bb_items`: the set of items that are waiting for the black-box parser. /// - `bb_items`: the set of items that are waiting for the black-box parser.
@ -509,9 +512,10 @@ fn token_name_eq(t1: &Token, t2: &Token) -> bool {
/// ///
/// # Returns /// # Returns
/// ///
/// `Some(result)` if everything is finished, `None` otherwise. Note that matches are kept track of /// `Some(result)` if everything is finished, `None` otherwise. Note that matches are kept
/// through the items generated. /// track of through the items generated.
fn parse_tt_inner<'root, 'tt>( fn parse_tt_inner<'root, 'tt>(
&self,
sess: &ParseSess, sess: &ParseSess,
ms: &[TokenTree], ms: &[TokenTree],
cur_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, cur_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>,
@ -519,8 +523,8 @@ fn parse_tt_inner<'root, 'tt>(
bb_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, bb_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>,
token: &Token, token: &Token,
) -> Option<NamedParseResult> { ) -> Option<NamedParseResult> {
// Matcher positions that would be valid if the macro invocation was over now. Only modified if // Matcher positions that would be valid if the macro invocation was over now. Only
// `token == Eof`. // modified if `token == Eof`.
let mut eof_items = EofItems::None; let mut eof_items = EofItems::None;
while let Some(mut item) = cur_items.pop() { while let Some(mut item) = cur_items.pop() {
@ -537,13 +541,14 @@ fn parse_tt_inner<'root, 'tt>(
} }
} }
// Get the current position of the "dot" (`idx`) in `item` and the number of token trees in // Get the current position of the "dot" (`idx`) in `item` and the number of token
// the matcher (`len`). // trees in the matcher (`len`).
let idx = item.idx; let idx = item.idx;
let len = item.top_elts.len(); let len = item.top_elts.len();
if idx < len { if idx < len {
// We are in the middle of a matcher. Compare the matcher's current tt against `token`. // We are in the middle of a matcher. Compare the matcher's current tt against
// `token`.
match item.top_elts.get_tt(idx) { match item.top_elts.get_tt(idx) {
TokenTree::Sequence(sp, seq) => { TokenTree::Sequence(sp, seq) => {
let op = seq.kleene.op; let op = seq.kleene.op;
@ -584,9 +589,9 @@ fn parse_tt_inner<'root, 'tt>(
seq @ (TokenTree::Delimited(..) seq @ (TokenTree::Delimited(..)
| TokenTree::Token(Token { kind: DocComment(..), .. })) => { | TokenTree::Token(Token { kind: DocComment(..), .. })) => {
// To descend into a delimited submatcher or a doc comment, we push the current // To descend into a delimited submatcher or a doc comment, we push the
// matcher onto a stack and push a new item containing the submatcher onto // current matcher onto a stack and push a new item containing the
// `cur_items`. // submatcher onto `cur_items`.
// //
// At the beginning of the loop, if we reach the end of the delimited // At the beginning of the loop, if we reach the end of the delimited
// submatcher, we pop the stack to backtrack out of the descent. // submatcher, we pop the stack to backtrack out of the descent.
@ -598,8 +603,8 @@ fn parse_tt_inner<'root, 'tt>(
} }
TokenTree::Token(t) => { TokenTree::Token(t) => {
// If the token matches, we can just advance the parser. Otherwise, this match // If the token matches, we can just advance the parser. Otherwise, this
// hash failed, there is nothing to do, and hopefully another item in // match hash failed, there is nothing to do, and hopefully another item in
// `cur_items` will match. // `cur_items` will match.
if token_name_eq(&t, token) { if token_name_eq(&t, token) {
item.idx += 1; item.idx += 1;
@ -645,7 +650,8 @@ fn parse_tt_inner<'root, 'tt>(
cur_items.push(item); cur_items.push(item);
} }
} else { } else {
// We are past the end of the matcher, and not in a repetition. Look for end of input. // We are past the end of the matcher, and not in a repetition. Look for end of
// input.
debug_assert_eq!(idx, len); debug_assert_eq!(idx, len);
if *token == token::Eof { if *token == token::Eof {
eof_items = match eof_items { eof_items = match eof_items {
@ -656,8 +662,8 @@ fn parse_tt_inner<'root, 'tt>(
} }
} }
// If we reached the end of input, check that there is EXACTLY ONE possible matcher. Otherwise, // If we reached the end of input, check that there is EXACTLY ONE possible matcher.
// either the parse is ambiguous (which is an error) or there is a syntax error. // Otherwise, either the parse is ambiguous (which is an error) or there is a syntax error.
if *token == token::Eof { if *token == token::Eof {
Some(match eof_items { Some(match eof_items {
EofItems::One(mut eof_item) => { EofItems::One(mut eof_item) => {
@ -681,21 +687,22 @@ fn parse_tt_inner<'root, 'tt>(
} }
} }
/// Use the given slice of token trees (`ms`) as a matcher. Match the token stream from the given /// Use the given slice of token trees (`ms`) as a matcher. Match the token stream from the
/// `parser` against it and return the match. /// given `parser` against it and return the match.
pub(super) fn parse_tt( pub(super) fn parse_tt(
&self,
parser: &mut Cow<'_, Parser<'_>>, parser: &mut Cow<'_, Parser<'_>>,
ms: &[TokenTree], ms: &[TokenTree],
macro_name: Ident, macro_name: Ident,
) -> NamedParseResult { ) -> NamedParseResult {
// A queue of possible matcher positions. We initialize it with the matcher position in which // A queue of possible matcher positions. We initialize it with the matcher position in
// the "dot" is before the first token of the first token tree in `ms`. `parse_tt_inner` then // which the "dot" is before the first token of the first token tree in `ms`.
// processes all of these possible matcher positions and produces possible next positions into // `parse_tt_inner` then processes all of these possible matcher positions and produces
// `next_items`. After some post-processing, the contents of `next_items` replenish `cur_items` // possible next positions into `next_items`. After some post-processing, the contents of
// and we start over again. // `next_items` replenish `cur_items` and we start over again.
// //
// This MatcherPos instance is allocated on the stack. All others -- and there are frequently // This MatcherPos instance is allocated on the stack. All others -- and there are
// *no* others! -- are allocated on the heap. // frequently *no* others! -- are allocated on the heap.
let mut initial = MatcherPos::new(ms); let mut initial = MatcherPos::new(ms);
let mut cur_items = smallvec![MatcherPosHandle::Ref(&mut initial)]; let mut cur_items = smallvec![MatcherPosHandle::Ref(&mut initial)];
@ -707,7 +714,7 @@ pub(super) fn parse_tt(
// Process `cur_items` until either we have finished the input or we need to get some // Process `cur_items` until either we have finished the input or we need to get some
// parsing from the black-box parser done. // parsing from the black-box parser done.
if let Some(result) = parse_tt_inner( if let Some(result) = self.parse_tt_inner(
parser.sess, parser.sess,
ms, ms,
&mut cur_items, &mut cur_items,
@ -726,7 +733,10 @@ pub(super) fn parse_tt(
(0, 0) => { (0, 0) => {
// There are no possible next positions AND we aren't waiting for the black-box // There are no possible next positions AND we aren't waiting for the black-box
// parser: syntax error. // parser: syntax error.
return Failure(parser.token.clone(), "no rules expected this token in macro call"); return Failure(
parser.token.clone(),
"no rules expected this token in macro call",
);
} }
(_, 0) => { (_, 0) => {
@ -739,7 +749,8 @@ pub(super) fn parse_tt(
(0, 1) => { (0, 1) => {
// We need to call the black-box parser to get some nonterminal. // We need to call the black-box parser to get some nonterminal.
let mut item = bb_items.pop().unwrap(); let mut item = bb_items.pop().unwrap();
if let TokenTree::MetaVarDecl(span, _, Some(kind)) = item.top_elts.get_tt(item.idx) if let TokenTree::MetaVarDecl(span, _, Some(kind)) =
item.top_elts.get_tt(item.idx)
{ {
let match_cur = item.match_cur; let match_cur = item.match_cur;
// We use the span of the metavariable declaration to determine any // We use the span of the metavariable declaration to determine any
@ -748,7 +759,9 @@ pub(super) fn parse_tt(
Err(mut err) => { Err(mut err) => {
err.span_label( err.span_label(
span, span,
format!("while parsing argument for this `{kind}` macro fragment"), format!(
"while parsing argument for this `{kind}` macro fragment"
),
) )
.emit(); .emit();
return ErrorReported; return ErrorReported;
@ -766,7 +779,7 @@ pub(super) fn parse_tt(
(_, _) => { (_, _) => {
// Too many possibilities! // Too many possibilities!
return bb_items_ambiguity_error( return self.bb_items_ambiguity_error(
macro_name, macro_name,
next_items, next_items,
bb_items, bb_items,
@ -780,6 +793,7 @@ pub(super) fn parse_tt(
} }
fn bb_items_ambiguity_error<'root, 'tt>( fn bb_items_ambiguity_error<'root, 'tt>(
&self,
macro_name: Ident, macro_name: Ident,
next_items: SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, next_items: SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>,
bb_items: SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, bb_items: SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>,
@ -808,3 +822,4 @@ fn bb_items_ambiguity_error<'root, 'tt>(
), ),
) )
} }
}

18
compiler/rustc_expand/src/mbe/macro_rules.rs
View file

@ -3,8 +3,7 @@ use crate::base::{SyntaxExtension, SyntaxExtensionKind};
use crate::expand::{ensure_complete_parse, parse_ast_fragment, AstFragment, AstFragmentKind}; use crate::expand::{ensure_complete_parse, parse_ast_fragment, AstFragment, AstFragmentKind};
use crate::mbe; use crate::mbe;
use crate::mbe::macro_check; use crate::mbe::macro_check;
use crate::mbe::macro_parser::parse_tt; use crate::mbe::macro_parser::{Error, ErrorReported, Failure, Success, TtParser};
use crate::mbe::macro_parser::{Error, ErrorReported, Failure, Success};
use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq}; use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq};
use crate::mbe::transcribe::transcribe; use crate::mbe::transcribe::transcribe;
@ -246,6 +245,7 @@ fn generic_extension<'cx>(
// this situation.) // this situation.)
let parser = parser_from_cx(sess, arg.clone()); let parser = parser_from_cx(sess, arg.clone());
let tt_parser = TtParser;
for (i, lhs) in lhses.iter().enumerate() { for (i, lhs) in lhses.iter().enumerate() {
// try each arm's matchers // try each arm's matchers
let lhs_tt = match *lhs { let lhs_tt = match *lhs {
@ -259,7 +259,7 @@ fn generic_extension<'cx>(
// are not recorded. On the first `Success(..)`ful matcher, the spans are merged. // are not recorded. On the first `Success(..)`ful matcher, the spans are merged.
let mut gated_spans_snapshot = mem::take(&mut *sess.gated_spans.spans.borrow_mut()); let mut gated_spans_snapshot = mem::take(&mut *sess.gated_spans.spans.borrow_mut());
match parse_tt(&mut Cow::Borrowed(&parser), lhs_tt, name) { match tt_parser.parse_tt(&mut Cow::Borrowed(&parser), lhs_tt, name) {
Success(named_matches) => { Success(named_matches) => {
// The matcher was `Success(..)`ful. // The matcher was `Success(..)`ful.
// Merge the gated spans from parsing the matcher with the pre-existing ones. // Merge the gated spans from parsing the matcher with the pre-existing ones.
@ -352,9 +352,11 @@ fn generic_extension<'cx>(
mbe::TokenTree::Delimited(_, ref delim) => &delim.tts, mbe::TokenTree::Delimited(_, ref delim) => &delim.tts,
_ => continue, _ => continue,
}; };
if let Success(_) = if let Success(_) = tt_parser.parse_tt(
parse_tt(&mut Cow::Borrowed(&parser_from_cx(sess, arg.clone())), lhs_tt, name) &mut Cow::Borrowed(&parser_from_cx(sess, arg.clone())),
{ lhs_tt,
name,
) {
if comma_span.is_dummy() { if comma_span.is_dummy() {
err.note("you might be missing a comma"); err.note("you might be missing a comma");
} else { } else {
@ -447,7 +449,9 @@ pub fn compile_declarative_macro(
]; ];
let parser = Parser::new(&sess.parse_sess, body, true, rustc_parse::MACRO_ARGUMENTS); let parser = Parser::new(&sess.parse_sess, body, true, rustc_parse::MACRO_ARGUMENTS);
let argument_map = match parse_tt(&mut Cow::Borrowed(&parser), &argument_gram, def.ident) { let tt_parser = TtParser;
let argument_map =
match tt_parser.parse_tt(&mut Cow::Borrowed(&parser), &argument_gram, def.ident) {
Success(m) => m, Success(m) => m,
Failure(token, msg) => { Failure(token, msg) => {
let s = parse_failure_msg(&token); let s = parse_failure_msg(&token);