bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Rename RefTokenTreeCursor.

Browse source 
Because `TokenStreamIter` is a much better name for a `TokenStream`
iterator. Also rename the `TokenStream::trees` method as
`TokenStream::iter`, and some local variables.
This commit is contained in:
Nicholas Nethercote 2024-12-11 14:38:09 +11:00
parent 3575e7943b
commit 809975c94a
17 changed files with 117 additions and 119 deletions
Download patch file Download diff file Expand all files Collapse all files

28
compiler/rustc_expand/src/mbe/metavar_expr.rs
View file

@ -1,5 +1,5 @@
use rustc_ast::token::{self, Delimiter, IdentIsRaw, Lit, Token, TokenKind};
use rustc_ast::tokenstream::{RefTokenTreeCursor, TokenStream, TokenTree};
use rustc_ast::tokenstream::{TokenStream, TokenStreamIter, TokenTree};
use rustc_ast::{LitIntType, LitKind};
use rustc_ast_pretty::pprust;
use rustc_errors::{Applicability, PResult};
@ -39,14 +39,14 @@ impl MetaVarExpr {
outer_span: Span,
psess: &'psess ParseSess,
) -> PResult<'psess, MetaVarExpr> {
let mut tts = input.trees();
let ident = parse_ident(&mut tts, psess, outer_span)?;
let Some(TokenTree::Delimited(.., Delimiter::Parenthesis, args)) = tts.next() else {
let mut iter = input.iter();
let ident = parse_ident(&mut iter, psess, outer_span)?;
let Some(TokenTree::Delimited(.., Delimiter::Parenthesis, args)) = iter.next() else {
let msg = "meta-variable expression parameter must be wrapped in parentheses";
return Err(psess.dcx().struct_span_err(ident.span, msg));
};
check_trailing_token(&mut tts, psess)?;
let mut iter = args.trees();
check_trailing_token(&mut iter, psess)?;
let mut iter = args.iter();
let rslt = match ident.as_str() {
"concat" => {
let mut result = Vec::new();
@ -143,7 +143,7 @@ pub(crate) enum MetaVarExprConcatElem {
// Checks if there are any remaining tokens. For example, `${ignore(ident ... a b c ...)}`
fn check_trailing_token<'psess>(
iter: &mut RefTokenTreeCursor<'_>,
iter: &mut TokenStreamIter<'_>,
psess: &'psess ParseSess,
) -> PResult<'psess, ()> {
if let Some(tt) = iter.next() {
@ -159,7 +159,7 @@ fn check_trailing_token<'psess>(
/// Parse a meta-variable `count` expression: `count(ident[, depth])`
fn parse_count<'psess>(
iter: &mut RefTokenTreeCursor<'_>,
iter: &mut TokenStreamIter<'_>,
psess: &'psess ParseSess,
span: Span,
) -> PResult<'psess, MetaVarExpr> {
@ -181,7 +181,7 @@ fn parse_count<'psess>(
/// Parses the depth used by index(depth) and len(depth).
fn parse_depth<'psess>(
iter: &mut RefTokenTreeCursor<'_>,
iter: &mut TokenStreamIter<'_>,
psess: &'psess ParseSess,
span: Span,
) -> PResult<'psess, usize> {
@ -204,7 +204,7 @@ fn parse_depth<'psess>(
/// Parses an generic ident
fn parse_ident<'psess>(
iter: &mut RefTokenTreeCursor<'_>,
iter: &mut TokenStreamIter<'_>,
psess: &'psess ParseSess,
fallback_span: Span,
) -> PResult<'psess, Ident> {
@ -236,7 +236,7 @@ fn parse_ident_from_token<'psess>(
}
fn parse_token<'psess, 't>(
iter: &mut RefTokenTreeCursor<'t>,
iter: &mut TokenStreamIter<'t>,
psess: &'psess ParseSess,
fallback_span: Span,
) -> PResult<'psess, &'t Token> {
@ -251,7 +251,7 @@ fn parse_token<'psess, 't>(
/// Tries to move the iterator forward returning `true` if there is a comma. If not, then the
/// iterator is not modified and the result is `false`.
fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool {
fn try_eat_comma(iter: &mut TokenStreamIter<'_>) -> bool {
if let Some(TokenTree::Token(Token { kind: token::Comma, .. }, _)) = iter.peek() {
let _ = iter.next();
return true;
@ -261,7 +261,7 @@ fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool {
/// Tries to move the iterator forward returning `true` if there is a dollar sign. If not, then the
/// iterator is not modified and the result is `false`.
fn try_eat_dollar(iter: &mut RefTokenTreeCursor<'_>) -> bool {
fn try_eat_dollar(iter: &mut TokenStreamIter<'_>) -> bool {
if let Some(TokenTree::Token(Token { kind: token::Dollar, .. }, _)) = iter.peek() {
let _ = iter.next();
return true;
@ -271,7 +271,7 @@ fn try_eat_dollar(iter: &mut RefTokenTreeCursor<'_>) -> bool {
/// Expects that the next item is a dollar sign.
fn eat_dollar<'psess>(
iter: &mut RefTokenTreeCursor<'_>,
iter: &mut TokenStreamIter<'_>,
psess: &'psess ParseSess,
span: Span,
) -> PResult<'psess, ()> {

42
compiler/rustc_expand/src/mbe/quoted.rs
View file

@ -49,25 +49,25 @@ pub(super) fn parse(
// For each token tree in `input`, parse the token into a `self::TokenTree`, consuming
// additional trees if need be.
let mut trees = input.trees().peekable();
while let Some(tree) = trees.next() {
let mut iter = input.iter().peekable();
while let Some(tree) = iter.next() {
// Given the parsed tree, if there is a metavar and we are expecting matchers, actually
// parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`).
let tree = parse_tree(tree, &mut trees, parsing_patterns, sess, node_id, features, edition);
let tree = parse_tree(tree, &mut iter, parsing_patterns, sess, node_id, features, edition);
match tree {
TokenTree::MetaVar(start_sp, ident) if parsing_patterns => {
// Not consuming the next token immediately, as it may not be a colon
let span = match trees.peek() {
let span = match iter.peek() {
Some(&tokenstream::TokenTree::Token(
Token { kind: token::Colon, span: colon_span },
_,
)) => {
// Consume the colon first
trees.next();
iter.next();
// It's ok to consume the next tree no matter how,
// since if it's not a token then it will be an invalid declaration.
match trees.next() {
match iter.next() {
Some(tokenstream::TokenTree::Token(token, _)) => match token.ident() {
Some((fragment, _)) => {
let span = token.span.with_lo(start_sp.lo());
@ -143,14 +143,14 @@ fn maybe_emit_macro_metavar_expr_concat_feature(features: &Features, sess: &Sess
/// # Parameters
///
/// - `tree`: the tree we wish to convert.
/// - `outer_trees`: an iterator over trees. We may need to read more tokens from it in order to finish
/// - `outer_iter`: an iterator over trees. We may need to read more tokens from it in order to finish
/// converting `tree`
/// - `parsing_patterns`: same as [parse].
/// - `sess`: the parsing session. Any errors will be emitted to this session.
/// - `features`: language features so we can do feature gating.
fn parse_tree<'a>(
tree: &'a tokenstream::TokenTree,
outer_trees: &mut impl Iterator<Item = &'a tokenstream::TokenTree>,
outer_iter: &mut impl Iterator<Item = &'a tokenstream::TokenTree>,
parsing_patterns: bool,
sess: &Session,
node_id: NodeId,
@ -163,14 +163,14 @@ fn parse_tree<'a>(
&tokenstream::TokenTree::Token(Token { kind: token::Dollar, span: dollar_span }, _) => {
// FIXME: Handle `Invisible`-delimited groups in a more systematic way
// during parsing.
let mut next = outer_trees.next();
let mut trees: Box<dyn Iterator<Item = &tokenstream::TokenTree>>;
let mut next = outer_iter.next();
let mut iter: Box<dyn Iterator<Item = &tokenstream::TokenTree>>;
match next {
Some(tokenstream::TokenTree::Delimited(.., delim, tts)) if delim.skip() => {
trees = Box::new(tts.trees());
next = trees.next();
iter = Box::new(tts.iter());
next = iter.next();
}
_ => trees = Box::new(outer_trees),
_ => iter = Box::new(outer_iter),
}
match next {
@ -230,7 +230,7 @@ fn parse_tree<'a>(
let sequence = parse(tts, parsing_patterns, sess, node_id, features, edition);
// Get the Kleene operator and optional separator
let (separator, kleene) =
parse_sep_and_kleene_op(&mut trees, delim_span.entire(), sess);
parse_sep_and_kleene_op(&mut iter, delim_span.entire(), sess);
// Count the number of captured "names" (i.e., named metavars)
let num_captures =
if parsing_patterns { count_metavar_decls(&sequence) } else { 0 };
@ -314,10 +314,10 @@ fn kleene_op(token: &Token) -> Option<KleeneOp> {
/// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp
/// - Err(span) if the next token tree is not a token
fn parse_kleene_op<'a>(
input: &mut impl Iterator<Item = &'a tokenstream::TokenTree>,
iter: &mut impl Iterator<Item = &'a tokenstream::TokenTree>,
span: Span,
) -> Result<Result<(KleeneOp, Span), Token>, Span> {
match input.next() {
match iter.next() {
Some(tokenstream::TokenTree::Token(token, _)) => match kleene_op(token) {
Some(op) => Ok(Ok((op, token.span))),
None => Ok(Err(token.clone())),
@ -334,22 +334,22 @@ fn parse_kleene_op<'a>(
/// itself. Note that here we are parsing the _macro_ itself, rather than trying to match some
/// stream of tokens in an invocation of a macro.
///
/// This function will take some input iterator `input` corresponding to `span` and a parsing
/// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene
/// This function will take some input iterator `iter` corresponding to `span` and a parsing
/// session `sess`. If the next one (or possibly two) tokens in `iter` correspond to a Kleene
/// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an
/// error with the appropriate span is emitted to `sess` and a dummy value is returned.
fn parse_sep_and_kleene_op<'a>(
input: &mut impl Iterator<Item = &'a tokenstream::TokenTree>,
iter: &mut impl Iterator<Item = &'a tokenstream::TokenTree>,
span: Span,
sess: &Session,
) -> (Option<Token>, KleeneToken) {
// We basically look at two token trees here, denoted as #1 and #2 below
let span = match parse_kleene_op(input, span) {
let span = match parse_kleene_op(iter, span) {
// #1 is a `?`, `+`, or `*` KleeneOp
Ok(Ok((op, span))) => return (None, KleeneToken::new(op, span)),
// #1 is a separator followed by #2, a KleeneOp
Ok(Err(token)) => match parse_kleene_op(input, token.span) {
Ok(Err(token)) => match parse_kleene_op(iter, token.span) {
// #2 is the `?` Kleene op, which does not take a separator (error)
Ok(Ok((KleeneOp::ZeroOrOne, span))) => {
// Error!

4
compiler/rustc_expand/src/proc_macro_server.rs
View file

@ -112,9 +112,9 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
// Estimate the capacity as `stream.len()` rounded up to the next power
// of two to limit the number of required reallocations.
let mut trees = Vec::with_capacity(stream.len().next_power_of_two());
let mut cursor = stream.trees();
let mut iter = stream.iter();
while let Some(tree) = cursor.next() {
while let Some(tree) = iter.next() {
let (Token { kind, span }, joint) = match tree.clone() {
tokenstream::TokenTree::Delimited(span, _, delim, tts) => {
let delimiter = pm::Delimiter::from_internal(delim);