1
Fork 0

Remove Peekable<TokenStreamIter> uses.

Currently there are two ways to peek at a `TokenStreamIter`.
- Wrap it in a `Peekable` and use that traits `peek` method.
- Use `TokenStreamIter`'s inherent `peek` method.

Some code uses one, some use the other. This commit converts all places
to the inherent method. This eliminates mixing of `TokenStreamIter` and
`Peekable<TokenStreamIter>` and some use of `impl Iterator` and `dyn
Iterator`.
This commit is contained in:
Nicholas Nethercote 2024-12-11 14:58:19 +11:00
parent 809975c94a
commit c82d5865f2
3 changed files with 27 additions and 31 deletions

View file

@ -1,7 +1,6 @@
//! Functions dealing with attributes and meta items. //! Functions dealing with attributes and meta items.
use std::fmt::Debug; use std::fmt::Debug;
use std::iter;
use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::atomic::{AtomicU32, Ordering};
use rustc_index::bit_set::GrowableBitSet; use rustc_index::bit_set::GrowableBitSet;
@ -17,7 +16,9 @@ use crate::ast::{
}; };
use crate::ptr::P; use crate::ptr::P;
use crate::token::{self, CommentKind, Delimiter, Token}; use crate::token::{self, CommentKind, Delimiter, Token};
use crate::tokenstream::{DelimSpan, LazyAttrTokenStream, Spacing, TokenStream, TokenTree}; use crate::tokenstream::{
DelimSpan, LazyAttrTokenStream, Spacing, TokenStream, TokenStreamIter, TokenTree,
};
use crate::util::comments; use crate::util::comments;
use crate::util::literal::escape_string_symbol; use crate::util::literal::escape_string_symbol;
@ -366,10 +367,7 @@ impl MetaItem {
} }
} }
fn from_tokens<'a, I>(iter: &mut iter::Peekable<I>) -> Option<MetaItem> fn from_tokens(iter: &mut TokenStreamIter<'_>) -> Option<MetaItem> {
where
I: Iterator<Item = &'a TokenTree>,
{
// FIXME: Share code with `parse_path`. // FIXME: Share code with `parse_path`.
let tt = iter.next().map(|tt| TokenTree::uninterpolate(tt)); let tt = iter.next().map(|tt| TokenTree::uninterpolate(tt));
let path = match tt.as_deref() { let path = match tt.as_deref() {
@ -439,7 +437,7 @@ impl MetaItem {
impl MetaItemKind { impl MetaItemKind {
// public because it can be called in the hir // public because it can be called in the hir
pub fn list_from_tokens(tokens: TokenStream) -> Option<ThinVec<MetaItemInner>> { pub fn list_from_tokens(tokens: TokenStream) -> Option<ThinVec<MetaItemInner>> {
let mut iter = tokens.iter().peekable(); let mut iter = tokens.iter();
let mut result = ThinVec::new(); let mut result = ThinVec::new();
while iter.peek().is_some() { while iter.peek().is_some() {
let item = MetaItemInner::from_tokens(&mut iter)?; let item = MetaItemInner::from_tokens(&mut iter)?;
@ -452,9 +450,7 @@ impl MetaItemKind {
Some(result) Some(result)
} }
fn name_value_from_tokens<'a>( fn name_value_from_tokens(iter: &mut TokenStreamIter<'_>) -> Option<MetaItemKind> {
iter: &mut impl Iterator<Item = &'a TokenTree>,
) -> Option<MetaItemKind> {
match iter.next() { match iter.next() {
Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) => { Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) => {
MetaItemKind::name_value_from_tokens(&mut inner_tokens.iter()) MetaItemKind::name_value_from_tokens(&mut inner_tokens.iter())
@ -466,9 +462,7 @@ impl MetaItemKind {
} }
} }
fn from_tokens<'a>( fn from_tokens(iter: &mut TokenStreamIter<'_>) -> Option<MetaItemKind> {
iter: &mut iter::Peekable<impl Iterator<Item = &'a TokenTree>>,
) -> Option<MetaItemKind> {
match iter.peek() { match iter.peek() {
Some(TokenTree::Delimited(.., Delimiter::Parenthesis, inner_tokens)) => { Some(TokenTree::Delimited(.., Delimiter::Parenthesis, inner_tokens)) => {
let inner_tokens = inner_tokens.clone(); let inner_tokens = inner_tokens.clone();
@ -594,10 +588,7 @@ impl MetaItemInner {
self.meta_item().is_some() self.meta_item().is_some()
} }
fn from_tokens<'a, I>(iter: &mut iter::Peekable<I>) -> Option<MetaItemInner> fn from_tokens(iter: &mut TokenStreamIter<'_>) -> Option<MetaItemInner> {
where
I: Iterator<Item = &'a TokenTree>,
{
match iter.peek() { match iter.peek() {
Some(TokenTree::Token(token, _)) if let Some(lit) = MetaItemLit::from_token(token) => { Some(TokenTree::Token(token, _)) if let Some(lit) = MetaItemLit::from_token(token) => {
iter.next(); iter.next();
@ -605,7 +596,7 @@ impl MetaItemInner {
} }
Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) => { Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) => {
iter.next(); iter.next();
return MetaItemInner::from_tokens(&mut inner_tokens.iter().peekable()); return MetaItemInner::from_tokens(&mut inner_tokens.iter());
} }
_ => {} _ => {}
} }

View file

@ -676,7 +676,10 @@ impl<'t> TokenStreamIter<'t> {
TokenStreamIter { stream, index: 0 } TokenStreamIter { stream, index: 0 }
} }
pub fn peek(&self) -> Option<&TokenTree> { // Peeking could be done via `Peekable`, but most iterators need peeking,
// and this is simple and avoids the need to use `peekable` and `Peekable`
// at all the use sites.
pub fn peek(&self) -> Option<&'t TokenTree> {
self.stream.0.get(self.index) self.stream.0.get(self.index)
} }
} }

View file

@ -1,4 +1,5 @@
use rustc_ast::token::{self, Delimiter, IdentIsRaw, NonterminalKind, Token}; use rustc_ast::token::{self, Delimiter, IdentIsRaw, NonterminalKind, Token};
use rustc_ast::tokenstream::TokenStreamIter;
use rustc_ast::{NodeId, tokenstream}; use rustc_ast::{NodeId, tokenstream};
use rustc_ast_pretty::pprust; use rustc_ast_pretty::pprust;
use rustc_feature::Features; use rustc_feature::Features;
@ -49,7 +50,7 @@ pub(super) fn parse(
// For each token tree in `input`, parse the token into a `self::TokenTree`, consuming // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming
// additional trees if need be. // additional trees if need be.
let mut iter = input.iter().peekable(); let mut iter = input.iter();
while let Some(tree) = iter.next() { while let Some(tree) = iter.next() {
// Given the parsed tree, if there is a metavar and we are expecting matchers, actually // Given the parsed tree, if there is a metavar and we are expecting matchers, actually
// parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`). // parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`).
@ -150,7 +151,7 @@ fn maybe_emit_macro_metavar_expr_concat_feature(features: &Features, sess: &Sess
/// - `features`: language features so we can do feature gating. /// - `features`: language features so we can do feature gating.
fn parse_tree<'a>( fn parse_tree<'a>(
tree: &'a tokenstream::TokenTree, tree: &'a tokenstream::TokenTree,
outer_iter: &mut impl Iterator<Item = &'a tokenstream::TokenTree>, outer_iter: &mut TokenStreamIter<'a>,
parsing_patterns: bool, parsing_patterns: bool,
sess: &Session, sess: &Session,
node_id: NodeId, node_id: NodeId,
@ -164,14 +165,15 @@ fn parse_tree<'a>(
// FIXME: Handle `Invisible`-delimited groups in a more systematic way // FIXME: Handle `Invisible`-delimited groups in a more systematic way
// during parsing. // during parsing.
let mut next = outer_iter.next(); let mut next = outer_iter.next();
let mut iter: Box<dyn Iterator<Item = &tokenstream::TokenTree>>; let mut iter_storage;
match next { let mut iter: &mut TokenStreamIter<'_> = match next {
Some(tokenstream::TokenTree::Delimited(.., delim, tts)) if delim.skip() => { Some(tokenstream::TokenTree::Delimited(.., delim, tts)) if delim.skip() => {
iter = Box::new(tts.iter()); iter_storage = tts.iter();
next = iter.next(); next = iter_storage.next();
&mut iter_storage
} }
_ => iter = Box::new(outer_iter), _ => outer_iter,
} };
match next { match next {
// `tree` is followed by a delimited set of token trees. // `tree` is followed by a delimited set of token trees.
@ -313,8 +315,8 @@ fn kleene_op(token: &Token) -> Option<KleeneOp> {
/// - Ok(Ok((op, span))) if the next token tree is a KleeneOp /// - Ok(Ok((op, span))) if the next token tree is a KleeneOp
/// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp /// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp
/// - Err(span) if the next token tree is not a token /// - Err(span) if the next token tree is not a token
fn parse_kleene_op<'a>( fn parse_kleene_op(
iter: &mut impl Iterator<Item = &'a tokenstream::TokenTree>, iter: &mut TokenStreamIter<'_>,
span: Span, span: Span,
) -> Result<Result<(KleeneOp, Span), Token>, Span> { ) -> Result<Result<(KleeneOp, Span), Token>, Span> {
match iter.next() { match iter.next() {
@ -338,8 +340,8 @@ fn parse_kleene_op<'a>(
/// session `sess`. If the next one (or possibly two) tokens in `iter` correspond to a Kleene /// session `sess`. If the next one (or possibly two) tokens in `iter` correspond to a Kleene
/// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an /// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an
/// error with the appropriate span is emitted to `sess` and a dummy value is returned. /// error with the appropriate span is emitted to `sess` and a dummy value is returned.
fn parse_sep_and_kleene_op<'a>( fn parse_sep_and_kleene_op(
iter: &mut impl Iterator<Item = &'a tokenstream::TokenTree>, iter: &mut TokenStreamIter<'_>,
span: Span, span: Span,
sess: &Session, sess: &Session,
) -> (Option<Token>, KleeneToken) { ) -> (Option<Token>, KleeneToken) {