bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

rustc_ast: Harmonize delimiter naming with proc_macro::Delimiter

Browse source
This commit is contained in:
Vadim Petrochenkov 2022-04-26 15:40:14 +03:00
parent 0e7915d11f
commit 2733ec1be3
41 changed files with 433 additions and 426 deletions
Download patch file Download diff file Expand all files Collapse all files

28
compiler/rustc_parse/src/parser/attr_wrapper.rs
View file

@ -1,5 +1,5 @@
use super::{Capturing, FlatToken, ForceCollect, Parser, ReplaceRange, TokenCursor, TrailingToken};
use rustc_ast::token::{self, DelimToken, Token, TokenKind};
use rustc_ast::token::{self, Delimiter, Token, TokenKind};
use rustc_ast::tokenstream::{AttrAnnotatedTokenStream, AttributesData, CreateTokenStream};
use rustc_ast::tokenstream::{AttrAnnotatedTokenTree, DelimSpan, LazyTokenStream, Spacing};
use rustc_ast::{self as ast};
@ -388,11 +388,11 @@ impl<'a> Parser<'a> {
/// Converts a flattened iterator of tokens (including open and close delimiter tokens)
/// into a `TokenStream`, creating a `TokenTree::Delimited` for each matching pair
/// of open and close delims.
// FIXME(#67062): Currently, we don't parse `None`-delimited groups correctly,
// which can cause us to end up with mismatched `None` delimiters in our
// FIXME(#67062): Currently, we don't parse `Invisible`-delimited groups correctly,
// which can cause us to end up with mismatched `Invisible` delimiters in our
// captured tokens. This function contains several hacks to work around this -
// essentially, we throw away mismatched `None` delimiters when we encounter them.
// Once we properly parse `None` delimiters, they can be captured just like any
// essentially, we throw away mismatched `Invisible` delimiters when we encounter them.
// Once we properly parse `Invisible` delimiters, they can be captured just like any
// other tokens, and these hacks can be removed.
fn make_token_stream(
mut iter: impl Iterator<Item = (FlatToken, Spacing)>,
@ -401,7 +401,7 @@ fn make_token_stream(
#[derive(Debug)]
struct FrameData {
// This is `None` for the first frame, `Some` for all others.
open_delim_sp: Option<(DelimToken, Span)>,
open_delim_sp: Option<(Delimiter, Span)>,
inner: Vec<(AttrAnnotatedTokenTree, Spacing)>,
}
let mut stack = vec![FrameData { open_delim_sp: None, inner: vec![] }];
@ -412,13 +412,13 @@ fn make_token_stream(
stack.push(FrameData { open_delim_sp: Some((delim, span)), inner: vec![] });
}
FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => {
// HACK: If we encounter a mismatched `None` delimiter at the top
// HACK: If we encounter a mismatched `Invisible` delimiter at the top
// level, just ignore it.
if matches!(delim, DelimToken::NoDelim)
if matches!(delim, Delimiter::Invisible)
&& (stack.len() == 1
|| !matches!(
stack.last_mut().unwrap().open_delim_sp.unwrap().0,
DelimToken::NoDelim
Delimiter::Invisible
))
{
token_and_spacing = iter.next();
@ -428,11 +428,11 @@ fn make_token_stream(
.pop()
.unwrap_or_else(|| panic!("Token stack was empty for token: {:?}", token));
// HACK: If our current frame has a mismatched opening `None` delimiter,
// HACK: If our current frame has a mismatched opening `Invisible` delimiter,
// merge our current frame with the one above it. That is, transform
// `[ { < first second } third ]` into `[ { first second } third ]`
if !matches!(delim, DelimToken::NoDelim)
&& matches!(frame_data.open_delim_sp.unwrap().0, DelimToken::NoDelim)
if !matches!(delim, Delimiter::Invisible)
&& matches!(frame_data.open_delim_sp.unwrap().0, Delimiter::Invisible)
{
stack.last_mut().unwrap().inner.extend(frame_data.inner);
// Process our closing delimiter again, this time at the previous
@ -472,10 +472,10 @@ fn make_token_stream(
}
token_and_spacing = iter.next();
}
// HACK: If we don't have a closing `None` delimiter for our last
// HACK: If we don't have a closing `Invisible` delimiter for our last
// frame, merge the frame with the top-level frame. That is,
// turn `< first second` into `first second`
if stack.len() == 2 && stack[1].open_delim_sp.unwrap().0 == DelimToken::NoDelim {
if stack.len() == 2 && stack[1].open_delim_sp.unwrap().0 == Delimiter::Invisible {
let temp_buf = stack.pop().unwrap();
stack.last_mut().unwrap().inner.extend(temp_buf.inner);
}