1
Fork 0

Rollup merge of #96421 - nnethercote:less-NoDelim, r=petrochenkov

Less `NoDelim`

Currently there are several places where `NoDelim` (which really means "implicit delimiter" or "invisible delimiter") is used to mean "no delimiter". The name `NoDelim` is a bit misleading, and may be a cause.

This PR changes these places, e.g. by changing a `DelimToken` to `Option<DelimToken>` and then using `None` to mean "no delimiter". As a result, the *only* place where `NoDelim` values are now produced is within:
- `Delimiter::to_internal()`, when converting from `Delimiter::None`.
- `FlattenNonterminals::process_token()`, when converting `TokenKind::Interpolated`.

r? ````@petrochenkov````
This commit is contained in:
Dylan DPC 2022-04-28 02:40:34 +02:00 committed by GitHub
commit 80045d65e1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 86 additions and 84 deletions

View file

@ -1542,10 +1542,10 @@ pub enum MacArgs {
} }
impl MacArgs { impl MacArgs {
pub fn delim(&self) -> DelimToken { pub fn delim(&self) -> Option<DelimToken> {
match self { match self {
MacArgs::Delimited(_, delim, _) => delim.to_token(), MacArgs::Delimited(_, delim, _) => Some(delim.to_token()),
MacArgs::Empty | MacArgs::Eq(..) => token::NoDelim, MacArgs::Empty | MacArgs::Eq(..) => None,
} }
} }

View file

@ -464,7 +464,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
Some(MacHeader::Path(&item.path)), Some(MacHeader::Path(&item.path)),
false, false,
None, None,
delim.to_token(), Some(delim.to_token()),
tokens, tokens,
true, true,
span, span,
@ -530,7 +530,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
None, None,
false, false,
None, None,
*delim, Some(*delim),
tts, tts,
convert_dollar_crate, convert_dollar_crate,
dspan.entire(), dspan.entire(),
@ -556,12 +556,12 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
header: Option<MacHeader<'_>>, header: Option<MacHeader<'_>>,
has_bang: bool, has_bang: bool,
ident: Option<Ident>, ident: Option<Ident>,
delim: DelimToken, delim: Option<DelimToken>,
tts: &TokenStream, tts: &TokenStream,
convert_dollar_crate: bool, convert_dollar_crate: bool,
span: Span, span: Span,
) { ) {
if delim == DelimToken::Brace { if delim == Some(DelimToken::Brace) {
self.cbox(INDENT_UNIT); self.cbox(INDENT_UNIT);
} }
match header { match header {
@ -577,7 +577,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
self.print_ident(ident); self.print_ident(ident);
} }
match delim { match delim {
DelimToken::Brace => { Some(DelimToken::Brace) => {
if header.is_some() || has_bang || ident.is_some() { if header.is_some() || has_bang || ident.is_some() {
self.nbsp(); self.nbsp();
} }
@ -585,23 +585,25 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
if !tts.is_empty() { if !tts.is_empty() {
self.space(); self.space();
} }
} self.ibox(0);
_ => { self.print_tts(tts, convert_dollar_crate);
let token_str = self.token_kind_to_string(&token::OpenDelim(delim)); self.end();
self.word(token_str)
}
}
self.ibox(0);
self.print_tts(tts, convert_dollar_crate);
self.end();
match delim {
DelimToken::Brace => {
let empty = tts.is_empty(); let empty = tts.is_empty();
self.bclose(span, empty); self.bclose(span, empty);
} }
_ => { Some(delim) => {
let token_str = self.token_kind_to_string(&token::OpenDelim(delim));
self.word(token_str);
self.ibox(0);
self.print_tts(tts, convert_dollar_crate);
self.end();
let token_str = self.token_kind_to_string(&token::CloseDelim(delim)); let token_str = self.token_kind_to_string(&token::CloseDelim(delim));
self.word(token_str) self.word(token_str);
}
None => {
self.ibox(0);
self.print_tts(tts, convert_dollar_crate);
self.end();
} }
} }
} }

View file

@ -260,16 +260,15 @@ fn generic_extension<'cx, 'tt>(
// Merge the gated spans from parsing the matcher with the pre-existing ones. // Merge the gated spans from parsing the matcher with the pre-existing ones.
sess.gated_spans.merge(gated_spans_snapshot); sess.gated_spans.merge(gated_spans_snapshot);
// Ignore the delimiters on the RHS. let (rhs, rhs_span): (&mbe::Delimited, DelimSpan) = match &rhses[i] {
let rhs = match &rhses[i] { mbe::TokenTree::Delimited(span, delimited) => (&delimited, *span),
mbe::TokenTree::Delimited(_, delimited) => &delimited.tts,
_ => cx.span_bug(sp, "malformed macro rhs"), _ => cx.span_bug(sp, "malformed macro rhs"),
}; };
let arm_span = rhses[i].span(); let arm_span = rhses[i].span();
let rhs_spans = rhs.iter().map(|t| t.span()).collect::<Vec<_>>(); let rhs_spans = rhs.tts.iter().map(|t| t.span()).collect::<Vec<_>>();
// rhs has holes ( `$id` and `$(...)` that need filled) // rhs has holes ( `$id` and `$(...)` that need filled)
let mut tts = match transcribe(cx, &named_matches, &rhs, transparency) { let mut tts = match transcribe(cx, &named_matches, &rhs, rhs_span, transparency) {
Ok(tts) => tts, Ok(tts) => tts,
Err(mut err) => { Err(mut err) => {
err.emit(); err.emit();

View file

@ -29,8 +29,8 @@ impl MutVisitor for Marker {
enum Frame<'a> { enum Frame<'a> {
Delimited { Delimited {
tts: &'a [mbe::TokenTree], tts: &'a [mbe::TokenTree],
delim_token: token::DelimToken,
idx: usize, idx: usize,
delim_token: token::DelimToken,
span: DelimSpan, span: DelimSpan,
}, },
Sequence { Sequence {
@ -42,8 +42,8 @@ enum Frame<'a> {
impl<'a> Frame<'a> { impl<'a> Frame<'a> {
/// Construct a new frame around the delimited set of tokens. /// Construct a new frame around the delimited set of tokens.
fn new(tts: &'a [mbe::TokenTree]) -> Frame<'a> { fn new(src: &'a mbe::Delimited, span: DelimSpan) -> Frame<'a> {
Frame::Delimited { tts, delim_token: token::NoDelim, idx: 0, span: DelimSpan::dummy() } Frame::Delimited { tts: &src.tts, idx: 0, delim_token: src.delim, span }
} }
} }
@ -85,17 +85,18 @@ impl<'a> Iterator for Frame<'a> {
pub(super) fn transcribe<'a>( pub(super) fn transcribe<'a>(
cx: &ExtCtxt<'a>, cx: &ExtCtxt<'a>,
interp: &FxHashMap<MacroRulesNormalizedIdent, NamedMatch>, interp: &FxHashMap<MacroRulesNormalizedIdent, NamedMatch>,
src: &[mbe::TokenTree], src: &mbe::Delimited,
src_span: DelimSpan,
transparency: Transparency, transparency: Transparency,
) -> PResult<'a, TokenStream> { ) -> PResult<'a, TokenStream> {
// Nothing for us to transcribe... // Nothing for us to transcribe...
if src.is_empty() { if src.tts.is_empty() {
return Ok(TokenStream::default()); return Ok(TokenStream::default());
} }
// We descend into the RHS (`src`), expanding things as we go. This stack contains the things // We descend into the RHS (`src`), expanding things as we go. This stack contains the things
// we have yet to expand/are still expanding. We start the stack off with the whole RHS. // we have yet to expand/are still expanding. We start the stack off with the whole RHS.
let mut stack: SmallVec<[Frame<'_>; 1]> = smallvec![Frame::new(&src)]; let mut stack: SmallVec<[Frame<'_>; 1]> = smallvec![Frame::new(&src, src_span)];
// As we descend in the RHS, we will need to be able to match nested sequences of matchers. // As we descend in the RHS, we will need to be able to match nested sequences of matchers.
// `repeats` keeps track of where we are in matching at each level, with the last element being // `repeats` keeps track of where we are in matching at each level, with the last element being

View file

@ -5,7 +5,7 @@ use rustc_ast::tokenstream::{AttrAnnotatedTokenTree, DelimSpan, LazyTokenStream,
use rustc_ast::{self as ast}; use rustc_ast::{self as ast};
use rustc_ast::{AstLike, AttrVec, Attribute}; use rustc_ast::{AstLike, AttrVec, Attribute};
use rustc_errors::PResult; use rustc_errors::PResult;
use rustc_span::{sym, Span, DUMMY_SP}; use rustc_span::{sym, Span};
use std::convert::TryInto; use std::convert::TryInto;
use std::ops::Range; use std::ops::Range;
@ -400,24 +400,26 @@ fn make_token_stream(
) -> AttrAnnotatedTokenStream { ) -> AttrAnnotatedTokenStream {
#[derive(Debug)] #[derive(Debug)]
struct FrameData { struct FrameData {
open: Span, // This is `None` for the first frame, `Some` for all others.
open_delim: DelimToken, open_delim_sp: Option<(DelimToken, Span)>,
inner: Vec<(AttrAnnotatedTokenTree, Spacing)>, inner: Vec<(AttrAnnotatedTokenTree, Spacing)>,
} }
let mut stack = let mut stack = vec![FrameData { open_delim_sp: None, inner: vec![] }];
vec![FrameData { open: DUMMY_SP, open_delim: DelimToken::NoDelim, inner: vec![] }];
let mut token_and_spacing = iter.next(); let mut token_and_spacing = iter.next();
while let Some((token, spacing)) = token_and_spacing { while let Some((token, spacing)) = token_and_spacing {
match token { match token {
FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span }) => { FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span }) => {
stack.push(FrameData { open: span, open_delim: delim, inner: vec![] }); stack.push(FrameData { open_delim_sp: Some((delim, span)), inner: vec![] });
} }
FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => { FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => {
// HACK: If we encounter a mismatched `None` delimiter at the top // HACK: If we encounter a mismatched `None` delimiter at the top
// level, just ignore it. // level, just ignore it.
if matches!(delim, DelimToken::NoDelim) if matches!(delim, DelimToken::NoDelim)
&& (stack.len() == 1 && (stack.len() == 1
|| !matches!(stack.last_mut().unwrap().open_delim, DelimToken::NoDelim)) || !matches!(
stack.last_mut().unwrap().open_delim_sp.unwrap().0,
DelimToken::NoDelim
))
{ {
token_and_spacing = iter.next(); token_and_spacing = iter.next();
continue; continue;
@ -430,7 +432,7 @@ fn make_token_stream(
// merge our current frame with the one above it. That is, transform // merge our current frame with the one above it. That is, transform
// `[ { < first second } third ]` into `[ { first second } third ]` // `[ { < first second } third ]` into `[ { first second } third ]`
if !matches!(delim, DelimToken::NoDelim) if !matches!(delim, DelimToken::NoDelim)
&& matches!(frame_data.open_delim, DelimToken::NoDelim) && matches!(frame_data.open_delim_sp.unwrap().0, DelimToken::NoDelim)
{ {
stack.last_mut().unwrap().inner.extend(frame_data.inner); stack.last_mut().unwrap().inner.extend(frame_data.inner);
// Process our closing delimiter again, this time at the previous // Process our closing delimiter again, this time at the previous
@ -439,12 +441,13 @@ fn make_token_stream(
continue; continue;
} }
let (open_delim, open_sp) = frame_data.open_delim_sp.unwrap();
assert_eq!( assert_eq!(
frame_data.open_delim, delim, open_delim, delim,
"Mismatched open/close delims: open={:?} close={:?}", "Mismatched open/close delims: open={:?} close={:?}",
frame_data.open, span open_delim, span
); );
let dspan = DelimSpan::from_pair(frame_data.open, span); let dspan = DelimSpan::from_pair(open_sp, span);
let stream = AttrAnnotatedTokenStream::new(frame_data.inner); let stream = AttrAnnotatedTokenStream::new(frame_data.inner);
let delimited = AttrAnnotatedTokenTree::Delimited(dspan, delim, stream); let delimited = AttrAnnotatedTokenTree::Delimited(dspan, delim, stream);
stack stack
@ -472,7 +475,7 @@ fn make_token_stream(
// HACK: If we don't have a closing `None` delimiter for our last // HACK: If we don't have a closing `None` delimiter for our last
// frame, merge the frame with the top-level frame. That is, // frame, merge the frame with the top-level frame. That is,
// turn `< first second` into `first second` // turn `< first second` into `first second`
if stack.len() == 2 && stack[1].open_delim == DelimToken::NoDelim { if stack.len() == 2 && stack[1].open_delim_sp.unwrap().0 == DelimToken::NoDelim {
let temp_buf = stack.pop().unwrap(); let temp_buf = stack.pop().unwrap();
stack.last_mut().unwrap().inner.extend(temp_buf.inner); stack.last_mut().unwrap().inner.extend(temp_buf.inner);
} }

View file

@ -2043,7 +2043,8 @@ impl<'a> Parser<'a> {
self.sess.gated_spans.gate(sym::async_closure, span); self.sess.gated_spans.gate(sym::async_closure, span);
} }
if self.token.kind == TokenKind::Semi && self.token_cursor.frame.delim == DelimToken::Paren if self.token.kind == TokenKind::Semi
&& matches!(self.token_cursor.frame.delim_sp, Some((DelimToken::Paren, _)))
{ {
// It is likely that the closure body is a block but where the // It is likely that the closure body is a block but where the
// braces have been removed. We will recover and eat the next // braces have been removed. We will recover and eat the next

View file

@ -244,14 +244,13 @@ struct TokenCursor {
#[derive(Clone)] #[derive(Clone)]
struct TokenCursorFrame { struct TokenCursorFrame {
delim: token::DelimToken, delim_sp: Option<(DelimToken, DelimSpan)>,
span: DelimSpan,
tree_cursor: tokenstream::Cursor, tree_cursor: tokenstream::Cursor,
} }
impl TokenCursorFrame { impl TokenCursorFrame {
fn new(span: DelimSpan, delim: DelimToken, tts: TokenStream) -> Self { fn new(delim_sp: Option<(DelimToken, DelimSpan)>, tts: TokenStream) -> Self {
TokenCursorFrame { delim, span, tree_cursor: tts.into_trees() } TokenCursorFrame { delim_sp, tree_cursor: tts.into_trees() }
} }
} }
@ -266,7 +265,7 @@ impl TokenCursor {
loop { loop {
// FIXME: we currently don't return `NoDelim` open/close delims. To fix #67062 we will // FIXME: we currently don't return `NoDelim` open/close delims. To fix #67062 we will
// need to, whereupon the `delim != DelimToken::NoDelim` conditions below can be // need to, whereupon the `delim != DelimToken::NoDelim` conditions below can be
// removed, as well as the loop. // removed.
if let Some((tree, spacing)) = self.frame.tree_cursor.next_with_spacing_ref() { if let Some((tree, spacing)) = self.frame.tree_cursor.next_with_spacing_ref() {
match tree { match tree {
&TokenTree::Token(ref token) => match (desugar_doc_comments, token) { &TokenTree::Token(ref token) => match (desugar_doc_comments, token) {
@ -277,7 +276,7 @@ impl TokenCursor {
}, },
&TokenTree::Delimited(sp, delim, ref tts) => { &TokenTree::Delimited(sp, delim, ref tts) => {
// Set `open_delim` to true here because we deal with it immediately. // Set `open_delim` to true here because we deal with it immediately.
let frame = TokenCursorFrame::new(sp, delim, tts.clone()); let frame = TokenCursorFrame::new(Some((delim, sp)), tts.clone());
self.stack.push(mem::replace(&mut self.frame, frame)); self.stack.push(mem::replace(&mut self.frame, frame));
if delim != DelimToken::NoDelim { if delim != DelimToken::NoDelim {
return (Token::new(token::OpenDelim(delim), sp.open), Spacing::Alone); return (Token::new(token::OpenDelim(delim), sp.open), Spacing::Alone);
@ -286,12 +285,11 @@ impl TokenCursor {
} }
}; };
} else if let Some(frame) = self.stack.pop() { } else if let Some(frame) = self.stack.pop() {
let delim = self.frame.delim; if let Some((delim, span)) = self.frame.delim_sp && delim != DelimToken::NoDelim {
let span = self.frame.span; self.frame = frame;
self.frame = frame;
if delim != DelimToken::NoDelim {
return (Token::new(token::CloseDelim(delim), span.close), Spacing::Alone); return (Token::new(token::CloseDelim(delim), span.close), Spacing::Alone);
} }
self.frame = frame;
// No close delimiter to return; continue on to the next iteration. // No close delimiter to return; continue on to the next iteration.
} else { } else {
return (Token::new(token::Eof, DUMMY_SP), Spacing::Alone); return (Token::new(token::Eof, DUMMY_SP), Spacing::Alone);
@ -330,8 +328,7 @@ impl TokenCursor {
self.stack.push(mem::replace( self.stack.push(mem::replace(
&mut self.frame, &mut self.frame,
TokenCursorFrame::new( TokenCursorFrame::new(
delim_span, None,
token::NoDelim,
if attr_style == AttrStyle::Inner { if attr_style == AttrStyle::Inner {
[TokenTree::token(token::Pound, span), TokenTree::token(token::Not, span), body] [TokenTree::token(token::Pound, span), TokenTree::token(token::Not, span), body]
.iter() .iter()
@ -431,10 +428,6 @@ impl<'a> Parser<'a> {
desugar_doc_comments: bool, desugar_doc_comments: bool,
subparser_name: Option<&'static str>, subparser_name: Option<&'static str>,
) -> Self { ) -> Self {
// Note: because of the way `TokenCursor::inlined_next` is structured, the `span` and
// `delim` arguments here are never used.
let start_frame = TokenCursorFrame::new(DelimSpan::dummy(), token::NoDelim, tokens);
let mut parser = Parser { let mut parser = Parser {
sess, sess,
token: Token::dummy(), token: Token::dummy(),
@ -444,7 +437,7 @@ impl<'a> Parser<'a> {
restrictions: Restrictions::empty(), restrictions: Restrictions::empty(),
expected_tokens: Vec::new(), expected_tokens: Vec::new(),
token_cursor: TokenCursor { token_cursor: TokenCursor {
frame: start_frame, frame: TokenCursorFrame::new(None, tokens),
stack: Vec::new(), stack: Vec::new(),
num_next_calls: 0, num_next_calls: 0,
desugar_doc_comments, desugar_doc_comments,
@ -1025,7 +1018,7 @@ impl<'a> Parser<'a> {
} }
let frame = &self.token_cursor.frame; let frame = &self.token_cursor.frame;
if frame.delim != DelimToken::NoDelim { if let Some((delim, span)) = frame.delim_sp && delim != DelimToken::NoDelim {
let all_normal = (0..dist).all(|i| { let all_normal = (0..dist).all(|i| {
let token = frame.tree_cursor.look_ahead(i); let token = frame.tree_cursor.look_ahead(i);
!matches!(token, Some(TokenTree::Delimited(_, DelimToken::NoDelim, _))) !matches!(token, Some(TokenTree::Delimited(_, DelimToken::NoDelim, _)))
@ -1038,7 +1031,7 @@ impl<'a> Parser<'a> {
looker(&Token::new(token::OpenDelim(*delim), dspan.open)) looker(&Token::new(token::OpenDelim(*delim), dspan.open))
} }
}, },
None => looker(&Token::new(token::CloseDelim(frame.delim), frame.span.close)), None => looker(&Token::new(token::CloseDelim(delim), span.close)),
}; };
} }
} }
@ -1198,8 +1191,7 @@ impl<'a> Parser<'a> {
// Grab the tokens from this frame. // Grab the tokens from this frame.
let frame = &self.token_cursor.frame; let frame = &self.token_cursor.frame;
let stream = frame.tree_cursor.stream.clone(); let stream = frame.tree_cursor.stream.clone();
let span = frame.span; let (delim, span) = frame.delim_sp.unwrap();
let delim = frame.delim;
// Advance the token cursor through the entire delimited // Advance the token cursor through the entire delimited
// sequence. After getting the `OpenDelim` we are *within* the // sequence. After getting the `OpenDelim` we are *within* the

View file

@ -164,25 +164,29 @@ impl<'a> Parser<'a> {
let delim = args.delim(); let delim = args.delim();
let hi = self.prev_token.span; let hi = self.prev_token.span;
let style = let style = match delim {
if delim == token::Brace { MacStmtStyle::Braces } else { MacStmtStyle::NoBraces }; Some(token::Brace) => MacStmtStyle::Braces,
Some(_) => MacStmtStyle::NoBraces,
None => unreachable!(),
};
let mac = MacCall { path, args, prior_type_ascription: self.last_type_ascription }; let mac = MacCall { path, args, prior_type_ascription: self.last_type_ascription };
let kind = let kind = if (style == MacStmtStyle::Braces
if (delim == token::Brace && self.token != token::Dot && self.token != token::Question) && self.token != token::Dot
|| self.token == token::Semi && self.token != token::Question)
|| self.token == token::Eof || self.token == token::Semi
{ || self.token == token::Eof
StmtKind::MacCall(P(MacCallStmt { mac, style, attrs, tokens: None })) {
} else { StmtKind::MacCall(P(MacCallStmt { mac, style, attrs, tokens: None }))
// Since none of the above applied, this is an expression statement macro. } else {
let e = self.mk_expr(lo.to(hi), ExprKind::MacCall(mac), AttrVec::new()); // Since none of the above applied, this is an expression statement macro.
let e = self.maybe_recover_from_bad_qpath(e, true)?; let e = self.mk_expr(lo.to(hi), ExprKind::MacCall(mac), AttrVec::new());
let e = self.parse_dot_or_call_expr_with(e, lo, attrs.into())?; let e = self.maybe_recover_from_bad_qpath(e, true)?;
let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?; let e = self.parse_dot_or_call_expr_with(e, lo, attrs.into())?;
StmtKind::Expr(e) let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?;
}; StmtKind::Expr(e)
};
Ok(self.mk_stmt(lo.to(hi), kind)) Ok(self.mk_stmt(lo.to(hi), kind))
} }

View file

@ -1325,7 +1325,7 @@ pub(crate) fn can_be_overflowed_expr(
} }
ast::ExprKind::MacCall(ref mac) => { ast::ExprKind::MacCall(ref mac) => {
match ( match (
rustc_ast::ast::MacDelimiter::from_token(mac.args.delim()), rustc_ast::ast::MacDelimiter::from_token(mac.args.delim().unwrap()),
context.config.overflow_delimited_expr(), context.config.overflow_delimited_expr(),
) { ) {
(Some(ast::MacDelimiter::Bracket), true) (Some(ast::MacDelimiter::Bracket), true)

View file

@ -562,7 +562,7 @@ fn delim_token_to_str(
("{ ", " }") ("{ ", " }")
} }
} }
DelimToken::NoDelim => ("", ""), DelimToken::NoDelim => unreachable!(),
}; };
if use_multiple_lines { if use_multiple_lines {
let indent_str = shape.indent.to_string_with_newline(context.config); let indent_str = shape.indent.to_string_with_newline(context.config);