bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Add spacing information to delimiters.

Browse source 
This is an extension of the previous commit. It means the output of
something like this:
```
stringify!(let a: Vec<u32> = vec![];)
```
goes from this:
```
let a: Vec<u32> = vec![] ;
```
With this PR, it now produces this string:
```
let a: Vec<u32> = vec![];
```
This commit is contained in:
Nicholas Nethercote 2023-10-12 15:36:14 +11:00
parent 925f7fad57
commit 4cfdbd328b
47 changed files with 307 additions and 231 deletions
Download patch file Download diff file Expand all files Collapse all files

10
compiler/rustc_expand/src/config.rs
View file

@ -6,8 +6,7 @@ use crate::errors::{
};
use rustc_ast::ptr::P;
use rustc_ast::token::{Delimiter, Token, TokenKind};
use rustc_ast::tokenstream::{AttrTokenStream, AttrTokenTree};
use rustc_ast::tokenstream::{DelimSpan, Spacing};
use rustc_ast::tokenstream::{AttrTokenStream, AttrTokenTree, DelimSpacing, DelimSpan, Spacing};
use rustc_ast::tokenstream::{LazyAttrTokenStream, TokenTree};
use rustc_ast::NodeId;
use rustc_ast::{self as ast, AttrStyle, Attribute, HasAttrs, HasTokens, MetaItem};
@ -242,7 +241,7 @@ impl<'a> StripUnconfigured<'a> {
stream.0.iter().all(|tree| match tree {
AttrTokenTree::Attributes(_) => false,
AttrTokenTree::Token(..) => true,
AttrTokenTree::Delimited(_, _, inner) => can_skip(inner),
AttrTokenTree::Delimited(.., inner) => can_skip(inner),
})
}
@ -266,9 +265,9 @@ impl<'a> StripUnconfigured<'a> {
None.into_iter()
}
}
AttrTokenTree::Delimited(sp, delim, mut inner) => {
AttrTokenTree::Delimited(sp, spacing, delim, mut inner) => {
inner = self.configure_tokens(&inner);
Some(AttrTokenTree::Delimited(sp, delim, inner)).into_iter()
Some(AttrTokenTree::Delimited(sp, spacing, delim, inner)).into_iter()
}
AttrTokenTree::Token(ref token, _)
if let TokenKind::Interpolated(nt) = &token.kind =>
@ -376,6 +375,7 @@ impl<'a> StripUnconfigured<'a> {
// in `#[attr]`, so just use the span of the `#` token.
let bracket_group = AttrTokenTree::Delimited(
DelimSpan::from_single(pound_span),
DelimSpacing::new(Spacing::JointHidden, Spacing::Alone),
Delimiter::Bracket,
item.tokens
.as_ref()

6
compiler/rustc_expand/src/mbe.rs
View file

@ -13,7 +13,7 @@ pub(crate) mod transcribe;
use metavar_expr::MetaVarExpr;
use rustc_ast::token::{Delimiter, NonterminalKind, Token, TokenKind};
use rustc_ast::tokenstream::DelimSpan;
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan};
use rustc_span::symbol::Ident;
use rustc_span::Span;
@ -68,7 +68,7 @@ pub(crate) enum KleeneOp {
enum TokenTree {
Token(Token),
/// A delimited sequence, e.g. `($e:expr)` (RHS) or `{ $e }` (LHS).
Delimited(DelimSpan, Delimited),
Delimited(DelimSpan, DelimSpacing, Delimited),
/// A kleene-style repetition sequence, e.g. `$($e:expr)*` (RHS) or `$($e),*` (LHS).
Sequence(DelimSpan, SequenceRepetition),
/// e.g., `$var`.
@ -99,7 +99,7 @@ impl TokenTree {
TokenTree::Token(Token { span, .. })
| TokenTree::MetaVar(span, _)
| TokenTree::MetaVarDecl(span, _, _) => span,
TokenTree::Delimited(span, _)
TokenTree::Delimited(span, ..)
| TokenTree::MetaVarExpr(span, _)
| TokenTree::Sequence(span, _) => span.entire(),
}

12
compiler/rustc_expand/src/mbe/macro_check.rs
View file

@ -290,7 +290,7 @@ fn check_binders(
}
// `MetaVarExpr` can not appear in the LHS of a macro arm
TokenTree::MetaVarExpr(..) => {}
TokenTree::Delimited(_, ref del) => {
TokenTree::Delimited(.., ref del) => {
for tt in &del.tts {
check_binders(sess, node_id, tt, macros, binders, ops, valid);
}
@ -353,7 +353,7 @@ fn check_occurrences(
};
check_ops_is_prefix(sess, node_id, macros, binders, ops, dl.entire(), name);
}
TokenTree::Delimited(_, ref del) => {
TokenTree::Delimited(.., ref del) => {
check_nested_occurrences(sess, node_id, &del.tts, macros, binders, ops, valid);
}
TokenTree::Sequence(_, ref seq) => {
@ -435,8 +435,8 @@ fn check_nested_occurrences(
// We check that the meta-variable is correctly used.
check_occurrences(sess, node_id, tt, macros, binders, ops, valid);
}
(NestedMacroState::MacroRulesNotName, TokenTree::Delimited(_, del))
| (NestedMacroState::MacroName, TokenTree::Delimited(_, del))
(NestedMacroState::MacroRulesNotName, TokenTree::Delimited(.., del))
| (NestedMacroState::MacroName, TokenTree::Delimited(.., del))
if del.delim == Delimiter::Brace =>
{
let macro_rules = state == NestedMacroState::MacroRulesNotName;
@ -466,7 +466,7 @@ fn check_nested_occurrences(
// We check that the meta-variable is correctly used.
check_occurrences(sess, node_id, tt, macros, binders, ops, valid);
}
(NestedMacroState::MacroName, TokenTree::Delimited(_, del))
(NestedMacroState::MacroName, TokenTree::Delimited(.., del))
if del.delim == Delimiter::Parenthesis =>
{
state = NestedMacroState::MacroNameParen;
@ -481,7 +481,7 @@ fn check_nested_occurrences(
valid,
);
}
(NestedMacroState::MacroNameParen, TokenTree::Delimited(_, del))
(NestedMacroState::MacroNameParen, TokenTree::Delimited(.., del))
if del.delim == Delimiter::Brace =>
{
state = NestedMacroState::Empty;

4
compiler/rustc_expand/src/mbe/macro_parser.rs
View file

@ -184,7 +184,7 @@ pub(super) fn compute_locs(matcher: &[TokenTree]) -> Vec<MatcherLoc> {
TokenTree::Token(token) => {
locs.push(MatcherLoc::Token { token: token.clone() });
}
TokenTree::Delimited(span, delimited) => {
TokenTree::Delimited(span, _, delimited) => {
let open_token = Token::new(token::OpenDelim(delimited.delim), span.open);
let close_token = Token::new(token::CloseDelim(delimited.delim), span.close);
@ -335,7 +335,7 @@ pub(super) fn count_metavar_decls(matcher: &[TokenTree]) -> usize {
.map(|tt| match tt {
TokenTree::MetaVarDecl(..) => 1,
TokenTree::Sequence(_, seq) => seq.num_captures,
TokenTree::Delimited(_, delim) => count_metavar_decls(&delim.tts),
TokenTree::Delimited(.., delim) => count_metavar_decls(&delim.tts),
TokenTree::Token(..) => 0,
TokenTree::MetaVar(..) | TokenTree::MetaVarExpr(..) => unreachable!(),
})

18
compiler/rustc_expand/src/mbe/macro_rules.rs
View file

@ -207,7 +207,7 @@ fn expand_macro<'cx>(
match try_success_result {
Ok((i, named_matches)) => {
let (rhs, rhs_span): (&mbe::Delimited, DelimSpan) = match &rhses[i] {
mbe::TokenTree::Delimited(span, delimited) => (&delimited, *span),
mbe::TokenTree::Delimited(span, _, delimited) => (&delimited, *span),
_ => cx.span_bug(sp, "malformed macro rhs"),
};
let arm_span = rhses[i].span();
@ -589,7 +589,7 @@ pub fn compile_declarative_macro(
.map(|lhs| {
// Ignore the delimiters around the matcher.
match lhs {
mbe::TokenTree::Delimited(_, delimited) => {
mbe::TokenTree::Delimited(.., delimited) => {
mbe::macro_parser::compute_locs(&delimited.tts)
}
_ => sess.diagnostic().span_bug(def.span, "malformed macro lhs"),
@ -615,7 +615,7 @@ pub fn compile_declarative_macro(
fn check_lhs_nt_follows(sess: &ParseSess, def: &ast::Item, lhs: &mbe::TokenTree) -> bool {
// lhs is going to be like TokenTree::Delimited(...), where the
// entire lhs is those tts. Or, it can be a "bare sequence", not wrapped in parens.
if let mbe::TokenTree::Delimited(_, delimited) = lhs {
if let mbe::TokenTree::Delimited(.., delimited) = lhs {
check_matcher(sess, def, &delimited.tts)
} else {
let msg = "invalid macro matcher; matchers must be contained in balanced delimiters";
@ -668,7 +668,7 @@ fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[mbe::TokenTree]) -> bool {
| TokenTree::MetaVar(..)
| TokenTree::MetaVarDecl(..)
| TokenTree::MetaVarExpr(..) => (),
TokenTree::Delimited(_, del) => {
TokenTree::Delimited(.., del) => {
if !check_lhs_no_empty_seq(sess, &del.tts) {
return false;
}
@ -709,14 +709,14 @@ fn check_matcher(sess: &ParseSess, def: &ast::Item, matcher: &[mbe::TokenTree])
fn has_compile_error_macro(rhs: &mbe::TokenTree) -> bool {
match rhs {
mbe::TokenTree::Delimited(_sp, d) => {
mbe::TokenTree::Delimited(.., d) => {
let has_compile_error = d.tts.array_windows::<3>().any(|[ident, bang, args]| {
if let mbe::TokenTree::Token(ident) = ident
&& let TokenKind::Ident(ident, _) = ident.kind
&& ident == sym::compile_error
&& let mbe::TokenTree::Token(bang) = bang
&& let TokenKind::Not = bang.kind
&& let mbe::TokenTree::Delimited(_, del) = args
&& let mbe::TokenTree::Delimited(.., del) = args
&& del.delim != Delimiter::Invisible
{
true
@ -773,7 +773,7 @@ impl<'tt> FirstSets<'tt> {
| TokenTree::MetaVarExpr(..) => {
first.replace_with(TtHandle::TtRef(tt));
}
TokenTree::Delimited(span, delimited) => {
TokenTree::Delimited(span, _, delimited) => {
build_recur(sets, &delimited.tts);
first.replace_with(TtHandle::from_token_kind(
token::OpenDelim(delimited.delim),
@ -842,7 +842,7 @@ impl<'tt> FirstSets<'tt> {
first.add_one(TtHandle::TtRef(tt));
return first;
}
TokenTree::Delimited(span, delimited) => {
TokenTree::Delimited(span, _, delimited) => {
first.add_one(TtHandle::from_token_kind(
token::OpenDelim(delimited.delim),
span.open,
@ -1087,7 +1087,7 @@ fn check_matcher_core<'tt>(
suffix_first = build_suffix_first();
}
}
TokenTree::Delimited(span, d) => {
TokenTree::Delimited(span, _, d) => {
let my_suffix = TokenSet::singleton(TtHandle::from_token_kind(
token::CloseDelim(d.delim),
span.close,

2
compiler/rustc_expand/src/mbe/metavar_expr.rs
View file

@ -35,7 +35,7 @@ impl MetaVarExpr {
) -> PResult<'sess, MetaVarExpr> {
let mut tts = input.trees();
let ident = parse_ident(&mut tts, sess, outer_span)?;
let Some(TokenTree::Delimited(_, Delimiter::Parenthesis, args)) = tts.next() else {
let Some(TokenTree::Delimited(.., Delimiter::Parenthesis, args)) = tts.next() else {
let msg = "meta-variable expression parameter must be wrapped in parentheses";
return Err(sess.span_diagnostic.struct_span_err(ident.span, msg));
};

7
compiler/rustc_expand/src/mbe/quoted.rs
View file

@ -151,7 +151,7 @@ fn parse_tree<'a>(
// during parsing.
let mut next = outer_trees.next();
let mut trees: Box<dyn Iterator<Item = &tokenstream::TokenTree>>;
if let Some(tokenstream::TokenTree::Delimited(_, Delimiter::Invisible, tts)) = next {
if let Some(tokenstream::TokenTree::Delimited(.., Delimiter::Invisible, tts)) = next {
trees = Box::new(tts.trees());
next = trees.next();
} else {
@ -160,7 +160,7 @@ fn parse_tree<'a>(
match next {
// `tree` is followed by a delimited set of token trees.
Some(&tokenstream::TokenTree::Delimited(delim_span, delim, ref tts)) => {
Some(&tokenstream::TokenTree::Delimited(delim_span, _, delim, ref tts)) => {
if parsing_patterns {
if delim != Delimiter::Parenthesis {
span_dollar_dollar_or_metavar_in_the_lhs_err(
@ -258,8 +258,9 @@ fn parse_tree<'a>(
// `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to
// descend into the delimited set and further parse it.
&tokenstream::TokenTree::Delimited(span, delim, ref tts) => TokenTree::Delimited(
&tokenstream::TokenTree::Delimited(span, spacing, delim, ref tts) => TokenTree::Delimited(
span,
spacing,
Delimited {
delim,
tts: parse(tts, parsing_patterns, sess, node_id, features, edition),

40
compiler/rustc_expand/src/mbe/transcribe.rs
View file

@ -7,7 +7,7 @@ use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, MatchedTokenTree,
use crate::mbe::{self, MetaVarExpr};
use rustc_ast::mut_visit::{self, MutVisitor};
use rustc_ast::token::{self, Delimiter, Token, TokenKind};
use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree};
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
use rustc_data_structures::fx::FxHashMap;
use rustc_errors::{pluralize, PResult};
use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed};
@ -31,14 +31,24 @@ impl MutVisitor for Marker {
/// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`).
enum Frame<'a> {
Delimited { tts: &'a [mbe::TokenTree], idx: usize, delim: Delimiter, span: DelimSpan },
Sequence { tts: &'a [mbe::TokenTree], idx: usize, sep: Option<Token> },
Delimited {
tts: &'a [mbe::TokenTree],
idx: usize,
delim: Delimiter,
span: DelimSpan,
spacing: DelimSpacing,
},
Sequence {
tts: &'a [mbe::TokenTree],
idx: usize,
sep: Option<Token>,
},
}
impl<'a> Frame<'a> {
/// Construct a new frame around the delimited set of tokens.
fn new(src: &'a mbe::Delimited, span: DelimSpan) -> Frame<'a> {
Frame::Delimited { tts: &src.tts, idx: 0, delim: src.delim, span }
fn new(src: &'a mbe::Delimited, span: DelimSpan, spacing: DelimSpacing) -> Frame<'a> {
Frame::Delimited { tts: &src.tts, idx: 0, delim: src.delim, span, spacing }
}
}
@ -89,8 +99,10 @@ pub(super) fn transcribe<'a>(
}
// We descend into the RHS (`src`), expanding things as we go. This stack contains the things
// we have yet to expand/are still expanding. We start the stack off with the whole RHS.
let mut stack: SmallVec<[Frame<'_>; 1]> = smallvec![Frame::new(src, src_span)];
// we have yet to expand/are still expanding. We start the stack off with the whole RHS. The
// choice of spacing values doesn't matter.
let mut stack: SmallVec<[Frame<'_>; 1]> =
smallvec![Frame::new(src, src_span, DelimSpacing::new(Spacing::Alone, Spacing::Alone))];
// As we descend in the RHS, we will need to be able to match nested sequences of matchers.
// `repeats` keeps track of where we are in matching at each level, with the last element being
@ -144,14 +156,19 @@ pub(super) fn transcribe<'a>(
// We are done processing a Delimited. If this is the top-level delimited, we are
// done. Otherwise, we unwind the result_stack to append what we have produced to
// any previous results.
Frame::Delimited { delim, span, .. } => {
Frame::Delimited { delim, span, mut spacing, .. } => {
// Hack to force-insert a space after `]` in certain case.
// See discussion of the `hex-literal` crate in #114571.
if delim == Delimiter::Bracket {
spacing.close = Spacing::Alone;
}
if result_stack.is_empty() {
// No results left to compute! We are back at the top-level.
return Ok(TokenStream::new(result));
}
// Step back into the parent Delimited.
let tree = TokenTree::Delimited(span, delim, TokenStream::new(result));
let tree = TokenTree::Delimited(span, spacing, delim, TokenStream::new(result));
result = result_stack.pop().unwrap();
result.push(tree);
}
@ -258,13 +275,14 @@ pub(super) fn transcribe<'a>(
// We will produce all of the results of the inside of the `Delimited` and then we will
// jump back out of the Delimited, pop the result_stack and add the new results back to
// the previous results (from outside the Delimited).
mbe::TokenTree::Delimited(mut span, delimited) => {
mbe::TokenTree::Delimited(mut span, spacing, delimited) => {
mut_visit::visit_delim_span(&mut span, &mut marker);
stack.push(Frame::Delimited {
tts: &delimited.tts,
delim: delimited.delim,
idx: 0,
span,
spacing: *spacing,
});
result_stack.push(mem::take(&mut result));
}
@ -374,7 +392,7 @@ fn lockstep_iter_size(
) -> LockstepIterSize {
use mbe::TokenTree;
match tree {
TokenTree::Delimited(_, delimited) => {
TokenTree::Delimited(.., delimited) => {
delimited.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| {
size.with(lockstep_iter_size(tt, interpolations, repeats))
})

15
compiler/rustc_expand/src/parse/tests.rs
View file

@ -4,7 +4,7 @@ use crate::tests::{
use rustc_ast::ptr::P;
use rustc_ast::token::{self, Delimiter, Token};
use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
use rustc_ast::visit;
use rustc_ast::{self as ast, PatKind};
use rustc_ast_pretty::pprust::item_to_string;
@ -77,14 +77,14 @@ fn string_to_tts_macro() {
TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }, _),
TokenTree::Token(Token { kind: token::Not, .. }, _),
TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }, _),
TokenTree::Delimited(_, macro_delim, macro_tts),
TokenTree::Delimited(.., macro_delim, macro_tts),
] if name_macro_rules == &kw::MacroRules && name_zip.as_str() == "zip" => {
let tts = &macro_tts.trees().collect::<Vec<_>>();
match &tts[..] {
[
TokenTree::Delimited(_, first_delim, first_tts),
TokenTree::Delimited(.., first_delim, first_tts),
TokenTree::Token(Token { kind: token::FatArrow, .. }, _),
TokenTree::Delimited(_, second_delim, second_tts),
TokenTree::Delimited(.., second_delim, second_tts),
] if macro_delim == &Delimiter::Parenthesis => {
let tts = &first_tts.trees().collect::<Vec<_>>();
match &tts[..] {
@ -123,6 +123,9 @@ fn string_to_tts_1() {
TokenTree::token_joint_hidden(token::Ident(Symbol::intern("a"), false), sp(3, 4)),
TokenTree::Delimited(
DelimSpan::from_pair(sp(4, 5), sp(11, 12)),
// `JointHidden` because the `(` is followed immediately by
// `b`, `Alone` because the `)` is followed by whitespace.
DelimSpacing::new(Spacing::JointHidden, Spacing::Alone),
Delimiter::Parenthesis,
TokenStream::new(vec![
TokenTree::token_joint(token::Ident(Symbol::intern("b"), false), sp(5, 6)),
@ -134,6 +137,10 @@ fn string_to_tts_1() {
),
TokenTree::Delimited(
DelimSpan::from_pair(sp(13, 14), sp(18, 19)),
// First `Alone` because the `{` is followed by whitespace,
// second `Alone` because the `}` is followed immediately by
// EOF.
DelimSpacing::new(Spacing::Alone, Spacing::Alone),
Delimiter::Brace,
TokenStream::new(vec![
TokenTree::token_joint(token::Ident(Symbol::intern("b"), false), sp(15, 16)),

14
compiler/rustc_expand/src/proc_macro_server.rs
View file

@ -5,7 +5,7 @@ use pm::bridge::{
use pm::{Delimiter, Level};
use rustc_ast as ast;
use rustc_ast::token;
use rustc_ast::tokenstream::{self, Spacing, TokenStream};
use rustc_ast::tokenstream::{self, DelimSpacing, Spacing, TokenStream};
use rustc_ast::util::literal::escape_byte_str_symbol;
use rustc_ast_pretty::pprust;
use rustc_data_structures::fx::FxHashMap;
@ -98,7 +98,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
while let Some(tree) = cursor.next() {
let (Token { kind, span }, joint) = match tree.clone() {
tokenstream::TokenTree::Delimited(span, delim, tts) => {
tokenstream::TokenTree::Delimited(span, _, delim, tts) => {
let delimiter = pm::Delimiter::from_internal(delim);
trees.push(TokenTree::Group(Group {
delimiter,
@ -284,10 +284,11 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
fn to_internal(self) -> SmallVec<[tokenstream::TokenTree; 2]> {
use rustc_ast::token::*;
// The code below is conservative and uses `token_alone` in most
// places. When the resulting code is pretty-printed by `print_tts` it
// ends up with spaces between most tokens, which is safe but ugly.
// It's hard in general to do better when working at the token level.
// The code below is conservative, using `token_alone`/`Spacing::Alone`
// in most places. When the resulting code is pretty-printed by
// `print_tts` it ends up with spaces between most tokens, which is
// safe but ugly. It's hard in general to do better when working at the
// token level.
let (tree, rustc) = self;
match tree {
TokenTree::Punct(Punct { ch, joint, span }) => {
@ -330,6 +331,7 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => {
smallvec![tokenstream::TokenTree::Delimited(
tokenstream::DelimSpan { open, close },
DelimSpacing::new(Spacing::Alone, Spacing::Alone),
delimiter.to_internal(),
stream.unwrap_or_default(),
)]