bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Add spacing information to delimiters.

Browse source 
This is an extension of the previous commit. It means the output of
something like this:
```
stringify!(let a: Vec<u32> = vec![];)
```
goes from this:
```
let a: Vec<u32> = vec![] ;
```
With this PR, it now produces this string:
```
let a: Vec<u32> = vec![];
```
This commit is contained in:
Nicholas Nethercote 2023-10-12 15:36:14 +11:00
parent 925f7fad57
commit 4cfdbd328b
47 changed files with 307 additions and 231 deletions
Download patch file Download diff file Expand all files Collapse all files

2
compiler/rustc_parse/src/lexer/mod.rs
View file

@ -67,7 +67,7 @@ pub(crate) fn parse_token_trees<'a>(
let (stream, res, unmatched_delims) =
tokentrees::TokenTreesReader::parse_all_token_trees(string_reader);
match res {
Ok(()) if unmatched_delims.is_empty() => Ok(stream),
Ok(_open_spacing) if unmatched_delims.is_empty() => Ok(stream),
_ => {
// Return error if there are unmatched delimiters or unclosed delimiters.
// We emit delimiter mismatch errors first, then emit the unclosing delimiter mismatch

87
compiler/rustc_parse/src/lexer/tokentrees.rs
View file

@ -3,7 +3,7 @@ use super::diagnostics::same_indentation_level;
use super::diagnostics::TokenTreeDiagInfo;
use super::{StringReader, UnmatchedDelim};
use rustc_ast::token::{self, Delimiter, Token};
use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree};
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
use rustc_ast_pretty::pprust::token_to_string;
use rustc_errors::{Applicability, PErr};
use rustc_span::symbol::kw;
@ -25,59 +25,46 @@ impl<'a> TokenTreesReader<'a> {
token: Token::dummy(),
diag_info: TokenTreeDiagInfo::default(),
};
let (stream, res) = tt_reader.parse_token_trees(/* is_delimited */ false);
let (_open_spacing, stream, res) =
tt_reader.parse_token_trees(/* is_delimited */ false);
(stream, res, tt_reader.diag_info.unmatched_delims)
}
// Parse a stream of tokens into a list of `TokenTree`s.
// Parse a stream of tokens into a list of `TokenTree`s. The `Spacing` in
// the result is that of the opening delimiter.
fn parse_token_trees(
&mut self,
is_delimited: bool,
) -> (TokenStream, Result<(), Vec<PErr<'a>>>) {
self.token = self.string_reader.next_token().0;
) -> (Spacing, TokenStream, Result<(), Vec<PErr<'a>>>) {
// Move past the opening delimiter.
let (_, open_spacing) = self.bump(false);
let mut buf = Vec::new();
loop {
match self.token.kind {
token::OpenDelim(delim) => {
buf.push(match self.parse_token_tree_open_delim(delim) {
Ok(val) => val,
Err(errs) => return (TokenStream::new(buf), Err(errs)),
Err(errs) => return (open_spacing, TokenStream::new(buf), Err(errs)),
})
}
token::CloseDelim(delim) => {
return (
open_spacing,
TokenStream::new(buf),
if is_delimited { Ok(()) } else { Err(vec![self.close_delim_err(delim)]) },
);
}
token::Eof => {
return (
open_spacing,
TokenStream::new(buf),
if is_delimited { Err(vec![self.eof_err()]) } else { Ok(()) },
);
}
_ => {
// Get the next normal token. This might require getting multiple adjacent
// single-char tokens and joining them together.
let (this_spacing, next_tok) = loop {
let (next_tok, is_next_tok_preceded_by_whitespace) =
self.string_reader.next_token();
if is_next_tok_preceded_by_whitespace {
break (Spacing::Alone, next_tok);
} else if let Some(glued) = self.token.glue(&next_tok) {
self.token = glued;
} else {
let this_spacing = if next_tok.is_punct() {
Spacing::Joint
} else if next_tok.kind == token::Eof {
Spacing::Alone
} else {
Spacing::JointHidden
};
break (this_spacing, next_tok);
}
};
let this_tok = std::mem::replace(&mut self.token, next_tok);
// Get the next normal token.
let (this_tok, this_spacing) = self.bump(true);
buf.push(TokenTree::Token(this_tok, this_spacing));
}
}
@ -121,7 +108,7 @@ impl<'a> TokenTreesReader<'a> {
// Parse the token trees within the delimiters.
// We stop at any delimiter so we can try to recover if the user
// uses an incorrect delimiter.
let (tts, res) = self.parse_token_trees(/* is_delimited */ true);
let (open_spacing, tts, res) = self.parse_token_trees(/* is_delimited */ true);
if let Err(errs) = res {
return Err(self.unclosed_delim_err(tts, errs));
}
@ -130,7 +117,7 @@ impl<'a> TokenTreesReader<'a> {
let delim_span = DelimSpan::from_pair(pre_span, self.token.span);
let sm = self.string_reader.sess.source_map();
match self.token.kind {
let close_spacing = match self.token.kind {
// Correct delimiter.
token::CloseDelim(close_delim) if close_delim == open_delim => {
let (open_brace, open_brace_span) = self.diag_info.open_braces.pop().unwrap();
@ -152,7 +139,7 @@ impl<'a> TokenTreesReader<'a> {
}
// Move past the closing delimiter.
self.token = self.string_reader.next_token().0;
self.bump(false).1
}
// Incorrect delimiter.
token::CloseDelim(close_delim) => {
@ -196,18 +183,50 @@ impl<'a> TokenTreesReader<'a> {
// bar(baz(
// } // Incorrect delimiter but matches the earlier `{`
if !self.diag_info.open_braces.iter().any(|&(b, _)| b == close_delim) {
self.token = self.string_reader.next_token().0;
self.bump(false).1
} else {
// The choice of value here doesn't matter.
Spacing::Alone
}
}
token::Eof => {
// Silently recover, the EOF token will be seen again
// and an error emitted then. Thus we don't pop from
// self.open_braces here.
// self.open_braces here. The choice of spacing value here
// doesn't matter.
Spacing::Alone
}
_ => unreachable!(),
}
};
Ok(TokenTree::Delimited(delim_span, open_delim, tts))
let spacing = DelimSpacing::new(open_spacing, close_spacing);
Ok(TokenTree::Delimited(delim_span, spacing, open_delim, tts))
}
// Move on to the next token, returning the current token and its spacing.
// Will glue adjacent single-char tokens together if `glue` is set.
fn bump(&mut self, glue: bool) -> (Token, Spacing) {
let (this_spacing, next_tok) = loop {
let (next_tok, is_next_tok_preceded_by_whitespace) = self.string_reader.next_token();
if is_next_tok_preceded_by_whitespace {
break (Spacing::Alone, next_tok);
} else if glue && let Some(glued) = self.token.glue(&next_tok) {
self.token = glued;
} else {
let this_spacing = if next_tok.is_punct() {
Spacing::Joint
} else if next_tok.kind == token::Eof {
Spacing::Alone
} else {
Spacing::JointHidden
};
break (this_spacing, next_tok);
}
};
let this_tok = std::mem::replace(&mut self.token, next_tok);
(this_tok, this_spacing)
}
fn unclosed_delim_err(&mut self, tts: TokenStream, mut errs: Vec<PErr<'a>>) -> Vec<PErr<'a>> {

14
compiler/rustc_parse/src/parser/attr_wrapper.rs
View file

@ -1,7 +1,7 @@
use super::{Capturing, FlatToken, ForceCollect, Parser, ReplaceRange, TokenCursor, TrailingToken};
use rustc_ast::token::{self, Delimiter, Token, TokenKind};
use rustc_ast::tokenstream::{AttrTokenStream, AttributesData, ToAttrTokenStream};
use rustc_ast::tokenstream::{AttrTokenTree, DelimSpan, LazyAttrTokenStream, Spacing};
use rustc_ast::tokenstream::{AttrTokenStream, AttrTokenTree, AttributesData, DelimSpacing};
use rustc_ast::tokenstream::{DelimSpan, LazyAttrTokenStream, Spacing, ToAttrTokenStream};
use rustc_ast::{self as ast};
use rustc_ast::{AttrVec, Attribute, HasAttrs, HasTokens};
use rustc_errors::PResult;
@ -389,7 +389,7 @@ fn make_token_stream(
#[derive(Debug)]
struct FrameData {
// This is `None` for the first frame, `Some` for all others.
open_delim_sp: Option<(Delimiter, Span)>,
open_delim_sp: Option<(Delimiter, Span, Spacing)>,
inner: Vec<AttrTokenTree>,
}
let mut stack = vec![FrameData { open_delim_sp: None, inner: vec![] }];
@ -397,21 +397,23 @@ fn make_token_stream(
while let Some((token, spacing)) = token_and_spacing {
match token {
FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span }) => {
stack.push(FrameData { open_delim_sp: Some((delim, span)), inner: vec![] });
stack
.push(FrameData { open_delim_sp: Some((delim, span, spacing)), inner: vec![] });
}
FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => {
let frame_data = stack
.pop()
.unwrap_or_else(|| panic!("Token stack was empty for token: {token:?}"));
let (open_delim, open_sp) = frame_data.open_delim_sp.unwrap();
let (open_delim, open_sp, open_spacing) = frame_data.open_delim_sp.unwrap();
assert_eq!(
open_delim, delim,
"Mismatched open/close delims: open={open_delim:?} close={span:?}"
);
let dspan = DelimSpan::from_pair(open_sp, span);
let dspacing = DelimSpacing::new(open_spacing, spacing);
let stream = AttrTokenStream::new(frame_data.inner);
let delimited = AttrTokenTree::Delimited(dspan, delim, stream);
let delimited = AttrTokenTree::Delimited(dspan, dspacing, delim, stream);
stack
.last_mut()
.unwrap_or_else(|| panic!("Bottom token frame is missing for token: {token:?}"))

2
compiler/rustc_parse/src/parser/expr.rs
View file

@ -2276,7 +2276,7 @@ impl<'a> Parser<'a> {
}
if self.token.kind == TokenKind::Semi
&& matches!(self.token_cursor.stack.last(), Some((_, Delimiter::Parenthesis, _)))
&& matches!(self.token_cursor.stack.last(), Some((.., Delimiter::Parenthesis)))
&& self.may_recover()
{
// It is likely that the closure body is a block but where the

35
compiler/rustc_parse/src/parser/mod.rs
View file

@ -20,7 +20,7 @@ pub use path::PathStyle;
use rustc_ast::ptr::P;
use rustc_ast::token::{self, Delimiter, Nonterminal, Token, TokenKind};
use rustc_ast::tokenstream::{AttributesData, DelimSpan, Spacing};
use rustc_ast::tokenstream::{AttributesData, DelimSpacing, DelimSpan, Spacing};
use rustc_ast::tokenstream::{TokenStream, TokenTree, TokenTreeCursor};
use rustc_ast::util::case::Case;
use rustc_ast::AttrId;
@ -240,7 +240,7 @@ struct TokenCursor {
// Token streams surrounding the current one. The delimiters for stack[n]'s
// tokens are in `stack[n-1]`. `stack[0]` (when present) has no delimiters
// because it's the outermost token stream which never has delimiters.
stack: Vec<(TokenTreeCursor, Delimiter, DelimSpan)>,
stack: Vec<(TokenTreeCursor, DelimSpan, DelimSpacing, Delimiter)>,
}
impl TokenCursor {
@ -264,24 +264,25 @@ impl TokenCursor {
));
return (token.clone(), spacing);
}
&TokenTree::Delimited(sp, delim, ref tts) => {
&TokenTree::Delimited(sp, spacing, delim, ref tts) => {
let trees = tts.clone().into_trees();
self.stack.push((mem::replace(&mut self.tree_cursor, trees), delim, sp));
self.stack.push((
mem::replace(&mut self.tree_cursor, trees),
sp,
spacing,
delim,
));
if delim != Delimiter::Invisible {
// FIXME: add two `Spacing` fields to `TokenTree::Delimited`
// and use the open delim one here.
return (Token::new(token::OpenDelim(delim), sp.open), Spacing::Alone);
return (Token::new(token::OpenDelim(delim), sp.open), spacing.open);
}
// No open delimiter to return; continue on to the next iteration.
}
};
} else if let Some((tree_cursor, delim, span)) = self.stack.pop() {
} else if let Some((tree_cursor, span, spacing, delim)) = self.stack.pop() {
// We have exhausted this token stream. Move back to its parent token stream.
self.tree_cursor = tree_cursor;
if delim != Delimiter::Invisible {
// FIXME: add two `Spacing` fields to `TokenTree::Delimited` and
// use the close delim one here.
return (Token::new(token::CloseDelim(delim), span.close), Spacing::Alone);
return (Token::new(token::CloseDelim(delim), span.close), spacing.close);
}
// No close delimiter to return; continue on to the next iteration.
} else {
@ -1074,7 +1075,7 @@ impl<'a> Parser<'a> {
return looker(&self.token);
}
if let Some(&(_, delim, span)) = self.token_cursor.stack.last()
if let Some(&(_, span, _, delim)) = self.token_cursor.stack.last()
&& delim != Delimiter::Invisible
{
// We are not in the outermost token stream, and the token stream
@ -1083,7 +1084,7 @@ impl<'a> Parser<'a> {
let tree_cursor = &self.token_cursor.tree_cursor;
let all_normal = (0..dist).all(|i| {
let token = tree_cursor.look_ahead(i);
!matches!(token, Some(TokenTree::Delimited(_, Delimiter::Invisible, _)))
!matches!(token, Some(TokenTree::Delimited(.., Delimiter::Invisible, _)))
});
if all_normal {
// There were no skipped delimiters. Do lookahead by plain indexing.
@ -1092,7 +1093,7 @@ impl<'a> Parser<'a> {
// Indexing stayed within the current token stream.
match tree {
TokenTree::Token(token, _) => looker(token),
TokenTree::Delimited(dspan, delim, _) => {
TokenTree::Delimited(dspan, _, delim, _) => {
looker(&Token::new(token::OpenDelim(*delim), dspan.open))
}
}
@ -1270,7 +1271,7 @@ impl<'a> Parser<'a> {
|| self.check(&token::OpenDelim(Delimiter::Brace));
delimited.then(|| {
let TokenTree::Delimited(dspan, delim, tokens) = self.parse_token_tree() else {
let TokenTree::Delimited(dspan, _, delim, tokens) = self.parse_token_tree() else {
unreachable!()
};
DelimArgs { dspan, delim, tokens }
@ -1294,7 +1295,7 @@ impl<'a> Parser<'a> {
token::OpenDelim(..) => {
// Grab the tokens within the delimiters.
let stream = self.token_cursor.tree_cursor.stream.clone();
let (_, delim, span) = *self.token_cursor.stack.last().unwrap();
let (_, span, spacing, delim) = *self.token_cursor.stack.last().unwrap();
// Advance the token cursor through the entire delimited
// sequence. After getting the `OpenDelim` we are *within* the
@ -1314,7 +1315,7 @@ impl<'a> Parser<'a> {
// Consume close delimiter
self.bump();
TokenTree::Delimited(span, delim, stream)
TokenTree::Delimited(span, spacing, delim, stream)
}
token::CloseDelim(_) | token::Eof => unreachable!(),
_ => {