bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Move Spacing into FlatToken.

Browse source 
It's only needed for the `FlatToken::Token` variant. This makes things a
little more concise.
This commit is contained in:
Nicholas Nethercote 2024-07-08 19:32:21 +10:00
parent a88c4d67d9
commit f5527949f2
2 changed files with 13 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

28
compiler/rustc_parse/src/parser/attr_wrapper.rs
View file

@ -103,11 +103,8 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl {
// produce an empty `TokenStream` if no calls were made, and omit the // produce an empty `TokenStream` if no calls were made, and omit the
// final token otherwise. // final token otherwise.
let mut cursor_snapshot = self.cursor_snapshot.clone(); let mut cursor_snapshot = self.cursor_snapshot.clone();
let tokens = iter::once((FlatToken::Token(self.start_token.0.clone()), self.start_token.1)) let tokens = iter::once(FlatToken::Token(self.start_token.clone()))
.chain(iter::repeat_with(|| { .chain(iter::repeat_with(|| FlatToken::Token(cursor_snapshot.next())))
let token = cursor_snapshot.next();
(FlatToken::Token(token.0), token.1)
}))
.take(self.num_calls as usize); .take(self.num_calls as usize);
if self.replace_ranges.is_empty() { if self.replace_ranges.is_empty() {
@ -156,11 +153,8 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl {
(range.start as usize)..(range.end as usize), (range.start as usize)..(range.end as usize),
target target
.into_iter() .into_iter()
.map(|target| (FlatToken::AttrsTarget(target), Spacing::Alone)) .map(|target| FlatToken::AttrsTarget(target))
.chain( .chain(iter::repeat(FlatToken::Empty).take(range.len() - target_len)),
iter::repeat((FlatToken::Empty, Spacing::Alone))
.take(range.len() - target_len),
),
); );
} }
make_attr_token_stream(tokens.into_iter(), self.break_last_token) make_attr_token_stream(tokens.into_iter(), self.break_last_token)
@ -367,7 +361,7 @@ impl<'a> Parser<'a> {
/// `AttrTokenStream`, creating an `AttrTokenTree::Delimited` for each matching pair of open and /// `AttrTokenStream`, creating an `AttrTokenTree::Delimited` for each matching pair of open and
/// close delims. /// close delims.
fn make_attr_token_stream( fn make_attr_token_stream(
iter: impl Iterator<Item = (FlatToken, Spacing)>, iter: impl Iterator<Item = FlatToken>,
break_last_token: bool, break_last_token: bool,
) -> AttrTokenStream { ) -> AttrTokenStream {
#[derive(Debug)] #[derive(Debug)]
@ -379,15 +373,15 @@ fn make_attr_token_stream(
// The stack always has at least one element. Storing it separately makes for shorter code. // The stack always has at least one element. Storing it separately makes for shorter code.
let mut stack_top = FrameData { open_delim_sp: None, inner: vec![] }; let mut stack_top = FrameData { open_delim_sp: None, inner: vec![] };
let mut stack_rest = vec![]; let mut stack_rest = vec![];
for (token, spacing) in iter { for flat_token in iter {
match token { match flat_token {
FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span }) => { FlatToken::Token((Token { kind: TokenKind::OpenDelim(delim), span }, spacing)) => {
stack_rest.push(mem::replace( stack_rest.push(mem::replace(
&mut stack_top, &mut stack_top,
FrameData { open_delim_sp: Some((delim, span, spacing)), inner: vec![] }, FrameData { open_delim_sp: Some((delim, span, spacing)), inner: vec![] },
)); ));
} }
FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => { FlatToken::Token((Token { kind: TokenKind::CloseDelim(delim), span }, spacing)) => {
let frame_data = mem::replace(&mut stack_top, stack_rest.pop().unwrap()); let frame_data = mem::replace(&mut stack_top, stack_rest.pop().unwrap());
let (open_delim, open_sp, open_spacing) = frame_data.open_delim_sp.unwrap(); let (open_delim, open_sp, open_spacing) = frame_data.open_delim_sp.unwrap();
assert_eq!( assert_eq!(
@ -400,7 +394,9 @@ fn make_attr_token_stream(
let delimited = AttrTokenTree::Delimited(dspan, dspacing, delim, stream); let delimited = AttrTokenTree::Delimited(dspan, dspacing, delim, stream);
stack_top.inner.push(delimited); stack_top.inner.push(delimited);
} }
FlatToken::Token(token) => stack_top.inner.push(AttrTokenTree::Token(token, spacing)), FlatToken::Token((token, spacing)) => {
stack_top.inner.push(AttrTokenTree::Token(token, spacing))
}
FlatToken::AttrsTarget(target) => { FlatToken::AttrsTarget(target) => {
stack_top.inner.push(AttrTokenTree::AttrsTarget(target)) stack_top.inner.push(AttrTokenTree::AttrsTarget(target))
} }

2
compiler/rustc_parse/src/parser/mod.rs
View file

@ -1607,7 +1607,7 @@ pub(crate) fn make_unclosed_delims_error(
enum FlatToken { enum FlatToken {
/// A token - this holds both delimiter (e.g. '{' and '}') /// A token - this holds both delimiter (e.g. '{' and '}')
/// and non-delimiter tokens /// and non-delimiter tokens
Token(Token), Token((Token, Spacing)),
/// Holds the `AttrsTarget` for an AST node. The `AttrsTarget` is inserted /// Holds the `AttrsTarget` for an AST node. The `AttrsTarget` is inserted
/// directly into the constructed `AttrTokenStream` as an /// directly into the constructed `AttrTokenStream` as an
/// `AttrTokenTree::AttrsTarget`. /// `AttrTokenTree::AttrsTarget`.