bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Move Spacing out of AttrAnnotatedTokenStream.

Browse source 
And into `AttrAnnotatedTokenTree::Token`.

PR #99887 did the same thing for `TokenStream`.
This commit is contained in:
Nicholas Nethercote 2022-09-09 11:51:23 +10:00
parent 1120c5e01d
commit 890e759ffc
5 changed files with 34 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

13
compiler/rustc_ast/src/attr/mod.rs
View file

@ -303,13 +303,12 @@ impl Attribute {
.as_ref() .as_ref()
.unwrap_or_else(|| panic!("attribute is missing tokens: {:?}", self)) .unwrap_or_else(|| panic!("attribute is missing tokens: {:?}", self))
.create_token_stream(), .create_token_stream(),
AttrKind::DocComment(comment_kind, data) => AttrAnnotatedTokenStream::from(( AttrKind::DocComment(comment_kind, data) => {
AttrAnnotatedTokenTree::Token(Token::new( AttrAnnotatedTokenStream::new(vec![AttrAnnotatedTokenTree::Token(
token::DocComment(comment_kind, self.style, data), Token::new(token::DocComment(comment_kind, self.style, data), self.span),
self.span, Spacing::Alone,
)), )])
Spacing::Alone, }
)),
} }
} }
} }

4
compiler/rustc_ast/src/mut_visit.rs
View file

@ -644,7 +644,7 @@ pub fn noop_flat_map_param<T: MutVisitor>(mut param: Param, vis: &mut T) -> Smal
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`. // No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
pub fn visit_attr_annotated_tt<T: MutVisitor>(tt: &mut AttrAnnotatedTokenTree, vis: &mut T) { pub fn visit_attr_annotated_tt<T: MutVisitor>(tt: &mut AttrAnnotatedTokenTree, vis: &mut T) {
match tt { match tt {
AttrAnnotatedTokenTree::Token(token) => { AttrAnnotatedTokenTree::Token(token, _) => {
visit_token(token, vis); visit_token(token, vis);
} }
AttrAnnotatedTokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => { AttrAnnotatedTokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
@ -696,7 +696,7 @@ pub fn visit_attr_annotated_tts<T: MutVisitor>(
) { ) {
if T::VISIT_TOKENS && !tts.is_empty() { if T::VISIT_TOKENS && !tts.is_empty() {
let tts = Lrc::make_mut(tts); let tts = Lrc::make_mut(tts);
visit_vec(tts, |(tree, _is_joint)| visit_attr_annotated_tt(tree, vis)); visit_vec(tts, |tree| visit_attr_annotated_tt(tree, vis));
} }
} }

23
compiler/rustc_ast/src/tokenstream.rs
View file

@ -177,12 +177,12 @@ impl<CTX> HashStable<CTX> for LazyTokenStream {
/// during expansion to perform early cfg-expansion, and to process attributes /// during expansion to perform early cfg-expansion, and to process attributes
/// during proc-macro invocations. /// during proc-macro invocations.
#[derive(Clone, Debug, Default, Encodable, Decodable)] #[derive(Clone, Debug, Default, Encodable, Decodable)]
pub struct AttrAnnotatedTokenStream(pub Lrc<Vec<(AttrAnnotatedTokenTree, Spacing)>>); pub struct AttrAnnotatedTokenStream(pub Lrc<Vec<AttrAnnotatedTokenTree>>);
/// Like `TokenTree`, but for `AttrAnnotatedTokenStream` /// Like `TokenTree`, but for `AttrAnnotatedTokenStream`
#[derive(Clone, Debug, Encodable, Decodable)] #[derive(Clone, Debug, Encodable, Decodable)]
pub enum AttrAnnotatedTokenTree { pub enum AttrAnnotatedTokenTree {
Token(Token), Token(Token, Spacing),
Delimited(DelimSpan, Delimiter, AttrAnnotatedTokenStream), Delimited(DelimSpan, Delimiter, AttrAnnotatedTokenStream),
/// Stores the attributes for an attribute target, /// Stores the attributes for an attribute target,
/// along with the tokens for that attribute target. /// along with the tokens for that attribute target.
@ -191,7 +191,7 @@ pub enum AttrAnnotatedTokenTree {
} }
impl AttrAnnotatedTokenStream { impl AttrAnnotatedTokenStream {
pub fn new(tokens: Vec<(AttrAnnotatedTokenTree, Spacing)>) -> AttrAnnotatedTokenStream { pub fn new(tokens: Vec<AttrAnnotatedTokenTree>) -> AttrAnnotatedTokenStream {
AttrAnnotatedTokenStream(Lrc::new(tokens)) AttrAnnotatedTokenStream(Lrc::new(tokens))
} }
@ -204,9 +204,9 @@ impl AttrAnnotatedTokenStream {
let trees: Vec<_> = self let trees: Vec<_> = self
.0 .0
.iter() .iter()
.flat_map(|tree| match &tree.0 { .flat_map(|tree| match &tree {
AttrAnnotatedTokenTree::Token(inner) => { AttrAnnotatedTokenTree::Token(inner, spacing) => {
smallvec![TokenTree::Token(inner.clone(), tree.1)].into_iter() smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter()
} }
AttrAnnotatedTokenTree::Delimited(span, delim, stream) => { AttrAnnotatedTokenTree::Delimited(span, delim, stream) => {
smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),] smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),]
@ -363,12 +363,6 @@ impl TokenStream {
} }
} }
impl From<(AttrAnnotatedTokenTree, Spacing)> for AttrAnnotatedTokenStream {
fn from((tree, spacing): (AttrAnnotatedTokenTree, Spacing)) -> AttrAnnotatedTokenStream {
AttrAnnotatedTokenStream::new(vec![(tree, spacing)])
}
}
impl iter::FromIterator<TokenTree> for TokenStream { impl iter::FromIterator<TokenTree> for TokenStream {
fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self { fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self {
TokenStream::new(iter.into_iter().collect::<Vec<TokenTree>>()) TokenStream::new(iter.into_iter().collect::<Vec<TokenTree>>())
@ -428,10 +422,7 @@ impl TokenStream {
} else { } else {
let attr_data = let attr_data =
AttributesData { attrs: attrs.iter().cloned().collect(), tokens: tokens.clone() }; AttributesData { attrs: attrs.iter().cloned().collect(), tokens: tokens.clone() };
AttrAnnotatedTokenStream::new(vec![( AttrAnnotatedTokenStream::new(vec![AttrAnnotatedTokenTree::Attributes(attr_data)])
AttrAnnotatedTokenTree::Attributes(attr_data),
Spacing::Alone,
)])
}; };
Some(attr_annotated.to_tokenstream()) Some(attr_annotated.to_tokenstream())
} }

22
compiler/rustc_expand/src/config.rs
View file

@ -276,9 +276,9 @@ impl<'a> StripUnconfigured<'a> {
/// Normal cfg-expansion operates on parsed AST nodes via the `configure` method /// Normal cfg-expansion operates on parsed AST nodes via the `configure` method
fn configure_tokens(&self, stream: &AttrAnnotatedTokenStream) -> AttrAnnotatedTokenStream { fn configure_tokens(&self, stream: &AttrAnnotatedTokenStream) -> AttrAnnotatedTokenStream {
fn can_skip(stream: &AttrAnnotatedTokenStream) -> bool { fn can_skip(stream: &AttrAnnotatedTokenStream) -> bool {
stream.0.iter().all(|(tree, _spacing)| match tree { stream.0.iter().all(|tree| match tree {
AttrAnnotatedTokenTree::Attributes(_) => false, AttrAnnotatedTokenTree::Attributes(_) => false,
AttrAnnotatedTokenTree::Token(_) => true, AttrAnnotatedTokenTree::Token(..) => true,
AttrAnnotatedTokenTree::Delimited(_, _, inner) => can_skip(inner), AttrAnnotatedTokenTree::Delimited(_, _, inner) => can_skip(inner),
}) })
} }
@ -290,7 +290,7 @@ impl<'a> StripUnconfigured<'a> {
let trees: Vec<_> = stream let trees: Vec<_> = stream
.0 .0
.iter() .iter()
.flat_map(|(tree, spacing)| match tree.clone() { .flat_map(|tree| match tree.clone() {
AttrAnnotatedTokenTree::Attributes(mut data) => { AttrAnnotatedTokenTree::Attributes(mut data) => {
data.attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr)); data.attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr));
@ -298,24 +298,24 @@ impl<'a> StripUnconfigured<'a> {
data.tokens = LazyTokenStream::new( data.tokens = LazyTokenStream::new(
self.configure_tokens(&data.tokens.create_token_stream()), self.configure_tokens(&data.tokens.create_token_stream()),
); );
Some((AttrAnnotatedTokenTree::Attributes(data), *spacing)).into_iter() Some(AttrAnnotatedTokenTree::Attributes(data)).into_iter()
} else { } else {
None.into_iter() None.into_iter()
} }
} }
AttrAnnotatedTokenTree::Delimited(sp, delim, mut inner) => { AttrAnnotatedTokenTree::Delimited(sp, delim, mut inner) => {
inner = self.configure_tokens(&inner); inner = self.configure_tokens(&inner);
Some((AttrAnnotatedTokenTree::Delimited(sp, delim, inner), *spacing)) Some(AttrAnnotatedTokenTree::Delimited(sp, delim, inner))
.into_iter() .into_iter()
} }
AttrAnnotatedTokenTree::Token(ref token) if let TokenKind::Interpolated(ref nt) = token.kind => { AttrAnnotatedTokenTree::Token(ref token, _) if let TokenKind::Interpolated(ref nt) = token.kind => {
panic!( panic!(
"Nonterminal should have been flattened at {:?}: {:?}", "Nonterminal should have been flattened at {:?}: {:?}",
token.span, nt token.span, nt
); );
} }
AttrAnnotatedTokenTree::Token(token) => { AttrAnnotatedTokenTree::Token(token, spacing) => {
Some((AttrAnnotatedTokenTree::Token(token), *spacing)).into_iter() Some(AttrAnnotatedTokenTree::Token(token, spacing)).into_iter()
} }
}) })
.collect(); .collect();
@ -404,13 +404,13 @@ impl<'a> StripUnconfigured<'a> {
}; };
let pound_span = pound_token.span; let pound_span = pound_token.span;
let mut trees = vec![(AttrAnnotatedTokenTree::Token(pound_token), Spacing::Alone)]; let mut trees = vec![AttrAnnotatedTokenTree::Token(pound_token, Spacing::Alone)];
if attr.style == AttrStyle::Inner { if attr.style == AttrStyle::Inner {
// For inner attributes, we do the same thing for the `!` in `#![some_attr]` // For inner attributes, we do the same thing for the `!` in `#![some_attr]`
let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) = orig_trees.next().unwrap() else { let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) = orig_trees.next().unwrap() else {
panic!("Bad tokens for attribute {:?}", attr); panic!("Bad tokens for attribute {:?}", attr);
}; };
trees.push((AttrAnnotatedTokenTree::Token(bang_token), Spacing::Alone)); trees.push(AttrAnnotatedTokenTree::Token(bang_token, Spacing::Alone));
} }
// We don't really have a good span to use for the synthesized `[]` // We don't really have a good span to use for the synthesized `[]`
// in `#[attr]`, so just use the span of the `#` token. // in `#[attr]`, so just use the span of the `#` token.
@ -422,7 +422,7 @@ impl<'a> StripUnconfigured<'a> {
.unwrap_or_else(|| panic!("Missing tokens for {:?}", item)) .unwrap_or_else(|| panic!("Missing tokens for {:?}", item))
.create_token_stream(), .create_token_stream(),
); );
trees.push((bracket_group, Spacing::Alone)); trees.push(bracket_group);
let tokens = Some(LazyTokenStream::new(AttrAnnotatedTokenStream::new(trees))); let tokens = Some(LazyTokenStream::new(AttrAnnotatedTokenStream::new(trees)));
let attr = attr::mk_attr_from_item(item, tokens, attr.style, item_span); let attr = attr::mk_attr_from_item(item, tokens, attr.style, item_span);
if attr.has_name(sym::crate_type) { if attr.has_name(sym::crate_type) {

16
compiler/rustc_parse/src/parser/attr_wrapper.rs
View file

@ -397,7 +397,7 @@ fn make_token_stream(
struct FrameData { struct FrameData {
// This is `None` for the first frame, `Some` for all others. // This is `None` for the first frame, `Some` for all others.
open_delim_sp: Option<(Delimiter, Span)>, open_delim_sp: Option<(Delimiter, Span)>,
inner: Vec<(AttrAnnotatedTokenTree, Spacing)>, inner: Vec<AttrAnnotatedTokenTree>,
} }
let mut stack = vec![FrameData { open_delim_sp: None, inner: vec![] }]; let mut stack = vec![FrameData { open_delim_sp: None, inner: vec![] }];
let mut token_and_spacing = iter.next(); let mut token_and_spacing = iter.next();
@ -426,34 +426,34 @@ fn make_token_stream(
panic!("Bottom token frame is missing for token: {:?}", token) panic!("Bottom token frame is missing for token: {:?}", token)
}) })
.inner .inner
.push((delimited, Spacing::Alone)); .push(delimited);
} }
FlatToken::Token(token) => stack FlatToken::Token(token) => stack
.last_mut() .last_mut()
.expect("Bottom token frame is missing!") .expect("Bottom token frame is missing!")
.inner .inner
.push((AttrAnnotatedTokenTree::Token(token), spacing)), .push(AttrAnnotatedTokenTree::Token(token, spacing)),
FlatToken::AttrTarget(data) => stack FlatToken::AttrTarget(data) => stack
.last_mut() .last_mut()
.expect("Bottom token frame is missing!") .expect("Bottom token frame is missing!")
.inner .inner
.push((AttrAnnotatedTokenTree::Attributes(data), spacing)), .push(AttrAnnotatedTokenTree::Attributes(data)),
FlatToken::Empty => {} FlatToken::Empty => {}
} }
token_and_spacing = iter.next(); token_and_spacing = iter.next();
} }
let mut final_buf = stack.pop().expect("Missing final buf!"); let mut final_buf = stack.pop().expect("Missing final buf!");
if break_last_token { if break_last_token {
let (last_token, spacing) = final_buf.inner.pop().unwrap(); let last_token = final_buf.inner.pop().unwrap();
if let AttrAnnotatedTokenTree::Token(last_token) = last_token { if let AttrAnnotatedTokenTree::Token(last_token, spacing) = last_token {
let unglued_first = last_token.kind.break_two_token_op().unwrap().0; let unglued_first = last_token.kind.break_two_token_op().unwrap().0;
// An 'unglued' token is always two ASCII characters // An 'unglued' token is always two ASCII characters
let mut first_span = last_token.span.shrink_to_lo(); let mut first_span = last_token.span.shrink_to_lo();
first_span = first_span.with_hi(first_span.lo() + rustc_span::BytePos(1)); first_span = first_span.with_hi(first_span.lo() + rustc_span::BytePos(1));
final_buf.inner.push(( final_buf.inner.push(AttrAnnotatedTokenTree::Token(
AttrAnnotatedTokenTree::Token(Token::new(unglued_first, first_span)), Token::new(unglued_first, first_span),
spacing, spacing,
)); ));
} else { } else {