bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Change AttrTokenStream::to_tokenstream to to_token_trees.

Browse source 
I.e. change the return type from `TokenStream` to `Vec<TokenTree>`.

Most of the callsites require a `TokenStream`, but the recursive call
used to create `target_tokens` requires a `Vec<TokenTree>`. It's easy
to convert a `Vec<TokenTree>` to a `TokenStream` (just call
`TokenStream::new`) but it's harder to convert a `TokenStream` to a
`Vec<TokenTree>` (either iterate/clone/collect, or use `Lrc::into_inner`
if appropriate).

So this commit changes the return value to simplify that `target_tokens`
call site.
This commit is contained in:
Nicholas Nethercote 2024-06-27 10:42:46 +10:00
parent d6c0b8117e
commit f852568fa6
2 changed files with 15 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

14
compiler/rustc_ast/src/attr/mod.rs
View file

@ -204,12 +204,14 @@ impl Attribute {
pub fn tokens(&self) -> TokenStream {
match &self.kind {
AttrKind::Normal(normal) => normal
.tokens
.as_ref()
.unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}"))
.to_attr_token_stream()
.to_tokenstream(),
AttrKind::Normal(normal) => TokenStream::new(
normal
.tokens
.as_ref()
.unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}"))
.to_attr_token_stream()
.to_token_trees(),
),
&AttrKind::DocComment(comment_kind, data) => TokenStream::token_alone(
token::DocComment(comment_kind, self.style, data),
self.span,

23
compiler/rustc_ast/src/tokenstream.rs
View file

@ -180,14 +180,13 @@ impl AttrTokenStream {
AttrTokenStream(Lrc::new(tokens))
}
/// Converts this `AttrTokenStream` to a plain `TokenStream`.
/// Converts this `AttrTokenStream` to a plain `Vec<TokenTree>`.
/// During conversion, `AttrTokenTree::Attributes` get 'flattened'
/// back to a `TokenStream` of the form `outer_attr attr_target`.
/// If there are inner attributes, they are inserted into the proper
/// place in the attribute target tokens.
pub fn to_tokenstream(&self) -> TokenStream {
let trees: Vec<_> = self
.0
pub fn to_token_trees(&self) -> Vec<TokenTree> {
self.0
.iter()
.flat_map(|tree| match &tree {
AttrTokenTree::Token(inner, spacing) => {
@ -198,7 +197,7 @@ impl AttrTokenStream {
*span,
*spacing,
*delim,
stream.to_tokenstream()
TokenStream::new(stream.to_token_trees())
),]
.into_iter()
}
@ -208,14 +207,7 @@ impl AttrTokenStream {
.partition_point(|attr| matches!(attr.style, crate::AttrStyle::Outer));
let (outer_attrs, inner_attrs) = data.attrs.split_at(idx);
let mut target_tokens: Vec<_> = data
.tokens
.to_attr_token_stream()
.to_tokenstream()
.0
.iter()
.cloned()
.collect();
let mut target_tokens = data.tokens.to_attr_token_stream().to_token_trees();
if !inner_attrs.is_empty() {
let mut found = false;
// Check the last two trees (to account for a trailing semi)
@ -260,8 +252,7 @@ impl AttrTokenStream {
flat.into_iter()
}
})
.collect();
TokenStream::new(trees)
.collect()
}
}
@ -461,7 +452,7 @@ impl TokenStream {
AttributesData { attrs: attrs.iter().cloned().collect(), tokens: tokens.clone() };
AttrTokenStream::new(vec![AttrTokenTree::Attributes(attr_data)])
};
attr_stream.to_tokenstream()
TokenStream::new(attr_stream.to_token_trees())
}
pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {