bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Implement token-based handling of attributes during expansion

Browse source 
This PR modifies the macro expansion infrastructure to handle attributes
in a fully token-based manner. As a result:

* Derives macros no longer lose spans when their input is modified
  by eager cfg-expansion. This is accomplished by performing eager
  cfg-expansion on the token stream that we pass to the derive
  proc-macro
* Inner attributes now preserve spans in all cases, including when we
  have multiple inner attributes in a row.

This is accomplished through the following changes:

* New structs `AttrAnnotatedTokenStream` and `AttrAnnotatedTokenTree` are introduced.
  These are very similar to a normal `TokenTree`, but they also track
  the position of attributes and attribute targets within the stream.
  They are built when we collect tokens during parsing.
  An `AttrAnnotatedTokenStream` is converted to a regular `TokenStream` when
  we invoke a macro.
* Token capturing and `LazyTokenStream` are modified to work with
  `AttrAnnotatedTokenStream`. A new `ReplaceRange` type is introduced, which
  is created during the parsing of a nested AST node to make the 'outer'
  AST node aware of the attributes and attribute target stored deeper in the token stream.
* When we need to perform eager cfg-expansion (either due to `#[derive]` or `#[cfg_eval]`),
we tokenize and reparse our target, capturing additional information about the locations of
`#[cfg]` and `#[cfg_attr]` attributes at any depth within the target.
This is a performance optimization, allowing us to perform less work
in the typical case where captured tokens never have eager cfg-expansion run.
This commit is contained in:
Aaron Hill 2020-11-28 18:33:17 -05:00
parent 25ea6be13e
commit a93c4f05de
No known key found for this signature in database
GPG key ID: B4087E510E98B164
33 changed files with 2046 additions and 1192 deletions
Download patch file Download diff file Expand all files Collapse all files

83
compiler/rustc_expand/src/base.rs
View file

@ -3,7 +3,7 @@ use crate::module::DirOwnership;
use rustc_ast::ptr::P;
use rustc_ast::token::{self, Nonterminal};
use rustc_ast::tokenstream::{CanSynthesizeMissingTokens, LazyTokenStream, TokenStream};
use rustc_ast::tokenstream::{CanSynthesizeMissingTokens, TokenStream};
use rustc_ast::visit::{AssocCtxt, Visitor};
use rustc_ast::{self as ast, AstLike, Attribute, Item, NodeId, PatKind};
use rustc_attr::{self as attr, Deprecation, Stability};
@ -46,62 +46,6 @@ pub enum Annotatable {
Variant(ast::Variant),
}
impl AstLike for Annotatable {
fn attrs(&self) -> &[Attribute] {
match *self {
Annotatable::Item(ref item) => &item.attrs,
Annotatable::TraitItem(ref trait_item) => &trait_item.attrs,
Annotatable::ImplItem(ref impl_item) => &impl_item.attrs,
Annotatable::ForeignItem(ref foreign_item) => &foreign_item.attrs,
Annotatable::Stmt(ref stmt) => stmt.attrs(),
Annotatable::Expr(ref expr) => &expr.attrs,
Annotatable::Arm(ref arm) => &arm.attrs,
Annotatable::ExprField(ref field) => &field.attrs,
Annotatable::PatField(ref fp) => &fp.attrs,
Annotatable::GenericParam(ref gp) => &gp.attrs,
Annotatable::Param(ref p) => &p.attrs,
Annotatable::FieldDef(ref sf) => &sf.attrs,
Annotatable::Variant(ref v) => &v.attrs(),
}
}
fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
match self {
Annotatable::Item(item) => item.visit_attrs(f),
Annotatable::TraitItem(trait_item) => trait_item.visit_attrs(f),
Annotatable::ImplItem(impl_item) => impl_item.visit_attrs(f),
Annotatable::ForeignItem(foreign_item) => foreign_item.visit_attrs(f),
Annotatable::Stmt(stmt) => stmt.visit_attrs(f),
Annotatable::Expr(expr) => expr.visit_attrs(f),
Annotatable::Arm(arm) => arm.visit_attrs(f),
Annotatable::ExprField(field) => field.visit_attrs(f),
Annotatable::PatField(fp) => fp.visit_attrs(f),
Annotatable::GenericParam(gp) => gp.visit_attrs(f),
Annotatable::Param(p) => p.visit_attrs(f),
Annotatable::FieldDef(sf) => sf.visit_attrs(f),
Annotatable::Variant(v) => v.visit_attrs(f),
}
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
match self {
Annotatable::Item(item) => item.tokens_mut(),
Annotatable::TraitItem(trait_item) => trait_item.tokens_mut(),
Annotatable::ImplItem(impl_item) => impl_item.tokens_mut(),
Annotatable::ForeignItem(foreign_item) => foreign_item.tokens_mut(),
Annotatable::Stmt(stmt) => stmt.tokens_mut(),
Annotatable::Expr(expr) => expr.tokens_mut(),
Annotatable::Arm(arm) => arm.tokens_mut(),
Annotatable::ExprField(field) => field.tokens_mut(),
Annotatable::PatField(fp) => fp.tokens_mut(),
Annotatable::GenericParam(gp) => gp.tokens_mut(),
Annotatable::Param(p) => p.tokens_mut(),
Annotatable::FieldDef(sf) => sf.tokens_mut(),
Annotatable::Variant(v) => v.tokens_mut(),
}
}
}
impl Annotatable {
pub fn span(&self) -> Span {
match *self {
@ -121,6 +65,24 @@ impl Annotatable {
}
}
pub fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
match self {
Annotatable::Item(item) => item.visit_attrs(f),
Annotatable::TraitItem(trait_item) => trait_item.visit_attrs(f),
Annotatable::ImplItem(impl_item) => impl_item.visit_attrs(f),
Annotatable::ForeignItem(foreign_item) => foreign_item.visit_attrs(f),
Annotatable::Stmt(stmt) => stmt.visit_attrs(f),
Annotatable::Expr(expr) => expr.visit_attrs(f),
Annotatable::Arm(arm) => arm.visit_attrs(f),
Annotatable::ExprField(field) => field.visit_attrs(f),
Annotatable::PatField(fp) => fp.visit_attrs(f),
Annotatable::GenericParam(gp) => gp.visit_attrs(f),
Annotatable::Param(p) => p.visit_attrs(f),
Annotatable::FieldDef(sf) => sf.visit_attrs(f),
Annotatable::Variant(v) => v.visit_attrs(f),
}
}
pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) {
match self {
Annotatable::Item(item) => visitor.visit_item(item),
@ -139,7 +101,7 @@ impl Annotatable {
}
}
crate fn into_nonterminal(self) -> Nonterminal {
pub fn into_nonterminal(self) -> Nonterminal {
match self {
Annotatable::Item(item) => token::NtItem(item),
Annotatable::TraitItem(item) | Annotatable::ImplItem(item) => {
@ -161,10 +123,7 @@ impl Annotatable {
}
crate fn into_tokens(self, sess: &ParseSess) -> TokenStream {
// Tokens of an attribute target may be invalidated by some outer `#[derive]` performing
// "full configuration" (attributes following derives on the same item should be the most
// common case), that's why synthesizing tokens is allowed.
nt_to_tokenstream(&self.into_nonterminal(), sess, CanSynthesizeMissingTokens::Yes)
nt_to_tokenstream(&self.into_nonterminal(), sess, CanSynthesizeMissingTokens::No)
}
pub fn expect_item(self) -> P<ast::Item> {

121
compiler/rustc_expand/src/config.rs
View file

@ -2,8 +2,10 @@
use rustc_ast::ptr::P;
use rustc_ast::token::{DelimToken, Token, TokenKind};
use rustc_ast::tokenstream::{DelimSpan, LazyTokenStream, Spacing, TokenStream, TokenTree};
use rustc_ast::{self as ast, AstLike, AttrItem, Attribute, MetaItem};
use rustc_ast::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree};
use rustc_ast::tokenstream::{DelimSpan, Spacing};
use rustc_ast::tokenstream::{LazyTokenStream, TokenTree};
use rustc_ast::{self as ast, AstLike, AttrItem, AttrStyle, Attribute, MetaItem};
use rustc_attr as attr;
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::map_in_place::MapInPlace;
@ -23,7 +25,10 @@ use rustc_span::{Span, DUMMY_SP};
pub struct StripUnconfigured<'a> {
pub sess: &'a Session,
pub features: Option<&'a Features>,
pub modified: bool,
/// If `true`, perform cfg-stripping on attached tokens.
/// This is only used for the input to derive macros,
/// which needs eager expansion of `cfg` and `cfg_attr`
pub config_tokens: bool,
}
fn get_features(
@ -194,7 +199,7 @@ fn get_features(
// `cfg_attr`-process the crate's attributes and compute the crate's features.
pub fn features(sess: &Session, mut krate: ast::Crate) -> (ast::Crate, Features) {
let mut strip_unconfigured = StripUnconfigured { sess, features: None, modified: false };
let mut strip_unconfigured = StripUnconfigured { sess, features: None, config_tokens: false };
let unconfigured_attrs = krate.attrs.clone();
let diag = &sess.parse_sess.span_diagnostic;
@ -241,24 +246,83 @@ impl<'a> StripUnconfigured<'a> {
pub fn configure<T: AstLike>(&mut self, mut node: T) -> Option<T> {
self.process_cfg_attrs(&mut node);
if self.in_cfg(node.attrs()) {
self.try_configure_tokens(&mut node);
Some(node)
} else {
self.modified = true;
None
}
}
fn try_configure_tokens<T: AstLike>(&mut self, node: &mut T) {
if self.config_tokens {
if let Some(Some(tokens)) = node.tokens_mut() {
let attr_annotated_tokens = tokens.create_token_stream();
*tokens = LazyTokenStream::new(self.configure_tokens(&attr_annotated_tokens));
}
}
}
fn configure_krate_attrs(
&mut self,
mut attrs: Vec<ast::Attribute>,
) -> Option<Vec<ast::Attribute>> {
attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr));
if self.in_cfg(&attrs) {
Some(attrs)
} else {
self.modified = true;
None
if self.in_cfg(&attrs) { Some(attrs) } else { None }
}
/// Performs cfg-expansion on `stream`, producing a new `AttrAnnotatedTokenStream`.
/// This is only used during the invocation of `derive` proc-macros,
/// which require that we cfg-expand their entire input.
/// Normal cfg-expansion operates on parsed AST nodes via the `configure` method
fn configure_tokens(&mut self, stream: &AttrAnnotatedTokenStream) -> AttrAnnotatedTokenStream {
fn can_skip(stream: &AttrAnnotatedTokenStream) -> bool {
stream.0.iter().all(|(tree, _spacing)| match tree {
AttrAnnotatedTokenTree::Attributes(_) => false,
AttrAnnotatedTokenTree::Token(_) => true,
AttrAnnotatedTokenTree::Delimited(_, _, inner) => can_skip(inner),
})
}
if can_skip(stream) {
return stream.clone();
}
let trees: Vec<_> = stream
.0
.iter()
.flat_map(|(tree, spacing)| match tree.clone() {
AttrAnnotatedTokenTree::Attributes(mut data) => {
let mut attrs: Vec<_> = std::mem::take(&mut data.attrs).into();
attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr));
data.attrs = attrs.into();
if self.in_cfg(&data.attrs) {
data.tokens = LazyTokenStream::new(
self.configure_tokens(&data.tokens.create_token_stream()),
);
Some((AttrAnnotatedTokenTree::Attributes(data), *spacing)).into_iter()
} else {
None.into_iter()
}
}
AttrAnnotatedTokenTree::Delimited(sp, delim, mut inner) => {
inner = self.configure_tokens(&inner);
Some((AttrAnnotatedTokenTree::Delimited(sp, delim, inner), *spacing))
.into_iter()
}
AttrAnnotatedTokenTree::Token(token) => {
if let TokenKind::Interpolated(nt) = token.kind {
panic!(
"Nonterminal should have been flattened at {:?}: {:?}",
token.span, nt
);
} else {
Some((AttrAnnotatedTokenTree::Token(token), *spacing)).into_iter()
}
}
})
.collect();
AttrAnnotatedTokenStream::new(trees)
}
/// Parse and expand all `cfg_attr` attributes into a list of attributes
@ -285,9 +349,6 @@ impl<'a> StripUnconfigured<'a> {
return vec![attr];
}
// A `#[cfg_attr]` either gets removed, or replaced with a new attribute
self.modified = true;
let (cfg_predicate, expanded_attrs) = match self.parse_cfg_attr(&attr) {
None => return vec![],
Some(r) => r,
@ -311,7 +372,7 @@ impl<'a> StripUnconfigured<'a> {
expanded_attrs
.into_iter()
.flat_map(|(item, span)| {
let orig_tokens = attr.tokens();
let orig_tokens = attr.tokens().to_tokenstream();
// We are taking an attribute of the form `#[cfg_attr(pred, attr)]`
// and producing an attribute of the form `#[attr]`. We
@ -321,25 +382,34 @@ impl<'a> StripUnconfigured<'a> {
// Use the `#` in `#[cfg_attr(pred, attr)]` as the `#` token
// for `attr` when we expand it to `#[attr]`
let pound_token = orig_tokens.trees().next().unwrap();
if !matches!(pound_token, TokenTree::Token(Token { kind: TokenKind::Pound, .. })) {
panic!("Bad tokens for attribute {:?}", attr);
let mut orig_trees = orig_tokens.trees();
let pound_token = match orig_trees.next().unwrap() {
TokenTree::Token(token @ Token { kind: TokenKind::Pound, .. }) => token,
_ => panic!("Bad tokens for attribute {:?}", attr),
};
let pound_span = pound_token.span;
let mut trees = vec![(AttrAnnotatedTokenTree::Token(pound_token), Spacing::Alone)];
if attr.style == AttrStyle::Inner {
// For inner attributes, we do the same thing for the `!` in `#![some_attr]`
let bang_token = match orig_trees.next().unwrap() {
TokenTree::Token(token @ Token { kind: TokenKind::Not, .. }) => token,
_ => panic!("Bad tokens for attribute {:?}", attr),
};
trees.push((AttrAnnotatedTokenTree::Token(bang_token), Spacing::Alone));
}
// We don't really have a good span to use for the syntheized `[]`
// in `#[attr]`, so just use the span of the `#` token.
let bracket_group = TokenTree::Delimited(
DelimSpan::from_single(pound_token.span()),
let bracket_group = AttrAnnotatedTokenTree::Delimited(
DelimSpan::from_single(pound_span),
DelimToken::Bracket,
item.tokens
.as_ref()
.unwrap_or_else(|| panic!("Missing tokens for {:?}", item))
.create_token_stream(),
);
let tokens = Some(LazyTokenStream::new(TokenStream::new(vec![
(pound_token, Spacing::Alone),
(bracket_group, Spacing::Alone),
])));
trees.push((bracket_group, Spacing::Alone));
let tokens = Some(LazyTokenStream::new(AttrAnnotatedTokenStream::new(trees)));
self.process_cfg_attr(attr::mk_attr_from_item(item, tokens, attr.style, span))
})
.collect()
@ -457,7 +527,8 @@ impl<'a> StripUnconfigured<'a> {
self.sess.parse_sess.span_diagnostic.span_err(attr.span, msg);
}
self.process_cfg_attrs(expr)
self.process_cfg_attrs(expr);
self.try_configure_tokens(&mut *expr);
}
}

38
compiler/rustc_expand/src/expand.rs
View file

@ -12,7 +12,7 @@ use rustc_ast::ptr::P;
use rustc_ast::token;
use rustc_ast::tokenstream::TokenStream;
use rustc_ast::visit::{self, AssocCtxt, Visitor};
use rustc_ast::{AstLike, AttrItem, AttrStyle, Block, Inline, ItemKind, LitKind, MacArgs};
use rustc_ast::{AstLike, AttrItem, Block, Inline, ItemKind, LitKind, MacArgs};
use rustc_ast::{MacCallStmt, MacStmtStyle, MetaItemKind, ModKind, NestedMetaItem};
use rustc_ast::{NodeId, PatKind, Path, StmtKind, Unsafe};
use rustc_ast_pretty::pprust;
@ -611,10 +611,15 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
let invocations = {
let mut collector = InvocationCollector {
// Non-derive macro invocations cannot see the results of cfg expansion - they
// will either be removed along with the item, or invoked before the cfg/cfg_attr
// attribute is expanded. Therefore, we don't need to configure the tokens
// Derive macros *can* see the results of cfg-expansion - they are handled
// specially in `fully_expand_fragment`
cfg: StripUnconfigured {
sess: &self.cx.sess,
features: self.cx.ecfg.features,
modified: false,
config_tokens: false,
},
cx: self.cx,
invocations: Vec::new(),
@ -709,13 +714,26 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
SyntaxExtensionKind::Attr(expander) => {
self.gate_proc_macro_input(&item);
self.gate_proc_macro_attr_item(span, &item);
let tokens = match attr.style {
AttrStyle::Outer => item.into_tokens(&self.cx.sess.parse_sess),
// FIXME: Properly collect tokens for inner attributes
AttrStyle::Inner => rustc_parse::fake_token_stream(
let mut fake_tokens = false;
if let Annotatable::Item(item_inner) = &item {
if let ItemKind::Mod(_, mod_kind) = &item_inner.kind {
// FIXME: Collect tokens and use them instead of generating
// fake ones. These are unstable, so it needs to be
// fixed prior to stabilization
// Fake tokens when we are invoking an inner attribute, and:
fake_tokens = matches!(attr.style, ast::AttrStyle::Inner) &&
// We are invoking an attribute on the crate root, or an outline
// module
(item_inner.ident.name.is_empty() || !matches!(mod_kind, ast::ModKind::Loaded(_, Inline::Yes, _)));
}
}
let tokens = if fake_tokens {
rustc_parse::fake_token_stream(
&self.cx.sess.parse_sess,
&item.into_nonterminal(),
),
)
} else {
item.into_tokens(&self.cx.sess.parse_sess)
};
let attr_item = attr.unwrap_normal_item();
if let MacArgs::Eq(..) = attr_item.args {
@ -897,21 +915,21 @@ pub fn parse_ast_fragment<'a>(
}
AstFragmentKind::TraitItems => {
let mut items = SmallVec::new();
while let Some(item) = this.parse_trait_item()? {
while let Some(item) = this.parse_trait_item(ForceCollect::No)? {
items.extend(item);
}
AstFragment::TraitItems(items)
}
AstFragmentKind::ImplItems => {
let mut items = SmallVec::new();
while let Some(item) = this.parse_impl_item()? {
while let Some(item) = this.parse_impl_item(ForceCollect::No)? {
items.extend(item);
}
AstFragment::ImplItems(items)
}
AstFragmentKind::ForeignItems => {
let mut items = SmallVec::new();
while let Some(item) = this.parse_foreign_item()? {
while let Some(item) = this.parse_foreign_item(ForceCollect::No)? {
items.extend(item);
}
AstFragment::ForeignItems(items)

2
compiler/rustc_expand/src/proc_macro.rs
View file

@ -94,7 +94,7 @@ impl MultiItemModifier for ProcMacroDerive {
{
TokenTree::token(token::Interpolated(Lrc::new(item)), DUMMY_SP).into()
} else {
nt_to_tokenstream(&item, &ecx.sess.parse_sess, CanSynthesizeMissingTokens::Yes)
nt_to_tokenstream(&item, &ecx.sess.parse_sess, CanSynthesizeMissingTokens::No)
};
let server = proc_macro_server::Rustc::new(ecx);