bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Implement token-based handling of attributes during expansion

Browse source 
This PR modifies the macro expansion infrastructure to handle attributes
in a fully token-based manner. As a result:

* Derives macros no longer lose spans when their input is modified
  by eager cfg-expansion. This is accomplished by performing eager
  cfg-expansion on the token stream that we pass to the derive
  proc-macro
* Inner attributes now preserve spans in all cases, including when we
  have multiple inner attributes in a row.

This is accomplished through the following changes:

* New structs `AttrAnnotatedTokenStream` and `AttrAnnotatedTokenTree` are introduced.
  These are very similar to a normal `TokenTree`, but they also track
  the position of attributes and attribute targets within the stream.
  They are built when we collect tokens during parsing.
  An `AttrAnnotatedTokenStream` is converted to a regular `TokenStream` when
  we invoke a macro.
* Token capturing and `LazyTokenStream` are modified to work with
  `AttrAnnotatedTokenStream`. A new `ReplaceRange` type is introduced, which
  is created during the parsing of a nested AST node to make the 'outer'
  AST node aware of the attributes and attribute target stored deeper in the token stream.
* When we need to perform eager cfg-expansion (either due to `#[derive]` or `#[cfg_eval]`),
we tokenize and reparse our target, capturing additional information about the locations of
`#[cfg]` and `#[cfg_attr]` attributes at any depth within the target.
This is a performance optimization, allowing us to perform less work
in the typical case where captured tokens never have eager cfg-expansion run.
This commit is contained in:
Aaron Hill 2020-11-28 18:33:17 -05:00
parent 25ea6be13e
commit a93c4f05de
No known key found for this signature in database
GPG key ID: B4087E510E98B164
33 changed files with 2046 additions and 1192 deletions
Download patch file Download diff file Expand all files Collapse all files

38
compiler/rustc_expand/src/expand.rs
View file

@ -12,7 +12,7 @@ use rustc_ast::ptr::P;
use rustc_ast::token;
use rustc_ast::tokenstream::TokenStream;
use rustc_ast::visit::{self, AssocCtxt, Visitor};
use rustc_ast::{AstLike, AttrItem, AttrStyle, Block, Inline, ItemKind, LitKind, MacArgs};
use rustc_ast::{AstLike, AttrItem, Block, Inline, ItemKind, LitKind, MacArgs};
use rustc_ast::{MacCallStmt, MacStmtStyle, MetaItemKind, ModKind, NestedMetaItem};
use rustc_ast::{NodeId, PatKind, Path, StmtKind, Unsafe};
use rustc_ast_pretty::pprust;
@ -611,10 +611,15 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
let invocations = {
let mut collector = InvocationCollector {
// Non-derive macro invocations cannot see the results of cfg expansion - they
// will either be removed along with the item, or invoked before the cfg/cfg_attr
// attribute is expanded. Therefore, we don't need to configure the tokens
// Derive macros *can* see the results of cfg-expansion - they are handled
// specially in `fully_expand_fragment`
cfg: StripUnconfigured {
sess: &self.cx.sess,
features: self.cx.ecfg.features,
modified: false,
config_tokens: false,
},
cx: self.cx,
invocations: Vec::new(),
@ -709,13 +714,26 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
SyntaxExtensionKind::Attr(expander) => {
self.gate_proc_macro_input(&item);
self.gate_proc_macro_attr_item(span, &item);
let tokens = match attr.style {
AttrStyle::Outer => item.into_tokens(&self.cx.sess.parse_sess),
// FIXME: Properly collect tokens for inner attributes
AttrStyle::Inner => rustc_parse::fake_token_stream(
let mut fake_tokens = false;
if let Annotatable::Item(item_inner) = &item {
if let ItemKind::Mod(_, mod_kind) = &item_inner.kind {
// FIXME: Collect tokens and use them instead of generating
// fake ones. These are unstable, so it needs to be
// fixed prior to stabilization
// Fake tokens when we are invoking an inner attribute, and:
fake_tokens = matches!(attr.style, ast::AttrStyle::Inner) &&
// We are invoking an attribute on the crate root, or an outline
// module
(item_inner.ident.name.is_empty() || !matches!(mod_kind, ast::ModKind::Loaded(_, Inline::Yes, _)));
}
}
let tokens = if fake_tokens {
rustc_parse::fake_token_stream(
&self.cx.sess.parse_sess,
&item.into_nonterminal(),
),
)
} else {
item.into_tokens(&self.cx.sess.parse_sess)
};
let attr_item = attr.unwrap_normal_item();
if let MacArgs::Eq(..) = attr_item.args {
@ -897,21 +915,21 @@ pub fn parse_ast_fragment<'a>(
}
AstFragmentKind::TraitItems => {
let mut items = SmallVec::new();
while let Some(item) = this.parse_trait_item()? {
while let Some(item) = this.parse_trait_item(ForceCollect::No)? {
items.extend(item);
}
AstFragment::TraitItems(items)
}
AstFragmentKind::ImplItems => {
let mut items = SmallVec::new();
while let Some(item) = this.parse_impl_item()? {
while let Some(item) = this.parse_impl_item(ForceCollect::No)? {
items.extend(item);
}
AstFragment::ImplItems(items)
}
AstFragmentKind::ForeignItems => {
let mut items = SmallVec::new();
while let Some(item) = this.parse_foreign_item()? {
while let Some(item) = this.parse_foreign_item(ForceCollect::No)? {
items.extend(item);
}
AstFragment::ForeignItems(items)