bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Implement token-based handling of attributes during expansion

Browse source 
This PR modifies the macro expansion infrastructure to handle attributes
in a fully token-based manner. As a result:

* Derives macros no longer lose spans when their input is modified
  by eager cfg-expansion. This is accomplished by performing eager
  cfg-expansion on the token stream that we pass to the derive
  proc-macro
* Inner attributes now preserve spans in all cases, including when we
  have multiple inner attributes in a row.

This is accomplished through the following changes:

* New structs `AttrAnnotatedTokenStream` and `AttrAnnotatedTokenTree` are introduced.
  These are very similar to a normal `TokenTree`, but they also track
  the position of attributes and attribute targets within the stream.
  They are built when we collect tokens during parsing.
  An `AttrAnnotatedTokenStream` is converted to a regular `TokenStream` when
  we invoke a macro.
* Token capturing and `LazyTokenStream` are modified to work with
  `AttrAnnotatedTokenStream`. A new `ReplaceRange` type is introduced, which
  is created during the parsing of a nested AST node to make the 'outer'
  AST node aware of the attributes and attribute target stored deeper in the token stream.
* When we need to perform eager cfg-expansion (either due to `#[derive]` or `#[cfg_eval]`),
we tokenize and reparse our target, capturing additional information about the locations of
`#[cfg]` and `#[cfg_attr]` attributes at any depth within the target.
This is a performance optimization, allowing us to perform less work
in the typical case where captured tokens never have eager cfg-expansion run.
This commit is contained in:
Aaron Hill 2020-11-28 18:33:17 -05:00
parent 25ea6be13e
commit a93c4f05de
No known key found for this signature in database
GPG key ID: B4087E510E98B164
33 changed files with 2046 additions and 1192 deletions
Download patch file Download diff file Expand all files Collapse all files

65
compiler/rustc_parse/src/parser/stmt.rs
View file

@ -48,39 +48,26 @@ impl<'a> Parser<'a> {
if let token::Interpolated(nt) = &self.token.kind {
if let token::NtStmt(stmt) = &**nt {
let mut stmt = stmt.clone();
return self.collect_tokens_trailing_token(
attrs,
force_collect,
|this, mut attrs| {
stmt.visit_attrs(|stmt_attrs| {
mem::swap(stmt_attrs, &mut attrs);
stmt_attrs.extend(attrs);
});
// Make sure we capture the token::Interpolated
this.bump();
Ok((Some(stmt), TrailingToken::None))
},
);
self.bump();
stmt.visit_attrs(|stmt_attrs| {
attrs.prepend_to_nt_inner(stmt_attrs);
});
return Ok(Some(stmt));
}
}
Ok(Some(if self.token.is_keyword(kw::Let) {
self.parse_local_mk(lo, attrs, capture_semi, force_collect)?
} else if self.is_kw_followed_by_ident(kw::Mut) {
self.recover_stmt_local(
lo,
attrs.take_for_recovery().into(),
"missing keyword",
"let mut",
)?
self.recover_stmt_local(lo, attrs, "missing keyword", "let mut")?
} else if self.is_kw_followed_by_ident(kw::Auto) {
self.bump(); // `auto`
let msg = "write `let` instead of `auto` to introduce a new variable";
self.recover_stmt_local(lo, attrs.take_for_recovery().into(), msg, "let")?
self.recover_stmt_local(lo, attrs, msg, "let")?
} else if self.is_kw_followed_by_ident(sym::var) {
self.bump(); // `var`
let msg = "write `let` instead of `var` to introduce a new variable";
self.recover_stmt_local(lo, attrs.take_for_recovery().into(), msg, "let")?
self.recover_stmt_local(lo, attrs, msg, "let")?
} else if self.check_path() && !self.token.is_qpath_start() && !self.is_path_start_item() {
// We have avoided contextual keywords like `union`, items with `crate` visibility,
// or `auto trait` items. We aim to parse an arbitrary path `a::b` but not something
@ -112,7 +99,7 @@ impl<'a> Parser<'a> {
attrs: AttrWrapper,
force_collect: ForceCollect,
) -> PResult<'a, Stmt> {
self.collect_tokens_trailing_token(attrs, force_collect, |this, attrs| {
let stmt = self.collect_tokens_trailing_token(attrs, force_collect, |this, attrs| {
let path = this.parse_path(PathStyle::Expr)?;
if this.eat(&token::Not) {
@ -132,14 +119,22 @@ impl<'a> Parser<'a> {
};
let expr = this.with_res(Restrictions::STMT_EXPR, |this| {
let expr = this.parse_dot_or_call_expr_with(expr, lo, attrs)?;
this.parse_dot_or_call_expr_with(expr, lo, attrs)
})?;
// `DUMMY_SP` will get overwritten later in this function
Ok((this.mk_stmt(rustc_span::DUMMY_SP, StmtKind::Expr(expr)), TrailingToken::None))
})?;
if let StmtKind::Expr(expr) = stmt.kind {
// Perform this outside of the `collect_tokens_trailing_token` closure,
// since our outer attributes do not apply to this part of the expression
let expr = self.with_res(Restrictions::STMT_EXPR, |this| {
this.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(expr))
})?;
Ok((
this.mk_stmt(lo.to(this.prev_token.span), StmtKind::Expr(expr)),
TrailingToken::None,
))
})
Ok(self.mk_stmt(lo.to(self.prev_token.span), StmtKind::Expr(expr)))
} else {
Ok(stmt)
}
}
/// Parses a statement macro `mac!(args)` provided a `path` representing `mac`.
@ -183,7 +178,7 @@ impl<'a> Parser<'a> {
fn recover_stmt_local(
&mut self,
lo: Span,
attrs: AttrVec,
attrs: AttrWrapper,
msg: &str,
sugg: &str,
) -> PResult<'a, Stmt> {
@ -213,9 +208,15 @@ impl<'a> Parser<'a> {
})
}
fn recover_local_after_let(&mut self, lo: Span, attrs: AttrVec) -> PResult<'a, Stmt> {
let local = self.parse_local(attrs)?;
Ok(self.mk_stmt(lo.to(self.prev_token.span), StmtKind::Local(local)))
fn recover_local_after_let(&mut self, lo: Span, attrs: AttrWrapper) -> PResult<'a, Stmt> {
self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| {
let local = this.parse_local(attrs.into())?;
// FIXME - maybe capture semicolon in recovery?
Ok((
this.mk_stmt(lo.to(this.prev_token.span), StmtKind::Local(local)),
TrailingToken::None,
))
})
}
/// Parses a local variable declaration.