bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Require passing an AttrWrapper to collect_tokens_trailing_token

Browse source 
This is a pure refactoring split out from #80689.
It represents the most invasive part of that PR, requiring changes in
every caller of `parse_outer_attributes`

In order to eagerly expand `#[cfg]` attributes while preserving the
original `TokenStream`, we need to know the range of tokens that
corresponds to every attribute target. This is accomplished by making
`parse_outer_attributes` return an opaque `AttrWrapper` struct. An
`AttrWrapper` must be converted to a plain `AttrVec` by passing it to
`collect_tokens_trailing_token`. This makes it difficult to accidentally
construct an AST node with attributes without calling `collect_tokens_trailing_token`,
since AST nodes store an `AttrVec`, not an `AttrWrapper`.

As a result, we now call `collect_tokens_trailing_token` for attribute
targets which only support inert attributes, such as generic arguments
and struct fields. Currently, the constructed `LazyTokenStream` is
simply discarded. Future PRs will record the token range corresponding
to the attribute target, allowing those tokens to be removed from an
enclosing `collect_tokens_trailing_token` call if necessary.
This commit is contained in:
Aaron Hill 2021-01-22 13:28:08 -05:00
parent 7e0241c637
commit 0b411f56e1
No known key found for this signature in database
GPG key ID: B4087E510E98B164
9 changed files with 621 additions and 407 deletions
Download patch file Download diff file Expand all files Collapse all files

46
compiler/rustc_parse/src/parser/nonterminal.rs
View file

@ -108,7 +108,9 @@ impl<'a> Parser<'a> {
}
},
NonterminalKind::Block => {
token::NtBlock(self.collect_tokens(|this| this.parse_block())?)
// While an block *expression* may have attributes (e.g. `#[my_attr] { ... }`),
// the ':block' matcher does not support them
token::NtBlock(self.collect_tokens_no_attrs(|this| this.parse_block())?)
}
NonterminalKind::Stmt => match self.parse_stmt(ForceCollect::Yes)? {
Some(s) => token::NtStmt(s),
@ -117,7 +119,7 @@ impl<'a> Parser<'a> {
}
},
NonterminalKind::Pat2018 { .. } | NonterminalKind::Pat2021 { .. } => {
token::NtPat(self.collect_tokens(|this| match kind {
token::NtPat(self.collect_tokens_no_attrs(|this| match kind {
NonterminalKind::Pat2018 { .. } => this.parse_pat(None),
NonterminalKind::Pat2021 { .. } => {
this.parse_top_pat(GateOr::Yes, RecoverComma::No)
@ -125,11 +127,33 @@ impl<'a> Parser<'a> {
_ => unreachable!(),
})?)
}
NonterminalKind::Expr => token::NtExpr(self.collect_tokens(|this| this.parse_expr())?),
NonterminalKind::Literal => {
token::NtLiteral(self.collect_tokens(|this| this.parse_literal_maybe_minus())?)
// If there are attributes present, then `parse_expr` will end up collecting tokens,
// turning the outer `collect_tokens_no_attrs` into a no-op due to the already present
// tokens. If there are *not* attributes present, then the outer
// `collect_tokens_no_attrs` will ensure that we will end up collecting tokens for the
// expressions.
//
// This is less efficient than it could be, since the outer `collect_tokens_no_attrs`
// still needs to snapshot the `TokenCursor` before calling `parse_expr`, even when
// `parse_expr` will end up collecting tokens. Ideally, this would work more like
// `parse_item`, and take in a `ForceCollect` parameter. However, this would require
// adding a `ForceCollect` parameter in a bunch of places in expression parsing
// for little gain. If the perf impact from this turns out to be noticeable, we should
// revisit this apporach.
NonterminalKind::Expr => {
token::NtExpr(self.collect_tokens_no_attrs(|this| this.parse_expr())?)
}
NonterminalKind::Literal => {
// The `:literal` matcher does not support attributes
token::NtLiteral(
self.collect_tokens_no_attrs(|this| this.parse_literal_maybe_minus())?,
)
}
NonterminalKind::Ty => {
token::NtTy(self.collect_tokens_no_attrs(|this| this.parse_ty())?)
}
NonterminalKind::Ty => token::NtTy(self.collect_tokens(|this| this.parse_ty())?),
// this could be handled like a token, since it is one
NonterminalKind::Ident => {
if let Some((ident, is_raw)) = get_macro_ident(&self.token) {
@ -141,15 +165,15 @@ impl<'a> Parser<'a> {
return Err(self.struct_span_err(self.token.span, msg));
}
}
NonterminalKind::Path => {
token::NtPath(self.collect_tokens(|this| this.parse_path(PathStyle::Type))?)
}
NonterminalKind::Path => token::NtPath(
self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))?,
),
NonterminalKind::Meta => {
token::NtMeta(P(self.collect_tokens(|this| this.parse_attr_item(false))?))
token::NtMeta(P(self.collect_tokens_no_attrs(|this| this.parse_attr_item(false))?))
}
NonterminalKind::TT => token::NtTT(self.parse_token_tree()),
NonterminalKind::Vis => token::NtVis(
self.collect_tokens(|this| this.parse_visibility(FollowedByType::Yes))?,
self.collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?,
),
NonterminalKind::Lifetime => {
if self.check_lifetime() {