bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Properly handle attributes on statements

Browse source 
We now collect tokens for the underlying node wrapped by `StmtKind`
instead of storing tokens directly in `Stmt`.

`LazyTokenStream` now supports capturing a trailing semicolon after it
is initially constructed. This allows us to avoid refactoring statement
parsing to wrap the parsing of the semicolon in `parse_tokens`.

Attributes on item statements
(e.g. `fn foo() { #[bar] struct MyStruct; }`) are now treated as
item attributes, not statement attributes, which is consistent with how
we handle attributes on other kinds of statements. The feature-gating
code is adjusted so that proc-macro attributes are still allowed on item
statements on stable.

Two built-in macros (`#[global_allocator]` and `#[test]`) needed to be
adjusted to support being passed `Annotatable::Stmt`.
This commit is contained in:
Aaron Hill 2020-11-17 14:27:44 -05:00
parent 72da5a9d85
commit de88bf148b
No known key found for this signature in database
GPG key ID: B4087E510E98B164
20 changed files with 485 additions and 187 deletions
Download patch file Download diff file Expand all files Collapse all files

23
compiler/rustc_parse/src/lib.rs
View file

@ -6,6 +6,7 @@
#![feature(or_patterns)]
use rustc_ast as ast;
use rustc_ast::attr::HasAttrs;
use rustc_ast::token::{self, DelimToken, Nonterminal, Token, TokenKind};
use rustc_ast::tokenstream::{self, LazyTokenStream, TokenStream, TokenTree};
use rustc_ast_pretty::pprust;
@ -249,29 +250,23 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
// before we fall back to the stringification.
let convert_tokens =
|tokens: &Option<LazyTokenStream>| tokens.as_ref().map(|t| t.create_token_stream());
|tokens: Option<&LazyTokenStream>| tokens.as_ref().map(|t| t.create_token_stream());
let tokens = match *nt {
Nonterminal::NtItem(ref item) => prepend_attrs(&item.attrs, item.tokens.as_ref()),
Nonterminal::NtBlock(ref block) => convert_tokens(&block.tokens),
Nonterminal::NtStmt(ref stmt) => {
// FIXME: We currently only collect tokens for `:stmt`
// matchers in `macro_rules!` macros. When we start collecting
// tokens for attributes on statements, we will need to prepend
// attributes here
convert_tokens(&stmt.tokens)
}
Nonterminal::NtPat(ref pat) => convert_tokens(&pat.tokens),
Nonterminal::NtTy(ref ty) => convert_tokens(&ty.tokens),
Nonterminal::NtBlock(ref block) => convert_tokens(block.tokens.as_ref()),
Nonterminal::NtStmt(ref stmt) => prepend_attrs(stmt.attrs(), stmt.tokens()),
Nonterminal::NtPat(ref pat) => convert_tokens(pat.tokens.as_ref()),
Nonterminal::NtTy(ref ty) => convert_tokens(ty.tokens.as_ref()),
Nonterminal::NtIdent(ident, is_raw) => {
Some(tokenstream::TokenTree::token(token::Ident(ident.name, is_raw), ident.span).into())
}
Nonterminal::NtLifetime(ident) => {
Some(tokenstream::TokenTree::token(token::Lifetime(ident.name), ident.span).into())
}
Nonterminal::NtMeta(ref attr) => convert_tokens(&attr.tokens),
Nonterminal::NtPath(ref path) => convert_tokens(&path.tokens),
Nonterminal::NtVis(ref vis) => convert_tokens(&vis.tokens),
Nonterminal::NtMeta(ref attr) => convert_tokens(attr.tokens.as_ref()),
Nonterminal::NtPath(ref path) => convert_tokens(path.tokens.as_ref()),
Nonterminal::NtVis(ref vis) => convert_tokens(vis.tokens.as_ref()),
Nonterminal::NtTT(ref tt) => Some(tt.clone().into()),
Nonterminal::NtExpr(ref expr) | Nonterminal::NtLiteral(ref expr) => {
if expr.tokens.is_none() {