bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Force token collection to run when parsing nonterminals

Browse source 
Fixes #81007

Previously, we would fail to collect tokens in the proper place when
only builtin attributes were present. As a result, we would end up with
attribute tokens in the collected `TokenStream`, leading to duplication
when we attempted to prepend the attributes from the AST node.

We now explicitly track when token collection must be performed due to
nomterminal parsing.
This commit is contained in:
Aaron Hill 2021-01-18 16:47:37 -05:00
parent a4cbb44ae2
commit 11b1e37016
No known key found for this signature in database
GPG key ID: B4087E510E98B164
13 changed files with 233 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

6
compiler/rustc_parse/src/parser/nonterminal.rs
View file

@ -5,7 +5,7 @@ use rustc_errors::PResult;
use rustc_span::symbol::{kw, Ident};
use crate::parser::pat::{GateOr, RecoverComma};
use crate::parser::{FollowedByType, Parser, PathStyle};
use crate::parser::{FollowedByType, ForceCollect, Parser, PathStyle};
impl<'a> Parser<'a> {
/// Checks whether a non-terminal may begin with a particular token.
@ -98,7 +98,7 @@ impl<'a> Parser<'a> {
// in advance whether or not a proc-macro will be (transitively) invoked,
// we always capture tokens for any `Nonterminal` which needs them.
Ok(match kind {
NonterminalKind::Item => match self.collect_tokens(|this| this.parse_item())? {
NonterminalKind::Item => match self.parse_item(ForceCollect::Yes)? {
Some(item) => token::NtItem(item),
None => {
return Err(self.struct_span_err(self.token.span, "expected an item keyword"));
@ -107,7 +107,7 @@ impl<'a> Parser<'a> {
NonterminalKind::Block => {
token::NtBlock(self.collect_tokens(|this| this.parse_block())?)
}
NonterminalKind::Stmt => match self.collect_tokens(|this| this.parse_stmt())? {
NonterminalKind::Stmt => match self.parse_stmt(ForceCollect::Yes)? {
Some(s) => token::NtStmt(s),
None => {
return Err(self.struct_span_err(self.token.span, "expected a statement"));