bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Overhaul token collection.

Browse source 
This commit does the following.

- Renames `collect_tokens_trailing_token` as `collect_tokens`, because
  (a) it's annoying long, and (b) the `_trailing_token` bit is less
  accurate now that its types have changed.

- In `collect_tokens`, adds a `Option<CollectPos>` argument and a
  `UsePreAttrPos` in the return type of `f`. These are used in
  `parse_expr_force_collect` (for vanilla expressions) and in
  `parse_stmt_without_recovery` (for two different cases of expression
  statements). Together these ensure are enough to fix all the problems
  with token collection and assoc expressions. The changes to the
  `stringify.rs` test demonstrate some of these.

- Adds a new test. The code in this test was causing an assertion
  failure prior to this commit, due to an invalid `NodeRange`.

The extra complexity is annoying, but necessary to fix the existing
problems.
This commit is contained in:
Nicholas Nethercote 2024-08-06 17:16:40 +10:00
parent fe460ac28b
commit 9d31f86f0d
12 changed files with 414 additions and 292 deletions
Download patch file Download diff file Expand all files Collapse all files

62
compiler/rustc_parse/src/parser/stmt.rs
View file

@ -21,7 +21,7 @@ use super::pat::{PatternLocation, RecoverComma};
use super::path::PathStyle;
use super::{
AttrWrapper, BlockMode, FnParseMode, ForceCollect, Parser, Restrictions, SemiColonMode,
Trailing,
Trailing, UsePreAttrPos,
};
use crate::errors::MalformedLoopLabel;
use crate::{errors, maybe_whole};
@ -46,6 +46,7 @@ impl<'a> Parser<'a> {
capture_semi: bool,
force_collect: ForceCollect,
) -> PResult<'a, Option<Stmt>> {
let pre_attr_pos = self.collect_pos();
let attrs = self.parse_outer_attributes()?;
let lo = self.token.span;
@ -66,11 +67,15 @@ impl<'a> Parser<'a> {
}
Ok(Some(if self.token.is_keyword(kw::Let) {
self.collect_tokens_trailing_token(attrs, force_collect, |this, attrs| {
self.collect_tokens(None, attrs, force_collect, |this, attrs| {
this.expect_keyword(kw::Let)?;
let local = this.parse_local(attrs)?;
let trailing = Trailing::from(capture_semi && this.token == token::Semi);
Ok((this.mk_stmt(lo.to(this.prev_token.span), StmtKind::Let(local)), trailing))
Ok((
this.mk_stmt(lo.to(this.prev_token.span), StmtKind::Let(local)),
trailing,
UsePreAttrPos::No,
))
})?
} else if self.is_kw_followed_by_ident(kw::Mut) && self.may_recover() {
self.recover_stmt_local_after_let(
@ -104,10 +109,18 @@ impl<'a> Parser<'a> {
// or `auto trait` items. We aim to parse an arbitrary path `a::b` but not something
// that starts like a path (1 token), but it fact not a path.
// Also, we avoid stealing syntax from `parse_item_`.
let stmt = self.collect_tokens_trailing_token(
//
// `UsePreAttrPos::Yes` here means the attribute belongs unconditionally to the
// expression, not the statement. (But the statement attributes/tokens are obtained
// from the expression anyway, because `Stmt` delegates `HasAttrs`/`HasTokens` to
// the things within `StmtKind`.)
let stmt = self.collect_tokens(
Some(pre_attr_pos),
AttrWrapper::empty(),
force_collect,
|this, _empty_attrs| Ok((this.parse_stmt_path_start(lo, attrs)?, Trailing::No)),
|this, _empty_attrs| {
Ok((this.parse_stmt_path_start(lo, attrs)?, Trailing::No, UsePreAttrPos::Yes))
},
);
match stmt {
Ok(stmt) => stmt,
@ -129,12 +142,15 @@ impl<'a> Parser<'a> {
self.error_outer_attrs(attrs);
self.mk_stmt(lo, StmtKind::Empty)
} else if self.token != token::CloseDelim(Delimiter::Brace) {
// Remainder are line-expr stmts.
let e = self.collect_tokens_trailing_token(
// Remainder are line-expr stmts. This is similar to the `parse_stmt_path_start` case
// above.
let e = self.collect_tokens(
Some(pre_attr_pos),
AttrWrapper::empty(),
force_collect,
|this, _empty_attrs| {
Ok((this.parse_expr_res(Restrictions::STMT_EXPR, attrs)?, Trailing::No))
let (expr, _) = this.parse_expr_res(Restrictions::STMT_EXPR, attrs)?;
Ok((expr, Trailing::No, UsePreAttrPos::Yes))
},
)?;
if matches!(e.kind, ExprKind::Assign(..)) && self.eat_keyword(kw::Else) {
@ -151,12 +167,16 @@ impl<'a> Parser<'a> {
}
fn parse_stmt_path_start(&mut self, lo: Span, attrs: AttrWrapper) -> PResult<'a, Stmt> {
let stmt = self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| {
let stmt = self.collect_tokens(None, attrs, ForceCollect::No, |this, attrs| {
let path = this.parse_path(PathStyle::Expr)?;
if this.eat(&token::Not) {
let stmt_mac = this.parse_stmt_mac(lo, attrs, path)?;
return Ok((stmt_mac, Trailing::from(this.token == token::Semi)));
return Ok((
stmt_mac,
Trailing::from(this.token == token::Semi),
UsePreAttrPos::No,
));
}
let expr = if this.eat(&token::OpenDelim(Delimiter::Brace)) {
@ -170,13 +190,17 @@ impl<'a> Parser<'a> {
this.parse_expr_dot_or_call_with(attrs, expr, lo)
})?;
// `DUMMY_SP` will get overwritten later in this function
Ok((this.mk_stmt(rustc_span::DUMMY_SP, StmtKind::Expr(expr)), Trailing::No))
Ok((
this.mk_stmt(rustc_span::DUMMY_SP, StmtKind::Expr(expr)),
Trailing::No,
UsePreAttrPos::No,
))
})?;
if let StmtKind::Expr(expr) = stmt.kind {
// Perform this outside of the `collect_tokens_trailing_token` closure,
// since our outer attributes do not apply to this part of the expression
let expr = self.with_res(Restrictions::STMT_EXPR, |this| {
// Perform this outside of the `collect_tokens` closure, since our
// outer attributes do not apply to this part of the expression.
let (expr, _) = self.with_res(Restrictions::STMT_EXPR, |this| {
this.parse_expr_assoc_rest_with(0, true, expr)
})?;
Ok(self.mk_stmt(lo.to(self.prev_token.span), StmtKind::Expr(expr)))
@ -210,7 +234,7 @@ impl<'a> Parser<'a> {
let e = self.mk_expr(lo.to(hi), ExprKind::MacCall(mac));
let e = self.maybe_recover_from_bad_qpath(e)?;
let e = self.parse_expr_dot_or_call_with(attrs, e, lo)?;
let e = self.parse_expr_assoc_rest_with(0, false, e)?;
let (e, _) = self.parse_expr_assoc_rest_with(0, false, e)?;
StmtKind::Expr(e)
};
Ok(self.mk_stmt(lo.to(hi), kind))
@ -240,10 +264,14 @@ impl<'a> Parser<'a> {
subdiagnostic: fn(Span) -> errors::InvalidVariableDeclarationSub,
force_collect: ForceCollect,
) -> PResult<'a, Stmt> {
let stmt = self.collect_tokens_trailing_token(attrs, force_collect, |this, attrs| {
let stmt = self.collect_tokens(None, attrs, force_collect, |this, attrs| {
let local = this.parse_local(attrs)?;
// FIXME - maybe capture semicolon in recovery?
Ok((this.mk_stmt(lo.to(this.prev_token.span), StmtKind::Let(local)), Trailing::No))
Ok((
this.mk_stmt(lo.to(this.prev_token.span), StmtKind::Let(local)),
Trailing::No,
UsePreAttrPos::No,
))
})?;
self.dcx()
.emit_err(errors::InvalidVariableDeclaration { span: lo, sub: subdiagnostic(lo) });