bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Overhaul token collection.

Browse source 
This commit does the following.

- Renames `collect_tokens_trailing_token` as `collect_tokens`, because
  (a) it's annoying long, and (b) the `_trailing_token` bit is less
  accurate now that its types have changed.

- In `collect_tokens`, adds a `Option<CollectPos>` argument and a
  `UsePreAttrPos` in the return type of `f`. These are used in
  `parse_expr_force_collect` (for vanilla expressions) and in
  `parse_stmt_without_recovery` (for two different cases of expression
  statements). Together these ensure are enough to fix all the problems
  with token collection and assoc expressions. The changes to the
  `stringify.rs` test demonstrate some of these.

- Adds a new test. The code in this test was causing an assertion
  failure prior to this commit, due to an invalid `NodeRange`.

The extra complexity is annoying, but necessary to fix the existing
problems.
This commit is contained in:
Nicholas Nethercote 2024-08-06 17:16:40 +10:00
parent fe460ac28b
commit 9d31f86f0d
12 changed files with 414 additions and 292 deletions
Download patch file Download diff file Expand all files Collapse all files

146
compiler/rustc_parse/src/parser/item.rs
View file

@ -20,7 +20,9 @@ use tracing::debug;
use super::diagnostics::{dummy_arg, ConsumeClosingDelim};
use super::ty::{AllowPlus, RecoverQPath, RecoverReturnSign};
use super::{AttrWrapper, FollowedByType, ForceCollect, Parser, PathStyle, Trailing};
use super::{
AttrWrapper, FollowedByType, ForceCollect, Parser, PathStyle, Trailing, UsePreAttrPos,
};
use crate::errors::{self, MacroExpandsToAdtField};
use crate::{fluent_generated as fluent, maybe_whole};
@ -127,7 +129,7 @@ impl<'a> Parser<'a> {
Some(item.into_inner())
});
self.collect_tokens_trailing_token(attrs, force_collect, |this, mut attrs| {
self.collect_tokens(None, attrs, force_collect, |this, mut attrs| {
let lo = this.token.span;
let vis = this.parse_visibility(FollowedByType::No)?;
let mut def = this.parse_defaultness();
@ -145,7 +147,7 @@ impl<'a> Parser<'a> {
let span = lo.to(this.prev_token.span);
let id = DUMMY_NODE_ID;
let item = Item { ident, attrs, id, kind, vis, span, tokens: None };
return Ok((Some(item), Trailing::No));
return Ok((Some(item), Trailing::No, UsePreAttrPos::No));
}
// At this point, we have failed to parse an item.
@ -160,7 +162,7 @@ impl<'a> Parser<'a> {
if !attrs_allowed {
this.recover_attrs_no_item(&attrs)?;
}
Ok((None, Trailing::No))
Ok((None, Trailing::No, UsePreAttrPos::No))
})
}
@ -1546,86 +1548,82 @@ impl<'a> Parser<'a> {
self.recover_vcs_conflict_marker();
let help = "enum variants can be `Variant`, `Variant = <integer>`, \
`Variant(Type, ..., TypeN)` or `Variant { fields: Types }`";
self.collect_tokens_trailing_token(
variant_attrs,
ForceCollect::No,
|this, variant_attrs| {
let vlo = this.token.span;
self.collect_tokens(None, variant_attrs, ForceCollect::No, |this, variant_attrs| {
let vlo = this.token.span;
let vis = this.parse_visibility(FollowedByType::No)?;
if !this.recover_nested_adt_item(kw::Enum)? {
return Ok((None, Trailing::No));
}
let ident = this.parse_field_ident("enum", vlo)?;
let vis = this.parse_visibility(FollowedByType::No)?;
if !this.recover_nested_adt_item(kw::Enum)? {
return Ok((None, Trailing::No, UsePreAttrPos::No));
}
let ident = this.parse_field_ident("enum", vlo)?;
if this.token == token::Not {
if let Err(err) = this.unexpected() {
err.with_note(fluent::parse_macro_expands_to_enum_variant).emit();
}
this.bump();
this.parse_delim_args()?;
return Ok((None, Trailing::from(this.token == token::Comma)));
if this.token == token::Not {
if let Err(err) = this.unexpected() {
err.with_note(fluent::parse_macro_expands_to_enum_variant).emit();
}
let struct_def = if this.check(&token::OpenDelim(Delimiter::Brace)) {
// Parse a struct variant.
let (fields, recovered) =
match this.parse_record_struct_body("struct", ident.span, false) {
Ok((fields, recovered)) => (fields, recovered),
Err(mut err) => {
if this.token == token::Colon {
// We handle `enum` to `struct` suggestion in the caller.
return Err(err);
}
this.eat_to_tokens(&[&token::CloseDelim(Delimiter::Brace)]);
this.bump(); // }
err.span_label(span, "while parsing this enum");
err.help(help);
let guar = err.emit();
(thin_vec![], Recovered::Yes(guar))
}
};
VariantData::Struct { fields, recovered: recovered.into() }
} else if this.check(&token::OpenDelim(Delimiter::Parenthesis)) {
let body = match this.parse_tuple_struct_body() {
Ok(body) => body,
this.bump();
this.parse_delim_args()?;
return Ok((None, Trailing::from(this.token == token::Comma), UsePreAttrPos::No));
}
let struct_def = if this.check(&token::OpenDelim(Delimiter::Brace)) {
// Parse a struct variant.
let (fields, recovered) =
match this.parse_record_struct_body("struct", ident.span, false) {
Ok((fields, recovered)) => (fields, recovered),
Err(mut err) => {
if this.token == token::Colon {
// We handle `enum` to `struct` suggestion in the caller.
return Err(err);
}
this.eat_to_tokens(&[&token::CloseDelim(Delimiter::Parenthesis)]);
this.bump(); // )
this.eat_to_tokens(&[&token::CloseDelim(Delimiter::Brace)]);
this.bump(); // }
err.span_label(span, "while parsing this enum");
err.help(help);
err.emit();
thin_vec![]
let guar = err.emit();
(thin_vec![], Recovered::Yes(guar))
}
};
VariantData::Tuple(body, DUMMY_NODE_ID)
} else {
VariantData::Unit(DUMMY_NODE_ID)
VariantData::Struct { fields, recovered: recovered.into() }
} else if this.check(&token::OpenDelim(Delimiter::Parenthesis)) {
let body = match this.parse_tuple_struct_body() {
Ok(body) => body,
Err(mut err) => {
if this.token == token::Colon {
// We handle `enum` to `struct` suggestion in the caller.
return Err(err);
}
this.eat_to_tokens(&[&token::CloseDelim(Delimiter::Parenthesis)]);
this.bump(); // )
err.span_label(span, "while parsing this enum");
err.help(help);
err.emit();
thin_vec![]
}
};
VariantData::Tuple(body, DUMMY_NODE_ID)
} else {
VariantData::Unit(DUMMY_NODE_ID)
};
let disr_expr =
if this.eat(&token::Eq) { Some(this.parse_expr_anon_const()?) } else { None };
let disr_expr =
if this.eat(&token::Eq) { Some(this.parse_expr_anon_const()?) } else { None };
let vr = ast::Variant {
ident,
vis,
id: DUMMY_NODE_ID,
attrs: variant_attrs,
data: struct_def,
disr_expr,
span: vlo.to(this.prev_token.span),
is_placeholder: false,
};
let vr = ast::Variant {
ident,
vis,
id: DUMMY_NODE_ID,
attrs: variant_attrs,
data: struct_def,
disr_expr,
span: vlo.to(this.prev_token.span),
is_placeholder: false,
};
Ok((Some(vr), Trailing::from(this.token == token::Comma)))
},
)
Ok((Some(vr), Trailing::from(this.token == token::Comma), UsePreAttrPos::No))
})
.map_err(|mut err| {
err.help(help);
err
@ -1777,7 +1775,7 @@ impl<'a> Parser<'a> {
// Unit like structs are handled in parse_item_struct function
self.parse_paren_comma_seq(|p| {
let attrs = p.parse_outer_attributes()?;
p.collect_tokens_trailing_token(attrs, ForceCollect::No, |p, attrs| {
p.collect_tokens(None, attrs, ForceCollect::No, |p, attrs| {
let mut snapshot = None;
if p.is_vcs_conflict_marker(&TokenKind::BinOp(token::Shl), &TokenKind::Lt) {
// Account for `<<<<<<<` diff markers. We can't proactively error here because
@ -1816,6 +1814,7 @@ impl<'a> Parser<'a> {
is_placeholder: false,
},
Trailing::from(p.token == token::Comma),
UsePreAttrPos::No,
))
})
})
@ -1827,11 +1826,11 @@ impl<'a> Parser<'a> {
self.recover_vcs_conflict_marker();
let attrs = self.parse_outer_attributes()?;
self.recover_vcs_conflict_marker();
self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| {
self.collect_tokens(None, attrs, ForceCollect::No, |this, attrs| {
let lo = this.token.span;
let vis = this.parse_visibility(FollowedByType::No)?;
this.parse_single_struct_field(adt_ty, lo, vis, attrs)
.map(|field| (field, Trailing::No))
.map(|field| (field, Trailing::No, UsePreAttrPos::No))
})
}
@ -2806,12 +2805,12 @@ impl<'a> Parser<'a> {
fn parse_param_general(&mut self, req_name: ReqName, first_param: bool) -> PResult<'a, Param> {
let lo = self.token.span;
let attrs = self.parse_outer_attributes()?;
self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| {
self.collect_tokens(None, attrs, ForceCollect::No, |this, attrs| {
// Possibly parse `self`. Recover if we parsed it and it wasn't allowed here.
if let Some(mut param) = this.parse_self_param()? {
param.attrs = attrs;
let res = if first_param { Ok(param) } else { this.recover_bad_self_param(param) };
return Ok((res?, Trailing::No));
return Ok((res?, Trailing::No, UsePreAttrPos::No));
}
let is_name_required = match this.token.kind {
@ -2827,7 +2826,7 @@ impl<'a> Parser<'a> {
this.parameter_without_type(&mut err, pat, is_name_required, first_param)
{
let guar = err.emit();
Ok((dummy_arg(ident, guar), Trailing::No))
Ok((dummy_arg(ident, guar), Trailing::No, UsePreAttrPos::No))
} else {
Err(err)
};
@ -2871,6 +2870,7 @@ impl<'a> Parser<'a> {
Ok((
Param { attrs, id: ast::DUMMY_NODE_ID, is_placeholder: false, pat, span, ty },
Trailing::No,
UsePreAttrPos::No,
))
})
}