bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Auto merge of #138478 - nnethercote:rm-NtExpr-NtLiteral, r=petrochenkov

Browse source 
Remove `NtExpr` and `NtLiteral`

The next part of #124141.

r? `@petrochenkov`
This commit is contained in:
bors 2025-04-02 00:36:04 +00:00
commit 70dab5a27c
33 changed files with 1095 additions and 709 deletions
Download patch file Download diff file Expand all files Collapse all files

2
compiler/rustc_ast/src/ast_traits.rs
View file

@ -209,13 +209,11 @@ impl HasTokens for Attribute {
impl HasTokens for Nonterminal {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
match self {
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens(),
Nonterminal::NtBlock(block) => block.tokens(),
}
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
match self {
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
Nonterminal::NtBlock(block) => block.tokens_mut(),
}
}

2
compiler/rustc_ast/src/mut_visit.rs
View file

@ -899,8 +899,6 @@ pub fn visit_token<T: MutVisitor>(vis: &mut T, t: &mut Token) {
fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
match nt {
token::NtBlock(block) => vis.visit_block(block),
token::NtExpr(expr) => vis.visit_expr(expr),
token::NtLiteral(expr) => vis.visit_expr(expr),
}
}

92
compiler/rustc_ast/src/token.rs
View file

@ -198,16 +198,17 @@ impl Lit {
}
}
/// Keep this in sync with `Token::can_begin_literal_maybe_minus` excluding unary negation.
/// Keep this in sync with `Token::can_begin_literal_maybe_minus` and
/// `Parser::eat_token_lit` (excluding unary negation).
pub fn from_token(token: &Token) -> Option<Lit> {
match token.uninterpolate().kind {
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)),
Literal(token_lit) => Some(token_lit),
Interpolated(ref nt)
if let NtExpr(expr) | NtLiteral(expr) = &**nt
&& let ast::ExprKind::Lit(token_lit) = expr.kind =>
{
Some(token_lit)
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
MetaVarKind::Literal | MetaVarKind::Expr { .. },
))) => {
// Unreachable with the current test suite.
panic!("from_token metavar");
}
_ => None,
}
@ -447,8 +448,9 @@ pub enum TokenKind {
/// Identifier token.
/// Do not forget about `NtIdent` when you want to match on identifiers.
/// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
/// treat regular and interpolated identifiers in the same way.
/// It's recommended to use `Token::{ident,uninterpolate}` and
/// `Parser::token_uninterpolated_span` to treat regular and interpolated
/// identifiers in the same way.
Ident(Symbol, IdentIsRaw),
/// This identifier (and its span) is the identifier passed to the
/// declarative macro. The span in the surrounding `Token` is the span of
@ -457,8 +459,9 @@ pub enum TokenKind {
/// Lifetime identifier token.
/// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
/// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
/// treat regular and interpolated lifetime identifiers in the same way.
/// It's recommended to use `Token::{ident,uninterpolate}` and
/// `Parser::token_uninterpolated_span` to treat regular and interpolated
/// identifiers in the same way.
Lifetime(Symbol, IdentIsRaw),
/// This identifier (and its span) is the lifetime passed to the
/// declarative macro. The span in the surrounding `Token` is the span of
@ -584,20 +587,6 @@ impl Token {
Token::new(Ident(ident.name, ident.is_raw_guess().into()), ident.span)
}
/// For interpolated tokens, returns a span of the fragment to which the interpolated
/// token refers. For all other tokens this is just a regular span.
/// It is particularly important to use this for identifiers and lifetimes
/// for which spans affect name resolution and edition checks.
/// Note that keywords are also identifiers, so they should use this
/// if they keep spans or perform edition checks.
pub fn uninterpolated_span(&self) -> Span {
match self.kind {
NtIdent(ident, _) | NtLifetime(ident, _) => ident.span,
Interpolated(ref nt) => nt.use_span(),
_ => self.span,
}
}
pub fn is_range_separator(&self) -> bool {
[DotDot, DotDotDot, DotDotEq].contains(&self.kind)
}
@ -642,12 +631,7 @@ impl Token {
PathSep | // global path
Lifetime(..) | // labeled loop
Pound => true, // expression attributes
Interpolated(ref nt) =>
matches!(&**nt,
NtBlock(..) |
NtExpr(..) |
NtLiteral(..)
),
Interpolated(ref nt) => matches!(&**nt, NtBlock(..)),
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
MetaVarKind::Block |
MetaVarKind::Expr { .. } |
@ -677,11 +661,6 @@ impl Token {
Lt | // path (UFCS constant)
Shl => true, // path (double UFCS)
Or => matches!(pat_kind, PatWithOr), // leading vert `|` or-pattern
Interpolated(nt) =>
matches!(&**nt,
| NtExpr(..)
| NtLiteral(..)
),
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
MetaVarKind::Expr { .. } |
MetaVarKind::Literal |
@ -724,7 +703,7 @@ impl Token {
match self.kind {
OpenDelim(Delimiter::Brace) | Literal(..) | Minus => true,
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
Interpolated(ref nt) => matches!(&**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
Interpolated(ref nt) => matches!(&**nt, NtBlock(..)),
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
MetaVarKind::Expr { .. } | MetaVarKind::Block | MetaVarKind::Literal,
))) => true,
@ -768,22 +747,12 @@ impl Token {
///
/// In other words, would this token be a valid start of `parse_literal_maybe_minus`?
///
/// Keep this in sync with and `Lit::from_token`, excluding unary negation.
/// Keep this in sync with `Lit::from_token` and `Parser::eat_token_lit`
/// (excluding unary negation).
pub fn can_begin_literal_maybe_minus(&self) -> bool {
match self.uninterpolate().kind {
Literal(..) | Minus => true,
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
Interpolated(ref nt) => match &**nt {
NtLiteral(_) => true,
NtExpr(e) => match &e.kind {
ast::ExprKind::Lit(_) => true,
ast::ExprKind::Unary(ast::UnOp::Neg, e) => {
matches!(&e.kind, ast::ExprKind::Lit(_))
}
_ => false,
},
_ => false,
},
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
MetaVarKind::Literal => true,
MetaVarKind::Expr { can_begin_literal_maybe_minus, .. } => {
@ -798,14 +767,6 @@ impl Token {
pub fn can_begin_string_literal(&self) -> bool {
match self.uninterpolate().kind {
Literal(..) => true,
Interpolated(ref nt) => match &**nt {
NtLiteral(_) => true,
NtExpr(e) => match &e.kind {
ast::ExprKind::Lit(_) => true,
_ => false,
},
_ => false,
},
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
MetaVarKind::Literal => true,
MetaVarKind::Expr { can_begin_string_literal, .. } => can_begin_string_literal,
@ -869,12 +830,17 @@ impl Token {
/// Is this a pre-parsed expression dropped into the token stream
/// (which happens while parsing the result of macro expansion)?
pub fn is_whole_expr(&self) -> bool {
pub fn is_metavar_expr(&self) -> bool {
#[allow(irrefutable_let_patterns)] // FIXME: temporary
if let Interpolated(nt) = &self.kind
&& let NtExpr(_) | NtLiteral(_) | NtBlock(_) = &**nt
&& let NtBlock(_) = &**nt
{
true
} else if matches!(
self.is_metavar_seq(),
Some(MetaVarKind::Expr { .. } | MetaVarKind::Literal | MetaVarKind::Path)
) {
true
} else {
matches!(self.is_metavar_seq(), Some(MetaVarKind::Path))
}
@ -882,6 +848,7 @@ impl Token {
/// Is the token an interpolated block (`$b:block`)?
pub fn is_whole_block(&self) -> bool {
#[allow(irrefutable_let_patterns)] // FIXME: temporary
if let Interpolated(nt) = &self.kind
&& let NtBlock(..) = &**nt
{
@ -1100,8 +1067,6 @@ pub enum NtExprKind {
/// For interpolation during macro expansion.
pub enum Nonterminal {
NtBlock(P<ast::Block>),
NtExpr(P<ast::Expr>),
NtLiteral(P<ast::Expr>),
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
@ -1191,15 +1156,12 @@ impl Nonterminal {
pub fn use_span(&self) -> Span {
match self {
NtBlock(block) => block.span,
NtExpr(expr) | NtLiteral(expr) => expr.span,
}
}
pub fn descr(&self) -> &'static str {
match self {
NtBlock(..) => "block",
NtExpr(..) => "expression",
NtLiteral(..) => "literal",
}
}
}
@ -1218,8 +1180,6 @@ impl fmt::Debug for Nonterminal {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match *self {
NtBlock(..) => f.pad("NtBlock(..)"),
NtExpr(..) => f.pad("NtExpr(..)"),
NtLiteral(..) => f.pad("NtLiteral(..)"),
}
}
}
@ -1242,7 +1202,7 @@ mod size_asserts {
// tidy-alphabetical-start
static_assert_size!(Lit, 12);
static_assert_size!(LitKind, 2);
static_assert_size!(Nonterminal, 16);
static_assert_size!(Nonterminal, 8);
static_assert_size!(Token, 24);
static_assert_size!(TokenKind, 16);
// tidy-alphabetical-end

62
compiler/rustc_ast/src/tokenstream.rs
View file

@ -233,35 +233,52 @@ fn attrs_and_tokens_to_token_trees(
// Insert inner attribute tokens.
if !inner_attrs.is_empty() {
let mut found = false;
// Check the last two trees (to account for a trailing semi)
for tree in res.iter_mut().rev().take(2) {
if let TokenTree::Delimited(span, spacing, delim, delim_tokens) = tree {
// Inner attributes are only supported on extern blocks, functions,
// impls, and modules. All of these have their inner attributes
// placed at the beginning of the rightmost outermost braced group:
// e.g. fn foo() { #![my_attr] }
//
// Therefore, we can insert them back into the right location
// without needing to do any extra position tracking.
//
// Note: Outline modules are an exception - they can
// have attributes like `#![my_attr]` at the start of a file.
// Support for custom attributes in this position is not
// properly implemented - we always synthesize fake tokens,
// so we never reach this code.
let found = insert_inner_attrs(inner_attrs, res);
assert!(found, "Failed to find trailing delimited group in: {res:?}");
}
// Inner attributes are only supported on blocks, functions, impls, and
// modules. All of these have their inner attributes placed at the
// beginning of the rightmost outermost braced group:
// e.g. `fn foo() { #![my_attr] }`. (Note: the braces may be within
// invisible delimiters.)
//
// Therefore, we can insert them back into the right location without
// needing to do any extra position tracking.
//
// Note: Outline modules are an exception - they can have attributes like
// `#![my_attr]` at the start of a file. Support for custom attributes in
// this position is not properly implemented - we always synthesize fake
// tokens, so we never reach this code.
fn insert_inner_attrs(inner_attrs: &[Attribute], tts: &mut Vec<TokenTree>) -> bool {
for tree in tts.iter_mut().rev() {
if let TokenTree::Delimited(span, spacing, Delimiter::Brace, stream) = tree {
// Found it: the rightmost, outermost braced group.
let mut tts = vec![];
for inner_attr in inner_attrs {
tts.extend(inner_attr.token_trees());
}
tts.extend(delim_tokens.0.iter().cloned());
tts.extend(stream.0.iter().cloned());
let stream = TokenStream::new(tts);
*tree = TokenTree::Delimited(*span, *spacing, *delim, stream);
found = true;
break;
*tree = TokenTree::Delimited(*span, *spacing, Delimiter::Brace, stream);
return true;
} else if let TokenTree::Delimited(span, spacing, Delimiter::Invisible(src), stream) =
tree
{
// Recurse inside invisible delimiters.
let mut vec: Vec<_> = stream.iter().cloned().collect();
if insert_inner_attrs(inner_attrs, &mut vec) {
*tree = TokenTree::Delimited(
*span,
*spacing,
Delimiter::Invisible(*src),
TokenStream::new(vec),
);
return true;
}
}
}
assert!(found, "Failed to find trailing delimited group in: {res:?}");
false
}
}
@ -462,7 +479,6 @@ impl TokenStream {
pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
match nt {
Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),
}
}

35
compiler/rustc_expand/src/mbe/transcribe.rs
View file

@ -3,11 +3,10 @@ use std::sync::Arc;
use rustc_ast::mut_visit::{self, MutVisitor};
use rustc_ast::token::{
self, Delimiter, IdentIsRaw, InvisibleOrigin, Lit, LitKind, MetaVarKind, Nonterminal, Token,
TokenKind,
self, Delimiter, IdentIsRaw, InvisibleOrigin, Lit, LitKind, MetaVarKind, Token, TokenKind,
};
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
use rustc_ast::{ExprKind, StmtKind, TyKind};
use rustc_ast::{ExprKind, StmtKind, TyKind, UnOp};
use rustc_data_structures::fx::FxHashMap;
use rustc_errors::{Diag, DiagCtxtHandle, PResult, pluralize};
use rustc_parse::lexer::nfc_normalize;
@ -340,6 +339,30 @@ pub(super) fn transcribe<'a>(
MetaVarKind::Pat(*pat_kind),
TokenStream::from_ast(pat),
),
MatchedSingle(ParseNtResult::Expr(expr, kind)) => {
let (can_begin_literal_maybe_minus, can_begin_string_literal) =
match &expr.kind {
ExprKind::Lit(_) => (true, true),
ExprKind::Unary(UnOp::Neg, e)
if matches!(&e.kind, ExprKind::Lit(_)) =>
{
(true, false)
}
_ => (false, false),
};
mk_delimited(
expr.span,
MetaVarKind::Expr {
kind: *kind,
can_begin_literal_maybe_minus,
can_begin_string_literal,
},
TokenStream::from_ast(expr),
)
}
MatchedSingle(ParseNtResult::Literal(lit)) => {
mk_delimited(lit.span, MetaVarKind::Literal, TokenStream::from_ast(lit))
}
MatchedSingle(ParseNtResult::Ty(ty)) => {
let is_path = matches!(&ty.kind, TyKind::Path(None, _path));
mk_delimited(
@ -869,10 +892,8 @@ fn extract_symbol_from_pnr<'a>(
},
_,
)) => Ok(*symbol),
ParseNtResult::Nt(nt)
if let Nonterminal::NtLiteral(expr) = &**nt
&& let ExprKind::Lit(Lit { kind: LitKind::Str, symbol, suffix: None }) =
&expr.kind =>
ParseNtResult::Literal(expr)
if let ExprKind::Lit(Lit { kind: LitKind::Str, symbol, suffix: None }) = &expr.kind =>
{
Ok(*symbol)
}

2
compiler/rustc_parse/messages.ftl
View file

@ -858,7 +858,7 @@ parse_unexpected_parentheses_in_match_arm_pattern = unexpected parentheses surro
parse_unexpected_self_in_generic_parameters = unexpected keyword `Self` in generic parameters
.note = you cannot use `Self` as a generic parameter because it is reserved for associated items
parse_unexpected_token_after_dot = unexpected token: `{$actual}`
parse_unexpected_token_after_dot = unexpected token: {$actual}
parse_unexpected_token_after_label = expected `while`, `for`, `loop` or `{"{"}` after a label
.suggestion_remove_label = consider removing the label

4
compiler/rustc_parse/src/errors.rs
View file

@ -1696,10 +1696,10 @@ pub(crate) struct SelfArgumentPointer {
#[derive(Diagnostic)]
#[diag(parse_unexpected_token_after_dot)]
pub(crate) struct UnexpectedTokenAfterDot<'a> {
pub(crate) struct UnexpectedTokenAfterDot {
#[primary_span]
pub span: Span,
pub actual: Cow<'a, str>,
pub actual: String,
}
#[derive(Diagnostic)]

234
compiler/rustc_parse/src/parser/expr.rs
View file

@ -4,10 +4,10 @@ use core::mem;
use core::ops::{Bound, ControlFlow};
use ast::mut_visit::{self, MutVisitor};
use ast::token::{IdentIsRaw, MetaVarKind};
use ast::token::IdentIsRaw;
use ast::{CoroutineKind, ForLoopKind, GenBlockKind, MatchKind, Pat, Path, PathSegment, Recovered};
use rustc_ast::ptr::P;
use rustc_ast::token::{self, Delimiter, Token, TokenKind};
use rustc_ast::token::{self, Delimiter, InvisibleOrigin, MetaVarKind, Token, TokenKind};
use rustc_ast::tokenstream::TokenTree;
use rustc_ast::util::case::Case;
use rustc_ast::util::classify;
@ -19,7 +19,6 @@ use rustc_ast::{
MetaItemLit, Movability, Param, RangeLimits, StmtKind, Ty, TyKind, UnOp, UnsafeBinderCastKind,
YieldKind,
};
use rustc_ast_pretty::pprust;
use rustc_data_structures::stack::ensure_sufficient_stack;
use rustc_errors::{Applicability, Diag, PResult, StashKey, Subdiagnostic};
use rustc_lexer::unescape::unescape_char;
@ -605,7 +604,7 @@ impl<'a> Parser<'a> {
// can't continue an expression after an ident
token::Ident(name, is_raw) => token::ident_can_begin_expr(name, t.span, is_raw),
token::Literal(..) | token::Pound => true,
_ => t.is_whole_expr(),
_ => t.is_metavar_expr(),
};
self.token.is_ident_named(sym::not) && self.look_ahead(1, token_cannot_continue_expr)
}
@ -641,6 +640,13 @@ impl<'a> Parser<'a> {
TokenKind::NtIdent(..) | TokenKind::NtLifetime(..) | TokenKind::Interpolated(..) => {
self.prev_token.span
}
TokenKind::CloseDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))) => {
// `expr.span` is the interpolated span, because invisible open
// and close delims both get marked with the same span, one
// that covers the entire thing between them. (See
// `rustc_expand::mbe::transcribe::transcribe`.)
self.prev_token.span
}
_ => expr.span,
}
}
@ -979,12 +985,30 @@ impl<'a> Parser<'a> {
}
fn error_unexpected_after_dot(&self) {
let actual = pprust::token_to_string(&self.token);
let actual = super::token_descr(&self.token);
let span = self.token.span;
let sm = self.psess.source_map();
let (span, actual) = match (&self.token.kind, self.subparser_name) {
(token::Eof, Some(_)) if let Ok(actual) = sm.span_to_snippet(sm.next_point(span)) => {
(span.shrink_to_hi(), actual.into())
(token::Eof, Some(_)) if let Ok(snippet) = sm.span_to_snippet(sm.next_point(span)) => {
(span.shrink_to_hi(), format!("`{}`", snippet))
}
(token::CloseDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))), _) => {
// No need to report an error. This case will only occur when parsing a pasted
// metavariable, and we should have emitted an error when parsing the macro call in
// the first place. E.g. in this code:
// ```
// macro_rules! m { ($e:expr) => { $e }; }
//
// fn main() {
// let f = 1;
// m!(f.);
// }
// ```
// we'll get an error "unexpected token: `)` when parsing the `m!(f.)`, so we don't
// want to issue a second error when parsing the expansion `«f.»` (where `«`/`»`
// represent the invisible delimiters).
self.dcx().span_delayed_bug(span, "bad dot expr in metavariable");
return;
}
_ => (span, actual),
};
@ -1294,7 +1318,7 @@ impl<'a> Parser<'a> {
/// Assuming we have just parsed `.`, continue parsing into an expression.
fn parse_dot_suffix(&mut self, self_arg: P<Expr>, lo: Span) -> PResult<'a, P<Expr>> {
if self.token.uninterpolated_span().at_least_rust_2018() && self.eat_keyword(exp!(Await)) {
if self.token_uninterpolated_span().at_least_rust_2018() && self.eat_keyword(exp!(Await)) {
return Ok(self.mk_await_expr(self_arg, lo));
}
@ -1364,17 +1388,33 @@ impl<'a> Parser<'a> {
let span = self.token.span;
if let token::Interpolated(nt) = &self.token.kind {
match &**nt {
token::NtExpr(e) | token::NtLiteral(e) => {
let e = e.clone();
self.bump();
return Ok(e);
}
token::NtBlock(block) => {
let block = block.clone();
self.bump();
return Ok(self.mk_expr(self.prev_token.span, ExprKind::Block(block, None)));
}
};
} else if let Some(expr) = self.eat_metavar_seq_with_matcher(
|mv_kind| matches!(mv_kind, MetaVarKind::Expr { .. }),
|this| {
// Force collection (as opposed to just `parse_expr`) is required to avoid the
// attribute duplication seen in #138478.
let expr = this.parse_expr_force_collect();
// FIXME(nnethercote) Sometimes with expressions we get a trailing comma, possibly
// related to the FIXME in `collect_tokens_for_expr`. Examples are the multi-line
// `assert_eq!` calls involving arguments annotated with `#[rustfmt::skip]` in
// `compiler/rustc_index/src/bit_set/tests.rs`.
if this.token.kind == token::Comma {
this.bump();
}
expr
},
) {
return Ok(expr);
} else if let Some(lit) =
self.eat_metavar_seq(MetaVarKind::Literal, |this| this.parse_literal_maybe_minus())
{
return Ok(lit);
} else if let Some(path) = self.eat_metavar_seq(MetaVarKind::Path, |this| {
this.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))
}) {
@ -1471,9 +1511,9 @@ impl<'a> Parser<'a> {
this.parse_expr_let(restrictions)
} else if this.eat_keyword(exp!(Underscore)) {
Ok(this.mk_expr(this.prev_token.span, ExprKind::Underscore))
} else if this.token.uninterpolated_span().at_least_rust_2018() {
} else if this.token_uninterpolated_span().at_least_rust_2018() {
// `Span::at_least_rust_2018()` is somewhat expensive; don't get it repeatedly.
if this.token.uninterpolated_span().at_least_rust_2024()
if this.token_uninterpolated_span().at_least_rust_2024()
// check for `gen {}` and `gen move {}`
// or `async gen {}` and `async gen move {}`
&& (this.is_gen_block(kw::Gen, 0)
@ -2062,87 +2102,107 @@ impl<'a> Parser<'a> {
.or_else(|()| self.handle_missing_lit(Parser::mk_meta_item_lit_char))
}
fn recover_after_dot(&mut self) -> Option<Token> {
let mut recovered = None;
fn recover_after_dot(&mut self) {
if self.token == token::Dot {
// Attempt to recover `.4` as `0.4`. We don't currently have any syntax where
// dot would follow an optional literal, so we do this unconditionally.
recovered = self.look_ahead(1, |next_token| {
let recovered = self.look_ahead(1, |next_token| {
// If it's an integer that looks like a float, then recover as such.
//
// We will never encounter the exponent part of a floating
// point literal here, since there's no use of the exponent
// syntax that also constitutes a valid integer, so we need
// not check for that.
if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) =
next_token.kind
&& suffix.is_none_or(|s| s == sym::f32 || s == sym::f64)
&& symbol.as_str().chars().all(|c| c.is_numeric() || c == '_')
&& self.token.span.hi() == next_token.span.lo()
{
// If this integer looks like a float, then recover as such.
//
// We will never encounter the exponent part of a floating
// point literal here, since there's no use of the exponent
// syntax that also constitutes a valid integer, so we need
// not check for that.
if suffix.is_none_or(|s| s == sym::f32 || s == sym::f64)
&& symbol.as_str().chars().all(|c| c.is_numeric() || c == '_')
&& self.token.span.hi() == next_token.span.lo()
{
let s = String::from("0.") + symbol.as_str();
let kind = TokenKind::lit(token::Float, Symbol::intern(&s), suffix);
return Some(Token::new(kind, self.token.span.to(next_token.span)));
}
let s = String::from("0.") + symbol.as_str();
let kind = TokenKind::lit(token::Float, Symbol::intern(&s), suffix);
Some(Token::new(kind, self.token.span.to(next_token.span)))
} else {
None
}
None
});
if let Some(token) = &recovered {
self.bump();
if let Some(recovered) = recovered {
self.dcx().emit_err(errors::FloatLiteralRequiresIntegerPart {
span: token.span,
suggestion: token.span.shrink_to_lo(),
span: recovered.span,
suggestion: recovered.span.shrink_to_lo(),
});
self.bump();
self.token = recovered;
}
}
}
recovered
/// Keep this in sync with `Token::can_begin_literal_maybe_minus` and
/// `Lit::from_token` (excluding unary negation).
fn eat_token_lit(&mut self) -> Option<token::Lit> {
match self.token.uninterpolate().kind {
token::Ident(name, IdentIsRaw::No) if name.is_bool_lit() => {
self.bump();
Some(token::Lit::new(token::Bool, name, None))
}
token::Literal(token_lit) => {
self.bump();
Some(token_lit)
}
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
MetaVarKind::Literal,
))) => {
let lit = self
.eat_metavar_seq(MetaVarKind::Literal, |this| this.parse_literal_maybe_minus())
.expect("metavar seq literal");
let ast::ExprKind::Lit(token_lit) = lit.kind else {
panic!("didn't reparse a literal");
};
Some(token_lit)
}
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
mv_kind @ MetaVarKind::Expr { can_begin_literal_maybe_minus: true, .. },
))) => {
let expr = self
.eat_metavar_seq(mv_kind, |this| this.parse_expr())
.expect("metavar seq expr");
let ast::ExprKind::Lit(token_lit) = expr.kind else {
panic!("didn't reparse an expr");
};
Some(token_lit)
}
_ => None,
}
}
/// Matches `lit = true | false | token_lit`.
/// Returns `None` if the next token is not a literal.
pub(super) fn parse_opt_token_lit(&mut self) -> Option<(token::Lit, Span)> {
let recovered = self.recover_after_dot();
let token = recovered.as_ref().unwrap_or(&self.token);
let span = token.span;
token::Lit::from_token(token).map(|token_lit| {
self.bump();
(token_lit, span)
})
fn parse_opt_token_lit(&mut self) -> Option<(token::Lit, Span)> {
self.recover_after_dot();
let span = self.token.span;
self.eat_token_lit().map(|token_lit| (token_lit, span))
}
/// Matches `lit = true | false | token_lit`.
/// Returns `None` if the next token is not a literal.
pub(super) fn parse_opt_meta_item_lit(&mut self) -> Option<MetaItemLit> {
let recovered = self.recover_after_dot();
let token = recovered.as_ref().unwrap_or(&self.token);
match token::Lit::from_token(token) {
Some(lit) => {
match MetaItemLit::from_token_lit(lit, token.span) {
Ok(lit) => {
self.bump();
Some(lit)
}
Err(err) => {
let span = token.uninterpolated_span();
self.bump();
let guar = report_lit_error(self.psess, err, lit, span);
// Pack possible quotes and prefixes from the original literal into
// the error literal's symbol so they can be pretty-printed faithfully.
let suffixless_lit = token::Lit::new(lit.kind, lit.symbol, None);
let symbol = Symbol::intern(&suffixless_lit.to_string());
let lit = token::Lit::new(token::Err(guar), symbol, lit.suffix);
Some(
MetaItemLit::from_token_lit(lit, span)
.unwrap_or_else(|_| unreachable!()),
)
}
fn parse_opt_meta_item_lit(&mut self) -> Option<MetaItemLit> {
self.recover_after_dot();
let span = self.token.span;
let uninterpolated_span = self.token_uninterpolated_span();
self.eat_token_lit().map(|token_lit| {
match MetaItemLit::from_token_lit(token_lit, span) {
Ok(lit) => lit,
Err(err) => {
let guar = report_lit_error(&self.psess, err, token_lit, uninterpolated_span);
// Pack possible quotes and prefixes from the original literal into
// the error literal's symbol so they can be pretty-printed faithfully.
let suffixless_lit = token::Lit::new(token_lit.kind, token_lit.symbol, None);
let symbol = Symbol::intern(&suffixless_lit.to_string());
let token_lit = token::Lit::new(token::Err(guar), symbol, token_lit.suffix);
MetaItemLit::from_token_lit(token_lit, uninterpolated_span).unwrap()
}
}
None => None,
}
})
}
pub(super) fn expect_no_tuple_index_suffix(&self, span: Span, suffix: Symbol) {
@ -2166,9 +2226,10 @@ impl<'a> Parser<'a> {
/// Matches `'-' lit | lit` (cf. `ast_validation::AstValidator::check_expr_within_pat`).
/// Keep this in sync with `Token::can_begin_literal_maybe_minus`.
pub fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P<Expr>> {
if let token::Interpolated(nt) = &self.token.kind {
match &**nt {
// FIXME(nnethercote) The `NtExpr` case should only match if
if let Some(expr) = self.eat_metavar_seq_with_matcher(
|mv_kind| matches!(mv_kind, MetaVarKind::Expr { .. }),
|this| {
// FIXME(nnethercote) The `expr` case should only match if
// `e` is an `ExprKind::Lit` or an `ExprKind::Unary` containing
// an `UnOp::Neg` and an `ExprKind::Lit`, like how
// `can_begin_literal_maybe_minus` works. But this method has
@ -2178,13 +2239,14 @@ impl<'a> Parser<'a> {
// `ExprKind::Path` must be accepted when parsing range
// patterns. That requires some care. So for now, we continue
// being less strict here than we should be.
token::NtExpr(e) | token::NtLiteral(e) => {
let e = e.clone();
self.bump();
return Ok(e);
}
_ => {}
};
this.parse_expr()
},
) {
return Ok(expr);
} else if let Some(lit) =
self.eat_metavar_seq(MetaVarKind::Literal, |this| this.parse_literal_maybe_minus())
{
return Ok(lit);
}
let lo = self.token.span;
@ -2330,7 +2392,7 @@ impl<'a> Parser<'a> {
let movability =
if self.eat_keyword(exp!(Static)) { Movability::Static } else { Movability::Movable };
let coroutine_kind = if self.token.uninterpolated_span().at_least_rust_2018() {
let coroutine_kind = if self.token_uninterpolated_span().at_least_rust_2018() {
self.parse_coroutine_kind(Case::Sensitive)
} else {
None
@ -2879,7 +2941,7 @@ impl<'a> Parser<'a> {
/// Parses `for await? <src_pat> in <src_expr> <src_loop_block>` (`for` token already eaten).
fn parse_expr_for(&mut self, opt_label: Option<Label>, lo: Span) -> PResult<'a, P<Expr>> {
let is_await =
self.token.uninterpolated_span().at_least_rust_2018() && self.eat_keyword(exp!(Await));
self.token_uninterpolated_span().at_least_rust_2018() && self.eat_keyword(exp!(Await));
if is_await {
self.psess.gated_spans.gate(sym::async_for_loop, self.prev_token.span);
@ -3469,7 +3531,7 @@ impl<'a> Parser<'a> {
self.token.is_keyword(kw::Try)
&& self
.look_ahead(1, |t| *t == token::OpenDelim(Delimiter::Brace) || t.is_whole_block())
&& self.token.uninterpolated_span().at_least_rust_2018()
&& self.token_uninterpolated_span().at_least_rust_2018()
}
/// Parses an `async move? {...}` or `gen move? {...}` expression.

63
compiler/rustc_parse/src/parser/item.rs
View file

@ -591,7 +591,7 @@ impl<'a> Parser<'a> {
}
// Parse stray `impl async Trait`
if (self.token.uninterpolated_span().at_least_rust_2018()
if (self.token_uninterpolated_span().at_least_rust_2018()
&& self.token.is_keyword(kw::Async))
|| self.is_kw_followed_by_ident(kw::Async)
{
@ -877,7 +877,7 @@ impl<'a> Parser<'a> {
&& self.look_ahead(1, |t| t.is_non_raw_ident_where(|i| i.name != kw::As))
{
self.bump(); // `default`
Defaultness::Default(self.prev_token.uninterpolated_span())
Defaultness::Default(self.prev_token_uninterpolated_span())
} else {
Defaultness::Final
}
@ -1208,7 +1208,7 @@ impl<'a> Parser<'a> {
attrs: &mut AttrVec,
mut safety: Safety,
) -> PResult<'a, ItemKind> {
let extern_span = self.prev_token.uninterpolated_span();
let extern_span = self.prev_token_uninterpolated_span();
let abi = self.parse_abi(); // ABI?
// FIXME: This recovery should be tested better.
if safety == Safety::Default
@ -1290,12 +1290,24 @@ impl<'a> Parser<'a> {
}
fn is_unsafe_foreign_mod(&self) -> bool {
self.token.is_keyword(kw::Unsafe)
&& self.is_keyword_ahead(1, &[kw::Extern])
&& self.look_ahead(
2 + self.look_ahead(2, |t| t.can_begin_string_literal() as usize),
|t| *t == token::OpenDelim(Delimiter::Brace),
)
// Look for `unsafe`.
if !self.token.is_keyword(kw::Unsafe) {
return false;
}
// Look for `extern`.
if !self.is_keyword_ahead(1, &[kw::Extern]) {
return false;
}
// Look for the optional ABI string literal.
let n = if self.look_ahead(2, |t| t.can_begin_string_literal()) { 3 } else { 2 };
// Look for the `{`. Use `tree_look_ahead` because the ABI (if present)
// might be a metavariable i.e. an invisible-delimited sequence, and
// `tree_look_ahead` will consider that a single element when looking
// ahead.
self.tree_look_ahead(n, |t| matches!(t, TokenTree::Delimited(_, _, Delimiter::Brace, _)))
== Some(true)
}
fn is_static_global(&mut self) -> bool {
@ -2604,13 +2616,36 @@ impl<'a> Parser<'a> {
})
// `extern ABI fn`
|| self.check_keyword_case(exp!(Extern), case)
// Use `tree_look_ahead` because `ABI` might be a metavariable,
// i.e. an invisible-delimited sequence, and `tree_look_ahead`
// will consider that a single element when looking ahead.
&& self.look_ahead(1, |t| t.can_begin_string_literal())
&& (self.look_ahead(2, |t| t.is_keyword_case(kw::Fn, case)) ||
&& (self.tree_look_ahead(2, |tt| {
match tt {
TokenTree::Token(t, _) => t.is_keyword_case(kw::Fn, case),
TokenTree::Delimited(..) => false,
}
}) == Some(true) ||
// This branch is only for better diagnostics; `pub`, `unsafe`, etc. are not
// allowed here.
(self.may_recover()
&& self.look_ahead(2, |t| ALL_QUALS.iter().any(|exp| t.is_keyword(exp.kw)))
&& self.look_ahead(3, |t| t.is_keyword_case(kw::Fn, case))))
&& self.tree_look_ahead(2, |tt| {
match tt {
TokenTree::Token(t, _) =>
ALL_QUALS.iter().any(|exp| {
t.is_keyword(exp.kw)
}),
TokenTree::Delimited(..) => false,
}
}) == Some(true)
&& self.tree_look_ahead(3, |tt| {
match tt {
TokenTree::Token(t, _) => t.is_keyword_case(kw::Fn, case),
TokenTree::Delimited(..) => false,
}
}) == Some(true)
)
)
}
/// Parses all the "front matter" (or "qualifiers") for a `fn` declaration,
@ -2746,7 +2781,7 @@ impl<'a> Parser<'a> {
.expect("Span extracted directly from keyword should always work");
err.span_suggestion(
self.token.uninterpolated_span(),
self.token_uninterpolated_span(),
format!("`{original_kw}` already used earlier, remove this one"),
"",
Applicability::MachineApplicable,
@ -2757,7 +2792,7 @@ impl<'a> Parser<'a> {
else if let Some(WrongKw::Misplaced(correct_pos_sp)) = wrong_kw {
let correct_pos_sp = correct_pos_sp.to(self.prev_token.span);
if let Ok(current_qual) = self.span_to_snippet(correct_pos_sp) {
let misplaced_qual_sp = self.token.uninterpolated_span();
let misplaced_qual_sp = self.token_uninterpolated_span();
let misplaced_qual = self.span_to_snippet(misplaced_qual_sp).unwrap();
err.span_suggestion(

65
compiler/rustc_parse/src/parser/mod.rs
View file

@ -24,8 +24,8 @@ pub use pat::{CommaRecoveryMode, RecoverColon, RecoverComma};
use path::PathStyle;
use rustc_ast::ptr::P;
use rustc_ast::token::{
self, Delimiter, IdentIsRaw, InvisibleOrigin, MetaVarKind, Nonterminal, NtPatKind, Token,
TokenKind,
self, Delimiter, IdentIsRaw, InvisibleOrigin, MetaVarKind, Nonterminal, NtExprKind, NtPatKind,
Token, TokenKind,
};
use rustc_ast::tokenstream::{AttrsTarget, Spacing, TokenStream, TokenTree};
use rustc_ast::util::case::Case;
@ -101,6 +101,7 @@ pub enum ForceCollect {
#[macro_export]
macro_rules! maybe_whole {
($p:expr, $constructor:ident, |$x:ident| $e:expr) => {
#[allow(irrefutable_let_patterns)] // FIXME: temporary
if let token::Interpolated(nt) = &$p.token.kind
&& let token::$constructor(x) = &**nt
{
@ -299,6 +300,10 @@ impl TokenTreeCursor {
self.stream.get(self.index)
}
fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
self.stream.get(self.index + n)
}
#[inline]
fn bump(&mut self) {
self.index += 1;
@ -1290,6 +1295,17 @@ impl<'a> Parser<'a> {
looker(&token)
}
/// Like `lookahead`, but skips over token trees rather than tokens. Useful
/// when looking past possible metavariable pasting sites.
pub fn tree_look_ahead<R>(
&self,
dist: usize,
looker: impl FnOnce(&TokenTree) -> R,
) -> Option<R> {
assert_ne!(dist, 0);
self.token_cursor.curr.look_ahead(dist - 1).map(looker)
}
/// Returns whether any of the given keywords are `dist` tokens ahead of the current one.
pub(crate) fn is_keyword_ahead(&self, dist: usize, kws: &[Symbol]) -> bool {
self.look_ahead(dist, |t| kws.iter().any(|&kw| t.is_keyword(kw)))
@ -1297,14 +1313,14 @@ impl<'a> Parser<'a> {
/// Parses asyncness: `async` or nothing.
fn parse_coroutine_kind(&mut self, case: Case) -> Option<CoroutineKind> {
let span = self.token.uninterpolated_span();
let span = self.token_uninterpolated_span();
if self.eat_keyword_case(exp!(Async), case) {
// FIXME(gen_blocks): Do we want to unconditionally parse `gen` and then
// error if edition <= 2024, like we do with async and edition <= 2018?
if self.token.uninterpolated_span().at_least_rust_2024()
if self.token_uninterpolated_span().at_least_rust_2024()
&& self.eat_keyword_case(exp!(Gen), case)
{
let gen_span = self.prev_token.uninterpolated_span();
let gen_span = self.prev_token_uninterpolated_span();
Some(CoroutineKind::AsyncGen {
span: span.to(gen_span),
closure_id: DUMMY_NODE_ID,
@ -1317,7 +1333,7 @@ impl<'a> Parser<'a> {
return_impl_trait_id: DUMMY_NODE_ID,
})
}
} else if self.token.uninterpolated_span().at_least_rust_2024()
} else if self.token_uninterpolated_span().at_least_rust_2024()
&& self.eat_keyword_case(exp!(Gen), case)
{
Some(CoroutineKind::Gen {
@ -1333,9 +1349,9 @@ impl<'a> Parser<'a> {
/// Parses fn unsafety: `unsafe`, `safe` or nothing.
fn parse_safety(&mut self, case: Case) -> Safety {
if self.eat_keyword_case(exp!(Unsafe), case) {
Safety::Unsafe(self.prev_token.uninterpolated_span())
Safety::Unsafe(self.prev_token_uninterpolated_span())
} else if self.eat_keyword_case(exp!(Safe), case) {
Safety::Safe(self.prev_token.uninterpolated_span())
Safety::Safe(self.prev_token_uninterpolated_span())
} else {
Safety::Default
}
@ -1362,7 +1378,7 @@ impl<'a> Parser<'a> {
.look_ahead(1, |t| *t == token::OpenDelim(Delimiter::Brace) || t.is_whole_block())
&& self.eat_keyword_case(exp!(Const), case)
{
Const::Yes(self.prev_token.uninterpolated_span())
Const::Yes(self.prev_token_uninterpolated_span())
} else {
Const::No
}
@ -1706,6 +1722,35 @@ impl<'a> Parser<'a> {
pub fn approx_token_stream_pos(&self) -> u32 {
self.num_bump_calls
}
/// For interpolated `self.token`, returns a span of the fragment to which
/// the interpolated token refers. For all other tokens this is just a
/// regular span. It is particularly important to use this for identifiers
/// and lifetimes for which spans affect name resolution and edition
/// checks. Note that keywords are also identifiers, so they should use
/// this if they keep spans or perform edition checks.
pub fn token_uninterpolated_span(&self) -> Span {
match &self.token.kind {
token::NtIdent(ident, _) | token::NtLifetime(ident, _) => ident.span,
token::Interpolated(nt) => nt.use_span(),
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))) => {
self.look_ahead(1, |t| t.span)
}
_ => self.token.span,
}
}
/// Like `token_uninterpolated_span`, but works on `self.prev_token`.
pub fn prev_token_uninterpolated_span(&self) -> Span {
match &self.prev_token.kind {
token::NtIdent(ident, _) | token::NtLifetime(ident, _) => ident.span,
token::Interpolated(nt) => nt.use_span(),
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))) => {
self.look_ahead(0, |t| t.span)
}
_ => self.prev_token.span,
}
}
}
pub(crate) fn make_unclosed_delims_error(
@ -1758,6 +1803,8 @@ pub enum ParseNtResult {
Item(P<ast::Item>),
Stmt(P<ast::Stmt>),
Pat(P<ast::Pat>, NtPatKind),
Expr(P<ast::Expr>, NtExprKind),
Literal(P<ast::Expr>),
Ty(P<ast::Ty>),
Meta(P<ast::AttrItem>),
Path(P<ast::Path>),

16
compiler/rustc_parse/src/parser/nonterminal.rs
View file

@ -48,10 +48,6 @@ impl<'a> Parser<'a> {
/// Old variant of `may_be_ident`. Being phased out.
fn nt_may_be_ident(nt: &Nonterminal) -> bool {
match nt {
NtExpr(_)
| NtLiteral(_) // `true`, `false`
=> true,
NtBlock(_) => false,
}
}
@ -95,7 +91,7 @@ impl<'a> Parser<'a> {
token::OpenDelim(Delimiter::Brace) => true,
token::NtLifetime(..) => true,
token::Interpolated(nt) => match &**nt {
NtBlock(_) | NtExpr(_) | NtLiteral(_) => true,
NtBlock(_) => true,
},
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(k))) => match k {
MetaVarKind::Block
@ -179,10 +175,14 @@ impl<'a> Parser<'a> {
pat_kind,
));
}
NonterminalKind::Expr(_) => NtExpr(self.parse_expr_force_collect()?),
NonterminalKind::Expr(expr_kind) => {
return Ok(ParseNtResult::Expr(self.parse_expr_force_collect()?, expr_kind));
}
NonterminalKind::Literal => {
// The `:literal` matcher does not support attributes
NtLiteral(self.collect_tokens_no_attrs(|this| this.parse_literal_maybe_minus())?)
// The `:literal` matcher does not support attributes.
return Ok(ParseNtResult::Literal(
self.collect_tokens_no_attrs(|this| this.parse_literal_maybe_minus())?,
));
}
NonterminalKind::Ty => {
return Ok(ParseNtResult::Ty(

2
compiler/rustc_parse/src/parser/pat.rs
View file

@ -1252,7 +1252,7 @@ impl<'a> Parser<'a> {
|| *t == token::Dot // e.g. `.5` for recovery;
|| matches!(t.kind, token::Literal(..) | token::Minus)
|| t.is_bool_lit()
|| t.is_whole_expr()
|| t.is_metavar_expr()
|| t.is_lifetime() // recover `'a` instead of `'a'`
|| (self.may_recover() // recover leading `(`
&& *t == token::OpenDelim(Delimiter::Parenthesis)

6
compiler/rustc_parse/src/parser/ty.rs
View file

@ -775,7 +775,7 @@ impl<'a> Parser<'a> {
/// Is a `dyn B0 + ... + Bn` type allowed here?
fn is_explicit_dyn_type(&mut self) -> bool {
self.check_keyword(exp!(Dyn))
&& (self.token.uninterpolated_span().at_least_rust_2018()
&& (self.token_uninterpolated_span().at_least_rust_2018()
|| self.look_ahead(1, |t| {
(can_begin_dyn_bound_in_edition_2015(t) || *t == TokenKind::Star)
&& !can_continue_type_after_non_fn_ident(t)
@ -998,13 +998,13 @@ impl<'a> Parser<'a> {
BoundConstness::Never
};
let asyncness = if self.token.uninterpolated_span().at_least_rust_2018()
let asyncness = if self.token_uninterpolated_span().at_least_rust_2018()
&& self.eat_keyword(exp!(Async))
{
self.psess.gated_spans.gate(sym::async_trait_bounds, self.prev_token.span);
BoundAsyncness::Async(self.prev_token.span)
} else if self.may_recover()
&& self.token.uninterpolated_span().is_rust_2015()
&& self.token_uninterpolated_span().is_rust_2015()
&& self.is_kw_followed_by_ident(kw::Async)
{
self.bump(); // eat `async`