bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Auto merge of #102944 - nnethercote:ast-Lit-third-time-lucky, r=petrochenkov

Browse source 
Use `token::Lit` in `ast::ExprKind::Lit`.

Instead of `ast::Lit`.

Literal lowering now happens at two different times. Expression literals are lowered when HIR is crated. Attribute literals are lowered during parsing.

r? `@petrochenkov`
This commit is contained in:
bors 2022-11-16 23:03:14 +00:00
commit bebd57a960
45 changed files with 786 additions and 507 deletions
Download patch file Download diff file Expand all files Collapse all files

90
compiler/rustc_parse/src/errors.rs
View file

@ -304,61 +304,6 @@ pub(crate) struct FloatLiteralRequiresIntegerPart {
pub correct: String,
}
#[derive(Diagnostic)]
#[diag(parser_invalid_int_literal_width)]
#[help]
pub(crate) struct InvalidIntLiteralWidth {
#[primary_span]
pub span: Span,
pub width: String,
}
#[derive(Diagnostic)]
#[diag(parser_invalid_num_literal_base_prefix)]
#[note]
pub(crate) struct InvalidNumLiteralBasePrefix {
#[primary_span]
#[suggestion(applicability = "maybe-incorrect", code = "{fixed}")]
pub span: Span,
pub fixed: String,
}
#[derive(Diagnostic)]
#[diag(parser_invalid_num_literal_suffix)]
#[help]
pub(crate) struct InvalidNumLiteralSuffix {
#[primary_span]
#[label]
pub span: Span,
pub suffix: String,
}
#[derive(Diagnostic)]
#[diag(parser_invalid_float_literal_width)]
#[help]
pub(crate) struct InvalidFloatLiteralWidth {
#[primary_span]
pub span: Span,
pub width: String,
}
#[derive(Diagnostic)]
#[diag(parser_invalid_float_literal_suffix)]
#[help]
pub(crate) struct InvalidFloatLiteralSuffix {
#[primary_span]
#[label]
pub span: Span,
pub suffix: String,
}
#[derive(Diagnostic)]
#[diag(parser_int_literal_too_large)]
pub(crate) struct IntLiteralTooLarge {
#[primary_span]
pub span: Span,
}
#[derive(Diagnostic)]
#[diag(parser_missing_semicolon_before_array)]
pub(crate) struct MissingSemicolonBeforeArray {
@ -740,41 +685,6 @@ pub(crate) struct InvalidInterpolatedExpression {
pub span: Span,
}
#[derive(Diagnostic)]
#[diag(parser_hexadecimal_float_literal_not_supported)]
pub(crate) struct HexadecimalFloatLiteralNotSupported {
#[primary_span]
#[label(parser_not_supported)]
pub span: Span,
}
#[derive(Diagnostic)]
#[diag(parser_octal_float_literal_not_supported)]
pub(crate) struct OctalFloatLiteralNotSupported {
#[primary_span]
#[label(parser_not_supported)]
pub span: Span,
}
#[derive(Diagnostic)]
#[diag(parser_binary_float_literal_not_supported)]
pub(crate) struct BinaryFloatLiteralNotSupported {
#[primary_span]
#[label(parser_not_supported)]
pub span: Span,
}
#[derive(Diagnostic)]
#[diag(parser_invalid_literal_suffix)]
pub(crate) struct InvalidLiteralSuffix {
#[primary_span]
#[label]
pub span: Span,
// FIXME(#100717)
pub kind: String,
pub suffix: Symbol,
}
#[derive(Diagnostic)]
#[diag(parser_invalid_literal_suffix_on_tuple_index)]
pub(crate) struct InvalidLiteralSuffixOnTupleIndex {

13
compiler/rustc_parse/src/lexer/mod.rs
View file

@ -661,6 +661,7 @@ impl<'a> StringReader<'a> {
prefix_len: u32,
postfix_len: u32,
) -> (token::LitKind, Symbol) {
let mut has_fatal_err = false;
let content_start = start + BytePos(prefix_len);
let content_end = end - BytePos(postfix_len);
let lit_content = self.str_from_to(content_start, content_end);
@ -672,6 +673,9 @@ impl<'a> StringReader<'a> {
let lo = content_start + BytePos(start);
let hi = lo + BytePos(end - start);
let span = self.mk_sp(lo, hi);
if err.is_fatal() {
has_fatal_err = true;
}
emit_unescape_error(
&self.sess.span_diagnostic,
lit_content,
@ -683,7 +687,14 @@ impl<'a> StringReader<'a> {
);
}
});
(kind, Symbol::intern(lit_content))
// We normally exclude the quotes for the symbol, but for errors we
// include it because it results in clearer error messages.
if !has_fatal_err {
(kind, Symbol::intern(lit_content))
} else {
(token::Err, self.symbol_from_to(start, end))
}
}
}

4
compiler/rustc_parse/src/parser/attr.rs
View file

@ -316,8 +316,8 @@ impl<'a> Parser<'a> {
}
pub(crate) fn parse_unsuffixed_lit(&mut self) -> PResult<'a, ast::Lit> {
let lit = self.parse_lit()?;
debug!("checking if {:?} is unusuffixed", lit);
let lit = self.parse_ast_lit()?;
debug!("checking if {:?} is unsuffixed", lit);
if !lit.kind.is_unsuffixed() {
self.sess.emit_err(SuffixedLiteralInAttribute { span: lit.span });

242
compiler/rustc_parse/src/parser/expr.rs
View file

@ -7,35 +7,30 @@ use super::{
};
use crate::errors::{
ArrayBracketsInsteadOfSpaces, ArrayBracketsInsteadOfSpacesSugg, AsyncMoveOrderIncorrect,
BinaryFloatLiteralNotSupported, BracesForStructLiteral, CatchAfterTry, CommaAfterBaseStruct,
ComparisonInterpretedAsGeneric, ComparisonOrShiftInterpretedAsGenericSugg,
DoCatchSyntaxRemoved, DotDotDot, EqFieldInit, ExpectedElseBlock, ExpectedEqForLetExpr,
ExpectedExpressionFoundLet, FieldExpressionWithGeneric, FloatLiteralRequiresIntegerPart,
FoundExprWouldBeStmt, HexadecimalFloatLiteralNotSupported, IfExpressionMissingCondition,
IfExpressionMissingThenBlock, IfExpressionMissingThenBlockSub, IntLiteralTooLarge,
BracesForStructLiteral, CatchAfterTry, CommaAfterBaseStruct, ComparisonInterpretedAsGeneric,
ComparisonOrShiftInterpretedAsGenericSugg, DoCatchSyntaxRemoved, DotDotDot, EqFieldInit,
ExpectedElseBlock, ExpectedEqForLetExpr, ExpectedExpressionFoundLet,
FieldExpressionWithGeneric, FloatLiteralRequiresIntegerPart, FoundExprWouldBeStmt,
IfExpressionMissingCondition, IfExpressionMissingThenBlock, IfExpressionMissingThenBlockSub,
InvalidBlockMacroSegment, InvalidComparisonOperator, InvalidComparisonOperatorSub,
InvalidFloatLiteralSuffix, InvalidFloatLiteralWidth, InvalidIntLiteralWidth,
InvalidInterpolatedExpression, InvalidLiteralSuffix, InvalidLiteralSuffixOnTupleIndex,
InvalidLogicalOperator, InvalidLogicalOperatorSub, InvalidNumLiteralBasePrefix,
InvalidNumLiteralSuffix, LabeledLoopInBreak, LeadingPlusNotSupported, LeftArrowOperator,
InvalidInterpolatedExpression, InvalidLiteralSuffixOnTupleIndex, InvalidLogicalOperator,
InvalidLogicalOperatorSub, LabeledLoopInBreak, LeadingPlusNotSupported, LeftArrowOperator,
LifetimeInBorrowExpression, MacroInvocationWithQualifiedPath, MalformedLoopLabel,
MatchArmBodyWithoutBraces, MatchArmBodyWithoutBracesSugg, MissingCommaAfterMatchArm,
MissingDotDot, MissingInInForLoop, MissingInInForLoopSub, MissingSemicolonBeforeArray,
NoFieldsForFnCall, NotAsNegationOperator, NotAsNegationOperatorSub,
OctalFloatLiteralNotSupported, OuterAttributeNotAllowedOnIfElse, ParenthesesWithStructFields,
OuterAttributeNotAllowedOnIfElse, ParenthesesWithStructFields,
RequireColonAfterLabeledExpression, ShiftInterpretedAsGeneric, StructLiteralNotAllowedHere,
StructLiteralNotAllowedHereSugg, TildeAsUnaryOperator, UnexpectedTokenAfterLabel,
UnexpectedTokenAfterLabelSugg, WrapExpressionInParentheses,
};
use crate::maybe_recover_from_interpolated_ty_qpath;
use core::mem;
use rustc_ast::ptr::P;
use rustc_ast::token::{self, Delimiter, Token, TokenKind};
use rustc_ast::tokenstream::Spacing;
use rustc_ast::util::case::Case;
use rustc_ast::util::classify;
use rustc_ast::util::literal::LitError;
use rustc_ast::util::parser::{prec_let_scrutinee_needs_par, AssocOp, Fixity};
use rustc_ast::visit::Visitor;
use rustc_ast::{self as ast, AttrStyle, AttrVec, CaptureBy, ExprField, Lit, UnOp, DUMMY_NODE_ID};
@ -47,7 +42,7 @@ use rustc_errors::{
Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic, PResult,
StashKey,
};
use rustc_session::errors::ExprParenthesesNeeded;
use rustc_session::errors::{report_lit_error, ExprParenthesesNeeded};
use rustc_session::lint::builtin::BREAK_WITH_LABEL_AND_LOOP;
use rustc_session::lint::BuiltinLintDiagnostics;
use rustc_span::source_map::{self, Span, Spanned};
@ -1415,9 +1410,9 @@ impl<'a> Parser<'a> {
fn parse_lit_expr(&mut self) -> PResult<'a, P<Expr>> {
let lo = self.token.span;
match self.parse_opt_lit() {
Some(literal) => {
let expr = self.mk_expr(lo.to(self.prev_token.span), ExprKind::Lit(literal));
match self.parse_opt_token_lit() {
Some((token_lit, _)) => {
let expr = self.mk_expr(lo.to(self.prev_token.span), ExprKind::Lit(token_lit));
self.maybe_recover_from_bad_qpath(expr)
}
None => self.try_macro_suggestion(),
@ -1548,7 +1543,7 @@ impl<'a> Parser<'a> {
})
});
consume_colon = false;
Ok(self.mk_expr(lo, ExprKind::Lit(lit)))
Ok(self.mk_expr(lo, ExprKind::Lit(lit.token_lit)))
} else if !ate_colon
&& (self.check_noexpect(&TokenKind::Comma) || self.check_noexpect(&TokenKind::Gt))
{
@ -1625,9 +1620,9 @@ impl<'a> Parser<'a> {
/// Emit an error when a char is parsed as a lifetime because of a missing quote
pub(super) fn recover_unclosed_char(
&mut self,
&self,
lifetime: Ident,
err: impl FnOnce(&mut Self) -> DiagnosticBuilder<'a, ErrorGuaranteed>,
err: impl FnOnce(&Self) -> DiagnosticBuilder<'a, ErrorGuaranteed>,
) -> ast::Lit {
if let Some(mut diag) =
self.sess.span_diagnostic.steal_diagnostic(lifetime.span, StashKey::LifetimeIsChar)
@ -1649,9 +1644,10 @@ impl<'a> Parser<'a> {
)
.emit();
}
let name = lifetime.without_first_quote().name;
ast::Lit {
token_lit: token::Lit::new(token::LitKind::Char, lifetime.name, None),
kind: ast::LitKind::Char(lifetime.name.as_str().chars().next().unwrap_or('_')),
token_lit: token::Lit::new(token::LitKind::Char, name, None),
kind: ast::LitKind::Char(name.as_str().chars().next().unwrap_or('_')),
span: lifetime.span,
}
}
@ -1765,7 +1761,7 @@ impl<'a> Parser<'a> {
/// In case of error returns `Some(lit)` if the next token is a literal with a wrong kind,
/// and returns `None` if the next token is not literal at all.
pub fn parse_str_lit(&mut self) -> Result<ast::StrLit, Option<Lit>> {
match self.parse_opt_lit() {
match self.parse_opt_ast_lit() {
Some(lit) => match lit.kind {
ast::LitKind::Str(symbol_unescaped, style) => Ok(ast::StrLit {
style,
@ -1780,41 +1776,47 @@ impl<'a> Parser<'a> {
}
}
pub(super) fn parse_lit(&mut self) -> PResult<'a, Lit> {
self.parse_opt_lit().ok_or(()).or_else(|()| {
if let token::Interpolated(inner) = &self.token.kind {
let expr = match inner.as_ref() {
token::NtExpr(expr) => Some(expr),
token::NtLiteral(expr) => Some(expr),
_ => None,
};
if let Some(expr) = expr {
if matches!(expr.kind, ExprKind::Err) {
let mut err = InvalidInterpolatedExpression { span: self.token.span }
.into_diagnostic(&self.sess.span_diagnostic);
err.downgrade_to_delayed_bug();
return Err(err);
}
fn handle_missing_lit(&mut self) -> PResult<'a, Lit> {
if let token::Interpolated(inner) = &self.token.kind {
let expr = match inner.as_ref() {
token::NtExpr(expr) => Some(expr),
token::NtLiteral(expr) => Some(expr),
_ => None,
};
if let Some(expr) = expr {
if matches!(expr.kind, ExprKind::Err) {
let mut err = InvalidInterpolatedExpression { span: self.token.span }
.into_diagnostic(&self.sess.span_diagnostic);
err.downgrade_to_delayed_bug();
return Err(err);
}
}
let token = self.token.clone();
let err = |self_: &mut Self| {
let msg = format!("unexpected token: {}", super::token_descr(&token));
self_.struct_span_err(token.span, &msg)
};
// On an error path, eagerly consider a lifetime to be an unclosed character lit
if self.token.is_lifetime() {
let lt = self.expect_lifetime();
Ok(self.recover_unclosed_char(lt.ident, err))
} else {
Err(err(self))
}
})
}
let token = self.token.clone();
let err = |self_: &Self| {
let msg = format!("unexpected token: {}", super::token_descr(&token));
self_.struct_span_err(token.span, &msg)
};
// On an error path, eagerly consider a lifetime to be an unclosed character lit
if self.token.is_lifetime() {
let lt = self.expect_lifetime();
Ok(self.recover_unclosed_char(lt.ident, err))
} else {
Err(err(self))
}
}
/// Matches `lit = true | false | token_lit`.
/// Returns `None` if the next token is not a literal.
pub(super) fn parse_opt_lit(&mut self) -> Option<Lit> {
pub(super) fn parse_token_lit(&mut self) -> PResult<'a, (token::Lit, Span)> {
self.parse_opt_token_lit()
.ok_or(())
.or_else(|()| self.handle_missing_lit().map(|lit| (lit.token_lit, lit.span)))
}
pub(super) fn parse_ast_lit(&mut self) -> PResult<'a, Lit> {
self.parse_opt_ast_lit().ok_or(()).or_else(|()| self.handle_missing_lit())
}
fn recover_after_dot(&mut self) -> Option<Token> {
let mut recovered = None;
if self.token == token::Dot {
// Attempt to recover `.4` as `0.4`. We don't currently have any syntax where
@ -1840,100 +1842,50 @@ impl<'a> Parser<'a> {
}
}
let token = recovered.as_ref().unwrap_or(&self.token);
match Lit::from_token(token) {
Ok(lit) => {
self.bump();
Some(lit)
}
Err(LitError::NotLiteral) => None,
Err(err) => {
let span = token.span;
let token::Literal(lit) = token.kind else {
unreachable!();
};
self.bump();
self.report_lit_error(err, lit, span);
// Pack possible quotes and prefixes from the original literal into
// the error literal's symbol so they can be pretty-printed faithfully.
let suffixless_lit = token::Lit::new(lit.kind, lit.symbol, None);
let symbol = Symbol::intern(&suffixless_lit.to_string());
let lit = token::Lit::new(token::Err, symbol, lit.suffix);
Some(Lit::from_token_lit(lit, span).unwrap_or_else(|_| unreachable!()))
}
}
recovered
}
fn report_lit_error(&self, err: LitError, lit: token::Lit, span: Span) {
// Checks if `s` looks like i32 or u1234 etc.
fn looks_like_width_suffix(first_chars: &[char], s: &str) -> bool {
s.len() > 1 && s.starts_with(first_chars) && s[1..].chars().all(|c| c.is_ascii_digit())
}
/// Matches `lit = true | false | token_lit`.
/// Returns `None` if the next token is not a literal.
pub(super) fn parse_opt_token_lit(&mut self) -> Option<(token::Lit, Span)> {
let recovered = self.recover_after_dot();
let token = recovered.as_ref().unwrap_or(&self.token);
let span = token.span;
token::Lit::from_token(token).map(|token_lit| {
self.bump();
(token_lit, span)
})
}
// Try to lowercase the prefix if it's a valid base prefix.
fn fix_base_capitalisation(s: &str) -> Option<String> {
if let Some(stripped) = s.strip_prefix('B') {
Some(format!("0b{stripped}"))
} else if let Some(stripped) = s.strip_prefix('O') {
Some(format!("0o{stripped}"))
} else if let Some(stripped) = s.strip_prefix('X') {
Some(format!("0x{stripped}"))
} else {
None
}
}
let token::Lit { kind, suffix, .. } = lit;
match err {
// `NotLiteral` is not an error by itself, so we don't report
// it and give the parser opportunity to try something else.
LitError::NotLiteral => {}
// `LexerError` *is* an error, but it was already reported
// by lexer, so here we don't report it the second time.
LitError::LexerError => {}
LitError::InvalidSuffix => {
if let Some(suffix) = suffix {
self.sess.emit_err(InvalidLiteralSuffix {
span,
kind: format!("{}", kind.descr()),
suffix,
});
/// Matches `lit = true | false | token_lit`.
/// Returns `None` if the next token is not a literal.
pub(super) fn parse_opt_ast_lit(&mut self) -> Option<Lit> {
let recovered = self.recover_after_dot();
let token = recovered.as_ref().unwrap_or(&self.token);
match token::Lit::from_token(token) {
Some(token_lit) => {
match Lit::from_token_lit(token_lit, token.span) {
Ok(lit) => {
self.bump();
Some(lit)
}
Err(err) => {
let span = token.span;
let token::Literal(lit) = token.kind else {
unreachable!();
};
self.bump();
report_lit_error(&self.sess, err, lit, span);
// Pack possible quotes and prefixes from the original literal into
// the error literal's symbol so they can be pretty-printed faithfully.
let suffixless_lit = token::Lit::new(lit.kind, lit.symbol, None);
let symbol = Symbol::intern(&suffixless_lit.to_string());
let lit = token::Lit::new(token::Err, symbol, lit.suffix);
Some(Lit::from_token_lit(lit, span).unwrap_or_else(|_| unreachable!()))
}
}
}
LitError::InvalidIntSuffix => {
let suf = suffix.expect("suffix error with no suffix");
let suf = suf.as_str();
if looks_like_width_suffix(&['i', 'u'], &suf) {
// If it looks like a width, try to be helpful.
self.sess.emit_err(InvalidIntLiteralWidth { span, width: suf[1..].into() });
} else if let Some(fixed) = fix_base_capitalisation(suf) {
self.sess.emit_err(InvalidNumLiteralBasePrefix { span, fixed });
} else {
self.sess.emit_err(InvalidNumLiteralSuffix { span, suffix: suf.to_string() });
}
}
LitError::InvalidFloatSuffix => {
let suf = suffix.expect("suffix error with no suffix");
let suf = suf.as_str();
if looks_like_width_suffix(&['f'], suf) {
// If it looks like a width, try to be helpful.
self.sess
.emit_err(InvalidFloatLiteralWidth { span, width: suf[1..].to_string() });
} else {
self.sess.emit_err(InvalidFloatLiteralSuffix { span, suffix: suf.to_string() });
}
}
LitError::NonDecimalFloat(base) => {
match base {
16 => self.sess.emit_err(HexadecimalFloatLiteralNotSupported { span }),
8 => self.sess.emit_err(OctalFloatLiteralNotSupported { span }),
2 => self.sess.emit_err(BinaryFloatLiteralNotSupported { span }),
_ => unreachable!(),
};
}
LitError::IntTooLarge => {
self.sess.emit_err(IntLiteralTooLarge { span });
}
None => None,
}
}
@ -1958,8 +1910,8 @@ impl<'a> Parser<'a> {
let lo = self.token.span;
let minus_present = self.eat(&token::BinOp(token::Minus));
let lit = self.parse_lit()?;
let expr = self.mk_expr(lit.span, ExprKind::Lit(lit));
let (token_lit, span) = self.parse_token_lit()?;
let expr = self.mk_expr(span, ExprKind::Lit(token_lit));
if minus_present {
Ok(self.mk_expr(lo.to(self.prev_token.span), self.mk_unary(UnOp::Neg, expr)))

2
compiler/rustc_parse/src/parser/pat.rs
View file

@ -420,7 +420,7 @@ impl<'a> Parser<'a> {
err.span_label(self_.token.span, format!("expected {}", expected));
err
});
PatKind::Lit(self.mk_expr(lo, ExprKind::Lit(lit)))
PatKind::Lit(self.mk_expr(lo, ExprKind::Lit(lit.token_lit)))
} else {
// Try to parse everything else as literal with optional minus
match self.parse_literal_maybe_minus() {

2
compiler/rustc_parse/src/parser/stmt.rs
View file

@ -359,7 +359,7 @@ impl<'a> Parser<'a> {
/// report error for `let 1x = 123`
pub fn report_invalid_identifier_error(&mut self) -> PResult<'a, ()> {
if let token::Literal(lit) = self.token.uninterpolate().kind &&
let Err(_) = rustc_ast::Lit::from_token(&self.token) &&
rustc_ast::Lit::from_token(&self.token).is_none() &&
(lit.kind == token::LitKind::Integer || lit.kind == token::LitKind::Float) &&
self.look_ahead(1, |t| matches!(t.kind, token::Eq) || matches!(t.kind, token::Colon ) ) {
return Err(self.sess.create_err(InvalidIdentiferStartsWithNumber { span: self.token.span }));

10
compiler/rustc_parse/src/validate_attr.rs
View file

@ -49,10 +49,12 @@ pub fn parse_meta<'a>(sess: &'a ParseSess, attr: &Attribute) -> PResult<'a, Meta
MetaItemKind::List(nmis)
}
MacArgs::Eq(_, MacArgsEq::Ast(expr)) => {
if let ast::ExprKind::Lit(lit) = &expr.kind {
if !lit.kind.is_unsuffixed() {
if let ast::ExprKind::Lit(token_lit) = expr.kind
&& let Ok(lit) = ast::Lit::from_token_lit(token_lit, expr.span)
{
if token_lit.suffix.is_some() {
let mut err = sess.span_diagnostic.struct_span_err(
lit.span,
expr.span,
"suffixed literals are not allowed in attributes",
);
err.help(
@ -61,7 +63,7 @@ pub fn parse_meta<'a>(sess: &'a ParseSess, attr: &Attribute) -> PResult<'a, Meta
);
return Err(err);
} else {
MetaItemKind::NameValue(lit.clone())
MetaItemKind::NameValue(lit)
}
} else {
// The non-error case can happen with e.g. `#[foo = 1+1]`. The error case can