bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Use token::Lit in ast::ExprKind::Lit.

Browse source 
Instead of `ast::Lit`.

Literal lowering now happens at two different times. Expression literals
are lowered when HIR is crated. Attribute literals are lowered during
parsing.

This commit changes the language very slightly. Some programs that used
to not compile now will compile. This is because some invalid literals
that are removed by `cfg` or attribute macros will no longer trigger
errors. See this comment for more details:
https://github.com/rust-lang/rust/pull/102944#issuecomment-1277476773
This commit is contained in:
Nicholas Nethercote 2022-10-10 13:40:56 +11:00
parent 01760265cb
commit 358a603f11
45 changed files with 701 additions and 581 deletions
Download patch file Download diff file Expand all files Collapse all files

90
compiler/rustc_parse/src/errors.rs
View file

@ -304,61 +304,6 @@ pub(crate) struct FloatLiteralRequiresIntegerPart {
pub correct: String,
}
#[derive(Diagnostic)]
#[diag(parser_invalid_int_literal_width)]
#[help]
pub(crate) struct InvalidIntLiteralWidth {
#[primary_span]
pub span: Span,
pub width: String,
}
#[derive(Diagnostic)]
#[diag(parser_invalid_num_literal_base_prefix)]
#[note]
pub(crate) struct InvalidNumLiteralBasePrefix {
#[primary_span]
#[suggestion(applicability = "maybe-incorrect", code = "{fixed}")]
pub span: Span,
pub fixed: String,
}
#[derive(Diagnostic)]
#[diag(parser_invalid_num_literal_suffix)]
#[help]
pub(crate) struct InvalidNumLiteralSuffix {
#[primary_span]
#[label]
pub span: Span,
pub suffix: String,
}
#[derive(Diagnostic)]
#[diag(parser_invalid_float_literal_width)]
#[help]
pub(crate) struct InvalidFloatLiteralWidth {
#[primary_span]
pub span: Span,
pub width: String,
}
#[derive(Diagnostic)]
#[diag(parser_invalid_float_literal_suffix)]
#[help]
pub(crate) struct InvalidFloatLiteralSuffix {
#[primary_span]
#[label]
pub span: Span,
pub suffix: String,
}
#[derive(Diagnostic)]
#[diag(parser_int_literal_too_large)]
pub(crate) struct IntLiteralTooLarge {
#[primary_span]
pub span: Span,
}
#[derive(Diagnostic)]
#[diag(parser_missing_semicolon_before_array)]
pub(crate) struct MissingSemicolonBeforeArray {
@ -740,41 +685,6 @@ pub(crate) struct InvalidInterpolatedExpression {
pub span: Span,
}
#[derive(Diagnostic)]
#[diag(parser_hexadecimal_float_literal_not_supported)]
pub(crate) struct HexadecimalFloatLiteralNotSupported {
#[primary_span]
#[label(parser_not_supported)]
pub span: Span,
}
#[derive(Diagnostic)]
#[diag(parser_octal_float_literal_not_supported)]
pub(crate) struct OctalFloatLiteralNotSupported {
#[primary_span]
#[label(parser_not_supported)]
pub span: Span,
}
#[derive(Diagnostic)]
#[diag(parser_binary_float_literal_not_supported)]
pub(crate) struct BinaryFloatLiteralNotSupported {
#[primary_span]
#[label(parser_not_supported)]
pub span: Span,
}
#[derive(Diagnostic)]
#[diag(parser_invalid_literal_suffix)]
pub(crate) struct InvalidLiteralSuffix {
#[primary_span]
#[label]
pub span: Span,
// FIXME(#100717)
pub kind: String,
pub suffix: Symbol,
}
#[derive(Diagnostic)]
#[diag(parser_invalid_literal_suffix_on_tuple_index)]
pub(crate) struct InvalidLiteralSuffixOnTupleIndex {

13
compiler/rustc_parse/src/lexer/mod.rs
View file

@ -661,6 +661,7 @@ impl<'a> StringReader<'a> {
prefix_len: u32,
postfix_len: u32,
) -> (token::LitKind, Symbol) {
let mut has_fatal_err = false;
let content_start = start + BytePos(prefix_len);
let content_end = end - BytePos(postfix_len);
let lit_content = self.str_from_to(content_start, content_end);
@ -672,6 +673,9 @@ impl<'a> StringReader<'a> {
let lo = content_start + BytePos(start);
let hi = lo + BytePos(end - start);
let span = self.mk_sp(lo, hi);
if err.is_fatal() {
has_fatal_err = true;
}
emit_unescape_error(
&self.sess.span_diagnostic,
lit_content,
@ -683,7 +687,14 @@ impl<'a> StringReader<'a> {
);
}
});
(kind, Symbol::intern(lit_content))
// We normally exclude the quotes for the symbol, but for errors we
// include it because it results in clearer error messages.
if !has_fatal_err {
(kind, Symbol::intern(lit_content))
} else {
(token::Err, self.symbol_from_to(start, end))
}
}
}

4
compiler/rustc_parse/src/parser/attr.rs
View file

@ -316,8 +316,8 @@ impl<'a> Parser<'a> {
}
pub(crate) fn parse_unsuffixed_lit(&mut self) -> PResult<'a, ast::Lit> {
let lit = self.parse_lit()?;
debug!("checking if {:?} is unusuffixed", lit);
let lit = self.parse_ast_lit()?;
debug!("checking if {:?} is unsuffixed", lit);
if !lit.kind.is_unsuffixed() {
self.sess.emit_err(SuffixedLiteralInAttribute { span: lit.span });

242
compiler/rustc_parse/src/parser/expr.rs
View file

@ -7,35 +7,30 @@ use super::{
};
use crate::errors::{
ArrayBracketsInsteadOfSpaces, ArrayBracketsInsteadOfSpacesSugg, AsyncMoveOrderIncorrect,
BinaryFloatLiteralNotSupported, BracesForStructLiteral, CatchAfterTry, CommaAfterBaseStruct,
ComparisonInterpretedAsGeneric, ComparisonOrShiftInterpretedAsGenericSugg,
DoCatchSyntaxRemoved, DotDotDot, EqFieldInit, ExpectedElseBlock, ExpectedEqForLetExpr,
ExpectedExpressionFoundLet, FieldExpressionWithGeneric, FloatLiteralRequiresIntegerPart,
FoundExprWouldBeStmt, HexadecimalFloatLiteralNotSupported, IfExpressionMissingCondition,
IfExpressionMissingThenBlock, IfExpressionMissingThenBlockSub, IntLiteralTooLarge,
BracesForStructLiteral, CatchAfterTry, CommaAfterBaseStruct, ComparisonInterpretedAsGeneric,
ComparisonOrShiftInterpretedAsGenericSugg, DoCatchSyntaxRemoved, DotDotDot, EqFieldInit,
ExpectedElseBlock, ExpectedEqForLetExpr, ExpectedExpressionFoundLet,
FieldExpressionWithGeneric, FloatLiteralRequiresIntegerPart, FoundExprWouldBeStmt,
IfExpressionMissingCondition, IfExpressionMissingThenBlock, IfExpressionMissingThenBlockSub,
InvalidBlockMacroSegment, InvalidComparisonOperator, InvalidComparisonOperatorSub,
InvalidFloatLiteralSuffix, InvalidFloatLiteralWidth, InvalidIntLiteralWidth,
InvalidInterpolatedExpression, InvalidLiteralSuffix, InvalidLiteralSuffixOnTupleIndex,
InvalidLogicalOperator, InvalidLogicalOperatorSub, InvalidNumLiteralBasePrefix,
InvalidNumLiteralSuffix, LabeledLoopInBreak, LeadingPlusNotSupported, LeftArrowOperator,
InvalidInterpolatedExpression, InvalidLiteralSuffixOnTupleIndex, InvalidLogicalOperator,
InvalidLogicalOperatorSub, LabeledLoopInBreak, LeadingPlusNotSupported, LeftArrowOperator,
LifetimeInBorrowExpression, MacroInvocationWithQualifiedPath, MalformedLoopLabel,
MatchArmBodyWithoutBraces, MatchArmBodyWithoutBracesSugg, MissingCommaAfterMatchArm,
MissingDotDot, MissingInInForLoop, MissingInInForLoopSub, MissingSemicolonBeforeArray,
NoFieldsForFnCall, NotAsNegationOperator, NotAsNegationOperatorSub,
OctalFloatLiteralNotSupported, OuterAttributeNotAllowedOnIfElse, ParenthesesWithStructFields,
OuterAttributeNotAllowedOnIfElse, ParenthesesWithStructFields,
RequireColonAfterLabeledExpression, ShiftInterpretedAsGeneric, StructLiteralNotAllowedHere,
StructLiteralNotAllowedHereSugg, TildeAsUnaryOperator, UnexpectedTokenAfterLabel,
UnexpectedTokenAfterLabelSugg, WrapExpressionInParentheses,
};
use crate::maybe_recover_from_interpolated_ty_qpath;
use core::mem;
use rustc_ast::ptr::P;
use rustc_ast::token::{self, Delimiter, Token, TokenKind};
use rustc_ast::tokenstream::Spacing;
use rustc_ast::util::case::Case;
use rustc_ast::util::classify;
use rustc_ast::util::literal::LitError;
use rustc_ast::util::parser::{prec_let_scrutinee_needs_par, AssocOp, Fixity};
use rustc_ast::visit::Visitor;
use rustc_ast::{self as ast, AttrStyle, AttrVec, CaptureBy, ExprField, Lit, UnOp, DUMMY_NODE_ID};
@ -47,7 +42,7 @@ use rustc_errors::{
Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic, PResult,
StashKey,
};
use rustc_session::errors::ExprParenthesesNeeded;
use rustc_session::errors::{report_lit_error, ExprParenthesesNeeded};
use rustc_session::lint::builtin::BREAK_WITH_LABEL_AND_LOOP;
use rustc_session::lint::BuiltinLintDiagnostics;
use rustc_span::source_map::{self, Span, Spanned};
@ -1415,9 +1410,9 @@ impl<'a> Parser<'a> {
fn parse_lit_expr(&mut self) -> PResult<'a, P<Expr>> {
let lo = self.token.span;
match self.parse_opt_lit() {
Some(literal) => {
let expr = self.mk_expr(lo.to(self.prev_token.span), ExprKind::Lit(literal));
match self.parse_opt_token_lit() {
Some((token_lit, _)) => {
let expr = self.mk_expr(lo.to(self.prev_token.span), ExprKind::Lit(token_lit));
self.maybe_recover_from_bad_qpath(expr)
}
None => self.try_macro_suggestion(),
@ -1548,7 +1543,7 @@ impl<'a> Parser<'a> {
})
});
consume_colon = false;
Ok(self.mk_expr(lo, ExprKind::Lit(lit)))
Ok(self.mk_expr(lo, ExprKind::Lit(lit.token_lit)))
} else if !ate_colon
&& (self.check_noexpect(&TokenKind::Comma) || self.check_noexpect(&TokenKind::Gt))
{
@ -1625,9 +1620,9 @@ impl<'a> Parser<'a> {
/// Emit an error when a char is parsed as a lifetime because of a missing quote
pub(super) fn recover_unclosed_char(
&mut self,
&self,
lifetime: Ident,
err: impl FnOnce(&mut Self) -> DiagnosticBuilder<'a, ErrorGuaranteed>,
err: impl FnOnce(&Self) -> DiagnosticBuilder<'a, ErrorGuaranteed>,
) -> ast::Lit {
if let Some(mut diag) =
self.sess.span_diagnostic.steal_diagnostic(lifetime.span, StashKey::LifetimeIsChar)
@ -1649,9 +1644,10 @@ impl<'a> Parser<'a> {
)
.emit();
}
let name = lifetime.without_first_quote().name;
ast::Lit {
token_lit: token::Lit::new(token::LitKind::Char, lifetime.name, None),
kind: ast::LitKind::Char(lifetime.name.as_str().chars().next().unwrap_or('_')),
token_lit: token::Lit::new(token::LitKind::Char, name, None),
kind: ast::LitKind::Char(name.as_str().chars().next().unwrap_or('_')),
span: lifetime.span,
}
}
@ -1765,7 +1761,7 @@ impl<'a> Parser<'a> {
/// In case of error returns `Some(lit)` if the next token is a literal with a wrong kind,
/// and returns `None` if the next token is not literal at all.
pub fn parse_str_lit(&mut self) -> Result<ast::StrLit, Option<Lit>> {
match self.parse_opt_lit() {
match self.parse_opt_ast_lit() {
Some(lit) => match lit.kind {
ast::LitKind::Str(symbol_unescaped, style) => Ok(ast::StrLit {
style,
@ -1780,41 +1776,47 @@ impl<'a> Parser<'a> {
}
}
pub(super) fn parse_lit(&mut self) -> PResult<'a, Lit> {
self.parse_opt_lit().ok_or(()).or_else(|()| {
if let token::Interpolated(inner) = &self.token.kind {
let expr = match inner.as_ref() {
token::NtExpr(expr) => Some(expr),
token::NtLiteral(expr) => Some(expr),
_ => None,
};
if let Some(expr) = expr {
if matches!(expr.kind, ExprKind::Err) {
let mut err = InvalidInterpolatedExpression { span: self.token.span }
.into_diagnostic(&self.sess.span_diagnostic);
err.downgrade_to_delayed_bug();
return Err(err);
}
fn handle_missing_lit(&mut self) -> PResult<'a, Lit> {
if let token::Interpolated(inner) = &self.token.kind {
let expr = match inner.as_ref() {
token::NtExpr(expr) => Some(expr),
token::NtLiteral(expr) => Some(expr),
_ => None,
};
if let Some(expr) = expr {
if matches!(expr.kind, ExprKind::Err) {
let mut err = InvalidInterpolatedExpression { span: self.token.span }
.into_diagnostic(&self.sess.span_diagnostic);
err.downgrade_to_delayed_bug();
return Err(err);
}
}
let token = self.token.clone();
let err = |self_: &mut Self| {
let msg = format!("unexpected token: {}", super::token_descr(&token));
self_.struct_span_err(token.span, &msg)
};
// On an error path, eagerly consider a lifetime to be an unclosed character lit
if self.token.is_lifetime() {
let lt = self.expect_lifetime();
Ok(self.recover_unclosed_char(lt.ident, err))
} else {
Err(err(self))
}
})
}
let token = self.token.clone();
let err = |self_: &Self| {
let msg = format!("unexpected token: {}", super::token_descr(&token));
self_.struct_span_err(token.span, &msg)
};
// On an error path, eagerly consider a lifetime to be an unclosed character lit
if self.token.is_lifetime() {
let lt = self.expect_lifetime();
Ok(self.recover_unclosed_char(lt.ident, err))
} else {
Err(err(self))
}
}
/// Matches `lit = true | false | token_lit`.
/// Returns `None` if the next token is not a literal.
pub(super) fn parse_opt_lit(&mut self) -> Option<Lit> {
pub(super) fn parse_token_lit(&mut self) -> PResult<'a, (token::Lit, Span)> {
self.parse_opt_token_lit()
.ok_or(())
.or_else(|()| self.handle_missing_lit().map(|lit| (lit.token_lit, lit.span)))
}
pub(super) fn parse_ast_lit(&mut self) -> PResult<'a, Lit> {
self.parse_opt_ast_lit().ok_or(()).or_else(|()| self.handle_missing_lit())
}
fn recover_after_dot(&mut self) -> Option<Token> {
let mut recovered = None;
if self.token == token::Dot {
// Attempt to recover `.4` as `0.4`. We don't currently have any syntax where
@ -1840,100 +1842,50 @@ impl<'a> Parser<'a> {
}
}
let token = recovered.as_ref().unwrap_or(&self.token);
match Lit::from_token(token) {
Ok(lit) => {
self.bump();
Some(lit)
}
Err(LitError::NotLiteral) => None,
Err(err) => {
let span = token.span;
let token::Literal(lit) = token.kind else {
unreachable!();
};
self.bump();
self.report_lit_error(err, lit, span);
// Pack possible quotes and prefixes from the original literal into
// the error literal's symbol so they can be pretty-printed faithfully.
let suffixless_lit = token::Lit::new(lit.kind, lit.symbol, None);
let symbol = Symbol::intern(&suffixless_lit.to_string());
let lit = token::Lit::new(token::Err, symbol, lit.suffix);
Some(Lit::from_token_lit(lit, span).unwrap_or_else(|_| unreachable!()))
}
}
recovered
}
fn report_lit_error(&self, err: LitError, lit: token::Lit, span: Span) {
// Checks if `s` looks like i32 or u1234 etc.
fn looks_like_width_suffix(first_chars: &[char], s: &str) -> bool {
s.len() > 1 && s.starts_with(first_chars) && s[1..].chars().all(|c| c.is_ascii_digit())
}
/// Matches `lit = true | false | token_lit`.
/// Returns `None` if the next token is not a literal.
pub(super) fn parse_opt_token_lit(&mut self) -> Option<(token::Lit, Span)> {
let recovered = self.recover_after_dot();
let token = recovered.as_ref().unwrap_or(&self.token);
let span = token.span;
token::Lit::from_token(token).map(|token_lit| {
self.bump();
(token_lit, span)
})
}
// Try to lowercase the prefix if it's a valid base prefix.
fn fix_base_capitalisation(s: &str) -> Option<String> {
if let Some(stripped) = s.strip_prefix('B') {
Some(format!("0b{stripped}"))
} else if let Some(stripped) = s.strip_prefix('O') {
Some(format!("0o{stripped}"))
} else if let Some(stripped) = s.strip_prefix('X') {
Some(format!("0x{stripped}"))
} else {
None
}
}
let token::Lit { kind, suffix, .. } = lit;
match err {
// `NotLiteral` is not an error by itself, so we don't report
// it and give the parser opportunity to try something else.
LitError::NotLiteral => {}
// `LexerError` *is* an error, but it was already reported
// by lexer, so here we don't report it the second time.
LitError::LexerError => {}
LitError::InvalidSuffix => {
if let Some(suffix) = suffix {
self.sess.emit_err(InvalidLiteralSuffix {
span,
kind: format!("{}", kind.descr()),
suffix,
});
/// Matches `lit = true | false | token_lit`.
/// Returns `None` if the next token is not a literal.
pub(super) fn parse_opt_ast_lit(&mut self) -> Option<Lit> {
let recovered = self.recover_after_dot();
let token = recovered.as_ref().unwrap_or(&self.token);
match token::Lit::from_token(token) {
Some(token_lit) => {
match Lit::from_token_lit(token_lit, token.span) {
Ok(lit) => {
self.bump();
Some(lit)
}
Err(err) => {
let span = token.span;
let token::Literal(lit) = token.kind else {
unreachable!();
};
self.bump();
report_lit_error(&self.sess, err, lit, span);
// Pack possible quotes and prefixes from the original literal into
// the error literal's symbol so they can be pretty-printed faithfully.
let suffixless_lit = token::Lit::new(lit.kind, lit.symbol, None);
let symbol = Symbol::intern(&suffixless_lit.to_string());
let lit = token::Lit::new(token::Err, symbol, lit.suffix);
Some(Lit::from_token_lit(lit, span).unwrap_or_else(|_| unreachable!()))
}
}
}
LitError::InvalidIntSuffix => {
let suf = suffix.expect("suffix error with no suffix");
let suf = suf.as_str();
if looks_like_width_suffix(&['i', 'u'], &suf) {
// If it looks like a width, try to be helpful.
self.sess.emit_err(InvalidIntLiteralWidth { span, width: suf[1..].into() });
} else if let Some(fixed) = fix_base_capitalisation(suf) {
self.sess.emit_err(InvalidNumLiteralBasePrefix { span, fixed });
} else {
self.sess.emit_err(InvalidNumLiteralSuffix { span, suffix: suf.to_string() });
}
}
LitError::InvalidFloatSuffix => {
let suf = suffix.expect("suffix error with no suffix");
let suf = suf.as_str();
if looks_like_width_suffix(&['f'], suf) {
// If it looks like a width, try to be helpful.
self.sess
.emit_err(InvalidFloatLiteralWidth { span, width: suf[1..].to_string() });
} else {
self.sess.emit_err(InvalidFloatLiteralSuffix { span, suffix: suf.to_string() });
}
}
LitError::NonDecimalFloat(base) => {
match base {
16 => self.sess.emit_err(HexadecimalFloatLiteralNotSupported { span }),
8 => self.sess.emit_err(OctalFloatLiteralNotSupported { span }),
2 => self.sess.emit_err(BinaryFloatLiteralNotSupported { span }),
_ => unreachable!(),
};
}
LitError::IntTooLarge => {
self.sess.emit_err(IntLiteralTooLarge { span });
}
None => None,
}
}
@ -1958,8 +1910,8 @@ impl<'a> Parser<'a> {
let lo = self.token.span;
let minus_present = self.eat(&token::BinOp(token::Minus));
let lit = self.parse_lit()?;
let expr = self.mk_expr(lit.span, ExprKind::Lit(lit));
let (token_lit, span) = self.parse_token_lit()?;
let expr = self.mk_expr(span, ExprKind::Lit(token_lit));
if minus_present {
Ok(self.mk_expr(lo.to(self.prev_token.span), self.mk_unary(UnOp::Neg, expr)))

2
compiler/rustc_parse/src/parser/pat.rs
View file

@ -420,7 +420,7 @@ impl<'a> Parser<'a> {
err.span_label(self_.token.span, format!("expected {}", expected));
err
});
PatKind::Lit(self.mk_expr(lo, ExprKind::Lit(lit)))
PatKind::Lit(self.mk_expr(lo, ExprKind::Lit(lit.token_lit)))
} else {
// Try to parse everything else as literal with optional minus
match self.parse_literal_maybe_minus() {

2
compiler/rustc_parse/src/parser/stmt.rs
View file

@ -359,7 +359,7 @@ impl<'a> Parser<'a> {
/// report error for `let 1x = 123`
pub fn report_invalid_identifier_error(&mut self) -> PResult<'a, ()> {
if let token::Literal(lit) = self.token.uninterpolate().kind &&
let Err(_) = rustc_ast::Lit::from_token(&self.token) &&
rustc_ast::Lit::from_token(&self.token).is_none() &&
(lit.kind == token::LitKind::Integer || lit.kind == token::LitKind::Float) &&
self.look_ahead(1, |t| matches!(t.kind, token::Eq) || matches!(t.kind, token::Colon ) ) {
return Err(self.sess.create_err(InvalidIdentiferStartsWithNumber { span: self.token.span }));

10
compiler/rustc_parse/src/validate_attr.rs
View file

@ -49,10 +49,12 @@ pub fn parse_meta<'a>(sess: &'a ParseSess, attr: &Attribute) -> PResult<'a, Meta
MetaItemKind::List(nmis)
}
MacArgs::Eq(_, MacArgsEq::Ast(expr)) => {
if let ast::ExprKind::Lit(lit) = &expr.kind {
if !lit.kind.is_unsuffixed() {
if let ast::ExprKind::Lit(token_lit) = expr.kind
&& let Ok(lit) = ast::Lit::from_token_lit(token_lit, expr.span)
{
if token_lit.suffix.is_some() {
let mut err = sess.span_diagnostic.struct_span_err(
lit.span,
expr.span,
"suffixed literals are not allowed in attributes",
);
err.help(
@ -61,7 +63,7 @@ pub fn parse_meta<'a>(sess: &'a ParseSess, attr: &Attribute) -> PResult<'a, Meta
);
return Err(err);
} else {
MetaItemKind::NameValue(lit.clone())
MetaItemKind::NameValue(lit)
}
} else {
// The non-error case can happen with e.g. `#[foo = 1+1]`. The error case can