bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Use token::Lit in ast::ExprKind::Lit.

Browse source 
Instead of `ast::Lit`.

Literal lowering now happens at two different times. Expression literals
are lowered when HIR is crated. Attribute literals are lowered during
parsing.

This commit changes the language very slightly. Some programs that used
to not compile now will compile. This is because some invalid literals
that are removed by `cfg` or attribute macros will no longer trigger
errors. See this comment for more details:
https://github.com/rust-lang/rust/pull/102944#issuecomment-1277476773
This commit is contained in:
Nicholas Nethercote 2022-10-10 13:40:56 +11:00
parent 01760265cb
commit 358a603f11
45 changed files with 701 additions and 581 deletions
Download patch file Download diff file Expand all files Collapse all files

24
compiler/rustc_ast/src/ast.rs
View file

@ -1332,7 +1332,7 @@ pub enum ExprKind {
/// A unary operation (e.g., `!x`, `*x`).
Unary(UnOp, P<Expr>),
/// A literal (e.g., `1`, `"foo"`).
Lit(Lit),
Lit(token::Lit),
/// A cast (e.g., `foo as f64`).
Cast(P<Expr>, P<Ty>),
/// A type ascription (e.g., `42: usize`).
@ -1698,16 +1698,12 @@ pub struct StrLit {
}
impl StrLit {
pub fn as_lit(&self) -> Lit {
pub fn as_token_lit(&self) -> token::Lit {
let token_kind = match self.style {
StrStyle::Cooked => token::Str,
StrStyle::Raw(n) => token::StrRaw(n),
};
Lit {
token_lit: token::Lit::new(token_kind, self.symbol, self.suffix),
span: self.span,
kind: LitKind::Str(self.symbol_unescaped, self.style),
}
token::Lit::new(token_kind, self.symbol, self.suffix)
}
}
@ -1733,9 +1729,10 @@ pub enum LitFloatType {
Unsuffixed,
}
/// Literal kind.
///
/// E.g., `"foo"`, `42`, `12.34`, or `bool`.
/// Note that the entire literal (including the suffix) is considered when
/// deciding the `LitKind`. This means that float literals like `1f32` are
/// classified by this type as `Float`. This is different to `token::LitKind`
/// which does *not* consider the suffix.
#[derive(Clone, Encodable, Decodable, Debug, Hash, Eq, PartialEq, HashStable_Generic)]
pub enum LitKind {
/// A string literal (`"foo"`). The symbol is unescaped, and so may differ
@ -1749,10 +1746,11 @@ pub enum LitKind {
Char(char),
/// An integer literal (`1`).
Int(u128, LitIntType),
/// A float literal (`1f64` or `1E10f64`). Stored as a symbol rather than
/// `f64` so that `LitKind` can impl `Eq` and `Hash`.
/// A float literal (`1.0`, `1f64` or `1E10f64`). The pre-suffix part is
/// stored as a symbol rather than `f64` so that `LitKind` can impl `Eq`
/// and `Hash`.
Float(Symbol, LitFloatType),
/// A boolean literal.
/// A boolean literal (`true`, `false`).
Bool(bool),
/// Placeholder for a literal that wasn't well-formed in some way.
Err,

12
compiler/rustc_ast/src/attr/mod.rs
View file

@ -533,7 +533,7 @@ impl MetaItemKind {
MetaItemKind::NameValue(lit) => {
let expr = P(ast::Expr {
id: ast::DUMMY_NODE_ID,
kind: ast::ExprKind::Lit(lit.clone()),
kind: ast::ExprKind::Lit(lit.token_lit.clone()),
span: lit.span,
attrs: ast::AttrVec::new(),
tokens: None,
@ -605,7 +605,7 @@ impl MetaItemKind {
MetaItemKind::name_value_from_tokens(&mut inner_tokens.into_trees())
}
Some(TokenTree::Token(token, _)) => {
Lit::from_token(&token).ok().map(MetaItemKind::NameValue)
Lit::from_token(&token).map(MetaItemKind::NameValue)
}
_ => None,
}
@ -618,8 +618,10 @@ impl MetaItemKind {
MetaItemKind::list_from_tokens(tokens.clone())
}
MacArgs::Delimited(..) => None,
MacArgs::Eq(_, MacArgsEq::Ast(expr)) => match &expr.kind {
ast::ExprKind::Lit(lit) => Some(MetaItemKind::NameValue(lit.clone())),
MacArgs::Eq(_, MacArgsEq::Ast(expr)) => match expr.kind {
ast::ExprKind::Lit(token_lit) => Some(MetaItemKind::NameValue(
Lit::from_token_lit(token_lit, expr.span).expect("token_lit in from_mac_args"),
)),
_ => None,
},
MacArgs::Eq(_, MacArgsEq::Hir(lit)) => Some(MetaItemKind::NameValue(lit.clone())),
@ -668,7 +670,7 @@ impl NestedMetaItem {
{
match tokens.peek() {
Some(TokenTree::Token(token, _))
if let Ok(lit) = Lit::from_token(token) =>
if let Some(lit) = Lit::from_token(token) =>
{
tokens.next();
return Some(NestedMetaItem::Literal(lit));

50
compiler/rustc_ast/src/token.rs
View file

@ -59,13 +59,17 @@ pub enum Delimiter {
Invisible,
}
// Note that the suffix is *not* considered when deciding the `LitKind` in this
// type. This means that float literals like `1f32` are classified by this type
// as `Int`. Only upon conversion to `ast::LitKind` will such a literal be
// given the `Float` kind.
#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
pub enum LitKind {
Bool, // AST only, must never appear in a `Token`
Byte,
Char,
Integer,
Float,
Integer, // e.g. `1`, `1u8`, `1f32`
Float, // e.g. `1.`, `1.0`, `1e3f32`
Str,
StrRaw(u8), // raw string delimited by `n` hash symbols
ByteStr,
@ -81,6 +85,42 @@ pub struct Lit {
pub suffix: Option<Symbol>,
}
impl Lit {
pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit {
Lit { kind, symbol, suffix }
}
/// Returns `true` if this is semantically a float literal. This includes
/// ones like `1f32` that have an `Integer` kind but a float suffix.
pub fn is_semantic_float(&self) -> bool {
match self.kind {
LitKind::Float => true,
LitKind::Integer => match self.suffix {
Some(sym) => sym == sym::f32 || sym == sym::f64,
None => false,
},
_ => false,
}
}
/// Keep this in sync with `Token::can_begin_literal_or_bool` excluding unary negation.
pub fn from_token(token: &Token) -> Option<Lit> {
match token.uninterpolate().kind {
Ident(name, false) if name.is_bool_lit() => {
Some(Lit::new(Bool, name, None))
}
Literal(token_lit) => Some(token_lit),
Interpolated(ref nt)
if let NtExpr(expr) | NtLiteral(expr) = &**nt
&& let ast::ExprKind::Lit(token_lit) = expr.kind =>
{
Some(token_lit.clone())
}
_ => None,
}
}
}
impl fmt::Display for Lit {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let Lit { kind, symbol, suffix } = *self;
@ -139,12 +179,6 @@ impl LitKind {
}
}
impl Lit {
pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit {
Lit { kind, symbol, suffix }
}
}
pub fn ident_can_begin_expr(name: Symbol, span: Span, is_raw: bool) -> bool {
let ident_token = Token::new(Ident(name, is_raw), span);

27
compiler/rustc_ast/src/util/literal.rs
View file

@ -8,8 +8,8 @@ use rustc_span::symbol::{kw, sym, Symbol};
use rustc_span::Span;
use std::ascii;
#[derive(Debug)]
pub enum LitError {
NotLiteral,
LexerError,
InvalidSuffix,
InvalidIntSuffix,
@ -202,27 +202,10 @@ impl Lit {
Ok(Lit { token_lit, kind: LitKind::from_token_lit(token_lit)?, span })
}
/// Converts arbitrary token into an AST literal.
///
/// Keep this in sync with `Token::can_begin_literal_or_bool` excluding unary negation.
pub fn from_token(token: &Token) -> Result<Lit, LitError> {
let lit = match token.uninterpolate().kind {
token::Ident(name, false) if name.is_bool_lit() => {
token::Lit::new(token::Bool, name, None)
}
token::Literal(lit) => lit,
token::Interpolated(ref nt) => {
if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt
&& let ast::ExprKind::Lit(lit) = &expr.kind
{
return Ok(lit.clone());
}
return Err(LitError::NotLiteral);
}
_ => return Err(LitError::NotLiteral),
};
Lit::from_token_lit(lit, token.span)
/// Converts an arbitrary token into an AST literal.
pub fn from_token(token: &Token) -> Option<Lit> {
token::Lit::from_token(token)
.and_then(|token_lit| Lit::from_token_lit(token_lit, token.span).ok())
}
/// Attempts to recover an AST literal from semantic literal.