bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Auto merge of #105160 - nnethercote:rm-Lit-token_lit, r=petrochenkov

Browse source 
Remove `token::Lit` from `ast::MetaItemLit`.

Currently `ast::MetaItemLit` represents the literal kind twice. This PR removes that redundancy. Best reviewed one commit at a time.

r? `@petrochenkov`
This commit is contained in:
bors 2022-12-12 05:16:50 +00:00
commit 2cd2070af7
28 changed files with 244 additions and 171 deletions
Download patch file Download diff file Expand all files Collapse all files

24
compiler/rustc_ast/src/ast.rs
View file

@ -1734,8 +1734,10 @@ pub enum StrStyle {
/// A literal in a meta item.
#[derive(Clone, Encodable, Decodable, Debug, HashStable_Generic)]
pub struct MetaItemLit {
/// The original literal token as written in source code.
pub token_lit: token::Lit,
/// The original literal as written in the source code.
pub symbol: Symbol,
/// The original suffix as written in the source code.
pub suffix: Option<Symbol>,
/// The "semantic" representation of the literal lowered from the original tokens.
/// Strings are unescaped, hexadecimal forms are eliminated, etc.
pub kind: LitKind,
@ -1745,13 +1747,14 @@ pub struct MetaItemLit {
/// Similar to `MetaItemLit`, but restricted to string literals.
#[derive(Clone, Copy, Encodable, Decodable, Debug)]
pub struct StrLit {
/// The original literal token as written in source code.
pub style: StrStyle,
/// The original literal as written in source code.
pub symbol: Symbol,
/// The original suffix as written in source code.
pub suffix: Option<Symbol>,
pub span: Span,
/// The unescaped "semantic" representation of the literal lowered from the original token.
/// The semantic (unescaped) representation of the literal.
pub symbol_unescaped: Symbol,
pub style: StrStyle,
pub span: Span,
}
impl StrLit {
@ -1797,8 +1800,9 @@ pub enum LitKind {
/// A string literal (`"foo"`). The symbol is unescaped, and so may differ
/// from the original token's symbol.
Str(Symbol, StrStyle),
/// A byte string (`b"foo"`).
ByteStr(Lrc<[u8]>),
/// A byte string (`b"foo"`). Not stored as a symbol because it might be
/// non-utf8, and symbols only allow utf8 strings.
ByteStr(Lrc<[u8]>, StrStyle),
/// A byte char (`b'f'`).
Byte(u8),
/// A character literal (`'a'`).
@ -1823,7 +1827,7 @@ impl LitKind {
/// Returns `true` if this literal is byte literal string.
pub fn is_bytestr(&self) -> bool {
matches!(self, LitKind::ByteStr(_))
matches!(self, LitKind::ByteStr(..))
}
/// Returns `true` if this is a numeric literal.
@ -3100,7 +3104,7 @@ mod size_asserts {
static_assert_size!(ItemKind, 112);
static_assert_size!(LitKind, 24);
static_assert_size!(Local, 72);
static_assert_size!(MetaItemLit, 48);
static_assert_size!(MetaItemLit, 40);
static_assert_size!(Param, 40);
static_assert_size!(Pat, 88);
static_assert_size!(Path, 24);

18
compiler/rustc_ast/src/attr/mod.rs
View file

@ -1,15 +1,15 @@
//! Functions dealing with attributes and meta items.
use crate::ast;
use crate::ast::{AttrArgs, AttrArgsEq, AttrId, AttrItem, AttrKind, AttrStyle, AttrVec, Attribute};
use crate::ast::{DelimArgs, Expr, ExprKind, LitKind, MetaItemLit};
use crate::ast::{MacDelimiter, MetaItem, MetaItemKind, NestedMetaItem, NormalAttr};
use crate::ast::{Path, PathSegment, StrStyle, DUMMY_NODE_ID};
use crate::ast::{Path, PathSegment, DUMMY_NODE_ID};
use crate::ptr::P;
use crate::token::{self, CommentKind, Delimiter, Token};
use crate::tokenstream::{DelimSpan, Spacing, TokenTree};
use crate::tokenstream::{LazyAttrTokenStream, TokenStream};
use crate::util::comments;
use crate::util::literal::escape_string_symbol;
use rustc_data_structures::sync::WorkerLocal;
use rustc_index::bit_set::GrowableBitSet;
use rustc_span::symbol::{sym, Ident, Symbol};
@ -321,18 +321,6 @@ impl Attribute {
}
}
/* Constructors */
pub fn mk_name_value_item_str(ident: Ident, str: Symbol, str_span: Span) -> MetaItem {
mk_name_value_item(ident, LitKind::Str(str, ast::StrStyle::Cooked), str_span)
}
pub fn mk_name_value_item(ident: Ident, kind: LitKind, lit_span: Span) -> MetaItem {
let lit = MetaItemLit { token_lit: kind.to_token_lit(), kind, span: lit_span };
let span = ident.span.to(lit_span);
MetaItem { path: Path::from_ident(ident), kind: MetaItemKind::NameValue(lit), span }
}
pub struct AttrIdGenerator(WorkerLocal<Cell<u32>>);
#[cfg(debug_assertions)]
@ -408,7 +396,7 @@ pub fn mk_attr_name_value_str(
val: Symbol,
span: Span,
) -> Attribute {
let lit = LitKind::Str(val, StrStyle::Cooked).to_token_lit();
let lit = token::Lit::new(token::Str, escape_string_symbol(val), None);
let expr = P(Expr {
id: DUMMY_NODE_ID,
kind: ExprKind::Lit(lit),

142
compiler/rustc_ast/src/util/literal.rs
View file

@ -1,11 +1,31 @@
//! Code related to parsing literals.
use crate::ast::{self, LitKind, MetaItemLit};
use crate::ast::{self, LitKind, MetaItemLit, StrStyle};
use crate::token::{self, Token};
use rustc_lexer::unescape::{byte_from_char, unescape_byte, unescape_char, unescape_literal, Mode};
use rustc_span::symbol::{kw, sym, Symbol};
use rustc_span::Span;
use std::ascii;
use std::{ascii, fmt, str};
// Escapes a string, represented as a symbol. Reuses the original symbol,
// avoiding interning, if no changes are required.
pub fn escape_string_symbol(symbol: Symbol) -> Symbol {
let s = symbol.as_str();
let escaped = s.escape_default().to_string();
if s == escaped { symbol } else { Symbol::intern(&escaped) }
}
// Escapes a char.
pub fn escape_char_symbol(ch: char) -> Symbol {
let s: String = ch.escape_default().map(Into::<char>::into).collect();
Symbol::intern(&s)
}
// Escapes a byte string.
pub fn escape_byte_str_symbol(bytes: &[u8]) -> Symbol {
let s = bytes.escape_ascii().to_string();
Symbol::intern(&s)
}
#[derive(Debug)]
pub enum LitError {
@ -115,9 +135,9 @@ impl LitKind {
}
});
error?;
LitKind::ByteStr(buf.into())
LitKind::ByteStr(buf.into(), StrStyle::Cooked)
}
token::ByteStrRaw(_) => {
token::ByteStrRaw(n) => {
let s = symbol.as_str();
let bytes = if s.contains('\r') {
let mut buf = Vec::with_capacity(s.len());
@ -136,69 +156,95 @@ impl LitKind {
symbol.to_string().into_bytes()
};
LitKind::ByteStr(bytes.into())
LitKind::ByteStr(bytes.into(), StrStyle::Raw(n))
}
token::Err => LitKind::Err,
})
}
}
/// Attempts to recover a token from semantic literal.
/// This function is used when the original token doesn't exist (e.g. the literal is created
/// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
pub fn to_token_lit(&self) -> token::Lit {
let (kind, symbol, suffix) = match *self {
LitKind::Str(symbol, ast::StrStyle::Cooked) => {
// Don't re-intern unless the escaped string is different.
let s = symbol.as_str();
let escaped = s.escape_default().to_string();
let symbol = if s == escaped { symbol } else { Symbol::intern(&escaped) };
(token::Str, symbol, None)
impl fmt::Display for LitKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match *self {
LitKind::Byte(b) => {
let b: String = ascii::escape_default(b).map(Into::<char>::into).collect();
write!(f, "b'{}'", b)?;
}
LitKind::Str(symbol, ast::StrStyle::Raw(n)) => (token::StrRaw(n), symbol, None),
LitKind::ByteStr(ref bytes) => {
let string = bytes.escape_ascii().to_string();
(token::ByteStr, Symbol::intern(&string), None)
LitKind::Char(ch) => write!(f, "'{}'", escape_char_symbol(ch))?,
LitKind::Str(sym, StrStyle::Cooked) => write!(f, "\"{}\"", escape_string_symbol(sym))?,
LitKind::Str(sym, StrStyle::Raw(n)) => write!(
f,
"r{delim}\"{string}\"{delim}",
delim = "#".repeat(n as usize),
string = sym
)?,
LitKind::ByteStr(ref bytes, StrStyle::Cooked) => {
write!(f, "b\"{}\"", escape_byte_str_symbol(bytes))?
}
LitKind::Byte(byte) => {
let string: String = ascii::escape_default(byte).map(Into::<char>::into).collect();
(token::Byte, Symbol::intern(&string), None)
}
LitKind::Char(ch) => {
let string: String = ch.escape_default().map(Into::<char>::into).collect();
(token::Char, Symbol::intern(&string), None)
LitKind::ByteStr(ref bytes, StrStyle::Raw(n)) => {
// Unwrap because raw byte string literals can only contain ASCII.
let symbol = str::from_utf8(bytes).unwrap();
write!(
f,
"br{delim}\"{string}\"{delim}",
delim = "#".repeat(n as usize),
string = symbol
)?;
}
LitKind::Int(n, ty) => {
let suffix = match ty {
ast::LitIntType::Unsigned(ty) => Some(ty.name()),
ast::LitIntType::Signed(ty) => Some(ty.name()),
ast::LitIntType::Unsuffixed => None,
};
(token::Integer, sym::integer(n), suffix)
write!(f, "{}", n)?;
match ty {
ast::LitIntType::Unsigned(ty) => write!(f, "{}", ty.name())?,
ast::LitIntType::Signed(ty) => write!(f, "{}", ty.name())?,
ast::LitIntType::Unsuffixed => {}
}
}
LitKind::Float(symbol, ty) => {
let suffix = match ty {
ast::LitFloatType::Suffixed(ty) => Some(ty.name()),
ast::LitFloatType::Unsuffixed => None,
};
(token::Float, symbol, suffix)
write!(f, "{}", symbol)?;
match ty {
ast::LitFloatType::Suffixed(ty) => write!(f, "{}", ty.name())?,
ast::LitFloatType::Unsuffixed => {}
}
}
LitKind::Bool(value) => {
let symbol = if value { kw::True } else { kw::False };
(token::Bool, symbol, None)
LitKind::Bool(b) => write!(f, "{}", if b { "true" } else { "false" })?,
LitKind::Err => {
// This only shows up in places like `-Zunpretty=hir` output, so we
// don't bother to produce something useful.
write!(f, "<bad-literal>")?;
}
// This only shows up in places like `-Zunpretty=hir` output, so we
// don't bother to produce something useful.
LitKind::Err => (token::Err, Symbol::intern("<bad-literal>"), None),
};
}
token::Lit::new(kind, symbol, suffix)
Ok(())
}
}
impl MetaItemLit {
/// Converts token literal into a meta item literal.
/// Converts a token literal into a meta item literal.
pub fn from_token_lit(token_lit: token::Lit, span: Span) -> Result<MetaItemLit, LitError> {
Ok(MetaItemLit { token_lit, kind: LitKind::from_token_lit(token_lit)?, span })
Ok(MetaItemLit {
symbol: token_lit.symbol,
suffix: token_lit.suffix,
kind: LitKind::from_token_lit(token_lit)?,
span,
})
}
/// Cheaply converts a meta item literal into a token literal.
pub fn as_token_lit(&self) -> token::Lit {
let kind = match self.kind {
LitKind::Bool(_) => token::Bool,
LitKind::Str(_, ast::StrStyle::Cooked) => token::Str,
LitKind::Str(_, ast::StrStyle::Raw(n)) => token::StrRaw(n),
LitKind::ByteStr(_, ast::StrStyle::Cooked) => token::ByteStr,
LitKind::ByteStr(_, ast::StrStyle::Raw(n)) => token::ByteStrRaw(n),
LitKind::Byte(_) => token::Byte,
LitKind::Char(_) => token::Char,
LitKind::Int(..) => token::Integer,
LitKind::Float(..) => token::Float,
LitKind::Err => token::Err,
};
token::Lit::new(kind, self.symbol, self.suffix)
}
/// Converts an arbitrary token into meta item literal.