diff --git a/src/librustc/hir/mod.rs b/src/librustc/hir/mod.rs index 9f7fa6c5557..a59322bbe4d 100644 --- a/src/librustc/hir/mod.rs +++ b/src/librustc/hir/mod.rs @@ -1353,7 +1353,7 @@ pub struct Expr { // `Expr` is used a lot. Make sure it doesn't unintentionally get bigger. #[cfg(target_arch = "x86_64")] -static_assert!(MEM_SIZE_OF_EXPR: std::mem::size_of::() == 72); +static_assert!(MEM_SIZE_OF_EXPR: std::mem::size_of::() == 80); impl Expr { pub fn precedence(&self) -> ExprPrecedence { diff --git a/src/librustc/ich/impls_syntax.rs b/src/librustc/ich/impls_syntax.rs index 88a2c295a6e..7ff546b7467 100644 --- a/src/librustc/ich/impls_syntax.rs +++ b/src/librustc/ich/impls_syntax.rs @@ -164,6 +164,8 @@ impl_stable_hash_for!(enum ::syntax::ast::LitIntType { impl_stable_hash_for!(struct ::syntax::ast::Lit { node, + token, + suffix, span }); @@ -284,6 +286,19 @@ for tokenstream::TokenStream { } } +impl_stable_hash_for!(enum token::Lit { + Bool(val), + Byte(val), + Char(val), + Err(val), + Integer(val), + Float(val), + Str_(val), + ByteStr(val), + StrRaw(val, n), + ByteStrRaw(val, n) +}); + fn hash_token<'a, 'gcx, W: StableHasherResult>( token: &token::Token, hcx: &mut StableHashingContext<'a>, @@ -331,22 +346,8 @@ fn hash_token<'a, 'gcx, W: StableHasherResult>( token::Token::CloseDelim(delim_token) => { std_hash::Hash::hash(&delim_token, hasher); } - token::Token::Literal(ref lit, ref opt_name) => { - mem::discriminant(lit).hash_stable(hcx, hasher); - match *lit { - token::Lit::Byte(val) | - token::Lit::Char(val) | - token::Lit::Err(val) | - token::Lit::Integer(val) | - token::Lit::Float(val) | - token::Lit::Str_(val) | - token::Lit::ByteStr(val) => val.hash_stable(hcx, hasher), - token::Lit::StrRaw(val, n) | - token::Lit::ByteStrRaw(val, n) => { - val.hash_stable(hcx, hasher); - n.hash_stable(hcx, hasher); - } - }; + token::Token::Literal(lit, opt_name) => { + lit.hash_stable(hcx, hasher); opt_name.hash_stable(hcx, hasher); } diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs index d66455f91ba..5bb06516ac4 100644 --- a/src/librustdoc/html/highlight.rs +++ b/src/librustdoc/html/highlight.rs @@ -318,6 +318,8 @@ impl<'a> Classifier<'a> { // Number literals. token::Integer(..) | token::Float(..) => Class::Number, + + token::Bool(..) => panic!("literal token contains `Lit::Bool`"), } } diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 783792cf197..04bc146e145 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -6,6 +6,7 @@ pub use crate::symbol::{Ident, Symbol as Name}; pub use crate::util::parser::ExprPrecedence; use crate::ext::hygiene::{Mark, SyntaxContext}; +use crate::parse::token; use crate::print::pprust; use crate::ptr::P; use crate::source_map::{dummy_spanned, respan, Spanned}; @@ -1354,6 +1355,8 @@ pub enum StrStyle { #[derive(Clone, RustcEncodable, RustcDecodable, Debug, Hash, PartialEq)] pub struct Lit { pub node: LitKind, + pub token: token::Lit, + pub suffix: Option, pub span: Span, } diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index e331a263354..2b874581083 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -350,7 +350,9 @@ impl Attribute { /* Constructors */ pub fn mk_name_value_item_str(ident: Ident, value: Spanned) -> MetaItem { - let value = Lit { node: LitKind::Str(value.node, ast::StrStyle::Cooked), span: value.span }; + let node = LitKind::Str(value.node, ast::StrStyle::Cooked); + let (token, suffix) = node.lit_token(); + let value = Lit { node, token, suffix, span: value.span }; mk_name_value_item(ident.span.to(value.span), ident, value) } @@ -417,7 +419,9 @@ pub fn mk_spanned_attr_outer(sp: Span, id: AttrId, item: MetaItem) -> Attribute pub fn mk_sugared_doc_attr(id: AttrId, text: Symbol, span: Span) -> Attribute { let style = doc_comment_style(&text.as_str()); - let lit = Lit { node: LitKind::Str(text, ast::StrStyle::Cooked), span }; + let node = LitKind::Str(text, ast::StrStyle::Cooked); + let (token, suffix) = node.lit_token(); + let lit = Lit { node, token, suffix, span }; Attribute { id, style, @@ -562,7 +566,7 @@ impl MetaItemKind { tokens.next(); return if let Some(TokenTree::Token(span, token)) = tokens.next() { LitKind::from_token(token) - .map(|node| MetaItemKind::NameValue(Lit { node, span })) + .map(|(node, token, suffix)| MetaItemKind::NameValue(Lit { node, token, suffix, span })) } else { None }; @@ -607,9 +611,9 @@ impl NestedMetaItem { where I: Iterator, { if let Some(TokenTree::Token(span, token)) = tokens.peek().cloned() { - if let Some(node) = LitKind::from_token(token) { + if let Some((node, token, suffix)) = LitKind::from_token(token) { tokens.next(); - return Some(NestedMetaItem::Literal(Lit { node, span })); + return Some(NestedMetaItem::Literal(Lit { node, token, suffix, span })); } } @@ -625,28 +629,35 @@ impl Lit { impl LitKind { fn token(&self) -> Token { + match self.lit_token() { + (token::Bool(symbol), _) => Token::Ident(Ident::with_empty_ctxt(symbol), false), + (lit, suffix) => Token::Literal(lit, suffix), + } + } + + pub(crate) fn lit_token(&self) -> (token::Lit, Option) { use std::ascii; match *self { LitKind::Str(string, ast::StrStyle::Cooked) => { let escaped = string.as_str().escape_default().to_string(); - Token::Literal(token::Lit::Str_(Symbol::intern(&escaped)), None) + (token::Lit::Str_(Symbol::intern(&escaped)), None) } LitKind::Str(string, ast::StrStyle::Raw(n)) => { - Token::Literal(token::Lit::StrRaw(string, n), None) + (token::Lit::StrRaw(string, n), None) } LitKind::ByteStr(ref bytes) => { let string = bytes.iter().cloned().flat_map(ascii::escape_default) .map(Into::::into).collect::(); - Token::Literal(token::Lit::ByteStr(Symbol::intern(&string)), None) + (token::Lit::ByteStr(Symbol::intern(&string)), None) } LitKind::Byte(byte) => { let string: String = ascii::escape_default(byte).map(Into::::into).collect(); - Token::Literal(token::Lit::Byte(Symbol::intern(&string)), None) + (token::Lit::Byte(Symbol::intern(&string)), None) } LitKind::Char(ch) => { let string: String = ch.escape_default().map(Into::::into).collect(); - Token::Literal(token::Lit::Char(Symbol::intern(&string)), None) + (token::Lit::Char(Symbol::intern(&string)), None) } LitKind::Int(n, ty) => { let suffix = match ty { @@ -654,38 +665,39 @@ impl LitKind { ast::LitIntType::Signed(ty) => Some(Symbol::intern(ty.ty_to_string())), ast::LitIntType::Unsuffixed => None, }; - Token::Literal(token::Lit::Integer(Symbol::intern(&n.to_string())), suffix) + (token::Lit::Integer(Symbol::intern(&n.to_string())), suffix) } LitKind::Float(symbol, ty) => { - Token::Literal(token::Lit::Float(symbol), Some(Symbol::intern(ty.ty_to_string()))) + (token::Lit::Float(symbol), Some(Symbol::intern(ty.ty_to_string()))) } - LitKind::FloatUnsuffixed(symbol) => Token::Literal(token::Lit::Float(symbol), None), - LitKind::Bool(value) => Token::Ident(Ident::with_empty_ctxt(Symbol::intern(if value { - "true" - } else { - "false" - })), false), - LitKind::Err(val) => Token::Literal(token::Lit::Err(val), None), + LitKind::FloatUnsuffixed(symbol) => (token::Lit::Float(symbol), None), + LitKind::Bool(value) => { + let kw = if value { keywords::True } else { keywords::False }; + (token::Lit::Bool(kw.name()), None) + } + LitKind::Err(val) => (token::Lit::Err(val), None), } } - fn from_token(token: Token) -> Option { + fn from_token(token: Token) -> Option<(LitKind, token::Lit, Option)> { match token { - Token::Ident(ident, false) if ident.name == "true" => Some(LitKind::Bool(true)), - Token::Ident(ident, false) if ident.name == "false" => Some(LitKind::Bool(false)), + Token::Ident(ident, false) if ident.name == keywords::True.name() => + Some((LitKind::Bool(true), token::Bool(ident.name), None)), + Token::Ident(ident, false) if ident.name == keywords::False.name() => + Some((LitKind::Bool(false), token::Bool(ident.name), None)), Token::Interpolated(nt) => match *nt { token::NtExpr(ref v) | token::NtLiteral(ref v) => match v.node { - ExprKind::Lit(ref lit) => Some(lit.node.clone()), + ExprKind::Lit(ref lit) => Some((lit.node.clone(), lit.token, lit.suffix)), _ => None, }, _ => None, }, Token::Literal(lit, suf) => { let (suffix_illegal, result) = parse::lit_token(lit, suf, None); - if suffix_illegal && suf.is_some() { + if result.is_none() || suffix_illegal && suf.is_some() { return None; } - result + Some((result.unwrap(), lit, suf)) } _ => None, } diff --git a/src/libsyntax/ext/build.rs b/src/libsyntax/ext/build.rs index 0fe85361b54..0f9977e8503 100644 --- a/src/libsyntax/ext/build.rs +++ b/src/libsyntax/ext/build.rs @@ -698,7 +698,8 @@ impl<'a> AstBuilder for ExtCtxt<'a> { } fn expr_lit(&self, span: Span, node: ast::LitKind) -> P { - self.expr(span, ast::ExprKind::Lit(ast::Lit { node, span })) + let (token, suffix) = node.lit_token(); + self.expr(span, ast::ExprKind::Lit(ast::Lit { node, token, suffix, span })) } fn expr_usize(&self, span: Span, i: usize) -> P { self.expr_lit(span, ast::LitKind::Int(i as u128, @@ -1166,8 +1167,9 @@ impl<'a> AstBuilder for ExtCtxt<'a> { fn meta_name_value(&self, span: Span, name: ast::Name, node: ast::LitKind) -> ast::MetaItem { + let (token, suffix) = node.lit_token(); attr::mk_name_value_item(span, Ident::with_empty_ctxt(name).with_span_pos(span), - ast::Lit { node, span }) + ast::Lit { node, token, suffix, span }) } fn item_use(&self, sp: Span, diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index be44b964ba5..4d4e99009a9 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -376,6 +376,7 @@ crate fn lit_token(lit: token::Lit, suf: Option, diag: Option<(Span, &Ha use ast::LitKind; match lit { + token::Bool(_) => panic!("literal token contains `Lit::Bool`"), token::Byte(i) => { let lit_kind = match unescape_byte(&i.as_str()) { Ok(c) => LitKind::Byte(c), diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 2b30d2db95e..b988cb1447d 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -2070,11 +2070,11 @@ impl<'a> Parser<'a> { } /// Matches `token_lit = LIT_INTEGER | ...`. - fn parse_lit_token(&mut self) -> PResult<'a, LitKind> { + fn parse_lit_token(&mut self) -> PResult<'a, (LitKind, token::Lit, Option)> { let out = match self.token { token::Interpolated(ref nt) => match **nt { token::NtExpr(ref v) | token::NtLiteral(ref v) => match v.node { - ExprKind::Lit(ref lit) => { lit.node.clone() } + ExprKind::Lit(ref lit) => { (lit.node.clone(), lit.token, lit.suffix) } _ => { return self.unexpected_last(&self.token); } }, _ => { return self.unexpected_last(&self.token); } @@ -2088,19 +2088,19 @@ impl<'a> Parser<'a> { self.expect_no_suffix(sp, &format!("a {}", lit.literal_name()), suf) } - result.unwrap() + (result.unwrap(), lit, suf) } token::Dot if self.look_ahead(1, |t| match t { - token::Literal(parse::token::Lit::Integer(_) , _) => true, + token::Literal(token::Lit::Integer(_) , _) => true, _ => false, }) => { // recover from `let x = .4;` let lo = self.span; self.bump(); if let token::Literal( - parse::token::Lit::Integer(val), + token::Lit::Integer(val), suffix, ) = self.token { - let suffix = suffix.and_then(|s| { + let float_suffix = suffix.and_then(|s| { let s = s.as_str(); if s == "f32" { Some("f32") @@ -2117,14 +2117,14 @@ impl<'a> Parser<'a> { err.span_suggestion( sp, "must have an integer part", - format!("0.{}{}", val, suffix), + format!("0.{}{}", val, float_suffix), Applicability::MachineApplicable, ); err.emit(); - return Ok(match suffix { - "f32" => ast::LitKind::Float(val, ast::FloatTy::F32), - "f64" => ast::LitKind::Float(val, ast::FloatTy::F64), - _ => ast::LitKind::FloatUnsuffixed(val), + return Ok(match float_suffix { + "f32" => (ast::LitKind::Float(val, ast::FloatTy::F32), token::Float(val), suffix), + "f64" => (ast::LitKind::Float(val, ast::FloatTy::F64), token::Float(val), suffix), + _ => (ast::LitKind::FloatUnsuffixed(val), token::Float(val), suffix), }); } else { unreachable!(); @@ -2140,14 +2140,14 @@ impl<'a> Parser<'a> { /// Matches `lit = true | false | token_lit`. crate fn parse_lit(&mut self) -> PResult<'a, Lit> { let lo = self.span; - let node = if self.eat_keyword(keywords::True) { - LitKind::Bool(true) + let (node, token, suffix) = if self.eat_keyword(keywords::True) { + (LitKind::Bool(true), token::Bool(keywords::True.name()), None) } else if self.eat_keyword(keywords::False) { - LitKind::Bool(false) + (LitKind::Bool(false), token::Bool(keywords::False.name()), None) } else { self.parse_lit_token()? }; - Ok(Lit { node, span: lo.to(self.prev_span) }) + Ok(Lit { node, token, suffix, span: lo.to(self.prev_span) }) } /// Matches `'-' lit | lit` (cf. `ast_validation::AstValidator::check_expr_within_pat`). diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index fd7a39c576d..48a949257ff 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -61,6 +61,7 @@ impl DelimToken { #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] pub enum Lit { + Bool(ast::Name), // AST only, must never appear in a `Token` Byte(ast::Name), Char(ast::Name), Err(ast::Name), @@ -72,9 +73,13 @@ pub enum Lit { ByteStrRaw(ast::Name, u16), /* raw byte str delimited by n hash symbols */ } +#[cfg(target_arch = "x86_64")] +static_assert!(MEM_SIZE_OF_LIT: mem::size_of::() == 8); + impl Lit { crate fn literal_name(&self) -> &'static str { match *self { + Bool(_) => panic!("literal token contains `Lit::Bool`"), Byte(_) => "byte literal", Char(_) => "char literal", Err(_) => "invalid literal", diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 682621d40ab..0e93b857708 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -225,6 +225,7 @@ pub fn token_to_string(tok: &Token) -> String { /* Literals */ token::Literal(lit, suf) => { let mut out = match lit { + token::Bool(_) => panic!("literal token contains `Lit::Bool`"), token::Byte(b) => format!("b'{}'", b), token::Char(c) => format!("'{}'", c), token::Err(c) => format!("'{}'", c),