syntax: don't parse numeric literals in the lexer

This removes a bunch of token types. Tokens now store the original, unaltered
numeric literal (that is still checked for correctness), which is parsed into
an actual number later, as needed, when creating the AST.

This can change how syntax extensions work, but otherwise poses no visible
changes.

[breaking-change]

src/librustc/middle/trans/consts.rs

@@ -42,6 +42,7 @@ use syntax::{ast, ast_util};
 pub fn const_lit(cx: &CrateContext, e: &ast::Expr, lit: ast::Lit)
     -> ValueRef {
     let _icx = push_ctxt("trans_lit");
+    debug!("const_lit: {}", lit);
     match lit.node {
         ast::LitByte(b) => C_integral(Type::uint_from_ty(cx, ast::TyU8), b as u64, false),
         ast::LitChar(i) => C_integral(Type::char(cx), i as u64, false),

src/librustdoc/html/highlight.rs

@@ -144,8 +144,7 @@ fn doit(sess: &parse::ParseSess, mut lexer: lexer::StringReader,
                 t::LIT_CHAR(..) | t::LIT_STR(..) | t::LIT_STR_RAW(..) => "string",
 
             // number literals
-            t::LIT_INT(..) | t::LIT_UINT(..) | t::LIT_INT_UNSUFFIXED(..) |
+            t::LIT_INTEGER(..) | t::LIT_FLOAT(..) => "number",
-                t::LIT_FLOAT(..) | t::LIT_FLOAT_UNSUFFIXED(..) => "number",
 
             // keywords are also included in the identifier set
             t::IDENT(ident, _is_mod_sep) => {

src/libsyntax/ast.rs

@@ -619,7 +619,7 @@ pub enum Mac_ {
     MacInvocTT(Path, Vec<TokenTree> , SyntaxContext),   // new macro-invocation
 }
 
-#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash)]
+#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)]
 pub enum StrStyle {
     CookedStr,
     RawStr(uint)
@@ -627,7 +627,7 @@ pub enum StrStyle {
 
 pub type Lit = Spanned<Lit_>;
 
-#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash)]
+#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)]
 pub enum Lit_ {
     LitStr(InternedString, StrStyle),
     LitBinary(Rc<Vec<u8> >),
@@ -697,6 +697,16 @@ impl fmt::Show for IntTy {
     }
 }
 
+impl IntTy {
+    pub fn suffix_len(&self) -> uint {
+        match *self {
+            TyI => 1,
+            TyI8 => 2,
+            TyI16 | TyI32 | TyI64  => 3,
+        }
+    }
+}
+
 #[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash)]
 pub enum UintTy {
     TyU,
@@ -706,6 +716,16 @@ pub enum UintTy {
     TyU64,
 }
 
+impl UintTy {
+    pub fn suffix_len(&self) -> uint {
+        match *self {
+            TyU => 1,
+            TyU8 => 2,
+            TyU16 | TyU32 | TyU64  => 3,
+        }
+    }
+}
+
 impl fmt::Show for UintTy {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "{}", ast_util::uint_ty_to_string(*self, None))
@@ -724,6 +744,14 @@ impl fmt::Show for FloatTy {
     }
 }
 
+impl FloatTy {
+    pub fn suffix_len(&self) -> uint {
+        match *self {
+            TyF32 | TyF64 => 3, // add F128 handling here
+        }
+    }
+}
+
 // NB PartialEq method appears below.
 #[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash)]
 pub struct Ty {

src/libsyntax/codemap.rs

@@ -96,7 +96,7 @@ pub struct Span {
 
 pub static DUMMY_SP: Span = Span { lo: BytePos(0), hi: BytePos(0), expn_info: None };
 
-#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash)]
+#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)]
 pub struct Spanned<T> {
     pub node: T,
     pub span: Span,

src/libsyntax/ext/quote.rs

@@ -412,45 +412,14 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> Gc<ast::Expr> {
             return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_CHAR"), vec!(e_char));
         }
 
-        LIT_INT(i, ity) => {
+        LIT_INTEGER(i) => {
-            let s_ity = match ity {
+            let e_int = mk_ident(cx, sp, i);
-                ast::TyI => "TyI",
+            return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_INTEGER"), vec!(e_int));
-                ast::TyI8 => "TyI8",
-                ast::TyI16 => "TyI16",
-                ast::TyI32 => "TyI32",
-                ast::TyI64 => "TyI64"
-            };
-            let e_ity = mk_ast_path(cx, sp, s_ity);
-            let e_i64 = cx.expr_lit(sp, ast::LitInt(i, ast::TyI64));
-            return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_INT"), vec!(e_i64, e_ity));
         }
 
-        LIT_UINT(u, uty) => {
+        LIT_FLOAT(fident) => {
-            let s_uty = match uty {
-                ast::TyU => "TyU",
-                ast::TyU8 => "TyU8",
-                ast::TyU16 => "TyU16",
-                ast::TyU32 => "TyU32",
-                ast::TyU64 => "TyU64"
-            };
-            let e_uty = mk_ast_path(cx, sp, s_uty);
-            let e_u64 = cx.expr_lit(sp, ast::LitUint(u, ast::TyU64));
-            return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_UINT"), vec!(e_u64, e_uty));
-        }
-
-        LIT_INT_UNSUFFIXED(i) => {
-            let e_i64 = cx.expr_lit(sp, ast::LitInt(i, ast::TyI64));
-            return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_INT_UNSUFFIXED"), vec!(e_i64));
-        }
-
-        LIT_FLOAT(fident, fty) => {
-            let s_fty = match fty {
-                ast::TyF32 => "TyF32",
-                ast::TyF64 => "TyF64",
-            };
-            let e_fty = mk_ast_path(cx, sp, s_fty);
             let e_fident = mk_ident(cx, sp, fident);
-            return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_FLOAT"), vec!(e_fident, e_fty));
+            return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_FLOAT"), vec!(e_fident));
         }
 
         LIT_STR(ident) => {

src/libsyntax/parse/lexer/mod.rs

@@ -18,7 +18,6 @@ use parse::token::{str_to_ident};
 
 use std::char;
 use std::mem::replace;
-use std::num::from_str_radix;
 use std::rc::Rc;
 use std::str;
 
@@ -491,204 +490,113 @@ impl<'a> StringReader<'a> {
         if res.is_some() { res } else { self.consume_whitespace_and_comments() }
     }
 
-    fn scan_exponent(&mut self, start_bpos: BytePos) -> Option<String> {
+    /// Scan through any digits (base `radix`) or underscores, and return how
-        // \x00 hits the `return None` case immediately, so this is fine.
+    /// many digits there were.
-        let mut c = self.curr.unwrap_or('\x00');
+    fn scan_digits(&mut self, radix: uint) -> uint {
-        let mut rslt = String::new();
+        let mut len = 0u;
-        if c == 'e' || c == 'E' {
-            rslt.push_char(c);
-            self.bump();
-            c = self.curr.unwrap_or('\x00');
-            if c == '-' || c == '+' {
-                rslt.push_char(c);
-                self.bump();
-            }
-            let exponent = self.scan_digits(10u);
-            if exponent.len() > 0u {
-                rslt.push_str(exponent.as_slice());
-                return Some(rslt);
-            } else {
-                let last_bpos = self.last_pos;
-                self.err_span_(start_bpos, last_bpos, "scan_exponent: bad fp literal");
-                rslt.push_str("1"); // arbitrary placeholder exponent
-                return Some(rslt);
-            }
-        } else {
-            return None::<String>;
-        }
-    }
-
-    fn scan_digits(&mut self, radix: uint) -> String {
-        let mut rslt = String::new();
         loop {
             let c = self.curr;
-            if c == Some('_') { self.bump(); continue; }
+            if c == Some('_') { debug!("skipping a _"); self.bump(); continue; }
             match c.and_then(|cc| char::to_digit(cc, radix)) {
                 Some(_) => {
-                rslt.push_char(c.unwrap());
+                    debug!("{} in scan_digits", c);
+                    len += 1;
                     self.bump();
                 }
-              _ => return rslt
+                _ => return len
             }
         };
     }
 
-    fn check_float_base(&mut self, start_bpos: BytePos, last_bpos: BytePos, base: uint) {
+    /// Lex a LIT_INTEGER or a LIT_FLOAT
-        match base {
-          16u => self.err_span_(start_bpos, last_bpos,
-                                "hexadecimal float literal is not supported"),
-          8u => self.err_span_(start_bpos, last_bpos, "octal float literal is not supported"),
-          2u => self.err_span_(start_bpos, last_bpos, "binary float literal is not supported"),
-          _ => ()
-        }
-    }
-
     fn scan_number(&mut self, c: char) -> token::Token {
-        let mut num_str;
+        let mut num_digits;
-        let mut base = 10u;
+        let mut base = 10;
-        let mut c = c;
-        let mut n = self.nextch().unwrap_or('\x00');
         let start_bpos = self.last_pos;
-        if c == '0' && n == 'x' {
-            self.bump();
-            self.bump();
-            base = 16u;
-        } else if c == '0' && n == 'o' {
-            self.bump();
-            self.bump();
-            base = 8u;
-        } else if c == '0' && n == 'b' {
-            self.bump();
-            self.bump();
-            base = 2u;
-        }
-        num_str = self.scan_digits(base);
-        c = self.curr.unwrap_or('\x00');
-        self.nextch();
-        if c == 'u' || c == 'i' {
-            enum Result { Signed(ast::IntTy), Unsigned(ast::UintTy) }
-            let signed = c == 'i';
-            let mut tp = {
-                if signed { Signed(ast::TyI) }
-                else { Unsigned(ast::TyU) }
-            };
-            self.bump();
-            c = self.curr.unwrap_or('\x00');
-            if c == '8' {
-                self.bump();
-                tp = if signed { Signed(ast::TyI8) }
-                          else { Unsigned(ast::TyU8) };
-            }
-            n = self.nextch().unwrap_or('\x00');
-            if c == '1' && n == '6' {
-                self.bump();
-                self.bump();
-                tp = if signed { Signed(ast::TyI16) }
-                          else { Unsigned(ast::TyU16) };
-            } else if c == '3' && n == '2' {
-                self.bump();
-                self.bump();
-                tp = if signed { Signed(ast::TyI32) }
-                          else { Unsigned(ast::TyU32) };
-            } else if c == '6' && n == '4' {
-                self.bump();
-                self.bump();
-                tp = if signed { Signed(ast::TyI64) }
-                          else { Unsigned(ast::TyU64) };
-            }
-            if num_str.len() == 0u {
-                let last_bpos = self.last_pos;
-                self.err_span_(start_bpos, last_bpos, "no valid digits found for number");
-                num_str = "1".to_string();
-            }
-            let parsed = match from_str_radix::<u64>(num_str.as_slice(),
-                                                     base as uint) {
-                Some(p) => p,
-                None => {
-                    let last_bpos = self.last_pos;
-                    self.err_span_(start_bpos, last_bpos, "int literal is too large");
-                    1
-                }
-            };
 
-            match tp {
-              Signed(t) => return token::LIT_INT(parsed as i64, t),
-              Unsigned(t) => return token::LIT_UINT(parsed, t)
-            }
-        }
-        let mut is_float = false;
-        if self.curr_is('.') && !(ident_start(self.nextch()) || self.nextch_is('.')) {
-            is_float = true;
         self.bump();
-            let dec_part = self.scan_digits(10u);
-            num_str.push_char('.');
-            num_str.push_str(dec_part.as_slice());
-        }
-        match self.scan_exponent(start_bpos) {
-          Some(ref s) => {
-            is_float = true;
-            num_str.push_str(s.as_slice());
-          }
-          None => ()
-        }
 
-        if self.curr_is('f') {
+        if c == '0' {
-            self.bump();
+            match self.curr.unwrap_or('\0') {
-            c = self.curr.unwrap_or('\x00');
+                'b' => { self.bump(); base = 2; num_digits = self.scan_digits(2); }
-            n = self.nextch().unwrap_or('\x00');
+                'o' => { self.bump(); base = 8; num_digits = self.scan_digits(8); }
-            if c == '3' && n == '2' {
+                'x' => { self.bump(); base = 16; num_digits = self.scan_digits(16); }
-                self.bump();
+                '0'..'9' | '_' | '.' => {
-                self.bump();
+                    num_digits = self.scan_digits(10) + 1;
-                let last_bpos = self.last_pos;
-                self.check_float_base(start_bpos, last_bpos, base);
-                return token::LIT_FLOAT(str_to_ident(num_str.as_slice()),
-                                        ast::TyF32);
-            } else if c == '6' && n == '4' {
-                self.bump();
-                self.bump();
-                let last_bpos = self.last_pos;
-                self.check_float_base(start_bpos, last_bpos, base);
-                return token::LIT_FLOAT(str_to_ident(num_str.as_slice()),
-                                        ast::TyF64);
-                /* FIXME (#2252): if this is out of range for either a
-                32-bit or 64-bit float, it won't be noticed till the
-                back-end.  */
                 }
-            let last_bpos = self.last_pos;
+                'u' | 'i' => {
-            self.err_span_(start_bpos, last_bpos, "expected `f32` or `f64` suffix");
+                    self.scan_int_suffix();
+                    return token::LIT_INTEGER(self.ident_from(start_bpos));
+                },
+                'f' => {
+                    let last_pos = self.last_pos;
+                    self.scan_float_suffix();
+                    self.check_float_base(start_bpos, last_pos, base);
+                    return token::LIT_FLOAT(self.ident_from(start_bpos));
                 }
-        if is_float {
+                _ => {
-            let last_bpos = self.last_pos;
+                    // just a 0
-            self.check_float_base(start_bpos, last_bpos, base);
+                    return token::LIT_INTEGER(self.ident_from(start_bpos));
-            return token::LIT_FLOAT_UNSUFFIXED(str_to_ident(
+                }
-                    num_str.as_slice()));
+            }
+        } else if c.is_digit_radix(10) {
+            num_digits = self.scan_digits(10) + 1;
         } else {
-            if num_str.len() == 0u {
+            num_digits = 0;
-                let last_bpos = self.last_pos;
-                self.err_span_(start_bpos, last_bpos, "no valid digits found for number");
-                num_str = "1".to_string();
         }
-            let parsed = match from_str_radix::<u64>(num_str.as_slice(),
-                                                     base as uint) {
-                Some(p) => p,
-                None => {
-                    let last_bpos = self.last_pos;
-                    self.err_span_(start_bpos, last_bpos, "int literal is too large");
-                    1
-                }
-            };
 
-            debug!("lexing {} as an unsuffixed integer literal",
+        if num_digits == 0 {
-                   num_str.as_slice());
+            self.err_span_(start_bpos, self.last_pos, "no valid digits found for number");
-            return token::LIT_INT_UNSUFFIXED(parsed as i64);
+            // eat any suffix
+            self.scan_int_suffix();
+            return token::LIT_INTEGER(str_to_ident("0"));
+        }
+
+        // might be a float, but don't be greedy if this is actually an
+        // integer literal followed by field/method access or a range pattern
+        // (`0..2` and `12.foo()`)
+        if self.curr_is('.') && !self.nextch_is('.') && !self.nextch().unwrap_or('\0')
+                                                             .is_XID_start() {
+            // might have stuff after the ., and if it does, it needs to start
+            // with a number
+            self.bump();
+            if self.curr.unwrap_or('\0').is_digit_radix(10) {
+                self.scan_digits(10);
+                self.scan_float_exponent();
+                self.scan_float_suffix();
+            }
+            let last_pos = self.last_pos;
+            self.check_float_base(start_bpos, last_pos, base);
+            return token::LIT_FLOAT(self.ident_from(start_bpos));
+        } else if self.curr_is('f') {
+            // or it might be an integer literal suffixed as a float
+            self.scan_float_suffix();
+            let last_pos = self.last_pos;
+            self.check_float_base(start_bpos, last_pos, base);
+            return token::LIT_FLOAT(self.ident_from(start_bpos));
+        } else {
+            // it might be a float if it has an exponent
+            if self.curr_is('e') || self.curr_is('E') {
+                self.scan_float_exponent();
+                self.scan_float_suffix();
+                let last_pos = self.last_pos;
+                self.check_float_base(start_bpos, last_pos, base);
+                return token::LIT_FLOAT(self.ident_from(start_bpos));
+            }
+            // but we certainly have an integer!
+            self.scan_int_suffix();
+            return token::LIT_INTEGER(self.ident_from(start_bpos));
         }
     }
 
-
+    /// Scan over `n_digits` hex digits, stopping at `delim`, reporting an
-    fn scan_numeric_escape(&mut self, n_hex_digits: uint, delim: char) -> bool {
+    /// error if too many or too few digits are encountered.
-        let mut accum_int = 0u32;
+    fn scan_hex_digits(&mut self, n_digits: uint, delim: char) -> bool {
+        debug!("scanning {} digits until {}", n_digits, delim);
         let start_bpos = self.last_pos;
-        for _ in range(0, n_hex_digits) {
+        let mut accum_int = 0;
+
+        for _ in range(0, n_digits) {
             if self.is_eof() {
                 let last_bpos = self.last_pos;
                 self.fatal_span_(start_bpos, last_bpos, "unterminated numeric character escape");
@@ -736,9 +644,9 @@ impl<'a> StringReader<'a> {
                     Some(e) => {
                         return match e {
                             'n' | 'r' | 't' | '\\' | '\'' | '"' | '0' => true,
-                            'x' => self.scan_numeric_escape(2u, delim),
+                            'x' => self.scan_hex_digits(2u, delim),
-                            'u' if !ascii_only => self.scan_numeric_escape(4u, delim),
+                            'u' if !ascii_only => self.scan_hex_digits(4u, delim),
-                            'U' if !ascii_only => self.scan_numeric_escape(8u, delim),
+                            'U' if !ascii_only => self.scan_hex_digits(8u, delim),
                             '\n' if delim == '"' => {
                                 self.consume_whitespace();
                                 true
@@ -791,6 +699,80 @@ impl<'a> StringReader<'a> {
         true
     }
 
+    /// Scan over an int literal suffix.
+    fn scan_int_suffix(&mut self) {
+        match self.curr {
+            Some('i') | Some('u') => {
+                self.bump();
+
+                if self.curr_is('8') {
+                    self.bump();
+                } else if self.curr_is('1') {
+                    if !self.nextch_is('6') {
+                        self.err_span_(self.last_pos, self.pos,
+                                      "illegal int suffix");
+                    } else {
+                        self.bump(); self.bump();
+                    }
+                } else if self.curr_is('3') {
+                    if !self.nextch_is('2') {
+                        self.err_span_(self.last_pos, self.pos,
+                                      "illegal int suffix");
+                    } else {
+                        self.bump(); self.bump();
+                    }
+                } else if self.curr_is('6') {
+                    if !self.nextch_is('4') {
+                        self.err_span_(self.last_pos, self.pos,
+                                      "illegal int suffix");
+                    } else {
+                        self.bump(); self.bump();
+                    }
+                }
+            },
+            _ => { }
+        }
+    }
+
+    /// Scan over a float literal suffix
+    fn scan_float_suffix(&mut self) {
+        if self.curr_is('f') {
+            if (self.nextch_is('3') && self.nextnextch_is('2'))
+            || (self.nextch_is('6') && self.nextnextch_is('4')) {
+                self.bump();
+                self.bump();
+                self.bump();
+            } else {
+                self.err_span_(self.last_pos, self.pos, "illegal float suffix");
+            }
+        }
+    }
+
+    /// Scan over a float exponent.
+    fn scan_float_exponent(&mut self) {
+        if self.curr_is('e') || self.curr_is('E') {
+            self.bump();
+            if self.curr_is('-') || self.curr_is('+') {
+                self.bump();
+            }
+            if self.scan_digits(10) == 0 {
+                self.err_span_(self.last_pos, self.pos, "expected at least one digit in exponent")
+            }
+        }
+    }
+
+    /// Check that a base is valid for a floating literal, emitting a nice
+    /// error if it isn't.
+    fn check_float_base(&mut self, start_bpos: BytePos, last_bpos: BytePos, base: uint) {
+        match base {
+            16u => self.err_span_(start_bpos, last_bpos, "hexadecimal float literal is not \
+                                 supported"),
+            8u => self.err_span_(start_bpos, last_bpos, "octal float literal is not supported"),
+            2u => self.err_span_(start_bpos, last_bpos, "binary float literal is not supported"),
+            _ => ()
+        }
+    }
+
     fn binop(&mut self, op: token::BinOp) -> token::Token {
         self.bump();
         if self.curr_is('=') {

src/libsyntax/parse/mod.rs

@@ -506,6 +506,115 @@ pub fn binary_lit(lit: &str) -> Rc<Vec<u8>> {
     Rc::new(res)
 }
 
+pub fn integer_lit(s: &str, sd: &SpanHandler, sp: Span) -> ast::Lit_ {
+    // s can only be ascii, byte indexing is fine
+
+    let s2 = s.chars().filter(|&c| c != '_').collect::<String>();
+    let mut s = s2.as_slice();
+
+    debug!("parse_integer_lit: {}", s);
+
+    if s.len() == 1 {
+        return ast::LitIntUnsuffixed((s.char_at(0)).to_digit(10).unwrap() as i64);
+    }
+
+    let mut base = 10;
+    let orig = s;
+
+    #[deriving(Show)]
+    enum Result {
+        Nothing,
+        Signed(ast::IntTy),
+        Unsigned(ast::UintTy)
+    }
+
+    impl Result {
+        fn suffix_len(&self) -> uint {
+            match *self {
+                Nothing => 0,
+                Signed(s) => s.suffix_len(),
+                Unsigned(u) => u.suffix_len()
+            }
+        }
+    }
+
+    let mut ty = Nothing;
+
+
+    if s.char_at(0) == '0' {
+        match s.char_at(1) {
+            'x' => base = 16,
+            'o' => base = 8,
+            'b' => base = 2,
+            _ => { }
+        }
+    }
+
+    if base != 10 {
+        s = s.slice_from(2);
+    }
+
+    let last = s.len() - 1;
+    match s.char_at(last) {
+        'i' => ty = Signed(ast::TyI),
+        'u' => ty = Unsigned(ast::TyU),
+        '8' => {
+            if s.len() > 2 {
+                match s.char_at(last - 1) {
+                    'i' => ty = Signed(ast::TyI8),
+                    'u' => ty = Unsigned(ast::TyU8),
+                    _ => { }
+                }
+            }
+        },
+        '6' => {
+            if s.len() > 3 && s.char_at(last - 1) == '1' {
+                match s.char_at(last - 2) {
+                    'i' => ty = Signed(ast::TyI16),
+                    'u' => ty = Unsigned(ast::TyU16),
+                    _ => { }
+                }
+            }
+        },
+        '2' => {
+            if s.len() > 3 && s.char_at(last - 1) == '3' {
+                match s.char_at(last - 2) {
+                    'i' => ty = Signed(ast::TyI32),
+                    'u' => ty = Unsigned(ast::TyU32),
+                    _ => { }
+                }
+            }
+        },
+        '4' => {
+            if s.len() > 3 && s.char_at(last - 1) == '6' {
+                match s.char_at(last - 2) {
+                    'i' => ty = Signed(ast::TyI64),
+                    'u' => ty = Unsigned(ast::TyU64),
+                    _ => { }
+                }
+            }
+        },
+        _ => { }
+    }
+
+
+    s = s.slice_to(s.len() - ty.suffix_len());
+
+    debug!("The suffix is {}, base {}, the new string is {}, the original \
+           string was {}", ty, base, s, orig);
+
+    let res: u64 = match ::std::num::from_str_radix(s, base) {
+        Some(r) => r,
+        None => { sd.span_err(sp, "int literal is too large"); 0 }
+    };
+
+    match ty {
+        Nothing => ast::LitIntUnsuffixed(res as i64),
+        Signed(t) => ast::LitInt(res as i64, t),
+        Unsigned(t) => ast::LitUint(res, t)
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

src/libsyntax/parse/parser.rs

@@ -33,8 +33,8 @@ use ast::{ForeignItem, ForeignItemStatic, ForeignItemFn, ForeignMod};
 use ast::{Ident, NormalFn, Inherited, Item, Item_, ItemStatic};
 use ast::{ItemEnum, ItemFn, ItemForeignMod, ItemImpl};
 use ast::{ItemMac, ItemMod, ItemStruct, ItemTrait, ItemTy, Lit, Lit_};
-use ast::{LitBool, LitFloat, LitFloatUnsuffixed, LitInt, LitChar, LitByte, LitBinary};
+use ast::{LitBool, LitChar, LitByte, LitBinary};
-use ast::{LitIntUnsuffixed, LitNil, LitStr, LitUint, Local, LocalLet};
+use ast::{LitNil, LitStr, LitUint, Local, LocalLet};
 use ast::{MutImmutable, MutMutable, Mac_, MacInvocTT, Matcher, MatchNonterminal};
 use ast::{MatchSeq, MatchTok, Method, MutTy, BiMul, Mutability};
 use ast::{NamedField, UnNeg, NoReturn, UnNot, P, Pat, PatEnum};
@@ -1541,20 +1541,14 @@ impl<'a> Parser<'a> {
         }
     }
 
-    /// Matches token_lit = LIT_INT | ...
+    /// Matches token_lit = LIT_INTEGER | ...
     pub fn lit_from_token(&mut self, tok: &token::Token) -> Lit_ {
         match *tok {
             token::LIT_BYTE(i) => LitByte(parse::byte_lit(i.as_str()).val0()),
             token::LIT_CHAR(i) => LitChar(parse::char_lit(i.as_str()).val0()),
-            token::LIT_INT(i, it) => LitInt(i, it),
+            token::LIT_INTEGER(s) => parse::integer_lit(self.id_to_interned_str(s).get(),
-            token::LIT_UINT(u, ut) => LitUint(u, ut),
+                                                       &self.sess.span_diagnostic, self.span),
-            token::LIT_INT_UNSUFFIXED(i) => LitIntUnsuffixed(i),
+            token::LIT_FLOAT(s) => parse::float_lit(s.as_str()),
-            token::LIT_FLOAT(s, ft) => {
-                LitFloat(self.id_to_interned_str(s), ft)
-            }
-            token::LIT_FLOAT_UNSUFFIXED(s) => {
-                LitFloatUnsuffixed(self.id_to_interned_str(s))
-            }
             token::LIT_STR(s) => {
                 LitStr(token::intern_and_get_ident(parse::str_lit(s.as_str()).as_slice()),
                        ast::CookedStr)
@@ -5398,3 +5392,4 @@ impl<'a> Parser<'a> {
         }
     }
 }
+

src/libsyntax/parse/token.rs

@@ -10,7 +10,6 @@
 
 use ast;
 use ast::{P, Ident, Name, Mrk};
-use ast_util;
 use ext::mtwt;
 use parse::token;
 use util::interner::{RcStr, StrInterner};
@@ -81,11 +80,8 @@ pub enum Token {
     /* Literals */
     LIT_BYTE(Ident),
     LIT_CHAR(Ident),
-    LIT_INT(i64, ast::IntTy),
+    LIT_INTEGER(Ident),
-    LIT_UINT(u64, ast::UintTy),
+    LIT_FLOAT(Ident),
-    LIT_INT_UNSUFFIXED(i64),
-    LIT_FLOAT(Ident, ast::FloatTy),
-    LIT_FLOAT_UNSUFFIXED(Ident),
     LIT_STR(Ident),
     LIT_STR_RAW(Ident, uint), /* raw str delimited by n hash symbols */
     LIT_BINARY(Ident),
@@ -206,24 +202,10 @@ pub fn to_string(t: &Token) -> String {
       LIT_CHAR(c) => {
           format!("'{}'", get_ident(c).get())
       }
-      LIT_INT(i, t) => ast_util::int_ty_to_string(t, Some(i)),
+      LIT_INTEGER(c) | LIT_FLOAT(c) => {
-      LIT_UINT(u, t) => ast_util::uint_ty_to_string(t, Some(u)),
+          get_ident(c).get().to_string()
-      LIT_INT_UNSUFFIXED(i) => { (i as u64).to_string() }
-      LIT_FLOAT(s, t) => {
-        let mut body = String::from_str(get_ident(s).get());
-        if body.as_slice().ends_with(".") {
-            body.push_char('0');  // `10.f` is not a float literal
-        }
-        body.push_str(ast_util::float_ty_to_string(t).as_slice());
-        body
-      }
-      LIT_FLOAT_UNSUFFIXED(s) => {
-        let mut body = String::from_str(get_ident(s).get());
-        if body.as_slice().ends_with(".") {
-            body.push_char('0');  // `10.f` is not a float literal
-        }
-        body
       }
+
       LIT_STR(s) => {
           format!("\"{}\"", get_ident(s).get())
       }
@@ -285,11 +267,8 @@ pub fn can_begin_expr(t: &Token) -> bool {
       TILDE => true,
       LIT_BYTE(_) => true,
       LIT_CHAR(_) => true,
-      LIT_INT(_, _) => true,
+      LIT_INTEGER(_) => true,
-      LIT_UINT(_, _) => true,
+      LIT_FLOAT(_) => true,
-      LIT_INT_UNSUFFIXED(_) => true,
-      LIT_FLOAT(_, _) => true,
-      LIT_FLOAT_UNSUFFIXED(_) => true,
       LIT_STR(_) => true,
       LIT_STR_RAW(_, _) => true,
       LIT_BINARY(_) => true,
@@ -326,11 +305,8 @@ pub fn is_lit(t: &Token) -> bool {
     match *t {
       LIT_BYTE(_) => true,
       LIT_CHAR(_) => true,
-      LIT_INT(_, _) => true,
+      LIT_INTEGER(_) => true,
-      LIT_UINT(_, _) => true,
+      LIT_FLOAT(_) => true,
-      LIT_INT_UNSUFFIXED(_) => true,
-      LIT_FLOAT(_, _) => true,
-      LIT_FLOAT_UNSUFFIXED(_) => true,
       LIT_STR(_) => true,
       LIT_STR_RAW(_, _) => true,
       LIT_BINARY(_) => true,