bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Switch to an independent enum for Lit* subtokens.

Browse source
This commit is contained in:
Huon Wilson 2014-11-19 10:17:40 +11:00
parent c8d6e3b2c2
commit 5b5638f686
9 changed files with 115 additions and 118 deletions
Download patch file Download diff file Expand all files Collapse all files

50
src/grammar/verify.rs
View file

@ -61,7 +61,7 @@ fn parse_token_list(file: &str) -> HashMap<String, Token> {
"SHL" => token::BinOp(token::Shl), "SHL" => token::BinOp(token::Shl),
"LBRACE" => token::OpenDelim(token::Brace), "LBRACE" => token::OpenDelim(token::Brace),
"RARROW" => token::Rarrow, "RARROW" => token::Rarrow,
"LIT_STR" => token::LitStr(Name(0)), "LIT_STR" => token::Literal(token::Str_(Name(0))),
"DOTDOT" => token::DotDot, "DOTDOT" => token::DotDot,
"MOD_SEP" => token::ModSep, "MOD_SEP" => token::ModSep,
"DOTDOTDOT" => token::DotDotDot, "DOTDOTDOT" => token::DotDotDot,
@ -71,7 +71,7 @@ fn parse_token_list(file: &str) -> HashMap<String, Token> {
"ANDAND" => token::AndAnd, "ANDAND" => token::AndAnd,
"AT" => token::At, "AT" => token::At,
"LBRACKET" => token::OpenDelim(token::Bracket), "LBRACKET" => token::OpenDelim(token::Bracket),
"LIT_STR_RAW" => token::LitStrRaw(Name(0), 0), "LIT_STR_RAW" => token::Literal(token::StrRaw(Name(0), 0)),
"RPAREN" => token::CloseDelim(token::Paren), "RPAREN" => token::CloseDelim(token::Paren),
"SLASH" => token::BinOp(token::Slash), "SLASH" => token::BinOp(token::Slash),
"COMMA" => token::Comma, "COMMA" => token::Comma,
@ -80,8 +80,8 @@ fn parse_token_list(file: &str) -> HashMap<String, Token> {
"TILDE" => token::Tilde, "TILDE" => token::Tilde,
"IDENT" => token::Id(), "IDENT" => token::Id(),
"PLUS" => token::BinOp(token::Plus), "PLUS" => token::BinOp(token::Plus),
"LIT_CHAR" => token::LitChar(Name(0)), "LIT_CHAR" => token::Literal(token::Char(Name(0))),
"LIT_BYTE" => token::LitByte(Name(0)), "LIT_BYTE" => token::Literal(token::Byte(Name(0))),
"EQ" => token::Eq, "EQ" => token::Eq,
"RBRACKET" => token::CloseDelim(token::Bracket), "RBRACKET" => token::CloseDelim(token::Bracket),
"COMMENT" => token::Comment, "COMMENT" => token::Comment,
@ -95,9 +95,9 @@ fn parse_token_list(file: &str) -> HashMap<String, Token> {
"BINOP" => token::BinOp(token::Plus), "BINOP" => token::BinOp(token::Plus),
"POUND" => token::Pound, "POUND" => token::Pound,
"OROR" => token::OrOr, "OROR" => token::OrOr,
"LIT_INTEGER" => token::LitInteger(Name(0)), "LIT_INTEGER" => token::Literal(token::Integer(Name(0))),
"BINOPEQ" => token::BinOpEq(token::Plus), "BINOPEQ" => token::BinOpEq(token::Plus),
"LIT_FLOAT" => token::LitFloat(Name(0)), "LIT_FLOAT" => token::Literal(token::Float(Name(0))),
"WHITESPACE" => token::Whitespace, "WHITESPACE" => token::Whitespace,
"UNDERSCORE" => token::Underscore, "UNDERSCORE" => token::Underscore,
"MINUS" => token::BinOp(token::Minus), "MINUS" => token::BinOp(token::Minus),
@ -107,8 +107,8 @@ fn parse_token_list(file: &str) -> HashMap<String, Token> {
"OR" => token::BinOp(token::Or), "OR" => token::BinOp(token::Or),
"GT" => token::Gt, "GT" => token::Gt,
"LE" => token::Le, "LE" => token::Le,
"LIT_BINARY" => token::LitBinary(Name(0)), "LIT_BINARY" => token::Literal(token::Binary(Name(0))),
"LIT_BINARY_RAW" => token::LitBinaryRaw(Name(0), 0), "LIT_BINARY_RAW" => token::Literal(token::BinaryRaw(Name(0), 0)),
_ => continue, _ => continue,
}; };
@ -189,15 +189,17 @@ fn parse_antlr_token(s: &str, tokens: &HashMap<String, Token>) -> TokenAndSpan {
token::BinOp(..) => token::BinOp(str_to_binop(content)), token::BinOp(..) => token::BinOp(str_to_binop(content)),
token::BinOpEq(..) => token::BinOpEq(str_to_binop(content.slice_to( token::BinOpEq(..) => token::BinOpEq(str_to_binop(content.slice_to(
content.len() - 1))), content.len() - 1))),
token::LitStr(..) => token::LitStr(fix(content)), token::Literal(token::Str_(..)) => token::Literal(token::Str_(fix(content))),
token::LitStrRaw(..) => token::LitStrRaw(fix(content), count(content)), token::Literal(token::StrRaw(..)) => token::Literal(token::StrRaw(fix(content),
token::LitChar(..) => token::LitChar(fixchar(content)), count(content))),
token::LitByte(..) => token::LitByte(fixchar(content)), token::Literal(token::Char(..)) => token::Literal(token::Char(fixchar(content))),
token::Literal(token::Byte(..)) => token::Literal(token::Byte(fixchar(content))),
token::DocComment(..) => token::DocComment(nm), token::DocComment(..) => token::DocComment(nm),
token::LitInteger(..) => token::LitInteger(nm), token::Literal(token::Integer(..)) => token::Literal(token::Integer(nm)),
token::LitFloat(..) => token::LitFloat(nm), token::Literal(token::Float(..)) => token::Literal(token::Float(nm)),
token::LitBinary(..) => token::LitBinary(nm), token::Literal(token::Binary(..)) => token::Literal(token::Binary(nm)),
token::LitBinaryRaw(..) => token::LitBinaryRaw(fix(content), count(content)), token::Literal(token::BinaryRaw(..)) => token::Literal(token::BinaryRaw(fix(content),
count(content))),
token::Ident(..) => token::Ident(ast::Ident { name: nm, ctxt: 0 }, token::Ident(..) => token::Ident(ast::Ident { name: nm, ctxt: 0 },
token::ModName), token::ModName),
token::Lifetime(..) => token::Lifetime(ast::Ident { name: nm, ctxt: 0 }), token::Lifetime(..) => token::Lifetime(ast::Ident { name: nm, ctxt: 0 }),
@ -284,14 +286,14 @@ fn main() {
) )
matches!( matches!(
LitByte(..), token::Literal(token::Byte(..)),
LitChar(..), token::Literal(token::Char(..)),
LitInteger(..), token::Literal(token::Integer(..)),
LitFloat(..), token::Literal(token::Float(..)),
LitStr(..), token::Literal(token::Str_(..)),
LitStrRaw(..), token::Literal(token::StrRaw(..)),
LitBinary(..), token::Literal(token::Binary(..)),
LitBinaryRaw(..), token::Literal(token::BinaryRaw(..)),
Ident(..), Ident(..),
Lifetime(..), Lifetime(..),
Interpolated(..), Interpolated(..),

7
src/librustdoc/html/highlight.rs
View file

@ -129,11 +129,12 @@ fn doit(sess: &parse::ParseSess, mut lexer: lexer::StringReader,
} }
// text literals // text literals
token::LitByte(..) | token::LitBinary(..) | token::LitBinaryRaw(..) | token::Literal(token::Byte(..)) | token::Literal(token::Char(..)) |
token::LitChar(..) | token::LitStr(..) | token::LitStrRaw(..) => "string", token::Literal(token::Binary(..)) | token::Literal(token::BinaryRaw(..)) |
token::Literal(token::Str_(..)) | token::Literal(token::StrRaw(..)) => "string",
// number literals // number literals
token::LitInteger(..) | token::LitFloat(..) => "number", token::Literal(token::Integer(..)) | token::Literal(token::Float(..)) => "number",
// keywords are also included in the identifier set // keywords are also included in the identifier set
token::Ident(ident, _is_mod_sep) => { token::Ident(ident, _is_mod_sep) => {

2
src/libsyntax/ast.rs
View file

@ -838,7 +838,7 @@ impl TokenTree {
tts: vec![TtToken(sp, token::Ident(token::str_to_ident("doc"), tts: vec![TtToken(sp, token::Ident(token::str_to_ident("doc"),
token::Plain)), token::Plain)),
TtToken(sp, token::Eq), TtToken(sp, token::Eq),
TtToken(sp, token::LitStr(name))], TtToken(sp, token::Literal(token::Str_(name)))],
close_span: sp, close_span: sp,
})) }))
} }

2
src/libsyntax/diagnostics/plugin.rs
View file

@ -87,7 +87,7 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt,
}, },
[ast::TtToken(_, token::Ident(ref code, _)), [ast::TtToken(_, token::Ident(ref code, _)),
ast::TtToken(_, token::Comma), ast::TtToken(_, token::Comma),
ast::TtToken(_, token::LitStrRaw(description, _))] => { ast::TtToken(_, token::Literal(token::StrRaw(description, _)))] => {
(code, Some(description)) (code, Some(description))
} }
_ => unreachable!() _ => unreachable!()

37
src/libsyntax/ext/quote.rs
View file

@ -542,6 +542,13 @@ fn mk_delim(cx: &ExtCtxt, sp: Span, delim: token::DelimToken) -> P<ast::Expr> {
#[allow(non_upper_case_globals)] #[allow(non_upper_case_globals)]
fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> { fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
macro_rules! mk_lit {
($name: expr, $($args: expr),*) => {{
let inner = cx.expr_call(sp, mk_token_path(cx, sp, $name), vec![$($args),*]);
cx.expr_call(sp, mk_token_path(cx, sp, "Literal"), vec![inner])
}}
}
match *tok { match *tok {
token::BinOp(binop) => { token::BinOp(binop) => {
return cx.expr_call(sp, mk_token_path(cx, sp, "BinOp"), vec!(mk_binop(cx, sp, binop))); return cx.expr_call(sp, mk_token_path(cx, sp, "BinOp"), vec!(mk_binop(cx, sp, binop)));
@ -560,38 +567,32 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
vec![mk_delim(cx, sp, delim)]); vec![mk_delim(cx, sp, delim)]);
} }
token::LitByte(i) => { token::Literal(token::Byte(i)) => {
let e_byte = mk_name(cx, sp, i.ident()); let e_byte = mk_name(cx, sp, i.ident());
return mk_lit!("Byte", e_byte);
return cx.expr_call(sp, mk_token_path(cx, sp, "LitByte"), vec!(e_byte));
} }
token::LitChar(i) => { token::Literal(token::Char(i)) => {
let e_char = mk_name(cx, sp, i.ident()); let e_char = mk_name(cx, sp, i.ident());
return mk_lit!("Char", e_char);
return cx.expr_call(sp, mk_token_path(cx, sp, "LitChar"), vec!(e_char));
} }
token::LitInteger(i) => { token::Literal(token::Integer(i)) => {
let e_int = mk_name(cx, sp, i.ident()); let e_int = mk_name(cx, sp, i.ident());
return cx.expr_call(sp, mk_token_path(cx, sp, "LitInteger"), vec!(e_int)); return mk_lit!("Integer", e_int);
} }
token::LitFloat(fident) => { token::Literal(token::Float(fident)) => {
let e_fident = mk_name(cx, sp, fident.ident()); let e_fident = mk_name(cx, sp, fident.ident());
return cx.expr_call(sp, mk_token_path(cx, sp, "LitFloat"), vec!(e_fident)); return mk_lit!("Float", e_fident);
} }
token::LitStr(ident) => { token::Literal(token::Str_(ident)) => {
return cx.expr_call(sp, return mk_lit!("Str_", mk_name(cx, sp, ident.ident()))
mk_token_path(cx, sp, "LitStr"),
vec!(mk_name(cx, sp, ident.ident())));
} }
token::LitStrRaw(ident, n) => { token::Literal(token::StrRaw(ident, n)) => {
return cx.expr_call(sp, return mk_lit!("StrRaw", mk_name(cx, sp, ident.ident()), cx.expr_uint(sp, n))
mk_token_path(cx, sp, "LitStrRaw"),
vec!(mk_name(cx, sp, ident.ident()), cx.expr_uint(sp, n)));
} }
token::Ident(ident, style) => { token::Ident(ident, style) => {

41
src/libsyntax/parse/lexer/mod.rs
View file

@ -655,17 +655,17 @@ impl<'a> StringReader<'a> {
} }
'u' | 'i' => { 'u' | 'i' => {
self.scan_int_suffix(); self.scan_int_suffix();
return token::LitInteger(self.name_from(start_bpos)); return token::Literal(token::Integer(self.name_from(start_bpos)));
}, },
'f' => { 'f' => {
let last_pos = self.last_pos; let last_pos = self.last_pos;
self.scan_float_suffix(); self.scan_float_suffix();
self.check_float_base(start_bpos, last_pos, base); self.check_float_base(start_bpos, last_pos, base);
return token::LitFloat(self.name_from(start_bpos)); return token::Literal(token::Float(self.name_from(start_bpos)));
} }
_ => { _ => {
// just a 0 // just a 0
return token::LitInteger(self.name_from(start_bpos)); return token::Literal(token::Integer(self.name_from(start_bpos)));
} }
} }
} else if c.is_digit_radix(10) { } else if c.is_digit_radix(10) {
@ -678,7 +678,7 @@ impl<'a> StringReader<'a> {
self.err_span_(start_bpos, self.last_pos, "no valid digits found for number"); self.err_span_(start_bpos, self.last_pos, "no valid digits found for number");
// eat any suffix // eat any suffix
self.scan_int_suffix(); self.scan_int_suffix();
return token::LitInteger(token::intern("0")); return token::Literal(token::Integer(token::intern("0")));
} }
// might be a float, but don't be greedy if this is actually an // might be a float, but don't be greedy if this is actually an
@ -696,13 +696,13 @@ impl<'a> StringReader<'a> {
} }
let last_pos = self.last_pos; let last_pos = self.last_pos;
self.check_float_base(start_bpos, last_pos, base); self.check_float_base(start_bpos, last_pos, base);
return token::LitFloat(self.name_from(start_bpos)); return token::Literal(token::Float(self.name_from(start_bpos)));
} else if self.curr_is('f') { } else if self.curr_is('f') {
// or it might be an integer literal suffixed as a float // or it might be an integer literal suffixed as a float
self.scan_float_suffix(); self.scan_float_suffix();
let last_pos = self.last_pos; let last_pos = self.last_pos;
self.check_float_base(start_bpos, last_pos, base); self.check_float_base(start_bpos, last_pos, base);
return token::LitFloat(self.name_from(start_bpos)); return token::Literal(token::Float(self.name_from(start_bpos)));
} else { } else {
// it might be a float if it has an exponent // it might be a float if it has an exponent
if self.curr_is('e') || self.curr_is('E') { if self.curr_is('e') || self.curr_is('E') {
@ -710,11 +710,11 @@ impl<'a> StringReader<'a> {
self.scan_float_suffix(); self.scan_float_suffix();
let last_pos = self.last_pos; let last_pos = self.last_pos;
self.check_float_base(start_bpos, last_pos, base); self.check_float_base(start_bpos, last_pos, base);
return token::LitFloat(self.name_from(start_bpos)); return token::Literal(token::Float(self.name_from(start_bpos)));
} }
// but we certainly have an integer! // but we certainly have an integer!
self.scan_int_suffix(); self.scan_int_suffix();
return token::LitInteger(self.name_from(start_bpos)); return token::Literal(token::Integer(self.name_from(start_bpos)));
} }
} }
@ -1126,7 +1126,7 @@ impl<'a> StringReader<'a> {
} }
let id = if valid { self.name_from(start) } else { token::intern("0") }; let id = if valid { self.name_from(start) } else { token::intern("0") };
self.bump(); // advance curr past token self.bump(); // advance curr past token
return token::LitChar(id); return token::Literal(token::Char(id));
} }
'b' => { 'b' => {
self.bump(); self.bump();
@ -1157,7 +1157,7 @@ impl<'a> StringReader<'a> {
let id = if valid { self.name_from(start_bpos + BytePos(1)) } let id = if valid { self.name_from(start_bpos + BytePos(1)) }
else { token::intern("??") }; else { token::intern("??") };
self.bump(); self.bump();
return token::LitStr(id); return token::Literal(token::Str_(id));
} }
'r' => { 'r' => {
let start_bpos = self.last_pos; let start_bpos = self.last_pos;
@ -1224,7 +1224,7 @@ impl<'a> StringReader<'a> {
} else { } else {
token::intern("??") token::intern("??")
}; };
return token::LitStrRaw(id, hash_count); return token::Literal(token::StrRaw(id, hash_count));
} }
'-' => { '-' => {
if self.nextch_is('>') { if self.nextch_is('>') {
@ -1314,7 +1314,7 @@ impl<'a> StringReader<'a> {
let id = if valid { self.name_from(start) } else { token::intern("??") }; let id = if valid { self.name_from(start) } else { token::intern("??") };
self.bump(); // advance curr past token self.bump(); // advance curr past token
return token::LitByte(id); return token::Literal(token::Byte(id));
} }
fn scan_byte_string(&mut self) -> token::Token { fn scan_byte_string(&mut self) -> token::Token {
@ -1336,7 +1336,7 @@ impl<'a> StringReader<'a> {
} }
let id = if valid { self.name_from(start) } else { token::intern("??") }; let id = if valid { self.name_from(start) } else { token::intern("??") };
self.bump(); self.bump();
return token::LitBinary(id); return token::Literal(token::Binary(id));
} }
fn scan_raw_byte_string(&mut self) -> token::Token { fn scan_raw_byte_string(&mut self) -> token::Token {
@ -1387,8 +1387,9 @@ impl<'a> StringReader<'a> {
self.bump(); self.bump();
} }
self.bump(); self.bump();
return token::LitBinaryRaw(self.name_from_to(content_start_bpos, content_end_bpos), return token::Literal(token::BinaryRaw(self.name_from_to(content_start_bpos,
hash_count); content_end_bpos),
hash_count));
} }
} }
@ -1535,17 +1536,17 @@ mod test {
#[test] fn character_a() { #[test] fn character_a() {
assert_eq!(setup(&mk_sh(), "'a'".to_string()).next_token().tok, assert_eq!(setup(&mk_sh(), "'a'".to_string()).next_token().tok,
token::LitChar(token::intern("a"))); token::Literal(token::Char(token::intern("a"))));
} }
#[test] fn character_space() { #[test] fn character_space() {
assert_eq!(setup(&mk_sh(), "' '".to_string()).next_token().tok, assert_eq!(setup(&mk_sh(), "' '".to_string()).next_token().tok,
token::LitChar(token::intern(" "))); token::Literal(token::Char(token::intern(" "))));
} }
#[test] fn character_escaped() { #[test] fn character_escaped() {
assert_eq!(setup(&mk_sh(), "'\\n'".to_string()).next_token().tok, assert_eq!(setup(&mk_sh(), "'\\n'".to_string()).next_token().tok,
token::LitChar(token::intern("\\n"))); token::Literal(token::Char(token::intern("\\n"))));
} }
#[test] fn lifetime_name() { #[test] fn lifetime_name() {
@ -1557,7 +1558,7 @@ mod test {
assert_eq!(setup(&mk_sh(), assert_eq!(setup(&mk_sh(),
"r###\"\"#a\\b\x00c\"\"###".to_string()).next_token() "r###\"\"#a\\b\x00c\"\"###".to_string()).next_token()
.tok, .tok,
token::LitStrRaw(token::intern("\"#a\\b\x00c\""), 3)); token::Literal(token::StrRaw(token::intern("\"#a\\b\x00c\""), 3)));
} }
#[test] fn line_doc_comments() { #[test] fn line_doc_comments() {
@ -1573,7 +1574,7 @@ mod test {
token::Comment => { }, token::Comment => { },
_ => panic!("expected a comment!") _ => panic!("expected a comment!")
} }
assert_eq!(lexer.next_token().tok, token::LitChar(token::intern("a"))); assert_eq!(lexer.next_token().tok, token::Literal(token::Char(token::intern("a"))));
} }
} }

28
src/libsyntax/parse/parser.rs
View file

@ -1640,23 +1640,23 @@ impl<'a> Parser<'a> {
/// Matches token_lit = LIT_INTEGER | ... /// Matches token_lit = LIT_INTEGER | ...
pub fn lit_from_token(&mut self, tok: &token::Token) -> Lit_ { pub fn lit_from_token(&mut self, tok: &token::Token) -> Lit_ {
match *tok { match *tok {
token::LitByte(i) => LitByte(parse::byte_lit(i.as_str()).val0()), token::Literal(token::Byte(i)) => LitByte(parse::byte_lit(i.as_str()).val0()),
token::LitChar(i) => LitChar(parse::char_lit(i.as_str()).val0()), token::Literal(token::Char(i)) => LitChar(parse::char_lit(i.as_str()).val0()),
token::LitInteger(s) => parse::integer_lit(s.as_str(), token::Literal(token::Integer(s)) => parse::integer_lit(s.as_str(),
&self.sess.span_diagnostic, &self.sess.span_diagnostic,
self.last_span), self.last_span),
token::LitFloat(s) => parse::float_lit(s.as_str()), token::Literal(token::Float(s)) => parse::float_lit(s.as_str()),
token::LitStr(s) => { token::Literal(token::Str_(s)) => {
LitStr(token::intern_and_get_ident(parse::str_lit(s.as_str()).as_slice()), LitStr(token::intern_and_get_ident(parse::str_lit(s.as_str()).as_slice()),
ast::CookedStr) ast::CookedStr)
} }
token::LitStrRaw(s, n) => { token::Literal(token::StrRaw(s, n)) => {
LitStr(token::intern_and_get_ident(parse::raw_str_lit(s.as_str()).as_slice()), LitStr(token::intern_and_get_ident(parse::raw_str_lit(s.as_str()).as_slice()),
ast::RawStr(n)) ast::RawStr(n))
} }
token::LitBinary(i) => token::Literal(token::Binary(i)) =>
LitBinary(parse::binary_lit(i.as_str())), LitBinary(parse::binary_lit(i.as_str())),
token::LitBinaryRaw(i, _) => token::Literal(token::BinaryRaw(i, _)) =>
LitBinary(Rc::new(i.as_str().as_bytes().iter().map(|&x| x).collect())), LitBinary(Rc::new(i.as_str().as_bytes().iter().map(|&x| x).collect())),
_ => { self.unexpected_last(tok); } _ => { self.unexpected_last(tok); }
} }
@ -2424,7 +2424,7 @@ impl<'a> Parser<'a> {
} }
} }
} }
token::LitInteger(n) => { token::Literal(token::Integer(n)) => {
let index = n.as_str(); let index = n.as_str();
let dot = self.last_span.hi; let dot = self.last_span.hi;
hi = self.span.hi; hi = self.span.hi;
@ -2449,7 +2449,7 @@ impl<'a> Parser<'a> {
} }
} }
} }
token::LitFloat(n) => { token::Literal(token::Float(n)) => {
self.bump(); self.bump();
let last_span = self.last_span; let last_span = self.last_span;
let fstr = n.as_str(); let fstr = n.as_str();
@ -5085,7 +5085,7 @@ impl<'a> Parser<'a> {
self.expect(&token::Semi); self.expect(&token::Semi);
(path, the_ident) (path, the_ident)
}, },
token::LitStr(..) | token::LitStrRaw(..) => { token::Literal(token::Str_(..)) | token::Literal(token::StrRaw(..)) => {
let path = self.parse_str(); let path = self.parse_str();
self.expect_keyword(keywords::As); self.expect_keyword(keywords::As);
let the_ident = self.parse_ident(); let the_ident = self.parse_ident();
@ -5267,7 +5267,7 @@ impl<'a> Parser<'a> {
/// the `extern` keyword, if one is found. /// the `extern` keyword, if one is found.
fn parse_opt_abi(&mut self) -> Option<abi::Abi> { fn parse_opt_abi(&mut self) -> Option<abi::Abi> {
match self.token { match self.token {
token::LitStr(s) | token::LitStrRaw(s, _) => { token::Literal(token::Str_(s)) | token::Literal(token::StrRaw(s, _)) => {
self.bump(); self.bump();
let the_string = s.as_str(); let the_string = s.as_str();
match abi::lookup(the_string) { match abi::lookup(the_string) {
@ -5904,8 +5904,8 @@ impl<'a> Parser<'a> {
pub fn parse_optional_str(&mut self) pub fn parse_optional_str(&mut self)
-> Option<(InternedString, ast::StrStyle)> { -> Option<(InternedString, ast::StrStyle)> {
let (s, style) = match self.token { let (s, style) = match self.token {
token::LitStr(s) => (self.id_to_interned_str(s.ident()), ast::CookedStr), token::Literal(token::Str_(s)) => (self.id_to_interned_str(s.ident()), ast::CookedStr),
token::LitStrRaw(s, n) => { token::Literal(token::StrRaw(s, n)) => {
(self.id_to_interned_str(s.ident()), ast::RawStr(n)) (self.id_to_interned_str(s.ident()), ast::RawStr(n))
} }
_ => return None _ => return None

40
src/libsyntax/parse/token.rs
View file

@ -12,6 +12,7 @@ pub use self::BinOpToken::*;
pub use self::Nonterminal::*; pub use self::Nonterminal::*;
pub use self::DelimToken::*; pub use self::DelimToken::*;
pub use self::IdentStyle::*; pub use self::IdentStyle::*;
pub use self::Lit::*;
pub use self::Token::*; pub use self::Token::*;
use ast; use ast;
@ -59,6 +60,18 @@ pub enum IdentStyle {
Plain, Plain,
} }
#[deriving(Clone, Encodable, Decodable, PartialEq, Eq, Hash, Show)]
pub enum Lit {
Byte(ast::Name),
Char(ast::Name),
Integer(ast::Name),
Float(ast::Name),
Str_(ast::Name),
StrRaw(ast::Name, uint), /* raw str delimited by n hash symbols */
Binary(ast::Name),
BinaryRaw(ast::Name, uint), /* raw binary str delimited by n hash symbols */
}
#[allow(non_camel_case_types)] #[allow(non_camel_case_types)]
#[deriving(Clone, Encodable, Decodable, PartialEq, Eq, Hash, Show)] #[deriving(Clone, Encodable, Decodable, PartialEq, Eq, Hash, Show)]
pub enum Token { pub enum Token {
@ -98,14 +111,7 @@ pub enum Token {
CloseDelim(DelimToken), CloseDelim(DelimToken),
/* Literals */ /* Literals */
LitByte(ast::Name), Literal(Lit),
LitChar(ast::Name),
LitInteger(ast::Name),
LitFloat(ast::Name),
LitStr(ast::Name),
LitStrRaw(ast::Name, uint), /* raw str delimited by n hash symbols */
LitBinary(ast::Name),
LitBinaryRaw(ast::Name, uint), /* raw binary str delimited by n hash symbols */
/* Name components */ /* Name components */
Ident(ast::Ident, IdentStyle), Ident(ast::Ident, IdentStyle),
@ -145,14 +151,7 @@ impl Token {
Ident(_, _) => true, Ident(_, _) => true,
Underscore => true, Underscore => true,
Tilde => true, Tilde => true,
LitByte(_) => true, Literal(_) => true,
LitChar(_) => true,
LitInteger(_) => true,
LitFloat(_) => true,
LitStr(_) => true,
LitStrRaw(_, _) => true,
LitBinary(_) => true,
LitBinaryRaw(_, _) => true,
Pound => true, Pound => true,
At => true, At => true,
Not => true, Not => true,
@ -173,14 +172,7 @@ impl Token {
/// Returns `true` if the token is any literal /// Returns `true` if the token is any literal
pub fn is_lit(&self) -> bool { pub fn is_lit(&self) -> bool {
match *self { match *self {
LitByte(_) => true, Literal(_) => true,
LitChar(_) => true,
LitInteger(_) => true,
LitFloat(_) => true,
LitStr(_) => true,
LitStrRaw(_, _) => true,
LitBinary(_) => true,
LitBinaryRaw(_, _) => true,
_ => false, _ => false,
} }
} }

16
src/libsyntax/print/pprust.rs
View file

@ -236,16 +236,16 @@ pub fn token_to_string(tok: &Token) -> String {
token::Question => "?".into_string(), token::Question => "?".into_string(),
/* Literals */ /* Literals */
token::LitByte(b) => format!("b'{}'", b.as_str()), token::Literal(token::Byte(b)) => format!("b'{}'", b.as_str()),
token::LitChar(c) => format!("'{}'", c.as_str()), token::Literal(token::Char(c)) => format!("'{}'", c.as_str()),
token::LitFloat(c) => c.as_str().into_string(), token::Literal(token::Float(c)) => c.as_str().into_string(),
token::LitInteger(c) => c.as_str().into_string(), token::Literal(token::Integer(c)) => c.as_str().into_string(),
token::LitStr(s) => format!("\"{}\"", s.as_str()), token::Literal(token::Str_(s)) => format!("\"{}\"", s.as_str()),
token::LitStrRaw(s, n) => format!("r{delim}\"{string}\"{delim}", token::Literal(token::StrRaw(s, n)) => format!("r{delim}\"{string}\"{delim}",
delim="#".repeat(n), delim="#".repeat(n),
string=s.as_str()), string=s.as_str()),
token::LitBinary(v) => format!("b\"{}\"", v.as_str()), token::Literal(token::Binary(v)) => format!("b\"{}\"", v.as_str()),
token::LitBinaryRaw(s, n) => format!("br{delim}\"{string}\"{delim}", token::Literal(token::BinaryRaw(s, n)) => format!("br{delim}\"{string}\"{delim}",
delim="#".repeat(n), delim="#".repeat(n),
string=s.as_str()), string=s.as_str()),