bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

rollup merge of #19103: huonw/literal-suffixes

Browse source 
Futureproof Rust for fancier suffixed literals. The Rust compiler tokenises a literal followed immediately (no whitespace) by an identifier as a single token: (for example) the text sequences `"foo"bar`, `1baz` and `1u1024` are now a single token rather than the pairs `"foo"` `bar`, `1` `baz` and `1u` `1024` respectively.

The compiler rejects all such suffixes in the parser, except for the 12 numeric suffixes we have now.

I'm fairly sure this will affect very few programs, since it's not currently legal to have `<literal><identifier>` in a Rust program, except in a macro invocation. Any macro invocation relying on this behaviour can simply separate the two tokens with whitespace: `foo!("bar"baz)` becomes `foo!("bar" baz)`.

This implements [RFC 463](https://github.com/rust-lang/rfcs/blob/master/text/0463-future-proof-literal-suffixes.md), and so closes https://github.com/rust-lang/rust/issues/19088.
This commit is contained in:
Jakub Bukaj 2014-11-19 22:41:05 +01:00
commit f71b852d38
15 changed files with 477 additions and 358 deletions
Download patch file Download diff file Expand all files Collapse all files

122
src/libsyntax/parse/parser.rs
View file

@ -646,6 +646,20 @@ impl<'a> Parser<'a> {
}
}
pub fn expect_no_suffix(&mut self, sp: Span, kind: &str, suffix: Option<ast::Name>) {
match suffix {
None => {/* everything ok */}
Some(suf) => {
let text = suf.as_str();
if text.is_empty() {
self.span_bug(sp, "found empty literal suffix in Some")
}
self.span_err(sp, &*format!("{} with a suffix is illegal", kind));
}
}
}
/// Attempt to consume a `<`. If `<<` is seen, replace it with a single
/// `<` and continue. If a `<` is not seen, return false.
///
@ -968,6 +982,9 @@ impl<'a> Parser<'a> {
pub fn span_err(&mut self, sp: Span, m: &str) {
self.sess.span_diagnostic.span_err(sp, m)
}
pub fn span_bug(&mut self, sp: Span, m: &str) -> ! {
self.sess.span_diagnostic.span_bug(sp, m)
}
pub fn abort_if_errors(&mut self) {
self.sess.span_diagnostic.handler().abort_if_errors();
}
@ -1640,24 +1657,53 @@ impl<'a> Parser<'a> {
/// Matches token_lit = LIT_INTEGER | ...
pub fn lit_from_token(&mut self, tok: &token::Token) -> Lit_ {
match *tok {
token::LitByte(i) => LitByte(parse::byte_lit(i.as_str()).val0()),
token::LitChar(i) => LitChar(parse::char_lit(i.as_str()).val0()),
token::LitInteger(s) => parse::integer_lit(s.as_str(),
&self.sess.span_diagnostic,
self.last_span),
token::LitFloat(s) => parse::float_lit(s.as_str()),
token::LitStr(s) => {
LitStr(token::intern_and_get_ident(parse::str_lit(s.as_str()).as_slice()),
ast::CookedStr)
token::Literal(lit, suf) => {
let (suffix_illegal, out) = match lit {
token::Byte(i) => (true, LitByte(parse::byte_lit(i.as_str()).val0())),
token::Char(i) => (true, LitChar(parse::char_lit(i.as_str()).val0())),
// there are some valid suffixes for integer and
// float literals, so all the handling is done
// internally.
token::Integer(s) => {
(false, parse::integer_lit(s.as_str(),
suf.as_ref().map(|s| s.as_str()),
&self.sess.span_diagnostic,
self.last_span))
}
token::Float(s) => {
(false, parse::float_lit(s.as_str(),
suf.as_ref().map(|s| s.as_str()),
&self.sess.span_diagnostic,
self.last_span))
}
token::Str_(s) => {
(true,
LitStr(token::intern_and_get_ident(parse::str_lit(s.as_str()).as_slice()),
ast::CookedStr))
}
token::StrRaw(s, n) => {
(true,
LitStr(
token::intern_and_get_ident(
parse::raw_str_lit(s.as_str()).as_slice()),
ast::RawStr(n)))
}
token::Binary(i) =>
(true, LitBinary(parse::binary_lit(i.as_str()))),
token::BinaryRaw(i, _) =>
(true,
LitBinary(Rc::new(i.as_str().as_bytes().iter().map(|&x| x).collect()))),
};
if suffix_illegal {
let sp = self.last_span;
self.expect_no_suffix(sp, &*format!("{} literal", lit.short_name()), suf)
}
out
}
token::LitStrRaw(s, n) => {
LitStr(token::intern_and_get_ident(parse::raw_str_lit(s.as_str()).as_slice()),
ast::RawStr(n))
}
token::LitBinary(i) =>
LitBinary(parse::binary_lit(i.as_str())),
token::LitBinaryRaw(i, _) =>
LitBinary(Rc::new(i.as_str().as_bytes().iter().map(|&x| x).collect())),
_ => { self.unexpected_last(tok); }
}
}
@ -2424,7 +2470,10 @@ impl<'a> Parser<'a> {
}
}
}
token::LitInteger(n) => {
token::Literal(token::Integer(n), suf) => {
let sp = self.span;
self.expect_no_suffix(sp, "tuple index", suf);
let index = n.as_str();
let dot = self.last_span.hi;
hi = self.span.hi;
@ -2449,7 +2498,7 @@ impl<'a> Parser<'a> {
}
}
}
token::LitFloat(n) => {
token::Literal(token::Float(n), _suf) => {
self.bump();
let last_span = self.last_span;
let fstr = n.as_str();
@ -5085,12 +5134,17 @@ impl<'a> Parser<'a> {
self.expect(&token::Semi);
(path, the_ident)
},
token::LitStr(..) | token::LitStrRaw(..) => {
let path = self.parse_str();
token::Literal(token::Str_(..), suf) | token::Literal(token::StrRaw(..), suf) => {
let sp = self.span;
self.expect_no_suffix(sp, "extern crate name", suf);
// forgo the internal suffix check of `parse_str` to
// avoid repeats (this unwrap will always succeed due
// to the restriction of the `match`)
let (s, style, _) = self.parse_optional_str().unwrap();
self.expect_keyword(keywords::As);
let the_ident = self.parse_ident();
self.expect(&token::Semi);
(Some(path), the_ident)
(Some((s, style)), the_ident)
},
_ => {
let span = self.span;
@ -5267,7 +5321,9 @@ impl<'a> Parser<'a> {
/// the `extern` keyword, if one is found.
fn parse_opt_abi(&mut self) -> Option<abi::Abi> {
match self.token {
token::LitStr(s) | token::LitStrRaw(s, _) => {
token::Literal(token::Str_(s), suf) | token::Literal(token::StrRaw(s, _), suf) => {
let sp = self.span;
self.expect_no_suffix(sp, "ABI spec", suf);
self.bump();
let the_string = s.as_str();
match abi::lookup(the_string) {
@ -5910,21 +5966,27 @@ impl<'a> Parser<'a> {
}
pub fn parse_optional_str(&mut self)
-> Option<(InternedString, ast::StrStyle)> {
let (s, style) = match self.token {
token::LitStr(s) => (self.id_to_interned_str(s.ident()), ast::CookedStr),
token::LitStrRaw(s, n) => {
(self.id_to_interned_str(s.ident()), ast::RawStr(n))
-> Option<(InternedString, ast::StrStyle, Option<ast::Name>)> {
let ret = match self.token {
token::Literal(token::Str_(s), suf) => {
(self.id_to_interned_str(s.ident()), ast::CookedStr, suf)
}
token::Literal(token::StrRaw(s, n), suf) => {
(self.id_to_interned_str(s.ident()), ast::RawStr(n), suf)
}
_ => return None
};
self.bump();
Some((s, style))
Some(ret)
}
pub fn parse_str(&mut self) -> (InternedString, StrStyle) {
match self.parse_optional_str() {
Some(s) => { s }
Some((s, style, suf)) => {
let sp = self.last_span;
self.expect_no_suffix(sp, "str literal", suf);
(s, style)
}
_ => self.fatal("expected string literal")
}
}