bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Rollup merge of #56964 - nnethercote:TokenStream-IsJoint, r=petrochenkov

Browse source 
Remove `TokenStream::JointTree`.

This is done by adding a new `IsJoint` field to `TokenStream::Tree`,
which simplifies a lot of `match` statements. And likewise for
`CursorKind`.

The commit also adds a new method `TokenTree:stream()` which can replace
a choice between `.into()` and `.joint()`.
This commit is contained in:
Mazdak Farrokhzad 2018-12-23 23:09:07 +01:00 • committed by GitHub
commit 61f50d9d2e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 46 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/libsyntax/parse/lexer/tokentrees.rs
View file

@ -11,7 +11,7 @@
use print::pprust::token_to_string; use print::pprust::token_to_string;
use parse::lexer::StringReader; use parse::lexer::StringReader;
use parse::{token, PResult}; use parse::{token, PResult};
use tokenstream::{DelimSpan, TokenStream, TokenTree}; use tokenstream::{DelimSpan, IsJoint::*, TokenStream, TokenTree};
impl<'a> StringReader<'a> { impl<'a> StringReader<'a> {
// Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`. // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
@ -186,8 +186,7 @@ impl<'a> StringReader<'a> {
let raw = self.span_src_raw; let raw = self.span_src_raw;
self.real_token(); self.real_token();
let is_joint = raw.hi() == self.span_src_raw.lo() && token::is_op(&self.token); let is_joint = raw.hi() == self.span_src_raw.lo() && token::is_op(&self.token);
Ok(TokenStream::Tree(tt, if is_joint { Joint } else { NonJoint }))
Ok(if is_joint { tt.joint() } else { tt.into() })
} }
} }
} }

93
src/libsyntax/tokenstream.rs
View file

@ -123,7 +123,7 @@ impl TokenTree {
} }
pub fn joint(self) -> TokenStream { pub fn joint(self) -> TokenStream {
TokenStream::JointTree(self) TokenStream::Tree(self, Joint)
} }
/// Returns the opening delimiter as a token tree. /// Returns the opening delimiter as a token tree.
@ -156,8 +156,7 @@ impl TokenTree {
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub enum TokenStream { pub enum TokenStream {
Empty, Empty,
Tree(TokenTree), Tree(TokenTree, IsJoint),
JointTree(TokenTree),
Stream(Lrc<Vec<TokenStream>>), Stream(Lrc<Vec<TokenStream>>),
} }
@ -165,6 +164,14 @@ pub enum TokenStream {
#[cfg(target_arch = "x86_64")] #[cfg(target_arch = "x86_64")]
static_assert!(MEM_SIZE_OF_TOKEN_STREAM: mem::size_of::<TokenStream>() == 32); static_assert!(MEM_SIZE_OF_TOKEN_STREAM: mem::size_of::<TokenStream>() == 32);
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum IsJoint {
Joint,
NonJoint
}
use self::IsJoint::*;
impl TokenStream { impl TokenStream {
/// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream` /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream`
/// separating the two arguments with a comma for diagnostic suggestions. /// separating the two arguments with a comma for diagnostic suggestions.
@ -176,16 +183,16 @@ impl TokenStream {
while let Some((pos, ts)) = iter.next() { while let Some((pos, ts)) = iter.next() {
if let Some((_, next)) = iter.peek() { if let Some((_, next)) = iter.peek() {
let sp = match (&ts, &next) { let sp = match (&ts, &next) {
(TokenStream::Tree(TokenTree::Token(_, token::Token::Comma)), _) | (TokenStream::Tree(TokenTree::Token(_, token::Token::Comma), NonJoint), _) |
(_, TokenStream::Tree(TokenTree::Token(_, token::Token::Comma))) => { (_, TokenStream::Tree(TokenTree::Token(_, token::Token::Comma), NonJoint))
continue; => continue,
} (TokenStream::Tree(TokenTree::Token(sp, _), NonJoint), _) => *sp,
(TokenStream::Tree(TokenTree::Token(sp, _)), _) => *sp, (TokenStream::Tree(TokenTree::Delimited(sp, ..), NonJoint), _) =>
(TokenStream::Tree(TokenTree::Delimited(sp, ..)), _) => sp.entire(), sp.entire(),
_ => continue, _ => continue,
}; };
let sp = sp.shrink_to_hi(); let sp = sp.shrink_to_hi();
let comma = TokenStream::Tree(TokenTree::Token(sp, token::Comma)); let comma = TokenStream::Tree(TokenTree::Token(sp, token::Comma), NonJoint);
suggestion = Some((pos, comma, sp)); suggestion = Some((pos, comma, sp));
} }
} }
@ -204,7 +211,7 @@ impl TokenStream {
impl From<TokenTree> for TokenStream { impl From<TokenTree> for TokenStream {
fn from(tt: TokenTree) -> TokenStream { fn from(tt: TokenTree) -> TokenStream {
TokenStream::Tree(tt) TokenStream::Tree(tt, NonJoint)
} }
} }
@ -232,7 +239,7 @@ impl Extend<TokenStream> for TokenStream {
vec.reserve(iter.size_hint().0); vec.reserve(iter.size_hint().0);
vec vec
} }
TokenStream::Tree(_) | TokenStream::JointTree(_) => { TokenStream::Tree(..) => {
let mut vec = Vec::new(); let mut vec = Vec::new();
vec.reserve(1 + iter.size_hint().0); vec.reserve(1 + iter.size_hint().0);
vec.push(this); vec.push(this);
@ -369,8 +376,7 @@ impl TokenStream {
/// Returns true if the token tree is a joint operation w.r.t. `proc_macro::TokenNode`. /// Returns true if the token tree is a joint operation w.r.t. `proc_macro::TokenNode`.
pub fn as_tree(self) -> (TokenTree, bool /* joint? */) { pub fn as_tree(self) -> (TokenTree, bool /* joint? */) {
match self { match self {
TokenStream::Tree(tree) => (tree, false), TokenStream::Tree(tree, is_joint) => (tree, is_joint == Joint),
TokenStream::JointTree(tree) => (tree, true),
_ => unreachable!(), _ => unreachable!(),
} }
} }
@ -381,8 +387,7 @@ impl TokenStream {
let mut i = 0; let mut i = 0;
while let Some(stream) = trees.next_as_stream() { while let Some(stream) = trees.next_as_stream() {
result.push(match stream { result.push(match stream {
TokenStream::Tree(tree) => f(i, tree).into(), TokenStream::Tree(tree, is_joint) => TokenStream::Tree(f(i, tree), is_joint),
TokenStream::JointTree(tree) => f(i, tree).joint(),
_ => unreachable!() _ => unreachable!()
}); });
i += 1; i += 1;
@ -395,27 +400,25 @@ impl TokenStream {
let mut result = Vec::new(); let mut result = Vec::new();
while let Some(stream) = trees.next_as_stream() { while let Some(stream) = trees.next_as_stream() {
result.push(match stream { result.push(match stream {
TokenStream::Tree(tree) => f(tree).into(), TokenStream::Tree(tree, is_joint) => TokenStream::Tree(f(tree), is_joint),
TokenStream::JointTree(tree) => f(tree).joint(),
_ => unreachable!() _ => unreachable!()
}); });
} }
TokenStream::new(result) TokenStream::new(result)
} }
fn first_tree_and_joint(&self) -> Option<(TokenTree, bool)> { fn first_tree_and_joint(&self) -> Option<(TokenTree, IsJoint)> {
match self { match self {
TokenStream::Empty => None, TokenStream::Empty => None,
TokenStream::Tree(ref tree) => Some((tree.clone(), false)), TokenStream::Tree(ref tree, is_joint) => Some((tree.clone(), *is_joint)),
TokenStream::JointTree(ref tree) => Some((tree.clone(), true)),
TokenStream::Stream(ref stream) => stream.first().unwrap().first_tree_and_joint(), TokenStream::Stream(ref stream) => stream.first().unwrap().first_tree_and_joint(),
} }
} }
fn last_tree_if_joint(&self) -> Option<TokenTree> { fn last_tree_if_joint(&self) -> Option<TokenTree> {
match self { match self {
TokenStream::Empty | TokenStream::Tree(..) => None, TokenStream::Empty | TokenStream::Tree(_, NonJoint) => None,
TokenStream::JointTree(ref tree) => Some(tree.clone()), TokenStream::Tree(ref tree, Joint) => Some(tree.clone()),
TokenStream::Stream(ref stream) => stream.last().unwrap().last_tree_if_joint(), TokenStream::Stream(ref stream) => stream.last().unwrap().last_tree_if_joint(),
} }
} }
@ -439,11 +442,7 @@ impl TokenStreamBuilder {
self.push_all_but_last_tree(&last_stream); self.push_all_but_last_tree(&last_stream);
let glued_span = last_span.to(span); let glued_span = last_span.to(span);
let glued_tt = TokenTree::Token(glued_span, glued_tok); let glued_tt = TokenTree::Token(glued_span, glued_tok);
let glued_tokenstream = if is_joint { let glued_tokenstream = TokenStream::Tree(glued_tt, is_joint);
glued_tt.joint()
} else {
glued_tt.into()
};
self.0.push(glued_tokenstream); self.0.push(glued_tokenstream);
self.push_all_but_first_tree(&stream); self.push_all_but_first_tree(&stream);
return return
@ -493,8 +492,7 @@ pub struct Cursor(CursorKind);
#[derive(Clone)] #[derive(Clone)]
enum CursorKind { enum CursorKind {
Empty, Empty,
Tree(TokenTree, bool /* consumed? */), Tree(TokenTree, IsJoint, bool /* consumed? */),
JointTree(TokenTree, bool /* consumed? */),
Stream(StreamCursor), Stream(StreamCursor),
} }
@ -516,9 +514,9 @@ impl StreamCursor {
self.index += 1; self.index += 1;
let next = self.stream[self.index - 1].clone(); let next = self.stream[self.index - 1].clone();
match next { match next {
TokenStream::Tree(..) | TokenStream::JointTree(..) => return Some(next),
TokenStream::Stream(stream) => self.insert(stream),
TokenStream::Empty => {} TokenStream::Empty => {}
TokenStream::Tree(..) => return Some(next),
TokenStream::Stream(stream) => self.insert(stream),
} }
} else if let Some((stream, index)) = self.stack.pop() { } else if let Some((stream, index)) = self.stack.pop() {
self.stream = stream; self.stream = stream;
@ -540,7 +538,7 @@ impl Iterator for Cursor {
fn next(&mut self) -> Option<TokenTree> { fn next(&mut self) -> Option<TokenTree> {
self.next_as_stream().map(|stream| match stream { self.next_as_stream().map(|stream| match stream {
TokenStream::Tree(tree) | TokenStream::JointTree(tree) => tree, TokenStream::Tree(tree, _) => tree,
_ => unreachable!() _ => unreachable!()
}) })
} }
@ -550,18 +548,15 @@ impl Cursor {
fn new(stream: TokenStream) -> Self { fn new(stream: TokenStream) -> Self {
Cursor(match stream { Cursor(match stream {
TokenStream::Empty => CursorKind::Empty, TokenStream::Empty => CursorKind::Empty,
TokenStream::Tree(tree) => CursorKind::Tree(tree, false), TokenStream::Tree(tree, is_joint) => CursorKind::Tree(tree, is_joint, false),
TokenStream::JointTree(tree) => CursorKind::JointTree(tree, false),
TokenStream::Stream(stream) => CursorKind::Stream(StreamCursor::new(stream)), TokenStream::Stream(stream) => CursorKind::Stream(StreamCursor::new(stream)),
}) })
} }
pub fn next_as_stream(&mut self) -> Option<TokenStream> { pub fn next_as_stream(&mut self) -> Option<TokenStream> {
let (stream, consumed) = match self.0 { let (stream, consumed) = match self.0 {
CursorKind::Tree(ref tree, ref mut consumed @ false) => CursorKind::Tree(ref tree, ref is_joint, ref mut consumed @ false) =>
(tree.clone().into(), consumed), (TokenStream::Tree(tree.clone(), *is_joint), consumed),
CursorKind::JointTree(ref tree, ref mut consumed @ false) =>
(tree.clone().joint(), consumed),
CursorKind::Stream(ref mut cursor) => return cursor.next_as_stream(), CursorKind::Stream(ref mut cursor) => return cursor.next_as_stream(),
_ => return None, _ => return None,
}; };
@ -574,7 +569,7 @@ impl Cursor {
match self.0 { match self.0 {
_ if stream.is_empty() => return, _ if stream.is_empty() => return,
CursorKind::Empty => *self = stream.trees(), CursorKind::Empty => *self = stream.trees(),
CursorKind::Tree(_, consumed) | CursorKind::JointTree(_, consumed) => { CursorKind::Tree(_, _, consumed) => {
*self = TokenStream::new(vec![self.original_stream(), stream]).trees(); *self = TokenStream::new(vec![self.original_stream(), stream]).trees();
if consumed { if consumed {
self.next(); self.next();
@ -589,8 +584,8 @@ impl Cursor {
pub fn original_stream(&self) -> TokenStream { pub fn original_stream(&self) -> TokenStream {
match self.0 { match self.0 {
CursorKind::Empty => TokenStream::empty(), CursorKind::Empty => TokenStream::empty(),
CursorKind::Tree(ref tree, _) => tree.clone().into(), CursorKind::Tree(ref tree, ref is_joint, _) =>
CursorKind::JointTree(ref tree, _) => tree.clone().joint(), TokenStream::Tree(tree.clone(), *is_joint),
CursorKind::Stream(ref cursor) => TokenStream::Stream( CursorKind::Stream(ref cursor) => TokenStream::Stream(
cursor.stack.get(0).cloned().map(|(stream, _)| stream) cursor.stack.get(0).cloned().map(|(stream, _)| stream)
.unwrap_or_else(|| cursor.stream.clone()) .unwrap_or_else(|| cursor.stream.clone())
@ -602,9 +597,8 @@ impl Cursor {
fn look_ahead(streams: &[TokenStream], mut n: usize) -> Result<TokenTree, usize> { fn look_ahead(streams: &[TokenStream], mut n: usize) -> Result<TokenTree, usize> {
for stream in streams { for stream in streams {
n = match stream { n = match stream {
TokenStream::Tree(ref tree) | TokenStream::JointTree(ref tree) TokenStream::Tree(ref tree, _) if n == 0 => return Ok(tree.clone()),
if n == 0 => return Ok(tree.clone()), TokenStream::Tree(..) => n - 1,
TokenStream::Tree(..) | TokenStream::JointTree(..) => n - 1,
TokenStream::Stream(ref stream) => match look_ahead(stream, n) { TokenStream::Stream(ref stream) => match look_ahead(stream, n) {
Ok(tree) => return Ok(tree), Ok(tree) => return Ok(tree),
Err(n) => n, Err(n) => n,
@ -617,10 +611,8 @@ impl Cursor {
match self.0 { match self.0 {
CursorKind::Empty | CursorKind::Empty |
CursorKind::Tree(_, true) | CursorKind::Tree(_, _, true) => Err(n),
CursorKind::JointTree(_, true) => Err(n), CursorKind::Tree(ref tree, _, false) => look_ahead(&[tree.clone().into()], n),
CursorKind::Tree(ref tree, false) |
CursorKind::JointTree(ref tree, false) => look_ahead(&[tree.clone().into()], n),
CursorKind::Stream(ref cursor) => { CursorKind::Stream(ref cursor) => {
look_ahead(&cursor.stream[cursor.index ..], n).or_else(|mut n| { look_ahead(&cursor.stream[cursor.index ..], n).or_else(|mut n| {
for &(ref stream, index) in cursor.stack.iter().rev() { for &(ref stream, index) in cursor.stack.iter().rev() {
@ -653,8 +645,7 @@ impl From<TokenStream> for ThinTokenStream {
fn from(stream: TokenStream) -> ThinTokenStream { fn from(stream: TokenStream) -> ThinTokenStream {
ThinTokenStream(match stream { ThinTokenStream(match stream {
TokenStream::Empty => None, TokenStream::Empty => None,
TokenStream::Tree(tree) => Some(Lrc::new(vec![tree.into()])), TokenStream::Tree(..) => Some(Lrc::new(vec![stream])),
TokenStream::JointTree(tree) => Some(Lrc::new(vec![tree.joint()])),
TokenStream::Stream(stream) => Some(stream), TokenStream::Stream(stream) => Some(stream),
}) })
} }

8
src/libsyntax_ext/proc_macro_server.rs
View file

@ -21,7 +21,7 @@ use syntax::ast;
use syntax::ext::base::ExtCtxt; use syntax::ext::base::ExtCtxt;
use syntax::parse::lexer::comments; use syntax::parse::lexer::comments;
use syntax::parse::{self, token, ParseSess}; use syntax::parse::{self, token, ParseSess};
use syntax::tokenstream::{self, DelimSpan, TokenStream}; use syntax::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream};
use syntax_pos::hygiene::{SyntaxContext, Transparency}; use syntax_pos::hygiene::{SyntaxContext, Transparency};
use syntax_pos::symbol::{keywords, Symbol}; use syntax_pos::symbol::{keywords, Symbol};
use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span}; use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span};
@ -278,11 +278,7 @@ impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> {
}; };
let tree = tokenstream::TokenTree::Token(span, token); let tree = tokenstream::TokenTree::Token(span, token);
if joint { TokenStream::Tree(tree, if joint { Joint } else { NonJoint })
tree.joint()
} else {
tree.into()
}
} }
} }