1
Fork 0

Remove TreeAndSpacing.

A `TokenStream` contains a `Lrc<Vec<(TokenTree, Spacing)>>`. But this is
not quite right. `Spacing` makes sense for `TokenTree::Token`, but does
not make sense for `TokenTree::Delimited`, because a
`TokenTree::Delimited` cannot be joined with another `TokenTree`.

This commit fixes this problem, by adding `Spacing` to `TokenTree::Token`,
changing `TokenStream` to contain a `Lrc<Vec<TokenTree>>`, and removing the
`TreeAndSpacing` typedef.

The commit removes these two impls:
- `impl From<TokenTree> for TokenStream`
- `impl From<TokenTree> for TreeAndSpacing`

These were useful, but also resulted in code with many `.into()` calls
that was hard to read, particularly for anyone not highly familiar with
the relevant types. This commit makes some other changes to compensate:
- `TokenTree::token()` becomes `TokenTree::token_{alone,joint}()`.
- `TokenStream::token_{alone,joint}()` are added.
- `TokenStream::delimited` is added.

This results in things like this:
```rust
TokenTree::token(token::Semi, stmt.span).into()
```
changing to this:
```rust
TokenStream::token_alone(token::Semi, stmt.span)
```
This makes the type of the result, and its spacing, clearer.

These changes also simplifies `Cursor` and `CursorRef`, because they no longer
need to distinguish between `next` and `next_with_spacing`.
This commit is contained in:
Nicholas Nethercote 2022-07-28 10:31:04 +10:00
parent bdf520fd41
commit 332dffb1f9
23 changed files with 317 additions and 307 deletions

View file

@ -401,7 +401,7 @@ impl<'a> StripUnconfigured<'a> {
// Use the `#` in `#[cfg_attr(pred, attr)]` as the `#` token
// for `attr` when we expand it to `#[attr]`
let mut orig_trees = orig_tokens.into_trees();
let TokenTree::Token(pound_token @ Token { kind: TokenKind::Pound, .. }) = orig_trees.next().unwrap() else {
let TokenTree::Token(pound_token @ Token { kind: TokenKind::Pound, .. }, _) = orig_trees.next().unwrap() else {
panic!("Bad tokens for attribute {:?}", attr);
};
let pound_span = pound_token.span;
@ -409,7 +409,7 @@ impl<'a> StripUnconfigured<'a> {
let mut trees = vec![(AttrAnnotatedTokenTree::Token(pound_token), Spacing::Alone)];
if attr.style == AttrStyle::Inner {
// For inner attributes, we do the same thing for the `!` in `#![some_attr]`
let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }) = orig_trees.next().unwrap() else {
let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) = orig_trees.next().unwrap() else {
panic!("Bad tokens for attribute {:?}", attr);
};
trees.push((AttrAnnotatedTokenTree::Token(bang_token), Spacing::Alone));

View file

@ -481,7 +481,7 @@ pub fn compile_declarative_macro(
.map(|m| {
if let MatchedTokenTree(ref tt) = *m {
let tt = mbe::quoted::parse(
tt.clone().into(),
TokenStream::new(vec![tt.clone()]),
true,
&sess.parse_sess,
def.id,
@ -505,7 +505,7 @@ pub fn compile_declarative_macro(
.map(|m| {
if let MatchedTokenTree(ref tt) = *m {
return mbe::quoted::parse(
tt.clone().into(),
TokenStream::new(vec![tt.clone()]),
false,
&sess.parse_sess,
def.id,

View file

@ -106,7 +106,7 @@ fn parse_depth<'sess>(
let Some(tt) = iter.next() else { return Ok(0) };
let TokenTree::Token(token::Token {
kind: token::TokenKind::Literal(lit), ..
}) = tt else {
}, _) = tt else {
return Err(sess.span_diagnostic.struct_span_err(
span,
"meta-variable expression depth must be a literal"
@ -130,7 +130,7 @@ fn parse_ident<'sess>(
sess: &'sess ParseSess,
span: Span,
) -> PResult<'sess, Ident> {
if let Some(tt) = iter.next() && let TokenTree::Token(token) = tt {
if let Some(tt) = iter.next() && let TokenTree::Token(token, _) = tt {
if let Some((elem, false)) = token.ident() {
return Ok(elem);
}
@ -153,7 +153,7 @@ fn parse_ident<'sess>(
/// Tries to move the iterator forward returning `true` if there is a comma. If not, then the
/// iterator is not modified and the result is `false`.
fn try_eat_comma(iter: &mut CursorRef<'_>) -> bool {
if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. })) = iter.look_ahead(0) {
if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. }, _)) = iter.look_ahead(0) {
let _ = iter.next();
return true;
}

View file

@ -56,9 +56,9 @@ pub(super) fn parse(
match tree {
TokenTree::MetaVar(start_sp, ident) if parsing_patterns => {
let span = match trees.next() {
Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => {
Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span }, _)) => {
match trees.next() {
Some(tokenstream::TokenTree::Token(token)) => match token.ident() {
Some(tokenstream::TokenTree::Token(token, _)) => match token.ident() {
Some((frag, _)) => {
let span = token.span.with_lo(start_sp.lo());
@ -146,7 +146,7 @@ fn parse_tree(
// Depending on what `tree` is, we could be parsing different parts of a macro
match tree {
// `tree` is a `$` token. Look at the next token in `trees`
tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => {
tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _) => {
// FIXME: Handle `Invisible`-delimited groups in a more systematic way
// during parsing.
let mut next = outer_trees.next();
@ -217,7 +217,7 @@ fn parse_tree(
// `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate`
// special metavariable that names the crate of the invocation.
Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => {
Some(tokenstream::TokenTree::Token(token, _)) if token.is_ident() => {
let (ident, is_raw) = token.ident().unwrap();
let span = ident.span.with_lo(span.lo());
if ident.name == kw::Crate && !is_raw {
@ -228,7 +228,7 @@ fn parse_tree(
}
// `tree` is followed by another `$`. This is an escaped `$`.
Some(tokenstream::TokenTree::Token(Token { kind: token::Dollar, span })) => {
Some(tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _)) => {
if parsing_patterns {
span_dollar_dollar_or_metavar_in_the_lhs_err(
sess,
@ -241,7 +241,7 @@ fn parse_tree(
}
// `tree` is followed by some other token. This is an error.
Some(tokenstream::TokenTree::Token(token)) => {
Some(tokenstream::TokenTree::Token(token, _)) => {
let msg = format!(
"expected identifier, found `{}`",
pprust::token_to_string(&token),
@ -256,7 +256,7 @@ fn parse_tree(
}
// `tree` is an arbitrary token. Keep it.
tokenstream::TokenTree::Token(token) => TokenTree::Token(token),
tokenstream::TokenTree::Token(token, _) => TokenTree::Token(token),
// `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to
// descend into the delimited set and further parse it.
@ -291,7 +291,7 @@ fn parse_kleene_op(
span: Span,
) -> Result<Result<(KleeneOp, Span), Token>, Span> {
match input.next() {
Some(tokenstream::TokenTree::Token(token)) => match kleene_op(&token) {
Some(tokenstream::TokenTree::Token(token, _)) => match kleene_op(&token) {
Some(op) => Ok(Ok((op, token.span))),
None => Ok(Err(token)),
},

View file

@ -3,7 +3,7 @@ use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, MatchedTokenTree,
use crate::mbe::{self, MetaVarExpr};
use rustc_ast::mut_visit::{self, MutVisitor};
use rustc_ast::token::{self, Delimiter, Token, TokenKind};
use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndSpacing};
use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree};
use rustc_data_structures::fx::FxHashMap;
use rustc_errors::{pluralize, PResult};
use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed};
@ -105,7 +105,7 @@ pub(super) fn transcribe<'a>(
//
// Thus, if we try to pop the `result_stack` and it is empty, we have reached the top-level
// again, and we are done transcribing.
let mut result: Vec<TreeAndSpacing> = Vec::new();
let mut result: Vec<TokenTree> = Vec::new();
let mut result_stack = Vec::new();
let mut marker = Marker(cx.current_expansion.id, transparency);
@ -123,7 +123,7 @@ pub(super) fn transcribe<'a>(
if repeat_idx < repeat_len {
*idx = 0;
if let Some(sep) = sep {
result.push(TokenTree::Token(sep.clone()).into());
result.push(TokenTree::Token(sep.clone(), Spacing::Alone));
}
continue;
}
@ -150,7 +150,7 @@ pub(super) fn transcribe<'a>(
// Step back into the parent Delimited.
let tree = TokenTree::Delimited(span, delim, TokenStream::new(result));
result = result_stack.pop().unwrap();
result.push(tree.into());
result.push(tree);
}
}
continue;
@ -227,15 +227,15 @@ pub(super) fn transcribe<'a>(
// `tt`s are emitted into the output stream directly as "raw tokens",
// without wrapping them into groups.
let token = tt.clone();
result.push(token.into());
result.push(token);
}
MatchedNonterminal(ref nt) => {
// Other variables are emitted into the output stream as groups with
// `Delimiter::Invisible` to maintain parsing priorities.
// `Interpolated` is currently used for such groups in rustc parser.
marker.visit_span(&mut sp);
let token = TokenTree::token(token::Interpolated(nt.clone()), sp);
result.push(token.into());
let token = TokenTree::token_alone(token::Interpolated(nt.clone()), sp);
result.push(token);
}
MatchedSeq(..) => {
// We were unable to descend far enough. This is an error.
@ -250,8 +250,11 @@ pub(super) fn transcribe<'a>(
// with modified syntax context. (I believe this supports nested macros).
marker.visit_span(&mut sp);
marker.visit_ident(&mut original_ident);
result.push(TokenTree::token(token::Dollar, sp).into());
result.push(TokenTree::Token(Token::from_ast_ident(original_ident)).into());
result.push(TokenTree::token_alone(token::Dollar, sp));
result.push(TokenTree::Token(
Token::from_ast_ident(original_ident),
Spacing::Alone,
));
}
}
@ -281,8 +284,8 @@ pub(super) fn transcribe<'a>(
mbe::TokenTree::Token(token) => {
let mut token = token.clone();
mut_visit::visit_token(&mut token, &mut marker);
let tt = TokenTree::Token(token);
result.push(tt.into());
let tt = TokenTree::Token(token, Spacing::Alone);
result.push(tt);
}
// There should be no meta-var declarations in the invocation of a macro.
@ -532,7 +535,7 @@ fn transcribe_metavar_expr<'a>(
interp: &FxHashMap<MacroRulesNormalizedIdent, NamedMatch>,
marker: &mut Marker,
repeats: &[(usize, usize)],
result: &mut Vec<TreeAndSpacing>,
result: &mut Vec<TokenTree>,
sp: &DelimSpan,
) -> PResult<'a, ()> {
let mut visited_span = || {
@ -544,11 +547,11 @@ fn transcribe_metavar_expr<'a>(
MetaVarExpr::Count(original_ident, depth_opt) => {
let matched = matched_from_ident(cx, original_ident, interp)?;
let count = count_repetitions(cx, depth_opt, matched, &repeats, sp)?;
let tt = TokenTree::token(
let tt = TokenTree::token_alone(
TokenKind::lit(token::Integer, sym::integer(count), None),
visited_span(),
);
result.push(tt.into());
result.push(tt);
}
MetaVarExpr::Ignore(original_ident) => {
// Used to ensure that `original_ident` is present in the LHS
@ -556,25 +559,19 @@ fn transcribe_metavar_expr<'a>(
}
MetaVarExpr::Index(depth) => match repeats.iter().nth_back(depth) {
Some((index, _)) => {
result.push(
TokenTree::token(
TokenKind::lit(token::Integer, sym::integer(*index), None),
visited_span(),
)
.into(),
);
result.push(TokenTree::token_alone(
TokenKind::lit(token::Integer, sym::integer(*index), None),
visited_span(),
));
}
None => return Err(out_of_bounds_err(cx, repeats.len(), sp.entire(), "index")),
},
MetaVarExpr::Length(depth) => match repeats.iter().nth_back(depth) {
Some((_, length)) => {
result.push(
TokenTree::token(
TokenKind::lit(token::Integer, sym::integer(*length), None),
visited_span(),
)
.into(),
);
result.push(TokenTree::token_alone(
TokenKind::lit(token::Integer, sym::integer(*length), None),
visited_span(),
));
}
None => return Err(out_of_bounds_err(cx, repeats.len(), sp.entire(), "length")),
},

View file

@ -66,23 +66,23 @@ fn string_to_tts_macro() {
match tts {
[
TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }),
TokenTree::Token(Token { kind: token::Not, .. }),
TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }),
TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }, _),
TokenTree::Token(Token { kind: token::Not, .. }, _),
TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }, _),
TokenTree::Delimited(_, macro_delim, macro_tts),
] if name_macro_rules == &kw::MacroRules && name_zip.as_str() == "zip" => {
let tts = &macro_tts.trees().collect::<Vec<_>>();
match &tts[..] {
[
TokenTree::Delimited(_, first_delim, first_tts),
TokenTree::Token(Token { kind: token::FatArrow, .. }),
TokenTree::Token(Token { kind: token::FatArrow, .. }, _),
TokenTree::Delimited(_, second_delim, second_tts),
] if macro_delim == &Delimiter::Parenthesis => {
let tts = &first_tts.trees().collect::<Vec<_>>();
match &tts[..] {
[
TokenTree::Token(Token { kind: token::Dollar, .. }),
TokenTree::Token(Token { kind: token::Ident(name, false), .. }),
TokenTree::Token(Token { kind: token::Dollar, .. }, _),
TokenTree::Token(Token { kind: token::Ident(name, false), .. }, _),
] if first_delim == &Delimiter::Parenthesis && name.as_str() == "a" => {
}
_ => panic!("value 3: {:?} {:?}", first_delim, first_tts),
@ -90,8 +90,8 @@ fn string_to_tts_macro() {
let tts = &second_tts.trees().collect::<Vec<_>>();
match &tts[..] {
[
TokenTree::Token(Token { kind: token::Dollar, .. }),
TokenTree::Token(Token { kind: token::Ident(name, false), .. }),
TokenTree::Token(Token { kind: token::Dollar, .. }, _),
TokenTree::Token(Token { kind: token::Ident(name, false), .. }, _),
] if second_delim == &Delimiter::Parenthesis
&& name.as_str() == "a" => {}
_ => panic!("value 4: {:?} {:?}", second_delim, second_tts),
@ -111,29 +111,27 @@ fn string_to_tts_1() {
let tts = string_to_stream("fn a (b : i32) { b; }".to_string());
let expected = TokenStream::new(vec![
TokenTree::token(token::Ident(kw::Fn, false), sp(0, 2)).into(),
TokenTree::token(token::Ident(Symbol::intern("a"), false), sp(3, 4)).into(),
TokenTree::token_alone(token::Ident(kw::Fn, false), sp(0, 2)),
TokenTree::token_alone(token::Ident(Symbol::intern("a"), false), sp(3, 4)),
TokenTree::Delimited(
DelimSpan::from_pair(sp(5, 6), sp(13, 14)),
Delimiter::Parenthesis,
TokenStream::new(vec![
TokenTree::token(token::Ident(Symbol::intern("b"), false), sp(6, 7)).into(),
TokenTree::token(token::Colon, sp(8, 9)).into(),
TokenTree::token(token::Ident(sym::i32, false), sp(10, 13)).into(),
TokenTree::token_alone(token::Ident(Symbol::intern("b"), false), sp(6, 7)),
TokenTree::token_alone(token::Colon, sp(8, 9)),
TokenTree::token_alone(token::Ident(sym::i32, false), sp(10, 13)),
])
.into(),
)
.into(),
),
TokenTree::Delimited(
DelimSpan::from_pair(sp(15, 16), sp(20, 21)),
Delimiter::Brace,
TokenStream::new(vec![
TokenTree::token(token::Ident(Symbol::intern("b"), false), sp(17, 18)).into(),
TokenTree::token(token::Semi, sp(18, 19)).into(),
TokenTree::token_joint(token::Ident(Symbol::intern("b"), false), sp(17, 18)),
TokenTree::token_alone(token::Semi, sp(18, 19)),
])
.into(),
)
.into(),
),
]);
assert_eq!(tts, expected);

View file

@ -4,7 +4,7 @@ use crate::proc_macro_server;
use rustc_ast as ast;
use rustc_ast::ptr::P;
use rustc_ast::token;
use rustc_ast::tokenstream::{TokenStream, TokenTree};
use rustc_ast::tokenstream::TokenStream;
use rustc_data_structures::sync::Lrc;
use rustc_errors::ErrorGuaranteed;
use rustc_parse::parser::ForceCollect;
@ -94,7 +94,7 @@ impl MultiItemModifier for DeriveProcMacro {
Annotatable::Stmt(stmt) => token::NtStmt(stmt),
_ => unreachable!(),
};
TokenTree::token(token::Interpolated(Lrc::new(nt)), DUMMY_SP).into()
TokenStream::token_alone(token::Interpolated(Lrc::new(nt)), DUMMY_SP)
} else {
item.to_tokens()
};

View file

@ -92,9 +92,8 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
let mut trees = Vec::with_capacity(stream.len().next_power_of_two());
let mut cursor = stream.into_trees();
while let Some((tree, spacing)) = cursor.next_with_spacing() {
let joint = spacing == Joint;
let Token { kind, span } = match tree {
while let Some(tree) = cursor.next() {
let (Token { kind, span }, joint) = match tree {
tokenstream::TokenTree::Delimited(span, delim, tts) => {
let delimiter = pm::Delimiter::from_internal(delim);
trees.push(TokenTree::Group(Group {
@ -108,7 +107,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
}));
continue;
}
tokenstream::TokenTree::Token(token) => token,
tokenstream::TokenTree::Token(token, spacing) => (token, spacing == Joint),
};
let mut op = |s: &str| {
@ -194,7 +193,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
TokenKind::lit(token::Str, Symbol::intern(&escaped), None),
]
.into_iter()
.map(|kind| tokenstream::TokenTree::token(kind, span))
.map(|kind| tokenstream::TokenTree::token_alone(kind, span))
.collect();
trees.push(TokenTree::Punct(Punct { ch: b'#', joint: false, span }));
if attr_style == ast::AttrStyle::Inner {
@ -246,16 +245,15 @@ impl ToInternal<TokenStream> for (TokenTree<TokenStream, Span, Symbol>, &mut Rus
let (ch, joint, span) = match tree {
TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span),
TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => {
return tokenstream::TokenTree::Delimited(
return tokenstream::TokenStream::delimited(
tokenstream::DelimSpan { open, close },
delimiter.to_internal(),
stream.unwrap_or_default(),
)
.into();
);
}
TokenTree::Ident(self::Ident { sym, is_raw, span }) => {
rustc.sess().symbol_gallery.insert(sym, span);
return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into();
return tokenstream::TokenStream::token_alone(Ident(sym, is_raw), span);
}
TokenTree::Literal(self::Literal {
kind: self::LitKind::Integer,
@ -266,8 +264,8 @@ impl ToInternal<TokenStream> for (TokenTree<TokenStream, Span, Symbol>, &mut Rus
let minus = BinOp(BinOpToken::Minus);
let symbol = Symbol::intern(&symbol.as_str()[1..]);
let integer = TokenKind::lit(token::Integer, symbol, suffix);
let a = tokenstream::TokenTree::token(minus, span);
let b = tokenstream::TokenTree::token(integer, span);
let a = tokenstream::TokenTree::token_alone(minus, span);
let b = tokenstream::TokenTree::token_alone(integer, span);
return [a, b].into_iter().collect();
}
TokenTree::Literal(self::Literal {
@ -279,16 +277,15 @@ impl ToInternal<TokenStream> for (TokenTree<TokenStream, Span, Symbol>, &mut Rus
let minus = BinOp(BinOpToken::Minus);
let symbol = Symbol::intern(&symbol.as_str()[1..]);
let float = TokenKind::lit(token::Float, symbol, suffix);
let a = tokenstream::TokenTree::token(minus, span);
let b = tokenstream::TokenTree::token(float, span);
let a = tokenstream::TokenTree::token_alone(minus, span);
let b = tokenstream::TokenTree::token_alone(float, span);
return [a, b].into_iter().collect();
}
TokenTree::Literal(self::Literal { kind, symbol, suffix, span }) => {
return tokenstream::TokenTree::token(
return tokenstream::TokenStream::token_alone(
TokenKind::lit(kind.to_internal(), symbol, suffix),
span,
)
.into();
);
}
};
@ -318,8 +315,11 @@ impl ToInternal<TokenStream> for (TokenTree<TokenStream, Span, Symbol>, &mut Rus
_ => unreachable!(),
};
let tree = tokenstream::TokenTree::token(kind, span);
TokenStream::new(vec![(tree, if joint { Joint } else { Alone })])
if joint {
tokenstream::TokenStream::token_joint(kind, span)
} else {
tokenstream::TokenStream::token_alone(kind, span)
}
}
}
@ -486,12 +486,11 @@ impl server::TokenStream for Rustc<'_, '_> {
// We don't use `TokenStream::from_ast` as the tokenstream currently cannot
// be recovered in the general case.
match &expr.kind {
ast::ExprKind::Lit(l) if l.token.kind == token::Bool => {
Ok(tokenstream::TokenTree::token(token::Ident(l.token.symbol, false), l.span)
.into())
}
ast::ExprKind::Lit(l) if l.token.kind == token::Bool => Ok(
tokenstream::TokenStream::token_alone(token::Ident(l.token.symbol, false), l.span),
),
ast::ExprKind::Lit(l) => {
Ok(tokenstream::TokenTree::token(token::Literal(l.token), l.span).into())
Ok(tokenstream::TokenStream::token_alone(token::Literal(l.token), l.span))
}
ast::ExprKind::Unary(ast::UnOp::Neg, e) => match &e.kind {
ast::ExprKind::Lit(l) => match l.token {
@ -499,8 +498,8 @@ impl server::TokenStream for Rustc<'_, '_> {
Ok(Self::TokenStream::from_iter([
// FIXME: The span of the `-` token is lost when
// parsing, so we cannot faithfully recover it here.
tokenstream::TokenTree::token(token::BinOp(token::Minus), e.span),
tokenstream::TokenTree::token(token::Literal(l.token), l.span),
tokenstream::TokenTree::token_alone(token::BinOp(token::Minus), e.span),
tokenstream::TokenTree::token_alone(token::Literal(l.token), l.span),
]))
}
_ => Err(()),

View file

@ -1,7 +1,7 @@
use crate::tests::string_to_stream;
use rustc_ast::token;
use rustc_ast::tokenstream::{Spacing, TokenStream, TokenStreamBuilder, TokenTree};
use rustc_ast::tokenstream::{TokenStream, TokenStreamBuilder};
use rustc_span::create_default_session_globals_then;
use rustc_span::{BytePos, Span, Symbol};
@ -13,10 +13,6 @@ fn sp(a: u32, b: u32) -> Span {
Span::with_root_ctxt(BytePos(a), BytePos(b))
}
fn joint(tree: TokenTree) -> TokenStream {
TokenStream::new(vec![(tree, Spacing::Joint)])
}
#[test]
fn test_concat() {
create_default_session_globals_then(|| {
@ -90,9 +86,8 @@ fn test_diseq_1() {
#[test]
fn test_is_empty() {
create_default_session_globals_then(|| {
let test0: TokenStream = Vec::<TokenTree>::new().into_iter().collect();
let test1: TokenStream =
TokenTree::token(token::Ident(Symbol::intern("a"), false), sp(0, 1)).into();
let test0 = TokenStream::default();
let test1 = TokenStream::token_alone(token::Ident(Symbol::intern("a"), false), sp(0, 1));
let test2 = string_to_ts("foo(bar::baz)");
assert_eq!(test0.is_empty(), true);
@ -105,9 +100,9 @@ fn test_is_empty() {
fn test_dotdotdot() {
create_default_session_globals_then(|| {
let mut builder = TokenStreamBuilder::new();
builder.push(joint(TokenTree::token(token::Dot, sp(0, 1))));
builder.push(joint(TokenTree::token(token::Dot, sp(1, 2))));
builder.push(TokenTree::token(token::Dot, sp(2, 3)));
builder.push(TokenStream::token_joint(token::Dot, sp(0, 1)));
builder.push(TokenStream::token_joint(token::Dot, sp(1, 2)));
builder.push(TokenStream::token_alone(token::Dot, sp(2, 3)));
let stream = builder.build();
assert!(stream.eq_unspanned(&string_to_ts("...")));
assert_eq!(stream.trees().count(), 1);