bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Move jointness censoring to proc_macro

Browse source 
Proc-macro API currently exposes jointness in `Punct` tokens. That is,
`+` in `+one` is **non** joint.

Our lexer produces jointness info for all tokens, so we need to censor
it *somewhere*

Previously we did this in a lexer, but it makes more sense to do this
in a proc-macro server.
This commit is contained in:
Aleksey Kladov 2020-09-03 15:22:07 +02:00
parent 08deb863bd
commit 850c3219fb
2 changed files with 12 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

15
compiler/rustc_expand/src/proc_macro_server.rs
View file

@ -47,15 +47,21 @@ impl ToInternal<token::DelimToken> for Delimiter {
} }
} }
impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)> impl FromInternal<(TreeAndJoint, Option<tokenstream::TokenTree>, &'_ ParseSess, &'_ mut Vec<Self>)>
for TokenTree<Group, Punct, Ident, Literal> for TokenTree<Group, Punct, Ident, Literal>
{ {
fn from_internal( fn from_internal(
((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec<Self>), ((tree, is_joint), look_ahead, sess, stack): (
TreeAndJoint,
Option<tokenstream::TokenTree>,
&ParseSess,
&mut Vec<Self>,
),
) -> Self { ) -> Self {
use rustc_ast::token::*; use rustc_ast::token::*;
let joint = is_joint == Joint; let joint = is_joint == Joint
&& matches!(look_ahead, Some(tokenstream::TokenTree::Token(t)) if t.is_op());
let Token { kind, span } = match tree { let Token { kind, span } = match tree {
tokenstream::TokenTree::Delimited(span, delim, tts) => { tokenstream::TokenTree::Delimited(span, delim, tts) => {
let delimiter = Delimiter::from_internal(delim); let delimiter = Delimiter::from_internal(delim);
@ -445,7 +451,8 @@ impl server::TokenStreamIter for Rustc<'_> {
loop { loop {
let tree = iter.stack.pop().or_else(|| { let tree = iter.stack.pop().or_else(|| {
let next = iter.cursor.next_with_joint()?; let next = iter.cursor.next_with_joint()?;
Some(TokenTree::from_internal((next, self.sess, &mut iter.stack))) let lookahead = iter.cursor.look_ahead(0);
Some(TokenTree::from_internal((next, lookahead, self.sess, &mut iter.stack)))
})?; })?;
// A hack used to pass AST fragments to attribute and derive macros // A hack used to pass AST fragments to attribute and derive macros
// as a single nonterminal token instead of a token stream. // as a single nonterminal token instead of a token stream.

5
compiler/rustc_parse/src/lexer/tokentrees.rs
View file

@ -262,10 +262,7 @@ impl<'a> TokenTreesReader<'a> {
} }
_ => { _ => {
let tt = TokenTree::Token(self.token.take()); let tt = TokenTree::Token(self.token.take());
let mut is_joint = self.bump(); let is_joint = self.bump();
if !self.token.is_op() {
is_joint = NonJoint;
}
Ok((tt, is_joint)) Ok((tt, is_joint))
} }
} }