bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Rollup merge of #95797 - nnethercote:rm-Delimited-all_tts, r=petrochenkov

Browse source 
Remove explicit delimiter token trees from `Delimited`.

They were introduced by the final commit in #95159 and gave a
performance win. But since the introduction of `MatcherLoc` they are no
longer needed. This commit reverts that change, making the code a bit
simpler.

r? `@petrochenkov`
This commit is contained in:
Dylan DPC 2022-04-09 05:58:45 +02:00 • committed by GitHub
commit 747bd16214
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 58 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

44
compiler/rustc_expand/src/mbe.rs
View file

@ -17,48 +17,24 @@ use rustc_data_structures::sync::Lrc;
use rustc_span::symbol::Ident; use rustc_span::symbol::Ident;
use rustc_span::Span; use rustc_span::Span;
/// Contains the sub-token-trees of a "delimited" token tree such as `(a b c)`. The delimiter itself /// Contains the sub-token-trees of a "delimited" token tree such as `(a b c)`. The delimiters
/// might be `NoDelim`. /// might be `NoDelim`, but they are not represented explicitly.
#[derive(Clone, PartialEq, Encodable, Decodable, Debug)] #[derive(Clone, PartialEq, Encodable, Decodable, Debug)]
struct Delimited { struct Delimited {
delim: token::DelimToken, delim: token::DelimToken,
/// Note: This contains the opening and closing delimiters tokens (e.g. `(` and `)`). Note that /// FIXME: #67062 has details about why this is sub-optimal.
/// these could be `NoDelim`. These token kinds must match `delim`, and the methods below tts: Vec<TokenTree>,
/// debug_assert this.
all_tts: Vec<TokenTree>,
} }
impl Delimited { impl Delimited {
/// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter. Panics if /// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter.
/// the delimiter is `NoDelim`. fn open_tt(&self, span: DelimSpan) -> TokenTree {
fn open_tt(&self) -> &TokenTree { TokenTree::token(token::OpenDelim(self.delim), span.open)
let tt = self.all_tts.first().unwrap();
debug_assert!(matches!(
tt,
&TokenTree::Token(token::Token { kind: token::OpenDelim(d), .. }) if d == self.delim
));
tt
} }
/// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter. Panics if /// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter.
/// the delimiter is `NoDelim`. fn close_tt(&self, span: DelimSpan) -> TokenTree {
fn close_tt(&self) -> &TokenTree { TokenTree::token(token::CloseDelim(self.delim), span.close)
let tt = self.all_tts.last().unwrap();
debug_assert!(matches!(
tt,
&TokenTree::Token(token::Token { kind: token::CloseDelim(d), .. }) if d == self.delim
));
tt
}
/// Returns the tts excluding the outer delimiters.
///
/// FIXME: #67062 has details about why this is sub-optimal.
fn inner_tts(&self) -> &[TokenTree] {
// These functions are called for the assertions within them.
let _open_tt = self.open_tt();
let _close_tt = self.close_tt();
&self.all_tts[1..self.all_tts.len() - 1]
} }
} }

16
compiler/rustc_expand/src/mbe/macro_check.rs
View file

@ -282,7 +282,7 @@ fn check_binders(
// `MetaVarExpr` can not appear in the LHS of a macro arm // `MetaVarExpr` can not appear in the LHS of a macro arm
TokenTree::MetaVarExpr(..) => {} TokenTree::MetaVarExpr(..) => {}
TokenTree::Delimited(_, ref del) => { TokenTree::Delimited(_, ref del) => {
for tt in del.inner_tts() { for tt in &del.tts {
check_binders(sess, node_id, tt, macros, binders, ops, valid); check_binders(sess, node_id, tt, macros, binders, ops, valid);
} }
} }
@ -345,7 +345,7 @@ fn check_occurrences(
check_ops_is_prefix(sess, node_id, macros, binders, ops, dl.entire(), name); check_ops_is_prefix(sess, node_id, macros, binders, ops, dl.entire(), name);
} }
TokenTree::Delimited(_, ref del) => { TokenTree::Delimited(_, ref del) => {
check_nested_occurrences(sess, node_id, del.inner_tts(), macros, binders, ops, valid); check_nested_occurrences(sess, node_id, &del.tts, macros, binders, ops, valid);
} }
TokenTree::Sequence(_, ref seq) => { TokenTree::Sequence(_, ref seq) => {
let ops = ops.push(seq.kleene); let ops = ops.push(seq.kleene);
@ -432,20 +432,14 @@ fn check_nested_occurrences(
{ {
let macro_rules = state == NestedMacroState::MacroRulesNotName; let macro_rules = state == NestedMacroState::MacroRulesNotName;
state = NestedMacroState::Empty; state = NestedMacroState::Empty;
let rest = check_nested_macro( let rest =
sess, check_nested_macro(sess, node_id, macro_rules, &del.tts, &nested_macros, valid);
node_id,
macro_rules,
del.inner_tts(),
&nested_macros,
valid,
);
// If we did not check the whole macro definition, then check the rest as if outside // If we did not check the whole macro definition, then check the rest as if outside
// the macro definition. // the macro definition.
check_nested_occurrences( check_nested_occurrences(
sess, sess,
node_id, node_id,
&del.inner_tts()[rest..], &del.tts[rest..],
macros, macros,
binders, binders,
ops, ops,

8
compiler/rustc_expand/src/mbe/macro_parser.rs
View file

@ -151,9 +151,11 @@ pub(super) fn compute_locs(sess: &ParseSess, matcher: &[TokenTree]) -> Vec<Match
TokenTree::Token(token) => { TokenTree::Token(token) => {
locs.push(MatcherLoc::Token { token: token.clone() }); locs.push(MatcherLoc::Token { token: token.clone() });
} }
TokenTree::Delimited(_, delimited) => { TokenTree::Delimited(span, delimited) => {
locs.push(MatcherLoc::Delimited); locs.push(MatcherLoc::Delimited);
inner(sess, &delimited.all_tts, locs, next_metavar, seq_depth); inner(sess, &[delimited.open_tt(*span)], locs, next_metavar, seq_depth);
inner(sess, &delimited.tts, locs, next_metavar, seq_depth);
inner(sess, &[delimited.close_tt(*span)], locs, next_metavar, seq_depth);
} }
TokenTree::Sequence(_, seq) => { TokenTree::Sequence(_, seq) => {
// We can't determine `idx_first_after` and construct the final // We can't determine `idx_first_after` and construct the final
@ -293,7 +295,7 @@ pub(super) fn count_metavar_decls(matcher: &[TokenTree]) -> usize {
.map(|tt| match tt { .map(|tt| match tt {
TokenTree::MetaVarDecl(..) => 1, TokenTree::MetaVarDecl(..) => 1,
TokenTree::Sequence(_, seq) => seq.num_captures, TokenTree::Sequence(_, seq) => seq.num_captures,
TokenTree::Delimited(_, delim) => count_metavar_decls(delim.inner_tts()), TokenTree::Delimited(_, delim) => count_metavar_decls(&delim.tts),
TokenTree::Token(..) => 0, TokenTree::Token(..) => 0,
TokenTree::MetaVar(..) | TokenTree::MetaVarExpr(..) => unreachable!(), TokenTree::MetaVar(..) | TokenTree::MetaVarExpr(..) => unreachable!(),
}) })

44
compiler/rustc_expand/src/mbe/macro_rules.rs
View file

@ -263,9 +263,7 @@ fn generic_extension<'cx, 'tt>(
// Ignore the delimiters on the RHS. // Ignore the delimiters on the RHS.
let rhs = match &rhses[i] { let rhs = match &rhses[i] {
mbe::TokenTree::Delimited(_, delimited) => { mbe::TokenTree::Delimited(_, delimited) => delimited.tts.to_vec(),
delimited.inner_tts().to_vec().clone()
}
_ => cx.span_bug(sp, "malformed macro rhs"), _ => cx.span_bug(sp, "malformed macro rhs"),
}; };
let arm_span = rhses[i].span(); let arm_span = rhses[i].span();
@ -470,17 +468,16 @@ pub fn compile_declarative_macro(
.iter() .iter()
.map(|m| { .map(|m| {
if let MatchedTokenTree(ref tt) = *m { if let MatchedTokenTree(ref tt) = *m {
let mut tts = vec![]; let tt = mbe::quoted::parse(
mbe::quoted::parse(
tt.clone().into(), tt.clone().into(),
true, true,
&sess.parse_sess, &sess.parse_sess,
def.id, def.id,
features, features,
edition, edition,
&mut tts, )
); .pop()
let tt = tts.pop().unwrap(); .unwrap();
valid &= check_lhs_nt_follows(&sess.parse_sess, features, &def, &tt); valid &= check_lhs_nt_follows(&sess.parse_sess, features, &def, &tt);
return tt; return tt;
} }
@ -495,17 +492,16 @@ pub fn compile_declarative_macro(
.iter() .iter()
.map(|m| { .map(|m| {
if let MatchedTokenTree(ref tt) = *m { if let MatchedTokenTree(ref tt) = *m {
let mut tts = vec![]; return mbe::quoted::parse(
mbe::quoted::parse(
tt.clone().into(), tt.clone().into(),
false, false,
&sess.parse_sess, &sess.parse_sess,
def.id, def.id,
features, features,
edition, edition,
&mut tts, )
); .pop()
return tts.pop().unwrap(); .unwrap();
} }
sess.parse_sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") sess.parse_sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs")
}) })
@ -544,7 +540,7 @@ pub fn compile_declarative_macro(
// Ignore the delimiters around the matcher. // Ignore the delimiters around the matcher.
match lhs { match lhs {
mbe::TokenTree::Delimited(_, delimited) => { mbe::TokenTree::Delimited(_, delimited) => {
mbe::macro_parser::compute_locs(&sess.parse_sess, delimited.inner_tts()) mbe::macro_parser::compute_locs(&sess.parse_sess, &delimited.tts)
} }
_ => sess.parse_sess.span_diagnostic.span_bug(def.span, "malformed macro lhs"), _ => sess.parse_sess.span_diagnostic.span_bug(def.span, "malformed macro lhs"),
} }
@ -576,7 +572,7 @@ fn check_lhs_nt_follows(
// lhs is going to be like TokenTree::Delimited(...), where the // lhs is going to be like TokenTree::Delimited(...), where the
// entire lhs is those tts. Or, it can be a "bare sequence", not wrapped in parens. // entire lhs is those tts. Or, it can be a "bare sequence", not wrapped in parens.
if let mbe::TokenTree::Delimited(_, delimited) = lhs { if let mbe::TokenTree::Delimited(_, delimited) = lhs {
check_matcher(sess, features, def, delimited.inner_tts()) check_matcher(sess, features, def, &delimited.tts)
} else { } else {
let msg = "invalid macro matcher; matchers must be contained in balanced delimiters"; let msg = "invalid macro matcher; matchers must be contained in balanced delimiters";
sess.span_diagnostic.span_err(lhs.span(), msg); sess.span_diagnostic.span_err(lhs.span(), msg);
@ -597,7 +593,7 @@ fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[mbe::TokenTree]) -> bool {
| TokenTree::MetaVarDecl(..) | TokenTree::MetaVarDecl(..)
| TokenTree::MetaVarExpr(..) => (), | TokenTree::MetaVarExpr(..) => (),
TokenTree::Delimited(_, ref del) => { TokenTree::Delimited(_, ref del) => {
if !check_lhs_no_empty_seq(sess, del.inner_tts()) { if !check_lhs_no_empty_seq(sess, &del.tts) {
return false; return false;
} }
} }
@ -692,9 +688,9 @@ impl FirstSets {
| TokenTree::MetaVarExpr(..) => { | TokenTree::MetaVarExpr(..) => {
first.replace_with(tt.clone()); first.replace_with(tt.clone());
} }
TokenTree::Delimited(_span, ref delimited) => { TokenTree::Delimited(span, ref delimited) => {
build_recur(sets, delimited.inner_tts()); build_recur(sets, &delimited.tts);
first.replace_with(delimited.open_tt().clone()); first.replace_with(delimited.open_tt(span));
} }
TokenTree::Sequence(sp, ref seq_rep) => { TokenTree::Sequence(sp, ref seq_rep) => {
let subfirst = build_recur(sets, &seq_rep.tts); let subfirst = build_recur(sets, &seq_rep.tts);
@ -758,8 +754,8 @@ impl FirstSets {
first.add_one(tt.clone()); first.add_one(tt.clone());
return first; return first;
} }
TokenTree::Delimited(_span, ref delimited) => { TokenTree::Delimited(span, ref delimited) => {
first.add_one(delimited.open_tt().clone()); first.add_one(delimited.open_tt(span));
return first; return first;
} }
TokenTree::Sequence(sp, ref seq_rep) => { TokenTree::Sequence(sp, ref seq_rep) => {
@ -945,9 +941,9 @@ fn check_matcher_core(
suffix_first = build_suffix_first(); suffix_first = build_suffix_first();
} }
} }
TokenTree::Delimited(_span, ref d) => { TokenTree::Delimited(span, ref d) => {
let my_suffix = TokenSet::singleton(d.close_tt().clone()); let my_suffix = TokenSet::singleton(d.close_tt(span));
check_matcher_core(sess, features, def, first_sets, d.inner_tts(), &my_suffix); check_matcher_core(sess, features, def, first_sets, &d.tts, &my_suffix);
// don't track non NT tokens // don't track non NT tokens
last.replace_with_irrelevant(); last.replace_with_irrelevant();

26
compiler/rustc_expand/src/mbe/quoted.rs
View file

@ -45,8 +45,10 @@ pub(super) fn parse(
node_id: NodeId, node_id: NodeId,
features: &Features, features: &Features,
edition: Edition, edition: Edition,
result: &mut Vec<TokenTree>, ) -> Vec<TokenTree> {
) { // Will contain the final collection of `self::TokenTree`
let mut result = Vec::new();
// For each token tree in `input`, parse the token into a `self::TokenTree`, consuming // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming
// additional trees if need be. // additional trees if need be.
let mut trees = input.trees(); let mut trees = input.trees();
@ -113,6 +115,7 @@ pub(super) fn parse(
_ => result.push(tree), _ => result.push(tree),
} }
} }
result
} }
/// Asks for the `macro_metavar_expr` feature if it is not already declared /// Asks for the `macro_metavar_expr` feature if it is not already declared
@ -205,8 +208,7 @@ fn parse_tree(
// If we didn't find a metavar expression above, then we must have a // If we didn't find a metavar expression above, then we must have a
// repetition sequence in the macro (e.g. `$(pat)*`). Parse the // repetition sequence in the macro (e.g. `$(pat)*`). Parse the
// contents of the sequence itself // contents of the sequence itself
let mut sequence = vec![]; let sequence = parse(tts, parsing_patterns, sess, node_id, features, edition);
parse(tts, parsing_patterns, sess, node_id, features, edition, &mut sequence);
// Get the Kleene operator and optional separator // Get the Kleene operator and optional separator
let (separator, kleene) = let (separator, kleene) =
parse_sep_and_kleene_op(&mut trees, delim_span.entire(), sess); parse_sep_and_kleene_op(&mut trees, delim_span.entire(), sess);
@ -269,15 +271,13 @@ fn parse_tree(
// `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to
// descend into the delimited set and further parse it. // descend into the delimited set and further parse it.
tokenstream::TokenTree::Delimited(span, delim, tts) => { tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
let mut all_tts = vec![]; span,
// Add the explicit open and close delimiters, which Lrc::new(Delimited {
// `tokenstream::TokenTree::Delimited` lacks. delim,
all_tts.push(TokenTree::token(token::OpenDelim(delim), span.open)); tts: parse(tts, parsing_patterns, sess, node_id, features, edition),
parse(tts, parsing_patterns, sess, node_id, features, edition, &mut all_tts); }),
all_tts.push(TokenTree::token(token::CloseDelim(delim), span.close)); ),
TokenTree::Delimited(span, Lrc::new(Delimited { delim, all_tts }))
}
} }
} }

16
compiler/rustc_expand/src/mbe/transcribe.rs
View file

@ -10,7 +10,7 @@ use rustc_errors::{pluralize, PResult};
use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed}; use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed};
use rustc_span::hygiene::{LocalExpnId, Transparency}; use rustc_span::hygiene::{LocalExpnId, Transparency};
use rustc_span::symbol::{sym, Ident, MacroRulesNormalizedIdent}; use rustc_span::symbol::{sym, Ident, MacroRulesNormalizedIdent};
use rustc_span::{Span, DUMMY_SP}; use rustc_span::Span;
use smallvec::{smallvec, SmallVec}; use smallvec::{smallvec, SmallVec};
use std::mem; use std::mem;
@ -34,14 +34,8 @@ enum Frame {
impl Frame { impl Frame {
/// Construct a new frame around the delimited set of tokens. /// Construct a new frame around the delimited set of tokens.
fn new(mut tts: Vec<mbe::TokenTree>) -> Frame { fn new(tts: Vec<mbe::TokenTree>) -> Frame {
// Need to add empty delimiters. let forest = Lrc::new(mbe::Delimited { delim: token::NoDelim, tts });
let open_tt = mbe::TokenTree::token(token::OpenDelim(token::NoDelim), DUMMY_SP);
let close_tt = mbe::TokenTree::token(token::CloseDelim(token::NoDelim), DUMMY_SP);
tts.insert(0, open_tt);
tts.push(close_tt);
let forest = Lrc::new(mbe::Delimited { delim: token::NoDelim, all_tts: tts });
Frame::Delimited { forest, idx: 0, span: DelimSpan::dummy() } Frame::Delimited { forest, idx: 0, span: DelimSpan::dummy() }
} }
} }
@ -52,7 +46,7 @@ impl Iterator for Frame {
fn next(&mut self) -> Option<mbe::TokenTree> { fn next(&mut self) -> Option<mbe::TokenTree> {
match *self { match *self {
Frame::Delimited { ref forest, ref mut idx, .. } => { Frame::Delimited { ref forest, ref mut idx, .. } => {
let res = forest.inner_tts().get(*idx).cloned(); let res = forest.tts.get(*idx).cloned();
*idx += 1; *idx += 1;
res res
} }
@ -388,7 +382,7 @@ fn lockstep_iter_size(
use mbe::TokenTree; use mbe::TokenTree;
match *tree { match *tree {
TokenTree::Delimited(_, ref delimited) => { TokenTree::Delimited(_, ref delimited) => {
delimited.inner_tts().iter().fold(LockstepIterSize::Unconstrained, |size, tt| { delimited.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| {
size.with(lockstep_iter_size(tt, interpolations, repeats)) size.with(lockstep_iter_size(tt, interpolations, repeats))
}) })
} }