Make macro-system type and constructor names more uniform; more comments.

src/libsyntax/ast.rs

@@ -362,33 +362,53 @@ type capture_item = @{
 #[auto_serialize]
 type capture_clause = @~[capture_item];
 
+//
+// When the main rust parser encounters a syntax-extension invocation, it
+// parses the arguments to the invocation as a token-tree. This is a very
+// loose structure, such that all sorts of different AST-fragments can
+// be passed to syntax extensions using a uniform type.
+//
+// If the syntax extension is an MBE macro, it will attempt to match its
+// LHS "matchers" against the provided token tree, and if it finds a
+// match, will transcribe the RHS token tree, splicing in any captured
+// early_parser::matched_nonterminals into the tt_nonterminals it finds.
+//
+// The RHS of an MBE macro is the only place a tt_nonterminal or tt_seq
+// makes any real sense. You could write them elsewhere but nothing
+// else knows what to do with them, so you'll probably get a syntax
+// error.
+//
 #[auto_serialize]
 #[doc="For macro invocations; parsing is delegated to the macro"]
 enum token_tree {
+    tt_tok(span, token::token),
     tt_delim(~[token_tree]),
-    tt_flat(span, token::token),
+    // These only make sense for right-hand-sides of MBE macros
-    /* These only make sense for right-hand-sides of MBE macros*/
+    tt_seq(span, ~[token_tree], option<token::token>, bool),
-    tt_dotdotdot(span, ~[token_tree], option<token::token>, bool),
+    tt_nonterminal(span, ident)
-    tt_interpolate(span, ident)
 }
 
-#[auto_serialize]
-type matcher = spanned<matcher_>;
-
-#[auto_serialize]
 //
 // Matchers are nodes defined-by and recognized-by the main rust parser and
-// language, but they're only ever found inside syntax-extension invocations.
+// language, but they're only ever found inside syntax-extension invocations;
-// They represent a small sub-language for pattern-matching token-trees, and
+// indeed, the only thing that ever _activates_ the rules in the rust parser
-// are thus primarily used by the macro-defining extension itself.
+// for parsing a matcher is a matcher looking for the 'mtcs' nonterminal
+// itself. Matchers represent a small sub-language for pattern-matching
+// token-trees, and are thus primarily used by the macro-defining extension
+// itself.
 //
-// mtc_tok   ===>   A matcher that matches a single token,
+// match_tok
-//                  denoted by the token itself. So long as
+// ---------
-//                  there's no $ involved.
+//
+//     A matcher that matches a single token, denoted by the token itself. So
+//     long as there's no $ involved.
 //
 //
-// mtc_rep   ===>   A matcher that matches a sequence of
+// match_seq
-//                  sub-matchers, denoted various ways:
+// ---------
+//
+//     A matcher that matches a sequence of sub-matchers, denoted various
+//     possible ways:
 //
 //             $(M)*       zero or more Ms
 //             $(M)+       one or more Ms
@@ -396,12 +416,14 @@ type matcher = spanned<matcher_>;
 //             $(A B C);*  zero or more semi-separated 'A B C' seqs
 //
 //
-// mtc_bb ===> A matcher that matches one of a few interesting named rust
+// match_nonterminal
-//             nonterminals, such as types, expressions, items, or raw
+// -----------------
-//             token-trees. A black-box matcher on expr, for example, binds an
+//
-//             expr to a given ident, and that ident can re-occur as an
+//     A matcher that matches one of a few interesting named rust
-//             interpolation in the RHS of a macro-by-example rule. For
+//     nonterminals, such as types, expressions, items, or raw token-trees. A
-//             example:
+//     black-box matcher on expr, for example, binds an expr to a given ident,
+//     and that ident can re-occur as an interpolation in the RHS of a
+//     macro-by-example rule. For example:
 //
 //        $foo:expr   =>     1 + $foo    // interpolate an expr
 //        $foo:tt     =>     $foo        // interpolate a token-tree
@@ -411,21 +433,25 @@ type matcher = spanned<matcher_>;
 //
 // As a final, horrifying aside, note that macro-by-example's input is
 // also matched by one of these matchers. Holy self-referential! It is matched
-// by an mtc_rep, specifically this one:
+// by an match_seq, specifically this one:
 //
 //                   $( $lhs:mtcs => $rhs:tt );+
 //
 // If you understand that, you have closed to loop and understand the whole
 // macro system. Congratulations.
 //
+#[auto_serialize]
+type matcher = spanned<matcher_>;
+
+#[auto_serialize]
 enum matcher_ {
-    /* match one token */
+    // match one token
-    mtc_tok(token::token),
+    match_tok(token::token),
-    /* match repetitions of a sequence: body, separator, zero ok?,
+    // match repetitions of a sequence: body, separator, zero ok?,
-    lo, hi position-in-match-array used: */
+    // lo, hi position-in-match-array used:
-    mtc_rep(~[matcher], option<token::token>, bool, uint, uint),
+    match_seq(~[matcher], option<token::token>, bool, uint, uint),
-    /* parse a Rust NT: name to bind, name of NT, position in match array : */
+    // parse a Rust NT: name to bind, name of NT, position in match array:
-    mtc_bb(ident, ident, uint)
+    match_nonterminal(ident, ident, uint)
 }
 
 #[auto_serialize]

src/libsyntax/ext/base.rs

@@ -4,18 +4,6 @@ import diagnostic::span_handler;
 import codemap::{codemap, span, expn_info, expanded_from};
 import std::map::str_hash;
 
-
-// Nomenclature / abbreviations in the ext modules:
-//
-//     ms: matcher span, wraps a matcher with fake span
-//    mtc: matcher
-//   mtcs: matchers
-//     tt: token tree
-//     bt: backtrace
-//     cx: expansion context
-//     mr: macro result
-//
-
 // obsolete old-style #macro code:
 //
 //    syntax_expander, normal, macro_defining, macro_definer,
@@ -288,17 +276,18 @@ fn get_mac_body(cx: ext_ctxt, sp: span, args: ast::mac_body)
 // using new syntax. This will be obsolete when #old_macros go away.
 fn tt_args_to_original_flavor(cx: ext_ctxt, sp: span, arg: ~[ast::token_tree])
     -> ast::mac_arg {
-    import ast::{matcher, matcher_, mtc_tok, mtc_rep, mtc_bb};
+    import ast::{matcher, matcher_, match_tok, match_seq, match_nonterminal};
     import parse::lexer::{new_tt_reader, tt_reader_as_reader, reader};
-    import tt::earley_parser::{parse_or_else, seq, leaf};
+    import tt::earley_parser::{parse_or_else, matched_seq,
+                               matched_nonterminal};
 
     // these spans won't matter, anyways
     fn ms(m: matcher_) -> matcher {
         {node: m, span: {lo: 0u, hi: 0u, expn_info: none}}
     }
 
-    let argument_gram = ~[ms(mtc_rep(~[
+    let argument_gram = ~[ms(match_seq(~[
-        ms(mtc_bb(@~"arg",@~"expr", 0u))
+        ms(match_nonterminal(@~"arg",@~"expr", 0u))
     ], some(parse::token::COMMA), true, 0u, 1u))];
 
     let arg_reader = new_tt_reader(cx.parse_sess().span_diagnostic,
@@ -306,10 +295,10 @@ fn tt_args_to_original_flavor(cx: ext_ctxt, sp: span, arg: ~[ast::token_tree])
     let args =
         alt parse_or_else(cx.parse_sess(), cx.cfg(), arg_reader as reader,
                           argument_gram).get(@~"arg") {
-          @seq(s, _) {
+          @matched_seq(s, _) {
             do s.map() |lf| {
                 alt lf {
-                  @leaf(parse::token::w_expr(arg)) {
+                  @matched_nonterminal(parse::token::nt_expr(arg)) {
                     arg /* whew! list of exprs, here we come! */
                   }
                   _ { fail ~"badly-structured parse result"; }

src/libsyntax/ext/expand.rs

@@ -1,7 +1,7 @@
 import std::map::hashmap;
 
 import ast::{crate, expr_, expr_mac, mac_invoc, mac_invoc_tt,
-             tt_delim, tt_flat, item_mac};
+             tt_delim, tt_tok, item_mac};
 import fold::*;
 import ext::base::*;
 import ext::qquote::{qq_helper};

src/libsyntax/ext/tt/earley_parser.rs

@@ -1,6 +1,6 @@
 // Earley-like parser for macros.
 import parse::token;
-import parse::token::{token, EOF, to_str, whole_nt};
+import parse::token::{token, EOF, to_str, nonterminal};
 import parse::lexer::*; //resolve bug?
 //import parse::lexer::{reader, tt_reader, tt_reader_as_reader};
 import parse::parser::{parser,SOURCE_FILE};
@@ -8,20 +8,22 @@ import parse::parser::{parser,SOURCE_FILE};
 import parse::common::*; //resolve bug?
 import parse::parse_sess;
 import dvec::{dvec, extensions};
-import ast::{matcher, mtc_tok, mtc_rep, mtc_bb, ident};
+import ast::{matcher, match_tok, match_seq, match_nonterminal, ident};
 import ast_util::mk_sp;
 import std::map::{hashmap, box_str_hash};
 
-/* This is an Earley-like parser, without support for nonterminals.  This
+/* This is an Earley-like parser, without support for in-grammar nonterminals,
-means that there are no completer or predictor rules, and therefore no need to
+onlyl calling out to the main rust parser for named nonterminals (which it
-store one column per token: instead, there's a set of current Earley items and
+commits to fully when it hits one in a grammar). This means that there are no
-a set of next ones. Instead of NTs, we have a special case for Kleene
+completer or predictor rules, and therefore no need to store one column per
-star. The big-O, in pathological cases, is worse than traditional Earley
+token: instead, there's a set of current Earley items and a set of next
-parsing, but it's an easier fit for Macro-by-Example-style rules, and I think
+ones. Instead of NTs, we have a special case for Kleene star. The big-O, in
-the overhead is lower. */
+pathological cases, is worse than traditional Earley parsing, but it's an
+easier fit for Macro-by-Example-style rules, and I think the overhead is
+lower. */
 
 
-/* to avoid costly uniqueness checks, we require that `mtc_rep` always has a
+/* to avoid costly uniqueness checks, we require that `match_seq` always has a
 nonempty body. */
 
 enum matcher_pos_up { /* to break a circularity */
@@ -40,7 +42,7 @@ type matcher_pos = ~{
     sep: option<token>,
     mut idx: uint,
     mut up: matcher_pos_up, // mutable for swapping only
-    matches: ~[dvec<@arb_depth>],
+    matches: ~[dvec<@named_match>],
     match_lo: uint, match_hi: uint,
     sp_lo: uint,
 };
@@ -55,9 +57,9 @@ fn copy_up(&& mpu: matcher_pos_up) -> matcher_pos {
 fn count_names(ms: &[matcher]) -> uint {
     vec::foldl(0u, ms, |ct, m| {
         ct + alt m.node {
-          mtc_tok(_) { 0u }
+          match_tok(_) { 0u }
-          mtc_rep(more_ms, _, _, _, _) { count_names(more_ms) }
+          match_seq(more_ms, _, _, _, _) { count_names(more_ms) }
-          mtc_bb(_,_,_) { 1u }
+          match_nonterminal(_,_,_) { 1u }
         }})
 }
 
@@ -67,9 +69,13 @@ fn initial_matcher_pos(ms: ~[matcher], sep: option<token>, lo: uint)
     let mut match_idx_hi = 0u;
     for ms.each() |elt| {
         alt elt.node {
-          mtc_tok(_) {}
+          match_tok(_) {}
-          mtc_rep(_,_,_,_,hi) { match_idx_hi = hi; } //it is monotonic...
+          match_seq(_,_,_,_,hi) {
-          mtc_bb(_,_,pos) { match_idx_hi = pos+1u; } //...so latest is highest
+            match_idx_hi = hi;       // it is monotonic...
+          }
+          match_nonterminal(_,_,pos) {
+            match_idx_hi = pos+1u;  // ...so latest is highest
+          }
         }
     }
     ~{elts: ms, sep: sep, mut idx: 0u, mut up: matcher_pos_up(none),
@@ -77,38 +83,42 @@ fn initial_matcher_pos(ms: ~[matcher], sep: option<token>, lo: uint)
       match_lo: 0u, match_hi: match_idx_hi, sp_lo: lo}
 }
 
-// arb_depth is a pattern-match result for a single black-box matcher
+// named_match is a pattern-match result for a single ast::match_nonterminal:
-// (ast::mtc_bb): so it is associated with a single ident in a parse, and all
+// so it is associated with a single ident in a parse, and all
-// leaves in the arb_depth have the same nonterminal type (expr, item,
+// matched_nonterminals in the named_match have the same nonterminal type
-// etc). All the leaves in a single arb_depth correspond to a single mtc_bb in
+// (expr, item, etc). All the leaves in a single named_match correspond to a
-// the ast::matcher that produced it.
+// single matcher_nonterminal in the ast::matcher that produced it.
 //
 // It should probably be renamed, it has more or less exact correspondence to
-// ast::match nodes, and the in-memory structure of a particular arb_depth
+// ast::match nodes, and the in-memory structure of a particular named_match
 // represents the match that occurred when a particular subset of an
-// ast::match -- those ast::matcher nodes leading to a single mtc_bb -- was
+// ast::match -- those ast::matcher nodes leading to a single
-// applied to a particular token tree.
+// match_nonterminal -- was applied to a particular token tree.
 //
-// The width of each seq in the arb_depth, and the identity of the leaf nodes,
+// The width of each matched_seq in the named_match, and the identity of the
-// will depend on the token tree it was applied to: each seq corresponds to a
+// matched_nonterminals, will depend on the token tree it was applied to: each
-// single mtc_rep in the originating ast::matcher. The depth of the arb_depth
+// matched_seq corresponds to a single match_seq in the originating
-// structure will therefore depend only on the nesting depth of mtc_reps in
+// ast::matcher. The depth of the named_match structure will therefore depend
-// the originating ast::matcher it was derived from.
+// only on the nesting depth of ast::match_seqs in the originating
+// ast::matcher it was derived from.
 
-enum arb_depth { leaf(whole_nt), seq(~[@arb_depth], codemap::span) }
+enum named_match {
+    matched_seq(~[@named_match], codemap::span),
+    matched_nonterminal(nonterminal)
+}
 
 type earley_item = matcher_pos;
 
-fn nameize(p_s: parse_sess, ms: ~[matcher], res: ~[@arb_depth])
+fn nameize(p_s: parse_sess, ms: ~[matcher], res: ~[@named_match])
-    -> hashmap<ident,@arb_depth> {
+    -> hashmap<ident,@named_match> {
-    fn n_rec(p_s: parse_sess, m: matcher, res: ~[@arb_depth],
+    fn n_rec(p_s: parse_sess, m: matcher, res: ~[@named_match],
-             ret_val: hashmap<ident, @arb_depth>) {
+             ret_val: hashmap<ident, @named_match>) {
         alt m {
-          {node: mtc_tok(_), span: _} { }
+          {node: match_tok(_), span: _} { }
-          {node: mtc_rep(more_ms, _, _, _, _), span: _} {
+          {node: match_seq(more_ms, _, _, _, _), span: _} {
             for more_ms.each() |next_m| { n_rec(p_s, next_m, res, ret_val) };
           }
-          {node: mtc_bb(bind_name, _, idx), span: sp} {
+          {node: match_nonterminal(bind_name, _, idx), span: sp} {
             if ret_val.contains_key(bind_name) {
                 p_s.span_diagnostic.span_fatal(sp, ~"Duplicated bind name: "
                                                + *bind_name)
@@ -117,18 +127,18 @@ fn nameize(p_s: parse_sess, ms: ~[matcher], res: ~[@arb_depth])
           }
         }
     }
-    let ret_val = box_str_hash::<@arb_depth>();
+    let ret_val = box_str_hash::<@named_match>();
     for ms.each() |m| { n_rec(p_s, m, res, ret_val) }
     ret ret_val;
 }
 
 enum parse_result {
-    success(hashmap<ident, @arb_depth>),
+    success(hashmap<ident, @named_match>),
     failure(codemap::span, ~str)
 }
 
 fn parse_or_else(sess: parse_sess, cfg: ast::crate_cfg, rdr: reader,
-                 ms: ~[matcher]) -> hashmap<ident, @arb_depth> {
+                 ms: ~[matcher]) -> hashmap<ident, @named_match> {
     alt parse(sess, cfg, rdr, ms) {
       success(m) { m }
       failure(sp, str) {
@@ -182,7 +192,9 @@ fn parse(sess: parse_sess, cfg: ast::crate_cfg, rdr: reader, ms: ~[matcher])
                         for uint::range(ei.match_lo, ei.match_hi) |idx| {
                             let sub = ei.matches[idx].get();
                             new_pos.matches[idx]
-                                .push(@seq(sub, mk_sp(ei.sp_lo,sp.hi)));
+                                .push(@matched_seq(sub,
+                                                   mk_sp(ei.sp_lo,
+                                                         sp.hi)));
                         }
 
                         new_pos.idx += 1u;
@@ -212,20 +224,21 @@ fn parse(sess: parse_sess, cfg: ast::crate_cfg, rdr: reader, ms: ~[matcher])
             } else {
                 alt copy ei.elts[idx].node {
                   /* need to descend into sequence */
-                  mtc_rep(matchers, sep, zero_ok, match_idx_lo, match_idx_hi){
+                  match_seq(matchers, sep, zero_ok,
+                            match_idx_lo, match_idx_hi){
                     if zero_ok {
                         let new_ei = copy ei;
                         new_ei.idx += 1u;
                         //we specifically matched zero repeats.
                         for uint::range(match_idx_lo, match_idx_hi) |idx| {
-                            new_ei.matches[idx].push(@seq(~[], sp));
+                            new_ei.matches[idx].push(@matched_seq(~[], sp));
                         }
 
                         vec::push(cur_eis, new_ei);
                     }
 
                     let matches = vec::map(ei.matches, // fresh, same size:
-                                           |_m| dvec::<@arb_depth>());
+                                           |_m| dvec::<@named_match>());
                     let ei_t <- ei;
                     vec::push(cur_eis, ~{
                         elts: matchers, sep: sep, mut idx: 0u,
@@ -235,8 +248,8 @@ fn parse(sess: parse_sess, cfg: ast::crate_cfg, rdr: reader, ms: ~[matcher])
                         sp_lo: sp.lo
                     });
                   }
-                  mtc_bb(_,_,_) { vec::push(bb_eis, ei) }
+                  match_nonterminal(_,_,_) { vec::push(bb_eis, ei) }
-                  mtc_tok(t) {
+                  match_tok(t) {
                     let ei_t <- ei;
                     if t == tok { ei_t.idx += 1u; vec::push(next_eis, ei_t)}
                   }
@@ -260,7 +273,7 @@ fn parse(sess: parse_sess, cfg: ast::crate_cfg, rdr: reader, ms: ~[matcher])
                 || bb_eis.len() > 1u {
                 let nts = str::connect(vec::map(bb_eis, |ei| {
                     alt ei.elts[ei.idx].node {
-                      mtc_bb(bind,name,_) {
+                      match_nonterminal(bind,name,_) {
                         #fmt["%s ('%s')", *name, *bind]
                       }
                       _ { fail; } } }), ~" or ");
@@ -282,8 +295,8 @@ fn parse(sess: parse_sess, cfg: ast::crate_cfg, rdr: reader, ms: ~[matcher])
 
                 let ei = vec::pop(bb_eis);
                 alt ei.elts[ei.idx].node {
-                  mtc_bb(_, name, idx) {
+                  match_nonterminal(_, name, idx) {
-                    ei.matches[idx].push(@leaf(
+                    ei.matches[idx].push(@matched_nonterminal(
                         parse_nt(rust_parser, *name)));
                     ei.idx += 1u;
                   }
@@ -305,31 +318,31 @@ fn parse(sess: parse_sess, cfg: ast::crate_cfg, rdr: reader, ms: ~[matcher])
     }
 }
 
-fn parse_nt(p: parser, name: ~str) -> whole_nt {
+fn parse_nt(p: parser, name: ~str) -> nonterminal {
     alt name {
       ~"item" { alt p.parse_item(~[], ast::public) {
-        some(i) { token::w_item(i) }
+        some(i) { token::nt_item(i) }
         none { p.fatal(~"expected an item keyword") }
       }}
-      ~"block" { token::w_block(p.parse_block()) }
+      ~"block" { token::nt_block(p.parse_block()) }
-      ~"stmt" { token::w_stmt(p.parse_stmt(~[])) }
+      ~"stmt" { token::nt_stmt(p.parse_stmt(~[])) }
-      ~"pat" { token::w_pat(p.parse_pat()) }
+      ~"pat" { token::nt_pat(p.parse_pat()) }
-      ~"expr" { token::w_expr(p.parse_expr()) }
+      ~"expr" { token::nt_expr(p.parse_expr()) }
-      ~"ty" { token::w_ty(p.parse_ty(false /* no need to disambiguate*/)) }
+      ~"ty" { token::nt_ty(p.parse_ty(false /* no need to disambiguate*/)) }
       // this could be handled like a token, since it is one
       ~"ident" { alt copy p.token {
-          token::IDENT(sn,b) { p.bump(); token::w_ident(sn,b) }
+          token::IDENT(sn,b) { p.bump(); token::nt_ident(sn,b) }
           _ { p.fatal(~"expected ident, found "
                       + token::to_str(*p.reader.interner(), copy p.token)) }
       } }
-      ~"path" { token::w_path(p.parse_path_with_tps(false)) }
+      ~"path" { token::nt_path(p.parse_path_with_tps(false)) }
       ~"tt" {
         p.quote_depth += 1u; //but in theory, non-quoted tts might be useful
-        let res = token::w_tt(@p.parse_token_tree());
+        let res = token::nt_tt(@p.parse_token_tree());
         p.quote_depth -= 1u;
         res
       }
-      ~"mtcs" { token::w_mtcs(p.parse_matchers()) }
+      ~"matchers" { token::nt_matchers(p.parse_matchers()) }
       _ { p.fatal(~"Unsupported builtin nonterminal parser: " + name)}
     }
 }

src/libsyntax/ext/tt/macro_rules.rs

@@ -1,10 +1,12 @@
 import base::{ext_ctxt, mac_result, mr_expr, mr_def, expr_tt};
 import codemap::span;
-import ast::{ident, matcher_, matcher, mtc_tok, mtc_bb, mtc_rep, tt_delim};
+import ast::{ident, matcher_, matcher, match_tok,
+             match_nonterminal, match_seq, tt_delim};
 import parse::lexer::{new_tt_reader, tt_reader_as_reader, reader};
-import parse::token::{FAT_ARROW, SEMI, LBRACE, RBRACE, w_mtcs, w_tt};
+import parse::token::{FAT_ARROW, SEMI, LBRACE, RBRACE, nt_matchers, nt_tt};
 import parse::parser::{parser, SOURCE_FILE};
-import earley_parser::{parse, success, failure, arb_depth, seq, leaf};
+import earley_parser::{parse, success, failure, named_match,
+                       matched_seq, matched_nonterminal};
 import std::map::hashmap;
 
 
@@ -17,10 +19,10 @@ fn add_new_extension(cx: ext_ctxt, sp: span, name: ident,
     }
 
     let argument_gram = ~[
-        ms(mtc_rep(~[
+        ms(match_seq(~[
-            ms(mtc_bb(@~"lhs",@~"mtcs", 0u)),
+            ms(match_nonterminal(@~"lhs",@~"matchers", 0u)),
-            ms(mtc_tok(FAT_ARROW)),
+            ms(match_tok(FAT_ARROW)),
-            ms(mtc_bb(@~"rhs",@~"tt", 1u)),
+            ms(match_nonterminal(@~"rhs",@~"tt", 1u)),
         ], some(SEMI), false, 0u, 2u))];
 
     let arg_reader = new_tt_reader(cx.parse_sess().span_diagnostic,
@@ -32,16 +34,16 @@ fn add_new_extension(cx: ext_ctxt, sp: span, name: ident,
     };
 
     let lhses = alt arguments.get(@~"lhs") {
-      @seq(s, sp) { s }
+      @matched_seq(s, sp) { s }
       _ { cx.span_bug(sp, ~"wrong-structured lhs") }
     };
     let rhses = alt arguments.get(@~"rhs") {
-      @seq(s, sp) { s }
+      @matched_seq(s, sp) { s }
       _ { cx.span_bug(sp, ~"wrong-structured rhs") }
     };
 
     fn generic_extension(cx: ext_ctxt, sp: span, arg: ~[ast::token_tree],
-                         lhses: ~[@arb_depth], rhses: ~[@arb_depth])
+                         lhses: ~[@named_match], rhses: ~[@named_match])
     -> mac_result {
         let mut best_fail_spot = {lo: 0u, hi: 0u, expn_info: none};
         let mut best_fail_msg = ~"internal error: ran no matchers";
@@ -51,12 +53,12 @@ fn add_new_extension(cx: ext_ctxt, sp: span, name: ident,
 
         for lhses.eachi() |i, lhs| {
             alt lhs {
-              @leaf(w_mtcs(mtcs)) {
+              @matched_nonterminal(nt_matchers(mtcs)) {
                 let arg_rdr = new_tt_reader(s_d, itr, none, arg) as reader;
                 alt parse(cx.parse_sess(), cx.cfg(), arg_rdr, mtcs) {
                   success(m) {
                     let rhs = alt rhses[i] {
-                      @leaf(w_tt(@tt)) { tt }
+                      @matched_nonterminal(nt_tt(@tt)) { tt }
                       _ { cx.span_bug(sp, ~"bad thing in rhs") }
                     };
                     let trncbr = new_tt_reader(s_d, itr, some(m), ~[rhs]);

src/libsyntax/ext/tt/transcribe.rs

@@ -1,10 +1,10 @@
 import util::interner::interner;
 import diagnostic::span_handler;
-import ast::{token_tree,tt_delim,tt_flat,tt_dotdotdot,tt_interpolate,ident};
+import ast::{token_tree, tt_delim, tt_tok, tt_seq, tt_nonterminal,ident};
-import earley_parser::{arb_depth,seq,leaf};
+import earley_parser::{named_match, matched_seq, matched_nonterminal};
 import codemap::span;
-import parse::token::{EOF,ACTUALLY,IDENT,token,w_ident};
+import parse::token::{EOF, INTERPOLATED, IDENT, token, nt_ident};
-import std::map::{hashmap,box_str_hash};
+import std::map::{hashmap, box_str_hash};
 
 export tt_reader,  new_tt_reader, dup_tt_reader, tt_next_token;
 
@@ -28,7 +28,7 @@ type tt_reader = @{
     interner: @interner<@~str>,
     mut cur: tt_frame,
     /* for MBE-style macro transcription */
-    interpolations: std::map::hashmap<ident, @arb_depth>,
+    interpolations: std::map::hashmap<ident, @named_match>,
     mut repeat_idx: ~[mut uint],
     mut repeat_len: ~[uint],
     /* cached: */
@@ -37,17 +37,17 @@ type tt_reader = @{
 };
 
 /** This can do Macro-By-Example transcription. On the other hand, if
- *  `src` contains no `tt_dotdotdot`s and `tt_interpolate`s, `interp` can (and
+ *  `src` contains no `tt_seq`s and `tt_nonterminal`s, `interp` can (and
  *  should) be none. */
 fn new_tt_reader(sp_diag: span_handler, itr: @interner<@~str>,
-                 interp: option<std::map::hashmap<ident,@arb_depth>>,
+                 interp: option<std::map::hashmap<ident,@named_match>>,
                  src: ~[ast::token_tree])
     -> tt_reader {
     let r = @{sp_diag: sp_diag, interner: itr,
               mut cur: @{readme: src, mut idx: 0u, dotdotdoted: false,
                          sep: none, up: tt_frame_up(option::none)},
               interpolations: alt interp { /* just a convienience */
-                none { std::map::box_str_hash::<@arb_depth>() }
+                none { std::map::box_str_hash::<@named_match>() }
                 some(x) { x }
               },
               mut repeat_idx: ~[mut], mut repeat_len: ~[],
@@ -79,18 +79,22 @@ pure fn dup_tt_reader(&&r: tt_reader) -> tt_reader {
 }
 
 
-pure fn lookup_cur_ad_by_ad(r: tt_reader, start: @arb_depth) -> @arb_depth {
+pure fn lookup_cur_matched_by_matched(r: tt_reader,
-    pure fn red(&&ad: @arb_depth, &&idx: uint) -> @arb_depth {
+                                      start: @named_match) -> @named_match {
+    pure fn red(&&ad: @named_match, &&idx: uint) -> @named_match {
         alt *ad {
-          leaf(_) { ad /* end of the line; duplicate henceforth */ }
+          matched_nonterminal(_) {
-          seq(ads, _) { ads[idx] }
+            // end of the line; duplicate henceforth
+            ad
+          }
+          matched_seq(ads, _) { ads[idx] }
         }
     }
     vec::foldl(start, r.repeat_idx, red)
 }
 
-fn lookup_cur_ad(r: tt_reader, name: ident) -> @arb_depth {
+fn lookup_cur_matched(r: tt_reader, name: ident) -> @named_match {
-    lookup_cur_ad_by_ad(r, r.interpolations.get(name))
+    lookup_cur_matched_by_matched(r, r.interpolations.get(name))
 }
 enum lis {
     lis_unconstrained, lis_constraint(uint, ident), lis_contradiction(~str)
@@ -116,15 +120,15 @@ fn lockstep_iter_size(&&t: token_tree, &&r: tt_reader) -> lis {
         }
     }
     alt t {
-      tt_delim(tts) | tt_dotdotdot(_, tts, _, _) {
+      tt_delim(tts) | tt_seq(_, tts, _, _) {
         vec::foldl(lis_unconstrained, tts, {|lis, tt|
             lis_merge(lis, lockstep_iter_size(tt, r)) })
       }
-      tt_flat(*) { lis_unconstrained }
+      tt_tok(*) { lis_unconstrained }
-      tt_interpolate(_, name) {
+      tt_nonterminal(_, name) {
-        alt *lookup_cur_ad(r, name) {
+        alt *lookup_cur_matched(r, name) {
-          leaf(_) { lis_unconstrained }
+          matched_nonterminal(_) { lis_unconstrained }
-          seq(ads, _) { lis_constraint(ads.len(), name) }
+          matched_seq(ads, _) { lis_constraint(ads.len(), name) }
         }
       }
     }
@@ -166,20 +170,20 @@ fn tt_next_token(&&r: tt_reader) -> {tok: token, sp: span} {
         }
     }
     loop { /* because it's easiest, this handles `tt_delim` not starting
-    with a `tt_flat`, even though it won't happen */
+    with a `tt_tok`, even though it won't happen */
         alt r.cur.readme[r.cur.idx] {
           tt_delim(tts) {
             r.cur = @{readme: tts, mut idx: 0u, dotdotdoted: false,
                       sep: none, up: tt_frame_up(option::some(r.cur)) };
             // if this could be 0-length, we'd need to potentially recur here
           }
-          tt_flat(sp, tok) {
+          tt_tok(sp, tok) {
             r.cur_span = sp; r.cur_tok = tok;
             r.cur.idx += 1u;
             ret ret_val;
           }
-          tt_dotdotdot(sp, tts, sep, zerok) {
+          tt_seq(sp, tts, sep, zerok) {
-            alt lockstep_iter_size(tt_dotdotdot(sp, tts, sep, zerok), r) {
+            alt lockstep_iter_size(tt_seq(sp, tts, sep, zerok), r) {
               lis_unconstrained {
                 r.sp_diag.span_fatal(
                     sp, /* blame macro writer */
@@ -211,22 +215,22 @@ fn tt_next_token(&&r: tt_reader) -> {tok: token, sp: span} {
             }
           }
           // FIXME #2887: think about span stuff here
-          tt_interpolate(sp, ident) {
+          tt_nonterminal(sp, ident) {
-            alt *lookup_cur_ad(r, ident) {
+            alt *lookup_cur_matched(r, ident) {
               /* sidestep the interpolation tricks for ident because
               (a) idents can be in lots of places, so it'd be a pain
               (b) we actually can, since it's a token. */
-              leaf(w_ident(sn,b)) {
+              matched_nonterminal(nt_ident(sn,b)) {
                 r.cur_span = sp; r.cur_tok = IDENT(sn,b);
                 r.cur.idx += 1u;
                 ret ret_val;
               }
-              leaf(w_nt) {
+              matched_nonterminal(nt) {
-                r.cur_span = sp; r.cur_tok = ACTUALLY(w_nt);
+                r.cur_span = sp; r.cur_tok = INTERPOLATED(nt);
                 r.cur.idx += 1u;
                 ret ret_val;
               }
-              seq(*) {
+              matched_seq(*) {
                 r.sp_diag.span_fatal(
                     copy r.cur_span, /* blame the macro writer */
                     #fmt["variable '%s' is still repeating at this depth",

src/libsyntax/parse/common.rs

@@ -86,7 +86,7 @@ impl parser_common of parser_common for parser {
     fn parse_ident() -> ast::ident {
         alt copy self.token {
           token::IDENT(i, _) { self.bump(); ret self.get_str(i); }
-          token::ACTUALLY(token::w_ident(*)) { self.bug(
+          token::INTERPOLATED(token::nt_ident(*)) { self.bug(
               ~"ident interpolation not converted to real token"); }
           _ { self.fatal(~"expected ident, found `"
                          + token_to_str(self.reader, self.token)

src/libsyntax/parse/parser.rs

@@ -3,7 +3,7 @@ import print::pprust::expr_to_str;
 import result::result;
 import either::{either, left, right};
 import std::map::{hashmap, str_hash};
-import token::{can_begin_expr, is_ident, is_plain_ident, ACTUALLY};
+import token::{can_begin_expr, is_ident, is_plain_ident, INTERPOLATED};
 import codemap::{span,fss_none};
 import util::interner;
 import ast_util::{spanned, respan, mk_sp, ident_to_path, operator_prec};
@@ -39,15 +39,15 @@ import ast::{_mod, add, alt_check, alt_exhaustive, arg, arm, attribute,
              item_ty, lit, lit_, lit_bool, lit_float, lit_int,
              lit_int_unsuffixed, lit_nil, lit_str, lit_uint, local, m_const,
              m_imm, m_mutbl, mac_, mac_aq, mac_ellipsis,
-             mac_invoc, mac_invoc_tt, mac_var, matcher,
+             mac_invoc, mac_invoc_tt, mac_var, matcher, match_nonterminal,
-             method, mode, mt, mtc_bb, mtc_rep, mtc_tok, mul, mutability, neg,
+             match_seq, match_tok, method, mode, mt, mul, mutability, neg,
              noreturn, not, pat, pat_box, pat_enum, pat_ident, pat_lit,
              pat_range, pat_rec, pat_tup, pat_uniq, pat_wild, path, private,
              proto, proto_any, proto_bare, proto_block, proto_box, proto_uniq,
              provided, public, pure_fn, purity, re_anon, re_named, region,
              rem, required, ret_style, return_val, shl, shr, stmt, stmt_decl,
              stmt_expr, stmt_semi, subtract, token_tree, trait_method,
-             trait_ref, tt_delim, tt_dotdotdot, tt_flat, tt_interpolate, ty,
+             trait_ref, tt_delim, tt_seq, tt_tok, tt_nonterminal, ty,
              ty_, ty_bot, ty_box, ty_field, ty_fn, ty_infer, ty_mac,
              ty_method, ty_nil, ty_param, ty_path, ty_ptr, ty_rec, ty_rptr,
              ty_tup, ty_u32, ty_uniq, ty_vec, ty_fixed_length, unchecked_blk,
@@ -104,14 +104,14 @@ type item_info = (ident, item_, option<~[attribute]>);
 
 /* The expr situation is not as complex as I thought it would be.
 The important thing is to make sure that lookahead doesn't balk
-at ACTUALLY tokens */
+at INTERPOLATED tokens */
-macro_rules! maybe_whole_expr{
+macro_rules! maybe_whole_expr {
     {$p:expr} => { alt copy $p.token {
-      ACTUALLY(token::w_expr(e)) {
+      INTERPOLATED(token::nt_expr(e)) {
         $p.bump();
         ret pexpr(e);
       }
-      ACTUALLY(token::w_path(pt)) {
+      INTERPOLATED(token::nt_path(pt)) {
         $p.bump();
         ret $p.mk_pexpr($p.span.lo, $p.span.lo,
                        expr_path(pt));
@@ -122,7 +122,7 @@ macro_rules! maybe_whole_expr{
 
 macro_rules! maybe_whole {
     {$p:expr, $constructor:path} => { alt copy $p.token {
-      ACTUALLY($constructor(x)) { $p.bump(); ret x; }
+      INTERPOLATED($constructor(x)) { $p.bump(); ret x; }
       _ {}
     }}
 }
@@ -133,7 +133,7 @@ fn dummy() {
     /* we will need this to bootstrap maybe_whole! */
     #macro[[#maybe_whole_path[p],
             alt p.token {
-                ACTUALLY(token::w_path(pt)) { p.bump(); ret pt; }
+                INTERPOLATED(token::nt_path(pt)) { p.bump(); ret pt; }
                 _ {} }]];
 }
 
@@ -1090,7 +1090,7 @@ class parser {
             }
         }
 
-        fn parse_tt_flat(p: parser, delim_ok: bool) -> token_tree {
+        fn parse_tt_tok(p: parser, delim_ok: bool) -> token_tree {
             alt p.token {
               token::RPAREN | token::RBRACE | token::RBRACKET
               if !delim_ok {
@@ -1110,14 +1110,14 @@ class parser {
                                           seq_sep_none(),
                                           |p| p.parse_token_tree());
                     let (s, z) = p.parse_sep_and_zerok();
-                    ret tt_dotdotdot(mk_sp(sp.lo ,p.span.hi), seq.node, s, z);
+                    ret tt_seq(mk_sp(sp.lo ,p.span.hi), seq.node, s, z);
                 } else {
-                    ret tt_interpolate(sp, p.parse_ident());
+                    ret tt_nonterminal(sp, p.parse_ident());
                 }
               }
               _ { /* ok */ }
             }
-            let res = tt_flat(p.span, p.token);
+            let res = tt_tok(p.span, p.token);
             p.bump();
             ret res;
         }
@@ -1126,14 +1126,14 @@ class parser {
           token::LPAREN | token::LBRACE | token::LBRACKET {
             let ket = flip(self.token);
             tt_delim(vec::append(
-                ~[parse_tt_flat(self, true)],
+                ~[parse_tt_tok(self, true)],
                 vec::append(
                     self.parse_seq_to_before_end(
                         ket, seq_sep_none(),
                         |p| p.parse_token_tree()),
-                    ~[parse_tt_flat(self, true)])))
+                    ~[parse_tt_tok(self, true)])))
           }
-          _ { parse_tt_flat(self, false) }
+          _ { parse_tt_tok(self, false) }
         };
     }
 
@@ -1177,17 +1177,17 @@ class parser {
                     self.fatal(~"repetition body must be nonempty");
                 }
                 let (sep, zerok) = self.parse_sep_and_zerok();
-                mtc_rep(ms, sep, zerok, name_idx_lo, *name_idx)
+                match_seq(ms, sep, zerok, name_idx_lo, *name_idx)
             } else {
                 let bound_to = self.parse_ident();
                 self.expect(token::COLON);
                 let nt_name = self.parse_ident();
-                let m = mtc_bb(bound_to, nt_name, *name_idx);
+                let m = match_nonterminal(bound_to, nt_name, *name_idx);
                 *name_idx += 1u;
                 m
             }
         } else {
-            let m = mtc_tok(self.token);
+            let m = match_tok(self.token);
             self.bump();
             m
         };

src/libsyntax/parse/token.rs

@@ -79,7 +79,7 @@ enum token {
     UNDERSCORE,
 
     /* For interpolation */
-    ACTUALLY(whole_nt),
+    INTERPOLATED(nonterminal),
 
     DOC_COMMENT(str_num),
     EOF,
@@ -87,17 +87,17 @@ enum token {
 
 #[auto_serialize]
 /// For interpolation during macro expansion.
-enum whole_nt {
+enum nonterminal {
-    w_item(@ast::item),
+    nt_item(@ast::item),
-    w_block(ast::blk),
+    nt_block(ast::blk),
-    w_stmt(@ast::stmt),
+    nt_stmt(@ast::stmt),
-    w_pat( @ast::pat),
+    nt_pat( @ast::pat),
-    w_expr(@ast::expr),
+    nt_expr(@ast::expr),
-    w_ty(  @ast::ty),
+    nt_ty(  @ast::ty),
-    w_ident(str_num, bool),
+    nt_ident(str_num, bool),
-    w_path(@ast::path),
+    nt_path(@ast::path),
-    w_tt(  @ast::token_tree), //needs @ed to break a circularity
+    nt_tt(  @ast::token_tree), //needs @ed to break a circularity
-    w_mtcs(~[ast::matcher])
+    nt_matchers(~[ast::matcher])
 }
 
 fn binop_to_str(o: binop) -> ~str {
@@ -184,14 +184,14 @@ fn to_str(in: interner<@~str>, t: token) -> ~str {
       /* Other */
       DOC_COMMENT(s) { *interner::get(in, s) }
       EOF { ~"<eof>" }
-      ACTUALLY(w_nt) {
+      INTERPOLATED(nt) {
         ~"an interpolated " +
-            alt w_nt {
+            alt nt {
-              w_item(*) { ~"item" } w_block(*) { ~"block" }
+              nt_item(*) { ~"item" } nt_block(*) { ~"block" }
-              w_stmt(*) { ~"statement" } w_pat(*) { ~"pattern" }
+              nt_stmt(*) { ~"statement" } nt_pat(*) { ~"pattern" }
-              w_expr(*) { ~"expression" } w_ty(*) { ~"type" }
+              nt_expr(*) { ~"expression" } nt_ty(*) { ~"type" }
-              w_ident(*) { ~"identifier" } w_path(*) { ~"path" }
+              nt_ident(*) { ~"identifier" } nt_path(*) { ~"path" }
-              w_tt(*) { ~"tt" } w_mtcs(*) { ~"matcher sequence" }
+              nt_tt(*) { ~"tt" } nt_matchers(*) { ~"matcher sequence" }
             }
       }
     }
@@ -219,8 +219,10 @@ pure fn can_begin_expr(t: token) -> bool {
       BINOP(OR) { true } // in lambda syntax
       OROR { true } // in lambda syntax
       MOD_SEP { true }
-      ACTUALLY(w_expr(*)) | ACTUALLY(w_ident(*)) | ACTUALLY(w_block(*))
+      INTERPOLATED(nt_expr(*))
-      | ACTUALLY(w_path(*)) { true }
+      | INTERPOLATED(nt_ident(*))
+      | INTERPOLATED(nt_block(*))
+      | INTERPOLATED(nt_path(*)) { true }
       _ { false }
     }
 }