add hygiene support fns, move them around.

also adds test cases

src/libsyntax/ast.rs

@@ -571,7 +571,7 @@ pub enum token_tree {
     // These only make sense for right-hand-sides of MBE macros:
 
     // a kleene-style repetition sequence with a span, a tt_forest,
-    // an optional separator (?), and a boolean where true indicates
+    // an optional separator, and a boolean where true indicates
     // zero or more (*), and false indicates one or more (+).
     tt_seq(Span, @mut ~[token_tree], Option<::parse::token::Token>, bool),
 

src/libsyntax/ast_util.rs

@@ -825,9 +825,6 @@ pub fn pat_is_ident(pat: @ast::Pat) -> bool {
 
 // HYGIENE FUNCTIONS
 
-/// Construct an identifier with the given name and an empty context:
-pub fn new_ident(name: Name) -> Ident { Ident {name: name, ctxt: 0}}
-
 /// Extend a syntax context with a given mark
 pub fn new_mark(m:Mrk, tail:SyntaxContext) -> SyntaxContext {
     new_mark_internal(m,tail,get_sctable())

src/libsyntax/ext/expand.rs

@@ -8,9 +8,9 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{Block, Crate, NodeId, Expr_, ExprMac, Ident, mac_invoc_tt};
+use ast::{Block, Crate, NodeId, DeclLocal, Expr_, ExprMac, Local, Ident, mac_invoc_tt};
-use ast::{item_mac, Stmt_, StmtMac, StmtExpr, StmtSemi};
+use ast::{item_mac, Mrk, Stmt_, StmtDecl, StmtMac, StmtExpr, StmtSemi};
-use ast::{ILLEGAL_CTXT};
+use ast::{ILLEGAL_CTXT, SCTable, token_tree};
 use ast;
 use ast_util::{new_rename, new_mark, mtwt_resolve};
 use attr;
@@ -23,7 +23,7 @@ use opt_vec;
 use parse;
 use parse::{parse_item_from_source_str};
 use parse::token;
-use parse::token::{ident_to_str, intern};
+use parse::token::{fresh_name, ident_to_str, intern};
 use visit;
 use visit::Visitor;
 
@@ -521,6 +521,71 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv,
 
 }
 
+// expand a non-macro stmt. this is essentially the fallthrough for
+// expand_stmt, above.
+fn expand_non_macro_stmt (exts: SyntaxEnv,
+                          s: &Stmt_,
+                          sp: Span,
+                          fld: @ast_fold,
+                          orig: @fn(&Stmt_, Span, @ast_fold) -> (Option<Stmt_>, Span))
+    -> (Option<Stmt_>,Span) {
+    // is it a let?
+    match *s {
+        StmtDecl(@Spanned{node: DeclLocal(ref local), span: stmt_span}, node_id) => {
+            let block_info = get_block_info(exts);
+            let pending_renames = block_info.pending_renames;
+
+            // take it apart:
+            let @Local{is_mutbl:is_mutbl,
+                       ty:_,
+                       pat:pat,
+                       init:init,
+                       id:id,
+                       span:span
+                      } = *local;
+            // types can't be copied automatically because of the owned ptr in ty_tup...
+            let ty = local.ty.clone();
+            // expand the pat (it might contain exprs... #:(o)>
+            let expanded_pat = fld.fold_pat(pat);
+            // find the pat_idents in the pattern:
+            // oh dear heaven... this is going to include the enum names, as well....
+            let idents = @mut ~[];
+            let name_finder = new_name_finder(idents);
+            name_finder.visit_pat(expanded_pat,());
+            // generate fresh names, push them to a new pending list
+            let new_pending_renames = @mut ~[];
+            for ident in idents.iter() {
+                let new_name = fresh_name(ident);
+                new_pending_renames.push((*ident,new_name));
+            }
+            let mut rename_fld = renames_to_fold(new_pending_renames);
+            // rewrite the pattern using the new names (the old ones
+            // have already been applied):
+            let rewritten_pat = rename_fld.fold_pat(expanded_pat);
+            // add them to the existing pending renames:
+            for pr in new_pending_renames.iter() {pending_renames.push(*pr)}
+            // also, don't forget to expand the init:
+            let new_init_opt = init.map(|e| fld.fold_expr(*e));
+            let rewritten_local =
+                @Local{is_mutbl:is_mutbl,
+                       ty:ty,
+                       pat:rewritten_pat,
+                       init:new_init_opt,
+                       id:id,
+                       span:span};
+            (Some(StmtDecl(@Spanned{node:DeclLocal(rewritten_local),
+                                     span: stmt_span},node_id)),
+             sp)
+        },
+        _ => {
+            orig(s, sp, fld)
+        }
+    }
+}
+
+// a visitor that extracts the pat_ident paths
+// from a given pattern and puts them in a mutable
+// array (passed in to the traversal)
 #[deriving(Clone)]
 struct NewNameFinderContext {
     ident_accumulator: @mut ~[ast::Ident],
@@ -674,30 +739,10 @@ pub fn new_name_finder(idents: @mut ~[ast::Ident]) -> @mut Visitor<()> {
     context as @mut Visitor<()>
 }
 
-pub fn expand_block(extsbox: @mut SyntaxEnv,
-                    _cx: @ExtCtxt,
-                    blk: &Block,
-                    fld: @ast_fold,
-                    orig: @fn(&Block, @ast_fold) -> Block)
-                 -> Block {
-    // see note below about treatment of exts table
-    with_exts_frame!(extsbox,false,orig(blk,fld))
-}
-
-
-// get the (innermost) BlockInfo from an exts stack
-fn get_block_info(exts : SyntaxEnv) -> BlockInfo {
-    match exts.find_in_topmost_frame(&intern(special_block_name)) {
-        Some(@BlockInfo(bi)) => bi,
-        _ => fail!(fmt!("special identifier %? was bound to a non-BlockInfo",
-                       @" block"))
-    }
-}
-
-
 // given a mutable list of renames, return a tree-folder that applies those
 // renames.
-fn renames_to_fold(renames : @mut ~[(ast::Ident,ast::Name)]) -> @ast_fold {
+// FIXME #4536: currently pub to allow testing
+pub fn renames_to_fold(renames : @mut ~[(ast::Ident,ast::Name)]) -> @ast_fold {
     let afp = default_ast_fold();
     let f_pre = @AstFoldFns {
         fold_ident: |id,_| {
@@ -713,15 +758,56 @@ fn renames_to_fold(renames : @mut ~[(ast::Ident,ast::Name)]) -> @ast_fold {
     make_fold(f_pre)
 }
 
-// perform a bunch of renames
+pub fn expand_block(extsbox: @mut SyntaxEnv,
-fn apply_pending_renames(folder : @ast_fold, stmt : ast::Stmt) -> @ast::Stmt {
+                    _cx: @ExtCtxt,
-    match folder.fold_stmt(&stmt) {
+                    blk: &Block,
-        Some(s) => s,
+                    fld: @ast_fold,
-        None => fail!(fmt!("renaming of stmt produced None"))
+                    orig: @fn(&Block, @ast_fold) -> Block)
-    }
+                 -> Block {
+    // see note below about treatment of exts table
+    with_exts_frame!(extsbox,false,orig(blk,fld))
 }
 
 
+pub fn expand_block_elts(exts: SyntaxEnv, b: &Block, fld: @ast_fold) -> Block {
+    let block_info = get_block_info(exts);
+    let pending_renames = block_info.pending_renames;
+    let mut rename_fld = renames_to_fold(pending_renames);
+    let new_view_items = b.view_items.map(|x| fld.fold_view_item(x));
+    let mut new_stmts = ~[];
+    for x in b.stmts.iter() {
+        match fld.fold_stmt(mustbesome(rename_fld.fold_stmt(*x))) {
+            Some(s) => new_stmts.push(s),
+            None => ()
+        }
+    }
+    let new_expr = b.expr.map(|x| fld.fold_expr(rename_fld.fold_expr(*x)));
+    Block{
+        view_items: new_view_items,
+        stmts: new_stmts,
+        expr: new_expr,
+        id: fld.new_id(b.id),
+        rules: b.rules,
+        span: b.span,
+    }
+}
+
+// rename_fold should never return "None".
+fn mustbesome<T>(val : Option<T>) -> T {
+    match val {
+        Some(v) => v,
+        None => fail!("rename_fold returned None")
+    }
+}
+
+// get the (innermost) BlockInfo from an exts stack
+fn get_block_info(exts : SyntaxEnv) -> BlockInfo {
+    match exts.find_in_topmost_frame(&intern(special_block_name)) {
+        Some(@BlockInfo(bi)) => bi,
+        _ => fail!(fmt!("special identifier %? was bound to a non-BlockInfo",
+                       @" block"))
+    }
+}
 
 pub fn new_span(cx: @ExtCtxt, sp: Span) -> Span {
     /* this discards information in the case of macro-defining macros */
@@ -1228,12 +1314,15 @@ mod test {
     use super::*;
     use ast;
     use ast::{Attribute_, AttrOuter, MetaWord, EMPTY_CTXT};
+    use ast_util::{get_sctable, new_rename};
     use codemap;
     use codemap::Spanned;
     use parse;
-    use parse::token::{intern, get_ident_interner};
+    use parse::token::{gensym, intern, get_ident_interner};
     use print::pprust;
-    use util::parser_testing::{string_to_item, string_to_pat, strs_to_idents};
+    use std;
+    use util::parser_testing::{string_to_crate_and_sess, string_to_item, string_to_pat};
+    use util::parser_testing::{strs_to_idents};
 
     // make sure that fail! is present
     #[test] fn fail_exists_test () {
@@ -1333,26 +1422,60 @@ mod test {
 
     #[test]
     fn renaming () {
-        let maybe_item_ast = string_to_item(@"fn a() -> int { let b = 13; b }");
+        let item_ast = string_to_item(@"fn a() -> int { let b = 13; b }").unwrap();
-        let item_ast = match maybe_item_ast {
-            Some(x) => x,
-            None => fail!("test case fail")
-        };
         let a_name = intern("a");
-        let a2_name = intern("a2");
+        let a2_name = gensym("a2");
         let renamer = new_ident_renamer(ast::Ident{name:a_name,ctxt:EMPTY_CTXT},
                                         a2_name);
         let renamed_ast = fun_to_ident_folder(renamer).fold_item(item_ast).unwrap();
         let resolver = new_ident_resolver();
-        let resolved_ast = fun_to_ident_folder(resolver).fold_item(renamed_ast).unwrap();
+        let resolver_fold = fun_to_ident_folder(resolver);
+        let resolved_ast = resolver_fold.fold_item(renamed_ast).unwrap();
         let resolved_as_str = pprust::item_to_str(resolved_ast,
                                                   get_ident_interner());
         assert_eq!(resolved_as_str,~"fn a2() -> int { let b = 13; b }");
 
+        // try a double-rename, with pending_renames.
+        let a3_name = gensym("a3");
+        let ctxt2 = new_rename(ast::Ident::new(a_name),a2_name,EMPTY_CTXT);
+        let pending_renames = @mut ~[(ast::Ident::new(a_name),a2_name),
+                                     (ast::Ident{name:a_name,ctxt:ctxt2},a3_name)];
+        let double_renamed = renames_to_fold(pending_renames).fold_item(item_ast).unwrap();
+        let resolved_again = resolver_fold.fold_item(double_renamed).unwrap();
+        let double_renamed_as_str = pprust::item_to_str(resolved_again,
+                                                        get_ident_interner());
+        assert_eq!(double_renamed_as_str,~"fn a3() -> int { let b = 13; b }");
 
     }
 
-    // sigh... it looks like I have two different renaming mechanisms, now...
+    fn fake_print_crate(s: @pprust::ps, crate: &ast::Crate) {
+        pprust::print_mod(s, &crate.module, crate.attrs);
+    }
+
+    // "fn a() -> int { let b = 13; let c = b; b+c }" --> b & c should get new names, in the expr too.
+    // "macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}" --> one should
+    //     be renamed, one should not.
+
+    fn expand_and_resolve_and_pretty_print (crate_str : @str) -> ~str {
+        let resolver = new_ident_resolver();
+        let resolver_fold = fun_to_ident_folder(resolver);
+        let (crate_ast,ps) = string_to_crate_and_sess(crate_str);
+        // the cfg argument actually does matter, here...
+        let expanded_ast = expand_crate(ps,~[],crate_ast);
+        // std::io::println(fmt!("expanded: %?\n",expanded_ast));
+        let resolved_ast = resolver_fold.fold_crate(expanded_ast);
+        pprust::to_str(&resolved_ast,fake_print_crate,get_ident_interner())
+    }
+
+    #[test]
+    fn automatic_renaming () {
+        let teststrs =
+            ~[@"fn a() -> int { let b = 13; let c = b; b+c }",
+              @"macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}"];
+        for s in teststrs.iter() {
+            std::io::println(expand_and_resolve_and_pretty_print(*s));
+        }
+    }
 
     #[test]
     fn pat_idents(){

src/libsyntax/parse/token.rs

@@ -552,9 +552,9 @@ pub fn gensym_ident(str : &str) -> ast::Ident {
 // by using a gensym with a name that has a random number
 // at the end. So, the gensym guarantees the uniqueness,
 // and the int helps to avoid confusion.
-pub fn fresh_name(src_name : &str) -> Name {
+pub fn fresh_name(src_name : &ast::Ident) -> Name {
     let num = rand::rng().gen_uint_range(0,0xffff);
-   gensym(fmt!("%s_%u",src_name,num))
+    gensym(fmt!("%s_%u",ident_to_str(src_name),num))
 }
 
 /**
@@ -697,9 +697,5 @@ pub fn is_reserved_keyword(tok: &Token) -> bool {
 #[cfg(test)]
 mod test {
     use super::*;
-    #[test] fn t1() {
+
-        let a = fresh_name("ghi");
-        printfln!("interned name: %u,\ntextual name: %s\n",
-                  a, interner_get(a));
-    }
 }

src/libsyntax/util/parser_testing.rs

@@ -40,12 +40,19 @@ fn with_error_checking_parse<T>(s: @str, f: &fn(&mut Parser) -> T) -> T {
     x
 }
 
+// parse a string, return a crate.
 pub fn string_to_crate (source_str : @str) -> @ast::Crate {
     do with_error_checking_parse(source_str) |p| {
         p.parse_crate_mod()
     }
 }
 
+// parse a string, return a crate and the ParseSess
+pub fn string_to_crate_and_sess (source_str : @str) -> (@ast::Crate,@mut ParseSess) {
+    let (p,ps) = string_to_parser_and_sess(source_str);
+    (p.parse_crate_mod(),ps)
+}
+
 // parse a string, return an expr
 pub fn string_to_expr (source_str : @str) -> @ast::Expr {
     do with_error_checking_parse(source_str) |p| {
@@ -60,14 +67,6 @@ pub fn string_to_item (source_str : @str) -> Option<@ast::item> {
     }
 }
 
-// parse a string, return an item and the ParseSess
-pub fn string_to_item_and_sess (source_str : @str) -> (Option<@ast::item>,@mut ParseSess) {
-    let (p,ps) = string_to_parser_and_sess(source_str);
-    let io = p.parse_item(~[]);
-    p.abort_if_errors();
-    (io,ps)
-}
-
 // parse a string, return a stmt
 pub fn string_to_stmt(source_str : @str) -> @ast::Stmt {
     do with_error_checking_parse(source_str) |p| {