bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Auto merge of #34570 - jseyfried:no_rename, r=nrc

Browse source 
Simplify the macro hygiene algorithm

This PR removes renaming from the hygiene algorithm and treats differently marked identifiers as unequal.

This change makes the scope of identifiers in `macro_rules!` items empty. That is, identifiers in `macro_rules!` definitions do not inherit any semantics from the `macro_rules!`'s scope.

Since `macro_rules!` macros are items, the scope of their identifiers "should" be the same as that of other items; in particular, the scope should contain only items. Since all items are unhygienic today, this would mean the scope should be empty.

However, the scope of an identifier in a `macro_rules!` statement today is the scope that the identifier would have if it replaced the `macro_rules!` (excluding anything unhygienic, i.e. locals only).

To continue to support this, this PR tracks the scope of each `macro_rules!` and uses it in `resolve` to ensure that an identifier expanded from a `macro_rules!` gets a chance to resolve to the locals in the `macro_rules!`'s scope.

This PR is a pure refactoring. After this PR,
 - `syntax::ext::expand` is much simpler.
 - We can expand macros in any order without causing problems for hygiene (needed for macro modularization).
 - We can deprecate or remove today's `macro_rules!` scope easily.
 - Expansion performance improves by 25%, post-expansion memory usage decreases by ~5%.
 - Expanding a block is no longer quadratic in the number of `let` statements (fixes #10607).

r? @nrc
This commit is contained in:
bors 2016-07-15 08:48:41 -07:00 • committed by GitHub
commit 4db1874f4c
10 changed files with 251 additions and 806 deletions
Download patch file Download diff file Expand all files Collapse all files

70
src/librustc_driver/driver.rs
View file

@ -50,7 +50,6 @@ use std::io::{self, Write};
use std::path::{Path, PathBuf};
use syntax::{ast, diagnostics, visit};
use syntax::attr::{self, AttrMetaMethods};
use syntax::fold::Folder;
use syntax::parse::{self, PResult, token};
use syntax::util::node_count::NodeCounter;
use syntax;
@ -695,6 +694,19 @@ pub fn phase_2_configure_and_expand<'a, F>(sess: &Session,
sess.diagnostic())
});
let resolver_arenas = Resolver::arenas();
let mut resolver = Resolver::new(sess, make_glob_map, &resolver_arenas);
let krate = time(sess.time_passes(), "assigning node ids", || resolver.assign_node_ids(krate));
if sess.opts.debugging_opts.input_stats {
println!("Post-expansion node count: {}", count_nodes(&krate));
}
if sess.opts.debugging_opts.ast_json {
println!("{}", json::as_json(&krate));
}
time(time_passes,
"checking for inline asm in case the target doesn't support it",
|| no_asm::check_crate(sess, &krate));
@ -710,15 +722,6 @@ pub fn phase_2_configure_and_expand<'a, F>(sess: &Session,
})
})?;
if sess.opts.debugging_opts.input_stats {
println!("Post-expansion node count: {}", count_nodes(&krate));
}
krate = assign_node_ids(sess, krate);
let resolver_arenas = Resolver::arenas();
let mut resolver = Resolver::new(sess, make_glob_map, &resolver_arenas);
// Collect defintions for def ids.
time(sess.time_passes(), "collecting defs", || resolver.definitions.collect(&krate));
@ -783,53 +786,6 @@ pub fn phase_2_configure_and_expand<'a, F>(sess: &Session,
})
}
pub fn assign_node_ids(sess: &Session, krate: ast::Crate) -> ast::Crate {
use syntax::ptr::P;
use syntax::util::move_map::MoveMap;
struct NodeIdAssigner<'a> {
sess: &'a Session,
}
impl<'a> Folder for NodeIdAssigner<'a> {
fn new_id(&mut self, old_id: ast::NodeId) -> ast::NodeId {
assert_eq!(old_id, ast::DUMMY_NODE_ID);
self.sess.next_node_id()
}
fn fold_block(&mut self, block: P<ast::Block>) -> P<ast::Block> {
block.map(|mut block| {
block.id = self.new_id(block.id);
let stmt = block.stmts.pop();
block.stmts = block.stmts.move_flat_map(|s| self.fold_stmt(s).into_iter());
if let Some(ast::Stmt { node: ast::StmtKind::Expr(expr), span, .. }) = stmt {
let expr = self.fold_expr(expr);
block.stmts.push(ast::Stmt {
id: expr.id,
node: ast::StmtKind::Expr(expr),
span: span,
});
} else if let Some(stmt) = stmt {
block.stmts.extend(self.fold_stmt(stmt));
}
block
})
}
}
let krate = time(sess.time_passes(),
"assigning node ids",
|| NodeIdAssigner { sess: sess }.fold_crate(krate));
if sess.opts.debugging_opts.ast_json {
println!("{}", json::as_json(&krate));
}
krate
}
/// Run the resolution, typechecking, region checking and other
/// miscellaneous analysis passes on the crate. Return various
/// structures carrying the results of the analysis.

92
src/librustc_resolve/assign_ids.rs Normal file
View file

@ -0,0 +1,92 @@
// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use Resolver;
use rustc::session::Session;
use syntax::ast;
use syntax::ext::mtwt;
use syntax::fold::{self, Folder};
use syntax::ptr::P;
use syntax::util::move_map::MoveMap;
use syntax::util::small_vector::SmallVector;
use std::collections::HashMap;
use std::mem;
impl<'a> Resolver<'a> {
pub fn assign_node_ids(&mut self, krate: ast::Crate) -> ast::Crate {
NodeIdAssigner {
sess: self.session,
macros_at_scope: &mut self.macros_at_scope,
}.fold_crate(krate)
}
}
struct NodeIdAssigner<'a> {
sess: &'a Session,
macros_at_scope: &'a mut HashMap<ast::NodeId, Vec<ast::Mrk>>,
}
impl<'a> Folder for NodeIdAssigner<'a> {
fn new_id(&mut self, old_id: ast::NodeId) -> ast::NodeId {
assert_eq!(old_id, ast::DUMMY_NODE_ID);
self.sess.next_node_id()
}
fn fold_block(&mut self, block: P<ast::Block>) -> P<ast::Block> {
block.map(|mut block| {
block.id = self.new_id(block.id);
let stmt = block.stmts.pop();
let mut macros = Vec::new();
block.stmts = block.stmts.move_flat_map(|stmt| {
if let ast::StmtKind::Item(ref item) = stmt.node {
if let ast::ItemKind::Mac(..) = item.node {
macros.push(mtwt::outer_mark(item.ident.ctxt));
return None;
}
}
let stmt = self.fold_stmt(stmt).pop().unwrap();
if !macros.is_empty() {
self.macros_at_scope.insert(stmt.id, mem::replace(&mut macros, Vec::new()));
}
Some(stmt)
});
stmt.and_then(|mut stmt| {
// Avoid wasting a node id on a trailing expression statement,
// which shares a HIR node with the expression itself.
if let ast::StmtKind::Expr(expr) = stmt.node {
let expr = self.fold_expr(expr);
stmt.id = expr.id;
stmt.node = ast::StmtKind::Expr(expr);
Some(stmt)
} else {
self.fold_stmt(stmt).pop()
}
}).map(|stmt| {
if !macros.is_empty() {
self.macros_at_scope.insert(stmt.id, mem::replace(&mut macros, Vec::new()));
}
block.stmts.push(stmt);
});
block
})
}
fn fold_item(&mut self, item: P<ast::Item>) -> SmallVector<P<ast::Item>> {
match item.node {
ast::ItemKind::Mac(..) => SmallVector::zero(),
_ => fold::noop_fold_item(item, self),
}
}
}

101
src/librustc_resolve/lib.rs
View file

@ -83,6 +83,7 @@ mod diagnostics;
mod check_unused;
mod build_reduced_graph;
mod resolve_imports;
mod assign_ids;
enum SuggestionType {
Macro(String),
@ -461,7 +462,7 @@ struct BindingInfo {
}
// Map from the name in a pattern to its binding mode.
type BindingMap = HashMap<Name, BindingInfo>;
type BindingMap = HashMap<ast::Ident, BindingInfo>;
#[derive(Copy, Clone, PartialEq, Eq, Debug)]
enum PatternSource {
@ -651,6 +652,9 @@ enum RibKind<'a> {
// We passed through a module.
ModuleRibKind(Module<'a>),
// We passed through a `macro_rules!` statement with the given expansion
MacroDefinition(ast::Mrk),
}
#[derive(Copy, Clone)]
@ -667,7 +671,7 @@ enum ModulePrefixResult<'a> {
/// One local scope.
#[derive(Debug)]
struct Rib<'a> {
bindings: HashMap<Name, Def>,
bindings: HashMap<ast::Ident, Def>,
kind: RibKind<'a>,
}
@ -927,6 +931,10 @@ pub struct Resolver<'a> {
pub definitions: Definitions,
// Maps the node id of a statement to the expansions of the `macro_rules!`s
// immediately above the statement (if appropriate).
macros_at_scope: HashMap<NodeId, Vec<ast::Mrk>>,
graph_root: Module<'a>,
prelude: Option<Module<'a>>,
@ -1113,6 +1121,7 @@ impl<'a> Resolver<'a> {
session: session,
definitions: Definitions::new(),
macros_at_scope: HashMap::new(),
// The outermost module has def ID 0; this is not reflected in the
// AST.
@ -1384,15 +1393,17 @@ impl<'a> Resolver<'a> {
/// Invariant: This must only be called during main resolution, not during
/// import resolution.
fn resolve_ident_in_lexical_scope(&mut self,
ident: ast::Ident,
mut ident: ast::Ident,
ns: Namespace,
record_used: bool)
-> Option<LexicalScopeBinding<'a>> {
let name = match ns { ValueNS => mtwt::resolve(ident), TypeNS => ident.name };
if ns == TypeNS {
ident = ast::Ident::with_empty_ctxt(ident.name);
}
// Walk backwards up the ribs in scope.
for i in (0 .. self.get_ribs(ns).len()).rev() {
if let Some(def) = self.get_ribs(ns)[i].bindings.get(&name).cloned() {
if let Some(def) = self.get_ribs(ns)[i].bindings.get(&ident).cloned() {
// The ident resolves to a type parameter or local variable.
return Some(LexicalScopeBinding::LocalDef(LocalDef {
ribs: Some((ns, i)),
@ -1419,6 +1430,16 @@ impl<'a> Resolver<'a> {
};
}
}
if let MacroDefinition(mac) = self.get_ribs(ns)[i].kind {
// If an invocation of this macro created `ident`, give up on `ident`
// and switch to `ident`'s source from the macro definition.
if let Some((source_ident, source_macro)) = mtwt::source(ident) {
if mac == source_macro {
ident = source_ident;
}
}
}
}
None
@ -1555,18 +1576,27 @@ impl<'a> Resolver<'a> {
/// Searches the current set of local scopes for labels.
/// Stops after meeting a closure.
fn search_label(&self, name: Name) -> Option<Def> {
fn search_label(&self, mut ident: ast::Ident) -> Option<Def> {
for rib in self.label_ribs.iter().rev() {
match rib.kind {
NormalRibKind => {
// Continue
}
MacroDefinition(mac) => {
// If an invocation of this macro created `ident`, give up on `ident`
// and switch to `ident`'s source from the macro definition.
if let Some((source_ident, source_macro)) = mtwt::source(ident) {
if mac == source_macro {
ident = source_ident;
}
}
}
_ => {
// Do not resolve labels across function boundary
return None;
}
}
let result = rib.bindings.get(&name).cloned();
let result = rib.bindings.get(&ident).cloned();
if result.is_some() {
return result;
}
@ -1715,7 +1745,7 @@ impl<'a> Resolver<'a> {
// plain insert (no renaming)
let def_id = self.definitions.local_def_id(type_parameter.id);
let def = Def::TyParam(space, index as u32, def_id, name);
function_type_rib.bindings.insert(name, def);
function_type_rib.bindings.insert(ast::Ident::with_empty_ctxt(name), def);
}
self.type_ribs.push(function_type_rib);
}
@ -1886,7 +1916,7 @@ impl<'a> Resolver<'a> {
let mut self_type_rib = Rib::new(NormalRibKind);
// plain insert (no renaming, types are not currently hygienic....)
self_type_rib.bindings.insert(keywords::SelfType.name(), self_def);
self_type_rib.bindings.insert(keywords::SelfType.ident(), self_def);
self.type_ribs.push(self_type_rib);
f(self);
self.type_ribs.pop();
@ -1997,7 +2027,7 @@ impl<'a> Resolver<'a> {
_ => false,
} {
let binding_info = BindingInfo { span: ident.span, binding_mode: binding_mode };
binding_map.insert(mtwt::resolve(ident.node), binding_info);
binding_map.insert(ident.node, binding_info);
}
}
true
@ -2019,15 +2049,14 @@ impl<'a> Resolver<'a> {
for (&key, &binding_0) in &map_0 {
match map_i.get(&key) {
None => {
resolve_error(self,
p.span,
ResolutionError::VariableNotBoundInPattern(key, 1, i + 1));
let error = ResolutionError::VariableNotBoundInPattern(key.name, 1, i + 1);
resolve_error(self, p.span, error);
}
Some(binding_i) => {
if binding_0.binding_mode != binding_i.binding_mode {
resolve_error(self,
binding_i.span,
ResolutionError::VariableBoundWithDifferentMode(key,
ResolutionError::VariableBoundWithDifferentMode(key.name,
i + 1));
}
}
@ -2038,7 +2067,7 @@ impl<'a> Resolver<'a> {
if !map_0.contains_key(&key) {
resolve_error(self,
binding.span,
ResolutionError::VariableNotBoundInPattern(key, i + 1, 1));
ResolutionError::VariableNotBoundInPattern(key.name, i + 1, 1));
}
}
}
@ -2068,6 +2097,7 @@ impl<'a> Resolver<'a> {
let orig_module = self.current_module;
let anonymous_module = self.module_map.get(&block.id).cloned(); // clones a reference
let mut num_macro_definition_ribs = 0;
if let Some(anonymous_module) = anonymous_module {
debug!("(resolving block) found anonymous module, moving down");
self.value_ribs.push(Rib::new(ModuleRibKind(anonymous_module)));
@ -2078,10 +2108,24 @@ impl<'a> Resolver<'a> {
}
// Descend into the block.
visit::walk_block(self, block);
for stmt in &block.stmts {
if let Some(marks) = self.macros_at_scope.remove(&stmt.id) {
num_macro_definition_ribs += marks.len() as u32;
for mark in marks {
self.value_ribs.push(Rib::new(MacroDefinition(mark)));
self.label_ribs.push(Rib::new(MacroDefinition(mark)));
}
}
self.visit_stmt(stmt);
}
// Move back up.
self.current_module = orig_module;
for _ in 0 .. num_macro_definition_ribs {
self.value_ribs.pop();
self.label_ribs.pop();
}
self.value_ribs.pop();
if let Some(_) = anonymous_module {
self.type_ribs.pop();
@ -2172,16 +2216,15 @@ impl<'a> Resolver<'a> {
pat_id: NodeId,
outer_pat_id: NodeId,
pat_src: PatternSource,
bindings: &mut HashMap<Name, NodeId>)
bindings: &mut HashMap<ast::Ident, NodeId>)
-> PathResolution {
// Add the binding to the local ribs, if it
// doesn't already exist in the bindings map. (We
// must not add it if it's in the bindings map
// because that breaks the assumptions later
// passes make about or-patterns.)
let renamed = mtwt::resolve(ident.node);
let mut def = Def::Local(self.definitions.local_def_id(pat_id), pat_id);
match bindings.get(&renamed).cloned() {
match bindings.get(&ident.node).cloned() {
Some(id) if id == outer_pat_id => {
// `Variant(a, a)`, error
resolve_error(
@ -2203,7 +2246,7 @@ impl<'a> Resolver<'a> {
Some(..) if pat_src == PatternSource::Match => {
// `Variant1(a) | Variant2(a)`, ok
// Reuse definition from the first `a`.
def = self.value_ribs.last_mut().unwrap().bindings[&renamed];
def = self.value_ribs.last_mut().unwrap().bindings[&ident.node];
}
Some(..) => {
span_bug!(ident.span, "two bindings with the same name from \
@ -2212,8 +2255,8 @@ impl<'a> Resolver<'a> {
None => {
// A completely fresh binding, add to the lists if it's valid.
if ident.node.name != keywords::Invalid.name() {
bindings.insert(renamed, outer_pat_id);
self.value_ribs.last_mut().unwrap().bindings.insert(renamed, def);
bindings.insert(ident.node, outer_pat_id);
self.value_ribs.last_mut().unwrap().bindings.insert(ident.node, def);
}
}
}
@ -2274,7 +2317,7 @@ impl<'a> Resolver<'a> {
pat_src: PatternSource,
// Maps idents to the node ID for the
// outermost pattern that binds them.
bindings: &mut HashMap<Name, NodeId>) {
bindings: &mut HashMap<ast::Ident, NodeId>) {
// Visit all direct subpatterns of this pattern.
let outer_pat_id = pat.id;
pat.walk(&mut |pat| {
@ -2497,7 +2540,7 @@ impl<'a> Resolver<'a> {
Def::Local(_, node_id) => {
for rib in ribs {
match rib.kind {
NormalRibKind | ModuleRibKind(..) => {
NormalRibKind | ModuleRibKind(..) | MacroDefinition(..) => {
// Nothing to do. Continue.
}
ClosureRibKind(function_id) => {
@ -2546,7 +2589,7 @@ impl<'a> Resolver<'a> {
for rib in ribs {
match rib.kind {
NormalRibKind | MethodRibKind(_) | ClosureRibKind(..) |
ModuleRibKind(..) => {
ModuleRibKind(..) | MacroDefinition(..) => {
// Nothing to do. Continue.
}
ItemRibKind => {
@ -2747,7 +2790,7 @@ impl<'a> Resolver<'a> {
let names = self.value_ribs
.iter()
.rev()
.flat_map(|rib| rib.bindings.keys());
.flat_map(|rib| rib.bindings.keys().map(|ident| &ident.name));
if let Some(found) = find_best_match_for_name(names, name, None) {
if name != found {
@ -2758,7 +2801,7 @@ impl<'a> Resolver<'a> {
fn resolve_labeled_block(&mut self, label: Option<ast::Ident>, id: NodeId, block: &Block) {
if let Some(label) = label {
let (label, def) = (mtwt::resolve(label), Def::Label(id));
let def = Def::Label(id);
self.with_label_rib(|this| {
this.label_ribs.last_mut().unwrap().bindings.insert(label, def);
this.visit_block(block);
@ -2965,7 +3008,7 @@ impl<'a> Resolver<'a> {
{
let rib = this.label_ribs.last_mut().unwrap();
rib.bindings.insert(mtwt::resolve(label.node), def);
rib.bindings.insert(label.node, def);
}
visit::walk_expr(this, expr);
@ -2973,7 +3016,7 @@ impl<'a> Resolver<'a> {
}
ExprKind::Break(Some(label)) | ExprKind::Continue(Some(label)) => {
match self.search_label(mtwt::resolve(label.node)) {
match self.search_label(label.node) {
None => {
self.record_def(expr.id, err_path_resolution());
resolve_error(self,

37
src/libsyntax/ast.rs
View file

@ -26,7 +26,6 @@ use tokenstream::{TokenTree};
use std::fmt;
use std::rc::Rc;
use std::hash::{Hash, Hasher};
use serialize::{Encodable, Decodable, Encoder, Decoder};
/// A name is a part of an identifier, representing a string or gensym. It's
@ -46,7 +45,7 @@ pub struct SyntaxContext(pub u32);
/// An identifier contains a Name (index into the interner
/// table) and a SyntaxContext to track renaming and
/// macro expansion per Flatt et al., "Macros That Work Together"
#[derive(Clone, Copy, Eq)]
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
pub struct Ident {
pub name: Name,
pub ctxt: SyntaxContext
@ -93,40 +92,6 @@ impl Ident {
}
}
impl PartialEq for Ident {
fn eq(&self, other: &Ident) -> bool {
if self.ctxt != other.ctxt {
// There's no one true way to compare Idents. They can be compared
// non-hygienically `id1.name == id2.name`, hygienically
// `mtwt::resolve(id1) == mtwt::resolve(id2)`, or even member-wise
// `(id1.name, id1.ctxt) == (id2.name, id2.ctxt)` depending on the situation.
// Ideally, PartialEq should not be implemented for Ident at all, but that
// would be too impractical, because many larger structures (Token, in particular)
// including Idents as their parts derive PartialEq and use it for non-hygienic
// comparisons. That's why PartialEq is implemented and defaults to non-hygienic
// comparison. Hash is implemented too and is consistent with PartialEq, i.e. only
// the name of Ident is hashed. Still try to avoid comparing idents in your code
// (especially as keys in hash maps), use one of the three methods listed above
// explicitly.
//
// If you see this panic, then some idents from different contexts were compared
// non-hygienically. It's likely a bug. Use one of the three comparison methods
// listed above explicitly.
panic!("idents with different contexts are compared with operator `==`: \
{:?}, {:?}.", self, other);
}
self.name == other.name
}
}
impl Hash for Ident {
fn hash<H: Hasher>(&self, state: &mut H) {
self.name.hash(state)
}
}
impl fmt::Debug for Ident {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{}#{}", self.name, self.ctxt.0)

4
src/libsyntax/ext/base.rs
View file

@ -26,7 +26,6 @@ use parse::token::{InternedString, intern, str_to_ident};
use ptr::P;
use util::small_vector::SmallVector;
use util::lev_distance::find_best_match_for_name;
use ext::mtwt;
use fold::Folder;
use std::collections::{HashMap, HashSet};
@ -483,15 +482,12 @@ pub type NamedSyntaxExtension = (Name, SyntaxExtension);
pub struct BlockInfo {
/// Should macros escape from this scope?
pub macros_escape: bool,
/// What are the pending renames?
pub pending_renames: mtwt::RenameList,
}
impl BlockInfo {
pub fn new() -> BlockInfo {
BlockInfo {
macros_escape: false,
pending_renames: Vec::new(),
}
}
}

592
src/libsyntax/ext/expand.rs
View file

@ -8,23 +8,21 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use ast::{Block, Crate, PatKind};
use ast::{Local, Ident, Mac_, Name, SpannedIdent};
use ast::{Block, Crate, Ident, Mac_, Name, PatKind};
use ast::{MacStmtStyle, Mrk, Stmt, StmtKind, ItemKind};
use ast;
use attr::HasAttrs;
use ext::mtwt;
use attr;
use attr::AttrMetaMethods;
use codemap::{Spanned, ExpnInfo, NameAndSpan, MacroBang, MacroAttribute};
use codemap::{dummy_spanned, Spanned, ExpnInfo, NameAndSpan, MacroBang, MacroAttribute};
use syntax_pos::{self, Span, ExpnId};
use config::StripUnconfigured;
use ext::base::*;
use feature_gate::{self, Features};
use fold;
use fold::*;
use util::move_map::MoveMap;
use parse::token::{fresh_mark, fresh_name, intern, keywords};
use parse::token::{fresh_mark, intern, keywords};
use ptr::P;
use tokenstream::TokenTree;
use util::small_vector::SmallVector;
@ -96,89 +94,32 @@ impl MacroGenerable for Option<P<ast::Expr>> {
}
}
pub fn expand_expr(mut expr: ast::Expr, fld: &mut MacroExpander) -> P<ast::Expr> {
pub fn expand_expr(expr: ast::Expr, fld: &mut MacroExpander) -> P<ast::Expr> {
match expr.node {
// expr_mac should really be expr_ext or something; it's the
// entry-point for all syntax extensions.
ast::ExprKind::Mac(mac) => {
return expand_mac_invoc(mac, None, expr.attrs.into(), expr.span, fld);
}
_ => P(noop_fold_expr(expr, fld)),
}
}
ast::ExprKind::While(cond, body, opt_ident) => {
let cond = fld.fold_expr(cond);
let (body, opt_ident) = expand_loop_block(body, opt_ident, fld);
expr.node = ast::ExprKind::While(cond, body, opt_ident);
}
ast::ExprKind::WhileLet(pat, cond, body, opt_ident) => {
let pat = fld.fold_pat(pat);
let cond = fld.fold_expr(cond);
// Hygienic renaming of the body.
let ((body, opt_ident), mut rewritten_pats) =
rename_in_scope(vec![pat],
fld,
(body, opt_ident),
|rename_fld, fld, (body, opt_ident)| {
expand_loop_block(rename_fld.fold_block(body), opt_ident, fld)
});
assert!(rewritten_pats.len() == 1);
expr.node = ast::ExprKind::WhileLet(rewritten_pats.remove(0), cond, body, opt_ident);
}
ast::ExprKind::Loop(loop_block, opt_ident) => {
let (loop_block, opt_ident) = expand_loop_block(loop_block, opt_ident, fld);
expr.node = ast::ExprKind::Loop(loop_block, opt_ident);
}
ast::ExprKind::ForLoop(pat, head, body, opt_ident) => {
let pat = fld.fold_pat(pat);
// Hygienic renaming of the for loop body (for loop binds its pattern).
let ((body, opt_ident), mut rewritten_pats) =
rename_in_scope(vec![pat],
fld,
(body, opt_ident),
|rename_fld, fld, (body, opt_ident)| {
expand_loop_block(rename_fld.fold_block(body), opt_ident, fld)
});
assert!(rewritten_pats.len() == 1);
let head = fld.fold_expr(head);
expr.node = ast::ExprKind::ForLoop(rewritten_pats.remove(0), head, body, opt_ident);
}
ast::ExprKind::IfLet(pat, sub_expr, body, else_opt) => {
let pat = fld.fold_pat(pat);
// Hygienic renaming of the body.
let (body, mut rewritten_pats) =
rename_in_scope(vec![pat],
fld,
body,
|rename_fld, fld, body| {
fld.fold_block(rename_fld.fold_block(body))
});
assert!(rewritten_pats.len() == 1);
let else_opt = else_opt.map(|else_opt| fld.fold_expr(else_opt));
let sub_expr = fld.fold_expr(sub_expr);
expr.node = ast::ExprKind::IfLet(rewritten_pats.remove(0), sub_expr, body, else_opt);
}
ast::ExprKind::Closure(capture_clause, fn_decl, block, fn_decl_span) => {
let (rewritten_fn_decl, rewritten_block)
= expand_and_rename_fn_decl_and_block(fn_decl, block, fld);
expr.node = ast::ExprKind::Closure(capture_clause,
rewritten_fn_decl,
rewritten_block,
fn_decl_span);
}
_ => expr = noop_fold_expr(expr, fld),
};
P(expr)
struct MacroScopePlaceholder;
impl MacResult for MacroScopePlaceholder {
fn make_items(self: Box<Self>) -> Option<SmallVector<P<ast::Item>>> {
Some(SmallVector::one(P(ast::Item {
ident: keywords::Invalid.ident(),
attrs: Vec::new(),
id: ast::DUMMY_NODE_ID,
node: ast::ItemKind::Mac(dummy_spanned(ast::Mac_ {
path: ast::Path { span: syntax_pos::DUMMY_SP, global: false, segments: Vec::new() },
tts: Vec::new(),
})),
vis: ast::Visibility::Inherited,
span: syntax_pos::DUMMY_SP,
})))
}
}
/// Expand a macro invocation. Returns the result of expansion.
@ -219,6 +160,7 @@ fn expand_mac_invoc<T>(mac: ast::Mac, ident: Option<Ident>, attrs: Vec<ast::Attr
};
let ident = ident.unwrap_or(keywords::Invalid.ident());
let marked_tts = mark_tts(&tts, mark);
match *extension {
NormalTT(ref expandfun, exp_span, allow_internal_unstable) => {
if ident.name != keywords::Invalid.name() {
@ -237,7 +179,6 @@ fn expand_mac_invoc<T>(mac: ast::Mac, ident: Option<Ident>, attrs: Vec<ast::Attr
},
});
let marked_tts = mark_tts(&tts, mark);
Some(expandfun.expand(fld.cx, call_site, &marked_tts))
}
@ -257,7 +198,6 @@ fn expand_mac_invoc<T>(mac: ast::Mac, ident: Option<Ident>, attrs: Vec<ast::Attr
}
});
let marked_tts = mark_tts(&tts, mark);
Some(expander.expand(fld.cx, call_site, ident, marked_tts))
}
@ -286,15 +226,14 @@ fn expand_mac_invoc<T>(mac: ast::Mac, ident: Option<Ident>, attrs: Vec<ast::Attr
span: call_site,
imported_from: None,
use_locally: true,
body: tts,
body: marked_tts,
export: attr::contains_name(&attrs, "macro_export"),
allow_internal_unstable: attr::contains_name(&attrs, "allow_internal_unstable"),
attrs: attrs,
});
// macro_rules! has a side effect but expands to nothing.
fld.cx.bt_pop();
None
Some(Box::new(MacroScopePlaceholder))
}
MultiDecorator(..) | MultiModifier(..) => {
@ -327,41 +266,6 @@ fn expand_mac_invoc<T>(mac: ast::Mac, ident: Option<Ident>, attrs: Vec<ast::Attr
fully_expanded
}
/// Rename loop label and expand its loop body
///
/// The renaming procedure for loop is different in the sense that the loop
/// body is in a block enclosed by loop head so the renaming of loop label
/// must be propagated to the enclosed context.
fn expand_loop_block(loop_block: P<Block>,
opt_ident: Option<SpannedIdent>,
fld: &mut MacroExpander) -> (P<Block>, Option<SpannedIdent>) {
match opt_ident {
Some(label) => {
let new_label = fresh_name(label.node);
let rename = (label.node, new_label);
// The rename *must not* be added to the pending list of current
// syntax context otherwise an unrelated `break` or `continue` in
// the same context will pick that up in the deferred renaming pass
// and be renamed incorrectly.
let mut rename_list = vec!(rename);
let mut rename_fld = IdentRenamer{renames: &mut rename_list};
let renamed_ident = rename_fld.fold_ident(label.node);
// The rename *must* be added to the enclosed syntax context for
// `break` or `continue` to pick up because by definition they are
// in a block enclosed by loop head.
fld.cx.syntax_env.push_frame();
fld.cx.syntax_env.info().pending_renames.push(rename);
let expanded_block = expand_block_elts(loop_block, fld);
fld.cx.syntax_env.pop_frame();
(expanded_block, Some(Spanned { node: renamed_ident, span: label.span }))
}
None => (fld.fold_block(loop_block), opt_ident)
}
}
// eval $e with a new exts frame.
// must be a macro so that $e isn't evaluated too early.
macro_rules! with_exts_frame {
@ -381,20 +285,6 @@ pub fn expand_item(it: P<ast::Item>, fld: &mut MacroExpander)
.into_iter().map(|i| i.expect_item()).collect()
}
/// Expand item_kind
fn expand_item_kind(item: ast::ItemKind, fld: &mut MacroExpander) -> ast::ItemKind {
match item {
ast::ItemKind::Fn(decl, unsafety, constness, abi, generics, body) => {
let (rewritten_fn_decl, rewritten_body)
= expand_and_rename_fn_decl_and_block(decl, body, fld);
let expanded_generics = fold::noop_fold_generics(generics,fld);
ast::ItemKind::Fn(rewritten_fn_decl, unsafety, constness, abi,
expanded_generics, rewritten_body)
}
_ => noop_fold_item_kind(item, fld)
}
}
// does this attribute list contain "macro_use" ?
fn contains_macro_use(fld: &mut MacroExpander, attrs: &[ast::Attribute]) -> bool {
for attr in attrs {
@ -425,16 +315,9 @@ fn contains_macro_use(fld: &mut MacroExpander, attrs: &[ast::Attribute]) -> bool
/// Expand a stmt
fn expand_stmt(stmt: Stmt, fld: &mut MacroExpander) -> SmallVector<Stmt> {
// perform all pending renames
let stmt = {
let pending_renames = &mut fld.cx.syntax_env.info().pending_renames;
let mut rename_fld = IdentRenamer{renames:pending_renames};
rename_fld.fold_stmt(stmt).expect_one("rename_fold didn't return one value")
};
let (mac, style, attrs) = match stmt.node {
StmtKind::Mac(mac) => mac.unwrap(),
_ => return expand_non_macro_stmt(stmt, fld)
_ => return noop_fold_stmt(stmt, fld)
};
let mut fully_expanded: SmallVector<ast::Stmt> =
@ -451,167 +334,6 @@ fn expand_stmt(stmt: Stmt, fld: &mut MacroExpander) -> SmallVector<Stmt> {
fully_expanded
}
// expand a non-macro stmt. this is essentially the fallthrough for
// expand_stmt, above.
fn expand_non_macro_stmt(stmt: Stmt, fld: &mut MacroExpander)
-> SmallVector<Stmt> {
// is it a let?
match stmt.node {
StmtKind::Local(local) => {
// take it apart:
let rewritten_local = local.map(|Local {id, pat, ty, init, span, attrs}| {
// expand the ty since TyKind::FixedLengthVec contains an Expr
// and thus may have a macro use
let expanded_ty = ty.map(|t| fld.fold_ty(t));
// expand the pat (it might contain macro uses):
let expanded_pat = fld.fold_pat(pat);
// find the PatIdents in the pattern:
// oh dear heaven... this is going to include the enum
// names, as well... but that should be okay, as long as
// the new names are gensyms for the old ones.
// generate fresh names, push them to a new pending list
let idents = pattern_bindings(&expanded_pat);
let mut new_pending_renames =
idents.iter().map(|ident| (*ident, fresh_name(*ident))).collect();
// rewrite the pattern using the new names (the old
// ones have already been applied):
let rewritten_pat = {
// nested binding to allow borrow to expire:
let mut rename_fld = IdentRenamer{renames: &mut new_pending_renames};
rename_fld.fold_pat(expanded_pat)
};
// add them to the existing pending renames:
fld.cx.syntax_env.info().pending_renames
.extend(new_pending_renames);
Local {
id: id,
ty: expanded_ty,
pat: rewritten_pat,
// also, don't forget to expand the init:
init: init.map(|e| fld.fold_expr(e)),
span: span,
attrs: fold::fold_thin_attrs(attrs, fld),
}
});
SmallVector::one(Stmt {
id: stmt.id,
node: StmtKind::Local(rewritten_local),
span: stmt.span,
})
}
_ => noop_fold_stmt(stmt, fld),
}
}
// expand the arm of a 'match', renaming for macro hygiene
fn expand_arm(arm: ast::Arm, fld: &mut MacroExpander) -> ast::Arm {
// expand pats... they might contain macro uses:
let expanded_pats = arm.pats.move_map(|pat| fld.fold_pat(pat));
if expanded_pats.is_empty() {
panic!("encountered match arm with 0 patterns");
}
// apply renaming and then expansion to the guard and the body:
let ((rewritten_guard, rewritten_body), rewritten_pats) =
rename_in_scope(expanded_pats,
fld,
(arm.guard, arm.body),
|rename_fld, fld, (ag, ab)|{
let rewritten_guard = ag.map(|g| fld.fold_expr(rename_fld.fold_expr(g)));
let rewritten_body = fld.fold_expr(rename_fld.fold_expr(ab));
(rewritten_guard, rewritten_body)
});
ast::Arm {
attrs: fold::fold_attrs(arm.attrs, fld),
pats: rewritten_pats,
guard: rewritten_guard,
body: rewritten_body,
}
}
fn rename_in_scope<X, F>(pats: Vec<P<ast::Pat>>,
fld: &mut MacroExpander,
x: X,
f: F)
-> (X, Vec<P<ast::Pat>>)
where F: Fn(&mut IdentRenamer, &mut MacroExpander, X) -> X
{
// all of the pats must have the same set of bindings, so use the
// first one to extract them and generate new names:
let idents = pattern_bindings(&pats[0]);
let new_renames = idents.into_iter().map(|id| (id, fresh_name(id))).collect();
// apply the renaming, but only to the PatIdents:
let mut rename_pats_fld = PatIdentRenamer{renames:&new_renames};
let rewritten_pats = pats.move_map(|pat| rename_pats_fld.fold_pat(pat));
let mut rename_fld = IdentRenamer{ renames:&new_renames };
(f(&mut rename_fld, fld, x), rewritten_pats)
}
/// A visitor that extracts the PatKind::Ident (binding) paths
/// from a given thingy and puts them in a mutable
/// array
#[derive(Clone)]
struct PatIdentFinder {
ident_accumulator: Vec<ast::Ident>
}
impl Visitor for PatIdentFinder {
fn visit_pat(&mut self, pattern: &ast::Pat) {
match *pattern {
ast::Pat { id: _, node: PatKind::Ident(_, ref path1, ref inner), span: _ } => {
self.ident_accumulator.push(path1.node);
// visit optional subpattern of PatKind::Ident:
if let Some(ref subpat) = *inner {
self.visit_pat(subpat)
}
}
// use the default traversal for non-PatIdents
_ => visit::walk_pat(self, pattern)
}
}
}
/// find the PatKind::Ident paths in a pattern
fn pattern_bindings(pat: &ast::Pat) -> Vec<ast::Ident> {
let mut name_finder = PatIdentFinder{ident_accumulator:Vec::new()};
name_finder.visit_pat(pat);
name_finder.ident_accumulator
}
/// find the PatKind::Ident paths in a
fn fn_decl_arg_bindings(fn_decl: &ast::FnDecl) -> Vec<ast::Ident> {
let mut pat_idents = PatIdentFinder{ident_accumulator:Vec::new()};
for arg in &fn_decl.inputs {
pat_idents.visit_pat(&arg.pat);
}
pat_idents.ident_accumulator
}
// expand a block. pushes a new exts_frame, then calls expand_block_elts
pub fn expand_block(blk: P<Block>, fld: &mut MacroExpander) -> P<Block> {
// see note below about treatment of exts table
with_exts_frame!(fld.cx.syntax_env,false,
expand_block_elts(blk, fld))
}
// expand the elements of a block.
pub fn expand_block_elts(b: P<Block>, fld: &mut MacroExpander) -> P<Block> {
b.map(|Block {id, stmts, rules, span}| {
let new_stmts = stmts.into_iter().flat_map(|x| {
// perform pending renames and expand macros in the statement
fld.fold_stmt(x).into_iter()
}).collect();
Block {
id: fld.new_id(id),
stmts: new_stmts,
rules: rules,
span: span
}
})
}
fn expand_pat(p: P<ast::Pat>, fld: &mut MacroExpander) -> P<ast::Pat> {
match p.node {
PatKind::Mac(_) => {}
@ -625,62 +347,16 @@ fn expand_pat(p: P<ast::Pat>, fld: &mut MacroExpander) -> P<ast::Pat> {
})
}
/// A tree-folder that applies every rename in its (mutable) list
/// to every identifier, including both bindings and varrefs
/// (and lots of things that will turn out to be neither)
pub struct IdentRenamer<'a> {
renames: &'a mtwt::RenameList,
}
impl<'a> Folder for IdentRenamer<'a> {
fn fold_ident(&mut self, id: Ident) -> Ident {
mtwt::apply_renames(self.renames, id)
}
fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac {
fold::noop_fold_mac(mac, self)
}
}
/// A tree-folder that applies every rename in its list to
/// the idents that are in PatKind::Ident patterns. This is more narrowly
/// focused than IdentRenamer, and is needed for FnDecl,
/// where we want to rename the args but not the fn name or the generics etc.
pub struct PatIdentRenamer<'a> {
renames: &'a mtwt::RenameList,
}
impl<'a> Folder for PatIdentRenamer<'a> {
fn fold_pat(&mut self, pat: P<ast::Pat>) -> P<ast::Pat> {
match pat.node {
PatKind::Ident(..) => {},
_ => return noop_fold_pat(pat, self)
}
pat.map(|ast::Pat {id, node, span}| match node {
PatKind::Ident(binding_mode, Spanned{span: sp, node: ident}, sub) => {
let new_ident = mtwt::apply_renames(self.renames, ident);
let new_node =
PatKind::Ident(binding_mode,
Spanned{span: sp, node: new_ident},
sub.map(|p| self.fold_pat(p)));
ast::Pat {
id: id,
node: new_node,
span: span,
}
},
_ => unreachable!()
})
}
fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac {
fold::noop_fold_mac(mac, self)
}
}
fn expand_multi_modified(a: Annotatable, fld: &mut MacroExpander) -> SmallVector<Annotatable> {
match a {
Annotatable::Item(it) => match it.node {
ast::ItemKind::Mac(..) => {
if match it.node {
ItemKind::Mac(ref mac) => mac.node.path.segments.is_empty(),
_ => unreachable!(),
} {
return SmallVector::one(Annotatable::Item(it));
}
it.and_then(|it| match it.node {
ItemKind::Mac(mac) =>
expand_mac_invoc(mac, Some(it.ident), it.attrs, it.span, fld),
@ -774,21 +450,6 @@ fn expand_annotatable(mut item: Annotatable, fld: &mut MacroExpander) -> SmallVe
fn expand_impl_item(ii: ast::ImplItem, fld: &mut MacroExpander)
-> SmallVector<ast::ImplItem> {
match ii.node {
ast::ImplItemKind::Method(..) => SmallVector::one(ast::ImplItem {
id: ii.id,
ident: ii.ident,
attrs: ii.attrs,
vis: ii.vis,
defaultness: ii.defaultness,
node: match ii.node {
ast::ImplItemKind::Method(sig, body) => {
let (sig, body) = expand_and_rename_method(sig, body, fld);
ast::ImplItemKind::Method(sig, body)
}
_ => unreachable!()
},
span: ii.span,
}),
ast::ImplItemKind::Macro(mac) => {
expand_mac_invoc(mac, None, ii.attrs, ii.span, fld)
}
@ -799,21 +460,6 @@ fn expand_impl_item(ii: ast::ImplItem, fld: &mut MacroExpander)
fn expand_trait_item(ti: ast::TraitItem, fld: &mut MacroExpander)
-> SmallVector<ast::TraitItem> {
match ti.node {
ast::TraitItemKind::Method(_, Some(_)) => {
SmallVector::one(ast::TraitItem {
id: ti.id,
ident: ti.ident,
attrs: ti.attrs,
node: match ti.node {
ast::TraitItemKind::Method(sig, Some(body)) => {
let (sig, body) = expand_and_rename_method(sig, body, fld);
ast::TraitItemKind::Method(sig, Some(body))
}
_ => unreachable!()
},
span: ti.span,
})
}
ast::TraitItemKind::Macro(mac) => {
expand_mac_invoc(mac, None, ti.attrs, ti.span, fld)
}
@ -821,39 +467,6 @@ fn expand_trait_item(ti: ast::TraitItem, fld: &mut MacroExpander)
}
}
/// Given a fn_decl and a block and a MacroExpander, expand the fn_decl, then use the
/// PatIdents in its arguments to perform renaming in the FnDecl and
/// the block, returning both the new FnDecl and the new Block.
fn expand_and_rename_fn_decl_and_block(fn_decl: P<ast::FnDecl>, block: P<ast::Block>,
fld: &mut MacroExpander)
-> (P<ast::FnDecl>, P<ast::Block>) {
let expanded_decl = fld.fold_fn_decl(fn_decl);
let idents = fn_decl_arg_bindings(&expanded_decl);
let renames =
idents.iter().map(|id| (*id,fresh_name(*id))).collect();
// first, a renamer for the PatIdents, for the fn_decl:
let mut rename_pat_fld = PatIdentRenamer{renames: &renames};
let rewritten_fn_decl = rename_pat_fld.fold_fn_decl(expanded_decl);
// now, a renamer for *all* idents, for the body:
let mut rename_fld = IdentRenamer{renames: &renames};
let rewritten_body = fld.fold_block(rename_fld.fold_block(block));
(rewritten_fn_decl,rewritten_body)
}
fn expand_and_rename_method(sig: ast::MethodSig, body: P<ast::Block>,
fld: &mut MacroExpander)
-> (ast::MethodSig, P<ast::Block>) {
let (rewritten_fn_decl, rewritten_body)
= expand_and_rename_fn_decl_and_block(sig.decl, body, fld);
(ast::MethodSig {
generics: fld.fold_generics(sig.generics),
abi: sig.abi,
unsafety: sig.unsafety,
constness: sig.constness,
decl: rewritten_fn_decl
}, rewritten_body)
}
pub fn expand_type(t: P<ast::Ty>, fld: &mut MacroExpander) -> P<ast::Ty> {
let t = match t.node.clone() {
ast::TyKind::Mac(mac) => {
@ -976,25 +589,17 @@ impl<'a, 'b> Folder for MacroExpander<'a, 'b> {
result
}
fn fold_item_kind(&mut self, item: ast::ItemKind) -> ast::ItemKind {
expand_item_kind(item, self)
}
fn fold_stmt(&mut self, stmt: ast::Stmt) -> SmallVector<ast::Stmt> {
expand_stmt(stmt, self)
}
fn fold_block(&mut self, block: P<Block>) -> P<Block> {
let was_in_block = ::std::mem::replace(&mut self.cx.in_block, true);
let result = expand_block(block, self);
let result = with_exts_frame!(self.cx.syntax_env, false, noop_fold_block(block, self));
self.cx.in_block = was_in_block;
result
}
fn fold_arm(&mut self, arm: ast::Arm) -> ast::Arm {
expand_arm(arm, self)
}
fn fold_trait_item(&mut self, i: ast::TraitItem) -> SmallVector<ast::TraitItem> {
expand_annotatable(Annotatable::TraitItem(P(i)), self)
.into_iter().map(|i| i.expect_trait_item()).collect()
@ -1145,18 +750,11 @@ fn mark_tts(tts: &[TokenTree], m: Mrk) -> Vec<TokenTree> {
#[cfg(test)]
mod tests {
use super::{pattern_bindings, expand_crate};
use super::{PatIdentFinder, IdentRenamer, PatIdentRenamer, ExpansionConfig};
use super::{expand_crate, ExpansionConfig};
use ast;
use ast::Name;
use syntax_pos;
use ext::base::{ExtCtxt, DummyMacroLoader};
use ext::mtwt;
use fold::Folder;
use parse;
use parse::token;
use util::parser_testing::{string_to_parser};
use util::parser_testing::{string_to_pat, string_to_crate, strs_to_idents};
use visit;
use visit::Visitor;
@ -1177,32 +775,6 @@ mod tests {
}
}
// find the variable references in a crate
fn crate_varrefs(the_crate : &ast::Crate) -> Vec<ast::Path> {
let mut path_finder = PathExprFinderContext{path_accumulator:Vec::new()};
visit::walk_crate(&mut path_finder, the_crate);
path_finder.path_accumulator
}
/// A Visitor that extracts the identifiers from a thingy.
// as a side note, I'm starting to want to abstract over these....
struct IdentFinder {
ident_accumulator: Vec<ast::Ident>
}
impl Visitor for IdentFinder {
fn visit_ident(&mut self, _: syntax_pos::Span, id: ast::Ident){
self.ident_accumulator.push(id);
}
}
/// Find the idents in a crate
fn crate_idents(the_crate: &ast::Crate) -> Vec<ast::Ident> {
let mut ident_finder = IdentFinder{ident_accumulator: Vec::new()};
visit::walk_crate(&mut ident_finder, the_crate);
ident_finder.ident_accumulator
}
// these following tests are quite fragile, in that they don't test what
// *kind* of failure occurs.
@ -1264,13 +836,6 @@ mod tests {
expand_crate(ecx, vec![], crate_ast).0
}
// find the pat_ident paths in a crate
fn crate_bindings(the_crate : &ast::Crate) -> Vec<ast::Ident> {
let mut name_finder = PatIdentFinder{ident_accumulator:Vec::new()};
visit::walk_crate(&mut name_finder, the_crate);
name_finder.ident_accumulator
}
#[test] fn macro_tokens_should_match(){
expand_crate_str(
"macro_rules! m((a)=>(13)) ;fn main(){m!(a);}".to_string());
@ -1287,93 +852,4 @@ mod tests {
// create a really evil test case where a $x appears inside a binding of $x
// but *shouldn't* bind because it was inserted by a different macro....
// can't write this test case until we have macro-generating macros.
#[test]
fn fmt_in_macro_used_inside_module_macro() {
let crate_str = "macro_rules! fmt_wrap(($b:expr)=>($b.to_string()));
macro_rules! foo_module (() => (mod generated { fn a() { let xx = 147; fmt_wrap!(xx);}}));
foo_module!();
".to_string();
let cr = expand_crate_str(crate_str);
// find the xx binding
let bindings = crate_bindings(&cr);
let cxbinds: Vec<&ast::Ident> =
bindings.iter().filter(|b| b.name.as_str() == "xx").collect();
let cxbinds: &[&ast::Ident] = &cxbinds[..];
let cxbind = match (cxbinds.len(), cxbinds.get(0)) {
(1, Some(b)) => *b,
_ => panic!("expected just one binding for ext_cx")
};
let resolved_binding = mtwt::resolve(*cxbind);
let varrefs = crate_varrefs(&cr);
// the xx binding should bind all of the xx varrefs:
for (idx,v) in varrefs.iter().filter(|p| {
p.segments.len() == 1
&& p.segments[0].identifier.name.as_str() == "xx"
}).enumerate() {
if mtwt::resolve(v.segments[0].identifier) != resolved_binding {
println!("uh oh, xx binding didn't match xx varref:");
println!("this is xx varref \\# {}", idx);
println!("binding: {}", cxbind);
println!("resolves to: {}", resolved_binding);
println!("varref: {}", v.segments[0].identifier);
println!("resolves to: {}",
mtwt::resolve(v.segments[0].identifier));
mtwt::with_sctable(|x| mtwt::display_sctable(x));
}
assert_eq!(mtwt::resolve(v.segments[0].identifier),
resolved_binding);
};
}
#[test]
fn pat_idents(){
let pat = string_to_pat(
"(a,Foo{x:c @ (b,9),y:Bar(4,d)})".to_string());
let idents = pattern_bindings(&pat);
assert_eq!(idents, strs_to_idents(vec!("a","c","b","d")));
}
// test the list of identifier patterns gathered by the visitor. Note that
// 'None' is listed as an identifier pattern because we don't yet know that
// it's the name of a 0-ary variant, and that 'i' appears twice in succession.
#[test]
fn crate_bindings_test(){
let the_crate = string_to_crate("fn main (a: i32) -> i32 {|b| {
match 34 {None => 3, Some(i) | i => j, Foo{k:z,l:y} => \"banana\"}} }".to_string());
let idents = crate_bindings(&the_crate);
assert_eq!(idents, strs_to_idents(vec!("a","b","None","i","i","z","y")));
}
// test the IdentRenamer directly
#[test]
fn ident_renamer_test () {
let the_crate = string_to_crate("fn f(x: i32){let x = x; x}".to_string());
let f_ident = token::str_to_ident("f");
let x_ident = token::str_to_ident("x");
let int_ident = token::str_to_ident("i32");
let renames = vec!((x_ident,Name(16)));
let mut renamer = IdentRenamer{renames: &renames};
let renamed_crate = renamer.fold_crate(the_crate);
let idents = crate_idents(&renamed_crate);
let resolved : Vec<ast::Name> = idents.iter().map(|id| mtwt::resolve(*id)).collect();
assert_eq!(resolved, [f_ident.name,Name(16),int_ident.name,Name(16),Name(16),Name(16)]);
}
// test the PatIdentRenamer; only PatIdents get renamed
#[test]
fn pat_ident_renamer_test () {
let the_crate = string_to_crate("fn f(x: i32){let x = x; x}".to_string());
let f_ident = token::str_to_ident("f");
let x_ident = token::str_to_ident("x");
let int_ident = token::str_to_ident("i32");
let renames = vec!((x_ident,Name(16)));
let mut renamer = PatIdentRenamer{renames: &renames};
let renamed_crate = renamer.fold_crate(the_crate);
let idents = crate_idents(&renamed_crate);
let resolved : Vec<ast::Name> = idents.iter().map(|id| mtwt::resolve(*id)).collect();
let x_name = x_ident.name;
assert_eq!(resolved, [f_ident.name,Name(16),int_ident.name,Name(16),x_name,x_name]);
}
}

120
src/libsyntax/ext/mtwt.rs
View file

@ -17,7 +17,7 @@
pub use self::SyntaxContext_::*;
use ast::{Ident, Mrk, Name, SyntaxContext};
use ast::{Ident, Mrk, SyntaxContext};
use std::cell::RefCell;
use std::collections::HashMap;
@ -25,29 +25,19 @@ use std::collections::HashMap;
/// The SCTable contains a table of SyntaxContext_'s. It
/// represents a flattened tree structure, to avoid having
/// managed pointers everywhere (that caused an ICE).
/// the `marks` and `renames` fields are side-tables
/// that ensure that adding the same mark to the same context
/// gives you back the same context as before. This should cut
/// down on memory use *a lot*; applying a mark to a tree containing
/// 50 identifiers would otherwise generate 50 new contexts.
/// The `marks` ensures that adding the same mark to the
/// same context gives you back the same context as before.
pub struct SCTable {
table: RefCell<Vec<SyntaxContext_>>,
marks: RefCell<HashMap<(SyntaxContext,Mrk),SyntaxContext>>,
renames: RefCell<HashMap<Name,SyntaxContext>>,
}
#[derive(PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy, Clone)]
pub enum SyntaxContext_ {
EmptyCtxt,
Mark (Mrk,SyntaxContext),
Rename (Name),
/// actually, IllegalCtxt may not be necessary.
IllegalCtxt
}
/// A list of ident->name renamings
pub type RenameList = Vec<(Ident, Name)>;
/// Extend a syntax context with a given mark
pub fn apply_mark(m: Mrk, ctxt: SyntaxContext) -> SyntaxContext {
with_sctable(|table| apply_mark_internal(m, ctxt, table))
@ -65,32 +55,6 @@ fn apply_mark_internal(m: Mrk, ctxt: SyntaxContext, table: &SCTable) -> SyntaxCo
}
}
/// Extend a syntax context with a given rename
pub fn apply_rename(from: Ident, to: Name, ident: Ident) -> Ident {
with_sctable(|table| apply_rename_internal(from, to, ident, table))
}
/// Extend a syntax context with a given rename and sctable (explicit memoization)
fn apply_rename_internal(from: Ident, to: Name, ident: Ident, table: &SCTable) -> Ident {
if (ident.name, ident.ctxt) != (from.name, from.ctxt) {
return ident;
}
let ctxt = *table.renames.borrow_mut().entry(to).or_insert_with(|| {
SyntaxContext(idx_push(&mut *table.table.borrow_mut(), Rename(to)))
});
Ident { ctxt: ctxt, ..ident }
}
/// Apply a list of renamings to a context
// if these rename lists get long, it would make sense
// to consider memoizing this fold. This may come up
// when we add hygiene to item names.
pub fn apply_renames(renames: &RenameList, ident: Ident) -> Ident {
renames.iter().fold(ident, |ident, &(from, to)| {
apply_rename(from, to, ident)
})
}
/// Fetch the SCTable from TLS, create one if it doesn't yet exist.
pub fn with_sctable<T, F>(op: F) -> T where
F: FnOnce(&SCTable) -> T,
@ -99,21 +63,11 @@ pub fn with_sctable<T, F>(op: F) -> T where
SCTABLE_KEY.with(move |slot| op(slot))
}
// Make a fresh syntax context table with EmptyCtxt in slot zero
// and IllegalCtxt in slot one.
// Make a fresh syntax context table with EmptyCtxt in slot zero.
fn new_sctable_internal() -> SCTable {
SCTable {
table: RefCell::new(vec!(EmptyCtxt, IllegalCtxt)),
table: RefCell::new(vec![EmptyCtxt]),
marks: RefCell::new(HashMap::new()),
renames: RefCell::new(HashMap::new()),
}
}
/// Print out an SCTable for debugging
pub fn display_sctable(table: &SCTable) {
error!("SC table:");
for (idx,val) in table.table.borrow().iter().enumerate() {
error!("{:4} : {:?}",idx,val);
}
}
@ -122,16 +76,14 @@ pub fn clear_tables() {
with_sctable(|table| {
*table.table.borrow_mut() = Vec::new();
*table.marks.borrow_mut() = HashMap::new();
*table.renames.borrow_mut() = HashMap::new();
});
}
/// Reset the tables to their initial state
pub fn reset_tables() {
with_sctable(|table| {
*table.table.borrow_mut() = vec!(EmptyCtxt, IllegalCtxt);
*table.table.borrow_mut() = vec![EmptyCtxt];
*table.marks.borrow_mut() = HashMap::new();
*table.renames.borrow_mut() = HashMap::new();
});
}
@ -141,25 +93,6 @@ fn idx_push<T>(vec: &mut Vec<T>, val: T) -> u32 {
(vec.len() - 1) as u32
}
/// Resolve a syntax object to a name, per MTWT.
pub fn resolve(id: Ident) -> Name {
with_sctable(|sctable| {
resolve_internal(id, sctable)
})
}
/// Resolve a syntax object to a name, per MTWT.
/// adding memoization to resolve 500+ seconds in resolve for librustc (!)
fn resolve_internal(id: Ident, table: &SCTable) -> Name {
match table.table.borrow()[id.ctxt.0 as usize] {
EmptyCtxt => id.name,
// ignore marks here:
Mark(_, subctxt) => resolve_internal(Ident::new(id.name, subctxt), table),
Rename(name) => name,
IllegalCtxt => panic!("expected resolvable context, got IllegalCtxt")
}
}
/// Return the outer mark for a context with a mark at the outside.
/// FAILS when outside is not a mark.
pub fn outer_mark(ctxt: SyntaxContext) -> Mrk {
@ -171,15 +104,24 @@ pub fn outer_mark(ctxt: SyntaxContext) -> Mrk {
})
}
/// If `ident` is macro expanded, return the source ident from the macro definition
/// and the mark of the expansion that created the macro definition.
pub fn source(ident: Ident) -> Option<(Ident /* source ident */, Mrk /* source macro */)> {
with_sctable(|sctable| {
let ctxts = sctable.table.borrow();
if let Mark(_expansion_mark, macro_ctxt) = ctxts[ident.ctxt.0 as usize] {
if let Mark(definition_mark, orig_ctxt) = ctxts[macro_ctxt.0 as usize] {
return Some((Ident::new(ident.name, orig_ctxt), definition_mark));
}
}
None
})
}
#[cfg(test)]
mod tests {
use ast::{EMPTY_CTXT, Ident, Mrk, Name, SyntaxContext};
use super::{resolve, apply_mark_internal, new_sctable_internal};
use super::{SCTable, Mark};
fn id(n: u32, s: SyntaxContext) -> Ident {
Ident::new(Name(n), s)
}
use ast::{EMPTY_CTXT, Mrk, SyntaxContext};
use super::{apply_mark_internal, new_sctable_internal, Mark, SCTable};
// extend a syntax context with a sequence of marks given
// in a vector. v[0] will be the outermost mark.
@ -192,27 +134,21 @@ mod tests {
#[test] fn unfold_marks_test() {
let mut t = new_sctable_internal();
assert_eq!(unfold_marks(vec!(3,7),EMPTY_CTXT,&mut t),SyntaxContext(3));
assert_eq!(unfold_marks(vec!(3,7),EMPTY_CTXT,&mut t),SyntaxContext(2));
{
let table = t.table.borrow();
assert!((*table)[2] == Mark(7,EMPTY_CTXT));
assert!((*table)[3] == Mark(3,SyntaxContext(2)));
assert!((*table)[1] == Mark(7,EMPTY_CTXT));
assert!((*table)[2] == Mark(3,SyntaxContext(1)));
}
}
#[test]
fn mtwt_resolve_test(){
let a = 40;
assert_eq!(resolve(id(a,EMPTY_CTXT)),Name(a));
}
#[test]
fn hashing_tests () {
let mut t = new_sctable_internal();
assert_eq!(apply_mark_internal(12,EMPTY_CTXT,&mut t),SyntaxContext(2));
assert_eq!(apply_mark_internal(13,EMPTY_CTXT,&mut t),SyntaxContext(3));
assert_eq!(apply_mark_internal(12,EMPTY_CTXT,&mut t),SyntaxContext(1));
assert_eq!(apply_mark_internal(13,EMPTY_CTXT,&mut t),SyntaxContext(2));
// using the same one again should result in the same index:
assert_eq!(apply_mark_internal(12,EMPTY_CTXT,&mut t),SyntaxContext(2));
assert_eq!(apply_mark_internal(12,EMPTY_CTXT,&mut t),SyntaxContext(1));
// I'm assuming that the rename table will behave the same....
}
}

30
src/libsyntax/parse/token.rs
View file

@ -15,7 +15,6 @@ pub use self::Lit::*;
pub use self::Token::*;
use ast::{self, BinOpKind};
use ext::mtwt;
use ptr::P;
use util::interner::Interner;
use tokenstream;
@ -313,17 +312,6 @@ impl Token {
_ => false,
}
}
/// Hygienic identifier equality comparison.
///
/// See `styntax::ext::mtwt`.
pub fn mtwt_eq(&self, other : &Token) -> bool {
match (self, other) {
(&Ident(id1), &Ident(id2)) | (&Lifetime(id1), &Lifetime(id2)) =>
mtwt::resolve(id1) == mtwt::resolve(id2),
_ => *self == *other
}
}
}
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash)]
@ -650,21 +638,3 @@ pub fn fresh_name(src: ast::Ident) -> ast::Name {
pub fn fresh_mark() -> ast::Mrk {
gensym("mark").0
}
#[cfg(test)]
mod tests {
use super::*;
use ast;
use ext::mtwt;
fn mark_ident(id : ast::Ident, m : ast::Mrk) -> ast::Ident {
ast::Ident::new(id.name, mtwt::apply_mark(m, id.ctxt))
}
#[test] fn mtwt_token_eq_test() {
assert!(Gt.mtwt_eq(&Gt));
let a = str_to_ident("bac");
let a1 = mark_ident(a,92);
assert!(Ident(a).mtwt_eq(&Ident(a1)));
}
}

4
src/libsyntax/test.rs
View file

@ -185,6 +185,8 @@ impl<'a> fold::Folder for TestHarnessGenerator<'a> {
mod_folded
}
fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac { mac }
}
struct EntryPointCleaner {
@ -234,6 +236,8 @@ impl fold::Folder for EntryPointCleaner {
SmallVector::one(folded)
}
fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac { mac }
}
fn mk_reexport_mod(cx: &mut TestCtxt, tests: Vec<ast::Ident>,

7
src/test/run-pass/hygiene.rs
View file

@ -106,6 +106,13 @@ fn match_hygiene() {
m!(Ok(x), x);
}
fn label_hygiene() {
'a: loop {
macro_rules! m { () => { break 'a; } }
m!();
}
}
fn main() {
f();
g();