Auto merge of #31279 - DanielJCampbell:MacroReferencing, r=nrc

r? @nrc

src/librustc/session/mod.rs

@@ -30,7 +30,7 @@ use rustc_back::target::Target;
 
 use std::path::{Path, PathBuf};
 use std::cell::{Cell, RefCell};
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
 use std::env;
 use std::rc::Rc;
 
@@ -77,6 +77,11 @@ pub struct Session {
     /// available in this crate
     pub available_macros: RefCell<HashSet<Name>>,
 
+    /// Map from imported macro spans (which consist of
+    /// the localized span for the macro body) to the
+    /// macro name and defintion span in the source crate.
+    pub imported_macro_spans: RefCell<HashMap<Span, (String, Span)>>,
+
     next_node_id: Cell<ast::NodeId>,
 }
 
@@ -479,6 +484,7 @@ pub fn build_session_(sopts: config::Options,
         next_node_id: Cell::new(1),
         injected_allocator: Cell::new(None),
         available_macros: RefCell::new(HashSet::new()),
+        imported_macro_spans: RefCell::new(HashMap::new()),
     };
 
     sess

src/librustc_metadata/common.rs

@@ -223,6 +223,8 @@ pub const tag_polarity: usize = 0x9d;
 pub const tag_macro_defs: usize = 0x10e; // top-level only
 pub const tag_macro_def: usize = 0x9e;
 pub const tag_macro_def_body: usize = 0x9f;
+pub const tag_macro_def_span_lo: usize = 0xa8;
+pub const tag_macro_def_span_hi: usize = 0xa9;
 
 pub const tag_paren_sugar: usize = 0xa0;
 

src/librustc_metadata/creader.rs

@@ -494,7 +494,7 @@ impl<'a> CrateReader<'a> {
         let mut macros = vec![];
         decoder::each_exported_macro(ekrate.metadata.as_slice(),
                                      &*self.cstore.intr,
-            |name, attrs, body| {
+            |name, attrs, span, body| {
                 // NB: Don't use parse::parse_tts_from_source_str because it parses with
                 // quote_depth > 0.
                 let mut p = parse::new_parser_from_source_str(&self.sess.parse_sess,
@@ -509,7 +509,7 @@ impl<'a> CrateReader<'a> {
                         panic!(FatalError);
                     }
                 };
-                let span = mk_sp(lo, p.last_span.hi);
+                let local_span = mk_sp(lo, p.last_span.hi);
 
                 // Mark the attrs as used
                 for attr in &attrs {
@@ -520,7 +520,7 @@ impl<'a> CrateReader<'a> {
                     ident: ast::Ident::with_empty_ctxt(name),
                     attrs: attrs,
                     id: ast::DUMMY_NODE_ID,
-                    span: span,
+                    span: local_span,
                     imported_from: Some(item.ident),
                     // overridden in plugin/load.rs
                     export: false,
@@ -529,6 +529,8 @@ impl<'a> CrateReader<'a> {
 
                     body: body,
                 });
+                self.sess.imported_macro_spans.borrow_mut()
+                    .insert(local_span, (name.as_str().to_string(), span));
                 true
             }
         );

src/librustc_metadata/decoder.rs

@@ -52,7 +52,7 @@ use syntax::parse::token::{IdentInterner, special_idents};
 use syntax::parse::token;
 use syntax::ast;
 use syntax::abi;
-use syntax::codemap::{self, Span};
+use syntax::codemap::{self, Span, BytePos, NO_EXPANSION};
 use syntax::print::pprust;
 use syntax::ptr::P;
 
@@ -1471,19 +1471,28 @@ pub fn get_plugin_registrar_fn(data: &[u8]) -> Option<DefIndex> {
 }
 
 pub fn each_exported_macro<F>(data: &[u8], intr: &IdentInterner, mut f: F) where
-    F: FnMut(ast::Name, Vec<ast::Attribute>, String) -> bool,
+    F: FnMut(ast::Name, Vec<ast::Attribute>, Span, String) -> bool,
 {
     let macros = reader::get_doc(rbml::Doc::new(data), tag_macro_defs);
     for macro_doc in reader::tagged_docs(macros, tag_macro_def) {
         let name = item_name(intr, macro_doc);
         let attrs = get_attributes(macro_doc);
+        let span = get_macro_span(macro_doc);
         let body = reader::get_doc(macro_doc, tag_macro_def_body);
-        if !f(name, attrs, body.as_str().to_string()) {
+        if !f(name, attrs, span, body.as_str().to_string()) {
             break;
         }
     }
 }
 
+pub fn get_macro_span(doc: rbml::Doc) -> Span {
+    let lo_doc = reader::get_doc(doc, tag_macro_def_span_lo);
+    let lo = BytePos(reader::doc_as_u32(lo_doc));
+    let hi_doc = reader::get_doc(doc, tag_macro_def_span_hi);
+    let hi = BytePos(reader::doc_as_u32(hi_doc));
+    return Span { lo: lo, hi: hi, expn_id: NO_EXPANSION };
+}
+
 pub fn get_dylib_dependency_formats(cdata: Cmd)
     -> Vec<(ast::CrateNum, LinkagePreference)>
 {

src/librustc_metadata/encoder.rs

@@ -42,6 +42,7 @@ use std::rc::Rc;
 use std::u32;
 use syntax::abi;
 use syntax::ast::{self, NodeId, Name, CRATE_NODE_ID, CrateNum};
+use syntax::codemap::BytePos;
 use syntax::attr;
 use syntax::attr::AttrMetaMethods;
 use syntax::errors::Handler;
@@ -1727,6 +1728,10 @@ fn encode_macro_defs(rbml_w: &mut Encoder,
 
         encode_name(rbml_w, def.name);
         encode_attributes(rbml_w, &def.attrs);
+        let &BytePos(lo) = &def.span.lo;
+        let &BytePos(hi) = &def.span.hi;
+        rbml_w.wr_tagged_u32(tag_macro_def_span_lo, lo);
+        rbml_w.wr_tagged_u32(tag_macro_def_span_hi, hi);
 
         rbml_w.wr_tagged_str(tag_macro_def_body,
                              &::syntax::print::pprust::tts_to_string(&def.body));

src/librustc_trans/save/dump_csv.rs

@@ -37,6 +37,8 @@ use middle::def_id::DefId;
 use middle::ty;
 
 use std::fs::File;
+use std::hash::*;
+use std::collections::HashSet;
 
 use syntax::ast::{self, NodeId};
 use syntax::codemap::*;
@@ -70,6 +72,14 @@ pub struct DumpCsvVisitor<'l, 'tcx: 'l> {
     fmt: FmtStrs<'l, 'tcx>,
 
     cur_scope: NodeId,
+
+    // Set of macro definition (callee) spans, and the set
+    // of macro use (callsite) spans. We store these to ensure
+    // we only write one macro def per unique macro definition, and
+    // one macro use per unique callsite span.
+    mac_defs: HashSet<Span>,
+    mac_uses: HashSet<Span>,
+
 }
 
 impl <'l, 'tcx> DumpCsvVisitor<'l, 'tcx> {
@@ -92,6 +102,8 @@ impl <'l, 'tcx> DumpCsvVisitor<'l, 'tcx> {
                               span_utils,
                               tcx),
             cur_scope: 0,
+            mac_defs: HashSet::new(),
+            mac_uses: HashSet::new(),
         }
     }
 
@@ -814,10 +826,41 @@ impl <'l, 'tcx> DumpCsvVisitor<'l, 'tcx> {
                                   &typ);
         }
     }
+
+    /// Extract macro use and definition information from the AST node defined
+    /// by the given NodeId, using the expansion information from the node's
+    /// span.
+    ///
+    /// If the span is not macro-generated, do nothing, else use callee and
+    /// callsite spans to record macro definition and use data, using the
+    /// mac_uses and mac_defs sets to prevent multiples.
+    fn process_macro_use(&mut self, span: Span, id: NodeId) {
+        let data = match self.save_ctxt.get_macro_use_data(span, id) {
+            None => return,
+            Some(data) => data,
+        };
+        let mut hasher = SipHasher::new();
+        data.callee_span.hash(&mut hasher);
+        let hash = hasher.finish();
+        let qualname = format!("{}::{}", data.name, hash);
+        // Don't write macro definition for imported macros
+        if !self.mac_defs.contains(&data.callee_span)
+            && !data.imported {
+            self.mac_defs.insert(data.callee_span);
+            self.fmt.macro_str(data.callee_span, data.callee_span,
+                               data.name.clone(), qualname.clone());
+        }
+        if !self.mac_uses.contains(&data.span) {
+             self.mac_uses.insert(data.span);
+             self.fmt.macro_use_str(data.span, data.span, data.name,
+                                   qualname, data.scope);
+        }
+    }
 }
 
 impl<'l, 'tcx, 'v> Visitor<'v> for DumpCsvVisitor<'l, 'tcx> {
     fn visit_item(&mut self, item: &ast::Item) {
+        self.process_macro_use(item.span, item.id);
         match item.node {
             ast::ItemUse(ref use_item) => {
                 match use_item.node {
@@ -970,6 +1013,7 @@ impl<'l, 'tcx, 'v> Visitor<'v> for DumpCsvVisitor<'l, 'tcx> {
     }
 
     fn visit_trait_item(&mut self, trait_item: &ast::TraitItem) {
+        self.process_macro_use(trait_item.span, trait_item.id);
         match trait_item.node {
             ast::ConstTraitItem(ref ty, Some(ref expr)) => {
                 self.process_const(trait_item.id,
@@ -991,6 +1035,7 @@ impl<'l, 'tcx, 'v> Visitor<'v> for DumpCsvVisitor<'l, 'tcx> {
     }
 
     fn visit_impl_item(&mut self, impl_item: &ast::ImplItem) {
+        self.process_macro_use(impl_item.span, impl_item.id);
         match impl_item.node {
             ast::ImplItemKind::Const(ref ty, ref expr) => {
                 self.process_const(impl_item.id,
@@ -1012,6 +1057,7 @@ impl<'l, 'tcx, 'v> Visitor<'v> for DumpCsvVisitor<'l, 'tcx> {
     }
 
     fn visit_ty(&mut self, t: &ast::Ty) {
+        self.process_macro_use(t.span, t.id);
         match t.node {
             ast::TyPath(_, ref path) => {
                 match self.lookup_type_ref(t.id) {
@@ -1031,6 +1077,7 @@ impl<'l, 'tcx, 'v> Visitor<'v> for DumpCsvVisitor<'l, 'tcx> {
     }
 
     fn visit_expr(&mut self, ex: &ast::Expr) {
+        self.process_macro_use(ex.span, ex.id);
         match ex.node {
             ast::ExprCall(ref _f, ref _args) => {
                 // Don't need to do anything for function calls,
@@ -1117,11 +1164,13 @@ impl<'l, 'tcx, 'v> Visitor<'v> for DumpCsvVisitor<'l, 'tcx> {
         }
     }
 
-    fn visit_mac(&mut self, _: &ast::Mac) {
+    fn visit_mac(&mut self, mac: &ast::Mac) {
-        // Just stop, macros are poison to us.
+        // These shouldn't exist in the AST at this point, log a span bug.
+        self.sess.span_bug(mac.span, "macro invocation should have been expanded out of AST");
     }
 
     fn visit_pat(&mut self, p: &ast::Pat) {
+        self.process_macro_use(p.span, p.id);
         self.process_pat(p);
     }
 
@@ -1177,10 +1226,13 @@ impl<'l, 'tcx, 'v> Visitor<'v> for DumpCsvVisitor<'l, 'tcx> {
     }
 
     fn visit_stmt(&mut self, s: &ast::Stmt) {
+        let id = s.node.id();
+        self.process_macro_use(s.span, id.unwrap());
         visit::walk_stmt(self, s)
     }
 
     fn visit_local(&mut self, l: &ast::Local) {
+        self.process_macro_use(l.span, l.id);
         let value = self.span.snippet(l.span);
         self.process_var_decl(&l.pat, value);
 

src/librustc_trans/save/mod.rs

@@ -73,6 +73,8 @@ pub enum Data {
     FunctionCallData(FunctionCallData),
     /// Data about a method call.
     MethodCallData(MethodCallData),
+    /// Data about a macro use.
+    MacroUseData(MacroUseData),
 }
 
 /// Data for all kinds of functions and methods.
@@ -174,6 +176,22 @@ pub struct MethodCallData {
     pub decl_id: Option<DefId>,
 }
 
+/// Data about a macro use.
+#[derive(Debug)]
+pub struct MacroUseData {
+    pub span: Span,
+    pub name: String,
+    // Because macro expansion happens before ref-ids are determined,
+    // we use the callee span to reference the associated macro definition.
+    pub callee_span: Span,
+    pub scope: NodeId,
+    pub imported: bool,
+}
+
+macro_rules! option_try(
+    ($e:expr) => (match $e { Some(e) => e, None => return None })
+);
+
 
 
 impl<'l, 'tcx: 'l> SaveContext<'l, 'tcx> {
@@ -655,6 +673,51 @@ impl<'l, 'tcx: 'l> SaveContext<'l, 'tcx> {
         })
     }
 
+    /// Attempt to return MacroUseData for any AST node.
+    ///
+    /// For a given piece of AST defined by the supplied Span and NodeId,
+    /// returns None if the node is not macro-generated or the span is malformed,
+    /// else uses the expansion callsite and callee to return some MacroUseData.
+    pub fn get_macro_use_data(&self, span: Span, id: NodeId) -> Option<MacroUseData> {
+        if !generated_code(span) {
+            return None;
+        }
+        // Note we take care to use the source callsite/callee, to handle
+        // nested expansions and ensure we only generate data for source-visible
+        // macro uses.
+        let callsite = self.tcx.sess.codemap().source_callsite(span);
+        let callee = self.tcx.sess.codemap().source_callee(span);
+        let callee = option_try!(callee);
+        let callee_span = option_try!(callee.span);
+
+        // Ignore attribute macros, their spans are usually mangled
+        if let MacroAttribute(_) = callee.format {
+            return None;
+        }
+
+        // If the callee is an imported macro from an external crate, need to get
+        // the source span and name from the session, as their spans are localized
+        // when read in, and no longer correspond to the source.
+        if let Some(mac) = self.tcx.sess.imported_macro_spans.borrow().get(&callee_span) {
+            let &(ref mac_name, mac_span) = mac;
+            return Some(MacroUseData {
+                                        span: callsite,
+                                        name: mac_name.clone(),
+                                        callee_span: mac_span,
+                                        scope: self.enclosing_scope(id),
+                                        imported: true,
+                                    });
+        }
+
+        Some(MacroUseData {
+            span: callsite,
+            name: callee.name().to_string(),
+            callee_span: callee_span,
+            scope: self.enclosing_scope(id),
+            imported: false,
+        })
+    }
+
     pub fn get_data_for_id(&self, _id: &NodeId) -> Data {
         // FIXME
         unimplemented!();

src/librustc_trans/save/recorder.rs

@@ -96,6 +96,8 @@ pub enum Row {
     VarRef,
     TypeRef,
     FnRef,
+    Macro,
+    MacroUse,
 }
 
 impl<'a, 'tcx: 'a> FmtStrs<'a, 'tcx> {
@@ -219,6 +221,14 @@ impl<'a, 'tcx: 'a> FmtStrs<'a, 'tcx> {
                       vec!("refid", "refidcrate", "qualname", "scopeid"),
                       true,
                       true),
+            Macro => ("macro",
+                         vec!("name", "qualname"),
+                         true,
+                         true),
+            MacroUse => ("macro_use",
+                         vec!("callee_name", "qualname", "scopeid"),
+                         true,
+                         true),
         }
     }
 
@@ -686,4 +696,19 @@ impl<'a, 'tcx: 'a> FmtStrs<'a, 'tcx> {
                               sub_span,
                               svec!(id.index.as_usize(), id.krate, "", scope_id));
     }
+
+    pub fn macro_str(&mut self, span: Span, sub_span: Span, name: String, qualname: String) {
+        self.record_with_span(Macro, span, sub_span, svec!(name, qualname));
+    }
+
+    pub fn macro_use_str(&mut self,
+                         span: Span,
+                         sub_span: Span,
+                         name: String,
+                         qualname: String,
+                         scope_id: NodeId) {
+        let scope_id = self.normalize_node_id(scope_id);
+        self.record_with_span(MacroUse, span, sub_span,
+                              svec!(name, qualname, scope_id));
+    }
 }

src/librustc_trans/save/span_utils.rs

@@ -378,6 +378,25 @@ impl<'a> SpanUtils<'a> {
         }
     }
 
+    // Given a macro_rules definition span, return the span of the macro's name.
+    pub fn span_for_macro_name(&self, span: Span) -> Option<Span> {
+        let mut toks = self.retokenise_span(span);
+        loop {
+            let ts = toks.real_token();
+            if ts.tok == token::Eof {
+                return None;
+            }
+            if ts.tok == token::Not {
+                let ts = toks.real_token();
+                if ts.tok.is_ident() {
+                    return self.make_sub_span(span, Some(ts.sp));
+                } else {
+                    return None;
+                }
+            }
+        }
+    }
+
     /// Return true if the span is generated code, and
     /// it is not a subspan of the root callsite.
     ///
@@ -395,10 +414,16 @@ impl<'a> SpanUtils<'a> {
         if sub_span.is_none() {
             return true;
         }
-        // A generated span is deemed invalid if it is not a sub-span of the root
+
+        //If the span comes from a fake filemap, filter it.
+        if !self.sess.codemap().lookup_char_pos(parent.lo).file.is_real_file() {
+            return true;
+        }
+
+        // Otherwise, a generated span is deemed invalid if it is not a sub-span of the root
         // callsite. This filters out macro internal variables and most malformed spans.
         let span = self.sess.codemap().source_callsite(parent);
-        !(parent.lo >= span.lo && parent.hi <= span.hi)
+        !(span.contains(parent))
     }
 }
 

src/libsyntax/codemap.rs

@@ -1064,6 +1064,27 @@ impl CodeMap {
         span
     }
 
+    /// Return the source callee.
+    ///
+    /// Returns None if the supplied span has no expansion trace,
+    /// else returns the NameAndSpan for the macro definition
+    /// corresponding to the source callsite.
+    pub fn source_callee(&self, sp: Span) -> Option<NameAndSpan> {
+        let mut span = sp;
+        while let Some(callsite) = self.with_expn_info(span.expn_id,
+                                            |ei| ei.map(|ei| ei.call_site.clone())) {
+            if let Some(_) = self.with_expn_info(callsite.expn_id,
+                                                |ei| ei.map(|ei| ei.call_site.clone())) {
+                span = callsite;
+            }
+            else {
+                return self.with_expn_info(span.expn_id,
+                                           |ei| ei.map(|ei| ei.callee.clone()));
+            }
+        }
+        None
+    }
+
     pub fn span_to_filename(&self, sp: Span) -> FileName {
         self.lookup_char_pos(sp.lo).file.name.to_string()
     }