Rollup merge of #38842 - abonander:proc_macro_attribute, r=jseyfried

Implement `#[proc_macro_attribute]`

This implements `#[proc_macro_attribute]` as described in https://github.com/rust-lang/rfcs/pull/1566

The following major (hopefully non-breaking) changes are included:

* Refactor `proc_macro::TokenStream` to use `syntax::tokenstream::TokenStream`.
    * `proc_macro::tokenstream::TokenStream` no longer emits newlines between items, this can be trivially restored if desired
    * `proc_macro::TokenStream::from_str` does not try to parse an item anymore, moved to `impl MultiItemModifier for CustomDerive` with more informative error message

* Implement `#[proc_macro_attribute]`, which expects functions of the kind `fn(TokenStream, TokenStream) -> TokenStream`
    * Reactivated `#![feature(proc_macro)]` and gated `#[proc_macro_attribute]` under it
    * `#![feature(proc_macro)]` and `#![feature(custom_attribute)]` are mutually exclusive
    * adding `#![feature(proc_macro)]` makes the expansion pass assume that any attributes that are not built-in, or introduced by existing syntax extensions, are proc-macro attributes

* Fix `feature_gate::find_lang_feature_issue()` to not use `unwrap()`

    * This change wasn't necessary for this PR, but it helped debugging a problem where I was using the wrong feature string.

* Move "completed feature gate checking" pass to after "name resolution" pass

    * This was necessary for proper feature-gating of `#[proc_macro_attribute]` invocations when the `proc_macro` feature flag isn't set.

Prototype/Litmus Test: [Implementation](https://github.com/abonander/anterofit/blob/proc_macro/service-attr/src/lib.rs#L13) -- [Usage](https://github.com/abonander/anterofit/blob/proc_macro/service-attr/examples/post_service.rs#L35)

src/libproc_macro/lib.rs

@@ -39,9 +39,9 @@ extern crate syntax;
 use std::fmt;
 use std::str::FromStr;
 
-use syntax::ast;
+use syntax::errors::DiagnosticBuilder;
 use syntax::parse;
-use syntax::ptr::P;
+use syntax::tokenstream::TokenStream as TokenStream_;
 
 /// The main type provided by this crate, representing an abstract stream of
 /// tokens.
@@ -54,7 +54,7 @@ use syntax::ptr::P;
 /// time!
 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
 pub struct TokenStream {
-    inner: Vec<P<ast::Item>>,
+    inner: TokenStream_,
 }
 
 /// Error returned from `TokenStream::from_str`.
@@ -77,17 +77,41 @@ pub struct LexError {
 #[doc(hidden)]
 pub mod __internal {
     use std::cell::Cell;
+    use std::rc::Rc;
 
     use syntax::ast;
     use syntax::ptr::P;
-    use syntax::parse::ParseSess;
+    use syntax::parse::{self, token, ParseSess};
-    use super::TokenStream;
+    use syntax::tokenstream::TokenStream as TokenStream_;
+
+    use super::{TokenStream, LexError};
 
     pub fn new_token_stream(item: P<ast::Item>) -> TokenStream {
-        TokenStream { inner: vec![item] }
+        TokenStream { inner: TokenStream_::from_tokens(vec![
+            token::Interpolated(Rc::new(token::NtItem(item)))
+        ])}
     }
 
-    pub fn token_stream_items(stream: TokenStream) -> Vec<P<ast::Item>> {
+    pub fn token_stream_wrap(inner: TokenStream_) -> TokenStream {
+        TokenStream {
+            inner: inner
+        }
+    }
+
+    pub fn token_stream_parse_items(stream: TokenStream) -> Result<Vec<P<ast::Item>>, LexError> {
+        with_parse_sess(move |sess| {
+            let mut parser = parse::new_parser_from_ts(sess, stream.inner);
+            let mut items = Vec::new();
+
+            while let Some(item) = try!(parser.parse_item().map_err(super::parse_to_lex_err)) {
+                items.push(item)
+            }
+
+            Ok(items)
+        })
+    }
+
+    pub fn token_stream_inner(stream: TokenStream) -> TokenStream_ {
         stream.inner
     }
 
@@ -96,6 +120,10 @@ pub mod __internal {
                                   trait_name: &str,
                                   expand: fn(TokenStream) -> TokenStream,
                                   attributes: &[&'static str]);
+
+        fn register_attr_proc_macro(&mut self,
+                                    name: &str,
+                                    expand: fn(TokenStream, TokenStream) -> TokenStream);
     }
 
     // Emulate scoped_thread_local!() here essentially
@@ -125,11 +153,17 @@ pub mod __internal {
         where F: FnOnce(&ParseSess) -> R
     {
         let p = CURRENT_SESS.with(|p| p.get());
-        assert!(!p.is_null());
+        assert!(!p.is_null(), "proc_macro::__internal::with_parse_sess() called \
+                               before set_parse_sess()!");
         f(unsafe { &*p })
     }
 }
 
+fn parse_to_lex_err(mut err: DiagnosticBuilder) -> LexError {
+    err.cancel();
+    LexError { _inner: () }
+}
+
 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
 impl FromStr for TokenStream {
     type Err = LexError;
@@ -138,18 +172,10 @@ impl FromStr for TokenStream {
         __internal::with_parse_sess(|sess| {
             let src = src.to_string();
             let name = "<proc-macro source code>".to_string();
-            let mut parser = parse::new_parser_from_source_str(sess, name, src);
+            let tts = try!(parse::parse_tts_from_source_str(name, src, sess)
-            let mut ret = TokenStream { inner: Vec::new() };
+                .map_err(parse_to_lex_err));
-            loop {
+
-                match parser.parse_item() {
+            Ok(__internal::token_stream_wrap(TokenStream_::from_tts(tts)))
-                    Ok(Some(item)) => ret.inner.push(item),
-                    Ok(None) => return Ok(ret),
-                    Err(mut err) => {
-                        err.cancel();
-                        return Err(LexError { _inner: () })
-                    }
-                }
-            }
         })
     }
 }
@@ -157,11 +183,6 @@ impl FromStr for TokenStream {
 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
 impl fmt::Display for TokenStream {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        for item in self.inner.iter() {
+        self.inner.fmt(f)
-            let item = syntax::print::pprust::item_to_string(item);
-            try!(f.write_str(&item));
-            try!(f.write_str("\n"));
-        }
-        Ok(())
     }
 }

src/librustc_driver/driver.rs

@@ -681,6 +681,7 @@ pub fn phase_2_configure_and_expand<F>(sess: &Session,
             should_test: sess.opts.test,
             ..syntax::ext::expand::ExpansionConfig::default(crate_name.to_string())
         };
+
         let mut ecx = ExtCtxt::new(&sess.parse_sess, cfg, &mut resolver);
         let err_count = ecx.parse_sess.span_diagnostic.err_count();
 
@@ -740,17 +741,6 @@ pub fn phase_2_configure_and_expand<F>(sess: &Session,
          "checking for inline asm in case the target doesn't support it",
          || no_asm::check_crate(sess, &krate));
 
-    // Needs to go *after* expansion to be able to check the results of macro expansion.
-    time(time_passes, "complete gated feature checking", || {
-        sess.track_errors(|| {
-            syntax::feature_gate::check_crate(&krate,
-                                              &sess.parse_sess,
-                                              &sess.features.borrow(),
-                                              &attributes,
-                                              sess.opts.unstable_features);
-        })
-    })?;
-
     time(sess.time_passes(),
          "early lint checks",
          || lint::check_ast_crate(sess, &krate));
@@ -768,6 +758,17 @@ pub fn phase_2_configure_and_expand<F>(sess: &Session,
         Ok(())
     })?;
 
+    // Needs to go *after* expansion to be able to check the results of macro expansion.
+    time(time_passes, "complete gated feature checking", || {
+        sess.track_errors(|| {
+            syntax::feature_gate::check_crate(&krate,
+                                              &sess.parse_sess,
+                                              &sess.features.borrow(),
+                                              &attributes,
+                                              sess.opts.unstable_features);
+        })
+    })?;
+
     // Lower ast -> hir.
     let hir_forest = time(sess.time_passes(), "lowering ast -> hir", || {
         let hir_crate = lower_crate(sess, &krate, &mut resolver);

src/librustc_metadata/creader.rs

@@ -578,6 +578,7 @@ impl<'a> CrateLoader<'a> {
         use proc_macro::__internal::Registry;
         use rustc_back::dynamic_lib::DynamicLibrary;
         use syntax_ext::deriving::custom::CustomDerive;
+        use syntax_ext::proc_macro_impl::AttrProcMacro;
 
         let path = match dylib {
             Some(dylib) => dylib,
@@ -613,6 +614,15 @@ impl<'a> CrateLoader<'a> {
                 );
                 self.0.push((Symbol::intern(trait_name), Rc::new(derive)));
             }
+
+            fn register_attr_proc_macro(&mut self,
+                                        name: &str,
+                                        expand: fn(TokenStream, TokenStream) -> TokenStream) {
+                let expand = SyntaxExtension::AttrProcMacro(
+                    Box::new(AttrProcMacro { inner: expand })
+                );
+                self.0.push((Symbol::intern(name), Rc::new(expand)));
+            }
         }
 
         let mut my_registrar = MyRegistrar(Vec::new());

src/librustc_resolve/lib.rs

@@ -61,7 +61,7 @@ use syntax::ast::{FnDecl, ForeignItem, ForeignItemKind, Generics};
 use syntax::ast::{Item, ItemKind, ImplItem, ImplItemKind};
 use syntax::ast::{Local, Mutability, Pat, PatKind, Path};
 use syntax::ast::{QSelf, TraitItemKind, TraitRef, Ty, TyKind};
-use syntax::feature_gate::{emit_feature_err, GateIssue};
+use syntax::feature_gate::{feature_err, emit_feature_err, GateIssue};
 
 use syntax_pos::{Span, DUMMY_SP, MultiSpan};
 use errors::DiagnosticBuilder;
@@ -1123,6 +1123,12 @@ pub struct Resolver<'a> {
 
     // Avoid duplicated errors for "name already defined".
     name_already_seen: FxHashMap<Name, Span>,
+
+    // If `#![feature(proc_macro)]` is set
+    proc_macro_enabled: bool,
+
+    // A set of procedural macros imported by `#[macro_use]` that have already been warned about
+    warned_proc_macros: FxHashSet<Name>,
 }
 
 pub struct ResolverArenas<'a> {
@@ -1227,6 +1233,8 @@ impl<'a> Resolver<'a> {
         invocations.insert(Mark::root(),
                            arenas.alloc_invocation_data(InvocationData::root(graph_root)));
 
+        let features = session.features.borrow();
+
         Resolver {
             session: session,
 
@@ -1284,7 +1292,9 @@ impl<'a> Resolver<'a> {
                 span: DUMMY_SP,
                 vis: ty::Visibility::Public,
             }),
-            use_extern_macros: session.features.borrow().use_extern_macros,
+
+            // `#![feature(proc_macro)]` implies `#[feature(extern_macros)]`
+            use_extern_macros: features.use_extern_macros || features.proc_macro,
 
             exported_macros: Vec::new(),
             crate_loader: crate_loader,
@@ -1296,6 +1306,8 @@ impl<'a> Resolver<'a> {
             invocations: invocations,
             name_already_seen: FxHashMap(),
             whitelisted_legacy_custom_derives: Vec::new(),
+            proc_macro_enabled: features.proc_macro,
+            warned_proc_macros: FxHashSet(),
         }
     }
 
@@ -1525,6 +1537,8 @@ impl<'a> Resolver<'a> {
 
         debug!("(resolving item) resolving {}", name);
 
+        self.check_proc_macro_attrs(&item.attrs);
+
         match item.node {
             ItemKind::Enum(_, ref generics) |
             ItemKind::Ty(_, ref generics) |
@@ -1554,6 +1568,8 @@ impl<'a> Resolver<'a> {
                         walk_list!(this, visit_ty_param_bound, bounds);
 
                         for trait_item in trait_items {
+                            this.check_proc_macro_attrs(&trait_item.attrs);
+
                             match trait_item.node {
                                 TraitItemKind::Const(_, ref default) => {
                                     // Only impose the restrictions of
@@ -1738,6 +1754,7 @@ impl<'a> Resolver<'a> {
                 this.with_self_rib(Def::SelfTy(trait_id, Some(item_def_id)), |this| {
                     this.with_current_self_type(self_type, |this| {
                         for impl_item in impl_items {
+                            this.check_proc_macro_attrs(&impl_item.attrs);
                             this.resolve_visibility(&impl_item.vis);
                             match impl_item.node {
                                 ImplItemKind::Const(..) => {
@@ -3184,6 +3201,31 @@ impl<'a> Resolver<'a> {
         let msg = "`self` no longer imports values".to_string();
         self.session.add_lint(lint::builtin::LEGACY_IMPORTS, id, span, msg);
     }
+
+    fn check_proc_macro_attrs(&mut self, attrs: &[ast::Attribute]) {
+        if self.proc_macro_enabled { return; }
+
+        for attr in attrs {
+            let maybe_binding = self.builtin_macros.get(&attr.name()).cloned().or_else(|| {
+                let ident = Ident::with_empty_ctxt(attr.name());
+                self.resolve_lexical_macro_path_segment(ident, MacroNS, None).ok()
+            });
+
+            if let Some(binding) = maybe_binding {
+                if let SyntaxExtension::AttrProcMacro(..) = *binding.get_macro(self) {
+                    attr::mark_known(attr);
+
+                    let msg = "attribute procedural macros are experimental";
+                    let feature = "proc_macro";
+
+                    feature_err(&self.session.parse_sess, feature,
+                                attr.span, GateIssue::Language, msg)
+                        .span_note(binding.span, "procedural macro imported here")
+                        .emit();
+                }
+            }
+        }
+    }
 }
 
 fn is_struct_like(def: Def) -> bool {

src/librustc_resolve/macros.rs

@@ -27,7 +27,7 @@ use syntax::ext::base::{NormalTT, Resolver as SyntaxResolver, SyntaxExtension};
 use syntax::ext::expand::{Expansion, mark_tts};
 use syntax::ext::hygiene::Mark;
 use syntax::ext::tt::macro_rules;
-use syntax::feature_gate::{emit_feature_err, GateIssue};
+use syntax::feature_gate::{emit_feature_err, GateIssue, is_builtin_attr};
 use syntax::fold::{self, Folder};
 use syntax::ptr::P;
 use syntax::symbol::keywords;
@@ -183,6 +183,10 @@ impl<'a> base::Resolver for Resolver<'a> {
                 },
                 None => {}
             }
+
+            if self.proc_macro_enabled && !is_builtin_attr(&attrs[i]) {
+                return Some(attrs.remove(i));
+            }
         }
         None
     }
@@ -373,6 +377,10 @@ impl<'a> Resolver<'a> {
             let resolution = self.resolve_lexical_macro_path_segment(ident, MacroNS, Some(span));
             let (legacy_resolution, resolution) = match (legacy_resolution, resolution) {
                 (Some(legacy_resolution), Ok(resolution)) => (legacy_resolution, resolution),
+                (Some(MacroBinding::Modern(binding)), Err(_)) => {
+                    self.err_if_macro_use_proc_macro(ident.name, span, binding);
+                    continue
+                },
                 _ => continue,
             };
             let (legacy_span, participle) = match legacy_resolution {
@@ -469,4 +477,37 @@ impl<'a> Resolver<'a> {
             self.exported_macros.push(def);
         }
     }
+
+    /// Error if `ext` is a Macros 1.1 procedural macro being imported by `#[macro_use]`
+    fn err_if_macro_use_proc_macro(&mut self, name: Name, use_span: Span,
+                                   binding: &NameBinding<'a>) {
+        use self::SyntaxExtension::*;
+
+        let krate = binding.def().def_id().krate;
+
+        // Plugin-based syntax extensions are exempt from this check
+        if krate == BUILTIN_MACROS_CRATE { return; }
+
+        let ext = binding.get_macro(self);
+
+        match *ext {
+            // If `ext` is a procedural macro, check if we've already warned about it
+            AttrProcMacro(_) | ProcMacro(_) => if !self.warned_proc_macros.insert(name) { return; },
+            _ => return,
+        }
+
+        let warn_msg = match *ext {
+            AttrProcMacro(_) => "attribute procedural macros cannot be \
+                                 imported with `#[macro_use]`",
+            ProcMacro(_) => "procedural macros cannot be imported with `#[macro_use]`",
+            _ => return,
+        };
+
+        let crate_name = self.session.cstore.crate_name(krate);
+
+        self.session.struct_span_err(use_span, warn_msg)
+            .help(&format!("instead, import the procedural macro like any other item: \
+                             `use {}::{};`", crate_name, name))
+            .emit();
+    }
 }

src/libsyntax/ext/expand.rs

@@ -364,7 +364,9 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                 kind.expect_from_annotatables(items)
             }
             SyntaxExtension::AttrProcMacro(ref mac) => {
-                let attr_toks = TokenStream::from_tts(tts_for_attr(&attr, &self.cx.parse_sess));
+                let attr_toks = TokenStream::from_tts(tts_for_attr_args(&attr,
+                                                                        &self.cx.parse_sess));
+
                 let item_toks = TokenStream::from_tts(tts_for_item(&item, &self.cx.parse_sess));
 
                 let tok_result = mac.expand(self.cx, attr.span, attr_toks, item_toks);
@@ -640,8 +642,30 @@ fn tts_for_item(item: &Annotatable, parse_sess: &ParseSess) -> Vec<TokenTree> {
     string_to_tts(text, parse_sess)
 }
 
-fn tts_for_attr(attr: &ast::Attribute, parse_sess: &ParseSess) -> Vec<TokenTree> {
+fn tts_for_attr_args(attr: &ast::Attribute, parse_sess: &ParseSess) -> Vec<TokenTree> {
-    string_to_tts(pprust::attr_to_string(attr), parse_sess)
+    use ast::MetaItemKind::*;
+    use print::pp::Breaks;
+    use print::pprust::PrintState;
+
+    let token_string = match attr.value.node {
+        // For `#[foo]`, an empty token
+        Word => return vec![],
+        // For `#[foo(bar, baz)]`, returns `(bar, baz)`
+        List(ref items) => pprust::to_string(|s| {
+            s.popen()?;
+            s.commasep(Breaks::Consistent,
+                       &items[..],
+                       |s, i| s.print_meta_list_item(&i))?;
+            s.pclose()
+        }),
+        // For `#[foo = "bar"]`, returns `= "bar"`
+        NameValue(ref lit) => pprust::to_string(|s| {
+            s.word_space("=")?;
+            s.print_literal(lit)
+        }),
+    };
+
+    string_to_tts(token_string, parse_sess)
 }
 
 fn string_to_tts(text: String, parse_sess: &ParseSess) -> Vec<TokenTree> {

src/libsyntax/feature_gate.rs

@@ -30,7 +30,7 @@ use ast::{self, NodeId, PatKind};
 use attr;
 use codemap::{CodeMap, Spanned};
 use syntax_pos::Span;
-use errors::{DiagnosticBuilder, Handler};
+use errors::{DiagnosticBuilder, Handler, FatalError};
 use visit::{self, FnKind, Visitor};
 use parse::ParseSess;
 use symbol::Symbol;
@@ -319,6 +319,9 @@ declare_features! (
     // The `unadjusted` ABI. Perma unstable.
     (active, abi_unadjusted, "1.16.0", None),
 
+    // Macros 1.1
+    (active, proc_macro, "1.16.0", Some(35900)),
+
     // Allows attributes on struct literal fields.
     (active, struct_field_attributes, "1.16.0", Some(38814)),
 
@@ -375,8 +378,6 @@ declare_features! (
     // Allows `..` in tuple (struct) patterns
     (accepted, dotdot_in_tuple_patterns, "1.14.0", Some(33627)),
     (accepted, item_like_imports, "1.14.0", Some(35120)),
-    // Macros 1.1
-    (accepted, proc_macro, "1.15.0", Some(35900)),
 );
 // (changing above list without updating src/doc/reference.md makes @cmr sad)
 
@@ -444,6 +445,10 @@ pub fn deprecated_attributes() -> Vec<&'static (&'static str, AttributeType, Att
     BUILTIN_ATTRIBUTES.iter().filter(|a| a.2.is_deprecated()).collect()
 }
 
+pub fn is_builtin_attr(attr: &ast::Attribute) -> bool {
+    BUILTIN_ATTRIBUTES.iter().any(|&(builtin_name, _, _)| attr.check_name(builtin_name))
+}
+
 // Attributes that have a special meaning to rustc or rustdoc
 pub const BUILTIN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeGate)] = &[
     // Normal attributes
@@ -737,6 +742,16 @@ pub const BUILTIN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeG
                                               is currently unstable",
                                              cfg_fn!(windows_subsystem))),
 
+    ("proc_macro_attribute", Normal, Gated(Stability::Unstable,
+                                           "proc_macro",
+                                           "attribute proc macros are currently unstable",
+                                           cfg_fn!(proc_macro))),
+
+    ("rustc_derive_registrar", Normal, Gated(Stability::Unstable,
+                                             "rustc_derive_registrar",
+                                             "used internally by rustc",
+                                             cfg_fn!(rustc_attrs))),
+
     // Crate level attributes
     ("crate_name", CrateLevel, Ungated),
     ("crate_type", CrateLevel, Ungated),
@@ -879,9 +894,10 @@ fn find_lang_feature_issue(feature: &str) -> Option<u32> {
         issue
     } else {
         // search in Accepted or Removed features
-        ACCEPTED_FEATURES.iter().chain(REMOVED_FEATURES.iter())
+        match ACCEPTED_FEATURES.iter().chain(REMOVED_FEATURES).find(|t| t.0 == feature) {
-            .find(|t| t.0 == feature)
+            Some(&(_, _, issue)) => issue,
-            .unwrap().2
+            None => panic!("Feature `{}` is not declared anywhere", feature),
+        }
     }
 }
 
@@ -1382,6 +1398,8 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
 pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute]) -> Features {
     let mut features = Features::new();
 
+    let mut feature_checker = MutexFeatureChecker::default();
+
     for attr in krate_attrs {
         if !attr.check_name("feature") {
             continue
@@ -1405,6 +1423,7 @@ pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute]) -> F
                     if let Some(&(_, _, _, setter)) = ACTIVE_FEATURES.iter()
                         .find(|& &(n, _, _, _)| name == n) {
                         *(setter(&mut features)) = true;
+                        feature_checker.collect(&features, mi.span);
                     }
                     else if let Some(&(_, _, _)) = REMOVED_FEATURES.iter()
                         .find(|& &(n, _, _)| name == n) {
@@ -1421,9 +1440,45 @@ pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute]) -> F
         }
     }
 
+    feature_checker.check(span_handler);
+
     features
 }
 
+// A collector for mutually-exclusive features and their flag spans
+#[derive(Default)]
+struct MutexFeatureChecker {
+    proc_macro: Option<Span>,
+    custom_attribute: Option<Span>,
+}
+
+impl MutexFeatureChecker {
+    // If this method turns out to be a hotspot due to branching,
+    // the branching can be eliminated by modifying `setter!()` to set these spans
+    // only for the features that need to be checked for mutual exclusion.
+    fn collect(&mut self, features: &Features, span: Span) {
+        if features.proc_macro {
+            // If self.proc_macro is None, set to Some(span)
+            self.proc_macro = self.proc_macro.or(Some(span));
+        }
+
+        if features.custom_attribute {
+            self.custom_attribute = self.custom_attribute.or(Some(span));
+        }
+    }
+
+    fn check(self, handler: &Handler) {
+        if let (Some(pm_span), Some(ca_span)) = (self.proc_macro, self.custom_attribute) {
+            handler.struct_span_err(pm_span, "Cannot use `#![feature(proc_macro)]` and \
+                                              `#![feature(custom_attribute)] at the same time")
+                .span_note(ca_span, "`#![feature(custom_attribute)]` declared here")
+                .emit();
+
+            panic!(FatalError);
+        }
+    }
+}
+
 pub fn check_crate(krate: &ast::Crate,
                    sess: &ParseSess,
                    features: &Features,

src/libsyntax_ext/deriving/custom.rs

@@ -77,8 +77,9 @@ impl MultiItemModifier for CustomDerive {
             let inner = self.inner;
             panic::catch_unwind(panic::AssertUnwindSafe(|| inner(input)))
         });
-        let new_items = match res {
+
-            Ok(stream) => __internal::token_stream_items(stream),
+        let stream = match res {
+            Ok(stream) => stream,
             Err(e) => {
                 let msg = "custom derive attribute panicked";
                 let mut err = ecx.struct_span_fatal(span, msg);
@@ -94,6 +95,18 @@ impl MultiItemModifier for CustomDerive {
             }
         };
 
+        let new_items = __internal::set_parse_sess(&ecx.parse_sess, || {
+            match __internal::token_stream_parse_items(stream) {
+                Ok(new_items) => new_items,
+                Err(_) => {
+                    // FIXME: handle this better
+                    let msg = "custom derive produced unparseable tokens";
+                    ecx.struct_span_fatal(span, msg).emit();
+                    panic!(FatalError);
+                }
+            }
+        });
+
         let mut res = vec![Annotatable::Item(item)];
         // Reassign spans of all expanded items to the input `item`
         // for better errors here.

src/libsyntax_ext/lib.rs

@@ -47,6 +47,8 @@ pub mod proc_macro_registrar;
 // for custom_derive
 pub mod deriving;
 
+pub mod proc_macro_impl;
+
 use std::rc::Rc;
 use syntax::ast;
 use syntax::ext::base::{MacroExpanderFn, NormalTT, MultiModifier, NamedSyntaxExtension};

src/libsyntax_ext/proc_macro_impl.rs

@@ -0,0 +1,58 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use std::panic;
+
+use errors::FatalError;
+
+use syntax::codemap::Span;
+use syntax::ext::base::*;
+use syntax::tokenstream::TokenStream;
+use syntax::ext::base;
+
+use proc_macro::TokenStream as TsShim;
+use proc_macro::__internal;
+
+pub struct AttrProcMacro {
+    pub inner: fn(TsShim, TsShim) -> TsShim,
+}
+
+impl base::AttrProcMacro for AttrProcMacro {
+    fn expand<'cx>(&self,
+                   ecx: &'cx mut ExtCtxt,
+                   span: Span,
+                   annotation: TokenStream,
+                   annotated: TokenStream)
+                   -> TokenStream {
+        let annotation = __internal::token_stream_wrap(annotation);
+        let annotated = __internal::token_stream_wrap(annotated);
+
+        let res = __internal::set_parse_sess(&ecx.parse_sess, || {
+            panic::catch_unwind(panic::AssertUnwindSafe(|| (self.inner)(annotation, annotated)))
+        });
+
+        match res {
+            Ok(stream) => __internal::token_stream_inner(stream),
+            Err(e) => {
+                let msg = "custom attribute panicked";
+                let mut err = ecx.struct_span_fatal(span, msg);
+                if let Some(s) = e.downcast_ref::<String>() {
+                    err.help(&format!("message: {}", s));
+                }
+                if let Some(s) = e.downcast_ref::<&'static str>() {
+                    err.help(&format!("message: {}", s));
+                }
+
+                err.emit();
+                panic!(FatalError);
+            }
+        }
+    }
+}

src/libsyntax_ext/proc_macro_registrar.rs

@@ -11,18 +11,20 @@
 use std::mem;
 
 use errors;
+
 use syntax::ast::{self, Ident, NodeId};
 use syntax::codemap::{ExpnInfo, NameAndSpan, MacroAttribute};
 use syntax::ext::base::ExtCtxt;
 use syntax::ext::build::AstBuilder;
 use syntax::ext::expand::ExpansionConfig;
-use syntax::parse::ParseSess;
 use syntax::fold::Folder;
+use syntax::parse::ParseSess;
 use syntax::ptr::P;
 use syntax::symbol::Symbol;
-use syntax_pos::{Span, DUMMY_SP};
 use syntax::visit::{self, Visitor};
 
+use syntax_pos::{Span, DUMMY_SP};
+
 use deriving;
 
 struct CustomDerive {
@@ -32,8 +34,14 @@ struct CustomDerive {
     attrs: Vec<ast::Name>,
 }
 
-struct CollectCustomDerives<'a> {
+struct AttrProcMacro {
+    function_name: Ident,
+    span: Span,
+}
+
+struct CollectProcMacros<'a> {
     derives: Vec<CustomDerive>,
+    attr_macros: Vec<AttrProcMacro>,
     in_root: bool,
     handler: &'a errors::Handler,
     is_proc_macro_crate: bool,
@@ -50,16 +58,17 @@ pub fn modify(sess: &ParseSess,
     let ecfg = ExpansionConfig::default("proc_macro".to_string());
     let mut cx = ExtCtxt::new(sess, ecfg, resolver);
 
-    let derives = {
+    let (derives, attr_macros) = {
-        let mut collect = CollectCustomDerives {
+        let mut collect = CollectProcMacros {
             derives: Vec::new(),
+            attr_macros: Vec::new(),
             in_root: true,
             handler: handler,
             is_proc_macro_crate: is_proc_macro_crate,
             is_test_crate: is_test_crate,
         };
         visit::walk_crate(&mut collect, &krate);
-        collect.derives
+        (collect.derives, collect.attr_macros)
     };
 
     if !is_proc_macro_crate {
@@ -74,7 +83,7 @@ pub fn modify(sess: &ParseSess,
         return krate;
     }
 
-    krate.module.items.push(mk_registrar(&mut cx, &derives));
+    krate.module.items.push(mk_registrar(&mut cx, &derives, &attr_macros));
 
     if krate.exported_macros.len() > 0 {
         handler.err("cannot export macro_rules! macros from a `proc-macro` \
@@ -84,7 +93,7 @@ pub fn modify(sess: &ParseSess,
     return krate
 }
 
-impl<'a> CollectCustomDerives<'a> {
+impl<'a> CollectProcMacros<'a> {
     fn check_not_pub_in_root(&self, vis: &ast::Visibility, sp: Span) {
         if self.is_proc_macro_crate &&
            self.in_root &&
@@ -92,61 +101,11 @@ impl<'a> CollectCustomDerives<'a> {
             self.handler.span_err(sp,
                                   "`proc-macro` crate types cannot \
                                    export any items other than functions \
-                                   tagged with `#[proc_macro_derive]` \
+                                   tagged with `#[proc_macro_derive]` currently");
-                                   currently");
         }
     }
-}
-
-impl<'a> Visitor<'a> for CollectCustomDerives<'a> {
-    fn visit_item(&mut self, item: &'a ast::Item) {
-        let mut attrs = item.attrs.iter().filter(|a| a.check_name("proc_macro_derive"));
-
-        // First up, make sure we're checking a bare function. If we're not then
-        // we're just not interested in this item.
-        //
-        // If we find one, try to locate a `#[proc_macro_derive]` attribute on
-        // it.
-        match item.node {
-            ast::ItemKind::Fn(..) => {}
-            _ => {
-                // Check for invalid use of proc_macro_derive
-                if let Some(attr) = attrs.next() {
-                    self.handler.span_err(attr.span(),
-                                          "the `#[proc_macro_derive]` \
-                                          attribute may only be used \
-                                          on bare functions");
-                    return;
-                }
-                self.check_not_pub_in_root(&item.vis, item.span);
-                return visit::walk_item(self, item)
-            }
-        }
-
-        let attr = match attrs.next() {
-            Some(attr) => attr,
-            None => {
-                self.check_not_pub_in_root(&item.vis, item.span);
-                return visit::walk_item(self, item)
-            }
-        };
-
-        if let Some(a) = attrs.next() {
-            self.handler.span_err(a.span(), "multiple `#[proc_macro_derive]` \
-                                             attributes found");
-        }
-
-        if self.is_test_crate {
-            return;
-        }
-
-        if !self.is_proc_macro_crate {
-            self.handler.span_err(attr.span(),
-                                  "the `#[proc_macro_derive]` attribute is \
-                                   only usable with crates of the `proc-macro` \
-                                   crate type");
-        }
 
+    fn collect_custom_derive(&mut self, item: &'a ast::Item, attr: &'a ast::Attribute) {
         // Once we've located the `#[proc_macro_derive]` attribute, verify
         // that it's of the form `#[proc_macro_derive(Foo)]` or
         // `#[proc_macro_derive(Foo, attributes(A, ..))]`
@@ -232,6 +191,101 @@ impl<'a> Visitor<'a> for CollectCustomDerives<'a> {
             };
             self.handler.span_err(item.span, msg);
         }
+    }
+
+    fn collect_attr_proc_macro(&mut self, item: &'a ast::Item, attr: &'a ast::Attribute) {
+        if let Some(_) = attr.meta_item_list() {
+            self.handler.span_err(attr.span, "`#[proc_macro_attribute]` attribute
+                cannot contain any meta items");
+            return;
+        }
+
+        if self.in_root && item.vis == ast::Visibility::Public {
+            self.attr_macros.push(AttrProcMacro {
+                span: item.span,
+                function_name: item.ident,
+            });
+        } else {
+            let msg = if !self.in_root {
+                "functions tagged with `#[proc_macro_attribute]` must \
+                 currently reside in the root of the crate"
+            } else {
+                "functions tagged with `#[proc_macro_attribute]` must be `pub`"
+            };
+            self.handler.span_err(item.span, msg);
+        }
+    }
+}
+
+impl<'a> Visitor<'a> for CollectProcMacros<'a> {
+    fn visit_item(&mut self, item: &'a ast::Item) {
+        // First up, make sure we're checking a bare function. If we're not then
+        // we're just not interested in this item.
+        //
+        // If we find one, try to locate a `#[proc_macro_derive]` attribute on
+        // it.
+        let is_fn = match item.node {
+            ast::ItemKind::Fn(..) => true,
+            _ => false,
+        };
+
+        let mut found_attr: Option<&'a ast::Attribute> = None;
+
+        for attr in &item.attrs {
+            if attr.check_name("proc_macro_derive") || attr.check_name("proc_macro_attribute") {
+                if let Some(prev_attr) = found_attr {
+                    let msg = if attr.name() == prev_attr.name() {
+                        format!("Only one `#[{}]` attribute is allowed on any given function",
+                                attr.name())
+                    } else {
+                        format!("`#[{}]` and `#[{}]` attributes cannot both be applied \
+                                to the same function", attr.name(), prev_attr.name())
+                    };
+
+                    self.handler.struct_span_err(attr.span(), &msg)
+                        .span_note(prev_attr.span(), "Previous attribute here")
+                        .emit();
+
+                    return;
+                }
+
+                found_attr = Some(attr);
+            }
+        }
+
+        let attr = match found_attr {
+            None => {
+                self.check_not_pub_in_root(&item.vis, item.span);
+                return visit::walk_item(self, item);
+            },
+            Some(attr) => attr,
+        };
+
+        if !is_fn {
+            let msg = format!("the `#[{}]` attribute may only be used on bare functions",
+                              attr.name());
+
+            self.handler.span_err(attr.span(), &msg);
+            return;
+        }
+
+        if self.is_test_crate {
+            return;
+        }
+
+        if !self.is_proc_macro_crate {
+            let msg = format!("the `#[{}]` attribute is only usable with crates of the \
+                              `proc-macro` crate type", attr.name());
+
+            self.handler.span_err(attr.span(), &msg);
+            return;
+        }
+
+        if attr.check_name("proc_macro_derive") {
+            self.collect_custom_derive(item, attr);
+        } else if attr.check_name("proc_macro_attribute") {
+            self.collect_attr_proc_macro(item, attr);
+        };
 
         visit::walk_item(self, item);
     }
@@ -265,7 +319,8 @@ impl<'a> Visitor<'a> for CollectCustomDerives<'a> {
 //          }
 //      }
 fn mk_registrar(cx: &mut ExtCtxt,
-                custom_derives: &[CustomDerive]) -> P<ast::Item> {
+                custom_derives: &[CustomDerive],
+                custom_attrs: &[AttrProcMacro]) -> P<ast::Item> {
     let eid = cx.codemap().record_expansion(ExpnInfo {
         call_site: DUMMY_SP,
         callee: NameAndSpan {
@@ -286,25 +341,36 @@ fn mk_registrar(cx: &mut ExtCtxt,
     let registry = Ident::from_str("Registry");
     let registrar = Ident::from_str("registrar");
     let register_custom_derive = Ident::from_str("register_custom_derive");
-    let stmts = custom_derives.iter().map(|cd| {
+    let register_attr_proc_macro = Ident::from_str("register_attr_proc_macro");
+
+    let mut stmts = custom_derives.iter().map(|cd| {
         let path = cx.path_global(cd.span, vec![cd.function_name]);
         let trait_name = cx.expr_str(cd.span, cd.trait_name);
         let attrs = cx.expr_vec_slice(
             span,
             cd.attrs.iter().map(|&s| cx.expr_str(cd.span, s)).collect::<Vec<_>>()
         );
-        (path, trait_name, attrs)
-    }).map(|(path, trait_name, attrs)| {
         let registrar = cx.expr_ident(span, registrar);
         let ufcs_path = cx.path(span, vec![proc_macro, __internal, registry,
                                            register_custom_derive]);
-        cx.expr_call(span,
+
-                     cx.expr_path(ufcs_path),
+        cx.stmt_expr(cx.expr_call(span, cx.expr_path(ufcs_path),
-                     vec![registrar, trait_name, cx.expr_path(path), attrs])
+                                  vec![registrar, trait_name, cx.expr_path(path), attrs]))
-    }).map(|expr| {
+
-        cx.stmt_expr(expr)
     }).collect::<Vec<_>>();
 
+    stmts.extend(custom_attrs.iter().map(|ca| {
+        let name = cx.expr_str(ca.span, ca.function_name.name);
+        let path = cx.path_global(ca.span, vec![ca.function_name]);
+        let registrar = cx.expr_ident(ca.span, registrar);
+
+        let ufcs_path = cx.path(span,
+                                vec![proc_macro, __internal, registry, register_attr_proc_macro]);
+
+        cx.stmt_expr(cx.expr_call(span, cx.expr_path(ufcs_path),
+                                  vec![registrar, name, cx.expr_path(path)]))
+    }));
+
     let path = cx.path(span, vec![proc_macro, __internal, registry]);
     let registrar_path = cx.ty_path(path);
     let arg_ty = cx.ty_rptr(span, registrar_path, None, ast::Mutability::Mutable);

src/test/compile-fail-fulldeps/proc-macro/auxiliary/attr_proc_macro.rs

@@ -0,0 +1,23 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// force-host
+// no-prefer-dynamic
+#![feature(proc_macro)]
+#![crate_type = "proc-macro"]
+
+extern crate proc_macro;
+
+use proc_macro::TokenStream;
+
+#[proc_macro_attribute]
+pub fn attr_proc_macro(_: TokenStream, input: TokenStream) -> TokenStream {
+    input
+}

src/test/compile-fail-fulldeps/proc-macro/derive-bad.rs

@@ -16,8 +16,7 @@ extern crate derive_bad;
 #[derive(
     A
 )]
-//~^^ ERROR: custom derive attribute panicked
+//~^^ ERROR: custom derive produced unparseable tokens
-//~| HELP: called `Result::unwrap()` on an `Err` value: LexError
 struct A;
 
 fn main() {}

src/test/compile-fail-fulldeps/proc-macro/feature-gate-proc_macro.rs

@@ -0,0 +1,24 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// aux-build:attr_proc_macro.rs
+// gate-test-proc_macro
+#![feature(use_extern_macros)]
+
+extern crate attr_proc_macro;
+use attr_proc_macro::attr_proc_macro;
+
+#[attr_proc_macro]
+//~^ ERROR: attribute procedural macros are experimental
+struct Foo;
+
+fn main() {
+    let _ = Foo;
+}

src/test/compile-fail-fulldeps/proc-macro/macro-use-attr.rs

@@ -0,0 +1,22 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// aux-build:attr_proc_macro.rs
+#![feature(proc_macro)]
+
+#[macro_use] extern crate attr_proc_macro;
+
+#[attr_proc_macro]
+//~^ ERROR: attribute procedural macros cannot be imported with `#[macro_use]`
+struct Foo;
+
+fn main() {
+    let _ = Foo;
+}

src/test/compile-fail-fulldeps/proc-macro/proc-macro-custom-attr-mutex.rs

@@ -0,0 +1,24 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// aux-build:attr_proc_macro.rs
+
+#![feature(proc_macro, custom_attribute)]
+//~^ ERROR Cannot use `#![feature(proc_macro)]` and `#![feature(custom_attribute)] at the same time
+
+extern crate attr_proc_macro;
+use attr_proc_macro::attr_proc_macro;
+
+#[attr_proc_macro]
+fn foo() {}
+
+fn main() {
+    foo();
+}

src/test/compile-fail/const-fn-stability.rs

@@ -16,7 +16,9 @@ const fn foo() -> usize { 0 } //~ ERROR const fn is unstable
 
 trait Foo {
     const fn foo() -> u32; //~ ERROR const fn is unstable
+                           //~| ERROR trait fns cannot be declared const
     const fn bar() -> u32 { 0 } //~ ERROR const fn is unstable
+                                //~| ERROR trait fns cannot be declared const
 }
 
 impl Foo {
@@ -25,6 +27,7 @@ impl Foo {
 
 impl Foo for u32 {
     const fn foo() -> u32 { 0 } //~ ERROR const fn is unstable
+                                //~| ERROR trait fns cannot be declared const
 }
 
 static FOO: usize = foo();

src/test/compile-fail/feature-gate-no-debug.rs

@@ -8,5 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+#![allow(deprecated)]
+
 #[no_debug] //~ ERROR the `#[no_debug]` attribute is
 fn main() {}

src/test/run-pass-fulldeps/proc-macro/attr-args.rs

@@ -0,0 +1,24 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// aux-build:attr-args.rs
+
+#![allow(warnings)]
+#![feature(proc_macro)]
+
+extern crate attr_args;
+use attr_args::attr_with_args;
+
+#[attr_with_args(text = "Hello, world!")]
+fn foo() {}
+
+fn main() {
+    assert_eq!(foo(), "Hello, world!");
+}

src/test/run-pass-fulldeps/proc-macro/auxiliary/attr-args.rs

@@ -0,0 +1,32 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// no-prefer-dynamic
+#![feature(proc_macro)]
+#![crate_type = "proc-macro"]
+
+extern crate proc_macro;
+
+use proc_macro::TokenStream;
+
+#[proc_macro_attribute]
+pub fn attr_with_args(args: TokenStream, input: TokenStream) -> TokenStream {
+    let args = args.to_string();
+
+    assert_eq!(args, r#"( text = "Hello, world!" )"#);
+
+    let input = input.to_string();
+
+    assert_eq!(input, "fn foo (  ) {  }");
+
+    r#"
+        fn foo() -> &'static str { "Hello, world!" }
+    "#.parse().unwrap()
+}

src/test/run-pass-fulldeps/proc-macro/auxiliary/derive-atob.rs

@@ -19,6 +19,6 @@ use proc_macro::TokenStream;
 #[proc_macro_derive(AToB)]
 pub fn derive(input: TokenStream) -> TokenStream {
     let input = input.to_string();
-    assert_eq!(input, "#[derive(Copy, Clone)]\nstruct A;\n");
+    assert_eq!(input, "#[derive(Copy, Clone)]\nstruct A;");
     "struct B;".parse().unwrap()
 }

src/test/run-pass-fulldeps/proc-macro/auxiliary/derive-ctod.rs

@@ -19,6 +19,6 @@ use proc_macro::TokenStream;
 #[proc_macro_derive(CToD)]
 pub fn derive(input: TokenStream) -> TokenStream {
     let input = input.to_string();
-    assert_eq!(input, "struct C;\n");
+    assert_eq!(input, "struct C;");
     "struct D;".parse().unwrap()
 }

src/test/run-pass-fulldeps/proc-macro/auxiliary/derive-same-struct.rs

@@ -18,12 +18,12 @@ use proc_macro::TokenStream;
 #[proc_macro_derive(AToB)]
 pub fn derive1(input: TokenStream) -> TokenStream {
     println!("input1: {:?}", input.to_string());
-    assert_eq!(input.to_string(), "struct A;\n");
+    assert_eq!(input.to_string(), "struct A;");
     "#[derive(BToC)] struct B;".parse().unwrap()
 }
 
 #[proc_macro_derive(BToC)]
 pub fn derive2(input: TokenStream) -> TokenStream {
-    assert_eq!(input.to_string(), "struct B;\n");
+    assert_eq!(input.to_string(), "struct B;");
     "struct C;".parse().unwrap()
 }

src/tools/tidy/src/features.rs

@@ -172,7 +172,7 @@ pub fn check(path: &Path, bad: &mut bool) {
         "use_extern_macros", "staged_api", "const_indexing",
         "unboxed_closures", "stmt_expr_attributes",
         "cfg_target_thread_local", "unwind_attributes",
-        "inclusive_range_syntax"
+        "inclusive_range_syntax", "proc_macro"
     ];
 
     // Only check the number of lang features.