bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

proc_macro: Add an expand_expr method to TokenStream

Browse source 
This feature is aimed at giving proc macros access to powers similar to
those used by builtin macros such as `format_args!` or `concat!`. These
macros are able to accept macros in place of string literal parameters,
such as the format string, as they perform recursive macro expansion
while being expanded.

This can be especially useful in many cases thanks to helper macros like
`concat!`, `stringify!` and `include_str!` which are often used to
construct string literals at compile-time in user code.

For now, this method only allows expanding macros which produce
literals, although more expresisons will be supported before the method
is stabilized.
This commit is contained in:
Nika Layzell 2021-07-18 15:53:06 -04:00
parent 800a156c1e
commit 3e4d3d2a29
8 changed files with 421 additions and 75 deletions
Download patch file Download diff file Expand all files Collapse all files

28
compiler/rustc_expand/src/proc_macro.rs
View file

@ -24,8 +24,9 @@ impl base::ProcMacro for BangProcMacro {
span: Span,
input: TokenStream,
) -> Result<TokenStream, ErrorReported> {
let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace;
let server = proc_macro_server::Rustc::new(ecx);
self.client.run(&EXEC_STRATEGY, server, input, ecx.ecfg.proc_macro_backtrace).map_err(|e| {
self.client.run(&EXEC_STRATEGY, server, input, proc_macro_backtrace).map_err(|e| {
let mut err = ecx.struct_span_err(span, "proc macro panicked");
if let Some(s) = e.as_str() {
err.help(&format!("message: {}", s));
@ -48,9 +49,10 @@ impl base::AttrProcMacro for AttrProcMacro {
annotation: TokenStream,
annotated: TokenStream,
) -> Result<TokenStream, ErrorReported> {
let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace;
let server = proc_macro_server::Rustc::new(ecx);
self.client
.run(&EXEC_STRATEGY, server, annotation, annotated, ecx.ecfg.proc_macro_backtrace)
.run(&EXEC_STRATEGY, server, annotation, annotated, proc_macro_backtrace)
.map_err(|e| {
let mut err = ecx.struct_span_err(span, "custom attribute panicked");
if let Some(s) = e.as_str() {
@ -97,19 +99,19 @@ impl MultiItemModifier for ProcMacroDerive {
nt_to_tokenstream(&item, &ecx.sess.parse_sess, CanSynthesizeMissingTokens::No)
};
let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace;
let server = proc_macro_server::Rustc::new(ecx);
let stream =
match self.client.run(&EXEC_STRATEGY, server, input, ecx.ecfg.proc_macro_backtrace) {
Ok(stream) => stream,
Err(e) => {
let mut err = ecx.struct_span_err(span, "proc-macro derive panicked");
if let Some(s) = e.as_str() {
err.help(&format!("message: {}", s));
}
err.emit();
return ExpandResult::Ready(vec![]);
let stream = match self.client.run(&EXEC_STRATEGY, server, input, proc_macro_backtrace) {
Ok(stream) => stream,
Err(e) => {
let mut err = ecx.struct_span_err(span, "proc-macro derive panicked");
if let Some(s) = e.as_str() {
err.help(&format!("message: {}", s));
}
};
err.emit();
return ExpandResult::Ready(vec![]);
}
};
let error_count_before = ecx.sess.parse_sess.span_diagnostic.err_count();
let mut parser =

162
compiler/rustc_expand/src/proc_macro_server.rs
View file

@ -1,4 +1,4 @@
use crate::base::{ExtCtxt, ResolverExpand};
use crate::base::ExtCtxt;
use rustc_ast as ast;
use rustc_ast::token::{self, Nonterminal, NtIdent};
@ -7,7 +7,7 @@ use rustc_ast::tokenstream::{DelimSpan, Spacing::*, TokenStream, TreeAndSpacing}
use rustc_ast_pretty::pprust;
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::sync::Lrc;
use rustc_errors::Diagnostic;
use rustc_errors::{Diagnostic, PResult};
use rustc_lint_defs::builtin::PROC_MACRO_BACK_COMPAT;
use rustc_lint_defs::BuiltinLintDiagnostics;
use rustc_parse::lexer::nfc_normalize;
@ -53,11 +53,11 @@ impl ToInternal<token::DelimToken> for Delimiter {
}
}
impl FromInternal<(TreeAndSpacing, &'_ mut Vec<Self>, &mut Rustc<'_>)>
impl FromInternal<(TreeAndSpacing, &'_ mut Vec<Self>, &mut Rustc<'_, '_>)>
for TokenTree<Group, Punct, Ident, Literal>
{
fn from_internal(
((tree, spacing), stack, rustc): (TreeAndSpacing, &mut Vec<Self>, &mut Rustc<'_>),
((tree, spacing), stack, rustc): (TreeAndSpacing, &mut Vec<Self>, &mut Rustc<'_, '_>),
) -> Self {
use rustc_ast::token::*;
@ -146,10 +146,10 @@ impl FromInternal<(TreeAndSpacing, &'_ mut Vec<Self>, &mut Rustc<'_>)>
SingleQuote => op!('\''),
Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()),
Ident(name, is_raw) => tt!(Ident::new(rustc.sess, name, is_raw)),
Ident(name, is_raw) => tt!(Ident::new(rustc.sess(), name, is_raw)),
Lifetime(name) => {
let ident = symbol::Ident::new(name, span).without_first_quote();
stack.push(tt!(Ident::new(rustc.sess, ident.name, false)));
stack.push(tt!(Ident::new(rustc.sess(), ident.name, false)));
tt!(Punct::new('\'', true))
}
Literal(lit) => tt!(Literal { lit }),
@ -181,15 +181,15 @@ impl FromInternal<(TreeAndSpacing, &'_ mut Vec<Self>, &mut Rustc<'_>)>
Interpolated(nt)
if let Some((name, is_raw)) = ident_name_compatibility_hack(&nt, span, rustc) =>
{
TokenTree::Ident(Ident::new(rustc.sess, name.name, is_raw, name.span))
TokenTree::Ident(Ident::new(rustc.sess(), name.name, is_raw, name.span))
}
Interpolated(nt) => {
let stream = nt_to_tokenstream(&nt, rustc.sess, CanSynthesizeMissingTokens::No);
let stream = nt_to_tokenstream(&nt, rustc.sess(), CanSynthesizeMissingTokens::No);
TokenTree::Group(Group {
delimiter: Delimiter::None,
stream,
span: DelimSpan::from_single(span),
flatten: crate::base::pretty_printing_compatibility_hack(&nt, rustc.sess),
flatten: crate::base::pretty_printing_compatibility_hack(&nt, rustc.sess()),
})
}
@ -355,38 +355,38 @@ pub struct Literal {
span: Span,
}
pub(crate) struct Rustc<'a> {
resolver: &'a dyn ResolverExpand,
sess: &'a ParseSess,
pub(crate) struct Rustc<'a, 'b> {
ecx: &'a mut ExtCtxt<'b>,
def_site: Span,
call_site: Span,
mixed_site: Span,
span_debug: bool,
krate: CrateNum,
rebased_spans: FxHashMap<usize, Span>,
}
impl<'a> Rustc<'a> {
pub fn new(cx: &'a ExtCtxt<'_>) -> Self {
let expn_data = cx.current_expansion.id.expn_data();
impl<'a, 'b> Rustc<'a, 'b> {
pub fn new(ecx: &'a mut ExtCtxt<'b>) -> Self {
let expn_data = ecx.current_expansion.id.expn_data();
Rustc {
resolver: cx.resolver,
sess: cx.parse_sess(),
def_site: cx.with_def_site_ctxt(expn_data.def_site),
call_site: cx.with_call_site_ctxt(expn_data.call_site),
mixed_site: cx.with_mixed_site_ctxt(expn_data.call_site),
span_debug: cx.ecfg.span_debug,
def_site: ecx.with_def_site_ctxt(expn_data.def_site),
call_site: ecx.with_call_site_ctxt(expn_data.call_site),
mixed_site: ecx.with_mixed_site_ctxt(expn_data.call_site),
krate: expn_data.macro_def_id.unwrap().krate,
rebased_spans: FxHashMap::default(),
ecx,
}
}
fn sess(&self) -> &ParseSess {
self.ecx.parse_sess()
}
fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Literal {
Literal { lit: token::Lit::new(kind, symbol, suffix), span: server::Span::call_site(self) }
}
}
impl server::Types for Rustc<'_> {
impl server::Types for Rustc<'_, '_> {
type FreeFunctions = FreeFunctions;
type TokenStream = TokenStream;
type TokenStreamBuilder = tokenstream::TokenStreamBuilder;
@ -401,17 +401,20 @@ impl server::Types for Rustc<'_> {
type Span = Span;
}
impl server::FreeFunctions for Rustc<'_> {
impl server::FreeFunctions for Rustc<'_, '_> {
fn track_env_var(&mut self, var: &str, value: Option<&str>) {
self.sess.env_depinfo.borrow_mut().insert((Symbol::intern(var), value.map(Symbol::intern)));
self.sess()
.env_depinfo
.borrow_mut()
.insert((Symbol::intern(var), value.map(Symbol::intern)));
}
fn track_path(&mut self, path: &str) {
self.sess.file_depinfo.borrow_mut().insert(Symbol::intern(path));
self.sess().file_depinfo.borrow_mut().insert(Symbol::intern(path));
}
}
impl server::TokenStream for Rustc<'_> {
impl server::TokenStream for Rustc<'_, '_> {
fn new(&mut self) -> Self::TokenStream {
TokenStream::default()
}
@ -422,13 +425,62 @@ impl server::TokenStream for Rustc<'_> {
parse_stream_from_source_str(
FileName::proc_macro_source_code(src),
src.to_string(),
self.sess,
self.sess(),
Some(self.call_site),
)
}
fn to_string(&mut self, stream: &Self::TokenStream) -> String {
pprust::tts_to_string(stream)
}
fn expand_expr(&mut self, stream: &Self::TokenStream) -> Result<Self::TokenStream, ()> {
// Parse the expression from our tokenstream.
let expr: PResult<'_, _> = try {
let mut p = rustc_parse::stream_to_parser(
self.sess(),
stream.clone(),
Some("proc_macro expand expr"),
);
let expr = p.parse_expr()?;
if p.token != token::Eof {
p.unexpected()?;
}
expr
};
let expr = expr.map_err(|mut err| err.emit())?;
// Perform eager expansion on the expression.
let expr = self
.ecx
.expander()
.fully_expand_fragment(crate::expand::AstFragment::Expr(expr))
.make_expr();
// NOTE: For now, limit `expand_expr` to exclusively expand to literals.
// This may be relaxed in the future.
// We don't use `nt_to_tokenstream` as the tokenstream currently cannot
// be recovered in the general case.
match &expr.kind {
ast::ExprKind::Lit(l) => {
Ok(tokenstream::TokenTree::token(token::Literal(l.token), l.span).into())
}
ast::ExprKind::Unary(ast::UnOp::Neg, e) => match &e.kind {
ast::ExprKind::Lit(l) => match l.token {
token::Lit { kind: token::Integer | token::Float, .. } => {
Ok(std::array::IntoIter::new([
// FIXME: The span of the `-` token is lost when
// parsing, so we cannot faithfully recover it here.
tokenstream::TokenTree::token(token::BinOp(token::Minus), e.span),
tokenstream::TokenTree::token(token::Literal(l.token), l.span),
])
.collect())
}
_ => Err(()),
},
_ => Err(()),
},
_ => Err(()),
}
}
fn from_token_tree(
&mut self,
tree: TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>,
@ -440,7 +492,7 @@ impl server::TokenStream for Rustc<'_> {
}
}
impl server::TokenStreamBuilder for Rustc<'_> {
impl server::TokenStreamBuilder for Rustc<'_, '_> {
fn new(&mut self) -> Self::TokenStreamBuilder {
tokenstream::TokenStreamBuilder::new()
}
@ -452,7 +504,7 @@ impl server::TokenStreamBuilder for Rustc<'_> {
}
}
impl server::TokenStreamIter for Rustc<'_> {
impl server::TokenStreamIter for Rustc<'_, '_> {
fn next(
&mut self,
iter: &mut Self::TokenStreamIter,
@ -477,7 +529,7 @@ impl server::TokenStreamIter for Rustc<'_> {
}
}
impl server::Group for Rustc<'_> {
impl server::Group for Rustc<'_, '_> {
fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group {
Group {
delimiter,
@ -506,7 +558,7 @@ impl server::Group for Rustc<'_> {
}
}
impl server::Punct for Rustc<'_> {
impl server::Punct for Rustc<'_, '_> {
fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct {
Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self))
}
@ -524,9 +576,9 @@ impl server::Punct for Rustc<'_> {
}
}
impl server::Ident for Rustc<'_> {
impl server::Ident for Rustc<'_, '_> {
fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident {
Ident::new(self.sess, Symbol::intern(string), is_raw, span)
Ident::new(self.sess(), Symbol::intern(string), is_raw, span)
}
fn span(&mut self, ident: Self::Ident) -> Self::Span {
ident.span
@ -536,10 +588,10 @@ impl server::Ident for Rustc<'_> {
}
}
impl server::Literal for Rustc<'_> {
impl server::Literal for Rustc<'_, '_> {
fn from_str(&mut self, s: &str) -> Result<Self::Literal, ()> {
let name = FileName::proc_macro_source_code(s);
let mut parser = rustc_parse::new_parser_from_source_str(self.sess, name, s.to_owned());
let mut parser = rustc_parse::new_parser_from_source_str(self.sess(), name, s.to_owned());
let first_span = parser.token.span.data();
let minus_present = parser.eat(&token::BinOp(token::Minus));
@ -675,7 +727,7 @@ impl server::Literal for Rustc<'_> {
}
}
impl server::SourceFile for Rustc<'_> {
impl server::SourceFile for Rustc<'_, '_> {
fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool {
Lrc::ptr_eq(file1, file2)
}
@ -695,7 +747,7 @@ impl server::SourceFile for Rustc<'_> {
}
}
impl server::MultiSpan for Rustc<'_> {
impl server::MultiSpan for Rustc<'_, '_> {
fn new(&mut self) -> Self::MultiSpan {
vec![]
}
@ -704,7 +756,7 @@ impl server::MultiSpan for Rustc<'_> {
}
}
impl server::Diagnostic for Rustc<'_> {
impl server::Diagnostic for Rustc<'_, '_> {
fn new(&mut self, level: Level, msg: &str, spans: Self::MultiSpan) -> Self::Diagnostic {
let mut diag = Diagnostic::new(level.to_internal(), msg);
diag.set_span(MultiSpan::from_spans(spans));
@ -720,13 +772,13 @@ impl server::Diagnostic for Rustc<'_> {
diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None);
}
fn emit(&mut self, diag: Self::Diagnostic) {
self.sess.span_diagnostic.emit_diagnostic(&diag);
self.sess().span_diagnostic.emit_diagnostic(&diag);
}
}
impl server::Span for Rustc<'_> {
impl server::Span for Rustc<'_, '_> {
fn debug(&mut self, span: Self::Span) -> String {
if self.span_debug {
if self.ecx.ecfg.span_debug {
format!("{:?}", span)
} else {
format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0)
@ -742,7 +794,7 @@ impl server::Span for Rustc<'_> {
self.mixed_site
}
fn source_file(&mut self, span: Self::Span) -> Self::SourceFile {
self.sess.source_map().lookup_char_pos(span.lo()).file
self.sess().source_map().lookup_char_pos(span.lo()).file
}
fn parent(&mut self, span: Self::Span) -> Option<Self::Span> {
span.parent_callsite()
@ -751,11 +803,11 @@ impl server::Span for Rustc<'_> {
span.source_callsite()
}
fn start(&mut self, span: Self::Span) -> LineColumn {
let loc = self.sess.source_map().lookup_char_pos(span.lo());
let loc = self.sess().source_map().lookup_char_pos(span.lo());
LineColumn { line: loc.line, column: loc.col.to_usize() }
}
fn end(&mut self, span: Self::Span) -> LineColumn {
let loc = self.sess.source_map().lookup_char_pos(span.hi());
let loc = self.sess().source_map().lookup_char_pos(span.hi());
LineColumn { line: loc.line, column: loc.col.to_usize() }
}
fn before(&mut self, span: Self::Span) -> Self::Span {
@ -765,8 +817,8 @@ impl server::Span for Rustc<'_> {
span.shrink_to_hi()
}
fn join(&mut self, first: Self::Span, second: Self::Span) -> Option<Self::Span> {
let self_loc = self.sess.source_map().lookup_char_pos(first.lo());
let other_loc = self.sess.source_map().lookup_char_pos(second.lo());
let self_loc = self.sess().source_map().lookup_char_pos(first.lo());
let other_loc = self.sess().source_map().lookup_char_pos(second.lo());
if self_loc.file.name != other_loc.file.name {
return None;
@ -778,7 +830,7 @@ impl server::Span for Rustc<'_> {
span.with_ctxt(at.ctxt())
}
fn source_text(&mut self, span: Self::Span) -> Option<String> {
self.sess.source_map().span_to_snippet(span).ok()
self.sess().source_map().span_to_snippet(span).ok()
}
/// Saves the provided span into the metadata of
/// *the crate we are currently compiling*, which must
@ -805,10 +857,10 @@ impl server::Span for Rustc<'_> {
/// since we've loaded `my_proc_macro` from disk in order to execute it).
/// In this way, we have obtained a span pointing into `my_proc_macro`
fn save_span(&mut self, span: Self::Span) -> usize {
self.sess.save_proc_macro_span(span)
self.sess().save_proc_macro_span(span)
}
fn recover_proc_macro_span(&mut self, id: usize) -> Self::Span {
let (resolver, krate, def_site) = (self.resolver, self.krate, self.def_site);
let (resolver, krate, def_site) = (&*self.ecx.resolver, self.krate, self.def_site);
*self.rebased_spans.entry(id).or_insert_with(|| {
// FIXME: `SyntaxContext` for spans from proc macro crates is lost during encoding,
// replace it with a def-site context until we are encoding it properly.
@ -821,11 +873,11 @@ impl server::Span for Rustc<'_> {
fn ident_name_compatibility_hack(
nt: &Nonterminal,
orig_span: Span,
rustc: &mut Rustc<'_>,
rustc: &mut Rustc<'_, '_>,
) -> Option<(rustc_span::symbol::Ident, bool)> {
if let NtIdent(ident, is_raw) = nt {
if let ExpnKind::Macro(_, macro_name) = orig_span.ctxt().outer_expn_data().kind {
let source_map = rustc.sess.source_map();
let source_map = rustc.sess().source_map();
let filename = source_map.span_to_filename(orig_span);
if let FileName::Real(RealFileName::LocalPath(path)) = filename {
let matches_prefix = |prefix, filename| {
@ -846,7 +898,7 @@ fn ident_name_compatibility_hack(
let snippet = source_map.span_to_snippet(orig_span);
if snippet.as_deref() == Ok("$name") {
if time_macros_impl {
rustc.sess.buffer_lint_with_diagnostic(
rustc.sess().buffer_lint_with_diagnostic(
&PROC_MACRO_BACK_COMPAT,
orig_span,
ast::CRATE_NODE_ID,
@ -871,7 +923,7 @@ fn ident_name_compatibility_hack(
.and_then(|c| c.parse::<u32>().ok())
.map_or(false, |v| v < 40)
{
rustc.sess.buffer_lint_with_diagnostic(
rustc.sess().buffer_lint_with_diagnostic(
&PROC_MACRO_BACK_COMPAT,
orig_span,
ast::CRATE_NODE_ID,
@ -894,7 +946,7 @@ fn ident_name_compatibility_hack(
source_map.span_to_filename(rustc.def_site)
{
if macro_path.to_string_lossy().contains("pin-project-internal-0.") {
rustc.sess.buffer_lint_with_diagnostic(
rustc.sess().buffer_lint_with_diagnostic(
&PROC_MACRO_BACK_COMPAT,
orig_span,
ast::CRATE_NODE_ID,