Generate example source files with corresponding links

Add display name

Fix remaining merge conflicts

Only embed code for items containing examples

src/librustdoc/config.rs

@@ -161,12 +161,9 @@ crate struct Options {
     /// Whether to skip capturing stdout and stderr of tests.
     crate nocapture: bool,
 
-    // Options for scraping call sites from examples/ directory
     /// Path to output file to write JSON of call sites. If this option is Some(..) then
     /// the compiler will scrape examples and not generate documentation.
     crate scrape_examples: Option<PathBuf>,
-    /// Path to the root of the workspace, used to generate workspace-relative file paths.
-    crate workspace_root: Option<PathBuf>,
 }
 
 impl fmt::Debug for Options {
@@ -290,7 +287,6 @@ crate struct RenderOptions {
     /// If `true`, HTML source pages will generate links for items to their definition.
     crate generate_link_to_definition: bool,
     crate call_locations: Option<AllCallLocations>,
-    crate repository_url: Option<String>,
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
@@ -682,9 +678,7 @@ impl Options {
             return Err(1);
         }
 
-        let repository_url = matches.opt_str("repository-url");
         let scrape_examples = matches.opt_str("scrape-examples").map(PathBuf::from);
-        let workspace_root = matches.opt_str("workspace-root").map(PathBuf::from);
         let with_examples = matches.opt_strs("with-examples");
         let each_call_locations = with_examples
             .into_iter()
@@ -777,13 +771,11 @@ impl Options {
                 emit,
                 generate_link_to_definition,
                 call_locations,
-                repository_url,
             },
             crate_name,
             output_format,
             json_unused_externs,
             scrape_examples,
-            workspace_root,
         })
     }
 

src/librustdoc/html/render/context.rs

@@ -351,6 +351,7 @@ impl<'tcx> Context<'tcx> {
         let hiline = span.hi(self.sess()).line;
         let lines =
             if loline == hiline { loline.to_string() } else { format!("{}-{}", loline, hiline) };
+
         Some(format!(
             "{root}src/{krate}/{path}#{lines}",
             root = Escape(&root),

src/librustdoc/html/render/mod.rs

@@ -70,7 +70,7 @@ use crate::html::format::{
 };
 use crate::html::markdown::{HeadingOffset, Markdown, MarkdownHtml, MarkdownSummaryLine};
 use crate::html::sources;
-use crate::scrape_examples::FnCallLocations;
+use crate::scrape_examples::{CallData, FnCallLocations};
 
 /// A pair of name and its optional document.
 crate type NameDoc = (String, Option<String>);
@@ -2451,6 +2451,8 @@ fn collect_paths_for_type(first_ty: clean::Type, cache: &Cache) -> Vec<String> {
     out
 }
 
+const MAX_FULL_EXAMPLES: usize = 5;
+
 fn render_call_locations(
     w: &mut Buffer,
     cx: &Context<'_>,
@@ -2463,29 +2465,7 @@ fn render_call_locations(
         }
     };
 
-    let filtered_locations: Vec<_> = call_locations
+    if call_locations.len() == 0 {
-        .iter()
-        .filter_map(|(file, locs)| {
-            // FIXME(wcrichto): file I/O should be cached
-            let mut contents = match fs::read_to_string(&file) {
-                Ok(contents) => contents,
-                Err(e) => {
-                    eprintln!("Failed to read file {}", e);
-                    return None;
-                }
-            };
-
-            // Remove the utf-8 BOM if any
-            if contents.starts_with('\u{feff}') {
-                contents.drain(..3);
-            }
-
-            Some((file, contents, locs))
-        })
-        .collect();
-
-    let n_examples = filtered_locations.len();
-    if n_examples == 0 {
         return;
     }
 
@@ -2499,35 +2479,55 @@ fn render_call_locations(
         id
     );
 
-    let write_example = |w: &mut Buffer, (file, contents, locs): (&String, String, _)| {
+    let example_url = |call_data: &CallData| -> String {
-        let ex_title = match cx.shared.repository_url.as_ref() {
+        format!(
-            Some(url) => format!(
+            r#"<a href="{root}{url}" target="_blank">{name}</a>"#,
-                r#"<a href="{url}/{file}" target="_blank">{file}</a>"#,
+            root = cx.root_path(),
-                file = file,
+            url = call_data.url,
-                url = url
+            name = call_data.display_name
-            ),
+        )
-            None => file.clone(),
+    };
-        };
+
+    let write_example = |w: &mut Buffer, (path, call_data): (&PathBuf, &CallData)| {
+        let mut contents =
+            fs::read_to_string(&path).expect(&format!("Failed to read file: {}", path.display()));
+
+        let min_loc =
+            call_data.locations.iter().min_by_key(|loc| loc.enclosing_item_span.0).unwrap();
+        let min_byte = min_loc.enclosing_item_span.0;
+        let min_line = min_loc.enclosing_item_lines.0;
+        let max_byte =
+            call_data.locations.iter().map(|loc| loc.enclosing_item_span.1).max().unwrap();
+        contents = contents[min_byte..max_byte].to_string();
+
+        let locations = call_data
+            .locations
+            .iter()
+            .map(|loc| (loc.call_span.0 - min_byte, loc.call_span.1 - min_byte))
+            .collect::<Vec<_>>();
+
         let edition = cx.shared.edition();
         write!(
             w,
             r#"<div class="scraped-example" data-code="{code}" data-locs="{locations}">
-           <strong>{title}</strong>
+                <strong>{title}</strong>
-           <div class="code-wrapper">"#,
+                 <div class="code-wrapper">"#,
             code = contents.replace("\"", "&quot;"),
-            locations = serde_json::to_string(&locs).unwrap(),
+            locations = serde_json::to_string(&locations).unwrap(),
-            title = ex_title,
+            title = example_url(call_data),
         );
         write!(w, r#"<span class="prev">&pr;</span> <span class="next">&sc;</span>"#);
         write!(w, r#"<span class="expand">&varr;</span>"#);
-        sources::print_src(w, &contents, edition);
+        let file_span = rustc_span::DUMMY_SP;
+        let root_path = "".to_string();
+        sources::print_src(w, &contents, edition, file_span, cx, &root_path, Some(min_line));
         write!(w, "</div></div>");
     };
 
-    let mut it = filtered_locations.into_iter();
+    let mut it = call_locations.into_iter().peekable();
     write_example(w, it.next().unwrap());
 
-    if n_examples > 1 {
+    if it.peek().is_some() {
         write!(
             w,
             r#"<details class="rustdoc-toggle more-examples-toggle">
@@ -2536,7 +2536,16 @@ fn render_call_locations(
                   </summary>
                   <div class="more-scraped-examples">"#
         );
-        it.for_each(|ex| write_example(w, ex));
+        (&mut it).take(MAX_FULL_EXAMPLES).for_each(|ex| write_example(w, ex));
+
+        if it.peek().is_some() {
+            write!(w, "Additional examples can be found in:<br /><ul>");
+            it.for_each(|(_, call_data)| {
+                write!(w, "<li>{}</li>", example_url(call_data));
+            });
+            write!(w, "</ul>");
+        }
+
         write!(w, "</div></details>");
     }
 

src/librustdoc/html/sources.rs

@@ -204,7 +204,15 @@ impl SourceCollector<'_, 'tcx> {
             &page,
             "",
             |buf: &mut _| {
-                print_src(buf, contents, self.cx.shared.edition(), file_span, &self.cx, &root_path)
+                print_src(
+                    buf,
+                    contents,
+                    self.cx.shared.edition(),
+                    file_span,
+                    &self.cx,
+                    &root_path,
+                    None,
+                )
             },
             &self.cx.shared.style_files,
         );
@@ -250,6 +258,7 @@ crate fn print_src(
     file_span: rustc_span::Span,
     context: &Context<'_>,
     root_path: &str,
+    offset: Option<usize>,
 ) {
     let lines = s.lines().count();
     let mut line_numbers = Buffer::empty_from(buf);
@@ -260,8 +269,9 @@ crate fn print_src(
         tmp /= 10;
     }
     line_numbers.write_str("<pre class=\"line-numbers\">");
+    let offset = offset.unwrap_or(0);
     for i in 1..=lines {
-        writeln!(line_numbers, "<span id=\"{0}\">{0:1$}</span>", i, cols);
+        writeln!(line_numbers, "<span id=\"{0}\">{0:1$}</span>", i + offset, cols);
     }
     line_numbers.write_str("</pre>");
     highlight::render_with_highlighting(

src/librustdoc/html/static/css/rustdoc.css

@@ -453,7 +453,7 @@ nav.sub {
 	text-decoration: underline;
 }
 
-.rustdoc:not(.source) .example-wrap > pre:not(.line-number) {
+.rustdoc:not(.source) .example-wrap > pre:not(.line-numbers) {
 	width: 100%;
 	overflow-x: auto;
 }

src/librustdoc/lib.rs

@@ -620,8 +620,6 @@ fn opts() -> Vec<RustcOptGroup> {
             )
         }),
         unstable("scrape-examples", |o| o.optopt("", "scrape-examples", "", "")),
-        unstable("workspace-root", |o| o.optopt("", "workspace-root", "", "")),
-        unstable("repository-url", |o| o.optopt("", "repository-url", "", "")),
         unstable("with-examples", |o| o.optmulti("", "with-examples", "", "")),
     ]
 }
@@ -705,17 +703,16 @@ fn run_renderer<'tcx, T: formats::FormatRenderer<'tcx>>(
 fn main_options(options: config::Options) -> MainResult {
     let diag = core::new_handler(options.error_format, None, &options.debugging_opts);
 
-    match (options.should_test, options.markdown_input(), options.scrape_examples.is_some()) {
+    match (options.should_test, options.markdown_input()) {
-        (_, _, true) => return scrape_examples::run(options),
+        (true, true) => return wrap_return(&diag, markdown::test(options)),
-        (true, true, false) => return wrap_return(&diag, markdown::test(options)),
+        (true, false) => return doctest::run(options),
-        (true, false, false) => return doctest::run(options),
+        (false, true) => {
-        (false, true, false) => {
             return wrap_return(
                 &diag,
                 markdown::render(&options.input, options.render_options, options.edition),
             );
         }
-        (false, false, false) => {}
+        (false, false) => {}
     }
 
     // need to move these items separately because we lose them by the time the closure is called,
@@ -737,6 +734,7 @@ fn main_options(options: config::Options) -> MainResult {
     // FIXME: fix this clone (especially render_options)
     let manual_passes = options.manual_passes.clone();
     let render_options = options.render_options.clone();
+    let scrape_examples = options.scrape_examples.clone();
     let config = core::create_config(options);
 
     interface::create_compiler_and_run(config, |compiler| {
@@ -773,6 +771,10 @@ fn main_options(options: config::Options) -> MainResult {
                 });
                 info!("finished with rustc");
 
+                if let Some(example_path) = scrape_examples {
+                    return scrape_examples::run(krate, render_opts, cache, tcx, example_path);
+                }
+
                 cache.crate_version = crate_version;
 
                 if show_coverage {

src/librustdoc/scrape_examples.rs

@@ -1,8 +1,12 @@
 //! This module analyzes provided crates to find examples of uses for items in the
 //! current crate being documented.
 
-use crate::config::Options;
+use crate::clean;
-use crate::doctest::make_rustc_config;
+use crate::config;
+use crate::formats;
+use crate::formats::renderer::FormatRenderer;
+use crate::html::render::Context;
+
 use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::{
     self as hir,
@@ -11,23 +15,33 @@ use rustc_hir::{
 use rustc_interface::interface;
 use rustc_middle::hir::map::Map;
 use rustc_middle::ty::{self, TyCtxt};
-use rustc_span::def_id::DefId;
+use rustc_span::{def_id::DefId, FileName};
+use serde::{Deserialize, Serialize};
 use std::fs;
+use std::path::PathBuf;
 
+#[derive(Serialize, Deserialize, Debug, Clone)]
+crate struct CallLocation {
+    crate call_span: (usize, usize),
+    crate enclosing_item_span: (usize, usize),
+    crate enclosing_item_lines: (usize, usize),
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+crate struct CallData {
+    crate locations: Vec<CallLocation>,
+    crate url: String,
+    crate display_name: String,
+}
 crate type DefIdCallKey = String;
-crate type FnCallLocations = FxHashMap<String, Vec<(usize, usize)>>;
+crate type FnCallLocations = FxHashMap<PathBuf, CallData>;
 crate type AllCallLocations = FxHashMap<DefIdCallKey, FnCallLocations>;
 
 /// Visitor for traversing a crate and finding instances of function calls.
 struct FindCalls<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     map: Map<'tcx>,
-
+    cx: Context<'tcx>,
-    /// Workspace-relative path to the root of the crate. Used to remember
-    /// which example a particular call came from.
-    file_path: String,
-
-    /// Data structure to accumulate call sites across all examples.
     calls: &'a mut AllCallLocations,
 }
 
@@ -68,51 +82,75 @@ where
             }
         };
 
+        if span.from_expansion() {
+            return;
+        }
+
         // Save call site if the function resolves to a concrete definition
         if let ty::FnDef(def_id, _) = ty.kind() {
-            let key = def_id_call_key(self.tcx, *def_id);
+            let fn_key = def_id_call_key(self.tcx, *def_id);
-            let entries = self.calls.entry(key).or_insert_with(FxHashMap::default);
+            let entries = self.calls.entry(fn_key).or_insert_with(FxHashMap::default);
-            entries
+            let file = self.tcx.sess.source_map().lookup_char_pos(span.lo()).file;
-                .entry(self.file_path.clone())
+            let file_path = match file.name.clone() {
-                .or_insert_with(Vec::new)
+                FileName::Real(real_filename) => real_filename.into_local_path(),
-                .push((span.lo().0 as usize, span.hi().0 as usize));
+                _ => None,
+            };
+
+            let get_pos =
+                |bytepos: rustc_span::BytePos| file.original_relative_byte_pos(bytepos).0 as usize;
+            let get_range = |span: rustc_span::Span| (get_pos(span.lo()), get_pos(span.hi()));
+            let get_line = |bytepos: rustc_span::BytePos| file.lookup_line(bytepos).unwrap();
+            let get_lines = |span: rustc_span::Span| (get_line(span.lo()), get_line(span.hi()));
+
+            if let Some(file_path) = file_path {
+                let abs_path = fs::canonicalize(file_path.clone()).unwrap();
+                let cx = &self.cx;
+                let enclosing_item_span =
+                    self.tcx.hir().span_with_body(self.tcx.hir().get_parent_item(ex.hir_id));
+                assert!(enclosing_item_span.contains(span));
+
+                let location = CallLocation {
+                    call_span: get_range(span),
+                    enclosing_item_span: get_range(enclosing_item_span),
+                    enclosing_item_lines: get_lines(enclosing_item_span),
+                };
+
+                entries
+                    .entry(abs_path)
+                    .or_insert_with(|| {
+                        let clean_span = crate::clean::types::Span::new(span);
+                        let url = cx.href_from_span(clean_span).unwrap();
+                        let display_name = file_path.display().to_string();
+                        CallData { locations: Vec::new(), url, display_name }
+                    })
+                    .locations
+                    .push(location);
+            }
         }
     }
 }
 
-crate fn run(options: Options) -> interface::Result<()> {
+crate fn run(
+    krate: clean::Crate,
+    renderopts: config::RenderOptions,
+    cache: formats::cache::Cache,
+    tcx: TyCtxt<'tcx>,
+    example_path: PathBuf,
+) -> interface::Result<()> {
     let inner = move || {
-        let config = make_rustc_config(&options);
+        // Generates source files for examples
+        let (cx, _) = Context::init(krate, renderopts, cache, tcx).map_err(|e| format!("{}", e))?;
 
-        // Get input file path as relative to workspace root
+        // Run call-finder on all items
-        let file_path = options
+        let mut calls = FxHashMap::default();
-            .input
+        let mut finder = FindCalls { calls: &mut calls, tcx, map: tcx.hir(), cx };
-            .strip_prefix(options.workspace_root.as_ref().unwrap())
+        tcx.hir().krate().visit_all_item_likes(&mut finder.as_deep_visitor());
-            .map_err(|e| format!("{}", e))?;
 
-        interface::run_compiler(config, |compiler| {
+        // Save output JSON to provided path
-            compiler.enter(|queries| {
+        let calls_json = serde_json::to_string(&calls).map_err(|e| format!("{}", e))?;
-                let mut global_ctxt = queries.global_ctxt().unwrap().take();
+        fs::write(example_path, &calls_json).map_err(|e| format!("{}", e))?;
-                global_ctxt.enter(|tcx| {
-                    // Run call-finder on all items
-                    let mut calls = FxHashMap::default();
-                    let mut finder = FindCalls {
-                        calls: &mut calls,
-                        tcx,
-                        map: tcx.hir(),
-                        file_path: file_path.display().to_string(),
-                    };
-                    tcx.hir().krate().visit_all_item_likes(&mut finder.as_deep_visitor());
 
-                    // Save output JSON to provided path
+        Ok(())
-                    let calls_json = serde_json::to_string(&calls).map_err(|e| format!("{}", e))?;
-                    fs::write(options.scrape_examples.as_ref().unwrap(), &calls_json)
-                        .map_err(|e| format!("{}", e))?;
-
-                    Ok(())
-                })
-            })
-        })
     };
 
     inner().map_err(|e: String| {