1
Fork 0

Move highlighting logic from JS to Rust

Continue migrating JS functionality

Cleanup

Fix compile error

Clean up the diff

Set toggle font to sans-serif
This commit is contained in:
Will Crichton 2021-08-26 14:43:12 -07:00
parent eea8f0a39a
commit 55bb51786e
10 changed files with 188 additions and 186 deletions

View file

@ -45,7 +45,7 @@ crate struct TestOptions {
crate attrs: Vec<String>,
}
crate fn make_rustc_config(options: &Options) -> interface::Config {
crate fn run(options: Options) -> Result<(), ErrorReported> {
let input = config::Input::File(options.input.clone());
let invalid_codeblock_attributes_name = crate::lint::INVALID_CODEBLOCK_ATTRIBUTES.name;
@ -87,7 +87,7 @@ crate fn make_rustc_config(options: &Options) -> interface::Config {
let mut cfgs = options.cfgs.clone();
cfgs.push("doc".to_owned());
cfgs.push("doctest".to_owned());
interface::Config {
let config = interface::Config {
opts: sessopts,
crate_cfg: interface::parse_cfgspecs(cfgs),
input,
@ -103,11 +103,7 @@ crate fn make_rustc_config(options: &Options) -> interface::Config {
override_queries: None,
make_codegen_backend: None,
registry: rustc_driver::diagnostics_registry(),
}
}
crate fn run(options: Options) -> Result<(), ErrorReported> {
let config = make_rustc_config(&options);
};
let test_args = options.test_args.clone();
let display_doctest_warnings = options.display_doctest_warnings;

View file

@ -12,6 +12,7 @@ use crate::html::render::Context;
use std::collections::VecDeque;
use std::fmt::{Display, Write};
use rustc_data_structures::fx::FxHashMap;
use rustc_lexer::{LiteralKind, TokenKind};
use rustc_span::edition::Edition;
use rustc_span::symbol::Symbol;
@ -30,6 +31,8 @@ crate struct ContextInfo<'a, 'b, 'c> {
crate root_path: &'c str,
}
crate type DecorationInfo = FxHashMap<&'static str, Vec<(u32, u32)>>;
/// Highlights `src`, returning the HTML output.
crate fn render_with_highlighting(
src: &str,
@ -40,6 +43,7 @@ crate fn render_with_highlighting(
edition: Edition,
extra_content: Option<Buffer>,
context_info: Option<ContextInfo<'_, '_, '_>>,
decoration_info: Option<DecorationInfo>,
) {
debug!("highlighting: ================\n{}\n==============", src);
if let Some((edition_info, class)) = tooltip {
@ -56,7 +60,7 @@ crate fn render_with_highlighting(
}
write_header(out, class, extra_content);
write_code(out, &src, edition, context_info);
write_code(out, &src, edition, context_info, decoration_info);
write_footer(out, playground_button);
}
@ -89,10 +93,16 @@ fn write_code(
src: &str,
edition: Edition,
context_info: Option<ContextInfo<'_, '_, '_>>,
decoration_info: Option<DecorationInfo>,
) {
// This replace allows to fix how the code source with DOS backline characters is displayed.
let src = src.replace("\r\n", "\n");
Classifier::new(&src, edition, context_info.as_ref().map(|c| c.file_span).unwrap_or(DUMMY_SP))
Classifier::new(
&src,
edition,
context_info.as_ref().map(|c| c.file_span).unwrap_or(DUMMY_SP),
decoration_info,
)
.highlight(&mut |highlight| {
match highlight {
Highlight::Token { text, class } => string(out, Escape(text), class, &context_info),
@ -127,6 +137,7 @@ enum Class {
PreludeTy,
PreludeVal,
QuestionMark,
Decoration(&'static str),
}
impl Class {
@ -150,6 +161,7 @@ impl Class {
Class::PreludeTy => "prelude-ty",
Class::PreludeVal => "prelude-val",
Class::QuestionMark => "question-mark",
Class::Decoration(kind) => kind,
}
}
@ -244,7 +256,28 @@ impl Iterator for PeekIter<'a> {
type Item = (TokenKind, &'a str);
fn next(&mut self) -> Option<Self::Item> {
self.peek_pos = 0;
if let Some(first) = self.stored.pop_front() { Some(first) } else { self.iter.next() }
if let Some(first) = self.stored.pop_front() {
Some(first)
} else {
self.iter.next()
}
}
}
/// Custom spans inserted into the source. Eg --scrape-examples uses this to highlight function calls
struct Decorations {
starts: Vec<(u32, &'static str)>,
ends: Vec<u32>,
}
impl Decorations {
fn new(info: DecorationInfo) -> Self {
let (starts, ends) = info
.into_iter()
.map(|(kind, ranges)| ranges.into_iter().map(move |(lo, hi)| ((lo, kind), hi)))
.flatten()
.unzip();
Decorations { starts, ends }
}
}
@ -259,12 +292,18 @@ struct Classifier<'a> {
byte_pos: u32,
file_span: Span,
src: &'a str,
decorations: Option<Decorations>,
}
impl<'a> Classifier<'a> {
/// Takes as argument the source code to HTML-ify, the rust edition to use and the source code
/// file span which will be used later on by the `span_correspondance_map`.
fn new(src: &str, edition: Edition, file_span: Span) -> Classifier<'_> {
fn new(
src: &str,
edition: Edition,
file_span: Span,
decoration_info: Option<DecorationInfo>,
) -> Classifier<'_> {
let tokens = PeekIter::new(TokenIter { src });
Classifier {
tokens,
@ -275,6 +314,7 @@ impl<'a> Classifier<'a> {
byte_pos: 0,
file_span,
src,
decorations,
}
}
@ -356,6 +396,19 @@ impl<'a> Classifier<'a> {
/// token is used.
fn highlight(mut self, sink: &mut dyn FnMut(Highlight<'a>)) {
loop {
if let Some(decs) = self.decorations.as_mut() {
let byte_pos = self.byte_pos;
let n_starts = decs.starts.iter().filter(|(i, _)| byte_pos >= *i).count();
for (_, kind) in decs.starts.drain(0..n_starts) {
sink(Highlight::EnterSpan { class: Class::Decoration(kind) });
}
let n_ends = decs.ends.iter().filter(|i| byte_pos >= **i).count();
for _ in decs.ends.drain(0..n_ends) {
sink(Highlight::ExitSpan);
}
}
if self
.tokens
.peek()

View file

@ -22,7 +22,7 @@ fn test_html_highlighting() {
let src = include_str!("fixtures/sample.rs");
let html = {
let mut out = Buffer::new();
write_code(&mut out, src, Edition::Edition2018, None);
write_code(&mut out, src, Edition::Edition2018, None, None);
format!("{}<pre><code>{}</code></pre>\n", STYLE, out.into_inner())
};
expect_file!["fixtures/sample.html"].assert_eq(&html);
@ -36,7 +36,7 @@ fn test_dos_backline() {
println!(\"foo\");\r\n\
}\r\n";
let mut html = Buffer::new();
write_code(&mut html, src, Edition::Edition2018, None);
write_code(&mut html, src, Edition::Edition2018, None, None);
expect_file!["fixtures/dos_line.html"].assert_eq(&html.into_inner());
});
}
@ -50,7 +50,7 @@ let x = super::b::foo;
let y = Self::whatever;";
let mut html = Buffer::new();
write_code(&mut html, src, Edition::Edition2018, None);
write_code(&mut html, src, Edition::Edition2018, None, None);
expect_file!["fixtures/highlight.html"].assert_eq(&html.into_inner());
});
}

View file

@ -360,6 +360,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'_, 'a, I> {
edition,
None,
None,
None,
);
Some(Event::Html(s.into_inner().into()))
}

View file

@ -46,7 +46,7 @@ use std::string::ToString;
use rustc_ast_pretty::pprust;
use rustc_attr::{ConstStability, Deprecation, StabilityLevel};
use rustc_data_structures::fx::FxHashSet;
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_hir as hir;
use rustc_hir::def::CtorKind;
use rustc_hir::def_id::DefId;
@ -2496,23 +2496,28 @@ fn render_call_locations(
// To reduce file sizes, we only want to embed the source code needed to understand the example, not
// the entire file. So we find the smallest byte range that covers all items enclosing examples.
assert!(call_data.locations.len() > 0);
let min_loc =
call_data.locations.iter().min_by_key(|loc| loc.enclosing_item_span.0).unwrap();
let min_byte = min_loc.enclosing_item_span.0;
let min_line = min_loc.enclosing_item_lines.0;
call_data.locations.iter().min_by_key(|loc| loc.enclosing_item.byte_span.0).unwrap();
let min_byte = min_loc.enclosing_item.byte_span.0;
let min_line = min_loc.enclosing_item.line_span.0;
let max_byte =
call_data.locations.iter().map(|loc| loc.enclosing_item_span.1).max().unwrap();
call_data.locations.iter().map(|loc| loc.enclosing_item.byte_span.1).max().unwrap();
// The output code is limited to that byte range.
let contents_subset = &contents[min_byte..max_byte];
let contents_subset = &contents[(min_byte as usize)..(max_byte as usize)];
// The call locations need to be updated to reflect that the size of the program has changed.
// Specifically, the ranges are all subtracted by `min_byte` since that's the new zero point.
let locations = call_data
let (byte_ranges, line_ranges): (Vec<_>, Vec<_>) = call_data
.locations
.iter()
.map(|loc| (loc.call_span.0 - min_byte, loc.call_span.1 - min_byte))
.collect::<Vec<_>>();
.map(|loc| {
let (byte_lo, byte_hi) = loc.call_expr.byte_span;
let (line_lo, line_hi) = loc.call_expr.line_span;
((byte_lo - min_byte, byte_hi - min_byte), (line_lo - min_line, line_hi - min_line))
})
.unzip();
let edition = cx.shared.edition();
write!(
@ -2524,7 +2529,7 @@ fn render_call_locations(
// The code and locations are encoded as data attributes, so they can be read
// later by the JS for interactions.
code = contents_subset.replace("\"", "&quot;"),
locations = serde_json::to_string(&locations).unwrap(),
locations = serde_json::to_string(&line_ranges).unwrap(),
);
write!(w, r#"<span class="prev">&pr;</span> <span class="next">&sc;</span>"#);
write!(w, r#"<span class="expand">&varr;</span>"#);
@ -2532,7 +2537,18 @@ fn render_call_locations(
// FIXME(wcrichto): where should file_span and root_path come from?
let file_span = rustc_span::DUMMY_SP;
let root_path = "".to_string();
sources::print_src(w, contents_subset, edition, file_span, cx, &root_path, Some(min_line));
let mut decoration_info = FxHashMap::default();
decoration_info.insert("highlight", byte_ranges);
sources::print_src(
w,
contents_subset,
edition,
file_span,
cx,
&root_path,
Some(min_line),
Some(decoration_info),
);
write!(w, "</div></div>");
};
@ -2542,7 +2558,8 @@ fn render_call_locations(
// understand at a glance.
let ordered_locations = {
let sort_criterion = |(_, call_data): &(_, &CallData)| {
let (lo, hi) = call_data.locations[0].enclosing_item_span;
// Use the first location because that's what the user will see initially
let (lo, hi) = call_data.locations[0].enclosing_item.byte_span;
hi - lo
};

View file

@ -1117,6 +1117,7 @@ fn item_macro(w: &mut Buffer, cx: &Context<'_>, it: &clean::Item, t: &clean::Mac
it.span(cx.tcx()).inner().edition(),
None,
None,
None,
);
});
document(w, cx, it, None, HeadingOffset::H2)

View file

@ -212,6 +212,7 @@ impl SourceCollector<'_, 'tcx> {
&self.cx,
&root_path,
None,
None,
)
},
&self.cx.shared.style_files,
@ -259,6 +260,7 @@ crate fn print_src(
context: &Context<'_>,
root_path: &str,
offset: Option<usize>,
decoration_info: Option<highlight::DecorationInfo>,
) {
let lines = s.lines().count();
let mut line_numbers = Buffer::empty_from(buf);
@ -283,5 +285,6 @@ crate fn print_src(
edition,
Some(line_numbers),
Some(highlight::ContextInfo { context, file_span, root_path }),
decoration_info,
);
}

View file

@ -1971,7 +1971,8 @@ details.undocumented[open] > summary::before {
}
}
/* This part is for the new "examples" components */
/* Begin: styles for --scrape-examples feature */
.scraped-example-title {
font-family: 'Fira Sans';
@ -2063,16 +2064,17 @@ details.undocumented[open] > summary::before {
overflow-y: hidden;
}
.scraped-example .line-numbers span.highlight {
background: #f6fdb0;
.scraped-example .example-wrap .rust span.highlight {
background: #fcffd6;
}
.scraped-example .example-wrap .rust span.highlight {
.scraped-example .example-wrap .rust span.highlight.focus {
background: #f6fdb0;
}
.more-examples-toggle summary {
color: #999;
font-family: 'Fira Sans';
}
.more-scraped-examples {
@ -2115,3 +2117,5 @@ h1 + .scraped-example {
.example-links ul {
margin-bottom: 0;
}
/* End: styles for --scrape-examples feature */

View file

@ -980,154 +980,55 @@ function hideThemeButtonState() {
window.addEventListener("hashchange", onHashChange);
searchState.setup();
/////// EXAMPLE ANALYZER
// Merge the full set of [from, to] offsets into a minimal set of non-overlapping
// [from, to] offsets.
// NB: This is such a archetypal software engineering interview question that
// I can't believe I actually had to write it. Yes, it's O(N) in the input length --
// but it does assume a sorted input!
function distinctRegions(locs) {
var start = -1;
var end = -1;
var output = [];
for (var i = 0; i < locs.length; i++) {
var loc = locs[i];
if (loc[0] > end) {
if (end > 0) {
output.push([start, end]);
}
start = loc[0];
end = loc[1];
} else {
end = Math.max(end, loc[1]);
}
}
if (end > 0) {
output.push([start, end]);
}
return output;
}
function convertLocsStartsToLineOffsets(code, locs) {
locs = distinctRegions(locs.slice(0).sort(function (a, b) {
return a[0] === b[0] ? a[1] - b[1] : a[0] - b[0];
})); // sort by start; use end if start is equal.
var codeLines = code.split("\n");
var lineIndex = 0;
var totalOffset = 0;
var output = [];
while (locs.length > 0 && lineIndex < codeLines.length) {
// +1 here and later is due to omitted \n
var lineLength = codeLines[lineIndex].length + 1;
while (locs.length > 0 && totalOffset + lineLength > locs[0][0]) {
var endIndex = lineIndex;
var charsRemaining = locs[0][1] - totalOffset;
while (endIndex < codeLines.length &&
charsRemaining > codeLines[endIndex].length + 1)
{
charsRemaining -= codeLines[endIndex].length + 1;
endIndex += 1;
}
output.push({
from: [lineIndex, locs[0][0] - totalOffset],
to: [endIndex, charsRemaining]
});
locs.shift();
}
lineIndex++;
totalOffset += lineLength;
}
return output;
}
// inserts str into html, *but* calculates idx by eliding anything in html that's not in raw.
// ideally this would work by walking the element tree...but this is good enough for now.
function insertStrAtRawIndex(raw, html, idx, str) {
if (idx > raw.length) {
return html;
}
if (idx == raw.length) {
return html + str;
}
var rawIdx = 0;
var htmlIdx = 0;
while (rawIdx < idx && rawIdx < raw.length) {
while (raw[rawIdx] !== html[htmlIdx] && htmlIdx < html.length) {
htmlIdx++;
}
rawIdx++;
htmlIdx++;
}
return html.substring(0, htmlIdx) + str + html.substr(htmlIdx);
}
/////// --scrape-examples interactions
// Scroll code block to put the given code location in the middle of the viewer
function scrollToLoc(elt, loc) {
var wrapper = elt.querySelector(".code-wrapper");
var halfHeight = wrapper.offsetHeight / 2;
var lines = elt.querySelector('.line-numbers');
var offsetMid = (lines.children[loc.from[0]].offsetTop
+ lines.children[loc.to[0]].offsetTop) / 2;
var offsetMid = (lines.children[loc[0]].offsetTop
+ lines.children[loc[1]].offsetTop) / 2;
var scrollOffset = offsetMid - halfHeight;
lines.scrollTo(0, scrollOffset);
elt.querySelector(".rust").scrollTo(0, scrollOffset);
}
function updateScrapedExample(example) {
var code = example.attributes.getNamedItem("data-code").textContent;
var codeLines = code.split("\n");
var locs = JSON.parse(example.attributes.getNamedItem("data-locs").textContent);
locs = convertLocsStartsToLineOffsets(code, locs);
// Add call-site highlights to code listings
var litParent = example.querySelector('.example-wrap pre.rust');
var litHtml = litParent.innerHTML.split("\n");
onEach(locs, function (loc) {
for (var i = loc.from[0]; i < loc.to[0] + 1; i++) {
addClass(example.querySelector('.line-numbers').children[i], "highlight");
}
litHtml[loc.to[0]] = insertStrAtRawIndex(
codeLines[loc.to[0]],
litHtml[loc.to[0]],
loc.to[1],
"</span>");
litHtml[loc.from[0]] = insertStrAtRawIndex(
codeLines[loc.from[0]],
litHtml[loc.from[0]],
loc.from[1],
'<span class="highlight" data-loc="' +
JSON.stringify(loc).replace(/"/g, "&quot;") +
'">');
}, true); // do this backwards to avoid shifting later offsets
litParent.innerHTML = litHtml.join('\n');
// Toggle through list of examples in a given file
var locIndex = 0;
var highlights = example.querySelectorAll('.highlight');
addClass(highlights[0], 'focus');
if (locs.length > 1) {
// Toggle through list of examples in a given file
var onChangeLoc = function(f) {
removeClass(highlights[locIndex], 'focus');
f();
scrollToLoc(example, locs[locIndex]);
addClass(highlights[locIndex], 'focus');
};
example.querySelector('.prev')
.addEventListener('click', function() {
onChangeLoc(function() {
locIndex = (locIndex - 1 + locs.length) % locs.length;
scrollToLoc(example, locs[locIndex]);
});
});
example.querySelector('.next')
.addEventListener('click', function() {
locIndex = (locIndex + 1) % locs.length;
scrollToLoc(example, locs[locIndex]);
onChangeLoc(function() { locIndex = (locIndex + 1) % locs.length; });
});
} else {
// Remove buttons if there's only one example in the file
example.querySelector('.prev').remove();
example.querySelector('.next').remove();
}
let codeEl = example.querySelector('.rust');
let expandButton = example.querySelector('.expand');
if (codeEl.scrollHeight == codeEl.clientHeight) {
addClass(example, 'expanded');
expandButton.remove();
} else {
// Show full code on expansion
var codeEl = example.querySelector('.rust');
var codeOverflows = codeEl.scrollHeight > codeEl.clientHeight;
var expandButton = example.querySelector('.expand');
if (codeOverflows) {
// If file is larger than default height, give option to expand the viewer
expandButton.addEventListener('click', function () {
if (hasClass(example, "expanded")) {
removeClass(example, "expanded");
@ -1136,6 +1037,10 @@ function hideThemeButtonState() {
addClass(example, "expanded");
}
});
} else {
// Otherwise remove expansion buttons
addClass(example, 'expanded');
expandButton.remove();
}
// Start with the first example in view
@ -1146,6 +1051,8 @@ function hideThemeButtonState() {
var firstExamples = document.querySelectorAll('.scraped-example-list > .scraped-example');
onEach(firstExamples, updateScrapedExample);
onEach(document.querySelectorAll('.more-examples-toggle'), function(toggle) {
// Allow users to click the left border of the <details> section to close it,
// since the section can be large and finding the [+] button is annoying.
toggle.querySelector('.toggle-line').addEventListener('click', function() {
toggle.open = false;
});

View file

@ -1,5 +1,4 @@
//! This module analyzes crates to find examples of uses for items in the
//! current crate being documented.
//! This module analyzes crates to find call sites that can serve as examples in the documentation.
use crate::clean;
use crate::config;
@ -11,20 +10,55 @@ use rustc_data_structures::fx::FxHashMap;
use rustc_hir::{
self as hir,
intravisit::{self, Visitor},
HirId,
};
use rustc_interface::interface;
use rustc_middle::hir::map::Map;
use rustc_middle::ty::{self, TyCtxt};
use rustc_span::{def_id::DefId, FileName};
use rustc_span::{def_id::DefId, BytePos, FileName, SourceFile};
use serde::{Deserialize, Serialize};
use std::fs;
use std::path::PathBuf;
#[derive(Serialize, Deserialize, Debug, Clone)]
crate struct SyntaxRange {
crate byte_span: (u32, u32),
crate line_span: (usize, usize),
}
impl SyntaxRange {
fn new(span: rustc_span::Span, file: &SourceFile) -> Self {
let get_pos = |bytepos: BytePos| file.original_relative_byte_pos(bytepos).0;
let get_line = |bytepos: BytePos| file.lookup_line(bytepos).unwrap();
SyntaxRange {
byte_span: (get_pos(span.lo()), get_pos(span.hi())),
line_span: (get_line(span.lo()), get_line(span.hi())),
}
}
}
#[derive(Serialize, Deserialize, Debug, Clone)]
crate struct CallLocation {
crate call_span: (usize, usize),
crate enclosing_item_span: (usize, usize),
crate enclosing_item_lines: (usize, usize),
crate call_expr: SyntaxRange,
crate enclosing_item: SyntaxRange,
}
impl CallLocation {
fn new(
tcx: TyCtxt<'_>,
expr_span: rustc_span::Span,
expr_id: HirId,
source_file: &rustc_span::SourceFile,
) -> Self {
let enclosing_item_span = tcx.hir().span_with_body(tcx.hir().get_parent_item(expr_id));
assert!(enclosing_item_span.contains(expr_span));
CallLocation {
call_expr: SyntaxRange::new(expr_span, source_file),
enclosing_item: SyntaxRange::new(enclosing_item_span, source_file),
}
}
}
#[derive(Serialize, Deserialize, Debug, Clone)]
@ -96,24 +130,10 @@ where
_ => None,
};
let get_pos =
|bytepos: rustc_span::BytePos| file.original_relative_byte_pos(bytepos).0 as usize;
let get_range = |span: rustc_span::Span| (get_pos(span.lo()), get_pos(span.hi()));
let get_line = |bytepos: rustc_span::BytePos| file.lookup_line(bytepos).unwrap();
let get_lines = |span: rustc_span::Span| (get_line(span.lo()), get_line(span.hi()));
if let Some(file_path) = file_path {
let abs_path = fs::canonicalize(file_path.clone()).unwrap();
let cx = &self.cx;
let enclosing_item_span =
self.tcx.hir().span_with_body(self.tcx.hir().get_parent_item(ex.hir_id));
assert!(enclosing_item_span.contains(span));
let location = CallLocation {
call_span: get_range(span),
enclosing_item_span: get_range(enclosing_item_span),
enclosing_item_lines: get_lines(enclosing_item_span),
};
let location = CallLocation::new(self.tcx, span, ex.hir_id, &file);
entries
.entry(abs_path)