bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Rollup merge of #67702 - crlf0710:normalize_ident2, r=petrochenkov

Browse source 
Add symbol normalization for proc_macro_server.

Follow up for #66670, finishing the first bullet point in #55467.

r? @petrochenkov
This commit is contained in:
Mazdak Farrokhzad 2019-12-31 19:19:34 +01:00 • committed by GitHub
commit bc5963d2eb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 14 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/librustc_expand/proc_macro_server.rs
View file

@ -1,5 +1,6 @@
use crate::base::ExtCtxt;
use rustc_parse::lexer::nfc_normalize;
use rustc_parse::{nt_to_tokenstream, parse_stream_from_source_str};
use syntax::ast;
use syntax::print::pprust;
@ -327,6 +328,7 @@ impl Ident {
}
}
fn new(sym: Symbol, is_raw: bool, span: Span) -> Ident {
let sym = nfc_normalize(&sym.as_str());
let string = sym.as_str();
if !Self::is_valid(&string) {
panic!("`{:?}` is not a valid identifier", string)

27
src/librustc_parse/lexer/mod.rs
View file

@ -220,7 +220,7 @@ impl<'a> StringReader<'a> {
if is_raw_ident {
ident_start = ident_start + BytePos(2);
}
let sym = self.nfc_symbol_from(ident_start);
let sym = nfc_normalize(self.str_from(ident_start));
if is_raw_ident {
let span = self.mk_sp(start, self.pos);
if !sym.can_be_raw() {
@ -469,20 +469,6 @@ impl<'a> StringReader<'a> {
Symbol::intern(self.str_from_to(start, end))
}
/// As symbol_from, with the text normalized into Unicode NFC form.
fn nfc_symbol_from(&self, start: BytePos) -> Symbol {
use unicode_normalization::{is_nfc_quick, IsNormalized, UnicodeNormalization};
debug!("taking an normalized ident from {:?} to {:?}", start, self.pos);
let sym = self.str_from(start);
match is_nfc_quick(sym.chars()) {
IsNormalized::Yes => Symbol::intern(sym),
_ => {
let sym_str: String = sym.chars().nfc().collect();
Symbol::intern(&sym_str)
}
}
}
/// Slice of the source text spanning from `start` up to but excluding `end`.
fn str_from_to(&self, start: BytePos, end: BytePos) -> &str {
&self.src[self.src_index(start)..self.src_index(end)]
@ -651,3 +637,14 @@ impl<'a> StringReader<'a> {
}
}
}
pub fn nfc_normalize(string: &str) -> Symbol {
use unicode_normalization::{is_nfc_quick, IsNormalized, UnicodeNormalization};
match is_nfc_quick(string.chars()) {
IsNormalized::Yes => Symbol::intern(string),
_ => {
let normalized_str: String = string.chars().nfc().collect();
Symbol::intern(&normalized_str)
}
}
}