bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Add newtype for raw idents

Browse source
This commit is contained in:
clubby789 2024-02-13 23:28:27 +00:00
parent cce6a6e22e
commit 06d6c62f80
24 changed files with 148 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

4
compiler/rustc_expand/src/mbe/macro_check.rs
View file

@ -107,7 +107,7 @@
use crate::errors;
use crate::mbe::{KleeneToken, TokenTree};
use rustc_ast::token::{Delimiter, Token, TokenKind};
use rustc_ast::token::{Delimiter, IdentIsRaw, Token, TokenKind};
use rustc_ast::{NodeId, DUMMY_NODE_ID};
use rustc_data_structures::fx::FxHashMap;
use rustc_errors::{DiagnosticMessage, MultiSpan};
@ -409,7 +409,7 @@ fn check_nested_occurrences(
match (state, tt) {
(
NestedMacroState::Empty,
&TokenTree::Token(Token { kind: TokenKind::Ident(name, false), .. }),
&TokenTree::Token(Token { kind: TokenKind::Ident(name, IdentIsRaw::No), .. }),
) => {
if name == kw::MacroRules {
state = NestedMacroState::MacroRules;

14
compiler/rustc_expand/src/mbe/macro_rules.rs
View file

@ -8,6 +8,7 @@ use crate::mbe::macro_parser::{Error, ErrorReported, Failure, Success, TtParser}
use crate::mbe::macro_parser::{MatchedSeq, MatchedTokenTree, MatcherLoc};
use crate::mbe::transcribe::transcribe;
use ast::token::IdentIsRaw;
use rustc_ast as ast;
use rustc_ast::token::{self, Delimiter, NonterminalKind, Token, TokenKind, TokenKind::*};
use rustc_ast::tokenstream::{DelimSpan, TokenStream};
@ -1302,7 +1303,9 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
match tok {
TokenTree::Token(token) => match token.kind {
FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes,
Ident(name, false) if name == kw::If || name == kw::In => IsInFollow::Yes,
Ident(name, IdentIsRaw::No) if name == kw::If || name == kw::In => {
IsInFollow::Yes
}
_ => IsInFollow::No(TOKENS),
},
_ => IsInFollow::No(TOKENS),
@ -1313,7 +1316,9 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
match tok {
TokenTree::Token(token) => match token.kind {
FatArrow | Comma | Eq => IsInFollow::Yes,
Ident(name, false) if name == kw::If || name == kw::In => IsInFollow::Yes,
Ident(name, IdentIsRaw::No) if name == kw::If || name == kw::In => {
IsInFollow::Yes
}
_ => IsInFollow::No(TOKENS),
},
_ => IsInFollow::No(TOKENS),
@ -1336,7 +1341,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
| BinOp(token::Shr)
| Semi
| BinOp(token::Or) => IsInFollow::Yes,
Ident(name, false) if name == kw::As || name == kw::Where => {
Ident(name, IdentIsRaw::No) if name == kw::As || name == kw::Where => {
IsInFollow::Yes
}
_ => IsInFollow::No(TOKENS),
@ -1364,7 +1369,8 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
match tok {
TokenTree::Token(token) => match token.kind {
Comma => IsInFollow::Yes,
Ident(name, is_raw) if is_raw || name != kw::Priv => IsInFollow::Yes,
Ident(_, IdentIsRaw::Yes) => IsInFollow::Yes,
Ident(name, _) if name != kw::Priv => IsInFollow::Yes,
_ => {
if token.can_begin_type() {
IsInFollow::Yes

4
compiler/rustc_expand/src/mbe/metavar_expr.rs
View file

@ -1,4 +1,4 @@
use rustc_ast::token::{self, Delimiter};
use rustc_ast::token::{self, Delimiter, IdentIsRaw};
use rustc_ast::tokenstream::{RefTokenTreeCursor, TokenStream, TokenTree};
use rustc_ast::{LitIntType, LitKind};
use rustc_ast_pretty::pprust;
@ -142,7 +142,7 @@ fn parse_ident<'sess>(
if let Some(tt) = iter.next()
&& let TokenTree::Token(token, _) = tt
{
if let Some((elem, false)) = token.ident() {
if let Some((elem, IdentIsRaw::No)) = token.ident() {
return Ok(elem);
}
let token_str = pprust::token_to_string(token);

4
compiler/rustc_expand/src/mbe/quoted.rs
View file

@ -2,7 +2,7 @@ use crate::errors;
use crate::mbe::macro_parser::count_metavar_decls;
use crate::mbe::{Delimited, KleeneOp, KleeneToken, MetaVarExpr, SequenceRepetition, TokenTree};
use rustc_ast::token::{self, Delimiter, Token};
use rustc_ast::token::{self, Delimiter, IdentIsRaw, Token};
use rustc_ast::{tokenstream, NodeId};
use rustc_ast_pretty::pprust;
use rustc_feature::Features;
@ -222,7 +222,7 @@ fn parse_tree<'a>(
Some(tokenstream::TokenTree::Token(token, _)) if token.is_ident() => {
let (ident, is_raw) = token.ident().unwrap();
let span = ident.span.with_lo(span.lo());
if ident.name == kw::Crate && !is_raw {
if ident.name == kw::Crate && matches!(is_raw, IdentIsRaw::No) {
TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span)
} else {
TokenTree::MetaVar(span, ident)

40
compiler/rustc_expand/src/parse/tests.rs
View file

@ -2,6 +2,7 @@ use crate::tests::{
matches_codepattern, string_to_stream, with_error_checking_parse, with_expected_parse_error,
};
use ast::token::IdentIsRaw;
use rustc_ast::ptr::P;
use rustc_ast::token::{self, Delimiter, Token};
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
@ -74,9 +75,12 @@ fn string_to_tts_macro() {
match tts {
[
TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }, _),
TokenTree::Token(
Token { kind: token::Ident(name_macro_rules, IdentIsRaw::No), .. },
_,
),
TokenTree::Token(Token { kind: token::Not, .. }, _),
TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }, _),
TokenTree::Token(Token { kind: token::Ident(name_zip, IdentIsRaw::No), .. }, _),
TokenTree::Delimited(.., macro_delim, macro_tts),
] if name_macro_rules == &kw::MacroRules && name_zip.as_str() == "zip" => {
let tts = &macro_tts.trees().collect::<Vec<_>>();
@ -90,7 +94,10 @@ fn string_to_tts_macro() {
match &tts[..] {
[
TokenTree::Token(Token { kind: token::Dollar, .. }, _),
TokenTree::Token(Token { kind: token::Ident(name, false), .. }, _),
TokenTree::Token(
Token { kind: token::Ident(name, IdentIsRaw::No), .. },
_,
),
] if first_delim == &Delimiter::Parenthesis && name.as_str() == "a" => {
}
_ => panic!("value 3: {:?} {:?}", first_delim, first_tts),
@ -99,7 +106,10 @@ fn string_to_tts_macro() {
match &tts[..] {
[
TokenTree::Token(Token { kind: token::Dollar, .. }, _),
TokenTree::Token(Token { kind: token::Ident(name, false), .. }, _),
TokenTree::Token(
Token { kind: token::Ident(name, IdentIsRaw::No), .. },
_,
),
] if second_delim == &Delimiter::Parenthesis
&& name.as_str() == "a" => {}
_ => panic!("value 4: {:?} {:?}", second_delim, second_tts),
@ -119,8 +129,11 @@ fn string_to_tts_1() {
let tts = string_to_stream("fn a(b: i32) { b; }".to_string());
let expected = TokenStream::new(vec![
TokenTree::token_alone(token::Ident(kw::Fn, false), sp(0, 2)),
TokenTree::token_joint_hidden(token::Ident(Symbol::intern("a"), false), sp(3, 4)),
TokenTree::token_alone(token::Ident(kw::Fn, IdentIsRaw::No), sp(0, 2)),
TokenTree::token_joint_hidden(
token::Ident(Symbol::intern("a"), IdentIsRaw::No),
sp(3, 4),
),
TokenTree::Delimited(
DelimSpan::from_pair(sp(4, 5), sp(11, 12)),
// `JointHidden` because the `(` is followed immediately by
@ -128,10 +141,16 @@ fn string_to_tts_1() {
DelimSpacing::new(Spacing::JointHidden, Spacing::Alone),
Delimiter::Parenthesis,
TokenStream::new(vec![
TokenTree::token_joint(token::Ident(Symbol::intern("b"), false), sp(5, 6)),
TokenTree::token_joint(
token::Ident(Symbol::intern("b"), IdentIsRaw::No),
sp(5, 6),
),
TokenTree::token_alone(token::Colon, sp(6, 7)),
// `JointHidden` because the `i32` is immediately followed by the `)`.
TokenTree::token_joint_hidden(token::Ident(sym::i32, false), sp(8, 11)),
TokenTree::token_joint_hidden(
token::Ident(sym::i32, IdentIsRaw::No),
sp(8, 11),
),
])
.into(),
),
@ -143,7 +162,10 @@ fn string_to_tts_1() {
DelimSpacing::new(Spacing::Alone, Spacing::Alone),
Delimiter::Brace,
TokenStream::new(vec![
TokenTree::token_joint(token::Ident(Symbol::intern("b"), false), sp(15, 16)),
TokenTree::token_joint(
token::Ident(Symbol::intern("b"), IdentIsRaw::No),
sp(15, 16),
),
// `Alone` because the `;` is followed by whitespace.
TokenTree::token_alone(token::Semi, sp(16, 17)),
])

13
compiler/rustc_expand/src/proc_macro_server.rs
View file

@ -1,4 +1,5 @@
use crate::base::ExtCtxt;
use ast::token::IdentIsRaw;
use pm::bridge::{
server, DelimSpan, Diagnostic, ExpnGlobals, Group, Ident, LitKind, Literal, Punct, TokenTree,
};
@ -216,7 +217,9 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
Question => op("?"),
SingleQuote => op("'"),
Ident(sym, is_raw) => trees.push(TokenTree::Ident(Ident { sym, is_raw, span })),
Ident(sym, is_raw) => {
trees.push(TokenTree::Ident(Ident { sym, is_raw: is_raw.into(), span }))
}
Lifetime(name) => {
let ident = symbol::Ident::new(name, span).without_first_quote();
trees.extend([
@ -238,7 +241,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
escaped.extend(ch.escape_debug());
}
let stream = [
Ident(sym::doc, false),
Ident(sym::doc, IdentIsRaw::No),
Eq,
TokenKind::lit(token::Str, Symbol::intern(&escaped), None),
]
@ -259,7 +262,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
Interpolated(ref nt) if let NtIdent(ident, is_raw) = &nt.0 => {
trees.push(TokenTree::Ident(Ident {
sym: ident.name,
is_raw: *is_raw,
is_raw: matches!(is_raw, IdentIsRaw::Yes),
span: ident.span,
}))
}
@ -352,7 +355,7 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
}
TokenTree::Ident(self::Ident { sym, is_raw, span }) => {
rustc.sess().symbol_gallery.insert(sym, span);
smallvec![tokenstream::TokenTree::token_alone(Ident(sym, is_raw), span)]
smallvec![tokenstream::TokenTree::token_alone(Ident(sym, is_raw.into()), span)]
}
TokenTree::Literal(self::Literal {
kind: self::LitKind::Integer,
@ -569,7 +572,7 @@ impl server::TokenStream for Rustc<'_, '_> {
match &expr.kind {
ast::ExprKind::Lit(token_lit) if token_lit.kind == token::Bool => {
Ok(tokenstream::TokenStream::token_alone(
token::Ident(token_lit.symbol, false),
token::Ident(token_lit.symbol, IdentIsRaw::No),
expr.span,
))
}

5
compiler/rustc_expand/src/tokenstream/tests.rs
View file

@ -1,6 +1,6 @@
use crate::tests::string_to_stream;
use rustc_ast::token;
use rustc_ast::token::{self, IdentIsRaw};
use rustc_ast::tokenstream::{TokenStream, TokenTree};
use rustc_span::create_default_session_globals_then;
use rustc_span::{BytePos, Span, Symbol};
@ -86,7 +86,8 @@ fn test_diseq_1() {
fn test_is_empty() {
create_default_session_globals_then(|| {
let test0 = TokenStream::default();
let test1 = TokenStream::token_alone(token::Ident(Symbol::intern("a"), false), sp(0, 1));
let test1 =
TokenStream::token_alone(token::Ident(Symbol::intern("a"), IdentIsRaw::No), sp(0, 1));
let test2 = string_to_ts("foo(bar::baz)");
assert_eq!(test0.is_empty(), true);