Auto merge of #49698 - SimonSapin:unicode-for-everyone, r=alexcrichton
Merge the std_unicode crate into the core crate [The standard library facade](https://github.com/rust-lang/rust/issues/27783) has historically contained a number of crates with different roles, but that number has decreased over time. `rand` and `libc` have moved to crates.io, and [`collections` was merged into `alloc`](https://github.com/rust-lang/rust/pull/42648). Today we have `core` that applies everywhere, `std` that expects a full operating system, and `alloc` in-between that only requires a memory allocator (which can be provided by users)… and `std_unicode`, which doesn’t really have a reason to be separate anymore. It contains functionality based on Unicode data tables that can be large, but as long as relevant functions are not called the tables should be removed from binaries by linkers. This deprecates the unstable `std_unicode` crate and moves all of its contents into `core`, replacing them with `pub use` reexports. The crate can be removed later. This also removes the `CharExt` trait (replaced with inherent methods in libcore) and `UnicodeStr` trait (merged into `StrExt`). There traits were both unstable and not intended to be used or named directly. A number of new items are newly-available in libcore and instantly stable there, but only if they were already stable in libstd. Fixes #49319.
This commit is contained in:
bors
commit
d26f9e42df
42 changed files with 1398 additions and 1537 deletions
1
src/Cargo.lock
generated
1
src/Cargo.lock
generated
|
|
@ -13,7 +13,6 @@ dependencies = [
|
|||
"compiler_builtins 0.0.0",
|
||||
"core 0.0.0",
|
||||
"rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"std_unicode 0.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
|
|
@ -34,4 +34,3 @@ ENV SCRIPT python2.7 /checkout/x.py test --target $TARGETS \
|
|||
src/test/mir-opt \
|
||||
src/test/codegen-units \
|
||||
src/libcore \
|
||||
src/libstd_unicode/ \
|
||||
|
|
|
|||
|
|
@ -243,7 +243,7 @@ the source code.
|
|||
- `usize`: `libcore/num/mod.rs`
|
||||
- `f32`: `libstd/f32.rs`
|
||||
- `f64`: `libstd/f64.rs`
|
||||
- `char`: `libstd_unicode/char.rs`
|
||||
- `char`: `libcore/char.rs`
|
||||
- `slice`: `liballoc/slice.rs`
|
||||
- `str`: `liballoc/str.rs`
|
||||
- `const_ptr`: `libcore/ptr.rs`
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ path = "lib.rs"
|
|||
|
||||
[dependencies]
|
||||
core = { path = "../libcore" }
|
||||
std_unicode = { path = "../libstd_unicode" }
|
||||
compiler_builtins = { path = "../rustc/compiler_builtins_shim" }
|
||||
|
||||
[dev-dependencies]
|
||||
|
|
|
|||
|
|
@ -113,7 +113,7 @@
|
|||
#![feature(trusted_len)]
|
||||
#![feature(try_reserve)]
|
||||
#![feature(unboxed_closures)]
|
||||
#![feature(unicode)]
|
||||
#![feature(unicode_internals)]
|
||||
#![feature(unsize)]
|
||||
#![feature(allocator_internals)]
|
||||
#![feature(on_unimplemented)]
|
||||
|
|
@ -135,8 +135,6 @@ extern crate test;
|
|||
#[cfg(test)]
|
||||
extern crate rand;
|
||||
|
||||
extern crate std_unicode;
|
||||
|
||||
// Module with internal macros used by other modules (needs to be included before other modules).
|
||||
#[macro_use]
|
||||
mod macros;
|
||||
|
|
|
|||
|
|
@ -45,12 +45,10 @@ use core::str::pattern::{Searcher, ReverseSearcher, DoubleEndedSearcher};
|
|||
use core::mem;
|
||||
use core::ptr;
|
||||
use core::iter::FusedIterator;
|
||||
use std_unicode::str::{UnicodeStr, Utf16Encoder};
|
||||
|
||||
use vec_deque::VecDeque;
|
||||
use borrow::{Borrow, ToOwned};
|
||||
use string::String;
|
||||
use std_unicode;
|
||||
use vec::Vec;
|
||||
use slice::{SliceConcatExt, SliceIndex};
|
||||
use boxed::Box;
|
||||
|
|
@ -75,7 +73,7 @@ pub use core::str::{from_utf8, from_utf8_mut, Chars, CharIndices, Bytes};
|
|||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub use core::str::{from_utf8_unchecked, from_utf8_unchecked_mut, ParseBoolError};
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub use std_unicode::str::SplitWhitespace;
|
||||
pub use core::str::SplitWhitespace;
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub use core::str::pattern;
|
||||
|
||||
|
|
@ -147,7 +145,8 @@ impl<S: Borrow<str>> SliceConcatExt<str> for [S] {
|
|||
#[derive(Clone)]
|
||||
#[stable(feature = "encode_utf16", since = "1.8.0")]
|
||||
pub struct EncodeUtf16<'a> {
|
||||
encoder: Utf16Encoder<Chars<'a>>,
|
||||
chars: Chars<'a>,
|
||||
extra: u16,
|
||||
}
|
||||
|
||||
#[stable(feature = "collection_debug", since = "1.17.0")]
|
||||
|
|
@ -163,12 +162,29 @@ impl<'a> Iterator for EncodeUtf16<'a> {
|
|||
|
||||
#[inline]
|
||||
fn next(&mut self) -> Option<u16> {
|
||||
self.encoder.next()
|
||||
if self.extra != 0 {
|
||||
let tmp = self.extra;
|
||||
self.extra = 0;
|
||||
return Some(tmp);
|
||||
}
|
||||
|
||||
let mut buf = [0; 2];
|
||||
self.chars.next().map(|ch| {
|
||||
let n = ch.encode_utf16(&mut buf).len();
|
||||
if n == 2 {
|
||||
self.extra = buf[1];
|
||||
}
|
||||
buf[0]
|
||||
})
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
self.encoder.size_hint()
|
||||
let (low, high) = self.chars.size_hint();
|
||||
// every char gets either one u16 or two u16,
|
||||
// so this iterator is between 1 or 2 times as
|
||||
// long as the underlying iterator.
|
||||
(low, high.and_then(|n| n.checked_mul(2)))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -801,7 +817,7 @@ impl str {
|
|||
#[stable(feature = "split_whitespace", since = "1.1.0")]
|
||||
#[inline]
|
||||
pub fn split_whitespace(&self) -> SplitWhitespace {
|
||||
UnicodeStr::split_whitespace(self)
|
||||
StrExt::split_whitespace(self)
|
||||
}
|
||||
|
||||
/// An iterator over the lines of a string, as string slices.
|
||||
|
|
@ -871,7 +887,7 @@ impl str {
|
|||
/// ```
|
||||
#[stable(feature = "encode_utf16", since = "1.8.0")]
|
||||
pub fn encode_utf16(&self) -> EncodeUtf16 {
|
||||
EncodeUtf16 { encoder: Utf16Encoder::new(self[..].chars()) }
|
||||
EncodeUtf16 { chars: self[..].chars(), extra: 0 }
|
||||
}
|
||||
|
||||
/// Returns `true` if the given pattern matches a sub-slice of
|
||||
|
|
@ -1571,7 +1587,7 @@ impl str {
|
|||
/// ```
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub fn trim(&self) -> &str {
|
||||
UnicodeStr::trim(self)
|
||||
StrExt::trim(self)
|
||||
}
|
||||
|
||||
/// Returns a string slice with leading whitespace removed.
|
||||
|
|
@ -1607,7 +1623,7 @@ impl str {
|
|||
/// ```
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub fn trim_left(&self) -> &str {
|
||||
UnicodeStr::trim_left(self)
|
||||
StrExt::trim_left(self)
|
||||
}
|
||||
|
||||
/// Returns a string slice with trailing whitespace removed.
|
||||
|
|
@ -1643,7 +1659,7 @@ impl str {
|
|||
/// ```
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub fn trim_right(&self) -> &str {
|
||||
UnicodeStr::trim_right(self)
|
||||
StrExt::trim_right(self)
|
||||
}
|
||||
|
||||
/// Returns a string slice with all prefixes and suffixes that match a
|
||||
|
|
@ -1960,7 +1976,7 @@ impl str {
|
|||
}
|
||||
|
||||
fn case_ignoreable_then_cased<I: Iterator<Item = char>>(iter: I) -> bool {
|
||||
use std_unicode::derived_property::{Cased, Case_Ignorable};
|
||||
use core::unicode::derived_property::{Cased, Case_Ignorable};
|
||||
match iter.skip_while(|&c| Case_Ignorable(c)).next() {
|
||||
Some(c) => Cased(c),
|
||||
None => false,
|
||||
|
|
@ -2142,7 +2158,7 @@ impl str {
|
|||
#[stable(feature = "unicode_methods_on_intrinsics", since = "1.27.0")]
|
||||
#[inline]
|
||||
pub fn is_whitespace(&self) -> bool {
|
||||
UnicodeStr::is_whitespace(self)
|
||||
StrExt::is_whitespace(self)
|
||||
}
|
||||
|
||||
/// Returns true if this `str` is entirely alphanumeric, and false otherwise.
|
||||
|
|
@ -2161,7 +2177,7 @@ impl str {
|
|||
#[stable(feature = "unicode_methods_on_intrinsics", since = "1.27.0")]
|
||||
#[inline]
|
||||
pub fn is_alphanumeric(&self) -> bool {
|
||||
UnicodeStr::is_alphanumeric(self)
|
||||
StrExt::is_alphanumeric(self)
|
||||
}
|
||||
|
||||
/// Checks if all characters in this string are within the ASCII range.
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@
|
|||
|
||||
#![stable(feature = "rust1", since = "1.0.0")]
|
||||
|
||||
use core::char::{decode_utf16, REPLACEMENT_CHARACTER};
|
||||
use core::fmt;
|
||||
use core::hash;
|
||||
use core::iter::{FromIterator, FusedIterator};
|
||||
|
|
@ -63,8 +64,7 @@ use core::ops::Bound::{Excluded, Included, Unbounded};
|
|||
use core::ops::{self, Add, AddAssign, Index, IndexMut, RangeBounds};
|
||||
use core::ptr;
|
||||
use core::str::pattern::Pattern;
|
||||
use std_unicode::lossy;
|
||||
use std_unicode::char::{decode_utf16, REPLACEMENT_CHARACTER};
|
||||
use core::str::lossy;
|
||||
|
||||
use borrow::{Cow, ToOwned};
|
||||
use str::{self, from_boxed_utf8_unchecked, FromStr, Utf8Error, Chars};
|
||||
|
|
|
|||
|
|
@ -24,12 +24,11 @@
|
|||
#![feature(string_retain)]
|
||||
#![feature(try_reserve)]
|
||||
#![feature(unboxed_closures)]
|
||||
#![feature(unicode)]
|
||||
#![feature(exact_chunks)]
|
||||
#![feature(inclusive_range_fields)]
|
||||
|
||||
extern crate alloc_system;
|
||||
extern crate std_unicode;
|
||||
extern crate core;
|
||||
extern crate rand;
|
||||
|
||||
use std::hash::{Hash, Hasher};
|
||||
|
|
|
|||
|
|
@ -1204,8 +1204,7 @@ fn test_rev_split_char_iterator_no_trailing() {
|
|||
|
||||
#[test]
|
||||
fn test_utf16_code_units() {
|
||||
use std_unicode::str::Utf16Encoder;
|
||||
assert_eq!(Utf16Encoder::new(vec!['é', '\u{1F4A9}'].into_iter()).collect::<Vec<u16>>(),
|
||||
assert_eq!("é\u{1F4A9}".encode_utf16().collect::<Vec<u16>>(),
|
||||
[0xE9, 0xD83D, 0xDCA9])
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -132,7 +132,7 @@ fn test_from_utf16() {
|
|||
let s_as_utf16 = s.encode_utf16().collect::<Vec<u16>>();
|
||||
let u_as_string = String::from_utf16(&u).unwrap();
|
||||
|
||||
assert!(::std_unicode::char::decode_utf16(u.iter().cloned()).all(|r| r.is_ok()));
|
||||
assert!(::core::char::decode_utf16(u.iter().cloned()).all(|r| r.is_ok()));
|
||||
assert_eq!(s_as_utf16, u);
|
||||
|
||||
assert_eq!(u_as_string, s);
|
||||
|
|
|
|||
|
|
@ -1,918 +0,0 @@
|
|||
// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
|
||||
// file at the top-level directory of this distribution and at
|
||||
// http://rust-lang.org/COPYRIGHT.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
||||
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
||||
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
||||
// option. This file may not be copied, modified, or distributed
|
||||
// except according to those terms.
|
||||
|
||||
//! Character manipulation.
|
||||
//!
|
||||
//! For more details, see ::std_unicode::char (a.k.a. std::char)
|
||||
|
||||
#![allow(non_snake_case)]
|
||||
#![stable(feature = "core_char", since = "1.2.0")]
|
||||
|
||||
use char_private::is_printable;
|
||||
use convert::TryFrom;
|
||||
use fmt::{self, Write};
|
||||
use slice;
|
||||
use str::{from_utf8_unchecked_mut, FromStr};
|
||||
use iter::FusedIterator;
|
||||
use mem::transmute;
|
||||
|
||||
// UTF-8 ranges and tags for encoding characters
|
||||
const TAG_CONT: u8 = 0b1000_0000;
|
||||
const TAG_TWO_B: u8 = 0b1100_0000;
|
||||
const TAG_THREE_B: u8 = 0b1110_0000;
|
||||
const TAG_FOUR_B: u8 = 0b1111_0000;
|
||||
const MAX_ONE_B: u32 = 0x80;
|
||||
const MAX_TWO_B: u32 = 0x800;
|
||||
const MAX_THREE_B: u32 = 0x10000;
|
||||
|
||||
/*
|
||||
Lu Uppercase_Letter an uppercase letter
|
||||
Ll Lowercase_Letter a lowercase letter
|
||||
Lt Titlecase_Letter a digraphic character, with first part uppercase
|
||||
Lm Modifier_Letter a modifier letter
|
||||
Lo Other_Letter other letters, including syllables and ideographs
|
||||
Mn Nonspacing_Mark a nonspacing combining mark (zero advance width)
|
||||
Mc Spacing_Mark a spacing combining mark (positive advance width)
|
||||
Me Enclosing_Mark an enclosing combining mark
|
||||
Nd Decimal_Number a decimal digit
|
||||
Nl Letter_Number a letterlike numeric character
|
||||
No Other_Number a numeric character of other type
|
||||
Pc Connector_Punctuation a connecting punctuation mark, like a tie
|
||||
Pd Dash_Punctuation a dash or hyphen punctuation mark
|
||||
Ps Open_Punctuation an opening punctuation mark (of a pair)
|
||||
Pe Close_Punctuation a closing punctuation mark (of a pair)
|
||||
Pi Initial_Punctuation an initial quotation mark
|
||||
Pf Final_Punctuation a final quotation mark
|
||||
Po Other_Punctuation a punctuation mark of other type
|
||||
Sm Math_Symbol a symbol of primarily mathematical use
|
||||
Sc Currency_Symbol a currency sign
|
||||
Sk Modifier_Symbol a non-letterlike modifier symbol
|
||||
So Other_Symbol a symbol of other type
|
||||
Zs Space_Separator a space character (of various non-zero widths)
|
||||
Zl Line_Separator U+2028 LINE SEPARATOR only
|
||||
Zp Paragraph_Separator U+2029 PARAGRAPH SEPARATOR only
|
||||
Cc Control a C0 or C1 control code
|
||||
Cf Format a format control character
|
||||
Cs Surrogate a surrogate code point
|
||||
Co Private_Use a private-use character
|
||||
Cn Unassigned a reserved unassigned code point or a noncharacter
|
||||
*/
|
||||
|
||||
/// The highest valid code point a `char` can have.
|
||||
///
|
||||
/// A [`char`] is a [Unicode Scalar Value], which means that it is a [Code
|
||||
/// Point], but only ones within a certain range. `MAX` is the highest valid
|
||||
/// code point that's a valid [Unicode Scalar Value].
|
||||
///
|
||||
/// [`char`]: ../../std/primitive.char.html
|
||||
/// [Unicode Scalar Value]: http://www.unicode.org/glossary/#unicode_scalar_value
|
||||
/// [Code Point]: http://www.unicode.org/glossary/#code_point
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub const MAX: char = '\u{10ffff}';
|
||||
|
||||
/// Converts a `u32` to a `char`.
|
||||
///
|
||||
/// Note that all [`char`]s are valid [`u32`]s, and can be cast to one with
|
||||
/// [`as`]:
|
||||
///
|
||||
/// ```
|
||||
/// let c = '💯';
|
||||
/// let i = c as u32;
|
||||
///
|
||||
/// assert_eq!(128175, i);
|
||||
/// ```
|
||||
///
|
||||
/// However, the reverse is not true: not all valid [`u32`]s are valid
|
||||
/// [`char`]s. `from_u32()` will return `None` if the input is not a valid value
|
||||
/// for a [`char`].
|
||||
///
|
||||
/// [`char`]: ../../std/primitive.char.html
|
||||
/// [`u32`]: ../../std/primitive.u32.html
|
||||
/// [`as`]: ../../book/first-edition/casting-between-types.html#as
|
||||
///
|
||||
/// For an unsafe version of this function which ignores these checks, see
|
||||
/// [`from_u32_unchecked`].
|
||||
///
|
||||
/// [`from_u32_unchecked`]: fn.from_u32_unchecked.html
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// Basic usage:
|
||||
///
|
||||
/// ```
|
||||
/// use std::char;
|
||||
///
|
||||
/// let c = char::from_u32(0x2764);
|
||||
///
|
||||
/// assert_eq!(Some('❤'), c);
|
||||
/// ```
|
||||
///
|
||||
/// Returning `None` when the input is not a valid [`char`]:
|
||||
///
|
||||
/// ```
|
||||
/// use std::char;
|
||||
///
|
||||
/// let c = char::from_u32(0x110000);
|
||||
///
|
||||
/// assert_eq!(None, c);
|
||||
/// ```
|
||||
#[inline]
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub fn from_u32(i: u32) -> Option<char> {
|
||||
char::try_from(i).ok()
|
||||
}
|
||||
|
||||
/// Converts a `u32` to a `char`, ignoring validity.
|
||||
///
|
||||
/// Note that all [`char`]s are valid [`u32`]s, and can be cast to one with
|
||||
/// [`as`]:
|
||||
///
|
||||
/// ```
|
||||
/// let c = '💯';
|
||||
/// let i = c as u32;
|
||||
///
|
||||
/// assert_eq!(128175, i);
|
||||
/// ```
|
||||
///
|
||||
/// However, the reverse is not true: not all valid [`u32`]s are valid
|
||||
/// [`char`]s. `from_u32_unchecked()` will ignore this, and blindly cast to
|
||||
/// [`char`], possibly creating an invalid one.
|
||||
///
|
||||
/// [`char`]: ../../std/primitive.char.html
|
||||
/// [`u32`]: ../../std/primitive.u32.html
|
||||
/// [`as`]: ../../book/first-edition/casting-between-types.html#as
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// This function is unsafe, as it may construct invalid `char` values.
|
||||
///
|
||||
/// For a safe version of this function, see the [`from_u32`] function.
|
||||
///
|
||||
/// [`from_u32`]: fn.from_u32.html
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// Basic usage:
|
||||
///
|
||||
/// ```
|
||||
/// use std::char;
|
||||
///
|
||||
/// let c = unsafe { char::from_u32_unchecked(0x2764) };
|
||||
///
|
||||
/// assert_eq!('❤', c);
|
||||
/// ```
|
||||
#[inline]
|
||||
#[stable(feature = "char_from_unchecked", since = "1.5.0")]
|
||||
pub unsafe fn from_u32_unchecked(i: u32) -> char {
|
||||
transmute(i)
|
||||
}
|
||||
|
||||
#[stable(feature = "char_convert", since = "1.13.0")]
|
||||
impl From<char> for u32 {
|
||||
#[inline]
|
||||
fn from(c: char) -> Self {
|
||||
c as u32
|
||||
}
|
||||
}
|
||||
|
||||
/// Maps a byte in 0x00...0xFF to a `char` whose code point has the same value, in U+0000 to U+00FF.
|
||||
///
|
||||
/// Unicode is designed such that this effectively decodes bytes
|
||||
/// with the character encoding that IANA calls ISO-8859-1.
|
||||
/// This encoding is compatible with ASCII.
|
||||
///
|
||||
/// Note that this is different from ISO/IEC 8859-1 a.k.a. ISO 8859-1 (with one less hyphen),
|
||||
/// which leaves some "blanks", byte values that are not assigned to any character.
|
||||
/// ISO-8859-1 (the IANA one) assigns them to the C0 and C1 control codes.
|
||||
///
|
||||
/// Note that this is *also* different from Windows-1252 a.k.a. code page 1252,
|
||||
/// which is a superset ISO/IEC 8859-1 that assigns some (not all!) blanks
|
||||
/// to punctuation and various Latin characters.
|
||||
///
|
||||
/// To confuse things further, [on the Web](https://encoding.spec.whatwg.org/)
|
||||
/// `ascii`, `iso-8859-1`, and `windows-1252` are all aliases
|
||||
/// for a superset of Windows-1252 that fills the remaining blanks with corresponding
|
||||
/// C0 and C1 control codes.
|
||||
#[stable(feature = "char_convert", since = "1.13.0")]
|
||||
impl From<u8> for char {
|
||||
#[inline]
|
||||
fn from(i: u8) -> Self {
|
||||
i as char
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// An error which can be returned when parsing a char.
|
||||
#[stable(feature = "char_from_str", since = "1.20.0")]
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ParseCharError {
|
||||
kind: CharErrorKind,
|
||||
}
|
||||
|
||||
impl ParseCharError {
|
||||
#[unstable(feature = "char_error_internals",
|
||||
reason = "this method should not be available publicly",
|
||||
issue = "0")]
|
||||
#[doc(hidden)]
|
||||
pub fn __description(&self) -> &str {
|
||||
match self.kind {
|
||||
CharErrorKind::EmptyString => {
|
||||
"cannot parse char from empty string"
|
||||
},
|
||||
CharErrorKind::TooManyChars => "too many characters in string"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
enum CharErrorKind {
|
||||
EmptyString,
|
||||
TooManyChars,
|
||||
}
|
||||
|
||||
#[stable(feature = "char_from_str", since = "1.20.0")]
|
||||
impl fmt::Display for ParseCharError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
self.__description().fmt(f)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#[stable(feature = "char_from_str", since = "1.20.0")]
|
||||
impl FromStr for char {
|
||||
type Err = ParseCharError;
|
||||
|
||||
#[inline]
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
let mut chars = s.chars();
|
||||
match (chars.next(), chars.next()) {
|
||||
(None, _) => {
|
||||
Err(ParseCharError { kind: CharErrorKind::EmptyString })
|
||||
},
|
||||
(Some(c), None) => Ok(c),
|
||||
_ => {
|
||||
Err(ParseCharError { kind: CharErrorKind::TooManyChars })
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#[stable(feature = "try_from", since = "1.26.0")]
|
||||
impl TryFrom<u32> for char {
|
||||
type Error = CharTryFromError;
|
||||
|
||||
#[inline]
|
||||
fn try_from(i: u32) -> Result<Self, Self::Error> {
|
||||
if (i > MAX as u32) || (i >= 0xD800 && i <= 0xDFFF) {
|
||||
Err(CharTryFromError(()))
|
||||
} else {
|
||||
Ok(unsafe { from_u32_unchecked(i) })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The error type returned when a conversion from u32 to char fails.
|
||||
#[stable(feature = "try_from", since = "1.26.0")]
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
pub struct CharTryFromError(());
|
||||
|
||||
#[stable(feature = "try_from", since = "1.26.0")]
|
||||
impl fmt::Display for CharTryFromError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
"converted integer out of range for `char`".fmt(f)
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts a digit in the given radix to a `char`.
|
||||
///
|
||||
/// A 'radix' here is sometimes also called a 'base'. A radix of two
|
||||
/// indicates a binary number, a radix of ten, decimal, and a radix of
|
||||
/// sixteen, hexadecimal, to give some common values. Arbitrary
|
||||
/// radices are supported.
|
||||
///
|
||||
/// `from_digit()` will return `None` if the input is not a digit in
|
||||
/// the given radix.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if given a radix larger than 36.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// Basic usage:
|
||||
///
|
||||
/// ```
|
||||
/// use std::char;
|
||||
///
|
||||
/// let c = char::from_digit(4, 10);
|
||||
///
|
||||
/// assert_eq!(Some('4'), c);
|
||||
///
|
||||
/// // Decimal 11 is a single digit in base 16
|
||||
/// let c = char::from_digit(11, 16);
|
||||
///
|
||||
/// assert_eq!(Some('b'), c);
|
||||
/// ```
|
||||
///
|
||||
/// Returning `None` when the input is not a digit:
|
||||
///
|
||||
/// ```
|
||||
/// use std::char;
|
||||
///
|
||||
/// let c = char::from_digit(20, 10);
|
||||
///
|
||||
/// assert_eq!(None, c);
|
||||
/// ```
|
||||
///
|
||||
/// Passing a large radix, causing a panic:
|
||||
///
|
||||
/// ```
|
||||
/// use std::thread;
|
||||
/// use std::char;
|
||||
///
|
||||
/// let result = thread::spawn(|| {
|
||||
/// // this panics
|
||||
/// let c = char::from_digit(1, 37);
|
||||
/// }).join();
|
||||
///
|
||||
/// assert!(result.is_err());
|
||||
/// ```
|
||||
#[inline]
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub fn from_digit(num: u32, radix: u32) -> Option<char> {
|
||||
if radix > 36 {
|
||||
panic!("from_digit: radix is too high (maximum 36)");
|
||||
}
|
||||
if num < radix {
|
||||
let num = num as u8;
|
||||
if num < 10 {
|
||||
Some((b'0' + num) as char)
|
||||
} else {
|
||||
Some((b'a' + num - 10) as char)
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
// NB: the stabilization and documentation for this trait is in
|
||||
// unicode/char.rs, not here
|
||||
#[allow(missing_docs)] // docs in libunicode/u_char.rs
|
||||
#[doc(hidden)]
|
||||
#[unstable(feature = "core_char_ext",
|
||||
reason = "the stable interface is `impl char` in later crate",
|
||||
issue = "32110")]
|
||||
pub trait CharExt {
|
||||
#[stable(feature = "core", since = "1.6.0")]
|
||||
fn is_digit(self, radix: u32) -> bool;
|
||||
#[stable(feature = "core", since = "1.6.0")]
|
||||
fn to_digit(self, radix: u32) -> Option<u32>;
|
||||
#[stable(feature = "core", since = "1.6.0")]
|
||||
fn escape_unicode(self) -> EscapeUnicode;
|
||||
#[stable(feature = "core", since = "1.6.0")]
|
||||
fn escape_default(self) -> EscapeDefault;
|
||||
#[stable(feature = "char_escape_debug", since = "1.20.0")]
|
||||
fn escape_debug(self) -> EscapeDebug;
|
||||
#[stable(feature = "core", since = "1.6.0")]
|
||||
fn len_utf8(self) -> usize;
|
||||
#[stable(feature = "core", since = "1.6.0")]
|
||||
fn len_utf16(self) -> usize;
|
||||
#[stable(feature = "unicode_encode_char", since = "1.15.0")]
|
||||
fn encode_utf8(self, dst: &mut [u8]) -> &mut str;
|
||||
#[stable(feature = "unicode_encode_char", since = "1.15.0")]
|
||||
fn encode_utf16(self, dst: &mut [u16]) -> &mut [u16];
|
||||
}
|
||||
|
||||
#[stable(feature = "core", since = "1.6.0")]
|
||||
impl CharExt for char {
|
||||
#[inline]
|
||||
fn is_digit(self, radix: u32) -> bool {
|
||||
self.to_digit(radix).is_some()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn to_digit(self, radix: u32) -> Option<u32> {
|
||||
if radix > 36 {
|
||||
panic!("to_digit: radix is too high (maximum 36)");
|
||||
}
|
||||
let val = match self {
|
||||
'0' ... '9' => self as u32 - '0' as u32,
|
||||
'a' ... 'z' => self as u32 - 'a' as u32 + 10,
|
||||
'A' ... 'Z' => self as u32 - 'A' as u32 + 10,
|
||||
_ => return None,
|
||||
};
|
||||
if val < radix { Some(val) }
|
||||
else { None }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn escape_unicode(self) -> EscapeUnicode {
|
||||
let c = self as u32;
|
||||
|
||||
// or-ing 1 ensures that for c==0 the code computes that one
|
||||
// digit should be printed and (which is the same) avoids the
|
||||
// (31 - 32) underflow
|
||||
let msb = 31 - (c | 1).leading_zeros();
|
||||
|
||||
// the index of the most significant hex digit
|
||||
let ms_hex_digit = msb / 4;
|
||||
EscapeUnicode {
|
||||
c: self,
|
||||
state: EscapeUnicodeState::Backslash,
|
||||
hex_digit_idx: ms_hex_digit as usize,
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn escape_default(self) -> EscapeDefault {
|
||||
let init_state = match self {
|
||||
'\t' => EscapeDefaultState::Backslash('t'),
|
||||
'\r' => EscapeDefaultState::Backslash('r'),
|
||||
'\n' => EscapeDefaultState::Backslash('n'),
|
||||
'\\' | '\'' | '"' => EscapeDefaultState::Backslash(self),
|
||||
'\x20' ... '\x7e' => EscapeDefaultState::Char(self),
|
||||
_ => EscapeDefaultState::Unicode(self.escape_unicode())
|
||||
};
|
||||
EscapeDefault { state: init_state }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn escape_debug(self) -> EscapeDebug {
|
||||
let init_state = match self {
|
||||
'\t' => EscapeDefaultState::Backslash('t'),
|
||||
'\r' => EscapeDefaultState::Backslash('r'),
|
||||
'\n' => EscapeDefaultState::Backslash('n'),
|
||||
'\\' | '\'' | '"' => EscapeDefaultState::Backslash(self),
|
||||
c if is_printable(c) => EscapeDefaultState::Char(c),
|
||||
c => EscapeDefaultState::Unicode(c.escape_unicode()),
|
||||
};
|
||||
EscapeDebug(EscapeDefault { state: init_state })
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn len_utf8(self) -> usize {
|
||||
let code = self as u32;
|
||||
if code < MAX_ONE_B {
|
||||
1
|
||||
} else if code < MAX_TWO_B {
|
||||
2
|
||||
} else if code < MAX_THREE_B {
|
||||
3
|
||||
} else {
|
||||
4
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn len_utf16(self) -> usize {
|
||||
let ch = self as u32;
|
||||
if (ch & 0xFFFF) == ch { 1 } else { 2 }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn encode_utf8(self, dst: &mut [u8]) -> &mut str {
|
||||
let code = self as u32;
|
||||
unsafe {
|
||||
let len =
|
||||
if code < MAX_ONE_B && !dst.is_empty() {
|
||||
*dst.get_unchecked_mut(0) = code as u8;
|
||||
1
|
||||
} else if code < MAX_TWO_B && dst.len() >= 2 {
|
||||
*dst.get_unchecked_mut(0) = (code >> 6 & 0x1F) as u8 | TAG_TWO_B;
|
||||
*dst.get_unchecked_mut(1) = (code & 0x3F) as u8 | TAG_CONT;
|
||||
2
|
||||
} else if code < MAX_THREE_B && dst.len() >= 3 {
|
||||
*dst.get_unchecked_mut(0) = (code >> 12 & 0x0F) as u8 | TAG_THREE_B;
|
||||
*dst.get_unchecked_mut(1) = (code >> 6 & 0x3F) as u8 | TAG_CONT;
|
||||
*dst.get_unchecked_mut(2) = (code & 0x3F) as u8 | TAG_CONT;
|
||||
3
|
||||
} else if dst.len() >= 4 {
|
||||
*dst.get_unchecked_mut(0) = (code >> 18 & 0x07) as u8 | TAG_FOUR_B;
|
||||
*dst.get_unchecked_mut(1) = (code >> 12 & 0x3F) as u8 | TAG_CONT;
|
||||
*dst.get_unchecked_mut(2) = (code >> 6 & 0x3F) as u8 | TAG_CONT;
|
||||
*dst.get_unchecked_mut(3) = (code & 0x3F) as u8 | TAG_CONT;
|
||||
4
|
||||
} else {
|
||||
panic!("encode_utf8: need {} bytes to encode U+{:X}, but the buffer has {}",
|
||||
from_u32_unchecked(code).len_utf8(),
|
||||
code,
|
||||
dst.len())
|
||||
};
|
||||
from_utf8_unchecked_mut(dst.get_unchecked_mut(..len))
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn encode_utf16(self, dst: &mut [u16]) -> &mut [u16] {
|
||||
let mut code = self as u32;
|
||||
unsafe {
|
||||
if (code & 0xFFFF) == code && !dst.is_empty() {
|
||||
// The BMP falls through (assuming non-surrogate, as it should)
|
||||
*dst.get_unchecked_mut(0) = code as u16;
|
||||
slice::from_raw_parts_mut(dst.as_mut_ptr(), 1)
|
||||
} else if dst.len() >= 2 {
|
||||
// Supplementary planes break into surrogates.
|
||||
code -= 0x1_0000;
|
||||
*dst.get_unchecked_mut(0) = 0xD800 | ((code >> 10) as u16);
|
||||
*dst.get_unchecked_mut(1) = 0xDC00 | ((code as u16) & 0x3FF);
|
||||
slice::from_raw_parts_mut(dst.as_mut_ptr(), 2)
|
||||
} else {
|
||||
panic!("encode_utf16: need {} units to encode U+{:X}, but the buffer has {}",
|
||||
from_u32_unchecked(code).len_utf16(),
|
||||
code,
|
||||
dst.len())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns an iterator that yields the hexadecimal Unicode escape of a
|
||||
/// character, as `char`s.
|
||||
///
|
||||
/// This `struct` is created by the [`escape_unicode`] method on [`char`]. See
|
||||
/// its documentation for more.
|
||||
///
|
||||
/// [`escape_unicode`]: ../../std/primitive.char.html#method.escape_unicode
|
||||
/// [`char`]: ../../std/primitive.char.html
|
||||
#[derive(Clone, Debug)]
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub struct EscapeUnicode {
|
||||
c: char,
|
||||
state: EscapeUnicodeState,
|
||||
|
||||
// The index of the next hex digit to be printed (0 if none),
|
||||
// i.e. the number of remaining hex digits to be printed;
|
||||
// increasing from the least significant digit: 0x543210
|
||||
hex_digit_idx: usize,
|
||||
}
|
||||
|
||||
// The enum values are ordered so that their representation is the
|
||||
// same as the remaining length (besides the hexadecimal digits). This
|
||||
// likely makes `len()` a single load from memory) and inline-worth.
|
||||
#[derive(Clone, Debug)]
|
||||
enum EscapeUnicodeState {
|
||||
Done,
|
||||
RightBrace,
|
||||
Value,
|
||||
LeftBrace,
|
||||
Type,
|
||||
Backslash,
|
||||
}
|
||||
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
impl Iterator for EscapeUnicode {
|
||||
type Item = char;
|
||||
|
||||
fn next(&mut self) -> Option<char> {
|
||||
match self.state {
|
||||
EscapeUnicodeState::Backslash => {
|
||||
self.state = EscapeUnicodeState::Type;
|
||||
Some('\\')
|
||||
}
|
||||
EscapeUnicodeState::Type => {
|
||||
self.state = EscapeUnicodeState::LeftBrace;
|
||||
Some('u')
|
||||
}
|
||||
EscapeUnicodeState::LeftBrace => {
|
||||
self.state = EscapeUnicodeState::Value;
|
||||
Some('{')
|
||||
}
|
||||
EscapeUnicodeState::Value => {
|
||||
let hex_digit = ((self.c as u32) >> (self.hex_digit_idx * 4)) & 0xf;
|
||||
let c = from_digit(hex_digit, 16).unwrap();
|
||||
if self.hex_digit_idx == 0 {
|
||||
self.state = EscapeUnicodeState::RightBrace;
|
||||
} else {
|
||||
self.hex_digit_idx -= 1;
|
||||
}
|
||||
Some(c)
|
||||
}
|
||||
EscapeUnicodeState::RightBrace => {
|
||||
self.state = EscapeUnicodeState::Done;
|
||||
Some('}')
|
||||
}
|
||||
EscapeUnicodeState::Done => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
let n = self.len();
|
||||
(n, Some(n))
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn count(self) -> usize {
|
||||
self.len()
|
||||
}
|
||||
|
||||
fn last(self) -> Option<char> {
|
||||
match self.state {
|
||||
EscapeUnicodeState::Done => None,
|
||||
|
||||
EscapeUnicodeState::RightBrace |
|
||||
EscapeUnicodeState::Value |
|
||||
EscapeUnicodeState::LeftBrace |
|
||||
EscapeUnicodeState::Type |
|
||||
EscapeUnicodeState::Backslash => Some('}'),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "exact_size_escape", since = "1.11.0")]
|
||||
impl ExactSizeIterator for EscapeUnicode {
|
||||
#[inline]
|
||||
fn len(&self) -> usize {
|
||||
// The match is a single memory access with no branching
|
||||
self.hex_digit_idx + match self.state {
|
||||
EscapeUnicodeState::Done => 0,
|
||||
EscapeUnicodeState::RightBrace => 1,
|
||||
EscapeUnicodeState::Value => 2,
|
||||
EscapeUnicodeState::LeftBrace => 3,
|
||||
EscapeUnicodeState::Type => 4,
|
||||
EscapeUnicodeState::Backslash => 5,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "fused", since = "1.26.0")]
|
||||
impl FusedIterator for EscapeUnicode {}
|
||||
|
||||
#[stable(feature = "char_struct_display", since = "1.16.0")]
|
||||
impl fmt::Display for EscapeUnicode {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
for c in self.clone() {
|
||||
f.write_char(c)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// An iterator that yields the literal escape code of a `char`.
|
||||
///
|
||||
/// This `struct` is created by the [`escape_default`] method on [`char`]. See
|
||||
/// its documentation for more.
|
||||
///
|
||||
/// [`escape_default`]: ../../std/primitive.char.html#method.escape_default
|
||||
/// [`char`]: ../../std/primitive.char.html
|
||||
#[derive(Clone, Debug)]
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub struct EscapeDefault {
|
||||
state: EscapeDefaultState
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
enum EscapeDefaultState {
|
||||
Done,
|
||||
Char(char),
|
||||
Backslash(char),
|
||||
Unicode(EscapeUnicode),
|
||||
}
|
||||
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
impl Iterator for EscapeDefault {
|
||||
type Item = char;
|
||||
|
||||
fn next(&mut self) -> Option<char> {
|
||||
match self.state {
|
||||
EscapeDefaultState::Backslash(c) => {
|
||||
self.state = EscapeDefaultState::Char(c);
|
||||
Some('\\')
|
||||
}
|
||||
EscapeDefaultState::Char(c) => {
|
||||
self.state = EscapeDefaultState::Done;
|
||||
Some(c)
|
||||
}
|
||||
EscapeDefaultState::Done => None,
|
||||
EscapeDefaultState::Unicode(ref mut iter) => iter.next(),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
let n = self.len();
|
||||
(n, Some(n))
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn count(self) -> usize {
|
||||
self.len()
|
||||
}
|
||||
|
||||
fn nth(&mut self, n: usize) -> Option<char> {
|
||||
match self.state {
|
||||
EscapeDefaultState::Backslash(c) if n == 0 => {
|
||||
self.state = EscapeDefaultState::Char(c);
|
||||
Some('\\')
|
||||
},
|
||||
EscapeDefaultState::Backslash(c) if n == 1 => {
|
||||
self.state = EscapeDefaultState::Done;
|
||||
Some(c)
|
||||
},
|
||||
EscapeDefaultState::Backslash(_) => {
|
||||
self.state = EscapeDefaultState::Done;
|
||||
None
|
||||
},
|
||||
EscapeDefaultState::Char(c) => {
|
||||
self.state = EscapeDefaultState::Done;
|
||||
|
||||
if n == 0 {
|
||||
Some(c)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
},
|
||||
EscapeDefaultState::Done => return None,
|
||||
EscapeDefaultState::Unicode(ref mut i) => return i.nth(n),
|
||||
}
|
||||
}
|
||||
|
||||
fn last(self) -> Option<char> {
|
||||
match self.state {
|
||||
EscapeDefaultState::Unicode(iter) => iter.last(),
|
||||
EscapeDefaultState::Done => None,
|
||||
EscapeDefaultState::Backslash(c) | EscapeDefaultState::Char(c) => Some(c),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "exact_size_escape", since = "1.11.0")]
|
||||
impl ExactSizeIterator for EscapeDefault {
|
||||
fn len(&self) -> usize {
|
||||
match self.state {
|
||||
EscapeDefaultState::Done => 0,
|
||||
EscapeDefaultState::Char(_) => 1,
|
||||
EscapeDefaultState::Backslash(_) => 2,
|
||||
EscapeDefaultState::Unicode(ref iter) => iter.len(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "fused", since = "1.26.0")]
|
||||
impl FusedIterator for EscapeDefault {}
|
||||
|
||||
#[stable(feature = "char_struct_display", since = "1.16.0")]
|
||||
impl fmt::Display for EscapeDefault {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
for c in self.clone() {
|
||||
f.write_char(c)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// An iterator that yields the literal escape code of a `char`.
|
||||
///
|
||||
/// This `struct` is created by the [`escape_debug`] method on [`char`]. See its
|
||||
/// documentation for more.
|
||||
///
|
||||
/// [`escape_debug`]: ../../std/primitive.char.html#method.escape_debug
|
||||
/// [`char`]: ../../std/primitive.char.html
|
||||
#[stable(feature = "char_escape_debug", since = "1.20.0")]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct EscapeDebug(EscapeDefault);
|
||||
|
||||
#[stable(feature = "char_escape_debug", since = "1.20.0")]
|
||||
impl Iterator for EscapeDebug {
|
||||
type Item = char;
|
||||
fn next(&mut self) -> Option<char> { self.0.next() }
|
||||
fn size_hint(&self) -> (usize, Option<usize>) { self.0.size_hint() }
|
||||
}
|
||||
|
||||
#[stable(feature = "char_escape_debug", since = "1.20.0")]
|
||||
impl ExactSizeIterator for EscapeDebug { }
|
||||
|
||||
#[stable(feature = "fused", since = "1.26.0")]
|
||||
impl FusedIterator for EscapeDebug {}
|
||||
|
||||
#[stable(feature = "char_escape_debug", since = "1.20.0")]
|
||||
impl fmt::Display for EscapeDebug {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
fmt::Display::fmt(&self.0, f)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/// An iterator over an iterator of bytes of the characters the bytes represent
|
||||
/// as UTF-8
|
||||
#[unstable(feature = "decode_utf8", issue = "33906")]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct DecodeUtf8<I: Iterator<Item = u8>>(::iter::Peekable<I>);
|
||||
|
||||
/// Decodes an `Iterator` of bytes as UTF-8.
|
||||
#[unstable(feature = "decode_utf8", issue = "33906")]
|
||||
#[inline]
|
||||
pub fn decode_utf8<I: IntoIterator<Item = u8>>(i: I) -> DecodeUtf8<I::IntoIter> {
|
||||
DecodeUtf8(i.into_iter().peekable())
|
||||
}
|
||||
|
||||
/// `<DecodeUtf8 as Iterator>::next` returns this for an invalid input sequence.
|
||||
#[unstable(feature = "decode_utf8", issue = "33906")]
|
||||
#[derive(PartialEq, Eq, Debug)]
|
||||
pub struct InvalidSequence(());
|
||||
|
||||
#[unstable(feature = "decode_utf8", issue = "33906")]
|
||||
impl<I: Iterator<Item = u8>> Iterator for DecodeUtf8<I> {
|
||||
type Item = Result<char, InvalidSequence>;
|
||||
#[inline]
|
||||
|
||||
fn next(&mut self) -> Option<Result<char, InvalidSequence>> {
|
||||
self.0.next().map(|first_byte| {
|
||||
// Emit InvalidSequence according to
|
||||
// Unicode §5.22 Best Practice for U+FFFD Substitution
|
||||
// http://www.unicode.org/versions/Unicode9.0.0/ch05.pdf#G40630
|
||||
|
||||
// Roughly: consume at least one byte,
|
||||
// then validate one byte at a time and stop before the first unexpected byte
|
||||
// (which might be the valid start of the next byte sequence).
|
||||
|
||||
let mut code_point;
|
||||
macro_rules! first_byte {
|
||||
($mask: expr) => {
|
||||
code_point = u32::from(first_byte & $mask)
|
||||
}
|
||||
}
|
||||
macro_rules! continuation_byte {
|
||||
() => { continuation_byte!(0x80...0xBF) };
|
||||
($range: pat) => {
|
||||
match self.0.peek() {
|
||||
Some(&byte @ $range) => {
|
||||
code_point = (code_point << 6) | u32::from(byte & 0b0011_1111);
|
||||
self.0.next();
|
||||
}
|
||||
_ => return Err(InvalidSequence(()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match first_byte {
|
||||
0x00...0x7F => {
|
||||
first_byte!(0b1111_1111);
|
||||
}
|
||||
0xC2...0xDF => {
|
||||
first_byte!(0b0001_1111);
|
||||
continuation_byte!();
|
||||
}
|
||||
0xE0 => {
|
||||
first_byte!(0b0000_1111);
|
||||
continuation_byte!(0xA0...0xBF); // 0x80...0x9F here are overlong
|
||||
continuation_byte!();
|
||||
}
|
||||
0xE1...0xEC | 0xEE...0xEF => {
|
||||
first_byte!(0b0000_1111);
|
||||
continuation_byte!();
|
||||
continuation_byte!();
|
||||
}
|
||||
0xED => {
|
||||
first_byte!(0b0000_1111);
|
||||
continuation_byte!(0x80...0x9F); // 0xA0..0xBF here are surrogates
|
||||
continuation_byte!();
|
||||
}
|
||||
0xF0 => {
|
||||
first_byte!(0b0000_0111);
|
||||
continuation_byte!(0x90...0xBF); // 0x80..0x8F here are overlong
|
||||
continuation_byte!();
|
||||
continuation_byte!();
|
||||
}
|
||||
0xF1...0xF3 => {
|
||||
first_byte!(0b0000_0111);
|
||||
continuation_byte!();
|
||||
continuation_byte!();
|
||||
continuation_byte!();
|
||||
}
|
||||
0xF4 => {
|
||||
first_byte!(0b0000_0111);
|
||||
continuation_byte!(0x80...0x8F); // 0x90..0xBF here are beyond char::MAX
|
||||
continuation_byte!();
|
||||
continuation_byte!();
|
||||
}
|
||||
_ => return Err(InvalidSequence(())) // Illegal first byte, overlong, or beyond MAX
|
||||
}
|
||||
unsafe {
|
||||
Ok(from_u32_unchecked(code_point))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
let (lower, upper) = self.0.size_hint();
|
||||
|
||||
// A code point is at most 4 bytes long.
|
||||
let min_code_points = lower / 4;
|
||||
|
||||
(min_code_points, upper)
|
||||
}
|
||||
}
|
||||
|
||||
#[unstable(feature = "decode_utf8", issue = "33906")]
|
||||
impl<I: FusedIterator<Item = u8>> FusedIterator for DecodeUtf8<I> {}
|
||||
304
src/libcore/char/convert.rs
Normal file
304
src/libcore/char/convert.rs
Normal file
|
|
@ -0,0 +1,304 @@
|
|||
// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
|
||||
// file at the top-level directory of this distribution and at
|
||||
// http://rust-lang.org/COPYRIGHT.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
||||
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
||||
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
||||
// option. This file may not be copied, modified, or distributed
|
||||
// except according to those terms.
|
||||
|
||||
//! Character conversions.
|
||||
|
||||
use convert::TryFrom;
|
||||
use fmt;
|
||||
use mem::transmute;
|
||||
use str::FromStr;
|
||||
use super::MAX;
|
||||
|
||||
/// Converts a `u32` to a `char`.
|
||||
///
|
||||
/// Note that all [`char`]s are valid [`u32`]s, and can be cast to one with
|
||||
/// [`as`]:
|
||||
///
|
||||
/// ```
|
||||
/// let c = '💯';
|
||||
/// let i = c as u32;
|
||||
///
|
||||
/// assert_eq!(128175, i);
|
||||
/// ```
|
||||
///
|
||||
/// However, the reverse is not true: not all valid [`u32`]s are valid
|
||||
/// [`char`]s. `from_u32()` will return `None` if the input is not a valid value
|
||||
/// for a [`char`].
|
||||
///
|
||||
/// [`char`]: ../../std/primitive.char.html
|
||||
/// [`u32`]: ../../std/primitive.u32.html
|
||||
/// [`as`]: ../../book/first-edition/casting-between-types.html#as
|
||||
///
|
||||
/// For an unsafe version of this function which ignores these checks, see
|
||||
/// [`from_u32_unchecked`].
|
||||
///
|
||||
/// [`from_u32_unchecked`]: fn.from_u32_unchecked.html
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// Basic usage:
|
||||
///
|
||||
/// ```
|
||||
/// use std::char;
|
||||
///
|
||||
/// let c = char::from_u32(0x2764);
|
||||
///
|
||||
/// assert_eq!(Some('❤'), c);
|
||||
/// ```
|
||||
///
|
||||
/// Returning `None` when the input is not a valid [`char`]:
|
||||
///
|
||||
/// ```
|
||||
/// use std::char;
|
||||
///
|
||||
/// let c = char::from_u32(0x110000);
|
||||
///
|
||||
/// assert_eq!(None, c);
|
||||
/// ```
|
||||
#[inline]
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub fn from_u32(i: u32) -> Option<char> {
|
||||
char::try_from(i).ok()
|
||||
}
|
||||
|
||||
/// Converts a `u32` to a `char`, ignoring validity.
|
||||
///
|
||||
/// Note that all [`char`]s are valid [`u32`]s, and can be cast to one with
|
||||
/// [`as`]:
|
||||
///
|
||||
/// ```
|
||||
/// let c = '💯';
|
||||
/// let i = c as u32;
|
||||
///
|
||||
/// assert_eq!(128175, i);
|
||||
/// ```
|
||||
///
|
||||
/// However, the reverse is not true: not all valid [`u32`]s are valid
|
||||
/// [`char`]s. `from_u32_unchecked()` will ignore this, and blindly cast to
|
||||
/// [`char`], possibly creating an invalid one.
|
||||
///
|
||||
/// [`char`]: ../../std/primitive.char.html
|
||||
/// [`u32`]: ../../std/primitive.u32.html
|
||||
/// [`as`]: ../../book/first-edition/casting-between-types.html#as
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// This function is unsafe, as it may construct invalid `char` values.
|
||||
///
|
||||
/// For a safe version of this function, see the [`from_u32`] function.
|
||||
///
|
||||
/// [`from_u32`]: fn.from_u32.html
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// Basic usage:
|
||||
///
|
||||
/// ```
|
||||
/// use std::char;
|
||||
///
|
||||
/// let c = unsafe { char::from_u32_unchecked(0x2764) };
|
||||
///
|
||||
/// assert_eq!('❤', c);
|
||||
/// ```
|
||||
#[inline]
|
||||
#[stable(feature = "char_from_unchecked", since = "1.5.0")]
|
||||
pub unsafe fn from_u32_unchecked(i: u32) -> char {
|
||||
transmute(i)
|
||||
}
|
||||
|
||||
#[stable(feature = "char_convert", since = "1.13.0")]
|
||||
impl From<char> for u32 {
|
||||
#[inline]
|
||||
fn from(c: char) -> Self {
|
||||
c as u32
|
||||
}
|
||||
}
|
||||
|
||||
/// Maps a byte in 0x00...0xFF to a `char` whose code point has the same value, in U+0000 to U+00FF.
|
||||
///
|
||||
/// Unicode is designed such that this effectively decodes bytes
|
||||
/// with the character encoding that IANA calls ISO-8859-1.
|
||||
/// This encoding is compatible with ASCII.
|
||||
///
|
||||
/// Note that this is different from ISO/IEC 8859-1 a.k.a. ISO 8859-1 (with one less hyphen),
|
||||
/// which leaves some "blanks", byte values that are not assigned to any character.
|
||||
/// ISO-8859-1 (the IANA one) assigns them to the C0 and C1 control codes.
|
||||
///
|
||||
/// Note that this is *also* different from Windows-1252 a.k.a. code page 1252,
|
||||
/// which is a superset ISO/IEC 8859-1 that assigns some (not all!) blanks
|
||||
/// to punctuation and various Latin characters.
|
||||
///
|
||||
/// To confuse things further, [on the Web](https://encoding.spec.whatwg.org/)
|
||||
/// `ascii`, `iso-8859-1`, and `windows-1252` are all aliases
|
||||
/// for a superset of Windows-1252 that fills the remaining blanks with corresponding
|
||||
/// C0 and C1 control codes.
|
||||
#[stable(feature = "char_convert", since = "1.13.0")]
|
||||
impl From<u8> for char {
|
||||
#[inline]
|
||||
fn from(i: u8) -> Self {
|
||||
i as char
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// An error which can be returned when parsing a char.
|
||||
#[stable(feature = "char_from_str", since = "1.20.0")]
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ParseCharError {
|
||||
kind: CharErrorKind,
|
||||
}
|
||||
|
||||
impl ParseCharError {
|
||||
#[unstable(feature = "char_error_internals",
|
||||
reason = "this method should not be available publicly",
|
||||
issue = "0")]
|
||||
#[doc(hidden)]
|
||||
pub fn __description(&self) -> &str {
|
||||
match self.kind {
|
||||
CharErrorKind::EmptyString => {
|
||||
"cannot parse char from empty string"
|
||||
},
|
||||
CharErrorKind::TooManyChars => "too many characters in string"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
enum CharErrorKind {
|
||||
EmptyString,
|
||||
TooManyChars,
|
||||
}
|
||||
|
||||
#[stable(feature = "char_from_str", since = "1.20.0")]
|
||||
impl fmt::Display for ParseCharError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
self.__description().fmt(f)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#[stable(feature = "char_from_str", since = "1.20.0")]
|
||||
impl FromStr for char {
|
||||
type Err = ParseCharError;
|
||||
|
||||
#[inline]
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
let mut chars = s.chars();
|
||||
match (chars.next(), chars.next()) {
|
||||
(None, _) => {
|
||||
Err(ParseCharError { kind: CharErrorKind::EmptyString })
|
||||
},
|
||||
(Some(c), None) => Ok(c),
|
||||
_ => {
|
||||
Err(ParseCharError { kind: CharErrorKind::TooManyChars })
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#[stable(feature = "try_from", since = "1.26.0")]
|
||||
impl TryFrom<u32> for char {
|
||||
type Error = CharTryFromError;
|
||||
|
||||
#[inline]
|
||||
fn try_from(i: u32) -> Result<Self, Self::Error> {
|
||||
if (i > MAX as u32) || (i >= 0xD800 && i <= 0xDFFF) {
|
||||
Err(CharTryFromError(()))
|
||||
} else {
|
||||
Ok(unsafe { from_u32_unchecked(i) })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The error type returned when a conversion from u32 to char fails.
|
||||
#[stable(feature = "try_from", since = "1.26.0")]
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
pub struct CharTryFromError(());
|
||||
|
||||
#[stable(feature = "try_from", since = "1.26.0")]
|
||||
impl fmt::Display for CharTryFromError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
"converted integer out of range for `char`".fmt(f)
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts a digit in the given radix to a `char`.
|
||||
///
|
||||
/// A 'radix' here is sometimes also called a 'base'. A radix of two
|
||||
/// indicates a binary number, a radix of ten, decimal, and a radix of
|
||||
/// sixteen, hexadecimal, to give some common values. Arbitrary
|
||||
/// radices are supported.
|
||||
///
|
||||
/// `from_digit()` will return `None` if the input is not a digit in
|
||||
/// the given radix.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if given a radix larger than 36.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// Basic usage:
|
||||
///
|
||||
/// ```
|
||||
/// use std::char;
|
||||
///
|
||||
/// let c = char::from_digit(4, 10);
|
||||
///
|
||||
/// assert_eq!(Some('4'), c);
|
||||
///
|
||||
/// // Decimal 11 is a single digit in base 16
|
||||
/// let c = char::from_digit(11, 16);
|
||||
///
|
||||
/// assert_eq!(Some('b'), c);
|
||||
/// ```
|
||||
///
|
||||
/// Returning `None` when the input is not a digit:
|
||||
///
|
||||
/// ```
|
||||
/// use std::char;
|
||||
///
|
||||
/// let c = char::from_digit(20, 10);
|
||||
///
|
||||
/// assert_eq!(None, c);
|
||||
/// ```
|
||||
///
|
||||
/// Passing a large radix, causing a panic:
|
||||
///
|
||||
/// ```
|
||||
/// use std::thread;
|
||||
/// use std::char;
|
||||
///
|
||||
/// let result = thread::spawn(|| {
|
||||
/// // this panics
|
||||
/// let c = char::from_digit(1, 37);
|
||||
/// }).join();
|
||||
///
|
||||
/// assert!(result.is_err());
|
||||
/// ```
|
||||
#[inline]
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub fn from_digit(num: u32, radix: u32) -> Option<char> {
|
||||
if radix > 36 {
|
||||
panic!("from_digit: radix is too high (maximum 36)");
|
||||
}
|
||||
if num < radix {
|
||||
let num = num as u8;
|
||||
if num < 10 {
|
||||
Some((b'0' + num) as char)
|
||||
} else {
|
||||
Some((b'a' + num - 10) as char)
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
259
src/libcore/char/decode.rs
Normal file
259
src/libcore/char/decode.rs
Normal file
|
|
@ -0,0 +1,259 @@
|
|||
// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
|
||||
// file at the top-level directory of this distribution and at
|
||||
// http://rust-lang.org/COPYRIGHT.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
||||
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
||||
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
||||
// option. This file may not be copied, modified, or distributed
|
||||
// except according to those terms.
|
||||
|
||||
//! UTF-8 and UTF-16 decoding iterators
|
||||
|
||||
use fmt;
|
||||
use iter::FusedIterator;
|
||||
use super::from_u32_unchecked;
|
||||
|
||||
/// An iterator over an iterator of bytes of the characters the bytes represent
|
||||
/// as UTF-8
|
||||
#[unstable(feature = "decode_utf8", issue = "33906")]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct DecodeUtf8<I: Iterator<Item = u8>>(::iter::Peekable<I>);
|
||||
|
||||
/// Decodes an `Iterator` of bytes as UTF-8.
|
||||
#[unstable(feature = "decode_utf8", issue = "33906")]
|
||||
#[inline]
|
||||
pub fn decode_utf8<I: IntoIterator<Item = u8>>(i: I) -> DecodeUtf8<I::IntoIter> {
|
||||
DecodeUtf8(i.into_iter().peekable())
|
||||
}
|
||||
|
||||
/// `<DecodeUtf8 as Iterator>::next` returns this for an invalid input sequence.
|
||||
#[unstable(feature = "decode_utf8", issue = "33906")]
|
||||
#[derive(PartialEq, Eq, Debug)]
|
||||
pub struct InvalidSequence(());
|
||||
|
||||
#[unstable(feature = "decode_utf8", issue = "33906")]
|
||||
impl<I: Iterator<Item = u8>> Iterator for DecodeUtf8<I> {
|
||||
type Item = Result<char, InvalidSequence>;
|
||||
#[inline]
|
||||
|
||||
fn next(&mut self) -> Option<Result<char, InvalidSequence>> {
|
||||
self.0.next().map(|first_byte| {
|
||||
// Emit InvalidSequence according to
|
||||
// Unicode §5.22 Best Practice for U+FFFD Substitution
|
||||
// http://www.unicode.org/versions/Unicode9.0.0/ch05.pdf#G40630
|
||||
|
||||
// Roughly: consume at least one byte,
|
||||
// then validate one byte at a time and stop before the first unexpected byte
|
||||
// (which might be the valid start of the next byte sequence).
|
||||
|
||||
let mut code_point;
|
||||
macro_rules! first_byte {
|
||||
($mask: expr) => {
|
||||
code_point = u32::from(first_byte & $mask)
|
||||
}
|
||||
}
|
||||
macro_rules! continuation_byte {
|
||||
() => { continuation_byte!(0x80...0xBF) };
|
||||
($range: pat) => {
|
||||
match self.0.peek() {
|
||||
Some(&byte @ $range) => {
|
||||
code_point = (code_point << 6) | u32::from(byte & 0b0011_1111);
|
||||
self.0.next();
|
||||
}
|
||||
_ => return Err(InvalidSequence(()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match first_byte {
|
||||
0x00...0x7F => {
|
||||
first_byte!(0b1111_1111);
|
||||
}
|
||||
0xC2...0xDF => {
|
||||
first_byte!(0b0001_1111);
|
||||
continuation_byte!();
|
||||
}
|
||||
0xE0 => {
|
||||
first_byte!(0b0000_1111);
|
||||
continuation_byte!(0xA0...0xBF); // 0x80...0x9F here are overlong
|
||||
continuation_byte!();
|
||||
}
|
||||
0xE1...0xEC | 0xEE...0xEF => {
|
||||
first_byte!(0b0000_1111);
|
||||
continuation_byte!();
|
||||
continuation_byte!();
|
||||
}
|
||||
0xED => {
|
||||
first_byte!(0b0000_1111);
|
||||
continuation_byte!(0x80...0x9F); // 0xA0..0xBF here are surrogates
|
||||
continuation_byte!();
|
||||
}
|
||||
0xF0 => {
|
||||
first_byte!(0b0000_0111);
|
||||
continuation_byte!(0x90...0xBF); // 0x80..0x8F here are overlong
|
||||
continuation_byte!();
|
||||
continuation_byte!();
|
||||
}
|
||||
0xF1...0xF3 => {
|
||||
first_byte!(0b0000_0111);
|
||||
continuation_byte!();
|
||||
continuation_byte!();
|
||||
continuation_byte!();
|
||||
}
|
||||
0xF4 => {
|
||||
first_byte!(0b0000_0111);
|
||||
continuation_byte!(0x80...0x8F); // 0x90..0xBF here are beyond char::MAX
|
||||
continuation_byte!();
|
||||
continuation_byte!();
|
||||
}
|
||||
_ => return Err(InvalidSequence(())) // Illegal first byte, overlong, or beyond MAX
|
||||
}
|
||||
unsafe {
|
||||
Ok(from_u32_unchecked(code_point))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
let (lower, upper) = self.0.size_hint();
|
||||
|
||||
// A code point is at most 4 bytes long.
|
||||
let min_code_points = lower / 4;
|
||||
|
||||
(min_code_points, upper)
|
||||
}
|
||||
}
|
||||
|
||||
#[unstable(feature = "decode_utf8", issue = "33906")]
|
||||
impl<I: FusedIterator<Item = u8>> FusedIterator for DecodeUtf8<I> {}
|
||||
|
||||
/// An iterator that decodes UTF-16 encoded code points from an iterator of `u16`s.
|
||||
#[stable(feature = "decode_utf16", since = "1.9.0")]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct DecodeUtf16<I>
|
||||
where I: Iterator<Item = u16>
|
||||
{
|
||||
iter: I,
|
||||
buf: Option<u16>,
|
||||
}
|
||||
|
||||
/// An error that can be returned when decoding UTF-16 code points.
|
||||
#[stable(feature = "decode_utf16", since = "1.9.0")]
|
||||
#[derive(Debug, Clone, Eq, PartialEq)]
|
||||
pub struct DecodeUtf16Error {
|
||||
code: u16,
|
||||
}
|
||||
|
||||
/// Create an iterator over the UTF-16 encoded code points in `iter`,
|
||||
/// returning unpaired surrogates as `Err`s.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// Basic usage:
|
||||
///
|
||||
/// ```
|
||||
/// use std::char::decode_utf16;
|
||||
///
|
||||
/// fn main() {
|
||||
/// // 𝄞mus<invalid>ic<invalid>
|
||||
/// let v = [0xD834, 0xDD1E, 0x006d, 0x0075,
|
||||
/// 0x0073, 0xDD1E, 0x0069, 0x0063,
|
||||
/// 0xD834];
|
||||
///
|
||||
/// assert_eq!(decode_utf16(v.iter().cloned())
|
||||
/// .map(|r| r.map_err(|e| e.unpaired_surrogate()))
|
||||
/// .collect::<Vec<_>>(),
|
||||
/// vec![Ok('𝄞'),
|
||||
/// Ok('m'), Ok('u'), Ok('s'),
|
||||
/// Err(0xDD1E),
|
||||
/// Ok('i'), Ok('c'),
|
||||
/// Err(0xD834)]);
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// A lossy decoder can be obtained by replacing `Err` results with the replacement character:
|
||||
///
|
||||
/// ```
|
||||
/// use std::char::{decode_utf16, REPLACEMENT_CHARACTER};
|
||||
///
|
||||
/// fn main() {
|
||||
/// // 𝄞mus<invalid>ic<invalid>
|
||||
/// let v = [0xD834, 0xDD1E, 0x006d, 0x0075,
|
||||
/// 0x0073, 0xDD1E, 0x0069, 0x0063,
|
||||
/// 0xD834];
|
||||
///
|
||||
/// assert_eq!(decode_utf16(v.iter().cloned())
|
||||
/// .map(|r| r.unwrap_or(REPLACEMENT_CHARACTER))
|
||||
/// .collect::<String>(),
|
||||
/// "𝄞mus<75>ic<69>");
|
||||
/// }
|
||||
/// ```
|
||||
#[stable(feature = "decode_utf16", since = "1.9.0")]
|
||||
#[inline]
|
||||
pub fn decode_utf16<I: IntoIterator<Item = u16>>(iter: I) -> DecodeUtf16<I::IntoIter> {
|
||||
DecodeUtf16 {
|
||||
iter: iter.into_iter(),
|
||||
buf: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "decode_utf16", since = "1.9.0")]
|
||||
impl<I: Iterator<Item = u16>> Iterator for DecodeUtf16<I> {
|
||||
type Item = Result<char, DecodeUtf16Error>;
|
||||
|
||||
fn next(&mut self) -> Option<Result<char, DecodeUtf16Error>> {
|
||||
let u = match self.buf.take() {
|
||||
Some(buf) => buf,
|
||||
None => self.iter.next()?
|
||||
};
|
||||
|
||||
if u < 0xD800 || 0xDFFF < u {
|
||||
// not a surrogate
|
||||
Some(Ok(unsafe { from_u32_unchecked(u as u32) }))
|
||||
} else if u >= 0xDC00 {
|
||||
// a trailing surrogate
|
||||
Some(Err(DecodeUtf16Error { code: u }))
|
||||
} else {
|
||||
let u2 = match self.iter.next() {
|
||||
Some(u2) => u2,
|
||||
// eof
|
||||
None => return Some(Err(DecodeUtf16Error { code: u })),
|
||||
};
|
||||
if u2 < 0xDC00 || u2 > 0xDFFF {
|
||||
// not a trailing surrogate so we're not a valid
|
||||
// surrogate pair, so rewind to redecode u2 next time.
|
||||
self.buf = Some(u2);
|
||||
return Some(Err(DecodeUtf16Error { code: u }));
|
||||
}
|
||||
|
||||
// all ok, so lets decode it.
|
||||
let c = (((u - 0xD800) as u32) << 10 | (u2 - 0xDC00) as u32) + 0x1_0000;
|
||||
Some(Ok(unsafe { from_u32_unchecked(c) }))
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
let (low, high) = self.iter.size_hint();
|
||||
// we could be entirely valid surrogates (2 elements per
|
||||
// char), or entirely non-surrogates (1 element per char)
|
||||
(low / 2, high)
|
||||
}
|
||||
}
|
||||
|
||||
impl DecodeUtf16Error {
|
||||
/// Returns the unpaired surrogate which caused this error.
|
||||
#[stable(feature = "decode_utf16", since = "1.9.0")]
|
||||
pub fn unpaired_surrogate(&self) -> u16 {
|
||||
self.code
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "decode_utf16", since = "1.9.0")]
|
||||
impl fmt::Display for DecodeUtf16Error {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
write!(f, "unpaired surrogate found: {:x}", self.code)
|
||||
}
|
||||
}
|
||||
|
|
@ -8,169 +8,13 @@
|
|||
// option. This file may not be copied, modified, or distributed
|
||||
// except according to those terms.
|
||||
|
||||
//! A character type.
|
||||
//!
|
||||
//! The `char` type represents a single character. More specifically, since
|
||||
//! 'character' isn't a well-defined concept in Unicode, `char` is a '[Unicode
|
||||
//! scalar value]', which is similar to, but not the same as, a '[Unicode code
|
||||
//! point]'.
|
||||
//!
|
||||
//! [Unicode scalar value]: http://www.unicode.org/glossary/#unicode_scalar_value
|
||||
//! [Unicode code point]: http://www.unicode.org/glossary/#code_point
|
||||
//!
|
||||
//! This module exists for technical reasons, the primary documentation for
|
||||
//! `char` is directly on [the `char` primitive type](../../std/primitive.char.html)
|
||||
//! itself.
|
||||
//!
|
||||
//! This module is the home of the iterator implementations for the iterators
|
||||
//! implemented on `char`, as well as some useful constants and conversion
|
||||
//! functions that convert various types to `char`.
|
||||
//! impl char {}
|
||||
|
||||
#![stable(feature = "rust1", since = "1.0.0")]
|
||||
|
||||
use core::char::CharExt as C;
|
||||
use core::iter::FusedIterator;
|
||||
use core::fmt::{self, Write};
|
||||
use tables::{conversions, derived_property, general_category, property};
|
||||
|
||||
// stable re-exports
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub use core::char::{MAX, from_digit, from_u32, from_u32_unchecked};
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub use core::char::{EscapeDebug, EscapeDefault, EscapeUnicode};
|
||||
#[stable(feature = "char_from_str", since = "1.20.0")]
|
||||
pub use core::char::ParseCharError;
|
||||
|
||||
// unstable re-exports
|
||||
#[stable(feature = "try_from", since = "1.26.0")]
|
||||
pub use core::char::CharTryFromError;
|
||||
#[unstable(feature = "decode_utf8", issue = "33906")]
|
||||
pub use core::char::{DecodeUtf8, decode_utf8};
|
||||
#[unstable(feature = "unicode", issue = "27783")]
|
||||
pub use tables::{UNICODE_VERSION};
|
||||
#[unstable(feature = "unicode", issue = "27783")]
|
||||
pub use version::UnicodeVersion;
|
||||
|
||||
/// Returns an iterator that yields the lowercase equivalent of a `char`.
|
||||
///
|
||||
/// This `struct` is created by the [`to_lowercase`] method on [`char`]. See
|
||||
/// its documentation for more.
|
||||
///
|
||||
/// [`to_lowercase`]: ../../std/primitive.char.html#method.to_lowercase
|
||||
/// [`char`]: ../../std/primitive.char.html
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ToLowercase(CaseMappingIter);
|
||||
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
impl Iterator for ToLowercase {
|
||||
type Item = char;
|
||||
fn next(&mut self) -> Option<char> {
|
||||
self.0.next()
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "fused", since = "1.26.0")]
|
||||
impl FusedIterator for ToLowercase {}
|
||||
|
||||
/// Returns an iterator that yields the uppercase equivalent of a `char`.
|
||||
///
|
||||
/// This `struct` is created by the [`to_uppercase`] method on [`char`]. See
|
||||
/// its documentation for more.
|
||||
///
|
||||
/// [`to_uppercase`]: ../../std/primitive.char.html#method.to_uppercase
|
||||
/// [`char`]: ../../std/primitive.char.html
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ToUppercase(CaseMappingIter);
|
||||
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
impl Iterator for ToUppercase {
|
||||
type Item = char;
|
||||
fn next(&mut self) -> Option<char> {
|
||||
self.0.next()
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "fused", since = "1.26.0")]
|
||||
impl FusedIterator for ToUppercase {}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
enum CaseMappingIter {
|
||||
Three(char, char, char),
|
||||
Two(char, char),
|
||||
One(char),
|
||||
Zero,
|
||||
}
|
||||
|
||||
impl CaseMappingIter {
|
||||
fn new(chars: [char; 3]) -> CaseMappingIter {
|
||||
if chars[2] == '\0' {
|
||||
if chars[1] == '\0' {
|
||||
CaseMappingIter::One(chars[0]) // Including if chars[0] == '\0'
|
||||
} else {
|
||||
CaseMappingIter::Two(chars[0], chars[1])
|
||||
}
|
||||
} else {
|
||||
CaseMappingIter::Three(chars[0], chars[1], chars[2])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Iterator for CaseMappingIter {
|
||||
type Item = char;
|
||||
fn next(&mut self) -> Option<char> {
|
||||
match *self {
|
||||
CaseMappingIter::Three(a, b, c) => {
|
||||
*self = CaseMappingIter::Two(b, c);
|
||||
Some(a)
|
||||
}
|
||||
CaseMappingIter::Two(b, c) => {
|
||||
*self = CaseMappingIter::One(c);
|
||||
Some(b)
|
||||
}
|
||||
CaseMappingIter::One(c) => {
|
||||
*self = CaseMappingIter::Zero;
|
||||
Some(c)
|
||||
}
|
||||
CaseMappingIter::Zero => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for CaseMappingIter {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
match *self {
|
||||
CaseMappingIter::Three(a, b, c) => {
|
||||
f.write_char(a)?;
|
||||
f.write_char(b)?;
|
||||
f.write_char(c)
|
||||
}
|
||||
CaseMappingIter::Two(b, c) => {
|
||||
f.write_char(b)?;
|
||||
f.write_char(c)
|
||||
}
|
||||
CaseMappingIter::One(c) => {
|
||||
f.write_char(c)
|
||||
}
|
||||
CaseMappingIter::Zero => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "char_struct_display", since = "1.16.0")]
|
||||
impl fmt::Display for ToLowercase {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
fmt::Display::fmt(&self.0, f)
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "char_struct_display", since = "1.16.0")]
|
||||
impl fmt::Display for ToUppercase {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
fmt::Display::fmt(&self.0, f)
|
||||
}
|
||||
}
|
||||
use slice;
|
||||
use str::from_utf8_unchecked_mut;
|
||||
use super::*;
|
||||
use unicode::printable::is_printable;
|
||||
use unicode::tables::{conversions, derived_property, general_category, property};
|
||||
|
||||
#[lang = "char"]
|
||||
impl char {
|
||||
|
|
@ -223,7 +67,7 @@ impl char {
|
|||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
#[inline]
|
||||
pub fn is_digit(self, radix: u32) -> bool {
|
||||
C::is_digit(self, radix)
|
||||
self.to_digit(radix).is_some()
|
||||
}
|
||||
|
||||
/// Converts a `char` to a digit in the given radix.
|
||||
|
|
@ -277,7 +121,17 @@ impl char {
|
|||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
#[inline]
|
||||
pub fn to_digit(self, radix: u32) -> Option<u32> {
|
||||
C::to_digit(self, radix)
|
||||
if radix > 36 {
|
||||
panic!("to_digit: radix is too high (maximum 36)");
|
||||
}
|
||||
let val = match self {
|
||||
'0' ... '9' => self as u32 - '0' as u32,
|
||||
'a' ... 'z' => self as u32 - 'a' as u32 + 10,
|
||||
'A' ... 'Z' => self as u32 - 'A' as u32 + 10,
|
||||
_ => return None,
|
||||
};
|
||||
if val < radix { Some(val) }
|
||||
else { None }
|
||||
}
|
||||
|
||||
/// Returns an iterator that yields the hexadecimal Unicode escape of a
|
||||
|
|
@ -317,7 +171,20 @@ impl char {
|
|||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
#[inline]
|
||||
pub fn escape_unicode(self) -> EscapeUnicode {
|
||||
C::escape_unicode(self)
|
||||
let c = self as u32;
|
||||
|
||||
// or-ing 1 ensures that for c==0 the code computes that one
|
||||
// digit should be printed and (which is the same) avoids the
|
||||
// (31 - 32) underflow
|
||||
let msb = 31 - (c | 1).leading_zeros();
|
||||
|
||||
// the index of the most significant hex digit
|
||||
let ms_hex_digit = msb / 4;
|
||||
EscapeUnicode {
|
||||
c: self,
|
||||
state: EscapeUnicodeState::Backslash,
|
||||
hex_digit_idx: ms_hex_digit as usize,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns an iterator that yields the literal escape code of a character
|
||||
|
|
@ -357,7 +224,15 @@ impl char {
|
|||
#[stable(feature = "char_escape_debug", since = "1.20.0")]
|
||||
#[inline]
|
||||
pub fn escape_debug(self) -> EscapeDebug {
|
||||
C::escape_debug(self)
|
||||
let init_state = match self {
|
||||
'\t' => EscapeDefaultState::Backslash('t'),
|
||||
'\r' => EscapeDefaultState::Backslash('r'),
|
||||
'\n' => EscapeDefaultState::Backslash('n'),
|
||||
'\\' | '\'' | '"' => EscapeDefaultState::Backslash(self),
|
||||
c if is_printable(c) => EscapeDefaultState::Char(c),
|
||||
c => EscapeDefaultState::Unicode(c.escape_unicode()),
|
||||
};
|
||||
EscapeDebug(EscapeDefault { state: init_state })
|
||||
}
|
||||
|
||||
/// Returns an iterator that yields the literal escape code of a character
|
||||
|
|
@ -412,7 +287,15 @@ impl char {
|
|||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
#[inline]
|
||||
pub fn escape_default(self) -> EscapeDefault {
|
||||
C::escape_default(self)
|
||||
let init_state = match self {
|
||||
'\t' => EscapeDefaultState::Backslash('t'),
|
||||
'\r' => EscapeDefaultState::Backslash('r'),
|
||||
'\n' => EscapeDefaultState::Backslash('n'),
|
||||
'\\' | '\'' | '"' => EscapeDefaultState::Backslash(self),
|
||||
'\x20' ... '\x7e' => EscapeDefaultState::Char(self),
|
||||
_ => EscapeDefaultState::Unicode(self.escape_unicode())
|
||||
};
|
||||
EscapeDefault { state: init_state }
|
||||
}
|
||||
|
||||
/// Returns the number of bytes this `char` would need if encoded in UTF-8.
|
||||
|
|
@ -463,7 +346,16 @@ impl char {
|
|||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
#[inline]
|
||||
pub fn len_utf8(self) -> usize {
|
||||
C::len_utf8(self)
|
||||
let code = self as u32;
|
||||
if code < MAX_ONE_B {
|
||||
1
|
||||
} else if code < MAX_TWO_B {
|
||||
2
|
||||
} else if code < MAX_THREE_B {
|
||||
3
|
||||
} else {
|
||||
4
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of 16-bit code units this `char` would need if
|
||||
|
|
@ -488,7 +380,8 @@ impl char {
|
|||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
#[inline]
|
||||
pub fn len_utf16(self) -> usize {
|
||||
C::len_utf16(self)
|
||||
let ch = self as u32;
|
||||
if (ch & 0xFFFF) == ch { 1 } else { 2 }
|
||||
}
|
||||
|
||||
/// Encodes this character as UTF-8 into the provided byte buffer,
|
||||
|
|
@ -530,7 +423,35 @@ impl char {
|
|||
#[stable(feature = "unicode_encode_char", since = "1.15.0")]
|
||||
#[inline]
|
||||
pub fn encode_utf8(self, dst: &mut [u8]) -> &mut str {
|
||||
C::encode_utf8(self, dst)
|
||||
let code = self as u32;
|
||||
unsafe {
|
||||
let len =
|
||||
if code < MAX_ONE_B && !dst.is_empty() {
|
||||
*dst.get_unchecked_mut(0) = code as u8;
|
||||
1
|
||||
} else if code < MAX_TWO_B && dst.len() >= 2 {
|
||||
*dst.get_unchecked_mut(0) = (code >> 6 & 0x1F) as u8 | TAG_TWO_B;
|
||||
*dst.get_unchecked_mut(1) = (code & 0x3F) as u8 | TAG_CONT;
|
||||
2
|
||||
} else if code < MAX_THREE_B && dst.len() >= 3 {
|
||||
*dst.get_unchecked_mut(0) = (code >> 12 & 0x0F) as u8 | TAG_THREE_B;
|
||||
*dst.get_unchecked_mut(1) = (code >> 6 & 0x3F) as u8 | TAG_CONT;
|
||||
*dst.get_unchecked_mut(2) = (code & 0x3F) as u8 | TAG_CONT;
|
||||
3
|
||||
} else if dst.len() >= 4 {
|
||||
*dst.get_unchecked_mut(0) = (code >> 18 & 0x07) as u8 | TAG_FOUR_B;
|
||||
*dst.get_unchecked_mut(1) = (code >> 12 & 0x3F) as u8 | TAG_CONT;
|
||||
*dst.get_unchecked_mut(2) = (code >> 6 & 0x3F) as u8 | TAG_CONT;
|
||||
*dst.get_unchecked_mut(3) = (code & 0x3F) as u8 | TAG_CONT;
|
||||
4
|
||||
} else {
|
||||
panic!("encode_utf8: need {} bytes to encode U+{:X}, but the buffer has {}",
|
||||
from_u32_unchecked(code).len_utf8(),
|
||||
code,
|
||||
dst.len())
|
||||
};
|
||||
from_utf8_unchecked_mut(dst.get_unchecked_mut(..len))
|
||||
}
|
||||
}
|
||||
|
||||
/// Encodes this character as UTF-16 into the provided `u16` buffer,
|
||||
|
|
@ -570,7 +491,25 @@ impl char {
|
|||
#[stable(feature = "unicode_encode_char", since = "1.15.0")]
|
||||
#[inline]
|
||||
pub fn encode_utf16(self, dst: &mut [u16]) -> &mut [u16] {
|
||||
C::encode_utf16(self, dst)
|
||||
let mut code = self as u32;
|
||||
unsafe {
|
||||
if (code & 0xFFFF) == code && !dst.is_empty() {
|
||||
// The BMP falls through (assuming non-surrogate, as it should)
|
||||
*dst.get_unchecked_mut(0) = code as u16;
|
||||
slice::from_raw_parts_mut(dst.as_mut_ptr(), 1)
|
||||
} else if dst.len() >= 2 {
|
||||
// Supplementary planes break into surrogates.
|
||||
code -= 0x1_0000;
|
||||
*dst.get_unchecked_mut(0) = 0xD800 | ((code >> 10) as u16);
|
||||
*dst.get_unchecked_mut(1) = 0xDC00 | ((code as u16) & 0x3FF);
|
||||
slice::from_raw_parts_mut(dst.as_mut_ptr(), 2)
|
||||
} else {
|
||||
panic!("encode_utf16: need {} units to encode U+{:X}, but the buffer has {}",
|
||||
from_u32_unchecked(code).len_utf16(),
|
||||
code,
|
||||
dst.len())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if this `char` is an alphabetic code point, and false if not.
|
||||
|
|
@ -1452,140 +1391,3 @@ impl char {
|
|||
self.is_ascii() && (*self as u8).is_ascii_control()
|
||||
}
|
||||
}
|
||||
|
||||
/// An iterator that decodes UTF-16 encoded code points from an iterator of `u16`s.
|
||||
#[stable(feature = "decode_utf16", since = "1.9.0")]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct DecodeUtf16<I>
|
||||
where I: Iterator<Item = u16>
|
||||
{
|
||||
iter: I,
|
||||
buf: Option<u16>,
|
||||
}
|
||||
|
||||
/// An error that can be returned when decoding UTF-16 code points.
|
||||
#[stable(feature = "decode_utf16", since = "1.9.0")]
|
||||
#[derive(Debug, Clone, Eq, PartialEq)]
|
||||
pub struct DecodeUtf16Error {
|
||||
code: u16,
|
||||
}
|
||||
|
||||
/// Create an iterator over the UTF-16 encoded code points in `iter`,
|
||||
/// returning unpaired surrogates as `Err`s.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// Basic usage:
|
||||
///
|
||||
/// ```
|
||||
/// use std::char::decode_utf16;
|
||||
///
|
||||
/// fn main() {
|
||||
/// // 𝄞mus<invalid>ic<invalid>
|
||||
/// let v = [0xD834, 0xDD1E, 0x006d, 0x0075,
|
||||
/// 0x0073, 0xDD1E, 0x0069, 0x0063,
|
||||
/// 0xD834];
|
||||
///
|
||||
/// assert_eq!(decode_utf16(v.iter().cloned())
|
||||
/// .map(|r| r.map_err(|e| e.unpaired_surrogate()))
|
||||
/// .collect::<Vec<_>>(),
|
||||
/// vec![Ok('𝄞'),
|
||||
/// Ok('m'), Ok('u'), Ok('s'),
|
||||
/// Err(0xDD1E),
|
||||
/// Ok('i'), Ok('c'),
|
||||
/// Err(0xD834)]);
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// A lossy decoder can be obtained by replacing `Err` results with the replacement character:
|
||||
///
|
||||
/// ```
|
||||
/// use std::char::{decode_utf16, REPLACEMENT_CHARACTER};
|
||||
///
|
||||
/// fn main() {
|
||||
/// // 𝄞mus<invalid>ic<invalid>
|
||||
/// let v = [0xD834, 0xDD1E, 0x006d, 0x0075,
|
||||
/// 0x0073, 0xDD1E, 0x0069, 0x0063,
|
||||
/// 0xD834];
|
||||
///
|
||||
/// assert_eq!(decode_utf16(v.iter().cloned())
|
||||
/// .map(|r| r.unwrap_or(REPLACEMENT_CHARACTER))
|
||||
/// .collect::<String>(),
|
||||
/// "𝄞mus<75>ic<69>");
|
||||
/// }
|
||||
/// ```
|
||||
#[stable(feature = "decode_utf16", since = "1.9.0")]
|
||||
#[inline]
|
||||
pub fn decode_utf16<I: IntoIterator<Item = u16>>(iter: I) -> DecodeUtf16<I::IntoIter> {
|
||||
DecodeUtf16 {
|
||||
iter: iter.into_iter(),
|
||||
buf: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "decode_utf16", since = "1.9.0")]
|
||||
impl<I: Iterator<Item = u16>> Iterator for DecodeUtf16<I> {
|
||||
type Item = Result<char, DecodeUtf16Error>;
|
||||
|
||||
fn next(&mut self) -> Option<Result<char, DecodeUtf16Error>> {
|
||||
let u = match self.buf.take() {
|
||||
Some(buf) => buf,
|
||||
None => self.iter.next()?
|
||||
};
|
||||
|
||||
if u < 0xD800 || 0xDFFF < u {
|
||||
// not a surrogate
|
||||
Some(Ok(unsafe { from_u32_unchecked(u as u32) }))
|
||||
} else if u >= 0xDC00 {
|
||||
// a trailing surrogate
|
||||
Some(Err(DecodeUtf16Error { code: u }))
|
||||
} else {
|
||||
let u2 = match self.iter.next() {
|
||||
Some(u2) => u2,
|
||||
// eof
|
||||
None => return Some(Err(DecodeUtf16Error { code: u })),
|
||||
};
|
||||
if u2 < 0xDC00 || u2 > 0xDFFF {
|
||||
// not a trailing surrogate so we're not a valid
|
||||
// surrogate pair, so rewind to redecode u2 next time.
|
||||
self.buf = Some(u2);
|
||||
return Some(Err(DecodeUtf16Error { code: u }));
|
||||
}
|
||||
|
||||
// all ok, so lets decode it.
|
||||
let c = (((u - 0xD800) as u32) << 10 | (u2 - 0xDC00) as u32) + 0x1_0000;
|
||||
Some(Ok(unsafe { from_u32_unchecked(c) }))
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
let (low, high) = self.iter.size_hint();
|
||||
// we could be entirely valid surrogates (2 elements per
|
||||
// char), or entirely non-surrogates (1 element per char)
|
||||
(low / 2, high)
|
||||
}
|
||||
}
|
||||
|
||||
impl DecodeUtf16Error {
|
||||
/// Returns the unpaired surrogate which caused this error.
|
||||
#[stable(feature = "decode_utf16", since = "1.9.0")]
|
||||
pub fn unpaired_surrogate(&self) -> u16 {
|
||||
self.code
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "decode_utf16", since = "1.9.0")]
|
||||
impl fmt::Display for DecodeUtf16Error {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
write!(f, "unpaired surrogate found: {:x}", self.code)
|
||||
}
|
||||
}
|
||||
|
||||
/// `U+FFFD REPLACEMENT CHARACTER` (<28>) is used in Unicode to represent a
|
||||
/// decoding error.
|
||||
///
|
||||
/// It can occur, for example, when giving ill-formed UTF-8 bytes to
|
||||
/// [`String::from_utf8_lossy`](../../std/string/struct.String.html#method.from_utf8_lossy).
|
||||
#[stable(feature = "decode_utf16", since = "1.9.0")]
|
||||
pub const REPLACEMENT_CHARACTER: char = '\u{FFFD}';
|
||||
506
src/libcore/char/mod.rs
Normal file
506
src/libcore/char/mod.rs
Normal file
|
|
@ -0,0 +1,506 @@
|
|||
// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
|
||||
// file at the top-level directory of this distribution and at
|
||||
// http://rust-lang.org/COPYRIGHT.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
||||
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
||||
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
||||
// option. This file may not be copied, modified, or distributed
|
||||
// except according to those terms.
|
||||
|
||||
//! A character type.
|
||||
//!
|
||||
//! The `char` type represents a single character. More specifically, since
|
||||
//! 'character' isn't a well-defined concept in Unicode, `char` is a '[Unicode
|
||||
//! scalar value]', which is similar to, but not the same as, a '[Unicode code
|
||||
//! point]'.
|
||||
//!
|
||||
//! [Unicode scalar value]: http://www.unicode.org/glossary/#unicode_scalar_value
|
||||
//! [Unicode code point]: http://www.unicode.org/glossary/#code_point
|
||||
//!
|
||||
//! This module exists for technical reasons, the primary documentation for
|
||||
//! `char` is directly on [the `char` primitive type](../../std/primitive.char.html)
|
||||
//! itself.
|
||||
//!
|
||||
//! This module is the home of the iterator implementations for the iterators
|
||||
//! implemented on `char`, as well as some useful constants and conversion
|
||||
//! functions that convert various types to `char`.
|
||||
|
||||
#![allow(non_snake_case)]
|
||||
#![stable(feature = "core_char", since = "1.2.0")]
|
||||
|
||||
mod convert;
|
||||
mod decode;
|
||||
mod methods;
|
||||
|
||||
// stable re-exports
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub use self::convert::{from_u32, from_digit};
|
||||
#[stable(feature = "char_from_unchecked", since = "1.5.0")]
|
||||
pub use self::convert::from_u32_unchecked;
|
||||
#[stable(feature = "char_from_str", since = "1.20.0")]
|
||||
pub use self::convert::ParseCharError;
|
||||
#[stable(feature = "try_from", since = "1.26.0")]
|
||||
pub use self::convert::CharTryFromError;
|
||||
#[stable(feature = "decode_utf16", since = "1.9.0")]
|
||||
pub use self::decode::{decode_utf16, DecodeUtf16, DecodeUtf16Error};
|
||||
|
||||
// unstable re-exports
|
||||
#[unstable(feature = "unicode_version", issue = "49726")]
|
||||
pub use unicode::tables::UNICODE_VERSION;
|
||||
#[unstable(feature = "unicode_version", issue = "49726")]
|
||||
pub use unicode::version::UnicodeVersion;
|
||||
#[unstable(feature = "decode_utf8", issue = "33906")]
|
||||
pub use self::decode::{decode_utf8, DecodeUtf8, InvalidSequence};
|
||||
|
||||
use fmt::{self, Write};
|
||||
use iter::FusedIterator;
|
||||
|
||||
// UTF-8 ranges and tags for encoding characters
|
||||
const TAG_CONT: u8 = 0b1000_0000;
|
||||
const TAG_TWO_B: u8 = 0b1100_0000;
|
||||
const TAG_THREE_B: u8 = 0b1110_0000;
|
||||
const TAG_FOUR_B: u8 = 0b1111_0000;
|
||||
const MAX_ONE_B: u32 = 0x80;
|
||||
const MAX_TWO_B: u32 = 0x800;
|
||||
const MAX_THREE_B: u32 = 0x10000;
|
||||
|
||||
/*
|
||||
Lu Uppercase_Letter an uppercase letter
|
||||
Ll Lowercase_Letter a lowercase letter
|
||||
Lt Titlecase_Letter a digraphic character, with first part uppercase
|
||||
Lm Modifier_Letter a modifier letter
|
||||
Lo Other_Letter other letters, including syllables and ideographs
|
||||
Mn Nonspacing_Mark a nonspacing combining mark (zero advance width)
|
||||
Mc Spacing_Mark a spacing combining mark (positive advance width)
|
||||
Me Enclosing_Mark an enclosing combining mark
|
||||
Nd Decimal_Number a decimal digit
|
||||
Nl Letter_Number a letterlike numeric character
|
||||
No Other_Number a numeric character of other type
|
||||
Pc Connector_Punctuation a connecting punctuation mark, like a tie
|
||||
Pd Dash_Punctuation a dash or hyphen punctuation mark
|
||||
Ps Open_Punctuation an opening punctuation mark (of a pair)
|
||||
Pe Close_Punctuation a closing punctuation mark (of a pair)
|
||||
Pi Initial_Punctuation an initial quotation mark
|
||||
Pf Final_Punctuation a final quotation mark
|
||||
Po Other_Punctuation a punctuation mark of other type
|
||||
Sm Math_Symbol a symbol of primarily mathematical use
|
||||
Sc Currency_Symbol a currency sign
|
||||
Sk Modifier_Symbol a non-letterlike modifier symbol
|
||||
So Other_Symbol a symbol of other type
|
||||
Zs Space_Separator a space character (of various non-zero widths)
|
||||
Zl Line_Separator U+2028 LINE SEPARATOR only
|
||||
Zp Paragraph_Separator U+2029 PARAGRAPH SEPARATOR only
|
||||
Cc Control a C0 or C1 control code
|
||||
Cf Format a format control character
|
||||
Cs Surrogate a surrogate code point
|
||||
Co Private_Use a private-use character
|
||||
Cn Unassigned a reserved unassigned code point or a noncharacter
|
||||
*/
|
||||
|
||||
/// The highest valid code point a `char` can have.
|
||||
///
|
||||
/// A [`char`] is a [Unicode Scalar Value], which means that it is a [Code
|
||||
/// Point], but only ones within a certain range. `MAX` is the highest valid
|
||||
/// code point that's a valid [Unicode Scalar Value].
|
||||
///
|
||||
/// [`char`]: ../../std/primitive.char.html
|
||||
/// [Unicode Scalar Value]: http://www.unicode.org/glossary/#unicode_scalar_value
|
||||
/// [Code Point]: http://www.unicode.org/glossary/#code_point
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub const MAX: char = '\u{10ffff}';
|
||||
|
||||
/// `U+FFFD REPLACEMENT CHARACTER` (<28>) is used in Unicode to represent a
|
||||
/// decoding error.
|
||||
///
|
||||
/// It can occur, for example, when giving ill-formed UTF-8 bytes to
|
||||
/// [`String::from_utf8_lossy`](../../std/string/struct.String.html#method.from_utf8_lossy).
|
||||
#[stable(feature = "decode_utf16", since = "1.9.0")]
|
||||
pub const REPLACEMENT_CHARACTER: char = '\u{FFFD}';
|
||||
|
||||
/// Returns an iterator that yields the hexadecimal Unicode escape of a
|
||||
/// character, as `char`s.
|
||||
///
|
||||
/// This `struct` is created by the [`escape_unicode`] method on [`char`]. See
|
||||
/// its documentation for more.
|
||||
///
|
||||
/// [`escape_unicode`]: ../../std/primitive.char.html#method.escape_unicode
|
||||
/// [`char`]: ../../std/primitive.char.html
|
||||
#[derive(Clone, Debug)]
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub struct EscapeUnicode {
|
||||
c: char,
|
||||
state: EscapeUnicodeState,
|
||||
|
||||
// The index of the next hex digit to be printed (0 if none),
|
||||
// i.e. the number of remaining hex digits to be printed;
|
||||
// increasing from the least significant digit: 0x543210
|
||||
hex_digit_idx: usize,
|
||||
}
|
||||
|
||||
// The enum values are ordered so that their representation is the
|
||||
// same as the remaining length (besides the hexadecimal digits). This
|
||||
// likely makes `len()` a single load from memory) and inline-worth.
|
||||
#[derive(Clone, Debug)]
|
||||
enum EscapeUnicodeState {
|
||||
Done,
|
||||
RightBrace,
|
||||
Value,
|
||||
LeftBrace,
|
||||
Type,
|
||||
Backslash,
|
||||
}
|
||||
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
impl Iterator for EscapeUnicode {
|
||||
type Item = char;
|
||||
|
||||
fn next(&mut self) -> Option<char> {
|
||||
match self.state {
|
||||
EscapeUnicodeState::Backslash => {
|
||||
self.state = EscapeUnicodeState::Type;
|
||||
Some('\\')
|
||||
}
|
||||
EscapeUnicodeState::Type => {
|
||||
self.state = EscapeUnicodeState::LeftBrace;
|
||||
Some('u')
|
||||
}
|
||||
EscapeUnicodeState::LeftBrace => {
|
||||
self.state = EscapeUnicodeState::Value;
|
||||
Some('{')
|
||||
}
|
||||
EscapeUnicodeState::Value => {
|
||||
let hex_digit = ((self.c as u32) >> (self.hex_digit_idx * 4)) & 0xf;
|
||||
let c = from_digit(hex_digit, 16).unwrap();
|
||||
if self.hex_digit_idx == 0 {
|
||||
self.state = EscapeUnicodeState::RightBrace;
|
||||
} else {
|
||||
self.hex_digit_idx -= 1;
|
||||
}
|
||||
Some(c)
|
||||
}
|
||||
EscapeUnicodeState::RightBrace => {
|
||||
self.state = EscapeUnicodeState::Done;
|
||||
Some('}')
|
||||
}
|
||||
EscapeUnicodeState::Done => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
let n = self.len();
|
||||
(n, Some(n))
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn count(self) -> usize {
|
||||
self.len()
|
||||
}
|
||||
|
||||
fn last(self) -> Option<char> {
|
||||
match self.state {
|
||||
EscapeUnicodeState::Done => None,
|
||||
|
||||
EscapeUnicodeState::RightBrace |
|
||||
EscapeUnicodeState::Value |
|
||||
EscapeUnicodeState::LeftBrace |
|
||||
EscapeUnicodeState::Type |
|
||||
EscapeUnicodeState::Backslash => Some('}'),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "exact_size_escape", since = "1.11.0")]
|
||||
impl ExactSizeIterator for EscapeUnicode {
|
||||
#[inline]
|
||||
fn len(&self) -> usize {
|
||||
// The match is a single memory access with no branching
|
||||
self.hex_digit_idx + match self.state {
|
||||
EscapeUnicodeState::Done => 0,
|
||||
EscapeUnicodeState::RightBrace => 1,
|
||||
EscapeUnicodeState::Value => 2,
|
||||
EscapeUnicodeState::LeftBrace => 3,
|
||||
EscapeUnicodeState::Type => 4,
|
||||
EscapeUnicodeState::Backslash => 5,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "fused", since = "1.26.0")]
|
||||
impl FusedIterator for EscapeUnicode {}
|
||||
|
||||
#[stable(feature = "char_struct_display", since = "1.16.0")]
|
||||
impl fmt::Display for EscapeUnicode {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
for c in self.clone() {
|
||||
f.write_char(c)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// An iterator that yields the literal escape code of a `char`.
|
||||
///
|
||||
/// This `struct` is created by the [`escape_default`] method on [`char`]. See
|
||||
/// its documentation for more.
|
||||
///
|
||||
/// [`escape_default`]: ../../std/primitive.char.html#method.escape_default
|
||||
/// [`char`]: ../../std/primitive.char.html
|
||||
#[derive(Clone, Debug)]
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub struct EscapeDefault {
|
||||
state: EscapeDefaultState
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
enum EscapeDefaultState {
|
||||
Done,
|
||||
Char(char),
|
||||
Backslash(char),
|
||||
Unicode(EscapeUnicode),
|
||||
}
|
||||
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
impl Iterator for EscapeDefault {
|
||||
type Item = char;
|
||||
|
||||
fn next(&mut self) -> Option<char> {
|
||||
match self.state {
|
||||
EscapeDefaultState::Backslash(c) => {
|
||||
self.state = EscapeDefaultState::Char(c);
|
||||
Some('\\')
|
||||
}
|
||||
EscapeDefaultState::Char(c) => {
|
||||
self.state = EscapeDefaultState::Done;
|
||||
Some(c)
|
||||
}
|
||||
EscapeDefaultState::Done => None,
|
||||
EscapeDefaultState::Unicode(ref mut iter) => iter.next(),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
let n = self.len();
|
||||
(n, Some(n))
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn count(self) -> usize {
|
||||
self.len()
|
||||
}
|
||||
|
||||
fn nth(&mut self, n: usize) -> Option<char> {
|
||||
match self.state {
|
||||
EscapeDefaultState::Backslash(c) if n == 0 => {
|
||||
self.state = EscapeDefaultState::Char(c);
|
||||
Some('\\')
|
||||
},
|
||||
EscapeDefaultState::Backslash(c) if n == 1 => {
|
||||
self.state = EscapeDefaultState::Done;
|
||||
Some(c)
|
||||
},
|
||||
EscapeDefaultState::Backslash(_) => {
|
||||
self.state = EscapeDefaultState::Done;
|
||||
None
|
||||
},
|
||||
EscapeDefaultState::Char(c) => {
|
||||
self.state = EscapeDefaultState::Done;
|
||||
|
||||
if n == 0 {
|
||||
Some(c)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
},
|
||||
EscapeDefaultState::Done => return None,
|
||||
EscapeDefaultState::Unicode(ref mut i) => return i.nth(n),
|
||||
}
|
||||
}
|
||||
|
||||
fn last(self) -> Option<char> {
|
||||
match self.state {
|
||||
EscapeDefaultState::Unicode(iter) => iter.last(),
|
||||
EscapeDefaultState::Done => None,
|
||||
EscapeDefaultState::Backslash(c) | EscapeDefaultState::Char(c) => Some(c),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "exact_size_escape", since = "1.11.0")]
|
||||
impl ExactSizeIterator for EscapeDefault {
|
||||
fn len(&self) -> usize {
|
||||
match self.state {
|
||||
EscapeDefaultState::Done => 0,
|
||||
EscapeDefaultState::Char(_) => 1,
|
||||
EscapeDefaultState::Backslash(_) => 2,
|
||||
EscapeDefaultState::Unicode(ref iter) => iter.len(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "fused", since = "1.26.0")]
|
||||
impl FusedIterator for EscapeDefault {}
|
||||
|
||||
#[stable(feature = "char_struct_display", since = "1.16.0")]
|
||||
impl fmt::Display for EscapeDefault {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
for c in self.clone() {
|
||||
f.write_char(c)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// An iterator that yields the literal escape code of a `char`.
|
||||
///
|
||||
/// This `struct` is created by the [`escape_debug`] method on [`char`]. See its
|
||||
/// documentation for more.
|
||||
///
|
||||
/// [`escape_debug`]: ../../std/primitive.char.html#method.escape_debug
|
||||
/// [`char`]: ../../std/primitive.char.html
|
||||
#[stable(feature = "char_escape_debug", since = "1.20.0")]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct EscapeDebug(EscapeDefault);
|
||||
|
||||
#[stable(feature = "char_escape_debug", since = "1.20.0")]
|
||||
impl Iterator for EscapeDebug {
|
||||
type Item = char;
|
||||
fn next(&mut self) -> Option<char> { self.0.next() }
|
||||
fn size_hint(&self) -> (usize, Option<usize>) { self.0.size_hint() }
|
||||
}
|
||||
|
||||
#[stable(feature = "char_escape_debug", since = "1.20.0")]
|
||||
impl ExactSizeIterator for EscapeDebug { }
|
||||
|
||||
#[stable(feature = "fused", since = "1.26.0")]
|
||||
impl FusedIterator for EscapeDebug {}
|
||||
|
||||
#[stable(feature = "char_escape_debug", since = "1.20.0")]
|
||||
impl fmt::Display for EscapeDebug {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
fmt::Display::fmt(&self.0, f)
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns an iterator that yields the lowercase equivalent of a `char`.
|
||||
///
|
||||
/// This `struct` is created by the [`to_lowercase`] method on [`char`]. See
|
||||
/// its documentation for more.
|
||||
///
|
||||
/// [`to_lowercase`]: ../../std/primitive.char.html#method.to_lowercase
|
||||
/// [`char`]: ../../std/primitive.char.html
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ToLowercase(CaseMappingIter);
|
||||
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
impl Iterator for ToLowercase {
|
||||
type Item = char;
|
||||
fn next(&mut self) -> Option<char> {
|
||||
self.0.next()
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "fused", since = "1.26.0")]
|
||||
impl FusedIterator for ToLowercase {}
|
||||
|
||||
/// Returns an iterator that yields the uppercase equivalent of a `char`.
|
||||
///
|
||||
/// This `struct` is created by the [`to_uppercase`] method on [`char`]. See
|
||||
/// its documentation for more.
|
||||
///
|
||||
/// [`to_uppercase`]: ../../std/primitive.char.html#method.to_uppercase
|
||||
/// [`char`]: ../../std/primitive.char.html
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ToUppercase(CaseMappingIter);
|
||||
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
impl Iterator for ToUppercase {
|
||||
type Item = char;
|
||||
fn next(&mut self) -> Option<char> {
|
||||
self.0.next()
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "fused", since = "1.26.0")]
|
||||
impl FusedIterator for ToUppercase {}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
enum CaseMappingIter {
|
||||
Three(char, char, char),
|
||||
Two(char, char),
|
||||
One(char),
|
||||
Zero,
|
||||
}
|
||||
|
||||
impl CaseMappingIter {
|
||||
fn new(chars: [char; 3]) -> CaseMappingIter {
|
||||
if chars[2] == '\0' {
|
||||
if chars[1] == '\0' {
|
||||
CaseMappingIter::One(chars[0]) // Including if chars[0] == '\0'
|
||||
} else {
|
||||
CaseMappingIter::Two(chars[0], chars[1])
|
||||
}
|
||||
} else {
|
||||
CaseMappingIter::Three(chars[0], chars[1], chars[2])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Iterator for CaseMappingIter {
|
||||
type Item = char;
|
||||
fn next(&mut self) -> Option<char> {
|
||||
match *self {
|
||||
CaseMappingIter::Three(a, b, c) => {
|
||||
*self = CaseMappingIter::Two(b, c);
|
||||
Some(a)
|
||||
}
|
||||
CaseMappingIter::Two(b, c) => {
|
||||
*self = CaseMappingIter::One(c);
|
||||
Some(b)
|
||||
}
|
||||
CaseMappingIter::One(c) => {
|
||||
*self = CaseMappingIter::Zero;
|
||||
Some(c)
|
||||
}
|
||||
CaseMappingIter::Zero => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for CaseMappingIter {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
match *self {
|
||||
CaseMappingIter::Three(a, b, c) => {
|
||||
f.write_char(a)?;
|
||||
f.write_char(b)?;
|
||||
f.write_char(c)
|
||||
}
|
||||
CaseMappingIter::Two(b, c) => {
|
||||
f.write_char(b)?;
|
||||
f.write_char(c)
|
||||
}
|
||||
CaseMappingIter::One(c) => {
|
||||
f.write_char(c)
|
||||
}
|
||||
CaseMappingIter::Zero => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "char_struct_display", since = "1.16.0")]
|
||||
impl fmt::Display for ToLowercase {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
fmt::Display::fmt(&self.0, f)
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "char_struct_display", since = "1.16.0")]
|
||||
impl fmt::Display for ToUppercase {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
fmt::Display::fmt(&self.0, f)
|
||||
}
|
||||
}
|
||||
|
|
@ -180,12 +180,13 @@ pub mod hash;
|
|||
pub mod fmt;
|
||||
pub mod time;
|
||||
|
||||
pub mod unicode;
|
||||
|
||||
/* Heap memory allocator trait */
|
||||
#[allow(missing_docs)]
|
||||
pub mod heap;
|
||||
|
||||
// note: does not need to be public
|
||||
mod char_private;
|
||||
mod iter_private;
|
||||
mod tuple;
|
||||
mod unit;
|
||||
|
|
|
|||
|
|
@ -62,6 +62,3 @@ pub use slice::SliceExt;
|
|||
#[stable(feature = "core_prelude", since = "1.4.0")]
|
||||
#[doc(no_inline)]
|
||||
pub use str::StrExt;
|
||||
#[stable(feature = "core_prelude", since = "1.4.0")]
|
||||
#[doc(no_inline)]
|
||||
pub use char::CharExt;
|
||||
|
|
|
|||
|
|
@ -8,12 +8,11 @@
|
|||
// option. This file may not be copied, modified, or distributed
|
||||
// except according to those terms.
|
||||
|
||||
use core::str as core_str;
|
||||
use core::fmt;
|
||||
use core::fmt::Write;
|
||||
use char;
|
||||
use core::mem;
|
||||
|
||||
use str as core_str;
|
||||
use fmt;
|
||||
use fmt::Write;
|
||||
use mem;
|
||||
|
||||
/// Lossy UTF-8 string.
|
||||
#[unstable(feature = "str_internals", issue = "0")]
|
||||
|
|
@ -19,13 +19,17 @@ use self::pattern::{Searcher, ReverseSearcher, DoubleEndedSearcher};
|
|||
|
||||
use char;
|
||||
use fmt;
|
||||
use iter::{Map, Cloned, FusedIterator, TrustedLen};
|
||||
use iter::{Map, Cloned, FusedIterator, TrustedLen, Filter};
|
||||
use iter_private::TrustedRandomAccess;
|
||||
use slice::{self, SliceIndex};
|
||||
use mem;
|
||||
|
||||
pub mod pattern;
|
||||
|
||||
#[unstable(feature = "str_internals", issue = "0")]
|
||||
#[allow(missing_docs)]
|
||||
pub mod lossy;
|
||||
|
||||
/// A trait to abstract the idea of creating a new instance of a type from a
|
||||
/// string.
|
||||
///
|
||||
|
|
@ -2212,6 +2216,18 @@ pub trait StrExt {
|
|||
fn is_empty(&self) -> bool;
|
||||
#[stable(feature = "core", since = "1.6.0")]
|
||||
fn parse<T: FromStr>(&self) -> Result<T, T::Err>;
|
||||
#[stable(feature = "split_whitespace", since = "1.1.0")]
|
||||
fn split_whitespace<'a>(&'a self) -> SplitWhitespace<'a>;
|
||||
#[stable(feature = "unicode_methods_on_intrinsics", since = "1.27.0")]
|
||||
fn is_whitespace(&self) -> bool;
|
||||
#[stable(feature = "unicode_methods_on_intrinsics", since = "1.27.0")]
|
||||
fn is_alphanumeric(&self) -> bool;
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
fn trim(&self) -> &str;
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
fn trim_left(&self) -> &str;
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
fn trim_right(&self) -> &str;
|
||||
}
|
||||
|
||||
// truncate `&str` to length at most equal to `max`
|
||||
|
|
@ -2532,6 +2548,36 @@ impl StrExt for str {
|
|||
|
||||
#[inline]
|
||||
fn parse<T: FromStr>(&self) -> Result<T, T::Err> { FromStr::from_str(self) }
|
||||
|
||||
#[inline]
|
||||
fn split_whitespace(&self) -> SplitWhitespace {
|
||||
SplitWhitespace { inner: self.split(IsWhitespace).filter(IsNotEmpty) }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn is_whitespace(&self) -> bool {
|
||||
self.chars().all(|c| c.is_whitespace())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn is_alphanumeric(&self) -> bool {
|
||||
self.chars().all(|c| c.is_alphanumeric())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn trim(&self) -> &str {
|
||||
self.trim_matches(|c: char| c.is_whitespace())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn trim_left(&self) -> &str {
|
||||
self.trim_left_matches(|c: char| c.is_whitespace())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn trim_right(&self) -> &str {
|
||||
self.trim_right_matches(|c: char| c.is_whitespace())
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
|
|
@ -2547,3 +2593,75 @@ impl<'a> Default for &'a str {
|
|||
/// Creates an empty str
|
||||
fn default() -> &'a str { "" }
|
||||
}
|
||||
|
||||
/// An iterator over the non-whitespace substrings of a string,
|
||||
/// separated by any amount of whitespace.
|
||||
///
|
||||
/// This struct is created by the [`split_whitespace`] method on [`str`].
|
||||
/// See its documentation for more.
|
||||
///
|
||||
/// [`split_whitespace`]: ../../std/primitive.str.html#method.split_whitespace
|
||||
/// [`str`]: ../../std/primitive.str.html
|
||||
#[stable(feature = "split_whitespace", since = "1.1.0")]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SplitWhitespace<'a> {
|
||||
inner: Filter<Split<'a, IsWhitespace>, IsNotEmpty>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct IsWhitespace;
|
||||
|
||||
impl FnOnce<(char, )> for IsWhitespace {
|
||||
type Output = bool;
|
||||
|
||||
#[inline]
|
||||
extern "rust-call" fn call_once(mut self, arg: (char, )) -> bool {
|
||||
self.call_mut(arg)
|
||||
}
|
||||
}
|
||||
|
||||
impl FnMut<(char, )> for IsWhitespace {
|
||||
#[inline]
|
||||
extern "rust-call" fn call_mut(&mut self, arg: (char, )) -> bool {
|
||||
arg.0.is_whitespace()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct IsNotEmpty;
|
||||
|
||||
impl<'a, 'b> FnOnce<(&'a &'b str, )> for IsNotEmpty {
|
||||
type Output = bool;
|
||||
|
||||
#[inline]
|
||||
extern "rust-call" fn call_once(mut self, arg: (&&str, )) -> bool {
|
||||
self.call_mut(arg)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, 'b> FnMut<(&'a &'b str, )> for IsNotEmpty {
|
||||
#[inline]
|
||||
extern "rust-call" fn call_mut(&mut self, arg: (&&str, )) -> bool {
|
||||
!arg.0.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#[stable(feature = "split_whitespace", since = "1.1.0")]
|
||||
impl<'a> Iterator for SplitWhitespace<'a> {
|
||||
type Item = &'a str;
|
||||
|
||||
fn next(&mut self) -> Option<&'a str> {
|
||||
self.inner.next()
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "split_whitespace", since = "1.1.0")]
|
||||
impl<'a> DoubleEndedIterator for SplitWhitespace<'a> {
|
||||
fn next_back(&mut self) -> Option<&'a str> {
|
||||
self.inner.next_back()
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "fused", since = "1.26.0")]
|
||||
impl<'a> FusedIterator for SplitWhitespace<'a> {}
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@
|
|||
#![feature(sort_internals)]
|
||||
#![feature(specialization)]
|
||||
#![feature(step_trait)]
|
||||
#![feature(str_internals)]
|
||||
#![feature(test)]
|
||||
#![feature(trusted_len)]
|
||||
#![feature(try_trait)]
|
||||
|
|
@ -68,4 +69,5 @@ mod ptr;
|
|||
mod result;
|
||||
mod slice;
|
||||
mod str;
|
||||
mod str_lossy;
|
||||
mod tuple;
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
// option. This file may not be copied, modified, or distributed
|
||||
// except according to those terms.
|
||||
|
||||
use std_unicode::lossy::*;
|
||||
use core::str::lossy::*;
|
||||
|
||||
#[test]
|
||||
fn chunks() {
|
||||
27
src/libcore/unicode/mod.rs
Normal file
27
src/libcore/unicode/mod.rs
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
|
||||
// file at the top-level directory of this distribution and at
|
||||
// http://rust-lang.org/COPYRIGHT.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
||||
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
||||
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
||||
// option. This file may not be copied, modified, or distributed
|
||||
// except according to those terms.
|
||||
|
||||
#![unstable(feature = "unicode_internals", issue = "0")]
|
||||
#![allow(missing_docs)]
|
||||
|
||||
mod bool_trie;
|
||||
pub(crate) mod printable;
|
||||
pub(crate) mod tables;
|
||||
pub(crate) mod version;
|
||||
|
||||
// For use in liballoc, not re-exported in libstd.
|
||||
pub mod derived_property {
|
||||
pub use unicode::tables::derived_property::{Case_Ignorable, Cased};
|
||||
}
|
||||
|
||||
// For use in libsyntax
|
||||
pub mod property {
|
||||
pub use unicode::tables::property::Pattern_White_Space;
|
||||
}
|
||||
|
|
@ -187,7 +187,7 @@ def main():
|
|||
// option. This file may not be copied, modified, or distributed
|
||||
// except according to those terms.
|
||||
|
||||
// NOTE: The following code was generated by "src/etc/char_private.py",
|
||||
// NOTE: The following code was generated by "src/libcore/unicode/printable.py",
|
||||
// do not edit directly!
|
||||
|
||||
fn check(x: u16, singletonuppers: &[(u8, u8)], singletonlowers: &[u8],
|
||||
|
|
@ -8,7 +8,7 @@
|
|||
// option. This file may not be copied, modified, or distributed
|
||||
// except according to those terms.
|
||||
|
||||
// NOTE: The following code was generated by "src/etc/char_private.py",
|
||||
// NOTE: The following code was generated by "src/libcore/unicode/printable.py",
|
||||
// do not edit directly!
|
||||
|
||||
fn check(x: u16, singletonuppers: &[(u8, u8)], singletonlowers: &[u8],
|
||||
|
|
@ -12,11 +12,12 @@
|
|||
|
||||
#![allow(missing_docs, non_upper_case_globals, non_snake_case)]
|
||||
|
||||
use version::UnicodeVersion;
|
||||
use bool_trie::{BoolTrie, SmallBoolTrie};
|
||||
use unicode::version::UnicodeVersion;
|
||||
use unicode::bool_trie::{BoolTrie, SmallBoolTrie};
|
||||
|
||||
/// The version of [Unicode](http://www.unicode.org/) that the Unicode parts of
|
||||
/// `CharExt` and `UnicodeStrPrelude` traits are based on.
|
||||
/// `char` and `str` methods are based on.
|
||||
#[unstable(feature = "unicode_version", issue = "49726")]
|
||||
pub const UNICODE_VERSION: UnicodeVersion = UnicodeVersion {
|
||||
major: 10,
|
||||
minor: 0,
|
||||
|
|
@ -1138,9 +1139,6 @@ pub mod property {
|
|||
}
|
||||
|
||||
pub mod conversions {
|
||||
use core::option::Option;
|
||||
use core::option::Option::{Some, None};
|
||||
|
||||
pub fn to_lower(c: char) -> [char; 3] {
|
||||
match bsearch_case_table(c, to_lowercase_table) {
|
||||
None => [c, '\0', '\0'],
|
||||
|
|
@ -39,8 +39,8 @@ preamble = '''// Copyright 2012-2016 The Rust Project Developers. See the COPYRI
|
|||
|
||||
#![allow(missing_docs, non_upper_case_globals, non_snake_case)]
|
||||
|
||||
use version::UnicodeVersion;
|
||||
use bool_trie::{BoolTrie, SmallBoolTrie};
|
||||
use unicode::version::UnicodeVersion;
|
||||
use unicode::bool_trie::{BoolTrie, SmallBoolTrie};
|
||||
'''
|
||||
|
||||
# Mapping taken from Table 12 from:
|
||||
|
|
@ -408,9 +408,6 @@ def emit_property_module(f, mod, tbl, emit):
|
|||
def emit_conversions_module(f, to_upper, to_lower, to_title):
|
||||
f.write("pub mod conversions {")
|
||||
f.write("""
|
||||
use core::option::Option;
|
||||
use core::option::Option::{Some, None};
|
||||
|
||||
pub fn to_lower(c: char) -> [char; 3] {
|
||||
match bsearch_case_table(c, to_lowercase_table) {
|
||||
None => [c, '\\0', '\\0'],
|
||||
|
|
@ -473,7 +470,8 @@ if __name__ == "__main__":
|
|||
unicode_version = re.search(pattern, readme.read()).groups()
|
||||
rf.write("""
|
||||
/// The version of [Unicode](http://www.unicode.org/) that the Unicode parts of
|
||||
/// `CharExt` and `UnicodeStrPrelude` traits are based on.
|
||||
/// `char` and `str` methods are based on.
|
||||
#[unstable(feature = "unicode_version", issue = "49726")]
|
||||
pub const UNICODE_VERSION: UnicodeVersion = UnicodeVersion {
|
||||
major: %s,
|
||||
minor: %s,
|
||||
|
|
@ -12,6 +12,7 @@
|
|||
///
|
||||
/// See also: <http://www.unicode.org/versions/>
|
||||
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
|
||||
#[unstable(feature = "unicode_version", issue = "49726")]
|
||||
pub struct UnicodeVersion {
|
||||
/// Major version.
|
||||
pub major: u32,
|
||||
|
|
@ -21,7 +21,6 @@
|
|||
#![feature(set_stdio)]
|
||||
#![feature(slice_sort_by_cached_key)]
|
||||
#![feature(test)]
|
||||
#![feature(unicode)]
|
||||
#![feature(vec_remove_item)]
|
||||
#![feature(entry_and_modify)]
|
||||
|
||||
|
|
@ -42,7 +41,6 @@ extern crate serialize;
|
|||
#[macro_use] extern crate syntax;
|
||||
extern crate syntax_pos;
|
||||
extern crate test as testing;
|
||||
extern crate std_unicode;
|
||||
#[macro_use] extern crate log;
|
||||
extern crate rustc_errors as errors;
|
||||
extern crate pulldown_cmark;
|
||||
|
|
|
|||
|
|
@ -307,7 +307,6 @@
|
|||
#![feature(toowned_clone_into)]
|
||||
#![feature(try_reserve)]
|
||||
#![feature(unboxed_closures)]
|
||||
#![feature(unicode)]
|
||||
#![feature(untagged_unions)]
|
||||
#![feature(unwind_attributes)]
|
||||
#![feature(vec_push_all)]
|
||||
|
|
@ -354,7 +353,6 @@ extern crate core as __core;
|
|||
#[macro_reexport(vec, format)]
|
||||
extern crate alloc;
|
||||
extern crate alloc_system;
|
||||
extern crate std_unicode;
|
||||
#[doc(masked)]
|
||||
extern crate libc;
|
||||
|
||||
|
|
@ -455,7 +453,7 @@ pub use alloc::string;
|
|||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub use alloc::vec;
|
||||
#[stable(feature = "rust1", since = "1.0.0")]
|
||||
pub use std_unicode::char;
|
||||
pub use core::char;
|
||||
#[stable(feature = "i128", since = "1.26.0")]
|
||||
pub use core::u128;
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ use rc::Rc;
|
|||
use sync::Arc;
|
||||
use sys_common::{AsInner, IntoInner};
|
||||
use sys_common::bytestring::debug_fmt_bytestring;
|
||||
use std_unicode::lossy::Utf8Lossy;
|
||||
use core::str::lossy::Utf8Lossy;
|
||||
|
||||
#[derive(Clone, Hash)]
|
||||
pub struct Buf {
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ use rc::Rc;
|
|||
use sync::Arc;
|
||||
use sys_common::{AsInner, IntoInner};
|
||||
use sys_common::bytestring::debug_fmt_bytestring;
|
||||
use std_unicode::lossy::Utf8Lossy;
|
||||
use core::str::lossy::Utf8Lossy;
|
||||
|
||||
#[derive(Clone, Hash)]
|
||||
pub struct Buf {
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ use rc::Rc;
|
|||
use sync::Arc;
|
||||
use sys_common::{AsInner, IntoInner};
|
||||
use sys_common::bytestring::debug_fmt_bytestring;
|
||||
use std_unicode::lossy::Utf8Lossy;
|
||||
use core::str::lossy::Utf8Lossy;
|
||||
|
||||
#[derive(Clone, Hash)]
|
||||
pub struct Buf {
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@
|
|||
#![allow(dead_code)]
|
||||
|
||||
use fmt::{Formatter, Result, Write};
|
||||
use std_unicode::lossy::{Utf8Lossy, Utf8LossyChunk};
|
||||
use core::str::lossy::{Utf8Lossy, Utf8LossyChunk};
|
||||
|
||||
pub fn debug_fmt_bytestring(slice: &[u8], f: &mut Formatter) -> Result {
|
||||
// Writes out a valid unicode string with the correct escape sequences
|
||||
|
|
|
|||
|
|
@ -9,10 +9,6 @@ path = "lib.rs"
|
|||
test = false
|
||||
bench = false
|
||||
|
||||
[[test]]
|
||||
name = "std_unicode_tests"
|
||||
path = "tests/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
core = { path = "../libcore" }
|
||||
compiler_builtins = { path = "../rustc/compiler_builtins_shim" }
|
||||
|
|
|
|||
|
|
@ -27,38 +27,10 @@
|
|||
html_playground_url = "https://play.rust-lang.org/",
|
||||
issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/",
|
||||
test(no_crate_inject, attr(allow(unused_variables), deny(warnings))))]
|
||||
#![deny(missing_debug_implementations)]
|
||||
#![no_std]
|
||||
|
||||
#![feature(ascii_ctype)]
|
||||
#![feature(core_char_ext)]
|
||||
#![feature(str_internals)]
|
||||
#![feature(decode_utf8)]
|
||||
#![feature(fn_traits)]
|
||||
#![feature(lang_items)]
|
||||
#![feature(non_exhaustive)]
|
||||
#![feature(unicode_internals)]
|
||||
#![feature(staged_api)]
|
||||
#![feature(unboxed_closures)]
|
||||
#![rustc_deprecated(since = "1.27.0", reason = "moved into libcore")]
|
||||
|
||||
mod bool_trie;
|
||||
mod tables;
|
||||
mod u_str;
|
||||
mod version;
|
||||
pub mod char;
|
||||
pub mod lossy;
|
||||
|
||||
#[allow(deprecated)]
|
||||
pub mod str {
|
||||
pub use u_str::{SplitWhitespace, UnicodeStr};
|
||||
pub use u_str::Utf16Encoder;
|
||||
}
|
||||
|
||||
// For use in liballoc, not re-exported in libstd.
|
||||
pub mod derived_property {
|
||||
pub use tables::derived_property::{Case_Ignorable, Cased};
|
||||
}
|
||||
|
||||
// For use in libsyntax
|
||||
pub mod property {
|
||||
pub use tables::property::Pattern_White_Space;
|
||||
}
|
||||
pub use core::unicode::*;
|
||||
|
|
|
|||
|
|
@ -1,15 +0,0 @@
|
|||
// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT
|
||||
// file at the top-level directory of this distribution and at
|
||||
// http://rust-lang.org/COPYRIGHT.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
||||
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
||||
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
||||
// option. This file may not be copied, modified, or distributed
|
||||
// except according to those terms.
|
||||
|
||||
#![feature(str_internals, unicode)]
|
||||
|
||||
extern crate std_unicode;
|
||||
|
||||
mod lossy;
|
||||
|
|
@ -1,189 +0,0 @@
|
|||
// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
|
||||
// file at the top-level directory of this distribution and at
|
||||
// http://rust-lang.org/COPYRIGHT.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
||||
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
||||
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
||||
// option. This file may not be copied, modified, or distributed
|
||||
// except according to those terms.
|
||||
|
||||
//! Unicode-intensive string manipulations.
|
||||
//!
|
||||
//! This module provides functionality to `str` that requires the Unicode
|
||||
//! methods provided by the unicode parts of the CharExt trait.
|
||||
|
||||
use core::char;
|
||||
use core::iter::{Filter, FusedIterator};
|
||||
use core::str::Split;
|
||||
|
||||
/// An iterator over the non-whitespace substrings of a string,
|
||||
/// separated by any amount of whitespace.
|
||||
///
|
||||
/// This struct is created by the [`split_whitespace`] method on [`str`].
|
||||
/// See its documentation for more.
|
||||
///
|
||||
/// [`split_whitespace`]: ../../std/primitive.str.html#method.split_whitespace
|
||||
/// [`str`]: ../../std/primitive.str.html
|
||||
#[stable(feature = "split_whitespace", since = "1.1.0")]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SplitWhitespace<'a> {
|
||||
inner: Filter<Split<'a, IsWhitespace>, IsNotEmpty>,
|
||||
}
|
||||
|
||||
/// Methods for Unicode string slices
|
||||
#[allow(missing_docs)] // docs in liballoc
|
||||
pub trait UnicodeStr {
|
||||
fn split_whitespace<'a>(&'a self) -> SplitWhitespace<'a>;
|
||||
fn is_whitespace(&self) -> bool;
|
||||
fn is_alphanumeric(&self) -> bool;
|
||||
fn trim(&self) -> &str;
|
||||
fn trim_left(&self) -> &str;
|
||||
fn trim_right(&self) -> &str;
|
||||
}
|
||||
|
||||
impl UnicodeStr for str {
|
||||
#[inline]
|
||||
fn split_whitespace(&self) -> SplitWhitespace {
|
||||
SplitWhitespace { inner: self.split(IsWhitespace).filter(IsNotEmpty) }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn is_whitespace(&self) -> bool {
|
||||
self.chars().all(|c| c.is_whitespace())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn is_alphanumeric(&self) -> bool {
|
||||
self.chars().all(|c| c.is_alphanumeric())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn trim(&self) -> &str {
|
||||
self.trim_matches(|c: char| c.is_whitespace())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn trim_left(&self) -> &str {
|
||||
self.trim_left_matches(|c: char| c.is_whitespace())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn trim_right(&self) -> &str {
|
||||
self.trim_right_matches(|c: char| c.is_whitespace())
|
||||
}
|
||||
}
|
||||
|
||||
/// Iterator adaptor for encoding `char`s to UTF-16.
|
||||
#[derive(Clone)]
|
||||
#[allow(missing_debug_implementations)]
|
||||
pub struct Utf16Encoder<I> {
|
||||
chars: I,
|
||||
extra: u16,
|
||||
}
|
||||
|
||||
impl<I> Utf16Encoder<I> {
|
||||
/// Create a UTF-16 encoder from any `char` iterator.
|
||||
pub fn new(chars: I) -> Utf16Encoder<I>
|
||||
where I: Iterator<Item = char>
|
||||
{
|
||||
Utf16Encoder {
|
||||
chars,
|
||||
extra: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I> Iterator for Utf16Encoder<I>
|
||||
where I: Iterator<Item = char>
|
||||
{
|
||||
type Item = u16;
|
||||
|
||||
#[inline]
|
||||
fn next(&mut self) -> Option<u16> {
|
||||
if self.extra != 0 {
|
||||
let tmp = self.extra;
|
||||
self.extra = 0;
|
||||
return Some(tmp);
|
||||
}
|
||||
|
||||
let mut buf = [0; 2];
|
||||
self.chars.next().map(|ch| {
|
||||
let n = CharExt::encode_utf16(ch, &mut buf).len();
|
||||
if n == 2 {
|
||||
self.extra = buf[1];
|
||||
}
|
||||
buf[0]
|
||||
})
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
let (low, high) = self.chars.size_hint();
|
||||
// every char gets either one u16 or two u16,
|
||||
// so this iterator is between 1 or 2 times as
|
||||
// long as the underlying iterator.
|
||||
(low, high.and_then(|n| n.checked_mul(2)))
|
||||
}
|
||||
}
|
||||
|
||||
impl<I> FusedIterator for Utf16Encoder<I>
|
||||
where I: FusedIterator<Item = char> {}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct IsWhitespace;
|
||||
|
||||
impl FnOnce<(char, )> for IsWhitespace {
|
||||
type Output = bool;
|
||||
|
||||
#[inline]
|
||||
extern "rust-call" fn call_once(mut self, arg: (char, )) -> bool {
|
||||
self.call_mut(arg)
|
||||
}
|
||||
}
|
||||
|
||||
impl FnMut<(char, )> for IsWhitespace {
|
||||
#[inline]
|
||||
extern "rust-call" fn call_mut(&mut self, arg: (char, )) -> bool {
|
||||
arg.0.is_whitespace()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct IsNotEmpty;
|
||||
|
||||
impl<'a, 'b> FnOnce<(&'a &'b str, )> for IsNotEmpty {
|
||||
type Output = bool;
|
||||
|
||||
#[inline]
|
||||
extern "rust-call" fn call_once(mut self, arg: (&&str, )) -> bool {
|
||||
self.call_mut(arg)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, 'b> FnMut<(&'a &'b str, )> for IsNotEmpty {
|
||||
#[inline]
|
||||
extern "rust-call" fn call_mut(&mut self, arg: (&&str, )) -> bool {
|
||||
!arg.0.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#[stable(feature = "split_whitespace", since = "1.1.0")]
|
||||
impl<'a> Iterator for SplitWhitespace<'a> {
|
||||
type Item = &'a str;
|
||||
|
||||
fn next(&mut self) -> Option<&'a str> {
|
||||
self.inner.next()
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "split_whitespace", since = "1.1.0")]
|
||||
impl<'a> DoubleEndedIterator for SplitWhitespace<'a> {
|
||||
fn next_back(&mut self) -> Option<&'a str> {
|
||||
self.inner.next_back()
|
||||
}
|
||||
}
|
||||
|
||||
#[stable(feature = "fused", since = "1.26.0")]
|
||||
impl<'a> FusedIterator for SplitWhitespace<'a> {}
|
||||
|
|
@ -19,7 +19,7 @@
|
|||
html_root_url = "https://doc.rust-lang.org/nightly/",
|
||||
test(attr(deny(warnings))))]
|
||||
|
||||
#![feature(unicode)]
|
||||
#![feature(unicode_internals)]
|
||||
#![feature(rustc_diagnostic_macros)]
|
||||
#![feature(slice_sort_by_cached_key)]
|
||||
#![feature(non_exhaustive)]
|
||||
|
|
@ -33,9 +33,9 @@
|
|||
extern crate rustc_cratesio_shim;
|
||||
|
||||
#[macro_use] extern crate bitflags;
|
||||
extern crate core;
|
||||
extern crate serialize;
|
||||
#[macro_use] extern crate log;
|
||||
extern crate std_unicode;
|
||||
pub extern crate rustc_errors as errors;
|
||||
extern crate syntax_pos;
|
||||
extern crate rustc_data_structures;
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ use errors::{FatalError, DiagnosticBuilder};
|
|||
use parse::{token, ParseSess};
|
||||
use str::char_at;
|
||||
use symbol::{Symbol, keywords};
|
||||
use std_unicode::property::Pattern_White_Space;
|
||||
use core::unicode::property::Pattern_White_Space;
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::char;
|
||||
|
|
|
|||
|
|
@ -15,9 +15,9 @@
|
|||
#![no_std]
|
||||
|
||||
// OK
|
||||
#[lang = "char"]
|
||||
impl char {}
|
||||
#[lang = "str"]
|
||||
impl str {}
|
||||
|
||||
impl char {
|
||||
//~^ error: only a single inherent implementation marked with `#[lang = "char"]` is allowed for the `char` primitive
|
||||
impl str {
|
||||
//~^ error: only a single inherent implementation marked with `#[lang = "str"]` is allowed for the `str` primitive
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,9 +8,7 @@
|
|||
// option. This file may not be copied, modified, or distributed
|
||||
// except according to those terms.
|
||||
|
||||
|
||||
#![feature(unicode)]
|
||||
|
||||
#![feature(unicode_version)]
|
||||
|
||||
/// Tests access to the internal Unicode Version type and value.
|
||||
pub fn main() {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue