bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Auto merge of #110861 - m-ou-se:thread-local-restructure, r=workingjubilee

Browse source 
Restructure and rename std thread_local internals to make it less of a maze

Every time I try to work on std's thread local internals, it feels like I'm trying to navigate a confusing maze made of macros, deeply nested modules, and types with multiple names/aliases. Time to clean it up a bit.

This PR:

- Exports `Key` with its own name (`Key`), instead of `__LocalKeyInner`
- Uses `pub macro` to put `__thread_local_inner` into a (unstable, hidden) module, removing `#[macro_export]`, removing it from the crate root.
- Removes the `__` from `__thread_local_inner`.
- Removes a few unnecessary `allow_internal_unstable` features from the macros
- Removes the `libstd_thread_internals` feature. (Merged with `thread_local_internals`.)
    - And removes it from the unstable book
- Gets rid of the deeply nested modules for the `Key` definitions (`mod fast` / `mod os` / `mod statik`).
- Turns a `#[cfg]` mess into a single `cfg_if`, now that there's no `#[macro_export]` anymore that breaks with `cfg_if`.
- Simplifies the `cfg_if` conditions to not repeat the conditions.
- Removes useless `normalize-stderr-test`, which were left over from when the `Key` types had different names on different platforms.
- Removes a seemingly unnecessary `realstd` re-export on `cfg(test)`.

This PR changes nothing about the thread local implementation. That's for a later PR. (Which should hopefully be easier once all this stuff is a bit cleaned up.)
This commit is contained in:
bors 2023-04-26 22:07:17 +00:00
commit cb9aa8c9c1
12 changed files with 313 additions and 348 deletions
Download patch file Download diff file Expand all files Collapse all files

279
library/std/src/sys/common/thread_local/fast_local.rs
View file

@ -1,13 +1,14 @@
use super::lazy::LazyKeyInner;
use crate::cell::Cell;
use crate::sys::thread_local_dtor::register_dtor;
use crate::{fmt, mem, panic};
#[doc(hidden)] #[doc(hidden)]
#[macro_export] #[allow_internal_unstable(thread_local_internals, cfg_target_thread_local, thread_local)]
#[allow_internal_unstable(
thread_local_internals,
cfg_target_thread_local,
thread_local,
libstd_thread_internals
)]
#[allow_internal_unsafe] #[allow_internal_unsafe]
macro_rules! __thread_local_inner { #[unstable(feature = "thread_local_internals", issue = "none")]
#[rustc_macro_transparency = "semitransparent"]
pub macro thread_local_inner {
// used to generate the `LocalKey` value for const-initialized thread locals // used to generate the `LocalKey` value for const-initialized thread locals
(@key $t:ty, const $init:expr) => {{ (@key $t:ty, const $init:expr) => {{
#[cfg_attr(not(bootstrap), inline)] #[cfg_attr(not(bootstrap), inline)]
@ -49,7 +50,7 @@ macro_rules! __thread_local_inner {
// 0 == we haven't registered a destructor, so do // 0 == we haven't registered a destructor, so do
// so now. // so now.
0 => { 0 => {
$crate::thread::__LocalKeyInner::<$t>::register_dtor( $crate::thread::local_impl::Key::<$t>::register_dtor(
$crate::ptr::addr_of_mut!(VAL) as *mut $crate::primitive::u8, $crate::ptr::addr_of_mut!(VAL) as *mut $crate::primitive::u8,
destroy, destroy,
); );
@ -69,7 +70,7 @@ macro_rules! __thread_local_inner {
unsafe { unsafe {
$crate::thread::LocalKey::new(__getit) $crate::thread::LocalKey::new(__getit)
} }
}}; }},
// used to generate the `LocalKey` value for `thread_local!` // used to generate the `LocalKey` value for `thread_local!`
(@key $t:ty, $init:expr) => { (@key $t:ty, $init:expr) => {
@ -82,8 +83,8 @@ macro_rules! __thread_local_inner {
init: $crate::option::Option<&mut $crate::option::Option<$t>>, init: $crate::option::Option<&mut $crate::option::Option<$t>>,
) -> $crate::option::Option<&'static $t> { ) -> $crate::option::Option<&'static $t> {
#[thread_local] #[thread_local]
static __KEY: $crate::thread::__LocalKeyInner<$t> = static __KEY: $crate::thread::local_impl::Key<$t> =
$crate::thread::__LocalKeyInner::<$t>::new(); $crate::thread::local_impl::Key::<$t>::new();
// FIXME: remove the #[allow(...)] marker when macros don't // FIXME: remove the #[allow(...)] marker when macros don't
// raise warning for missing/extraneous unsafe blocks anymore. // raise warning for missing/extraneous unsafe blocks anymore.
@ -107,148 +108,140 @@ macro_rules! __thread_local_inner {
$crate::thread::LocalKey::new(__getit) $crate::thread::LocalKey::new(__getit)
} }
} }
}; },
($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => { ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => {
$(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> = $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> =
$crate::__thread_local_inner!(@key $t, $($init)*); $crate::thread::local_impl::thread_local_inner!(@key $t, $($init)*);
} },
} }
#[doc(hidden)] #[derive(Copy, Clone)]
pub mod fast { enum DtorState {
use super::super::lazy::LazyKeyInner; Unregistered,
use crate::cell::Cell; Registered,
use crate::sys::thread_local_dtor::register_dtor; RunningOrHasRun,
use crate::{fmt, mem, panic}; }
#[derive(Copy, Clone)] // This data structure has been carefully constructed so that the fast path
enum DtorState { // only contains one branch on x86. That optimization is necessary to avoid
Unregistered, // duplicated tls lookups on OSX.
Registered, //
RunningOrHasRun, // LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722
} pub struct Key<T> {
// If `LazyKeyInner::get` returns `None`, that indicates either:
// This data structure has been carefully constructed so that the fast path // * The value has never been initialized
// only contains one branch on x86. That optimization is necessary to avoid // * The value is being recursively initialized
// duplicated tls lookups on OSX. // * The value has already been destroyed or is being destroyed
// To determine which kind of `None`, check `dtor_state`.
// //
// This is very optimizer friendly for the fast path - initialized but
// not yet dropped.
inner: LazyKeyInner<T>,
// Metadata to keep track of the state of the destructor. Remember that
// this variable is thread-local, not global.
dtor_state: Cell<DtorState>,
}
impl<T> fmt::Debug for Key<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Key").finish_non_exhaustive()
}
}
impl<T> Key<T> {
pub const fn new() -> Key<T> {
Key { inner: LazyKeyInner::new(), dtor_state: Cell::new(DtorState::Unregistered) }
}
// note that this is just a publicly-callable function only for the
// const-initialized form of thread locals, basically a way to call the
// free `register_dtor` function defined elsewhere in std.
pub unsafe fn register_dtor(a: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) {
unsafe {
register_dtor(a, dtor);
}
}
pub unsafe fn get<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> {
// SAFETY: See the definitions of `LazyKeyInner::get` and
// `try_initialize` for more information.
//
// The caller must ensure no mutable references are ever active to
// the inner cell or the inner T when this is called.
// The `try_initialize` is dependant on the passed `init` function
// for this.
unsafe {
match self.inner.get() {
Some(val) => Some(val),
None => self.try_initialize(init),
}
}
}
// `try_initialize` is only called once per fast thread local variable,
// except in corner cases where thread_local dtors reference other
// thread_local's, or it is being recursively initialized.
//
// Macos: Inlining this function can cause two `tlv_get_addr` calls to
// be performed for every call to `Key::get`.
// LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722 // LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722
pub struct Key<T> { #[inline(never)]
// If `LazyKeyInner::get` returns `None`, that indicates either: unsafe fn try_initialize<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> {
// * The value has never been initialized // SAFETY: See comment above (this function doc).
// * The value is being recursively initialized if !mem::needs_drop::<T>() || unsafe { self.try_register_dtor() } {
// * The value has already been destroyed or is being destroyed
// To determine which kind of `None`, check `dtor_state`.
//
// This is very optimizer friendly for the fast path - initialized but
// not yet dropped.
inner: LazyKeyInner<T>,
// Metadata to keep track of the state of the destructor. Remember that
// this variable is thread-local, not global.
dtor_state: Cell<DtorState>,
}
impl<T> fmt::Debug for Key<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Key").finish_non_exhaustive()
}
}
impl<T> Key<T> {
pub const fn new() -> Key<T> {
Key { inner: LazyKeyInner::new(), dtor_state: Cell::new(DtorState::Unregistered) }
}
// note that this is just a publicly-callable function only for the
// const-initialized form of thread locals, basically a way to call the
// free `register_dtor` function defined elsewhere in std.
pub unsafe fn register_dtor(a: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) {
unsafe {
register_dtor(a, dtor);
}
}
pub unsafe fn get<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> {
// SAFETY: See the definitions of `LazyKeyInner::get` and
// `try_initialize` for more information.
//
// The caller must ensure no mutable references are ever active to
// the inner cell or the inner T when this is called.
// The `try_initialize` is dependant on the passed `init` function
// for this.
unsafe {
match self.inner.get() {
Some(val) => Some(val),
None => self.try_initialize(init),
}
}
}
// `try_initialize` is only called once per fast thread local variable,
// except in corner cases where thread_local dtors reference other
// thread_local's, or it is being recursively initialized.
//
// Macos: Inlining this function can cause two `tlv_get_addr` calls to
// be performed for every call to `Key::get`.
// LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722
#[inline(never)]
unsafe fn try_initialize<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> {
// SAFETY: See comment above (this function doc). // SAFETY: See comment above (this function doc).
if !mem::needs_drop::<T>() || unsafe { self.try_register_dtor() } { Some(unsafe { self.inner.initialize(init) })
// SAFETY: See comment above (this function doc). } else {
Some(unsafe { self.inner.initialize(init) }) None
} else {
None
}
}
// `try_register_dtor` is only called once per fast thread local
// variable, except in corner cases where thread_local dtors reference
// other thread_local's, or it is being recursively initialized.
unsafe fn try_register_dtor(&self) -> bool {
match self.dtor_state.get() {
DtorState::Unregistered => {
// SAFETY: dtor registration happens before initialization.
// Passing `self` as a pointer while using `destroy_value<T>`
// is safe because the function will build a pointer to a
// Key<T>, which is the type of self and so find the correct
// size.
unsafe { register_dtor(self as *const _ as *mut u8, destroy_value::<T>) };
self.dtor_state.set(DtorState::Registered);
true
}
DtorState::Registered => {
// recursively initialized
true
}
DtorState::RunningOrHasRun => false,
}
} }
} }
unsafe extern "C" fn destroy_value<T>(ptr: *mut u8) { // `try_register_dtor` is only called once per fast thread local
let ptr = ptr as *mut Key<T>; // variable, except in corner cases where thread_local dtors reference
// other thread_local's, or it is being recursively initialized.
// SAFETY: unsafe fn try_register_dtor(&self) -> bool {
// match self.dtor_state.get() {
// The pointer `ptr` has been built just above and comes from DtorState::Unregistered => {
// `try_register_dtor` where it is originally a Key<T> coming from `self`, // SAFETY: dtor registration happens before initialization.
// making it non-NUL and of the correct type. // Passing `self` as a pointer while using `destroy_value<T>`
// // is safe because the function will build a pointer to a
// Right before we run the user destructor be sure to set the // Key<T>, which is the type of self and so find the correct
// `Option<T>` to `None`, and `dtor_state` to `RunningOrHasRun`. This // size.
// causes future calls to `get` to run `try_initialize_drop` again, unsafe { register_dtor(self as *const _ as *mut u8, destroy_value::<T>) };
// which will now fail, and return `None`. self.dtor_state.set(DtorState::Registered);
// true
// Wrap the call in a catch to ensure unwinding is caught in the event }
// a panic takes place in a destructor. DtorState::Registered => {
if let Err(_) = panic::catch_unwind(panic::AssertUnwindSafe(|| unsafe { // recursively initialized
let value = (*ptr).inner.take(); true
(*ptr).dtor_state.set(DtorState::RunningOrHasRun); }
drop(value); DtorState::RunningOrHasRun => false,
})) {
rtabort!("thread local panicked on drop");
} }
} }
} }
unsafe extern "C" fn destroy_value<T>(ptr: *mut u8) {
let ptr = ptr as *mut Key<T>;
// SAFETY:
//
// The pointer `ptr` has been built just above and comes from
// `try_register_dtor` where it is originally a Key<T> coming from `self`,
// making it non-NUL and of the correct type.
//
// Right before we run the user destructor be sure to set the
// `Option<T>` to `None`, and `dtor_state` to `RunningOrHasRun`. This
// causes future calls to `get` to run `try_initialize_drop` again,
// which will now fail, and return `None`.
//
// Wrap the call in a catch to ensure unwinding is caught in the event
// a panic takes place in a destructor.
if let Err(_) = panic::catch_unwind(panic::AssertUnwindSafe(|| unsafe {
let value = (*ptr).inner.take();
(*ptr).dtor_state.set(DtorState::RunningOrHasRun);
drop(value);
})) {
rtabort!("thread local panicked on drop");
}
}

33
library/std/src/sys/common/thread_local/mod.rs
View file

@ -1,35 +1,24 @@
//! The following module declarations are outside cfg_if because the internal
//! `__thread_local_internal` macro does not seem to be exported properly when using cfg_if
#![unstable(feature = "thread_local_internals", reason = "should not be necessary", issue = "none")] #![unstable(feature = "thread_local_internals", reason = "should not be necessary", issue = "none")]
#[cfg(all(target_thread_local, not(all(target_family = "wasm", not(target_feature = "atomics")))))]
mod fast_local;
#[cfg(all(
not(target_thread_local),
not(all(target_family = "wasm", not(target_feature = "atomics")))
))]
mod os_local;
#[cfg(all(target_family = "wasm", not(target_feature = "atomics")))]
mod static_local;
#[cfg(not(test))]
cfg_if::cfg_if! { cfg_if::cfg_if! {
if #[cfg(all(target_family = "wasm", not(target_feature = "atomics")))] { if #[cfg(all(target_family = "wasm", not(target_feature = "atomics")))] {
#[doc(hidden)] #[doc(hidden)]
pub use static_local::statik::Key; mod static_local;
} else if #[cfg(all(target_thread_local, not(all(target_family = "wasm", not(target_feature = "atomics")))))] {
#[doc(hidden)] #[doc(hidden)]
pub use fast_local::fast::Key; pub use static_local::{Key, thread_local_inner};
} else if #[cfg(all(not(target_thread_local), not(all(target_family = "wasm", not(target_feature = "atomics")))))] { } else if #[cfg(all(target_thread_local))] {
#[doc(hidden)] #[doc(hidden)]
pub use os_local::os::Key; mod fast_local;
#[doc(hidden)]
pub use fast_local::{Key, thread_local_inner};
} else {
#[doc(hidden)]
mod os_local;
#[doc(hidden)]
pub use os_local::{Key, thread_local_inner};
} }
} }
#[doc(hidden)]
#[cfg(test)]
pub use realstd::thread::__LocalKeyInner as Key;
mod lazy { mod lazy {
use crate::cell::UnsafeCell; use crate::cell::UnsafeCell;
use crate::hint; use crate::hint;

219
library/std/src/sys/common/thread_local/os_local.rs
View file

@ -1,13 +1,14 @@
use super::lazy::LazyKeyInner;
use crate::cell::Cell;
use crate::sys_common::thread_local_key::StaticKey as OsStaticKey;
use crate::{fmt, marker, panic, ptr};
#[doc(hidden)] #[doc(hidden)]
#[macro_export] #[allow_internal_unstable(thread_local_internals)]
#[allow_internal_unstable(
thread_local_internals,
cfg_target_thread_local,
thread_local,
libstd_thread_internals
)]
#[allow_internal_unsafe] #[allow_internal_unsafe]
macro_rules! __thread_local_inner { #[unstable(feature = "thread_local_internals", issue = "none")]
#[rustc_macro_transparency = "semitransparent"]
pub macro thread_local_inner {
// used to generate the `LocalKey` value for const-initialized thread locals // used to generate the `LocalKey` value for const-initialized thread locals
(@key $t:ty, const $init:expr) => {{ (@key $t:ty, const $init:expr) => {{
#[cfg_attr(not(bootstrap), inline)] #[cfg_attr(not(bootstrap), inline)]
@ -21,8 +22,8 @@ macro_rules! __thread_local_inner {
// same implementation as below for os thread locals. // same implementation as below for os thread locals.
#[inline] #[inline]
const fn __init() -> $t { INIT_EXPR } const fn __init() -> $t { INIT_EXPR }
static __KEY: $crate::thread::__LocalKeyInner<$t> = static __KEY: $crate::thread::local_impl::Key<$t> =
$crate::thread::__LocalKeyInner::new(); $crate::thread::local_impl::Key::new();
#[allow(unused_unsafe)] #[allow(unused_unsafe)]
unsafe { unsafe {
__KEY.get(move || { __KEY.get(move || {
@ -41,7 +42,7 @@ macro_rules! __thread_local_inner {
unsafe { unsafe {
$crate::thread::LocalKey::new(__getit) $crate::thread::LocalKey::new(__getit)
} }
}}; }},
// used to generate the `LocalKey` value for `thread_local!` // used to generate the `LocalKey` value for `thread_local!`
(@key $t:ty, $init:expr) => { (@key $t:ty, $init:expr) => {
@ -55,8 +56,8 @@ macro_rules! __thread_local_inner {
unsafe fn __getit( unsafe fn __getit(
init: $crate::option::Option<&mut $crate::option::Option<$t>>, init: $crate::option::Option<&mut $crate::option::Option<$t>>,
) -> $crate::option::Option<&'static $t> { ) -> $crate::option::Option<&'static $t> {
static __KEY: $crate::thread::__LocalKeyInner<$t> = static __KEY: $crate::thread::local_impl::Key<$t> =
$crate::thread::__LocalKeyInner::new(); $crate::thread::local_impl::Key::new();
// FIXME: remove the #[allow(...)] marker when macros don't // FIXME: remove the #[allow(...)] marker when macros don't
// raise warning for missing/extraneous unsafe blocks anymore. // raise warning for missing/extraneous unsafe blocks anymore.
@ -80,118 +81,110 @@ macro_rules! __thread_local_inner {
$crate::thread::LocalKey::new(__getit) $crate::thread::LocalKey::new(__getit)
} }
} }
}; },
($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => { ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => {
$(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> = $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> =
$crate::__thread_local_inner!(@key $t, $($init)*); $crate::thread::local_impl::thread_local_inner!(@key $t, $($init)*);
},
}
/// Use a regular global static to store this key; the state provided will then be
/// thread-local.
pub struct Key<T> {
// OS-TLS key that we'll use to key off.
os: OsStaticKey,
marker: marker::PhantomData<Cell<T>>,
}
impl<T> fmt::Debug for Key<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Key").finish_non_exhaustive()
} }
} }
#[doc(hidden)] unsafe impl<T> Sync for Key<T> {}
pub mod os {
use super::super::lazy::LazyKeyInner;
use crate::cell::Cell;
use crate::sys_common::thread_local_key::StaticKey as OsStaticKey;
use crate::{fmt, marker, panic, ptr};
/// Use a regular global static to store this key; the state provided will then be struct Value<T: 'static> {
/// thread-local. inner: LazyKeyInner<T>,
pub struct Key<T> { key: &'static Key<T>,
// OS-TLS key that we'll use to key off. }
os: OsStaticKey,
marker: marker::PhantomData<Cell<T>>, impl<T: 'static> Key<T> {
#[rustc_const_unstable(feature = "thread_local_internals", issue = "none")]
pub const fn new() -> Key<T> {
Key { os: OsStaticKey::new(Some(destroy_value::<T>)), marker: marker::PhantomData }
} }
impl<T> fmt::Debug for Key<T> { /// It is a requirement for the caller to ensure that no mutable
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { /// reference is active when this method is called.
f.debug_struct("Key").finish_non_exhaustive() pub unsafe fn get(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> {
} // SAFETY: See the documentation for this method.
} let ptr = unsafe { self.os.get() as *mut Value<T> };
if ptr.addr() > 1 {
unsafe impl<T> Sync for Key<T> {} // SAFETY: the check ensured the pointer is safe (its destructor
// is not running) + it is coming from a trusted source (self).
struct Value<T: 'static> { if let Some(ref value) = unsafe { (*ptr).inner.get() } {
inner: LazyKeyInner<T>, return Some(value);
key: &'static Key<T>,
}
impl<T: 'static> Key<T> {
#[rustc_const_unstable(feature = "thread_local_internals", issue = "none")]
pub const fn new() -> Key<T> {
Key { os: OsStaticKey::new(Some(destroy_value::<T>)), marker: marker::PhantomData }
}
/// It is a requirement for the caller to ensure that no mutable
/// reference is active when this method is called.
pub unsafe fn get(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> {
// SAFETY: See the documentation for this method.
let ptr = unsafe { self.os.get() as *mut Value<T> };
if ptr.addr() > 1 {
// SAFETY: the check ensured the pointer is safe (its destructor
// is not running) + it is coming from a trusted source (self).
if let Some(ref value) = unsafe { (*ptr).inner.get() } {
return Some(value);
}
} }
// SAFETY: At this point we are sure we have no value and so
// initializing (or trying to) is safe.
unsafe { self.try_initialize(init) }
}
// `try_initialize` is only called once per os thread local variable,
// except in corner cases where thread_local dtors reference other
// thread_local's, or it is being recursively initialized.
unsafe fn try_initialize(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> {
// SAFETY: No mutable references are ever handed out meaning getting
// the value is ok.
let ptr = unsafe { self.os.get() as *mut Value<T> };
if ptr.addr() == 1 {
// destructor is running
return None;
}
let ptr = if ptr.is_null() {
// If the lookup returned null, we haven't initialized our own
// local copy, so do that now.
let ptr = Box::into_raw(Box::new(Value { inner: LazyKeyInner::new(), key: self }));
// SAFETY: At this point we are sure there is no value inside
// ptr so setting it will not affect anyone else.
unsafe {
self.os.set(ptr as *mut u8);
}
ptr
} else {
// recursive initialization
ptr
};
// SAFETY: ptr has been ensured as non-NUL just above an so can be
// dereferenced safely.
unsafe { Some((*ptr).inner.initialize(init)) }
} }
// SAFETY: At this point we are sure we have no value and so
// initializing (or trying to) is safe.
unsafe { self.try_initialize(init) }
} }
unsafe extern "C" fn destroy_value<T: 'static>(ptr: *mut u8) { // `try_initialize` is only called once per os thread local variable,
// SAFETY: // except in corner cases where thread_local dtors reference other
// // thread_local's, or it is being recursively initialized.
// The OS TLS ensures that this key contains a null value when this unsafe fn try_initialize(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> {
// destructor starts to run. We set it back to a sentinel value of 1 to // SAFETY: No mutable references are ever handed out meaning getting
// ensure that any future calls to `get` for this thread will return // the value is ok.
// `None`. let ptr = unsafe { self.os.get() as *mut Value<T> };
// if ptr.addr() == 1 {
// Note that to prevent an infinite loop we reset it back to null right // destructor is running
// before we return from the destructor ourselves. return None;
//
// Wrap the call in a catch to ensure unwinding is caught in the event
// a panic takes place in a destructor.
if let Err(_) = panic::catch_unwind(|| unsafe {
let ptr = Box::from_raw(ptr as *mut Value<T>);
let key = ptr.key;
key.os.set(ptr::invalid_mut(1));
drop(ptr);
key.os.set(ptr::null_mut());
}) {
rtabort!("thread local panicked on drop");
} }
let ptr = if ptr.is_null() {
// If the lookup returned null, we haven't initialized our own
// local copy, so do that now.
let ptr = Box::into_raw(Box::new(Value { inner: LazyKeyInner::new(), key: self }));
// SAFETY: At this point we are sure there is no value inside
// ptr so setting it will not affect anyone else.
unsafe {
self.os.set(ptr as *mut u8);
}
ptr
} else {
// recursive initialization
ptr
};
// SAFETY: ptr has been ensured as non-NUL just above an so can be
// dereferenced safely.
unsafe { Some((*ptr).inner.initialize(init)) }
}
}
unsafe extern "C" fn destroy_value<T: 'static>(ptr: *mut u8) {
// SAFETY:
//
// The OS TLS ensures that this key contains a null value when this
// destructor starts to run. We set it back to a sentinel value of 1 to
// ensure that any future calls to `get` for this thread will return
// `None`.
//
// Note that to prevent an infinite loop we reset it back to null right
// before we return from the destructor ourselves.
//
// Wrap the call in a catch to ensure unwinding is caught in the event
// a panic takes place in a destructor.
if let Err(_) = panic::catch_unwind(|| unsafe {
let ptr = Box::from_raw(ptr as *mut Value<T>);
let key = ptr.key;
key.os.set(ptr::invalid_mut(1));
drop(ptr);
key.os.set(ptr::null_mut());
}) {
rtabort!("thread local panicked on drop");
} }
} }

88
library/std/src/sys/common/thread_local/static_local.rs
View file

@ -1,13 +1,12 @@
use super::lazy::LazyKeyInner;
use crate::fmt;
#[doc(hidden)] #[doc(hidden)]
#[macro_export] #[allow_internal_unstable(thread_local_internals)]
#[allow_internal_unstable(
thread_local_internals,
cfg_target_thread_local,
thread_local,
libstd_thread_internals
)]
#[allow_internal_unsafe] #[allow_internal_unsafe]
macro_rules! __thread_local_inner { #[unstable(feature = "thread_local_internals", issue = "none")]
#[rustc_macro_transparency = "semitransparent"]
pub macro thread_local_inner {
// used to generate the `LocalKey` value for const-initialized thread locals // used to generate the `LocalKey` value for const-initialized thread locals
(@key $t:ty, const $init:expr) => {{ (@key $t:ty, const $init:expr) => {{
#[inline] // see comments below #[inline] // see comments below
@ -30,7 +29,7 @@ macro_rules! __thread_local_inner {
unsafe { unsafe {
$crate::thread::LocalKey::new(__getit) $crate::thread::LocalKey::new(__getit)
} }
}}; }},
// used to generate the `LocalKey` value for `thread_local!` // used to generate the `LocalKey` value for `thread_local!`
(@key $t:ty, $init:expr) => { (@key $t:ty, $init:expr) => {
@ -41,8 +40,8 @@ macro_rules! __thread_local_inner {
unsafe fn __getit( unsafe fn __getit(
init: $crate::option::Option<&mut $crate::option::Option<$t>>, init: $crate::option::Option<&mut $crate::option::Option<$t>>,
) -> $crate::option::Option<&'static $t> { ) -> $crate::option::Option<&'static $t> {
static __KEY: $crate::thread::__LocalKeyInner<$t> = static __KEY: $crate::thread::local_impl::Key<$t> =
$crate::thread::__LocalKeyInner::new(); $crate::thread::local_impl::Key::new();
// FIXME: remove the #[allow(...)] marker when macros don't // FIXME: remove the #[allow(...)] marker when macros don't
// raise warning for missing/extraneous unsafe blocks anymore. // raise warning for missing/extraneous unsafe blocks anymore.
@ -66,50 +65,45 @@ macro_rules! __thread_local_inner {
$crate::thread::LocalKey::new(__getit) $crate::thread::LocalKey::new(__getit)
} }
} }
}; },
($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => { ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => {
$(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> = $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> =
$crate::__thread_local_inner!(@key $t, $($init)*); $crate::thread::local_impl::thread_local_inner!(@key $t, $($init)*);
} },
} }
/// On some targets like wasm there's no threads, so no need to generate /// On some targets like wasm there's no threads, so no need to generate
/// thread locals and we can instead just use plain statics! /// thread locals and we can instead just use plain statics!
#[doc(hidden)]
pub mod statik {
use super::super::lazy::LazyKeyInner;
use crate::fmt;
pub struct Key<T> { pub struct Key<T> {
inner: LazyKeyInner<T>, inner: LazyKeyInner<T>,
} }
unsafe impl<T> Sync for Key<T> {} unsafe impl<T> Sync for Key<T> {}
impl<T> fmt::Debug for Key<T> { impl<T> fmt::Debug for Key<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Key").finish_non_exhaustive() f.debug_struct("Key").finish_non_exhaustive()
} }
} }
impl<T> Key<T> { impl<T> Key<T> {
pub const fn new() -> Key<T> { pub const fn new() -> Key<T> {
Key { inner: LazyKeyInner::new() } Key { inner: LazyKeyInner::new() }
} }
pub unsafe fn get(&self, init: impl FnOnce() -> T) -> Option<&'static T> { pub unsafe fn get(&self, init: impl FnOnce() -> T) -> Option<&'static T> {
// SAFETY: The caller must ensure no reference is ever handed out to // SAFETY: The caller must ensure no reference is ever handed out to
// the inner cell nor mutable reference to the Option<T> inside said // the inner cell nor mutable reference to the Option<T> inside said
// cell. This make it safe to hand a reference, though the lifetime // cell. This make it safe to hand a reference, though the lifetime
// of 'static is itself unsafe, making the get method unsafe. // of 'static is itself unsafe, making the get method unsafe.
let value = unsafe { let value = unsafe {
match self.inner.get() { match self.inner.get() {
Some(ref value) => value, Some(ref value) => value,
None => self.inner.initialize(init), None => self.inner.initialize(init),
} }
}; };
Some(value) Some(value)
}
} }
} }

8
library/std/src/thread/local.rs
View file

@ -153,23 +153,23 @@ macro_rules! thread_local {
() => {}; () => {};
($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = const { $init:expr }; $($rest:tt)*) => ( ($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = const { $init:expr }; $($rest:tt)*) => (
$crate::__thread_local_inner!($(#[$attr])* $vis $name, $t, const $init); $crate::thread::local_impl::thread_local_inner!($(#[$attr])* $vis $name, $t, const $init);
$crate::thread_local!($($rest)*); $crate::thread_local!($($rest)*);
); );
($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = const { $init:expr }) => ( ($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = const { $init:expr }) => (
$crate::__thread_local_inner!($(#[$attr])* $vis $name, $t, const $init); $crate::thread::local_impl::thread_local_inner!($(#[$attr])* $vis $name, $t, const $init);
); );
// process multiple declarations // process multiple declarations
($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = $init:expr; $($rest:tt)*) => ( ($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = $init:expr; $($rest:tt)*) => (
$crate::__thread_local_inner!($(#[$attr])* $vis $name, $t, $init); $crate::thread::local_impl::thread_local_inner!($(#[$attr])* $vis $name, $t, $init);
$crate::thread_local!($($rest)*); $crate::thread_local!($($rest)*);
); );
// handle a single declaration // handle a single declaration
($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = $init:expr) => ( ($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = $init:expr) => (
$crate::__thread_local_inner!($(#[$attr])* $vis $name, $t, $init); $crate::thread::local_impl::thread_local_inner!($(#[$attr])* $vis $name, $t, $init);
); );
} }

7
library/std/src/thread/mod.rs
View file

@ -204,9 +204,12 @@ pub use self::local::{AccessError, LocalKey};
// by the elf linker. "static" is for single-threaded platforms where a global // by the elf linker. "static" is for single-threaded platforms where a global
// static is sufficient. // static is sufficient.
// Implementation details used by the thread_local!{} macro.
#[doc(hidden)] #[doc(hidden)]
#[unstable(feature = "libstd_thread_internals", issue = "none")] #[unstable(feature = "thread_local_internals", issue = "none")]
pub use crate::sys::common::thread_local::Key as __LocalKeyInner; pub mod local_impl {
pub use crate::sys::common::thread_local::{thread_local_inner, Key};
}
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
// Builder // Builder

5
src/doc/unstable-book/src/library-features/libstd-thread-internals.md
View file

@ -1,5 +0,0 @@
# `libstd_thread_internals`
This feature is internal to the Rust compiler and is not intended for general use.
------------------------

2
tests/ui/macros/macro-local-data-key-priv.stderr
View file

@ -9,7 +9,7 @@ note: the constant `baz` is defined here
| |
LL | thread_local!(static baz: f64 = 0.0); LL | thread_local!(static baz: f64 = 0.0);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
= note: this error originates in the macro `$crate::__thread_local_inner` which comes from the expansion of the macro `thread_local` (in Nightly builds, run with -Z macro-backtrace for more info) = note: this error originates in the macro `$crate::thread::local_impl::thread_local_inner` which comes from the expansion of the macro `thread_local` (in Nightly builds, run with -Z macro-backtrace for more info)
error: aborting due to previous error error: aborting due to previous error

2
tests/ui/threads-sendsync/issue-43733-2.rs
View file

@ -21,7 +21,7 @@ impl<T> Key<T> {
} }
#[cfg(target_thread_local)] #[cfg(target_thread_local)]
use std::thread::__LocalKeyInner as Key; use std::thread::local_impl::Key;
static __KEY: Key<()> = Key::new(); static __KEY: Key<()> = Key::new();
//~^ ERROR `UnsafeCell<Option<()>>` cannot be shared between threads //~^ ERROR `UnsafeCell<Option<()>>` cannot be shared between threads

4
tests/ui/threads-sendsync/issue-43733.mir.stderr
View file

@ -1,5 +1,5 @@
error[E0133]: call to unsafe function is unsafe and requires unsafe function or block error[E0133]: call to unsafe function is unsafe and requires unsafe function or block
--> $DIR/issue-43733.rs:21:5 --> $DIR/issue-43733.rs:19:5
| |
LL | __KEY.get(Default::default) LL | __KEY.get(Default::default)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^ call to unsafe function | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ call to unsafe function
@ -7,7 +7,7 @@ LL | __KEY.get(Default::default)
= note: consult the function's documentation for information on how to avoid undefined behavior = note: consult the function's documentation for information on how to avoid undefined behavior
error[E0133]: call to unsafe function is unsafe and requires unsafe function or block error[E0133]: call to unsafe function is unsafe and requires unsafe function or block
--> $DIR/issue-43733.rs:26:42 --> $DIR/issue-43733.rs:24:42
| |
LL | static FOO: std::thread::LocalKey<Foo> = std::thread::LocalKey::new(__getit); LL | static FOO: std::thread::LocalKey<Foo> = std::thread::LocalKey::new(__getit);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ call to unsafe function | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ call to unsafe function

8
tests/ui/threads-sendsync/issue-43733.rs
View file

@ -1,8 +1,6 @@
// ignore-wasm32 // ignore-wasm32
// revisions: mir thir // revisions: mir thir
// [thir]compile-flags: -Z thir-unsafeck // [thir]compile-flags: -Z thir-unsafeck
// normalize-stderr-test: "__LocalKeyInner::<T>::get" -> "$$LOCALKEYINNER::<T>::get"
// normalize-stderr-test: "__LocalKeyInner::<T>::get" -> "$$LOCALKEYINNER::<T>::get"
#![feature(thread_local)] #![feature(thread_local)]
#![feature(cfg_target_thread_local, thread_local_internals)] #![feature(cfg_target_thread_local, thread_local_internals)]
@ -12,15 +10,15 @@ type Foo = std::cell::RefCell<String>;
#[cfg(target_thread_local)] #[cfg(target_thread_local)]
#[thread_local] #[thread_local]
static __KEY: std::thread::__LocalKeyInner<Foo> = std::thread::__LocalKeyInner::new(); static __KEY: std::thread::local_impl::Key<Foo> = std::thread::local_impl::Key::new();
#[cfg(not(target_thread_local))] #[cfg(not(target_thread_local))]
static __KEY: std::thread::__LocalKeyInner<Foo> = std::thread::__LocalKeyInner::new(); static __KEY: std::thread::local_impl::Key<Foo> = std::thread::local_impl::Key::new();
fn __getit(_: Option<&mut Option<RefCell<String>>>) -> std::option::Option<&'static Foo> { fn __getit(_: Option<&mut Option<RefCell<String>>>) -> std::option::Option<&'static Foo> {
__KEY.get(Default::default) __KEY.get(Default::default)
//[mir]~^ ERROR call to unsafe function is unsafe //[mir]~^ ERROR call to unsafe function is unsafe
//[thir]~^^ ERROR call to unsafe function `__ //[thir]~^^ ERROR call to unsafe function `Key::<T>::get`
} }
static FOO: std::thread::LocalKey<Foo> = std::thread::LocalKey::new(__getit); static FOO: std::thread::LocalKey<Foo> = std::thread::LocalKey::new(__getit);

6
tests/ui/threads-sendsync/issue-43733.thir.stderr
View file

@ -1,5 +1,5 @@
error[E0133]: call to unsafe function `$LOCALKEYINNER::<T>::get` is unsafe and requires unsafe function or block error[E0133]: call to unsafe function `Key::<T>::get` is unsafe and requires unsafe function or block
--> $DIR/issue-43733.rs:21:5 --> $DIR/issue-43733.rs:19:5
| |
LL | __KEY.get(Default::default) LL | __KEY.get(Default::default)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^ call to unsafe function | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ call to unsafe function
@ -7,7 +7,7 @@ LL | __KEY.get(Default::default)
= note: consult the function's documentation for information on how to avoid undefined behavior = note: consult the function's documentation for information on how to avoid undefined behavior
error[E0133]: call to unsafe function `LocalKey::<T>::new` is unsafe and requires unsafe function or block error[E0133]: call to unsafe function `LocalKey::<T>::new` is unsafe and requires unsafe function or block
--> $DIR/issue-43733.rs:26:42 --> $DIR/issue-43733.rs:24:42
| |
LL | static FOO: std::thread::LocalKey<Foo> = std::thread::LocalKey::new(__getit); LL | static FOO: std::thread::LocalKey<Foo> = std::thread::LocalKey::new(__getit);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ call to unsafe function | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ call to unsafe function