1
Fork 0

Rollup merge of #97423 - m-ou-se:memory-ordering-intrinsics, r=tmiasko

Simplify memory ordering intrinsics

This changes the names of the atomic intrinsics to always fully include their memory ordering arguments.

```diff
- atomic_cxchg
+ atomic_cxchg_seqcst_seqcst

- atomic_cxchg_acqrel
+ atomic_cxchg_acqrel_release

- atomic_cxchg_acqrel_failrelaxed
+ atomic_cxchg_acqrel_relaxed

// And so on.
```

- `seqcst` is no longer implied
- The failure ordering on chxchg is no longer implied in some cases, but now always explicitly part of the name.
- `release` is no longer shortened to just `rel`. That was especially confusing, since `relaxed` also starts with `rel`.
- `acquire` is no longer shortened to just `acq`, such that the names now all match the `std::sync::atomic::Ordering` variants exactly.
- This now allows for more combinations on the compare exchange operations, such as `atomic_cxchg_acquire_release`, which is necessary for #68464.
- This PR only exposes the new possibilities through unstable intrinsics, but not yet through the stable API. That's for [a separate PR](https://github.com/rust-lang/rust/pull/98383) that requires an FCP.

Suffixes for operations with a single memory order:

| Order   | Before       | After      |
|---------|--------------|------------|
| Relaxed | `_relaxed`   | `_relaxed` |
| Acquire | `_acq`       | `_acquire` |
| Release | `_rel`       | `_release` |
| AcqRel  | `_acqrel`    | `_acqrel`  |
| SeqCst  | (none)       | `_seqcst`  |

Suffixes for compare-and-exchange operations with two memory orderings:

| Success | Failure | Before                   | After              |
|---------|---------|--------------------------|--------------------|
| Relaxed | Relaxed | `_relaxed`               | `_relaxed_relaxed` |
| Relaxed | Acquire |                       | `_relaxed_acquire` |
| Relaxed | SeqCst  |                       | `_relaxed_seqcst`  |
| Acquire | Relaxed | `_acq_failrelaxed`       | `_acquire_relaxed` |
| Acquire | Acquire | `_acq`                   | `_acquire_acquire` |
| Acquire | SeqCst  |                       | `_acquire_seqcst`  |
| Release | Relaxed | `_rel`                   | `_release_relaxed` |
| Release | Acquire |                       | `_release_acquire` |
| Release | SeqCst  |                       | `_release_seqcst`  |
| AcqRel  | Relaxed | `_acqrel_failrelaxed`    | `_acqrel_relaxed`  |
| AcqRel  | Acquire | `_acqrel`                | `_acqrel_acquire`  |
| AcqRel  | SeqCst  |                       | `_acqrel_seqcst`   |
| SeqCst  | Relaxed | `_failrelaxed`           | `_seqcst_relaxed`  |
| SeqCst  | Acquire | `_failacq`               | `_seqcst_acquire`  |
| SeqCst  | SeqCst  | (none)                   | `_seqcst_seqcst`   |
This commit is contained in:
Dylan DPC 2022-06-29 10:28:18 +05:30 committed by GitHub
commit 45740acd34
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 733 additions and 413 deletions

View file

@ -2575,11 +2575,11 @@ unsafe fn atomic_store<T: Copy>(dst: *mut T, val: T, order: Ordering) {
// SAFETY: the caller must uphold the safety contract for `atomic_store`.
unsafe {
match order {
Release => intrinsics::atomic_store_rel(dst, val),
Relaxed => intrinsics::atomic_store_relaxed(dst, val),
SeqCst => intrinsics::atomic_store(dst, val),
Release => intrinsics::atomic_store_release(dst, val),
SeqCst => intrinsics::atomic_store_seqcst(dst, val),
Acquire => panic!("there is no such thing as an acquire store"),
AcqRel => panic!("there is no such thing as an acquire/release store"),
AcqRel => panic!("there is no such thing as an acquire-release store"),
}
}
}
@ -2589,11 +2589,11 @@ unsafe fn atomic_load<T: Copy>(dst: *const T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_load`.
unsafe {
match order {
Acquire => intrinsics::atomic_load_acq(dst),
Relaxed => intrinsics::atomic_load_relaxed(dst),
SeqCst => intrinsics::atomic_load(dst),
Acquire => intrinsics::atomic_load_acquire(dst),
SeqCst => intrinsics::atomic_load_seqcst(dst),
Release => panic!("there is no such thing as a release load"),
AcqRel => panic!("there is no such thing as an acquire/release load"),
AcqRel => panic!("there is no such thing as an acquire-release load"),
}
}
}
@ -2604,11 +2604,11 @@ unsafe fn atomic_swap<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_swap`.
unsafe {
match order {
Acquire => intrinsics::atomic_xchg_acq(dst, val),
Release => intrinsics::atomic_xchg_rel(dst, val),
AcqRel => intrinsics::atomic_xchg_acqrel(dst, val),
Relaxed => intrinsics::atomic_xchg_relaxed(dst, val),
SeqCst => intrinsics::atomic_xchg(dst, val),
Acquire => intrinsics::atomic_xchg_acquire(dst, val),
Release => intrinsics::atomic_xchg_release(dst, val),
AcqRel => intrinsics::atomic_xchg_acqrel(dst, val),
SeqCst => intrinsics::atomic_xchg_seqcst(dst, val),
}
}
}
@ -2620,11 +2620,11 @@ unsafe fn atomic_add<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_add`.
unsafe {
match order {
Acquire => intrinsics::atomic_xadd_acq(dst, val),
Release => intrinsics::atomic_xadd_rel(dst, val),
AcqRel => intrinsics::atomic_xadd_acqrel(dst, val),
Relaxed => intrinsics::atomic_xadd_relaxed(dst, val),
SeqCst => intrinsics::atomic_xadd(dst, val),
Acquire => intrinsics::atomic_xadd_acquire(dst, val),
Release => intrinsics::atomic_xadd_release(dst, val),
AcqRel => intrinsics::atomic_xadd_acqrel(dst, val),
SeqCst => intrinsics::atomic_xadd_seqcst(dst, val),
}
}
}
@ -2636,11 +2636,11 @@ unsafe fn atomic_sub<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_sub`.
unsafe {
match order {
Acquire => intrinsics::atomic_xsub_acq(dst, val),
Release => intrinsics::atomic_xsub_rel(dst, val),
AcqRel => intrinsics::atomic_xsub_acqrel(dst, val),
Relaxed => intrinsics::atomic_xsub_relaxed(dst, val),
SeqCst => intrinsics::atomic_xsub(dst, val),
Acquire => intrinsics::atomic_xsub_acquire(dst, val),
Release => intrinsics::atomic_xsub_release(dst, val),
AcqRel => intrinsics::atomic_xsub_acqrel(dst, val),
SeqCst => intrinsics::atomic_xsub_seqcst(dst, val),
}
}
}
@ -2657,16 +2657,22 @@ unsafe fn atomic_compare_exchange<T: Copy>(
// SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange`.
let (val, ok) = unsafe {
match (success, failure) {
(Acquire, Acquire) => intrinsics::atomic_cxchg_acq(dst, old, new),
(Release, Relaxed) => intrinsics::atomic_cxchg_rel(dst, old, new),
(AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel(dst, old, new),
(Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed(dst, old, new),
(SeqCst, SeqCst) => intrinsics::atomic_cxchg(dst, old, new),
(Acquire, Relaxed) => intrinsics::atomic_cxchg_acq_failrelaxed(dst, old, new),
(AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_failrelaxed(dst, old, new),
(SeqCst, Relaxed) => intrinsics::atomic_cxchg_failrelaxed(dst, old, new),
(SeqCst, Acquire) => intrinsics::atomic_cxchg_failacq(dst, old, new),
(_, AcqRel) => panic!("there is no such thing as an acquire/release failure ordering"),
(Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed_relaxed(dst, old, new),
//(Relaxed, Acquire) => intrinsics::atomic_cxchg_relaxed_acquire(dst, old, new),
//(Relaxed, SeqCst) => intrinsics::atomic_cxchg_relaxed_seqcst(dst, old, new),
(Acquire, Relaxed) => intrinsics::atomic_cxchg_acquire_relaxed(dst, old, new),
(Acquire, Acquire) => intrinsics::atomic_cxchg_acquire_acquire(dst, old, new),
//(Acquire, SeqCst) => intrinsics::atomic_cxchg_acquire_seqcst(dst, old, new),
(Release, Relaxed) => intrinsics::atomic_cxchg_release_relaxed(dst, old, new),
//(Release, Acquire) => intrinsics::atomic_cxchg_release_acquire(dst, old, new),
//(Release, SeqCst) => intrinsics::atomic_cxchg_release_seqcst(dst, old, new),
(AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_relaxed(dst, old, new),
(AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel_acquire(dst, old, new),
//(AcqRel, SeqCst) => intrinsics::atomic_cxchg_acqrel_seqcst(dst, old, new),
(SeqCst, Relaxed) => intrinsics::atomic_cxchg_seqcst_relaxed(dst, old, new),
(SeqCst, Acquire) => intrinsics::atomic_cxchg_seqcst_acquire(dst, old, new),
(SeqCst, SeqCst) => intrinsics::atomic_cxchg_seqcst_seqcst(dst, old, new),
(_, AcqRel) => panic!("there is no such thing as an acquire-release failure ordering"),
(_, Release) => panic!("there is no such thing as a release failure ordering"),
_ => panic!("a failure ordering can't be stronger than a success ordering"),
}
@ -2686,16 +2692,22 @@ unsafe fn atomic_compare_exchange_weak<T: Copy>(
// SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange_weak`.
let (val, ok) = unsafe {
match (success, failure) {
(Acquire, Acquire) => intrinsics::atomic_cxchgweak_acq(dst, old, new),
(Release, Relaxed) => intrinsics::atomic_cxchgweak_rel(dst, old, new),
(AcqRel, Acquire) => intrinsics::atomic_cxchgweak_acqrel(dst, old, new),
(Relaxed, Relaxed) => intrinsics::atomic_cxchgweak_relaxed(dst, old, new),
(SeqCst, SeqCst) => intrinsics::atomic_cxchgweak(dst, old, new),
(Acquire, Relaxed) => intrinsics::atomic_cxchgweak_acq_failrelaxed(dst, old, new),
(AcqRel, Relaxed) => intrinsics::atomic_cxchgweak_acqrel_failrelaxed(dst, old, new),
(SeqCst, Relaxed) => intrinsics::atomic_cxchgweak_failrelaxed(dst, old, new),
(SeqCst, Acquire) => intrinsics::atomic_cxchgweak_failacq(dst, old, new),
(_, AcqRel) => panic!("there is no such thing as an acquire/release failure ordering"),
(Relaxed, Relaxed) => intrinsics::atomic_cxchgweak_relaxed_relaxed(dst, old, new),
//(Relaxed, Acquire) => intrinsics::atomic_cxchgweak_relaxed_acquire(dst, old, new),
//(Relaxed, SeqCst) => intrinsics::atomic_cxchgweak_relaxed_seqcst(dst, old, new),
(Acquire, Relaxed) => intrinsics::atomic_cxchgweak_acquire_relaxed(dst, old, new),
(Acquire, Acquire) => intrinsics::atomic_cxchgweak_acquire_acquire(dst, old, new),
//(Acquire, SeqCst) => intrinsics::atomic_cxchgweak_acquire_seqcst(dst, old, new),
(Release, Relaxed) => intrinsics::atomic_cxchgweak_release_relaxed(dst, old, new),
//(Release, Acquire) => intrinsics::atomic_cxchgweak_release_acquire(dst, old, new),
//(Release, SeqCst) => intrinsics::atomic_cxchgweak_release_seqcst(dst, old, new),
(AcqRel, Relaxed) => intrinsics::atomic_cxchgweak_acqrel_relaxed(dst, old, new),
(AcqRel, Acquire) => intrinsics::atomic_cxchgweak_acqrel_acquire(dst, old, new),
//(AcqRel, SeqCst) => intrinsics::atomic_cxchgweak_acqrel_seqcst(dst, old, new),
(SeqCst, Relaxed) => intrinsics::atomic_cxchgweak_seqcst_relaxed(dst, old, new),
(SeqCst, Acquire) => intrinsics::atomic_cxchgweak_seqcst_acquire(dst, old, new),
(SeqCst, SeqCst) => intrinsics::atomic_cxchgweak_seqcst_seqcst(dst, old, new),
(_, AcqRel) => panic!("there is no such thing as an acquire-release failure ordering"),
(_, Release) => panic!("there is no such thing as a release failure ordering"),
_ => panic!("a failure ordering can't be stronger than a success ordering"),
}
@ -2709,11 +2721,11 @@ unsafe fn atomic_and<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_and`
unsafe {
match order {
Acquire => intrinsics::atomic_and_acq(dst, val),
Release => intrinsics::atomic_and_rel(dst, val),
AcqRel => intrinsics::atomic_and_acqrel(dst, val),
Relaxed => intrinsics::atomic_and_relaxed(dst, val),
SeqCst => intrinsics::atomic_and(dst, val),
Acquire => intrinsics::atomic_and_acquire(dst, val),
Release => intrinsics::atomic_and_release(dst, val),
AcqRel => intrinsics::atomic_and_acqrel(dst, val),
SeqCst => intrinsics::atomic_and_seqcst(dst, val),
}
}
}
@ -2724,11 +2736,11 @@ unsafe fn atomic_nand<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_nand`
unsafe {
match order {
Acquire => intrinsics::atomic_nand_acq(dst, val),
Release => intrinsics::atomic_nand_rel(dst, val),
AcqRel => intrinsics::atomic_nand_acqrel(dst, val),
Relaxed => intrinsics::atomic_nand_relaxed(dst, val),
SeqCst => intrinsics::atomic_nand(dst, val),
Acquire => intrinsics::atomic_nand_acquire(dst, val),
Release => intrinsics::atomic_nand_release(dst, val),
AcqRel => intrinsics::atomic_nand_acqrel(dst, val),
SeqCst => intrinsics::atomic_nand_seqcst(dst, val),
}
}
}
@ -2739,11 +2751,11 @@ unsafe fn atomic_or<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_or`
unsafe {
match order {
Acquire => intrinsics::atomic_or_acq(dst, val),
Release => intrinsics::atomic_or_rel(dst, val),
SeqCst => intrinsics::atomic_or_seqcst(dst, val),
Acquire => intrinsics::atomic_or_acquire(dst, val),
Release => intrinsics::atomic_or_release(dst, val),
AcqRel => intrinsics::atomic_or_acqrel(dst, val),
Relaxed => intrinsics::atomic_or_relaxed(dst, val),
SeqCst => intrinsics::atomic_or(dst, val),
}
}
}
@ -2754,11 +2766,11 @@ unsafe fn atomic_xor<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_xor`
unsafe {
match order {
Acquire => intrinsics::atomic_xor_acq(dst, val),
Release => intrinsics::atomic_xor_rel(dst, val),
SeqCst => intrinsics::atomic_xor_seqcst(dst, val),
Acquire => intrinsics::atomic_xor_acquire(dst, val),
Release => intrinsics::atomic_xor_release(dst, val),
AcqRel => intrinsics::atomic_xor_acqrel(dst, val),
Relaxed => intrinsics::atomic_xor_relaxed(dst, val),
SeqCst => intrinsics::atomic_xor(dst, val),
}
}
}
@ -2770,11 +2782,11 @@ unsafe fn atomic_max<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_max`
unsafe {
match order {
Acquire => intrinsics::atomic_max_acq(dst, val),
Release => intrinsics::atomic_max_rel(dst, val),
AcqRel => intrinsics::atomic_max_acqrel(dst, val),
Relaxed => intrinsics::atomic_max_relaxed(dst, val),
SeqCst => intrinsics::atomic_max(dst, val),
Acquire => intrinsics::atomic_max_acquire(dst, val),
Release => intrinsics::atomic_max_release(dst, val),
AcqRel => intrinsics::atomic_max_acqrel(dst, val),
SeqCst => intrinsics::atomic_max_seqcst(dst, val),
}
}
}
@ -2786,11 +2798,11 @@ unsafe fn atomic_min<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_min`
unsafe {
match order {
Acquire => intrinsics::atomic_min_acq(dst, val),
Release => intrinsics::atomic_min_rel(dst, val),
AcqRel => intrinsics::atomic_min_acqrel(dst, val),
Relaxed => intrinsics::atomic_min_relaxed(dst, val),
SeqCst => intrinsics::atomic_min(dst, val),
Acquire => intrinsics::atomic_min_acquire(dst, val),
Release => intrinsics::atomic_min_release(dst, val),
AcqRel => intrinsics::atomic_min_acqrel(dst, val),
SeqCst => intrinsics::atomic_min_seqcst(dst, val),
}
}
}
@ -2802,11 +2814,11 @@ unsafe fn atomic_umax<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_umax`
unsafe {
match order {
Acquire => intrinsics::atomic_umax_acq(dst, val),
Release => intrinsics::atomic_umax_rel(dst, val),
AcqRel => intrinsics::atomic_umax_acqrel(dst, val),
Relaxed => intrinsics::atomic_umax_relaxed(dst, val),
SeqCst => intrinsics::atomic_umax(dst, val),
Acquire => intrinsics::atomic_umax_acquire(dst, val),
Release => intrinsics::atomic_umax_release(dst, val),
AcqRel => intrinsics::atomic_umax_acqrel(dst, val),
SeqCst => intrinsics::atomic_umax_seqcst(dst, val),
}
}
}
@ -2818,11 +2830,11 @@ unsafe fn atomic_umin<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_umin`
unsafe {
match order {
Acquire => intrinsics::atomic_umin_acq(dst, val),
Release => intrinsics::atomic_umin_rel(dst, val),
AcqRel => intrinsics::atomic_umin_acqrel(dst, val),
Relaxed => intrinsics::atomic_umin_relaxed(dst, val),
SeqCst => intrinsics::atomic_umin(dst, val),
Acquire => intrinsics::atomic_umin_acquire(dst, val),
Release => intrinsics::atomic_umin_release(dst, val),
AcqRel => intrinsics::atomic_umin_acqrel(dst, val),
SeqCst => intrinsics::atomic_umin_seqcst(dst, val),
}
}
}
@ -2908,10 +2920,10 @@ pub fn fence(order: Ordering) {
// SAFETY: using an atomic fence is safe.
unsafe {
match order {
Acquire => intrinsics::atomic_fence_acq(),
Release => intrinsics::atomic_fence_rel(),
Acquire => intrinsics::atomic_fence_acquire(),
Release => intrinsics::atomic_fence_release(),
AcqRel => intrinsics::atomic_fence_acqrel(),
SeqCst => intrinsics::atomic_fence(),
SeqCst => intrinsics::atomic_fence_seqcst(),
Relaxed => panic!("there is no such thing as a relaxed fence"),
}
}
@ -2990,10 +3002,10 @@ pub fn compiler_fence(order: Ordering) {
// SAFETY: using an atomic fence is safe.
unsafe {
match order {
Acquire => intrinsics::atomic_singlethreadfence_acq(),
Release => intrinsics::atomic_singlethreadfence_rel(),
Acquire => intrinsics::atomic_singlethreadfence_acquire(),
Release => intrinsics::atomic_singlethreadfence_release(),
AcqRel => intrinsics::atomic_singlethreadfence_acqrel(),
SeqCst => intrinsics::atomic_singlethreadfence(),
SeqCst => intrinsics::atomic_singlethreadfence_seqcst(),
Relaxed => panic!("there is no such thing as a relaxed compiler fence"),
}
}