public/j7s_diagnostics_py
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Vendor deps.

Browse Source
This commit is contained in:
jimmy committed 2026-09-16 20:10:35 -04:00
1 parent 294527085e
commit 91e18fde3a
1714 files changed
+556667 -7
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

vendor/portable-atomic/src/cfgs.rs
Vendored
+631
View File
@@ -0,0 +1,631 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
#![allow(missing_docs)]
#[cfg(not(all(
portable_atomic_no_atomic_load_store,
not(any(
target_arch = "avr",
target_arch = "msp430",
target_arch = "riscv32",
target_arch = "riscv64",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
)),
)))]
#[macro_use]
mod atomic_8_16_macros {
#[macro_export]
macro_rules! cfg_has_atomic_8 {
($($tt:tt)*) => {
$($tt)*
};
}
#[macro_export]
macro_rules! cfg_no_atomic_8 {
($($tt:tt)*) => {};
}
#[macro_export]
macro_rules! cfg_has_atomic_16 {
($($tt:tt)*) => {
$($tt)*
};
}
#[macro_export]
macro_rules! cfg_no_atomic_16 {
($($tt:tt)*) => {};
}
}
#[cfg(all(
portable_atomic_no_atomic_load_store,
not(any(
target_arch = "avr",
target_arch = "msp430",
target_arch = "riscv32",
target_arch = "riscv64",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
)),
))]
#[macro_use]
mod atomic_8_16_macros {
#[macro_export]
macro_rules! cfg_has_atomic_8 {
($($tt:tt)*) => {};
}
#[macro_export]
macro_rules! cfg_no_atomic_8 {
($($tt:tt)*) => {
$($tt)*
};
}
#[macro_export]
macro_rules! cfg_has_atomic_16 {
($($tt:tt)*) => {};
}
#[macro_export]
macro_rules! cfg_no_atomic_16 {
($($tt:tt)*) => {
$($tt)*
};
}
}
#[cfg(all(
any(not(target_pointer_width = "16"), feature = "fallback"),
not(all(
portable_atomic_no_atomic_load_store,
not(any(
target_arch = "avr",
target_arch = "msp430",
target_arch = "riscv32",
target_arch = "riscv64",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
)),
)),
))]
#[macro_use]
mod atomic_32_macros {
#[macro_export]
macro_rules! cfg_has_atomic_32 {
($($tt:tt)*) => {
$($tt)*
};
}
#[macro_export]
macro_rules! cfg_no_atomic_32 {
($($tt:tt)*) => {};
}
}
#[cfg(not(all(
any(not(target_pointer_width = "16"), feature = "fallback"),
not(all(
portable_atomic_no_atomic_load_store,
not(any(
target_arch = "avr",
target_arch = "msp430",
target_arch = "riscv32",
target_arch = "riscv64",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
)),
)),
)))]
#[macro_use]
mod atomic_32_macros {
#[macro_export]
macro_rules! cfg_has_atomic_32 {
($($tt:tt)*) => {};
}
#[macro_export]
macro_rules! cfg_no_atomic_32 {
($($tt:tt)*) => {
$($tt)*
};
}
}
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(any(
all(
feature = "fallback",
any(
not(portable_atomic_no_atomic_cas),
target_arch = "avr",
target_arch = "msp430",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
),
),
not(portable_atomic_no_atomic_64),
not(any(target_pointer_width = "16", target_pointer_width = "32")),
all(
target_arch = "riscv32",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
),
))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(any(
all(
feature = "fallback",
any(
target_has_atomic = "ptr",
target_arch = "avr",
target_arch = "msp430",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
),
),
target_has_atomic = "64",
not(any(target_pointer_width = "16", target_pointer_width = "32")),
all(
target_arch = "riscv32",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
),
))
)]
#[macro_use]
mod atomic_64_macros {
#[macro_export]
macro_rules! cfg_has_atomic_64 {
($($tt:tt)*) => {
$($tt)*
};
}
#[macro_export]
macro_rules! cfg_no_atomic_64 {
($($tt:tt)*) => {};
}
}
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(not(any(
all(
feature = "fallback",
any(
not(portable_atomic_no_atomic_cas),
target_arch = "avr",
target_arch = "msp430",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
),
),
not(portable_atomic_no_atomic_64),
not(any(target_pointer_width = "16", target_pointer_width = "32")),
all(
target_arch = "riscv32",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
),
)))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(not(any(
all(
feature = "fallback",
any(
target_has_atomic = "ptr",
target_arch = "avr",
target_arch = "msp430",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
),
),
target_has_atomic = "64",
not(any(target_pointer_width = "16", target_pointer_width = "32")),
all(
target_arch = "riscv32",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
),
)))
)]
#[macro_use]
mod atomic_64_macros {
#[macro_export]
macro_rules! cfg_has_atomic_64 {
($($tt:tt)*) => {};
}
#[macro_export]
macro_rules! cfg_no_atomic_64 {
($($tt:tt)*) => {
$($tt)*
};
}
}
#[cfg_attr(
not(feature = "fallback"),
cfg(any(
all(
target_arch = "aarch64",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
),
all(
target_arch = "arm64ec",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
not(portable_atomic_no_asm),
),
all(
target_arch = "x86_64",
not(all(
any(miri, portable_atomic_sanitize_thread),
portable_atomic_no_cmpxchg16b_intrinsic,
)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
),
all(
target_arch = "riscv64",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
),
all(
target_arch = "powerpc64",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
not(portable_atomic_no_asm),
any(
target_feature = "quadword-atomics",
portable_atomic_target_feature = "quadword-atomics",
),
),
all(
target_arch = "s390x",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
not(portable_atomic_no_asm),
),
))
)]
#[cfg_attr(
all(feature = "fallback", portable_atomic_no_cfg_target_has_atomic),
cfg(any(
not(portable_atomic_no_atomic_cas),
target_arch = "avr",
target_arch = "msp430",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
))
)]
#[cfg_attr(
all(feature = "fallback", not(portable_atomic_no_cfg_target_has_atomic)),
cfg(any(
target_has_atomic = "ptr",
target_arch = "avr",
target_arch = "msp430",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
))
)]
#[macro_use]
mod atomic_128_macros {
#[macro_export]
macro_rules! cfg_has_atomic_128 {
($($tt:tt)*) => {
$($tt)*
};
}
#[macro_export]
macro_rules! cfg_no_atomic_128 {
($($tt:tt)*) => {};
}
}
#[cfg_attr(
not(feature = "fallback"),
cfg(not(any(
all(
target_arch = "aarch64",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
),
all(
target_arch = "arm64ec",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
not(portable_atomic_no_asm),
),
all(
target_arch = "x86_64",
not(all(
any(miri, portable_atomic_sanitize_thread),
portable_atomic_no_cmpxchg16b_intrinsic,
)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
),
all(
target_arch = "riscv64",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
),
all(
target_arch = "powerpc64",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
not(portable_atomic_no_asm),
any(
target_feature = "quadword-atomics",
portable_atomic_target_feature = "quadword-atomics",
),
),
all(
target_arch = "s390x",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
not(portable_atomic_no_asm),
),
)))
)]
#[cfg_attr(
all(feature = "fallback", portable_atomic_no_cfg_target_has_atomic),
cfg(not(any(
not(portable_atomic_no_atomic_cas),
target_arch = "avr",
target_arch = "msp430",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
)))
)]
#[cfg_attr(
all(feature = "fallback", not(portable_atomic_no_cfg_target_has_atomic)),
cfg(not(any(
target_has_atomic = "ptr",
target_arch = "avr",
target_arch = "msp430",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
)))
)]
#[macro_use]
mod atomic_128_macros {
#[macro_export]
macro_rules! cfg_has_atomic_128 {
($($tt:tt)*) => {};
}
#[macro_export]
macro_rules! cfg_no_atomic_128 {
($($tt:tt)*) => {
$($tt)*
};
}
}
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(any(
not(portable_atomic_no_atomic_cas),
target_arch = "avr",
target_arch = "msp430",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(any(
target_has_atomic = "ptr",
target_arch = "avr",
target_arch = "msp430",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
))
)]
#[macro_use]
mod atomic_cas_macros {
#[macro_export]
macro_rules! cfg_has_atomic_cas {
($($tt:tt)*) => {
$($tt)*
};
}
#[macro_export]
macro_rules! cfg_no_atomic_cas {
($($tt:tt)*) => {};
}
// private
macro_rules! cfg_has_atomic_cas_or_amo32 {
($($tt:tt)*) => {
$($tt)*
};
}
macro_rules! cfg_has_atomic_cas_or_amo8 {
($($tt:tt)*) => {
$($tt)*
};
}
}
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(not(any(
not(portable_atomic_no_atomic_cas),
target_arch = "avr",
target_arch = "msp430",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
)))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(not(any(
target_has_atomic = "ptr",
target_arch = "avr",
target_arch = "msp430",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
)))
)]
#[macro_use]
mod atomic_cas_macros {
#[macro_export]
macro_rules! cfg_has_atomic_cas {
($($tt:tt)*) => {};
}
#[macro_export]
macro_rules! cfg_no_atomic_cas {
($($tt:tt)*) => {
$($tt)*
};
}
// private
#[cfg_attr(
any(target_arch = "riscv32", target_arch = "riscv64"),
cfg(not(any(target_feature = "zaamo", portable_atomic_target_feature = "zaamo")))
)]
macro_rules! cfg_has_atomic_cas_or_amo32 {
($($tt:tt)*) => {};
}
#[cfg_attr(
any(target_arch = "riscv32", target_arch = "riscv64"),
cfg(not(any(target_feature = "zaamo", portable_atomic_target_feature = "zaamo")))
)]
macro_rules! cfg_no_atomic_cas_or_amo32 {
($($tt:tt)*) => {
$($tt)*
};
}
#[cfg(all(
any(target_arch = "riscv32", target_arch = "riscv64"),
any(target_feature = "zaamo", portable_atomic_target_feature = "zaamo"),
))]
macro_rules! cfg_has_atomic_cas_or_amo32 {
($($tt:tt)*) => {
$($tt)*
};
}
#[cfg(all(
any(target_arch = "riscv32", target_arch = "riscv64"),
any(target_feature = "zaamo", portable_atomic_target_feature = "zaamo"),
))]
macro_rules! cfg_no_atomic_cas_or_amo32 {
($($tt:tt)*) => {};
}
#[cfg_attr(
any(target_arch = "riscv32", target_arch = "riscv64"),
cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))
)]
#[allow(unused_macros)]
macro_rules! cfg_has_atomic_cas_or_amo8 {
($($tt:tt)*) => {};
}
#[cfg_attr(
any(target_arch = "riscv32", target_arch = "riscv64"),
cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))
)]
#[cfg_attr(target_arch = "bpf", allow(unused_macros))]
macro_rules! cfg_no_atomic_cas_or_amo8 {
($($tt:tt)*) => {
$($tt)*
};
}
#[cfg(all(
any(target_arch = "riscv32", target_arch = "riscv64"),
any(target_feature = "zabha", portable_atomic_target_feature = "zabha"),
))]
macro_rules! cfg_has_atomic_cas_or_amo8 {
($($tt:tt)*) => {
$($tt)*
};
}
#[cfg(all(
any(target_arch = "riscv32", target_arch = "riscv64"),
any(target_feature = "zabha", portable_atomic_target_feature = "zabha"),
))]
macro_rules! cfg_no_atomic_cas_or_amo8 {
($($tt:tt)*) => {};
}
}
macro_rules! cfg_core_atomic {
($($tt:tt)*) => {
#[cfg(not(any(
target_arch = "avr",
target_arch = "msp430",
all(
portable_atomic_no_atomic_load_store,
not(all(target_arch = "bpf", not(feature = "critical-section"))),
),
)))]
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(not(all(
any(
target_arch = "riscv32",
target_arch = "riscv64",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
),
portable_atomic_no_atomic_cas,
)))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(not(all(
any(
target_arch = "riscv32",
target_arch = "riscv64",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
),
not(target_has_atomic = "ptr"),
)))
)]
items!($($tt)*);
};
}
// Check that all cfg_ macros work.
mod check {
crate::cfg_has_atomic_8! { type _Atomic8 = (); }
crate::cfg_no_atomic_8! { type _Atomic8 = (); }
crate::cfg_has_atomic_16! { type _Atomic16 = (); }
crate::cfg_no_atomic_16! { type _Atomic16 = (); }
crate::cfg_has_atomic_32! { type _Atomic32 = (); }
crate::cfg_no_atomic_32! { type _Atomic32 = (); }
crate::cfg_has_atomic_64! { type _Atomic64 = (); }
crate::cfg_no_atomic_64! { type _Atomic64 = (); }
crate::cfg_has_atomic_128! { type _Atomic128 = (); }
crate::cfg_no_atomic_128! { type _Atomic128 = (); }
crate::cfg_has_atomic_ptr! { type _AtomicPtr = (); }
crate::cfg_no_atomic_ptr! { type _AtomicPtr = (); }
crate::cfg_has_atomic_cas! { type __AtomicPtr = (); }
crate::cfg_no_atomic_cas! { type __AtomicPtr = (); }
#[allow(unused_imports)]
use self::{
__AtomicPtr as _, _Atomic8 as _, _Atomic16 as _, _Atomic32 as _, _Atomic64 as _,
_Atomic128 as _, _AtomicPtr as _,
};
}
vendor/portable-atomic/src/gen/build.rs
Vendored
+80
View File
@@ -0,0 +1,80 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
// This file is @generated by target_spec.sh.
// It is not intended for manual editing.
// Note: This is the list as of nightly-2022-02-10. We don't refer to this in
// nightly-2022-02-11+ because feature(cfg_target_has_atomic) stabilized.
#[rustfmt::skip]
pub(crate) static NO_ATOMIC_CAS: &[&str] = &[
"avr-unknown-gnu-atmega328",
"bpfeb-unknown-none",
"bpfel-unknown-none",
"msp430-none-elf",
"riscv32i-unknown-none-elf",
"riscv32imc-unknown-none-elf",
"thumbv4t-none-eabi",
"thumbv6m-none-eabi",
];
// Note: This is the list as of nightly-2022-02-10. We don't refer to this in
// nightly-2022-02-11+ because feature(cfg_target_has_atomic) stabilized.
#[rustfmt::skip]
pub(crate) static NO_ATOMIC_64: &[&str] = &[
"arm-linux-androideabi",
"armebv7r-none-eabi",
"armebv7r-none-eabihf",
"armv4t-unknown-linux-gnueabi",
"armv5te-unknown-linux-gnueabi",
"armv5te-unknown-linux-musleabi",
"armv5te-unknown-linux-uclibceabi",
"armv6k-nintendo-3ds",
"armv7r-none-eabi",
"armv7r-none-eabihf",
"avr-unknown-gnu-atmega328",
"hexagon-unknown-linux-musl",
"m68k-unknown-linux-gnu",
"mips-unknown-linux-gnu",
"mips-unknown-linux-musl",
"mips-unknown-linux-uclibc",
"mipsel-sony-psp",
"mipsel-unknown-linux-gnu",
"mipsel-unknown-linux-musl",
"mipsel-unknown-linux-uclibc",
"mipsel-unknown-none",
"mipsisa32r6-unknown-linux-gnu",
"mipsisa32r6el-unknown-linux-gnu",
"msp430-none-elf",
"powerpc-unknown-freebsd",
"powerpc-unknown-linux-gnu",
"powerpc-unknown-linux-gnuspe",
"powerpc-unknown-linux-musl",
"powerpc-unknown-netbsd",
"powerpc-unknown-openbsd",
"powerpc-wrs-vxworks",
"powerpc-wrs-vxworks-spe",
"riscv32gc-unknown-linux-gnu",
"riscv32gc-unknown-linux-musl",
"riscv32i-unknown-none-elf",
"riscv32imac-unknown-none-elf",
"riscv32imc-esp-espidf",
"riscv32imc-unknown-none-elf",
"thumbv4t-none-eabi",
"thumbv6m-none-eabi",
"thumbv7em-none-eabi",
"thumbv7em-none-eabihf",
"thumbv7m-none-eabi",
"thumbv8m.base-none-eabi",
"thumbv8m.main-none-eabi",
"thumbv8m.main-none-eabihf",
];
#[rustfmt::skip]
pub(crate) static NO_ATOMIC: &[&str] = &[
"armv4t-none-eabi",
"armv5te-none-eabi",
"bpfeb-unknown-none",
"bpfel-unknown-none",
"mipsel-sony-psx",
"thumbv4t-none-eabi",
"thumbv5te-none-eabi",
];
vendor/portable-atomic/src/gen/utils.rs
Vendored
+168
View File
@@ -0,0 +1,168 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
// This file is @generated by target_spec.sh.
// It is not intended for manual editing.
#![allow(dead_code, unused_macros)]
// On AArch64, the base register of memory-related instructions must be 64-bit.
// Passing a 32-bit value to `in(reg)` on AArch64 results in the upper bits
// having an undefined value, but to work correctly with ILP32 ABI, the upper
// bits must be zero, which is handled here by casting to u64. Another way to
// handle this is to pass it as a pointer and clear the upper bits inside asm,
// but it is easier to overlook than cast, which can catch overlooks by
// asm_sub_register lint.
// See also https://github.com/ARM-software/abi-aa/blob/2025Q4/aapcs64/aapcs64.rst#pointers
//
// Except for x86_64, which can use 32-bit registers in the destination operand
// (on x86_64, we use the ptr_modifier macro to handle this), we need to do the
// same for ILP32 ABI on other 64-bit architectures. (At least, as far as I can
// see from the assembly generated by LLVM, this is also required for MIPS64 N32
// ABI. I don't know about other similar ABIs, but in any case, this should be
// a safe default for such ABIs).
//
// Known architectures that have such ABI are x86_64 (X32), AArch64 (ILP32),
// mips64/mips64r6 (N32), riscv64 (RV64ILP32*), PowerPC64 (Lv2). (As of
// 2026-03-22, only the former two are supported by rustc.) However, we list
// all known 64-bit architectures because similar ABIs may exist or future added
// for other architectures.
#[cfg(all(
target_pointer_width = "32",
any(
target_arch = "aarch64",
target_arch = "amdgpu",
target_arch = "arm64ec",
target_arch = "bpf",
target_arch = "loongarch64",
target_arch = "mips64",
target_arch = "mips64r6",
target_arch = "nvptx64",
target_arch = "powerpc64",
target_arch = "riscv64",
target_arch = "s390x",
target_arch = "sparc64",
target_arch = "wasm64",
target_arch = "x86_64",
),
))]
#[macro_use]
mod imp {
#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
macro_rules! ptr_reg {
($ptr:ident) => {{
let _: *const _ = $ptr; // ensure $ptr is a pointer (*mut _ or *const _)
#[cfg(not(portable_atomic_no_asm_maybe_uninit))]
#[allow(clippy::ptr_as_ptr)]
{
// If we cast to u64 here, the provenance will be lost,
// so we convert to MaybeUninit<u64> via zero extend helper.
crate::utils::zero_extend64_ptr($ptr as *mut ())
}
#[cfg(portable_atomic_no_asm_maybe_uninit)]
{
// Use cast on old rustc because it does not support MaybeUninit
// registers. This is still permissive-provenance compatible and
// is sound.
$ptr as u64
}
}};
}
pub(crate) type RegISize = i64;
pub(crate) type RegSize = u64;
}
#[cfg(not(all(
target_pointer_width = "32",
any(
target_arch = "aarch64",
target_arch = "amdgpu",
target_arch = "arm64ec",
target_arch = "bpf",
target_arch = "loongarch64",
target_arch = "mips64",
target_arch = "mips64r6",
target_arch = "nvptx64",
target_arch = "powerpc64",
target_arch = "riscv64",
target_arch = "s390x",
target_arch = "sparc64",
target_arch = "wasm64",
target_arch = "x86_64",
),
)))]
#[macro_use]
mod imp {
#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
macro_rules! ptr_reg {
($ptr:ident) => {{
let _: *const _ = $ptr; // ensure $ptr is a pointer (*mut _ or *const _)
$ptr // cast is unnecessary here.
}};
}
pub(crate) type RegISize = isize;
pub(crate) type RegSize = usize;
}
pub(crate) use self::imp::{RegISize, RegSize};
// Some 64-bit architectures have ABI with 32-bit pointer width (e.g., x86_64 X32 ABI,
// AArch64 ILP32 ABI, MIPS64 N32 ABI). On those targets, AtomicU64 is available
// and fast, so use it to implement normal sequence lock.
//
// See ptr_reg macro for the reason why all known 64-bit architectures are listed.
//
// Note: These macros are cfg_*!({ }), not cfg_*! { }.
// An extra brace is used in input to make contents rustfmt-able.
#[cfg(any(
not(any(target_pointer_width = "16", target_pointer_width = "32")), // i.e., 64-bit or greater
target_arch = "aarch64",
target_arch = "amdgpu",
target_arch = "arm64ec",
target_arch = "bpf",
target_arch = "loongarch64",
target_arch = "mips64",
target_arch = "mips64r6",
target_arch = "nvptx64",
target_arch = "powerpc64",
target_arch = "riscv64",
target_arch = "s390x",
target_arch = "sparc64",
target_arch = "wasm64",
target_arch = "x86_64",
))]
#[macro_use]
mod fast_atomic_64_macros {
macro_rules! cfg_has_fast_atomic_64 {
({$($tt:tt)*}) => {
$($tt)*
};
}
macro_rules! cfg_no_fast_atomic_64 {
({$($tt:tt)*}) => {};
}
}
#[cfg(not(any(
not(any(target_pointer_width = "16", target_pointer_width = "32")), // i.e., 64-bit or greater
target_arch = "aarch64",
target_arch = "amdgpu",
target_arch = "arm64ec",
target_arch = "bpf",
target_arch = "loongarch64",
target_arch = "mips64",
target_arch = "mips64r6",
target_arch = "nvptx64",
target_arch = "powerpc64",
target_arch = "riscv64",
target_arch = "s390x",
target_arch = "sparc64",
target_arch = "wasm64",
target_arch = "x86_64",
)))]
#[macro_use]
mod fast_atomic_64_macros {
macro_rules! cfg_has_fast_atomic_64 {
({$($tt:tt)*}) => {};
}
macro_rules! cfg_no_fast_atomic_64 {
({$($tt:tt)*}) => {
$($tt)*
};
}
}
vendor/portable-atomic/src/imp/arm_linux.rs
Vendored
+257
View File
@@ -0,0 +1,257 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Atomic operations implementation on pre-v6 ARM Linux/Android.
This module optimizes cases where LLVM code generation is not optimal,
and provides workaround for core::sync::atomic bug (https://github.com/rust-lang/compiler-builtins/pull/1234).
See "Atomic operation overview by architecture" in atomic-maybe-uninit for a more comprehensive and
detailed description of the atomic and synchronize instructions in this architecture:
https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#arm
Refs:
- https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/arm/kernel_user_helpers.rst
- https://github.com/rust-lang/compiler-builtins/blob/compiler_builtins-v0.1.124/src/arm_linux.rs
- https://github.com/taiki-e/atomic-maybe-uninit
Note: __kuser_memory_barrier is always SeqCst.
https://github.com/taiki-e/atomic-maybe-uninit/blob/v0.3.21/src/arch/arm.rs#L215
Note: On Miri and ThreadSanitizer which do not go through compiler-builtins, we don't use
this module and use normal core::sync::atomic instead.
*/
use core::{mem, sync::atomic::Ordering};
// https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/arm/kernel_user_helpers.rst
// __kuser_helper_version >= 3 (kernel version 2.6.15+)
const KUSER_MEMORY_BARRIER: usize = 0xFFFF0FA0;
#[inline]
fn __kuser_memory_barrier() {
// SAFETY: kernel docs specify a known address with the given signature.
// core assumes __kuser_helper_version >= 3.
unsafe {
let f: extern "C" fn() =
mem::transmute(crate::utils::ptr::with_exposed_provenance::<()>(KUSER_MEMORY_BARRIER));
f();
}
}
/// An atomic fence.
///
/// # Panics
///
/// Panics if `order` is [`Relaxed`](Ordering::Relaxed).
#[cfg_attr(doc, allow(dead_code))]
#[inline]
#[cfg_attr(any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri), track_caller)]
pub fn fence(order: Ordering) {
// Call __kuser_memory_barrier without going through __sync_synchronize.
match order {
Ordering::Relaxed => panic!("there is no such thing as a relaxed fence"),
_ => __kuser_memory_barrier(),
}
}
cfg_core_atomic!({
use super::core_atomic::{
AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, AtomicU32,
AtomicUsize,
};
macro_rules! atomic {
($([$($generics:tt)*])? $atomic_type:ident, $value_type:ty) => {
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn load(&self, order: Ordering) -> $value_type {
crate::utils::assert_load_ordering(order);
match order {
Ordering::Relaxed => self.inner.load(Ordering::Relaxed),
// Acquire and SeqCst loads are equivalent.
// This matches with LLVM, but GCC emits `dmb; ldr*; dmb` for SeqCst load.
Ordering::Acquire | Ordering::SeqCst => {
let out = self.inner.load(Ordering::Relaxed);
__kuser_memory_barrier();
out
}
_ => unreachable!(),
}
}
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn store(&self, val: $value_type, order: Ordering) {
crate::utils::assert_store_ordering(order);
match order {
Ordering::Relaxed => self.inner.store(val, Ordering::Relaxed),
Ordering::Release => {
__kuser_memory_barrier();
self.inner.store(val, Ordering::Relaxed);
}
Ordering::SeqCst => {
__kuser_memory_barrier();
self.inner.store(val, Ordering::Relaxed);
__kuser_memory_barrier();
}
_ => unreachable!(),
}
}
}
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(not(portable_atomic_no_atomic_cas))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(target_has_atomic = "ptr")
)]
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn compare_exchange(
&self,
current: $value_type,
new: $value_type,
success: Ordering,
failure: Ordering,
) -> Result<$value_type, $value_type> {
crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)
#[cfg(portable_atomic_no_stronger_failure_ordering)]
let success = crate::utils::upgrade_success_ordering(success, failure);
let res = self.inner.compare_exchange(current, new, success, failure);
if failure != Ordering::Relaxed && res.is_err() {
__kuser_memory_barrier();
}
res
}
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn compare_exchange_weak(
&self,
current: $value_type,
new: $value_type,
success: Ordering,
failure: Ordering,
) -> Result<$value_type, $value_type> {
crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)
#[cfg(portable_atomic_no_stronger_failure_ordering)]
let success = crate::utils::upgrade_success_ordering(success, failure);
let res = self.inner.compare_exchange_weak(current, new, success, failure);
if failure != Ordering::Relaxed && res.is_err() {
__kuser_memory_barrier();
}
res
}
}
};
}
atomic!(AtomicI8, i8);
atomic!(AtomicU8, u8);
atomic!(AtomicI16, i16);
atomic!(AtomicU16, u16);
atomic!(AtomicI32, i32);
atomic!(AtomicU32, u32);
atomic!(AtomicIsize, isize);
atomic!(AtomicUsize, usize);
atomic!([T] AtomicPtr, *mut T);
});
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_64)))]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(any(
target_has_atomic = "64",
not(any(target_pointer_width = "16", target_pointer_width = "32")),
))
)]
items!({
// This is not normally reached on this platform.
// It is only reached if something strange occurs, such as using target-feature=-v6 on armv6 target.
// Since 64-bit atomic operations differ from 32-bit or smaller ones on Arm, similar optimizations
// or workarounds are useless here, so do the same as atomic_int! macro in core_atomic.rs.
// NB: Sync with atomic_int! macro in core_atomic.rs
use super::core_atomic::{AtomicI64, AtomicU64};
macro_rules! atomic_int {
($atomic_type:ident, $int_type:ident) => {
impl $atomic_type {
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn load(&self, order: Ordering) -> $int_type {
crate::utils::assert_load_ordering(order); // for track_caller (compiler can omit double check)
self.inner.load(order)
}
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn store(&self, val: $int_type, order: Ordering) {
crate::utils::assert_store_ordering(order); // for track_caller (compiler can omit double check)
self.inner.store(val, order);
}
}
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(not(portable_atomic_no_atomic_cas))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(target_has_atomic = "ptr")
)]
impl $atomic_type {
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn compare_exchange(
&self,
current: $int_type,
new: $int_type,
success: Ordering,
failure: Ordering,
) -> Result<$int_type, $int_type> {
crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)
#[cfg(portable_atomic_no_stronger_failure_ordering)]
let success = crate::utils::upgrade_success_ordering(success, failure);
self.inner.compare_exchange(current, new, success, failure)
}
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn compare_exchange_weak(
&self,
current: $int_type,
new: $int_type,
success: Ordering,
failure: Ordering,
) -> Result<$int_type, $int_type> {
crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)
#[cfg(portable_atomic_no_stronger_failure_ordering)]
let success = crate::utils::upgrade_success_ordering(success, failure);
self.inner.compare_exchange_weak(current, new, success, failure)
}
}
};
}
atomic_int!(AtomicI64, i64);
atomic_int!(AtomicU64, u64);
});
vendor/portable-atomic/src/imp/atomic128/README.md
Vendored
+39
View File
@@ -0,0 +1,39 @@
# 128-bit atomic implementations on 64-bit architectures
(See the [`atomic64` module](../atomic64) for 64-bit atomic implementations on 32-bit architectures.)
## 128-bit atomic instructions
Here is the table of targets that support 128-bit atomics and the instructions used:
| target_arch | load | store | CAS | RMW | note |
| ----------- | ---- | ----- | --- | --- | ---- |
| x86_64 | cmpxchg16b or vmovdqa | cmpxchg16b or vmovdqa | cmpxchg16b | cmpxchg16b | Requires `cmpxchg16b` target feature (enabled by default on Apple, Windows (except Windows 7, since Rust 1.78), and Fuchsia (since Rust 1.87) targets). vmovdqa requires `avx` target feature. <br> Both compile-time and run-time detection are supported. <br> Requires Rust 1.59+ |
| aarch64/arm64ec | ldxp/stxp or casp or ldp/ldiapp | ldxp/stxp or casp or stp/stilp/swpp | ldxp/stxp or casp | ldxp/stxp or casp/swpp/ldclrp/ldsetp | casp requires `lse` target feature, ldp/stp requires `lse2` target feature, ldiapp/stilp requires `lse2` and `rcpc3` target features, swpp/ldclrp/ldsetp requires `lse128` target feature. <br> Both compile-time and run-time detection are supported. <br> Requires Rust 1.59+ (aarch64) / 1.84+ (arm64ec) |
| riscv64 | amocas.q | amocas.q | amocas.q | amocas.q | Requires `zacas` target feature. Both compile-time and run-time detection are supported. <br> Requires Rust 1.59+ |
| powerpc64 | lq | stq | lqarx/stqcx. | lqarx/stqcx. | Requires `quadword-atomics` target feature (enabled by default on powerpc64le). Both compile-time and run-time detection are supported. <br> Requires Rust 1.95+ |
| s390x | lpq | stpq | cdsg | cdsg | Requires Rust 1.84+ |
| loongarch64 | sc.q | sc.q | sc.q | sc.q | Unimplemented. Requires `scq` target feature. |
| mips64r6 | lldp | lldp/scdp | lldp/scdp | lldp/scdp | Unimplemented (unsupported in LLVM). Requires Release 6 Paired LL/SC family of instructions |
| nvptx64 | ld.b128 | st.b128 | atom.cas.b128 | atom.exch.b128/atom.cas.b128 | Unimplemented. Requires `ptx83` and `sm_90`. |
On compiler versions or platforms where these are not supported, the fallback implementation is used.
See [aarch64.rs](aarch64.rs) module-level comments for more details on the instructions used on AArch64.
## Comparison with core::intrinsics::atomic_\* (core::sync::atomic::Atomic{I,U}128)
This directory has target-specific implementations with inline assembly ([x86_64.rs](x86_64.rs), [aarch64.rs](aarch64.rs), [riscv64.rs](riscv64.rs), [powerpc64.rs](powerpc64.rs), [s390x.rs](s390x.rs)) and an implementation without inline assembly ([intrinsics.rs](intrinsics.rs)). The latter currently always needs nightly compilers and is only used for Miri and ThreadSanitizer, which do not support inline assembly.
Implementations with inline assembly generate assemblies almost equivalent to the `core::intrinsics::atomic_*` (used in `core::sync::atomic::Atomic{I,U}128`) for many operations, but some operations may or may not generate more efficient code. For example:
- On x86_64 and AArch64, implementation with inline assembly contains additional optimizations (e.g., [#16](https://github.com/taiki-e/portable-atomic/pull/16), [#126](https://github.com/taiki-e/portable-atomic/pull/126)) and is much faster for some operations.
- On AArch64, implementation with inline assembly supports outline-atomics on more operating systems, and may be faster in environments where outline-atomics can improve performance.
- On powerpc64, LLVM does not support generating some 128-bit atomic operations (see [intrinsics.rs](intrinsics.rs) module-level comments), and we use CAS loop to implement them, so implementation with inline assembly may be faster for those operations.
- In implementations without inline assembly, the compiler may reuse condition flags that have changed as a result of the operation, or use immediate values instead of registers, depending on the situation.
As 128-bit atomics-related APIs stabilize in the standard library, implementations with inline assembly are planned to be updated to get the benefits of both.
## Run-time CPU feature detection
See the [`detect` module's readme](../detect/README.md) for run-time CPU feature detection.
vendor/portable-atomic/src/imp/atomic128/aarch64.rs
Vendored
+2239
View File
File diff suppressed because it is too large. Load diff
vendor/portable-atomic/src/imp/atomic128/intrinsics.rs
Vendored
+508
View File
@@ -0,0 +1,508 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
128-bit atomic implementation without inline assembly.
Adapted from https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs.
Note: This module is currently only enabled on Miri and ThreadSanitizer which
do not support inline assembly.
This uses `core::arch::x86_64::cmpxchg16b` on x86_64 and
`core::intrinsics::atomic_*` on aarch64, powerpc64, and s390x.
See README.md of this directory for performance comparison with the
implementation with inline assembly.
Note:
- This currently needs Rust 1.70 on x86_64, otherwise nightly compilers.
- On powerpc64, this requires LLVM 15+ and quadword-atomics target feature:
https://github.com/llvm/llvm-project/commit/549e118e93c666914a1045fde38a2cac33e1e445
- On s390x, old LLVM (pre-18) generates libcalls for operations other than load/store/cmpxchg:
https://github.com/llvm/llvm-project/commit/c568927f3e2e7d9804ea74ecbf11c16c014ddcbc
- On aarch64 big-endian, LLVM (as of 17) generates broken code. (wrong result in stress test)
(on cfg(miri)/cfg(sanitize) it may be fine though)
- On powerpc64, LLVM (as of 17) doesn't support 128-bit atomic min/max:
https://github.com/llvm/llvm-project/issues/68390
- On powerpc64le, LLVM (as of 17) generates broken code. (wrong result from fetch_add)
- On riscv64, LLVM does not automatically use 128-bit atomic instructions even if zacas feature is
enabled, because doing it changes the ABI. (If the ability to do that is provided by LLVM in the
future, it should probably be controlled by another ABI feature similar to forced-atomics.)
*/
include!("macros.rs");
#[allow(dead_code)] // we only use compare_exchange.
#[cfg(target_arch = "x86_64")]
#[cfg(not(target_feature = "cmpxchg16b"))]
#[path = "../fallback/outline_atomics.rs"]
mod fallback;
#[cfg(target_arch = "x86_64")]
#[cfg(not(target_feature = "cmpxchg16b"))]
#[path = "../detect/x86_64.rs"]
mod detect;
#[cfg(not(target_arch = "x86_64"))]
use core::intrinsics;
use core::sync::atomic::Ordering::{self, AcqRel, Acquire, Relaxed, Release, SeqCst};
#[cfg(target_arch = "x86_64")]
#[inline]
fn strongest_failure_ordering(order: Ordering) -> Ordering {
match order {
Release | Relaxed => Relaxed,
SeqCst => SeqCst,
Acquire | AcqRel => Acquire,
_ => unreachable!(),
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 {
#[cfg(target_arch = "x86_64")]
// SAFETY: the caller must uphold the safety contract.
unsafe {
let fail_order = strongest_failure_ordering(order);
match atomic_compare_exchange(src, 0, 0, order, fail_order) {
Ok(v) | Err(v) => v,
}
}
#[cfg(not(target_arch = "x86_64"))]
// SAFETY: the caller must uphold the safety contract.
unsafe {
match order {
Acquire => intrinsics::atomic_load_acquire(src),
Relaxed => intrinsics::atomic_load_relaxed(src),
SeqCst => intrinsics::atomic_load_seqcst(src),
_ => unreachable!(),
}
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {
#[cfg(target_arch = "x86_64")]
// SAFETY: the caller must uphold the safety contract.
unsafe {
atomic_swap(dst, val, order);
}
#[cfg(not(target_arch = "x86_64"))]
// SAFETY: the caller must uphold the safety contract.
unsafe {
match order {
Release => intrinsics::atomic_store_release(dst, val),
Relaxed => intrinsics::atomic_store_relaxed(dst, val),
SeqCst => intrinsics::atomic_store_seqcst(dst, val),
_ => unreachable!(),
}
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_compare_exchange(
dst: *mut u128,
old: u128,
new: u128,
success: Ordering,
failure: Ordering,
) -> Result<u128, u128> {
#[cfg(target_arch = "x86_64")]
let (val, ok) = {
#[target_feature(enable = "cmpxchg16b")]
#[cfg_attr(target_feature = "cmpxchg16b", inline)]
#[cfg_attr(not(target_feature = "cmpxchg16b"), inline(never))]
unsafe fn cmpxchg16b(
dst: *mut u128,
old: u128,
new: u128,
success: Ordering,
failure: Ordering,
) -> (u128, bool) {
#[cfg(portable_atomic_no_strict_provenance)]
use crate::utils::ptr::PtrExt as _;
debug_assert!(dst.addr() % 16 == 0);
#[cfg(not(target_feature = "cmpxchg16b"))]
{
debug_assert!(detect::detect().cmpxchg16b());
}
// SAFETY: the caller must guarantee that `dst` is valid for both writes and
// reads, 16-byte aligned (required by CMPXCHG16B), that there are no
// concurrent non-atomic operations, and that the CPU supports CMPXCHG16B.
let prev = unsafe { core::arch::x86_64::cmpxchg16b(dst, old, new, success, failure) };
(prev, prev == old)
}
#[cfg(target_feature = "cmpxchg16b")]
// SAFETY: the caller must guarantee that `dst` is valid for both writes and
// reads, 16-byte aligned, that there are no concurrent non-atomic operations,
// and cfg guarantees that CMPXCHG16B is available at compile-time.
unsafe {
cmpxchg16b(dst, old, new, success, failure)
}
#[cfg(not(target_feature = "cmpxchg16b"))]
// SAFETY: the caller must guarantee that `dst` is valid for both writes and
// reads, 16-byte aligned, and that there are no different kinds of concurrent accesses.
unsafe {
ifunc!(unsafe fn(
dst: *mut u128, old: u128, new: u128, success: Ordering, failure: Ordering
) -> (u128, bool) {
if detect::detect().cmpxchg16b() {
cmpxchg16b
} else {
fallback::atomic_compare_exchange
}
})
}
};
#[cfg(not(target_arch = "x86_64"))]
// SAFETY: the caller must uphold the safety contract.
let (val, ok) = unsafe {
match (success, failure) {
(Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed_relaxed(dst, old, new),
(Relaxed, Acquire) => intrinsics::atomic_cxchg_relaxed_acquire(dst, old, new),
(Relaxed, SeqCst) => intrinsics::atomic_cxchg_relaxed_seqcst(dst, old, new),
(Acquire, Relaxed) => intrinsics::atomic_cxchg_acquire_relaxed(dst, old, new),
(Acquire, Acquire) => intrinsics::atomic_cxchg_acquire_acquire(dst, old, new),
(Acquire, SeqCst) => intrinsics::atomic_cxchg_acquire_seqcst(dst, old, new),
(Release, Relaxed) => intrinsics::atomic_cxchg_release_relaxed(dst, old, new),
(Release, Acquire) => intrinsics::atomic_cxchg_release_acquire(dst, old, new),
(Release, SeqCst) => intrinsics::atomic_cxchg_release_seqcst(dst, old, new),
(AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_relaxed(dst, old, new),
(AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel_acquire(dst, old, new),
(AcqRel, SeqCst) => intrinsics::atomic_cxchg_acqrel_seqcst(dst, old, new),
(SeqCst, Relaxed) => intrinsics::atomic_cxchg_seqcst_relaxed(dst, old, new),
(SeqCst, Acquire) => intrinsics::atomic_cxchg_seqcst_acquire(dst, old, new),
(SeqCst, SeqCst) => intrinsics::atomic_cxchg_seqcst_seqcst(dst, old, new),
_ => unreachable!(),
}
};
if ok { Ok(val) } else { Err(val) }
}
#[cfg(target_arch = "x86_64")]
use self::atomic_compare_exchange as atomic_compare_exchange_weak;
#[cfg(not(target_arch = "x86_64"))]
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_compare_exchange_weak(
dst: *mut u128,
old: u128,
new: u128,
success: Ordering,
failure: Ordering,
) -> Result<u128, u128> {
// SAFETY: the caller must uphold the safety contract.
let (val, ok) = unsafe {
match (success, failure) {
(Relaxed, Relaxed) => intrinsics::atomic_cxchgweak_relaxed_relaxed(dst, old, new),
(Relaxed, Acquire) => intrinsics::atomic_cxchgweak_relaxed_acquire(dst, old, new),
(Relaxed, SeqCst) => intrinsics::atomic_cxchgweak_relaxed_seqcst(dst, old, new),
(Acquire, Relaxed) => intrinsics::atomic_cxchgweak_acquire_relaxed(dst, old, new),
(Acquire, Acquire) => intrinsics::atomic_cxchgweak_acquire_acquire(dst, old, new),
(Acquire, SeqCst) => intrinsics::atomic_cxchgweak_acquire_seqcst(dst, old, new),
(Release, Relaxed) => intrinsics::atomic_cxchgweak_release_relaxed(dst, old, new),
(Release, Acquire) => intrinsics::atomic_cxchgweak_release_acquire(dst, old, new),
(Release, SeqCst) => intrinsics::atomic_cxchgweak_release_seqcst(dst, old, new),
(AcqRel, Relaxed) => intrinsics::atomic_cxchgweak_acqrel_relaxed(dst, old, new),
(AcqRel, Acquire) => intrinsics::atomic_cxchgweak_acqrel_acquire(dst, old, new),
(AcqRel, SeqCst) => intrinsics::atomic_cxchgweak_acqrel_seqcst(dst, old, new),
(SeqCst, Relaxed) => intrinsics::atomic_cxchgweak_seqcst_relaxed(dst, old, new),
(SeqCst, Acquire) => intrinsics::atomic_cxchgweak_seqcst_acquire(dst, old, new),
(SeqCst, SeqCst) => intrinsics::atomic_cxchgweak_seqcst_seqcst(dst, old, new),
_ => unreachable!(),
}
};
if ok { Ok(val) } else { Err(val) }
}
#[inline(always)]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_update<F>(dst: *mut u128, order: Ordering, mut f: F) -> u128
where
F: FnMut(u128) -> u128,
{
// SAFETY: the caller must uphold the safety contract.
unsafe {
// This is a private function and all instances of `f` only operate on the value
// loaded, so there is no need to synchronize the first load/failed CAS.
let mut prev = atomic_load(dst, Ordering::Relaxed);
loop {
let next = f(prev);
match atomic_compare_exchange_weak(dst, prev, next, order, Ordering::Relaxed) {
Ok(x) => return x,
Err(x) => prev = x,
}
}
}
}
// On x86_64, we use core::arch::x86_64::cmpxchg16b instead of core::intrinsics.
// - On s390x, old LLVM (pre-18) generates libcalls for operations other than load/store/cmpxchg (see also module-level comment).
#[cfg(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18)))]
atomic_rmw_by_atomic_update!();
// On powerpc64, LLVM doesn't support 128-bit atomic min/max (see also module-level comment).
#[cfg(target_arch = "powerpc64")]
atomic_rmw_by_atomic_update!(cmp);
#[cfg(not(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18))))]
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_swap(dst: *mut u128, val: u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe {
match order {
Acquire => intrinsics::atomic_xchg_acquire(dst, val),
Release => intrinsics::atomic_xchg_release(dst, val),
AcqRel => intrinsics::atomic_xchg_acqrel(dst, val),
Relaxed => intrinsics::atomic_xchg_relaxed(dst, val),
SeqCst => intrinsics::atomic_xchg_seqcst(dst, val),
_ => unreachable!(),
}
}
}
#[cfg(not(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18))))]
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_add(dst: *mut u128, val: u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe {
match order {
Acquire => intrinsics::atomic_xadd_acquire(dst, val),
Release => intrinsics::atomic_xadd_release(dst, val),
AcqRel => intrinsics::atomic_xadd_acqrel(dst, val),
Relaxed => intrinsics::atomic_xadd_relaxed(dst, val),
SeqCst => intrinsics::atomic_xadd_seqcst(dst, val),
_ => unreachable!(),
}
}
}
#[cfg(not(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18))))]
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_sub(dst: *mut u128, val: u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe {
match order {
Acquire => intrinsics::atomic_xsub_acquire(dst, val),
Release => intrinsics::atomic_xsub_release(dst, val),
AcqRel => intrinsics::atomic_xsub_acqrel(dst, val),
Relaxed => intrinsics::atomic_xsub_relaxed(dst, val),
SeqCst => intrinsics::atomic_xsub_seqcst(dst, val),
_ => unreachable!(),
}
}
}
#[cfg(not(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18))))]
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_and(dst: *mut u128, val: u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe {
match order {
Acquire => intrinsics::atomic_and_acquire(dst, val),
Release => intrinsics::atomic_and_release(dst, val),
AcqRel => intrinsics::atomic_and_acqrel(dst, val),
Relaxed => intrinsics::atomic_and_relaxed(dst, val),
SeqCst => intrinsics::atomic_and_seqcst(dst, val),
_ => unreachable!(),
}
}
}
#[cfg(not(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18))))]
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_nand(dst: *mut u128, val: u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe {
match order {
Acquire => intrinsics::atomic_nand_acquire(dst, val),
Release => intrinsics::atomic_nand_release(dst, val),
AcqRel => intrinsics::atomic_nand_acqrel(dst, val),
Relaxed => intrinsics::atomic_nand_relaxed(dst, val),
SeqCst => intrinsics::atomic_nand_seqcst(dst, val),
_ => unreachable!(),
}
}
}
#[cfg(not(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18))))]
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_or(dst: *mut u128, val: u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe {
match order {
Acquire => intrinsics::atomic_or_acquire(dst, val),
Release => intrinsics::atomic_or_release(dst, val),
AcqRel => intrinsics::atomic_or_acqrel(dst, val),
Relaxed => intrinsics::atomic_or_relaxed(dst, val),
SeqCst => intrinsics::atomic_or_seqcst(dst, val),
_ => unreachable!(),
}
}
}
#[cfg(not(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18))))]
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_xor(dst: *mut u128, val: u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe {
match order {
Acquire => intrinsics::atomic_xor_acquire(dst, val),
Release => intrinsics::atomic_xor_release(dst, val),
AcqRel => intrinsics::atomic_xor_acqrel(dst, val),
Relaxed => intrinsics::atomic_xor_relaxed(dst, val),
SeqCst => intrinsics::atomic_xor_seqcst(dst, val),
_ => unreachable!(),
}
}
}
#[cfg(not(any(
target_arch = "x86_64",
target_arch = "powerpc64",
all(target_arch = "s390x", portable_atomic_pre_llvm_18),
)))]
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_max(dst: *mut u128, val: u128, order: Ordering) -> i128 {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: the caller must uphold the safety contract.
unsafe {
match order {
Acquire => intrinsics::atomic_max_acquire(dst.cast::<i128>(), val as i128),
Release => intrinsics::atomic_max_release(dst.cast::<i128>(), val as i128),
AcqRel => intrinsics::atomic_max_acqrel(dst.cast::<i128>(), val as i128),
Relaxed => intrinsics::atomic_max_relaxed(dst.cast::<i128>(), val as i128),
SeqCst => intrinsics::atomic_max_seqcst(dst.cast::<i128>(), val as i128),
_ => unreachable!(),
}
}
}
#[cfg(not(any(
target_arch = "x86_64",
target_arch = "powerpc64",
all(target_arch = "s390x", portable_atomic_pre_llvm_18),
)))]
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_min(dst: *mut u128, val: u128, order: Ordering) -> i128 {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: the caller must uphold the safety contract.
unsafe {
match order {
Acquire => intrinsics::atomic_min_acquire(dst.cast::<i128>(), val as i128),
Release => intrinsics::atomic_min_release(dst.cast::<i128>(), val as i128),
AcqRel => intrinsics::atomic_min_acqrel(dst.cast::<i128>(), val as i128),
Relaxed => intrinsics::atomic_min_relaxed(dst.cast::<i128>(), val as i128),
SeqCst => intrinsics::atomic_min_seqcst(dst.cast::<i128>(), val as i128),
_ => unreachable!(),
}
}
}
#[cfg(not(any(
target_arch = "x86_64",
target_arch = "powerpc64",
all(target_arch = "s390x", portable_atomic_pre_llvm_18),
)))]
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_umax(dst: *mut u128, val: u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe {
match order {
Acquire => intrinsics::atomic_umax_acquire(dst, val),
Release => intrinsics::atomic_umax_release(dst, val),
AcqRel => intrinsics::atomic_umax_acqrel(dst, val),
Relaxed => intrinsics::atomic_umax_relaxed(dst, val),
SeqCst => intrinsics::atomic_umax_seqcst(dst, val),
_ => unreachable!(),
}
}
}
#[cfg(not(any(
target_arch = "x86_64",
target_arch = "powerpc64",
all(target_arch = "s390x", portable_atomic_pre_llvm_18),
)))]
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_umin(dst: *mut u128, val: u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe {
match order {
Acquire => intrinsics::atomic_umin_acquire(dst, val),
Release => intrinsics::atomic_umin_release(dst, val),
AcqRel => intrinsics::atomic_umin_acqrel(dst, val),
Relaxed => intrinsics::atomic_umin_relaxed(dst, val),
SeqCst => intrinsics::atomic_umin_seqcst(dst, val),
_ => unreachable!(),
}
}
}
#[cfg(not(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18))))]
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_not(dst: *mut u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe { atomic_xor(dst, !0, order) }
}
#[cfg(not(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18))))]
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_neg(dst: *mut u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe { atomic_update(dst, order, u128::wrapping_neg) }
}
#[cfg(not(target_arch = "x86_64"))]
#[inline]
const fn is_lock_free() -> bool {
IS_ALWAYS_LOCK_FREE
}
#[cfg(not(target_arch = "x86_64"))]
const IS_ALWAYS_LOCK_FREE: bool = true;
#[cfg(target_arch = "x86_64")]
#[inline]
fn is_lock_free() -> bool {
#[cfg(target_feature = "cmpxchg16b")]
{
// CMPXCHG16B is available at compile-time.
true
}
#[cfg(not(target_feature = "cmpxchg16b"))]
{
detect::detect().cmpxchg16b()
}
}
#[cfg(target_arch = "x86_64")]
const IS_ALWAYS_LOCK_FREE: bool = cfg!(target_feature = "cmpxchg16b");
atomic128!(AtomicI128, i128, atomic_max, atomic_min);
atomic128!(AtomicU128, u128, atomic_umax, atomic_umin);
#[cfg(test)]
mod tests {
use super::*;
test_atomic_int!(i128);
test_atomic_int!(u128);
// load/store/swap implementation is not affected by signedness, so it is
// enough to test only unsigned types.
stress_test!(u128);
}
vendor/portable-atomic/src/imp/atomic128/macros.rs
Vendored
+349
View File
@@ -0,0 +1,349 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
macro_rules! atomic128 {
($atomic_type:ident, $int_type:ident, $atomic_max:ident, $atomic_min:ident) => {
#[repr(C, align(16))]
pub(crate) struct $atomic_type {
v: core::cell::UnsafeCell<$int_type>,
}
// Send is implicitly implemented.
// SAFETY: any data races are prevented by atomic intrinsics.
unsafe impl Sync for $atomic_type {}
impl_default_no_fetch_ops!($atomic_type, $int_type);
impl_default_bit_opts!($atomic_type, $int_type);
impl $atomic_type {
#[inline]
pub(crate) const fn new(v: $int_type) -> Self {
Self { v: core::cell::UnsafeCell::new(v) }
}
#[inline]
pub(crate) fn is_lock_free() -> bool {
is_lock_free()
}
pub(crate) const IS_ALWAYS_LOCK_FREE: bool = IS_ALWAYS_LOCK_FREE;
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn load(&self, order: Ordering) -> $int_type {
crate::utils::assert_load_ordering(order);
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
atomic_load(self.v.get().cast::<u128>(), order) as $int_type
}
}
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn store(&self, val: $int_type, order: Ordering) {
crate::utils::assert_store_ordering(order);
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
atomic_store(self.v.get().cast::<u128>(), val as u128, order)
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn swap(&self, val: $int_type, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
atomic_swap(self.v.get().cast::<u128>(), val as u128, order) as $int_type
}
}
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn compare_exchange(
&self,
current: $int_type,
new: $int_type,
success: Ordering,
failure: Ordering,
) -> Result<$int_type, $int_type> {
crate::utils::assert_compare_exchange_ordering(success, failure);
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
match atomic_compare_exchange(
self.v.get().cast::<u128>(),
current as u128,
new as u128,
success,
failure,
) {
Ok(v) => Ok(v as $int_type),
Err(v) => Err(v as $int_type),
}
}
}
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn compare_exchange_weak(
&self,
current: $int_type,
new: $int_type,
success: Ordering,
failure: Ordering,
) -> Result<$int_type, $int_type> {
crate::utils::assert_compare_exchange_ordering(success, failure);
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
match atomic_compare_exchange_weak(
self.v.get().cast::<u128>(),
current as u128,
new as u128,
success,
failure,
) {
Ok(v) => Ok(v as $int_type),
Err(v) => Err(v as $int_type),
}
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
atomic_add(self.v.get().cast::<u128>(), val as u128, order) as $int_type
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
atomic_sub(self.v.get().cast::<u128>(), val as u128, order) as $int_type
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
atomic_and(self.v.get().cast::<u128>(), val as u128, order) as $int_type
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_nand(&self, val: $int_type, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
atomic_nand(self.v.get().cast::<u128>(), val as u128, order) as $int_type
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
atomic_or(self.v.get().cast::<u128>(), val as u128, order) as $int_type
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
atomic_xor(self.v.get().cast::<u128>(), val as u128, order) as $int_type
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
$atomic_max(self.v.get().cast::<u128>(), val as u128, order) as $int_type
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
$atomic_min(self.v.get().cast::<u128>(), val as u128, order) as $int_type
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_not(&self, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
atomic_not(self.v.get().cast::<u128>(), order) as $int_type
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn not(&self, order: Ordering) {
self.fetch_not(order);
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_neg(&self, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
atomic_neg(self.v.get().cast::<u128>(), order) as $int_type
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn neg(&self, order: Ordering) {
self.fetch_neg(order);
}
#[inline]
pub(crate) const fn as_ptr(&self) -> *mut $int_type {
self.v.get()
}
}
};
}
#[cfg(any(target_arch = "powerpc64", target_arch = "s390x", target_arch = "x86_64"))]
#[allow(unused_macros)] // also used by intrinsics.rs
macro_rules! atomic_rmw_by_atomic_update {
() => {
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_swap(dst: *mut u128, val: u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe { atomic_update(dst, order, |_| val) }
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_add(dst: *mut u128, val: u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe { atomic_update(dst, order, |x| x.wrapping_add(val)) }
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_sub(dst: *mut u128, val: u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe { atomic_update(dst, order, |x| x.wrapping_sub(val)) }
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_and(dst: *mut u128, val: u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe { atomic_update(dst, order, |x| x & val) }
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_nand(dst: *mut u128, val: u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe { atomic_update(dst, order, |x| !(x & val)) }
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_or(dst: *mut u128, val: u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe { atomic_update(dst, order, |x| x | val) }
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_xor(dst: *mut u128, val: u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe { atomic_update(dst, order, |x| x ^ val) }
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_not(dst: *mut u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe { atomic_update(dst, order, |x| !x) }
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_neg(dst: *mut u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe { atomic_update(dst, order, u128::wrapping_neg) }
}
atomic_rmw_by_atomic_update!(cmp);
};
(cmp) => {
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_max(dst: *mut u128, val: u128, order: Ordering) -> u128 {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: the caller must uphold the safety contract.
unsafe {
atomic_update(dst, order, |x| core::cmp::max(x as i128, val as i128) as u128)
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_umax(dst: *mut u128, val: u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe { atomic_update(dst, order, |x| core::cmp::max(x, val)) }
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_min(dst: *mut u128, val: u128, order: Ordering) -> u128 {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: the caller must uphold the safety contract.
unsafe {
atomic_update(dst, order, |x| core::cmp::min(x as i128, val as i128) as u128)
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_umin(dst: *mut u128, val: u128, order: Ordering) -> u128 {
// SAFETY: the caller must uphold the safety contract.
unsafe { atomic_update(dst, order, |x| core::cmp::min(x, val)) }
}
};
}
vendor/portable-atomic/src/imp/atomic128/mod.rs
Vendored
+131
View File
@@ -0,0 +1,131 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
128-bit atomic implementations on 64-bit architectures
See README.md for details.
*/
// AArch64
#[cfg(any(
all(
target_arch = "aarch64",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
),
all(
target_arch = "arm64ec",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
not(portable_atomic_no_asm),
),
))]
// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.
#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = "intrinsics.rs")]
pub(super) mod aarch64;
// powerpc64
#[cfg(all(
target_arch = "powerpc64",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
not(portable_atomic_no_asm),
any(
target_feature = "quadword-atomics",
portable_atomic_target_feature = "quadword-atomics",
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
any(
all(
target_os = "linux",
any(
all(
target_env = "gnu",
any(target_endian = "little", not(target_feature = "crt-static")),
),
all(
target_env = "musl",
any(not(target_feature = "crt-static"), feature = "std"),
),
target_env = "ohos",
all(target_env = "uclibc", not(target_feature = "crt-static")),
portable_atomic_outline_atomics,
),
),
target_os = "android",
all(
target_os = "freebsd",
any(
target_endian = "little",
not(target_feature = "crt-static"),
portable_atomic_outline_atomics,
),
),
target_os = "openbsd",
all(
target_os = "aix",
not(portable_atomic_pre_llvm_20),
any(test, portable_atomic_outline_atomics), // TODO(aix): currently disabled by default
),
),
not(any(miri, portable_atomic_sanitize_thread)),
),
),
))]
// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.
#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = "intrinsics.rs")]
pub(super) mod powerpc64;
// riscv64
#[cfg(all(
target_arch = "riscv64",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(
target_feature = "zacas",
portable_atomic_target_feature = "zacas",
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
any(target_os = "linux", target_os = "android"),
),
),
))]
pub(super) mod riscv64;
// s390x
#[cfg(all(
target_arch = "s390x",
not(all(any(miri, portable_atomic_sanitize_thread), not(portable_atomic_atomic_intrinsics))),
not(portable_atomic_no_asm),
))]
// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.
#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = "intrinsics.rs")]
pub(super) mod s390x;
// x86_64
#[cfg(all(
target_arch = "x86_64",
not(all(any(miri, portable_atomic_sanitize_thread), portable_atomic_no_cmpxchg16b_intrinsic)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(
target_feature = "cmpxchg16b",
portable_atomic_target_feature = "cmpxchg16b",
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
not(any(target_env = "sgx", miri)),
),
),
))]
// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.
#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = "intrinsics.rs")]
pub(super) mod x86_64;
vendor/portable-atomic/src/imp/atomic128/powerpc64.rs
Vendored
+1046
View File
File diff suppressed because it is too large. Load diff
vendor/portable-atomic/src/imp/atomic128/riscv64.rs
Vendored
+624
View File
@@ -0,0 +1,624 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
128-bit atomic implementation on riscv64.
This architecture provides the following 128-bit atomic instructions:
- amocas.q: CAS (Zacas extension)
See "Atomic operation overview by architecture" in atomic-maybe-uninit for a more comprehensive and
detailed description of the atomic and synchronize instructions in this architecture:
https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#risc-v
Note: On Miri and ThreadSanitizer which do not support inline assembly, we don't use
this module and use fallback implementation instead.
Refs:
- RISC-V Instruction Set Manual
"Zacas" Extension for Atomic Compare-and-Swap (CAS) Instructions
https://docs.riscv.org/reference/isa/v20260120/unpriv/zacas.html
- RISC-V Atomics ABI Specification
https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/draft-20250812-301374e92976e298e676e7129a6212926b2299ce/riscv-atomic.adoc
See tests/asm-test/asm/portable-atomic for generated assembly.
*/
include!("macros.rs");
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
#[path = "../fallback/outline_atomics.rs"]
mod fallback;
#[cfg(not(portable_atomic_no_outline_atomics))]
#[cfg(any(test, not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))))]
#[cfg(any(target_os = "linux", target_os = "android"))]
#[path = "../detect/riscv_linux.rs"]
mod detect;
#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
use core::sync::atomic::Ordering;
#[cfg(portable_atomic_no_strict_provenance)]
use crate::utils::ptr::PtrExt as _;
use crate::utils::{Pair, U128};
macro_rules! debug_assert_zacas {
() => {
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
{
debug_assert!(detect::detect().zacas());
}
};
}
// `.option arch, +zacas` directive requires LLVM 20, so we use .4byte directive for old LLVM.
// Note that `.insn <value>` directive requires LLVM 19.
// https://github.com/llvm/llvm-project/commit/2a086dce691e3cc34a2fc27f4fb255bb2cbbfac9
// https://github.com/riscv-non-isa/riscv-asm-manual/blob/v0.0.1/src/asm-manual.adoc#arch
#[cfg(not(portable_atomic_pre_llvm_20))]
macro_rules! start_zacas {
() => {
".option push\n.option arch, +zacas"
};
}
#[cfg(not(portable_atomic_pre_llvm_20))]
macro_rules! end_zacas {
() => {
".option pop"
};
}
#[cfg(not(portable_atomic_pre_llvm_20))]
macro_rules! atomic_rmw_amocas_order {
($op:ident, $order:ident) => {
atomic_rmw_amocas_order!($op, $order, failure = $order)
};
($op:ident, $order:ident, failure = $failure:ident) => {
match $order {
Ordering::Relaxed => $op!("", ""),
Ordering::Acquire => $op!("", ".aq"),
Ordering::Release => $op!("", ".rl"),
Ordering::AcqRel => $op!("", ".aqrl"),
Ordering::SeqCst if $failure == Ordering::SeqCst => $op!("fence rw,rw", ".aqrl"),
Ordering::SeqCst => $op!("", ".aqrl"),
_ => unreachable!(),
}
};
}
#[cfg(portable_atomic_pre_llvm_20)]
macro_rules! atomic_rmw_amocas_order_insn {
($op:ident, $order:ident) => {
atomic_rmw_amocas_order_insn!($op, $order, failure = $order)
};
($op:ident, $order:ident, failure = $failure:ident) => {
match $order {
Ordering::Relaxed => $op!("", "8"),
Ordering::Acquire => $op!("", "c"),
Ordering::Release => $op!("", "a"),
Ordering::AcqRel => $op!("", "e"),
Ordering::SeqCst if $failure == Ordering::SeqCst => $op!("fence rw,rw", "e"),
Ordering::SeqCst => $op!("", "e"),
_ => unreachable!(),
}
};
}
// If zacas is available at compile-time, we can always use zacas_fn.
#[cfg(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))]
use self::atomic_load_zacas as atomic_load;
// Otherwise, we need to do run-time detection and can use zacas_fn only if zacas is available.
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
#[inline]
unsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 {
fn_alias! {
// inline(never) is just a hint and also not strictly necessary
// because we use ifunc helper macro, but used for clarity.
#[inline(never)]
unsafe fn(src: *mut u128) -> u128;
atomic_load_zacas_relaxed = atomic_load_zacas(Ordering::Relaxed);
atomic_load_zacas_acquire = atomic_load_zacas(Ordering::Acquire);
atomic_load_zacas_seqcst = atomic_load_zacas(Ordering::SeqCst);
}
// SAFETY: the caller must uphold the safety contract.
// we only calls atomic_load_zacas if zacas is available.
unsafe {
match order {
Ordering::Relaxed => {
ifunc!(unsafe fn(src: *mut u128) -> u128 {
if detect::detect().zacas() {
atomic_load_zacas_relaxed
} else {
fallback::atomic_load_non_seqcst
}
})
}
Ordering::Acquire => {
ifunc!(unsafe fn(src: *mut u128) -> u128 {
if detect::detect().zacas() {
atomic_load_zacas_acquire
} else {
fallback::atomic_load_non_seqcst
}
})
}
Ordering::SeqCst => {
ifunc!(unsafe fn(src: *mut u128) -> u128 {
if detect::detect().zacas() {
atomic_load_zacas_seqcst
} else {
fallback::atomic_load_seqcst
}
})
}
_ => unreachable!(),
}
}
}
#[inline]
unsafe fn atomic_load_zacas(src: *mut u128, order: Ordering) -> u128 {
debug_assert!(src.addr() % 16 == 0);
debug_assert_zacas!();
let (out_lo, out_hi);
// SAFETY: the caller must uphold the safety contract.
unsafe {
#[cfg(not(portable_atomic_pre_llvm_20))]
macro_rules! load {
($fence:tt, $asm_order:tt) => {
asm!(
start_zacas!(),
$fence, // fence
concat!("amocas.q", $asm_order, " a2, a2, 0({src})"), // atomic { if *dst == a2:a3 { *dst = a2:a3 } else { a2:a3 = *dst } }
end_zacas!(),
src = in(reg) ptr_reg!(src),
inout("a2") 0_u64 => out_lo,
inout("a3") 0_u64 => out_hi,
options(nostack, preserves_flags),
)
};
}
#[cfg(not(portable_atomic_pre_llvm_20))]
atomic_rmw_amocas_order!(load, order);
#[cfg(portable_atomic_pre_llvm_20)]
macro_rules! load {
($fence:tt, $insn_order:tt) => {
asm!(
$fence, // fence
// amocas.q{,.aq,.rl,.aqrl} a2, a2, (a0) // atomic { if *a0 == a2:a3 { *a0 = a2:a3 } else { a2:a3 = *a0 } }
concat!(".4byte 0x2", $insn_order, "c5462f"),
in("a0") ptr_reg!(src),
inout("a2") 0_u64 => out_lo,
inout("a3") 0_u64 => out_hi,
options(nostack, preserves_flags),
)
};
}
#[cfg(portable_atomic_pre_llvm_20)]
atomic_rmw_amocas_order_insn!(load, order);
U128 { pair: Pair { lo: out_lo, hi: out_hi } }.whole
}
}
#[inline]
unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {
// SAFETY: the caller must uphold the safety contract.
unsafe {
atomic_swap(dst, val, order);
}
}
#[inline]
unsafe fn atomic_compare_exchange(
dst: *mut u128,
old: u128,
new: u128,
success: Ordering,
failure: Ordering,
) -> Result<u128, u128> {
#[cfg(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))]
// SAFETY: the caller must uphold the safety contract.
// cfg guarantees that zacas instructions are available at compile-time.
let (prev, ok) = unsafe { atomic_compare_exchange_zacas(dst, old, new, success, failure) };
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
let (prev, ok) = {
fn_alias! {
// inline(never) is just a hint and also not strictly necessary
// because we use ifunc helper macro, but used for clarity.
#[inline(never)]
unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool);
zacas_relaxed_fn = atomic_compare_exchange_zacas(Ordering::Relaxed, Ordering::Relaxed);
zacas_acquire_fn = atomic_compare_exchange_zacas(Ordering::Acquire, Ordering::Acquire);
zacas_release_fn = atomic_compare_exchange_zacas(Ordering::Release, Ordering::Relaxed);
zacas_acqrel_fn = atomic_compare_exchange_zacas(Ordering::AcqRel, Ordering::Acquire);
zacas_seqcst_fn = atomic_compare_exchange_zacas(Ordering::SeqCst, Ordering::SeqCst);
}
let order = crate::utils::upgrade_success_ordering(success, failure);
// SAFETY: the caller must uphold the safety contract.
// we only calls atomic_compare_exchange_zacas if zacas is available.
unsafe {
match order {
Ordering::Relaxed => {
ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {
if detect::detect().zacas() {
zacas_relaxed_fn
} else {
fallback::atomic_compare_exchange_non_seqcst
}
})
}
Ordering::Acquire => {
ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {
if detect::detect().zacas() {
zacas_acquire_fn
} else {
fallback::atomic_compare_exchange_non_seqcst
}
})
}
Ordering::Release => {
ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {
if detect::detect().zacas() {
zacas_release_fn
} else {
fallback::atomic_compare_exchange_non_seqcst
}
})
}
Ordering::AcqRel => {
ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {
if detect::detect().zacas() {
zacas_acqrel_fn
} else {
fallback::atomic_compare_exchange_non_seqcst
}
})
}
Ordering::SeqCst => {
ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {
if detect::detect().zacas() {
zacas_seqcst_fn
} else {
fallback::atomic_compare_exchange_seqcst
}
})
}
_ => unreachable!(),
}
}
};
if ok { Ok(prev) } else { Err(prev) }
}
#[inline]
unsafe fn atomic_compare_exchange_zacas(
dst: *mut u128,
old: u128,
new: u128,
success: Ordering,
failure: Ordering,
) -> (u128, bool) {
debug_assert!(dst.addr() % 16 == 0);
debug_assert_zacas!();
let order = crate::utils::upgrade_success_ordering(success, failure);
let old = U128 { whole: old };
let new = U128 { whole: new };
let (prev_lo, prev_hi);
// SAFETY: the caller must uphold the safety contract.
unsafe {
#[cfg(not(portable_atomic_pre_llvm_20))]
macro_rules! cmpxchg {
($fence:tt, $asm_order:tt) => {
asm!(
start_zacas!(),
$fence, // fence
concat!("amocas.q", $asm_order, " a4, a2, 0({dst})"), // atomic { if *dst == a4:a5 { *dst = a2:a3 } else { a4:a5 = *dst } }
end_zacas!(),
dst = in(reg) ptr_reg!(dst),
// must be allocated to even/odd register pair
inout("a4") old.pair.lo => prev_lo,
inout("a5") old.pair.hi => prev_hi,
// must be allocated to even/odd register pair
in("a2") new.pair.lo,
in("a3") new.pair.hi,
options(nostack, preserves_flags),
)
};
}
#[cfg(not(portable_atomic_pre_llvm_20))]
atomic_rmw_amocas_order!(cmpxchg, order, failure = failure);
#[cfg(portable_atomic_pre_llvm_20)]
macro_rules! cmpxchg {
($fence:tt, $insn_order:tt) => {
asm!(
$fence, // fence
// amocas.q{,.aq,.rl,.aqrl} a4, a2, (a0) // atomic { if *a0 == a4:a5 { *a0 = a2:a3 } else { a4:a5 = *a0 } }
concat!(".4byte 0x2", $insn_order, "c5472f"),
in("a0") ptr_reg!(dst),
// must be allocated to even/odd register pair
inout("a4") old.pair.lo => prev_lo,
inout("a5") old.pair.hi => prev_hi,
// must be allocated to even/odd register pair
in("a2") new.pair.lo,
in("a3") new.pair.hi,
options(nostack, preserves_flags),
)
};
}
#[cfg(portable_atomic_pre_llvm_20)]
atomic_rmw_amocas_order_insn!(cmpxchg, order, failure = failure);
let prev = U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole;
(prev, prev == old.whole)
}
}
// amocas is always strong.
use self::atomic_compare_exchange as atomic_compare_exchange_weak;
// 128-bit atomic load by two 64-bit atomic loads.
#[inline]
unsafe fn byte_wise_atomic_load(src: *const u128) -> u128 {
let (out_lo, out_hi);
// SAFETY: the caller must uphold the safety contract.
unsafe {
asm!(
"ld {out_lo}, ({src})", // atomic { out_lo = *src }
"ld {out_hi}, 8({src})", // atomic { out_hi = *src.byte_add(8) }
src = in(reg) ptr_reg!(src),
out_lo = out(reg) out_lo,
out_hi = out(reg) out_hi,
options(pure, nostack, preserves_flags, readonly),
);
U128 { pair: Pair { lo: out_lo, hi: out_hi } }.whole
}
}
macro_rules! select_atomic_rmw {
(
unsafe fn $name:ident(dst: *mut u128 $(, $($arg:tt)*)?) $(-> $ret_ty:ty)? {
|$zacas_fn_binding:ident| $($zacas_fn_body:tt)*
}
zacas = $zacas_fn:ident;
non_seqcst_fallback = $non_seqcst_fallback_fn:ident;
seqcst_fallback = $seqcst_fallback_fn:ident;
) => {
#[inline]
unsafe fn $zacas_fn(dst: *mut u128 $(, $($arg)*)?, order: Ordering) $(-> $ret_ty)? {
// SAFETY: the caller must uphold the safety contract.
unsafe {
// This is not single-copy atomic reads, but this is ok because subsequent
// CAS will check for consistency.
//
// Note that the C++20 memory model does not allow mixed-sized atomic access,
// so we must use inline assembly to implement byte_wise_atomic_load.
// (i.e., byte-wise atomic based on the standard library's atomic types
// cannot be used here).
let mut prev = byte_wise_atomic_load(dst);
loop {
let next = {
let $zacas_fn_binding = prev;
$($zacas_fn_body)*
};
match atomic_compare_exchange_weak(dst, prev, next, order, Ordering::Relaxed) {
Ok(x) => return x,
Err(x) => prev = x,
}
}
}
}
// If zacas is available at compile-time, we can always use zacas_fn.
#[cfg(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))]
use self::$zacas_fn as $name;
// Otherwise, we need to do run-time detection and can use zacas_fn only if zacas is available.
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
#[inline]
unsafe fn $name(dst: *mut u128 $(, $($arg)*)?, order: Ordering) $(-> $ret_ty)? {
fn_alias! {
// inline(never) is just a hint and also not strictly necessary
// because we use ifunc helper macro, but used for clarity.
#[inline(never)]
unsafe fn(dst: *mut u128 $(, $($arg)*)?) $(-> $ret_ty)?;
zacas_relaxed_fn = $zacas_fn(Ordering::Relaxed);
zacas_acquire_fn = $zacas_fn(Ordering::Acquire);
zacas_release_fn = $zacas_fn(Ordering::Release);
zacas_acqrel_fn = $zacas_fn(Ordering::AcqRel);
zacas_seqcst_fn = $zacas_fn(Ordering::SeqCst);
}
// SAFETY: the caller must uphold the safety contract.
// we only calls zacas_fn if zacas is available.
unsafe {
match order {
Ordering::Relaxed => {
ifunc!(unsafe fn(dst: *mut u128 $(, $($arg)*)?) $(-> $ret_ty)? {
if detect::detect().zacas() {
zacas_relaxed_fn
} else {
fallback::$non_seqcst_fallback_fn
}
})
}
Ordering::Acquire => {
ifunc!(unsafe fn(dst: *mut u128 $(, $($arg)*)?) $(-> $ret_ty)? {
if detect::detect().zacas() {
zacas_acquire_fn
} else {
fallback::$non_seqcst_fallback_fn
}
})
}
Ordering::Release => {
ifunc!(unsafe fn(dst: *mut u128 $(, $($arg)*)?) $(-> $ret_ty)? {
if detect::detect().zacas() {
zacas_release_fn
} else {
fallback::$non_seqcst_fallback_fn
}
})
}
Ordering::AcqRel => {
ifunc!(unsafe fn(dst: *mut u128 $(, $($arg)*)?) $(-> $ret_ty)? {
if detect::detect().zacas() {
zacas_acqrel_fn
} else {
fallback::$non_seqcst_fallback_fn
}
})
}
Ordering::SeqCst => {
ifunc!(unsafe fn(dst: *mut u128 $(, $($arg)*)?) $(-> $ret_ty)? {
if detect::detect().zacas() {
zacas_seqcst_fn
} else {
fallback::$seqcst_fallback_fn
}
})
}
_ => unreachable!(),
}
}
}
};
}
select_atomic_rmw! {
unsafe fn atomic_swap(dst: *mut u128, val: u128) -> u128 {
|_x| val
}
zacas = atomic_swap_zacas;
non_seqcst_fallback = atomic_swap_non_seqcst;
seqcst_fallback = atomic_swap_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_add(dst: *mut u128, val: u128) -> u128 {
|x| x.wrapping_add(val)
}
zacas = atomic_add_zacas;
non_seqcst_fallback = atomic_add_non_seqcst;
seqcst_fallback = atomic_add_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_sub(dst: *mut u128, val: u128) -> u128 {
|x| x.wrapping_sub(val)
}
zacas = atomic_sub_zacas;
non_seqcst_fallback = atomic_sub_non_seqcst;
seqcst_fallback = atomic_sub_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_and(dst: *mut u128, val: u128) -> u128 {
|x| x & val
}
zacas = atomic_and_zacas;
non_seqcst_fallback = atomic_and_non_seqcst;
seqcst_fallback = atomic_and_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_nand(dst: *mut u128, val: u128) -> u128 {
|x| !(x & val)
}
zacas = atomic_nand_zacas;
non_seqcst_fallback = atomic_nand_non_seqcst;
seqcst_fallback = atomic_nand_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_or(dst: *mut u128, val: u128) -> u128 {
|x| x | val
}
zacas = atomic_or_zacas;
non_seqcst_fallback = atomic_or_non_seqcst;
seqcst_fallback = atomic_or_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_xor(dst: *mut u128, val: u128) -> u128 {
|x| x ^ val
}
zacas = atomic_xor_zacas;
non_seqcst_fallback = atomic_xor_non_seqcst;
seqcst_fallback = atomic_xor_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_max(dst: *mut u128, val: u128) -> u128 {
|x| {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
{ core::cmp::max(x as i128, val as i128) as u128 }
}
}
zacas = atomic_max_zacas;
non_seqcst_fallback = atomic_max_non_seqcst;
seqcst_fallback = atomic_max_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_umax(dst: *mut u128, val: u128) -> u128 {
|x| core::cmp::max(x, val)
}
zacas = atomic_umax_zacas;
non_seqcst_fallback = atomic_umax_non_seqcst;
seqcst_fallback = atomic_umax_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_min(dst: *mut u128, val: u128) -> u128 {
|x| {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
{ core::cmp::min(x as i128, val as i128) as u128 }
}
}
zacas = atomic_min_zacas;
non_seqcst_fallback = atomic_min_non_seqcst;
seqcst_fallback = atomic_min_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_umin(dst: *mut u128, val: u128) -> u128 {
|x| core::cmp::min(x, val)
}
zacas = atomic_umin_zacas;
non_seqcst_fallback = atomic_umin_non_seqcst;
seqcst_fallback = atomic_umin_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_not(dst: *mut u128) -> u128 {
|x| !x
}
zacas = atomic_not_zacas;
non_seqcst_fallback = atomic_not_non_seqcst;
seqcst_fallback = atomic_not_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_neg(dst: *mut u128) -> u128 {
|x| x.wrapping_neg()
}
zacas = atomic_neg_zacas;
non_seqcst_fallback = atomic_neg_non_seqcst;
seqcst_fallback = atomic_neg_seqcst;
}
#[inline]
fn is_lock_free() -> bool {
#[cfg(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))]
{
// zacas is available at compile-time.
true
}
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
{
detect::detect().zacas()
}
}
const IS_ALWAYS_LOCK_FREE: bool =
cfg!(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"));
atomic128!(AtomicI128, i128, atomic_max, atomic_min);
atomic128!(AtomicU128, u128, atomic_umax, atomic_umin);
#[cfg(test)]
mod tests {
use super::*;
test_atomic_int!(i128);
test_atomic_int!(u128);
// load/store/swap implementation is not affected by signedness, so it is
// enough to test only unsigned types.
stress_test!(u128);
}
vendor/portable-atomic/src/imp/atomic128/s390x.rs
Vendored
+548
View File
@@ -0,0 +1,548 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
128-bit atomic implementation on s390x.
This architecture provides the following 128-bit atomic instructions:
- LPQ/STPQ: load/store (arch1 or later, i.e., baseline)
- CDSG: CAS (arch1 or later, i.e., baseline)
See "Atomic operation overview by architecture" in atomic-maybe-uninit for a more comprehensive and
detailed description of the atomic and synchronize instructions in this architecture:
https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#s390x
LLVM's minimal supported architecture level is arch8 (z10):
https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/SystemZ/SystemZProcessors.td#L16-L17
This does not appear to have changed since the current s390x backend was added in LLVM 3.3:
https://github.com/llvm/llvm-project/commit/5f613dfd1f7edb0ae95d521b7107b582d9df5103#diff-cbaef692b3958312e80fd5507a7e2aff071f1acb086f10e8a96bc06a7bb289db
We could use asm byte_wise_atomic_load + asm atomic_compare_exchange pattern for RMW here, but using an
inline assembly allows omitting the comparison of results and the storing/comparing of condition flags.
Note: On Miri and ThreadSanitizer which do not support inline assembly, we don't use
this module and use intrinsics.rs instead.
Refs:
- z/Architecture Principles of Operation, Fifteenth Edition (SA22-7832-14)
https://www.ibm.com/docs/en/module_1678991624569/pdf/SA22-7832-14.pdf
- atomic-maybe-uninit
https://github.com/taiki-e/atomic-maybe-uninit
See tests/asm-test/asm/portable-atomic for generated assembly.
*/
include!("macros.rs");
use core::{arch::asm, sync::atomic::Ordering};
#[cfg(portable_atomic_no_strict_provenance)]
use crate::utils::ptr::PtrExt as _;
use crate::utils::{Pair, U128};
// bcr 14,0 requires fast-BCR-serialization facility added in arch9 (z196).
#[cfg(any(
target_feature = "fast-serialization",
portable_atomic_target_feature = "fast-serialization",
))]
macro_rules! serialization {
() => {
"bcr 14, 0"
};
}
#[cfg(not(any(
target_feature = "fast-serialization",
portable_atomic_target_feature = "fast-serialization",
)))]
macro_rules! serialization {
() => {
"bcr 15, 0"
};
}
// Use distinct operands on z196 or later, otherwise split to lgr and $op.
#[cfg(any(target_feature = "distinct-ops", portable_atomic_target_feature = "distinct-ops"))]
macro_rules! distinct_op {
($op:tt, $a0:tt, $a1:tt, $a2:tt) => {
concat!($op, "k ", $a0, ", ", $a1, ", ", $a2)
};
}
#[cfg(not(any(target_feature = "distinct-ops", portable_atomic_target_feature = "distinct-ops")))]
macro_rules! distinct_op {
($op:tt, $a0:tt, $a1:tt, $a2:tt) => {
concat!("lgr ", $a0, ", ", $a1, "\n", $op, " ", $a0, ", ", $a2)
};
}
// Use selgr$cond on z15 or later, otherwise split to locgr$cond and $op.
#[cfg(any(
target_feature = "miscellaneous-extensions-3",
portable_atomic_target_feature = "miscellaneous-extensions-3",
))]
#[cfg(any(
target_feature = "load-store-on-cond",
portable_atomic_target_feature = "load-store-on-cond",
))]
macro_rules! select_op {
($cond:tt, $a0:tt, $a1:tt, $a2:tt) => {
concat!("selgr", $cond, " ", $a0, ", ", $a1, ", ", $a2)
};
}
#[cfg(not(any(
target_feature = "miscellaneous-extensions-3",
portable_atomic_target_feature = "miscellaneous-extensions-3",
)))]
#[cfg(any(
target_feature = "load-store-on-cond",
portable_atomic_target_feature = "load-store-on-cond",
))]
macro_rules! select_op {
($cond:tt, $a0:tt, $a1:tt, $a2:tt) => {
concat!("lgr ", $a0, ", ", $a2, "\n", "locgr", $cond, " ", $a0, ", ", $a1)
};
}
// Extracts and checks condition code.
#[inline]
fn extract_cc(r: i64) -> bool {
r.wrapping_add(-268435456) & (1 << 31) != 0
}
#[inline]
unsafe fn atomic_load(src: *mut u128, _order: Ordering) -> u128 {
debug_assert!(src.addr() % 16 == 0);
let (out_hi, out_lo);
// SAFETY: the caller must uphold the safety contract.
unsafe {
// atomic load is always SeqCst.
asm!(
"lpq %r0, 0({src})", // atomic { r0:r1 = *src }
src = in(reg) ptr_reg!(src),
// Quadword atomic instructions work with even/odd pair of specified register and subsequent register.
out("r0") out_hi,
out("r1") out_lo,
options(nostack, preserves_flags),
);
U128 { pair: Pair { hi: out_hi, lo: out_lo } }.whole
}
}
#[inline]
unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {
debug_assert!(dst.addr() % 16 == 0);
let val = U128 { whole: val };
// SAFETY: the caller must uphold the safety contract.
unsafe {
macro_rules! atomic_store {
($acquire:expr) => {
asm!(
"stpq %r0, 0({dst})", // atomic { *dst = r0:r1 }
$acquire, // fence
dst = in(reg) ptr_reg!(dst),
// Quadword atomic instructions work with even/odd pair of specified register and subsequent register.
in("r0") val.pair.hi,
in("r1") val.pair.lo,
options(nostack, preserves_flags),
)
};
}
match order {
// Relaxed and Release stores are equivalent.
Ordering::Relaxed | Ordering::Release => atomic_store!(""),
Ordering::SeqCst => atomic_store!(serialization!()),
_ => unreachable!(),
}
}
}
#[inline]
unsafe fn atomic_compare_exchange(
dst: *mut u128,
old: u128,
new: u128,
_success: Ordering,
_failure: Ordering,
) -> Result<u128, u128> {
debug_assert!(dst.addr() % 16 == 0);
let old = U128 { whole: old };
let new = U128 { whole: new };
let (prev_hi, prev_lo);
let r;
// SAFETY: the caller must uphold the safety contract.
// CDSG has SeqCst semantics.
let prev = unsafe {
macro_rules! cmpxchg {
($new_hi:tt, $new_lo:tt) => {
asm!(
concat!("cdsg %r0, %", $new_hi, ", 0({dst})"), // atomic { if *dst == r0:r1 { cc = 0; *dst = new_hi:new_lo } else { cc = 1; r0:r1 = *dst } }
"ipm {r}", // r[:] = cc
dst = in(reg) ptr_reg!(dst),
r = lateout(reg) r,
// Quadword atomic instructions work with even/odd pair of specified register and subsequent register.
inout("r0") old.pair.hi => prev_hi,
inout("r1") old.pair.lo => prev_lo,
in($new_hi) new.pair.hi,
in($new_lo) new.pair.lo,
// Do not use `preserves_flags` because CDSG modifies the condition code.
options(nostack),
)
};
}
// r4 is stack pointer on z/OS.
#[cfg(not(target_os = "zos"))]
cmpxchg!("r4", "r5");
#[cfg(target_os = "zos")]
cmpxchg!("r12", "r13");
U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole
};
if extract_cc(r) { Ok(prev) } else { Err(prev) }
}
// cdsg is always strong.
use self::atomic_compare_exchange as atomic_compare_exchange_weak;
#[inline]
unsafe fn atomic_swap(dst: *mut u128, val: u128, _order: Ordering) -> u128 {
debug_assert!(dst.addr() % 16 == 0);
let val = U128 { whole: val };
let (mut prev_hi, mut prev_lo);
// SAFETY: the caller must uphold the safety contract.
// CDSG has SeqCst semantics.
//
// Do not use atomic_rmw_cas_3 because it needs extra LGR to implement swap.
unsafe {
macro_rules! swap {
($val_hi:tt, $val_lo:tt) => {
asm!(
"lg %r1, 8({dst})", // atomic { r1 = *dst.byte_add(8) }
"lg %r0, 0({dst})", // atomic { r0 = *dst }
"2:", // 'retry:
concat!("cdsg %r0, %", $val_hi, ", 0({dst})"), // atomic { if *dst == r0:r1 { cc = 0; *dst = val_hi:val_lo } else { cc = 1; r0:r1 = *dst } }
"jl 2b", // if cc == 1 { jump 'retry }
dst = in(reg) ptr_reg!(dst),
// Quadword atomic instructions work with even/odd pair of specified register and subsequent register.
out("r0") prev_hi,
out("r1") prev_lo,
in($val_hi) val.pair.hi,
in($val_lo) val.pair.lo,
// Do not use `preserves_flags` because CDSG modifies the condition code.
options(nostack),
)
};
}
// r4 is stack pointer on z/OS.
#[cfg(not(target_os = "zos"))]
swap!("r4", "r5");
#[cfg(target_os = "zos")]
swap!("r12", "r13");
U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole
}
}
/// Atomic RMW by CAS loop (3 arguments)
/// `unsafe fn(dst: *mut u128, val: u128, order: Ordering) -> u128;`
///
/// `$op` can use the following registers:
/// - val_hi/val_lo pair: val argument (read-only for `$op`)
/// - r0/r1 pair: previous value loaded (read-only for `$op`)
/// - r12/r13 pair: new value that will be stored
macro_rules! atomic_rmw_cas_3 {
($name:ident, [$($reg:tt)*], $($op:tt)*) => {
#[inline]
unsafe fn $name(dst: *mut u128, val: u128, _order: Ordering) -> u128 {
debug_assert!(dst.addr() % 16 == 0);
let val = U128 { whole: val };
let (mut prev_hi, mut prev_lo);
// SAFETY: the caller must uphold the safety contract.
// CDSG has SeqCst semantics.
unsafe {
asm!(
"lg %r1, 8({dst})", // atomic { r1 = *dst.byte_add(8) }
"lg %r0, 0({dst})", // atomic { r0 = *dst }
"2:", // 'retry:
$($op)*
"cdsg %r0, %r12, 0({dst})", // atomic { if *dst == r0:r1 { cc = 0; *dst = r12:r13 } else { cc = 1; r0:r1 = *dst } }
"jl 2b", // if cc == 1 { jump 'retry }
dst = in(reg) ptr_reg!(dst),
val_hi = in(reg) val.pair.hi,
val_lo = in(reg) val.pair.lo,
$($reg)*
// Quadword atomic instructions work with even/odd pair of specified register and subsequent register.
out("r0") prev_hi,
out("r1") prev_lo,
out("r12") _,
out("r13") _,
// Do not use `preserves_flags` because CDSG modifies the condition code.
options(nostack),
);
U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole
}
}
};
}
/// Atomic RMW by CAS loop (2 arguments)
/// `unsafe fn(dst: *mut u128, order: Ordering) -> u128;`
///
/// `$op` can use the following registers:
/// - r0/r1 pair: previous value loaded (read-only for `$op`)
/// - r12/r13 pair: new value that will be stored
macro_rules! atomic_rmw_cas_2 {
($name:ident, [$($reg:tt)*], $($op:tt)*) => {
#[inline]
unsafe fn $name(dst: *mut u128, _order: Ordering) -> u128 {
debug_assert!(dst.addr() % 16 == 0);
let (mut prev_hi, mut prev_lo);
// SAFETY: the caller must uphold the safety contract.
// CDSG has SeqCst semantics.
unsafe {
asm!(
"lg %r1, 8({dst})", // atomic { r1 = *dst.byte_add(8) }
"lg %r0, 0({dst})", // atomic { r0 = *dst }
"2:", // 'retry:
$($op)*
"cdsg %r0, %r12, 0({dst})", // atomic { if *dst == r0:r1 { cc = 0; *dst = r12:r13 } else { cc = 1; r0:r1 = *dst } }
"jl 2b", // if cc == 1 { jump 'retry }
dst = in(reg) ptr_reg!(dst),
$($reg)*
// Quadword atomic instructions work with even/odd pair of specified register and subsequent register.
out("r0") prev_hi,
out("r1") prev_lo,
out("r12") _,
out("r13") _,
// Do not use `preserves_flags` because CDSG modifies the condition code.
options(nostack),
);
U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole
}
}
};
}
atomic_rmw_cas_3! {
atomic_add, [],
distinct_op!("algr", "%r13", "%r1", "{val_lo}"), // r13 = r1 + val_lo; cc = zero | carry
"lgr %r12, %r0", // r12 = r0
"alcgr %r12, {val_hi}", // r12 += val_hi + carry
}
atomic_rmw_cas_3! {
atomic_sub, [],
distinct_op!("slgr", "%r13", "%r1", "{val_lo}"), // r13 = r1 - val_lo; cc = zero | borrow
"lgr %r12, %r0", // r12 = r0
"slbgr %r12, {val_hi}", // r12 -= val_hi + borrow
}
atomic_rmw_cas_3! {
atomic_and, [],
distinct_op!("ngr", "%r13", "%r1", "{val_lo}"), // r13 = r1 & val_lo
distinct_op!("ngr", "%r12", "%r0", "{val_hi}"), // r12 = r0 & val_hi
}
// Use nngrk on z15 or later.
#[cfg(any(
target_feature = "miscellaneous-extensions-3",
portable_atomic_target_feature = "miscellaneous-extensions-3",
))]
atomic_rmw_cas_3! {
atomic_nand, [],
"nngrk %r13, %r1, {val_lo}", // r13 = !(r1 & val_lo)
"nngrk %r12, %r0, {val_hi}", // r12 = !(r0 & val_hi)
}
#[cfg(not(any(
target_feature = "miscellaneous-extensions-3",
portable_atomic_target_feature = "miscellaneous-extensions-3",
)))]
atomic_rmw_cas_3! {
atomic_nand, [],
distinct_op!("ngr", "%r13", "%r1", "{val_lo}"), // r13 = r1 & val_lo
distinct_op!("ngr", "%r12", "%r0", "{val_hi}"), // r12 = r0 & val_hi
"lcgr %r13, %r13", // r13 = !r13 + 1
"aghi %r13, -1", // r13 -= 1
"lcgr %r12, %r12", // r12 = !r12 + 1
"aghi %r12, -1", // r12 -= 1
}
atomic_rmw_cas_3! {
atomic_or, [],
distinct_op!("ogr", "%r13", "%r1", "{val_lo}"), // r13 = r1 | val_lo
distinct_op!("ogr", "%r12", "%r0", "{val_hi}"), // r12 = r0 | val_hi
}
atomic_rmw_cas_3! {
atomic_xor, [],
distinct_op!("xgr", "%r13", "%r1", "{val_lo}"), // r13 = r1 ^ val_lo
distinct_op!("xgr", "%r12", "%r0", "{val_hi}"), // r12 = r0 ^ val_hi
}
cfg_sel!({
#[cfg(any(
target_feature = "load-store-on-cond",
portable_atomic_target_feature = "load-store-on-cond",
))]
{
atomic_rmw_cas_3! {
atomic_max, [],
"clgr %r1, {val_lo}", // if r1(u) < val_lo(u) { cc = 1 } else if r1(u) > val_lo(u) { cc = 2 } else { cc = 0 }
select_op!("h", "%r12", "%r1", "{val_lo}"), // if cc == 2 { r12 = r1 } else { r12 = val_lo }
"cgr %r0, {val_hi}", // if r0(i) < val_hi(i) { cc = 1 } else if r0(i) > val_hi(i) { cc = 2 } else { cc = 0 }
select_op!("h", "%r13", "%r1", "{val_lo}"), // if cc == 2 { r13 = r1 } else { r13 = val_lo }
"locgre %r13, %r12", // if cc == 0 { r13 = r12 }
select_op!("h", "%r12", "%r0", "{val_hi}"), // if cc == 2 { r12 = r0 } else { r12 = val_hi }
}
atomic_rmw_cas_3! {
atomic_umax, [tmp = out(reg) _,],
"clgr %r1, {val_lo}", // if r1(u) < val_lo(u) { cc = 1 } else if r1(u) > val_lo(u) { cc = 2 } else { cc = 0 }
select_op!("h", "{tmp}", "%r1", "{val_lo}"), // if cc == 2 { tmp = r1 } else { tmp = val_lo }
"clgr %r0, {val_hi}", // if r0(u) < val_hi(u) { cc = 1 } else if r0(u) > val_hi(u) { cc = 2 } else { cc = 0 }
select_op!("h", "%r12", "%r0", "{val_hi}"), // if cc == 2 { r12 = r0 } else { r12 = val_hi }
select_op!("h", "%r13", "%r1", "{val_lo}"), // if cc == 2 { r13 = r1 } else { r13 = val_lo }
"cgr %r0, {val_hi}", // if r0(i) < val_hi(i) { cc = 1 } else if r0(i) > val_hi(i) { cc = 2 } else { cc = 0 }
"locgre %r13, {tmp}", // if cc == 0 { r13 = tmp }
}
atomic_rmw_cas_3! {
atomic_min, [],
"clgr %r1, {val_lo}", // if r1(u) < val_lo(u) { cc = 1 } else if r1(u) > val_lo(u) { cc = 2 } else { cc = 0 }
select_op!("l", "%r12", "%r1", "{val_lo}"), // if cc == 1 { r12 = r1 } else { r12 = val_lo }
"cgr %r0, {val_hi}", // if r0(i) < val_hi(i) { cc = 1 } else if r0(i) > val_hi(i) { cc = 2 } else { cc = 0 }
select_op!("l", "%r13", "%r1", "{val_lo}"), // if cc == 1 { r13 = r1 } else { r13 = val_lo }
"locgre %r13, %r12", // if cc == 0 { r13 = r12 }
select_op!("l", "%r12", "%r0", "{val_hi}"), // if cc == 1 { r12 = r0 } else { r12 = val_hi }
}
atomic_rmw_cas_3! {
atomic_umin, [tmp = out(reg) _,],
"clgr %r1, {val_lo}", // if r1(u) < val_lo(u) { cc = 1 } else if r1(u) > val_lo(u) { cc = 2 } else { cc = 0 }
select_op!("l", "{tmp}", "%r1", "{val_lo}"), // if cc == 1 { tmp = r1 } else { tmp = val_lo }
"clgr %r0, {val_hi}", // if r0(u) < val_hi(u) { cc = 1 } else if r0(u) > val_hi(u) { cc = 2 } else { cc = 0 }
select_op!("l", "%r12", "%r0", "{val_hi}"), // if cc == 1 { r12 = r0 } else { r12 = val_hi }
select_op!("l", "%r13", "%r1", "{val_lo}"), // if cc == 1 { r13 = r1 } else { r13 = val_lo }
"cgr %r0, {val_hi}", // if r0(i) < val_hi(i) { cc = 1 } else if r0(i) > val_hi(i) { cc = 2 } else { cc = 0 }
"locgre %r13, {tmp}", // if cc == 0 { r13 = tmp }
}
}
#[cfg(else)]
{
#[rustfmt::skip]
macro_rules! atomic_cmp {
($name:ident, $cmp_hi1:tt, $cmp_hi2:tt, $cmp_lo:tt,) => {
#[inline]
unsafe fn $name(dst: *mut u128, val: u128, _order: Ordering) -> u128 {
debug_assert!(dst.addr() % 16 == 0);
let val = U128 { whole: val };
let (mut prev_hi, mut prev_lo);
// SAFETY: the caller must uphold the safety contract.
// CDSG has SeqCst semantics.
unsafe {
macro_rules! cmp {
($val_hi:tt, $val_lo:tt) => {
asm!(
"lg %r1, 8({dst})", // atomic { r1 = *dst.byte_add(8) }
"lg %r0, 0({dst})", // atomic { r0 = *dst }
"2:", // 'retry:
concat!($cmp_hi1, " %", $val_hi, ", %r0, 4f"), // if $cmp_hi1(val_hi, r0) { jump 'val }
concat!($cmp_hi2, " %", $val_hi, ", %r0, 3f"), // if $cmp_hi2(val_hi, r0) { jump 'prev }
concat!($cmp_lo, " %", $val_lo, ", %r1, 4f"), // if $cmp_lo(val_lo, r1) { jump 'val }
"3:", // 'prev:
"cdsg %r0, %r0, 0({dst})", // atomic { if *dst == r0:r1 { cc = 0; *dst = r0:r1 } else { cc = 1; r0:r1 = *dst } }
"jl 2b", // if cc == 1 { jump 'retry }
"j 5f",
"4:", // 'val:
concat!("cdsg %r0, %", $val_hi, ", 0({dst})"), // atomic { if *dst == r0:r1 { cc = 0; *dst = val_hi:val_lo } else { cc = 1; r0:r1 = *dst } }
"jl 2b", // if cc == 1 { jump 'retry }
"5:",
dst = in(reg) ptr_reg!(dst),
// Quadword atomic instructions work with even/odd pair of specified register and subsequent register.
out("r0") prev_hi,
out("r1") prev_lo,
in($val_hi) val.pair.hi,
in($val_lo) val.pair.lo,
// Do not use `preserves_flags` because CDSG modifies the condition code.
options(nostack),
)
};
}
// r4 is stack pointer on z/OS.
#[cfg(not(target_os = "zos"))]
cmp!("r4", "r5");
#[cfg(target_os = "zos")]
cmp!("r12", "r13");
U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole
}
}
};
}
// loc{,g}r requires z196 or later.
atomic_cmp! {
atomic_max,
"cgrjh", // signed >
"cgrjl", // signed <
"clgrjhe", // unsigned >=
}
atomic_cmp! {
atomic_umax,
"clgrjh", // unsigned >
"clgrjl", // unsigned <
"clgrjhe", // unsigned >=
}
atomic_cmp! {
atomic_min,
"cgrjl", // signed <
"cgrjh", // signed >
"clgrjle", // unsigned <=
}
atomic_cmp! {
atomic_umin,
"clgrjl", // unsigned <
"clgrjh", // unsigned >
"clgrjle", // unsigned <=
}
}
});
atomic_rmw_cas_2! {
atomic_not, [],
"lcgr %r13, %r1", // r13 = !r1 + 1
"aghi %r13, -1", // r13 -= 1
"lcgr %r12, %r0", // r12 = !r0 + 1
"aghi %r12, -1", // r12 -= 1
}
#[cfg(any(target_feature = "distinct-ops", portable_atomic_target_feature = "distinct-ops"))]
atomic_rmw_cas_2! {
atomic_neg, [zero = in(reg) 0_u64,],
"slgrk %r13, {zero}, %r1", // r13 = 0 - r1; cc = zero | borrow
"lghi %r12, 0", // r12 = 0
"slbgr %r12, %r0", // r12 -= r0 + borrow
}
#[cfg(not(any(target_feature = "distinct-ops", portable_atomic_target_feature = "distinct-ops")))]
atomic_rmw_cas_2! {
atomic_neg, [],
"lghi %r13, 0", // r13 = 0
"slgr %r13, %r1", // r13 -= r1; cc = zero | borrow
"lghi %r12, 0", // r12 = 0
"slbgr %r12, %r0", // r12 -= r0 + borrow
}
#[inline]
const fn is_lock_free() -> bool {
IS_ALWAYS_LOCK_FREE
}
const IS_ALWAYS_LOCK_FREE: bool = true;
atomic128!(AtomicI128, i128, atomic_max, atomic_min);
atomic128!(AtomicU128, u128, atomic_umax, atomic_umin);
#[cfg(test)]
mod tests {
use super::*;
test_atomic_int!(i128);
test_atomic_int!(u128);
// load/store/swap implementation is not affected by signedness, so it is
// enough to test only unsigned types.
stress_test!(u128);
}
vendor/portable-atomic/src/imp/atomic128/x86_64.rs
Vendored
+1009
View File
File diff suppressed because it is too large. Load diff
vendor/portable-atomic/src/imp/atomic64/README.md
Vendored
+24
View File
@@ -0,0 +1,24 @@
# 64-bit atomic implementations on 32-bit architectures
(See the [`atomic128` module](../atomic128) for 128-bit atomic implementations on 64-bit architectures.)
## 64-bit atomic instructions
Here is the table of targets that support 64-bit atomics and the instructions used:
| target_arch | load | store | CAS | RMW | note |
| ----------- | ---- | ----- | --- | --- | ---- |
| x86 | cmpxchg8b or fild or movlps or movq | cmpxchg8b or fistp or movlps | cmpxchg8b | cmpxchg8b | provided by `core::sync::atomic` |
| arm | ldrexd | ldrexd/strexd | ldrexd/strexd | ldrexd/strexd | provided by `core::sync::atomic` for Armv6+, otherwise provided by us for Linux/Android using kuser_cmpxchg64 (see [arm_linux.rs](arm_linux.rs) for more) |
| riscv32 | amocas.d | amocas.d | amocas.d | amocas.d | Requires `zacas` target feature. Both compile-time and run-time detection are supported. <br> Requires Rust 1.59+ |
| hexagon | memd | memd | memd_locked | memd_locked | Unimplemented |
| sparc | ldx | stx | casx | casx | Unimplemented (unsupported in LLVM). Requires `v8plus` and `v9` target feature (Linux is v8plus+v9 by default) |
| m68k | cas2 | cas2 | cas2 | cas2 | Unimplemented (unsupported in LLVM). Requires M68020 or later (Linux is M68020 by default) |
| mips32r6 | llwp | llwp/scwp | llwp/scwp | llwp/scwp | Unimplemented (unsupported in LLVM). Requires Release 6 Paired LL/SC family of instructions |
If `core::sync::atomic` provides 64-bit atomics, we use them.
On compiler versions or platforms where these are not supported, the fallback implementation is used.
## Run-time CPU feature detection
See the [`detect` module's readme](../detect/README.md) for run-time CPU feature detection.
vendor/portable-atomic/src/imp/atomic64/arm_linux.rs
Vendored
+330
View File
@@ -0,0 +1,330 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
64-bit atomic implementation using kuser_cmpxchg64 on pre-v6 Arm Linux/Android.
See "Atomic operation overview by architecture" in atomic-maybe-uninit for a more comprehensive and
detailed description of the atomic and synchronize instructions in this architecture:
https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#arm
Refs:
- https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/arm/kernel_user_helpers.rst
- https://github.com/rust-lang/compiler-builtins/blob/compiler_builtins-v0.1.124/src/arm_linux.rs
- https://github.com/taiki-e/atomic-maybe-uninit
Note: __kuser_cmpxchg64 is always SeqCst.
https://github.com/taiki-e/atomic-maybe-uninit/blob/v0.3.21/src/arch/arm.rs#L230
Note: On Miri and ThreadSanitizer which do not support inline assembly, we don't use
this module and use fallback implementation instead.
*/
// TODO: Since Rust 1.64, the Linux kernel requirement for Rust when using std is 3.2+, so it should
// be possible to omit the dynamic kernel version check if the std feature is enabled on Rust 1.64+.
// https://blog.rust-lang.org/2022/08/01/Increasing-glibc-kernel-requirements
include!("macros.rs");
#[path = "../fallback/outline_atomics.rs"]
mod fallback;
#[cfg(test)] // test-only (unused)
#[cfg(not(portable_atomic_no_outline_atomics))]
#[cfg(any(
all(
target_os = "linux",
any(
target_env = "gnu",
target_env = "musl",
target_env = "ohos",
all(target_env = "uclibc", not(target_feature = "crt-static")),
),
),
target_os = "android",
target_os = "freebsd",
target_os = "openbsd",
))]
#[path = "../detect/auxv.rs"]
mod test_detect_auxv;
#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
use core::{mem, sync::atomic::Ordering};
#[cfg(portable_atomic_no_strict_provenance)]
use crate::utils::ptr::PtrExt as _;
use crate::utils::{Pair, U64};
// https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/arm/kernel_user_helpers.rst
const KUSER_HELPER_VERSION: usize = 0xFFFF0FFC;
// __kuser_helper_version >= 5 (kernel version 3.1+)
const KUSER_CMPXCHG64: usize = 0xFFFF0F60;
#[inline]
fn __kuser_helper_version() -> i32 {
// SAFETY: core assumes that at least __kuser_memory_barrier (__kuser_helper_version >= 3) is
// available on this platform. __kuser_helper_version is always available on such a platform.
unsafe { crate::utils::ptr::with_exposed_provenance::<i32>(KUSER_HELPER_VERSION).read() }
}
#[inline]
fn has_kuser_cmpxchg64() -> bool {
// Note: detect_false cfg is intended to make it easy for developers to test
// cases where features usually available is not available, and is not a public API.
if cfg!(portable_atomic_test_detect_false) {
return false;
}
__kuser_helper_version() >= 5
}
#[inline]
unsafe fn __kuser_cmpxchg64(old_val: *const u64, new_val: *const u64, ptr: *mut u64) -> bool {
// SAFETY: kernel docs specify a known address with the given signature.
// And the caller must guarantee that the pointer is valid for read and write,
// aligned to the element size, and __kuser_helper_version >= 5.
unsafe {
let f: extern "C" fn(*const u64, *const u64, *mut u64) -> u32 =
mem::transmute(crate::utils::ptr::with_exposed_provenance::<()>(KUSER_CMPXCHG64));
f(old_val, new_val, ptr) == 0
}
}
// 64-bit atomic load by two 32-bit atomic loads.
#[inline]
unsafe fn byte_wise_atomic_load(src: *const u64) -> u64 {
// SAFETY: the caller must uphold the safety contract.
unsafe {
let (out_lo, out_hi);
asm!(
"ldr {out_lo}, [{src}]", // atomic { out_lo = *src }
"ldr {out_hi}, [{src}, #4]", // atomic { out_hi = *src.byte_add(4) }
src = in(reg) src,
out_lo = out(reg) out_lo,
out_hi = out(reg) out_hi,
options(pure, nostack, preserves_flags, readonly),
);
U64 { pair: Pair { lo: out_lo, hi: out_hi } }.whole
}
}
macro_rules! select_atomic {
(
unsafe fn $name:ident($dst:ident: *mut u64 $(, $($arg:tt)*)?) $(-> $ret_ty:ty)? {
|$kuser_cmpxchg64_fn_binding:ident| $($kuser_cmpxchg64_fn_body:tt)*
}
fallback = $seqcst_fallback_fn:ident
) => {
#[inline]
unsafe fn $name($dst: *mut u64 $(, $($arg)*)?, _: Ordering) $(-> $ret_ty)? {
unsafe fn kuser_cmpxchg64_fn($dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {
debug_assert!($dst.addr() % 8 == 0);
debug_assert!(has_kuser_cmpxchg64());
// SAFETY: the caller must uphold the safety contract.
unsafe {
loop {
// This is not single-copy atomic reads, but this is ok because subsequent
// CAS will check for consistency.
//
// Arm's memory model allow mixed-sized atomic access.
// https://github.com/rust-lang/unsafe-code-guidelines/issues/345#issuecomment-1172891466
//
// Note that the C++20 memory model does not allow mixed-sized atomic access,
// so we must use inline assembly to implement byte_wise_atomic_load.
// (i.e., byte-wise atomic based on the standard library's atomic types
// cannot be used here).
let prev = byte_wise_atomic_load($dst);
let next = {
let $kuser_cmpxchg64_fn_binding = prev;
$($kuser_cmpxchg64_fn_body)*
};
if __kuser_cmpxchg64(&prev, &next, $dst) {
return prev;
}
}
}
}
// SAFETY: the caller must uphold the safety contract.
// we only calls __kuser_cmpxchg64 if it is available.
unsafe {
ifunc!(unsafe fn($dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {
if has_kuser_cmpxchg64() {
kuser_cmpxchg64_fn
} else {
// Use SeqCst because __kuser_cmpxchg64 is always SeqCst.
fallback::$seqcst_fallback_fn
}
})
}
}
};
}
select_atomic! {
unsafe fn atomic_load(src: *mut u64) -> u64 {
|old| old
}
fallback = atomic_load_seqcst
}
#[inline]
unsafe fn atomic_store(dst: *mut u64, val: u64, order: Ordering) {
// SAFETY: the caller must uphold the safety contract.
unsafe {
atomic_swap(dst, val, order);
}
}
select_atomic! {
unsafe fn atomic_swap(dst: *mut u64, val: u64) -> u64 {
|_x| val
}
fallback = atomic_swap_seqcst
}
#[inline]
unsafe fn atomic_compare_exchange(
dst: *mut u64,
old: u64,
new: u64,
_: Ordering,
_: Ordering,
) -> Result<u64, u64> {
unsafe fn kuser_cmpxchg64_fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {
debug_assert!(dst.addr() % 8 == 0);
debug_assert!(has_kuser_cmpxchg64());
// SAFETY: the caller must uphold the safety contract.
unsafe {
loop {
// See select_atomic! for more.
let prev = byte_wise_atomic_load(dst);
let next = if prev == old { new } else { prev };
if __kuser_cmpxchg64(&prev, &next, dst) {
return (prev, prev == old);
}
}
}
}
// SAFETY: the caller must uphold the safety contract.
// we only calls __kuser_cmpxchg64 if it is available.
let (prev, ok) = unsafe {
ifunc!(unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {
if has_kuser_cmpxchg64() {
kuser_cmpxchg64_fn
} else {
// Use SeqCst because __kuser_cmpxchg64 is always SeqCst.
fallback::atomic_compare_exchange_seqcst
}
})
};
if ok { Ok(prev) } else { Err(prev) }
}
use self::atomic_compare_exchange as atomic_compare_exchange_weak;
select_atomic! {
unsafe fn atomic_add(dst: *mut u64, val: u64) -> u64 {
|x| x.wrapping_add(val)
}
fallback = atomic_add_seqcst
}
select_atomic! {
unsafe fn atomic_sub(dst: *mut u64, val: u64) -> u64 {
|x| x.wrapping_sub(val)
}
fallback = atomic_sub_seqcst
}
select_atomic! {
unsafe fn atomic_and(dst: *mut u64, val: u64) -> u64 {
|x| x & val
}
fallback = atomic_and_seqcst
}
select_atomic! {
unsafe fn atomic_nand(dst: *mut u64, val: u64) -> u64 {
|x| !(x & val)
}
fallback = atomic_nand_seqcst
}
select_atomic! {
unsafe fn atomic_or(dst: *mut u64, val: u64) -> u64 {
|x| x | val
}
fallback = atomic_or_seqcst
}
select_atomic! {
unsafe fn atomic_xor(dst: *mut u64, val: u64) -> u64 {
|x| x ^ val
}
fallback = atomic_xor_seqcst
}
select_atomic! {
unsafe fn atomic_max(dst: *mut u64, val: u64) -> u64 {
|x| {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
{ core::cmp::max(x as i64, val as i64) as u64 }
}
}
fallback = atomic_max_seqcst
}
select_atomic! {
unsafe fn atomic_umax(dst: *mut u64, val: u64) -> u64 {
|x| core::cmp::max(x, val)
}
fallback = atomic_umax_seqcst
}
select_atomic! {
unsafe fn atomic_min(dst: *mut u64, val: u64) -> u64 {
|x| {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
{ core::cmp::min(x as i64, val as i64) as u64 }
}
}
fallback = atomic_min_seqcst
}
select_atomic! {
unsafe fn atomic_umin(dst: *mut u64, val: u64) -> u64 {
|x| core::cmp::min(x, val)
}
fallback = atomic_umin_seqcst
}
select_atomic! {
unsafe fn atomic_not(dst: *mut u64) -> u64 {
|x| !x
}
fallback = atomic_not_seqcst
}
select_atomic! {
unsafe fn atomic_neg(dst: *mut u64) -> u64 {
|x| x.wrapping_neg()
}
fallback = atomic_neg_seqcst
}
#[inline]
fn is_lock_free() -> bool {
has_kuser_cmpxchg64()
}
const IS_ALWAYS_LOCK_FREE: bool = false;
atomic64!(AtomicI64, i64, atomic_max, atomic_min);
atomic64!(AtomicU64, u64, atomic_umax, atomic_umin);
#[allow(
clippy::alloc_instead_of_core,
clippy::std_instead_of_alloc,
clippy::std_instead_of_core,
clippy::undocumented_unsafe_blocks,
clippy::wildcard_imports
)]
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn kuser_helper_version() {
let version = __kuser_helper_version();
assert!(version >= 5, "{:?}", version);
assert_eq!(version, unsafe {
crate::utils::ptr::with_exposed_provenance::<i32>(KUSER_HELPER_VERSION).read()
});
}
test_atomic_int!(i64);
test_atomic_int!(u64);
// load/store/swap implementation is not affected by signedness, so it is
// enough to test only unsigned types.
stress_test!(u64);
}
vendor/portable-atomic/src/imp/atomic64/macros.rs
Vendored
+230
View File
@@ -0,0 +1,230 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
macro_rules! atomic64 {
($atomic_type:ident, $int_type:ident, $atomic_max:ident, $atomic_min:ident) => {
#[repr(C, align(8))]
pub(crate) struct $atomic_type {
v: core::cell::UnsafeCell<$int_type>,
}
// Send is implicitly implemented.
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock.
unsafe impl Sync for $atomic_type {}
impl_default_no_fetch_ops!($atomic_type, $int_type);
impl_default_bit_opts!($atomic_type, $int_type);
impl $atomic_type {
#[inline]
pub(crate) const fn new(v: $int_type) -> Self {
Self { v: core::cell::UnsafeCell::new(v) }
}
#[inline]
pub(crate) fn is_lock_free() -> bool {
is_lock_free()
}
pub(crate) const IS_ALWAYS_LOCK_FREE: bool = IS_ALWAYS_LOCK_FREE;
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub(crate) fn load(&self, order: Ordering) -> $int_type {
crate::utils::assert_load_ordering(order);
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
// and the raw pointer passed in is valid because we got it from a reference.
unsafe {
atomic_load(self.v.get().cast::<u64>(), order) as $int_type
}
}
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub(crate) fn store(&self, val: $int_type, order: Ordering) {
crate::utils::assert_store_ordering(order);
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
// and the raw pointer passed in is valid because we got it from a reference.
unsafe {
atomic_store(self.v.get().cast::<u64>(), val as u64, order)
}
}
#[inline]
pub(crate) fn swap(&self, val: $int_type, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
// and the raw pointer passed in is valid because we got it from a reference.
unsafe {
atomic_swap(self.v.get().cast::<u64>(), val as u64, order) as $int_type
}
}
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub(crate) fn compare_exchange(
&self,
current: $int_type,
new: $int_type,
success: Ordering,
failure: Ordering,
) -> Result<$int_type, $int_type> {
crate::utils::assert_compare_exchange_ordering(success, failure);
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
// and the raw pointer passed in is valid because we got it from a reference.
unsafe {
match atomic_compare_exchange(
self.v.get().cast::<u64>(),
current as u64,
new as u64,
success,
failure,
) {
Ok(v) => Ok(v as $int_type),
Err(v) => Err(v as $int_type),
}
}
}
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub(crate) fn compare_exchange_weak(
&self,
current: $int_type,
new: $int_type,
success: Ordering,
failure: Ordering,
) -> Result<$int_type, $int_type> {
crate::utils::assert_compare_exchange_ordering(success, failure);
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
// and the raw pointer passed in is valid because we got it from a reference.
unsafe {
match atomic_compare_exchange_weak(
self.v.get().cast::<u64>(),
current as u64,
new as u64,
success,
failure,
) {
Ok(v) => Ok(v as $int_type),
Err(v) => Err(v as $int_type),
}
}
}
#[inline]
pub(crate) fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
// and the raw pointer passed in is valid because we got it from a reference.
unsafe {
atomic_add(self.v.get().cast::<u64>(), val as u64, order) as $int_type
}
}
#[inline]
pub(crate) fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
// and the raw pointer passed in is valid because we got it from a reference.
unsafe {
atomic_sub(self.v.get().cast::<u64>(), val as u64, order) as $int_type
}
}
#[inline]
pub(crate) fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
// and the raw pointer passed in is valid because we got it from a reference.
unsafe {
atomic_and(self.v.get().cast::<u64>(), val as u64, order) as $int_type
}
}
#[inline]
pub(crate) fn fetch_nand(&self, val: $int_type, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
// and the raw pointer passed in is valid because we got it from a reference.
unsafe {
atomic_nand(self.v.get().cast::<u64>(), val as u64, order) as $int_type
}
}
#[inline]
pub(crate) fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
// and the raw pointer passed in is valid because we got it from a reference.
unsafe {
atomic_or(self.v.get().cast::<u64>(), val as u64, order) as $int_type
}
}
#[inline]
pub(crate) fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
// and the raw pointer passed in is valid because we got it from a reference.
unsafe {
atomic_xor(self.v.get().cast::<u64>(), val as u64, order) as $int_type
}
}
#[inline]
pub(crate) fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
// and the raw pointer passed in is valid because we got it from a reference.
unsafe {
$atomic_max(self.v.get().cast::<u64>(), val as u64, order) as $int_type
}
}
#[inline]
pub(crate) fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
// and the raw pointer passed in is valid because we got it from a reference.
unsafe {
$atomic_min(self.v.get().cast::<u64>(), val as u64, order) as $int_type
}
}
#[inline]
pub(crate) fn fetch_not(&self, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
// and the raw pointer passed in is valid because we got it from a reference.
unsafe {
atomic_not(self.v.get().cast::<u64>(), order) as $int_type
}
}
#[inline]
pub(crate) fn not(&self, order: Ordering) {
self.fetch_not(order);
}
#[inline]
pub(crate) fn fetch_neg(&self, order: Ordering) -> $int_type {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
// and the raw pointer passed in is valid because we got it from a reference.
unsafe {
atomic_neg(self.v.get().cast::<u64>(), order) as $int_type
}
}
#[inline]
pub(crate) fn neg(&self, order: Ordering) {
self.fetch_neg(order);
}
#[inline]
pub(crate) const fn as_ptr(&self) -> *mut $int_type {
self.v.get()
}
}
};
}
vendor/portable-atomic/src/imp/atomic64/mod.rs
Vendored
+43
View File
@@ -0,0 +1,43 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
64-bit atomic implementations on 32-bit architectures
See README.md for details.
*/
// pre-v6 Arm Linux
// Miri and Sanitizer do not support inline assembly.
#[cfg(all(
feature = "fallback",
target_arch = "arm",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_os = "linux", target_os = "android"),
any(test, not(any(target_feature = "v6", portable_atomic_target_feature = "v6"))),
not(portable_atomic_no_outline_atomics),
))]
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(any(test, portable_atomic_no_atomic_64)))]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(any(test, not(target_has_atomic = "64")))
)]
pub(super) mod arm_linux;
// riscv32
// Miri and Sanitizer do not support inline assembly.
#[cfg(all(
target_arch = "riscv32",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(
target_feature = "zacas",
portable_atomic_target_feature = "zacas",
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
any(target_os = "linux", target_os = "android"),
),
),
))]
pub(super) mod riscv32;
vendor/portable-atomic/src/imp/atomic64/riscv32.rs
Vendored
+626
View File
@@ -0,0 +1,626 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
64-bit atomic implementation on riscv32.
This architecture provides the following 64-bit atomic instructions:
- amocas.d: CAS (Zacas extension)
See "Atomic operation overview by architecture" in atomic-maybe-uninit for a more comprehensive and
detailed description of the atomic and synchronize instructions in this architecture:
https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#risc-v
Note: On Miri and ThreadSanitizer which do not support inline assembly, we don't use
this module and use fallback implementation instead.
Refs:
- RISC-V Instruction Set Manual
"Zacas" Extension for Atomic Compare-and-Swap (CAS) Instructions
https://docs.riscv.org/reference/isa/v20260120/unpriv/zacas.html
- RISC-V Atomics ABI Specification
https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/draft-20250812-301374e92976e298e676e7129a6212926b2299ce/riscv-atomic.adoc
See tests/asm-test/asm/portable-atomic for generated assembly.
*/
// TODO: merge duplicated code with atomic128/riscv64.rs
include!("macros.rs");
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
#[path = "../fallback/outline_atomics.rs"]
mod fallback;
#[cfg(not(portable_atomic_no_outline_atomics))]
#[cfg(any(test, not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))))]
#[cfg(any(target_os = "linux", target_os = "android"))]
#[path = "../detect/riscv_linux.rs"]
mod detect;
#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
use core::sync::atomic::Ordering;
#[cfg(portable_atomic_no_strict_provenance)]
use crate::utils::ptr::PtrExt as _;
use crate::utils::{Pair, U64};
macro_rules! debug_assert_zacas {
() => {
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
{
debug_assert!(detect::detect().zacas());
}
};
}
// `.option arch, +zacas` directive requires LLVM 20, so we use .4byte directive for old LLVM.
// Note that `.insn <value>` directive requires LLVM 19.
// https://github.com/llvm/llvm-project/commit/2a086dce691e3cc34a2fc27f4fb255bb2cbbfac9
// https://github.com/riscv-non-isa/riscv-asm-manual/blob/v0.0.1/src/asm-manual.adoc#arch
#[cfg(not(portable_atomic_pre_llvm_20))]
macro_rules! start_zacas {
() => {
".option push\n.option arch, +zacas"
};
}
#[cfg(not(portable_atomic_pre_llvm_20))]
macro_rules! end_zacas {
() => {
".option pop"
};
}
#[cfg(not(portable_atomic_pre_llvm_20))]
macro_rules! atomic_rmw_amocas_order {
($op:ident, $order:ident) => {
atomic_rmw_amocas_order!($op, $order, failure = $order)
};
($op:ident, $order:ident, failure = $failure:ident) => {
match $order {
Ordering::Relaxed => $op!("", ""),
Ordering::Acquire => $op!("", ".aq"),
Ordering::Release => $op!("", ".rl"),
Ordering::AcqRel => $op!("", ".aqrl"),
Ordering::SeqCst if $failure == Ordering::SeqCst => $op!("fence rw,rw", ".aqrl"),
Ordering::SeqCst => $op!("", ".aqrl"),
_ => unreachable!(),
}
};
}
#[cfg(portable_atomic_pre_llvm_20)]
macro_rules! atomic_rmw_amocas_order_insn {
($op:ident, $order:ident) => {
atomic_rmw_amocas_order_insn!($op, $order, failure = $order)
};
($op:ident, $order:ident, failure = $failure:ident) => {
match $order {
Ordering::Relaxed => $op!("", "8"),
Ordering::Acquire => $op!("", "c"),
Ordering::Release => $op!("", "a"),
Ordering::AcqRel => $op!("", "e"),
Ordering::SeqCst if $failure == Ordering::SeqCst => $op!("fence rw,rw", "e"),
Ordering::SeqCst => $op!("", "e"),
_ => unreachable!(),
}
};
}
// If zacas is available at compile-time, we can always use zacas_fn.
#[cfg(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))]
use self::atomic_load_zacas as atomic_load;
// Otherwise, we need to do run-time detection and can use zacas_fn only if zacas is available.
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
#[inline]
unsafe fn atomic_load(src: *mut u64, order: Ordering) -> u64 {
fn_alias! {
// inline(never) is just a hint and also not strictly necessary
// because we use ifunc helper macro, but used for clarity.
#[inline(never)]
unsafe fn(src: *mut u64) -> u64;
atomic_load_zacas_relaxed = atomic_load_zacas(Ordering::Relaxed);
atomic_load_zacas_acquire = atomic_load_zacas(Ordering::Acquire);
atomic_load_zacas_seqcst = atomic_load_zacas(Ordering::SeqCst);
}
// SAFETY: the caller must uphold the safety contract.
// we only calls atomic_load_zacas if zacas is available.
unsafe {
match order {
Ordering::Relaxed => {
ifunc!(unsafe fn(src: *mut u64) -> u64 {
if detect::detect().zacas() {
atomic_load_zacas_relaxed
} else {
fallback::atomic_load_non_seqcst
}
})
}
Ordering::Acquire => {
ifunc!(unsafe fn(src: *mut u64) -> u64 {
if detect::detect().zacas() {
atomic_load_zacas_acquire
} else {
fallback::atomic_load_non_seqcst
}
})
}
Ordering::SeqCst => {
ifunc!(unsafe fn(src: *mut u64) -> u64 {
if detect::detect().zacas() {
atomic_load_zacas_seqcst
} else {
fallback::atomic_load_seqcst
}
})
}
_ => unreachable!(),
}
}
}
#[inline]
unsafe fn atomic_load_zacas(src: *mut u64, order: Ordering) -> u64 {
debug_assert!(src.addr() % 8 == 0);
debug_assert_zacas!();
let (out_lo, out_hi);
// SAFETY: the caller must uphold the safety contract.
unsafe {
#[cfg(not(portable_atomic_pre_llvm_20))]
macro_rules! load {
($fence:tt, $asm_order:tt) => {
asm!(
start_zacas!(),
$fence, // fence
concat!("amocas.d", $asm_order, " a2, a2, 0({src})"), // atomic { if *dst == a2:a3 { *dst = a2:a3 } else { a2:a3 = *dst } }
end_zacas!(),
src = in(reg) ptr_reg!(src),
inout("a2") 0_u32 => out_lo,
inout("a3") 0_u32 => out_hi,
options(nostack, preserves_flags),
)
};
}
#[cfg(not(portable_atomic_pre_llvm_20))]
atomic_rmw_amocas_order!(load, order);
#[cfg(portable_atomic_pre_llvm_20)]
macro_rules! load {
($fence:tt, $insn_order:tt) => {
asm!(
$fence, // fence
// amocas.d{,.aq,.rl,.aqrl} a2, a2, (a0) // atomic { if *a0 == a2:a3 { *a0 = a2:a3 } else { a2:a3 = *a0 } }
concat!(".4byte 0x2", $insn_order, "c5362f"),
in("a0") ptr_reg!(src),
inout("a2") 0_u32 => out_lo,
inout("a3") 0_u32 => out_hi,
options(nostack, preserves_flags),
)
};
}
#[cfg(portable_atomic_pre_llvm_20)]
atomic_rmw_amocas_order_insn!(load, order);
U64 { pair: Pair { lo: out_lo, hi: out_hi } }.whole
}
}
#[inline]
unsafe fn atomic_store(dst: *mut u64, val: u64, order: Ordering) {
// SAFETY: the caller must uphold the safety contract.
unsafe {
atomic_swap(dst, val, order);
}
}
#[inline]
unsafe fn atomic_compare_exchange(
dst: *mut u64,
old: u64,
new: u64,
success: Ordering,
failure: Ordering,
) -> Result<u64, u64> {
#[cfg(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))]
// SAFETY: the caller must uphold the safety contract.
// cfg guarantees that zacas instructions are available at compile-time.
let (prev, ok) = unsafe { atomic_compare_exchange_zacas(dst, old, new, success, failure) };
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
let (prev, ok) = {
fn_alias! {
// inline(never) is just a hint and also not strictly necessary
// because we use ifunc helper macro, but used for clarity.
#[inline(never)]
unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool);
zacas_relaxed_fn = atomic_compare_exchange_zacas(Ordering::Relaxed, Ordering::Relaxed);
zacas_acquire_fn = atomic_compare_exchange_zacas(Ordering::Acquire, Ordering::Acquire);
zacas_release_fn = atomic_compare_exchange_zacas(Ordering::Release, Ordering::Relaxed);
zacas_acqrel_fn = atomic_compare_exchange_zacas(Ordering::AcqRel, Ordering::Acquire);
zacas_seqcst_fn = atomic_compare_exchange_zacas(Ordering::SeqCst, Ordering::SeqCst);
}
let order = crate::utils::upgrade_success_ordering(success, failure);
// SAFETY: the caller must uphold the safety contract.
// we only calls atomic_compare_exchange_zacas if zacas is available.
unsafe {
match order {
Ordering::Relaxed => {
ifunc!(unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {
if detect::detect().zacas() {
zacas_relaxed_fn
} else {
fallback::atomic_compare_exchange_non_seqcst
}
})
}
Ordering::Acquire => {
ifunc!(unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {
if detect::detect().zacas() {
zacas_acquire_fn
} else {
fallback::atomic_compare_exchange_non_seqcst
}
})
}
Ordering::Release => {
ifunc!(unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {
if detect::detect().zacas() {
zacas_release_fn
} else {
fallback::atomic_compare_exchange_non_seqcst
}
})
}
Ordering::AcqRel => {
ifunc!(unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {
if detect::detect().zacas() {
zacas_acqrel_fn
} else {
fallback::atomic_compare_exchange_non_seqcst
}
})
}
Ordering::SeqCst => {
ifunc!(unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {
if detect::detect().zacas() {
zacas_seqcst_fn
} else {
fallback::atomic_compare_exchange_seqcst
}
})
}
_ => unreachable!(),
}
}
};
if ok { Ok(prev) } else { Err(prev) }
}
#[inline]
unsafe fn atomic_compare_exchange_zacas(
dst: *mut u64,
old: u64,
new: u64,
success: Ordering,
failure: Ordering,
) -> (u64, bool) {
debug_assert!(dst.addr() % 8 == 0);
debug_assert_zacas!();
let order = crate::utils::upgrade_success_ordering(success, failure);
let old = U64 { whole: old };
let new = U64 { whole: new };
let (prev_lo, prev_hi);
// SAFETY: the caller must uphold the safety contract.
unsafe {
#[cfg(not(portable_atomic_pre_llvm_20))]
macro_rules! cmpxchg {
($fence:tt, $asm_order:tt) => {
asm!(
start_zacas!(),
$fence, // fence
concat!("amocas.d", $asm_order, " a4, a2, 0({dst})"), // atomic { if *dst == a4:a5 { *dst = a2:a3 } else { a4:a5 = *dst } }
end_zacas!(),
dst = in(reg) ptr_reg!(dst),
// must be allocated to even/odd register pair
inout("a4") old.pair.lo => prev_lo,
inout("a5") old.pair.hi => prev_hi,
// must be allocated to even/odd register pair
in("a2") new.pair.lo,
in("a3") new.pair.hi,
options(nostack, preserves_flags),
)
};
}
#[cfg(not(portable_atomic_pre_llvm_20))]
atomic_rmw_amocas_order!(cmpxchg, order, failure = failure);
#[cfg(portable_atomic_pre_llvm_20)]
macro_rules! cmpxchg {
($fence:tt, $insn_order:tt) => {
asm!(
$fence, // fence
// amocas.d{,.aq,.rl,.aqrl} a4, a2, (a0) // atomic { if *a0 == a4:a5 { *a0 = a2:a3 } else { a4:a5 = *a0 } }
concat!(".4byte 0x2", $insn_order, "c5372f"),
in("a0") ptr_reg!(dst),
// must be allocated to even/odd register pair
inout("a4") old.pair.lo => prev_lo,
inout("a5") old.pair.hi => prev_hi,
// must be allocated to even/odd register pair
in("a2") new.pair.lo,
in("a3") new.pair.hi,
options(nostack, preserves_flags),
)
};
}
#[cfg(portable_atomic_pre_llvm_20)]
atomic_rmw_amocas_order_insn!(cmpxchg, order, failure = failure);
let prev = U64 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole;
(prev, prev == old.whole)
}
}
// amocas is always strong.
use self::atomic_compare_exchange as atomic_compare_exchange_weak;
// 64-bit atomic load by two 32-bit atomic loads.
#[inline]
unsafe fn byte_wise_atomic_load(src: *const u64) -> u64 {
let (out_lo, out_hi);
// SAFETY: the caller must uphold the safety contract.
unsafe {
asm!(
"lw {out_lo}, ({src})", // atomic { out_lo = *src }
"lw {out_hi}, 4({src})", // atomic { out_hi = *src.byte_add(4) }
src = in(reg) ptr_reg!(src),
out_lo = out(reg) out_lo,
out_hi = out(reg) out_hi,
options(pure, nostack, preserves_flags, readonly),
);
U64 { pair: Pair { lo: out_lo, hi: out_hi } }.whole
}
}
macro_rules! select_atomic_rmw {
(
unsafe fn $name:ident(dst: *mut u64 $(, $($arg:tt)*)?) $(-> $ret_ty:ty)? {
|$zacas_fn_binding:ident| $($zacas_fn_body:tt)*
}
zacas = $zacas_fn:ident;
non_seqcst_fallback = $non_seqcst_fallback_fn:ident;
seqcst_fallback = $seqcst_fallback_fn:ident;
) => {
#[inline]
unsafe fn $zacas_fn(dst: *mut u64 $(, $($arg)*)?, order: Ordering) $(-> $ret_ty)? {
// SAFETY: the caller must uphold the safety contract.
unsafe {
// This is not single-copy atomic reads, but this is ok because subsequent
// CAS will check for consistency.
//
// Note that the C++20 memory model does not allow mixed-sized atomic access,
// so we must use inline assembly to implement byte_wise_atomic_load.
// (i.e., byte-wise atomic based on the standard library's atomic types
// cannot be used here).
let mut prev = byte_wise_atomic_load(dst);
loop {
let next = {
let $zacas_fn_binding = prev;
$($zacas_fn_body)*
};
match atomic_compare_exchange_weak(dst, prev, next, order, Ordering::Relaxed) {
Ok(x) => return x,
Err(x) => prev = x,
}
}
}
}
// If zacas is available at compile-time, we can always use zacas_fn.
#[cfg(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))]
use self::$zacas_fn as $name;
// Otherwise, we need to do run-time detection and can use zacas_fn only if zacas is available.
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
#[inline]
unsafe fn $name(dst: *mut u64 $(, $($arg)*)?, order: Ordering) $(-> $ret_ty)? {
fn_alias! {
// inline(never) is just a hint and also not strictly necessary
// because we use ifunc helper macro, but used for clarity.
#[inline(never)]
unsafe fn(dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)?;
zacas_relaxed_fn = $zacas_fn(Ordering::Relaxed);
zacas_acquire_fn = $zacas_fn(Ordering::Acquire);
zacas_release_fn = $zacas_fn(Ordering::Release);
zacas_acqrel_fn = $zacas_fn(Ordering::AcqRel);
zacas_seqcst_fn = $zacas_fn(Ordering::SeqCst);
}
// SAFETY: the caller must uphold the safety contract.
// we only calls zacas_fn if zacas is available.
unsafe {
match order {
Ordering::Relaxed => {
ifunc!(unsafe fn(dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {
if detect::detect().zacas() {
zacas_relaxed_fn
} else {
fallback::$non_seqcst_fallback_fn
}
})
}
Ordering::Acquire => {
ifunc!(unsafe fn(dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {
if detect::detect().zacas() {
zacas_acquire_fn
} else {
fallback::$non_seqcst_fallback_fn
}
})
}
Ordering::Release => {
ifunc!(unsafe fn(dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {
if detect::detect().zacas() {
zacas_release_fn
} else {
fallback::$non_seqcst_fallback_fn
}
})
}
Ordering::AcqRel => {
ifunc!(unsafe fn(dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {
if detect::detect().zacas() {
zacas_acqrel_fn
} else {
fallback::$non_seqcst_fallback_fn
}
})
}
Ordering::SeqCst => {
ifunc!(unsafe fn(dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {
if detect::detect().zacas() {
zacas_seqcst_fn
} else {
fallback::$seqcst_fallback_fn
}
})
}
_ => unreachable!(),
}
}
}
};
}
select_atomic_rmw! {
unsafe fn atomic_swap(dst: *mut u64, val: u64) -> u64 {
|_x| val
}
zacas = atomic_swap_zacas;
non_seqcst_fallback = atomic_swap_non_seqcst;
seqcst_fallback = atomic_swap_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_add(dst: *mut u64, val: u64) -> u64 {
|x| x.wrapping_add(val)
}
zacas = atomic_add_zacas;
non_seqcst_fallback = atomic_add_non_seqcst;
seqcst_fallback = atomic_add_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_sub(dst: *mut u64, val: u64) -> u64 {
|x| x.wrapping_sub(val)
}
zacas = atomic_sub_zacas;
non_seqcst_fallback = atomic_sub_non_seqcst;
seqcst_fallback = atomic_sub_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_and(dst: *mut u64, val: u64) -> u64 {
|x| x & val
}
zacas = atomic_and_zacas;
non_seqcst_fallback = atomic_and_non_seqcst;
seqcst_fallback = atomic_and_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_nand(dst: *mut u64, val: u64) -> u64 {
|x| !(x & val)
}
zacas = atomic_nand_zacas;
non_seqcst_fallback = atomic_nand_non_seqcst;
seqcst_fallback = atomic_nand_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_or(dst: *mut u64, val: u64) -> u64 {
|x| x | val
}
zacas = atomic_or_zacas;
non_seqcst_fallback = atomic_or_non_seqcst;
seqcst_fallback = atomic_or_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_xor(dst: *mut u64, val: u64) -> u64 {
|x| x ^ val
}
zacas = atomic_xor_zacas;
non_seqcst_fallback = atomic_xor_non_seqcst;
seqcst_fallback = atomic_xor_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_max(dst: *mut u64, val: u64) -> u64 {
|x| {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
{ core::cmp::max(x as i64, val as i64) as u64 }
}
}
zacas = atomic_max_zacas;
non_seqcst_fallback = atomic_max_non_seqcst;
seqcst_fallback = atomic_max_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_umax(dst: *mut u64, val: u64) -> u64 {
|x| core::cmp::max(x, val)
}
zacas = atomic_umax_zacas;
non_seqcst_fallback = atomic_umax_non_seqcst;
seqcst_fallback = atomic_umax_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_min(dst: *mut u64, val: u64) -> u64 {
|x| {
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
{ core::cmp::min(x as i64, val as i64) as u64 }
}
}
zacas = atomic_min_zacas;
non_seqcst_fallback = atomic_min_non_seqcst;
seqcst_fallback = atomic_min_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_umin(dst: *mut u64, val: u64) -> u64 {
|x| core::cmp::min(x, val)
}
zacas = atomic_umin_zacas;
non_seqcst_fallback = atomic_umin_non_seqcst;
seqcst_fallback = atomic_umin_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_not(dst: *mut u64) -> u64 {
|x| !x
}
zacas = atomic_not_zacas;
non_seqcst_fallback = atomic_not_non_seqcst;
seqcst_fallback = atomic_not_seqcst;
}
select_atomic_rmw! {
unsafe fn atomic_neg(dst: *mut u64) -> u64 {
|x| x.wrapping_neg()
}
zacas = atomic_neg_zacas;
non_seqcst_fallback = atomic_neg_non_seqcst;
seqcst_fallback = atomic_neg_seqcst;
}
#[inline]
fn is_lock_free() -> bool {
#[cfg(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))]
{
// zacas is available at compile-time.
true
}
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
{
detect::detect().zacas()
}
}
const IS_ALWAYS_LOCK_FREE: bool =
cfg!(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"));
atomic64!(AtomicI64, i64, atomic_max, atomic_min);
atomic64!(AtomicU64, u64, atomic_umax, atomic_umin);
#[cfg(test)]
mod tests {
use super::*;
test_atomic_int!(i64);
test_atomic_int!(u64);
// load/store/swap implementation is not affected by signedness, so it is
// enough to test only unsigned types.
stress_test!(u64);
}
vendor/portable-atomic/src/imp/avr.rs
Vendored
+160
View File
@@ -0,0 +1,160 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Atomic load/store implementation on AVR.
Operations not supported here are provided by disabling interrupts.
See also src/imp/interrupt/avr.rs.
See "Atomic operation overview by architecture" in atomic-maybe-uninit for a more comprehensive and
detailed description of the atomic and synchronize instructions in this architecture:
https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#avr
Note: Ordering is always SeqCst.
Refs:
- AVR® Instruction Set Manual, Rev. DS40002198B
https://ww1.microchip.com/downloads/en/DeviceDoc/AVR-InstructionSet-Manual-DS40002198.pdf
- atomic-maybe-uninit
https://github.com/taiki-e/atomic-maybe-uninit
See tests/asm-test/asm/portable-atomic for generated assembly.
*/
use core::{arch::asm, cell::UnsafeCell, sync::atomic::Ordering};
macro_rules! atomic8 {
($atomic_type:ident, $value_type:ty) => {
#[repr(transparent)]
pub(crate) struct $atomic_type {
v: UnsafeCell<$value_type>,
}
// Send is implicitly implemented for atomic integers, but not for atomic pointers.
// SAFETY: any data races are prevented by atomic operations.
unsafe impl Send for $atomic_type {}
// SAFETY: any data races are prevented by atomic operations.
unsafe impl Sync for $atomic_type {}
impl $atomic_type {
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub(crate) fn load(&self, order: Ordering) -> $value_type {
crate::utils::assert_load_ordering(order);
let src = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
let out;
asm!(
"ld {out}, Z", // atomic { out = *Z }
out = out(reg) out,
in("Z") src,
options(nostack, preserves_flags),
);
out
}
}
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub(crate) fn store(&self, val: $value_type, order: Ordering) {
crate::utils::assert_store_ordering(order);
let dst = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
asm!(
"st Z, {val}", // atomic { *Z = val }
val = in(reg) val,
in("Z") dst,
options(nostack, preserves_flags),
);
}
}
}
#[cfg(any(target_feature = "rmw", portable_atomic_target_feature = "rmw"))]
impl $atomic_type {
#[inline]
pub(crate) fn swap(&self, val: $value_type, _order: Ordering) -> $value_type {
let dst = self.v.get();
let out;
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
// cfg guarantee that the CPU supports RMW instructions.
unsafe {
asm!(
"xch Z, {val}", // atomic { _x = *Z; *Z = val; val = _x }
val = inout(reg) val => out,
in("Z") dst,
options(nostack, preserves_flags),
);
}
out
}
#[inline]
pub(crate) fn fetch_and(&self, val: $value_type, _order: Ordering) -> $value_type {
let dst = self.v.get();
let out;
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
// cfg guarantee that the CPU supports RMW instructions.
unsafe {
asm!(
"lac Z, {val}", // atomic { _x = *Z; *Z &= !val; val = _x }
val = inout(reg) !val => out,
in("Z") dst,
options(nostack, preserves_flags),
);
}
out
}
#[inline]
pub(crate) fn fetch_or(&self, val: $value_type, _order: Ordering) -> $value_type {
let dst = self.v.get();
let out;
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
// cfg guarantee that the CPU supports RMW instructions.
unsafe {
asm!(
"las Z, {val}", // atomic { _x = *Z; *Z |= val; val = _x }
val = inout(reg) val => out,
in("Z") dst,
options(nostack, preserves_flags),
);
}
out
}
#[inline]
pub(crate) fn fetch_xor(&self, val: $value_type, _order: Ordering) -> $value_type {
let dst = self.v.get();
let out;
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
// cfg guarantee that the CPU supports RMW instructions.
unsafe {
asm!(
"lat Z, {val}", // atomic { _x = *Z; *Z ^= val; val = _x }
val = inout(reg) val => out,
in("Z") dst,
options(nostack, preserves_flags),
);
}
out
}
#[inline]
pub(crate) fn fetch_not(&self, order: Ordering) -> $value_type {
self.fetch_xor(!0, order)
}
}
};
}
atomic8!(AtomicI8, i8);
atomic8!(AtomicU8, u8);
vendor/portable-atomic/src/imp/core_atomic.rs
Vendored
+574
View File
@@ -0,0 +1,574 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Wrap the standard library's atomic types in newtype.
This is not a reexport, because we want to backport changes like
https://github.com/rust-lang/rust/pull/98383 to old compilers.
*/
use core::{cell::UnsafeCell, marker::PhantomData, sync::atomic::Ordering};
// core::panic::RefUnwindSafe is only available on Rust 1.56+, so on pre-1.56
// Rust, we implement RefUnwindSafe when "std" feature is enabled.
// However, on pre-1.56 Rust, the standard library's atomic types implement
// RefUnwindSafe when "linked to std", and that's behavior that our other atomic
// implementations can't emulate, so use PhantomData<NotRefUnwindSafe> to match
// conditions where our other atomic implementations implement RefUnwindSafe.
//
// If we do not do this, for example, downstream that is only tested on x86_64
// may incorrectly assume that AtomicU64 always implements RefUnwindSafe even on
// older rustc, and may be broken on platforms where std AtomicU64 is not available.
struct NotRefUnwindSafe(UnsafeCell<()>);
// SAFETY: this is a marker type and we'll never access the value.
unsafe impl Sync for NotRefUnwindSafe {}
#[repr(transparent)]
pub(crate) struct AtomicPtr<T> {
pub(super) inner: core::sync::atomic::AtomicPtr<T>,
// Prevent RefUnwindSafe from being propagated from the std atomic type. See NotRefUnwindSafe for more.
_not_ref_unwind_safe: PhantomData<NotRefUnwindSafe>,
}
impl<T> AtomicPtr<T> {
#[inline]
pub(crate) const fn new(v: *mut T) -> Self {
Self { inner: core::sync::atomic::AtomicPtr::new(v), _not_ref_unwind_safe: PhantomData }
}
#[inline]
pub(crate) fn is_lock_free() -> bool {
Self::IS_ALWAYS_LOCK_FREE
}
pub(crate) const IS_ALWAYS_LOCK_FREE: bool = true;
const_fn! {
const_if: #[cfg(not(portable_atomic_no_const_raw_ptr_deref))];
#[inline]
pub(crate) const fn as_ptr(&self) -> *mut *mut T {
// SAFETY: Self is #[repr(C)] and internally UnsafeCell<*mut T>.
// See also https://github.com/rust-lang/rust/pull/66705 and
// https://github.com/rust-lang/rust/issues/66136#issuecomment-557867116.
unsafe { (*(self as *const Self as *const UnsafeCell<*mut T>)).get() }
}
}
}
#[cfg(not(all(
target_arch = "arm",
not(any(miri, portable_atomic_sanitize_thread)),
any(target_os = "linux", target_os = "android"),
not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
)))]
impl<T> AtomicPtr<T> {
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn load(&self, order: Ordering) -> *mut T {
crate::utils::assert_load_ordering(order); // for track_caller (compiler can omit double check)
self.inner.load(order)
}
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn store(&self, ptr: *mut T, order: Ordering) {
crate::utils::assert_store_ordering(order); // for track_caller (compiler can omit double check)
self.inner.store(ptr, order);
}
}
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
items!({
#[cfg(not(all(
target_arch = "arm",
not(any(miri, portable_atomic_sanitize_thread)),
any(target_os = "linux", target_os = "android"),
not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
)))]
impl<T> AtomicPtr<T> {
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn compare_exchange(
&self,
current: *mut T,
new: *mut T,
success: Ordering,
failure: Ordering,
) -> Result<*mut T, *mut T> {
crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)
#[cfg(portable_atomic_no_stronger_failure_ordering)]
let success = crate::utils::upgrade_success_ordering(success, failure);
self.inner.compare_exchange(current, new, success, failure)
}
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn compare_exchange_weak(
&self,
current: *mut T,
new: *mut T,
success: Ordering,
failure: Ordering,
) -> Result<*mut T, *mut T> {
crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)
#[cfg(portable_atomic_no_stronger_failure_ordering)]
let success = crate::utils::upgrade_success_ordering(success, failure);
self.inner.compare_exchange_weak(current, new, success, failure)
}
}
// Ideally, we would always use AtomicPtr::fetch_* since it is strict-provenance
// compatible, but it requires 1.91+. So, for now emulate it only on cfg(miri).
// Code using AtomicUsize::fetch_* via casts is still permissive-provenance
// compatible and is sound.
#[cfg(portable_atomic_no_strict_provenance_atomic_ptr)]
impl<T> AtomicPtr<T> {
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_byte_add(&self, val: usize, order: Ordering) -> *mut T {
#[cfg(miri)]
{
#[cfg(portable_atomic_no_strict_provenance)]
use crate::utils::ptr::PtrExt as _;
self.fetch_update_(order, |x| x.with_addr(x.addr().wrapping_add(val)))
}
#[cfg(not(miri))]
{
crate::utils::ptr::with_exposed_provenance_mut(
self.as_atomic_usize().fetch_add(val, order),
)
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_byte_sub(&self, val: usize, order: Ordering) -> *mut T {
#[cfg(miri)]
{
#[cfg(portable_atomic_no_strict_provenance)]
use crate::utils::ptr::PtrExt as _;
self.fetch_update_(order, |x| x.with_addr(x.addr().wrapping_sub(val)))
}
#[cfg(not(miri))]
{
crate::utils::ptr::with_exposed_provenance_mut(
self.as_atomic_usize().fetch_sub(val, order),
)
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_or(&self, val: usize, order: Ordering) -> *mut T {
#[cfg(miri)]
{
#[cfg(portable_atomic_no_strict_provenance)]
use crate::utils::ptr::PtrExt as _;
self.fetch_update_(order, |x| x.with_addr(x.addr() | val))
}
#[cfg(not(miri))]
{
crate::utils::ptr::with_exposed_provenance_mut(
self.as_atomic_usize().fetch_or(val, order),
)
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_and(&self, val: usize, order: Ordering) -> *mut T {
#[cfg(miri)]
{
#[cfg(portable_atomic_no_strict_provenance)]
use crate::utils::ptr::PtrExt as _;
self.fetch_update_(order, |x| x.with_addr(x.addr() & val))
}
#[cfg(not(miri))]
{
crate::utils::ptr::with_exposed_provenance_mut(
self.as_atomic_usize().fetch_and(val, order),
)
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_xor(&self, val: usize, order: Ordering) -> *mut T {
#[cfg(miri)]
{
#[cfg(portable_atomic_no_strict_provenance)]
use crate::utils::ptr::PtrExt as _;
self.fetch_update_(order, |x| x.with_addr(x.addr() ^ val))
}
#[cfg(not(miri))]
{
crate::utils::ptr::with_exposed_provenance_mut(
self.as_atomic_usize().fetch_xor(val, order),
)
}
}
#[cfg(miri)]
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
fn fetch_update_<F>(&self, order: Ordering, mut f: F) -> *mut T
where
F: FnMut(*mut T) -> *mut T,
{
// This is a private function and all instances of `f` only operate on the value
// loaded, so there is no need to synchronize the first load/failed CAS.
let mut prev = self.load(Ordering::Relaxed);
loop {
let next = f(prev);
match self.compare_exchange_weak(prev, next, order, Ordering::Relaxed) {
Ok(x) => return x,
Err(next_prev) => prev = next_prev,
}
}
}
#[cfg(not(miri))]
#[inline(always)]
fn as_atomic_usize(&self) -> &AtomicUsize {
static_assert!(
core::mem::size_of::<AtomicPtr<()>>() == core::mem::size_of::<AtomicUsize>()
);
static_assert!(
core::mem::align_of::<AtomicPtr<()>>() == core::mem::align_of::<AtomicUsize>()
);
// SAFETY: AtomicPtr and AtomicUsize have the same layout,
// and both access data in the same way.
unsafe { &*(self as *const Self as *const AtomicUsize) }
}
}
#[cfg(not(all(
any(target_arch = "x86", target_arch = "x86_64"),
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
)))]
impl_default_bit_opts!(AtomicPtr, usize);
});
impl<T> core::ops::Deref for AtomicPtr<T> {
type Target = core::sync::atomic::AtomicPtr<T>;
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
fn deref(&self) -> &Self::Target {
&self.inner
}
}
// NB: Sync with atomic_int! macro in arm_linux.rs
macro_rules! atomic_int {
($atomic_type:ident, $int_type:ident) => {
#[repr(transparent)]
pub(crate) struct $atomic_type {
pub(super) inner: core::sync::atomic::$atomic_type,
// Prevent RefUnwindSafe from being propagated from the std atomic type. See NotRefUnwindSafe for more.
_not_ref_unwind_safe: PhantomData<NotRefUnwindSafe>,
}
impl $atomic_type {
#[inline]
pub(crate) const fn new(v: $int_type) -> Self {
Self {
inner: core::sync::atomic::$atomic_type::new(v),
_not_ref_unwind_safe: PhantomData,
}
}
#[inline]
pub(crate) fn is_lock_free() -> bool {
Self::IS_ALWAYS_LOCK_FREE
}
// ESP-IDF targets' 64-bit atomics are not lock-free.
// https://github.com/rust-lang/rust/pull/115577#issuecomment-1732259297
pub(crate) const IS_ALWAYS_LOCK_FREE: bool = cfg!(not(all(
any(target_arch = "riscv32", target_arch = "xtensa"),
target_os = "espidf",
))) | (core::mem::size_of::<$int_type>()
< 8);
const_fn! {
const_if: #[cfg(not(portable_atomic_no_const_raw_ptr_deref))];
#[inline]
pub(crate) const fn as_ptr(&self) -> *mut $int_type {
// SAFETY: Self is #[repr(C)] and internally UnsafeCell<$int_type>.
// See also https://github.com/rust-lang/rust/pull/66705 and
// https://github.com/rust-lang/rust/issues/66136#issuecomment-557867116.
unsafe {
(*(self as *const Self as *const UnsafeCell<$int_type>)).get()
}
}
}
}
#[cfg(not(all(
target_arch = "arm",
not(any(miri, portable_atomic_sanitize_thread)),
any(target_os = "linux", target_os = "android"),
not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
)))]
impl $atomic_type {
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn load(&self, order: Ordering) -> $int_type {
crate::utils::assert_load_ordering(order); // for track_caller (compiler can omit double check)
self.inner.load(order)
}
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn store(&self, val: $int_type, order: Ordering) {
crate::utils::assert_store_ordering(order); // for track_caller (compiler can omit double check)
self.inner.store(val, order);
}
}
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(not(portable_atomic_no_atomic_cas))
)]
#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
items!({
impl_default_no_fetch_ops!($atomic_type, $int_type);
#[cfg(not(all(
target_arch = "arm",
not(any(miri, portable_atomic_sanitize_thread)),
any(target_os = "linux", target_os = "android"),
not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
)))]
impl $atomic_type {
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn compare_exchange(
&self,
current: $int_type,
new: $int_type,
success: Ordering,
failure: Ordering,
) -> Result<$int_type, $int_type> {
crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)
#[cfg(portable_atomic_no_stronger_failure_ordering)]
let success = crate::utils::upgrade_success_ordering(success, failure);
self.inner.compare_exchange(current, new, success, failure)
}
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn compare_exchange_weak(
&self,
current: $int_type,
new: $int_type,
success: Ordering,
failure: Ordering,
) -> Result<$int_type, $int_type> {
crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)
#[cfg(portable_atomic_no_stronger_failure_ordering)]
let success = crate::utils::upgrade_success_ordering(success, failure);
self.inner.compare_exchange_weak(current, new, success, failure)
}
}
impl $atomic_type {
#[allow(dead_code)]
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
fn fetch_update_<F>(&self, order: Ordering, mut f: F) -> $int_type
where
F: FnMut($int_type) -> $int_type,
{
// This is a private function and all instances of `f` only operate on the value
// loaded, so there is no need to synchronize the first load/failed CAS.
let mut prev = self.load(Ordering::Relaxed);
loop {
let next = f(prev);
match self.compare_exchange_weak(prev, next, order, Ordering::Relaxed) {
Ok(x) => return x,
Err(next_prev) => prev = next_prev,
}
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type {
#[cfg(not(portable_atomic_no_atomic_min_max))]
{
#[cfg(any(
all(
any(target_arch = "aarch64", target_arch = "arm64ec"),
any(
target_feature = "lse",
portable_atomic_target_feature = "lse",
),
),
all(
target_arch = "arm",
not(any(
target_feature = "v6",
portable_atomic_target_feature = "v6",
)),
),
target_arch = "mips",
target_arch = "mips32r6",
target_arch = "mips64",
target_arch = "mips64r6",
target_arch = "powerpc",
target_arch = "powerpc64",
))]
{
// HACK: the following operations are currently broken (at least on qemu-user):
// - aarch64's `AtomicI{8,16}::fetch_{max,min}` (release mode + lse)
// - armv5te's `Atomic{I,U}{8,16}::fetch_{max,min}`
// - mips's `AtomicI8::fetch_{max,min}` (release mode)
// - mipsel's `AtomicI{8,16}::fetch_{max,min}` (debug mode, at least)
// - mips64's `AtomicI8::fetch_{max,min}` (release mode)
// - mips64el's `AtomicI{8,16}::fetch_{max,min}` (debug mode, at least)
// - powerpc's `AtomicI{8,16}::fetch_{max,min}`
// - powerpc64's `AtomicI{8,16}::fetch_{max,min}` (debug mode, at least)
// - powerpc64le's `AtomicU{8,16}::fetch_{max,min}` (release mode + fat LTO)
// See also:
// https://github.com/llvm/llvm-project/issues/61880
// https://github.com/llvm/llvm-project/issues/61881
// https://github.com/llvm/llvm-project/issues/61882
// https://github.com/taiki-e/portable-atomic/issues/2
// https://github.com/rust-lang/rust/issues/100650
if core::mem::size_of::<$int_type>() <= 2 {
return self.fetch_update_(order, |x| core::cmp::max(x, val));
}
}
self.inner.fetch_max(val, order)
}
#[cfg(portable_atomic_no_atomic_min_max)]
{
self.fetch_update_(order, |x| core::cmp::max(x, val))
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type {
#[cfg(not(portable_atomic_no_atomic_min_max))]
{
#[cfg(any(
all(
any(target_arch = "aarch64", target_arch = "arm64ec"),
any(
target_feature = "lse",
portable_atomic_target_feature = "lse",
),
),
all(
target_arch = "arm",
not(any(
target_feature = "v6",
portable_atomic_target_feature = "v6",
)),
),
target_arch = "mips",
target_arch = "mips32r6",
target_arch = "mips64",
target_arch = "mips64r6",
target_arch = "powerpc",
target_arch = "powerpc64",
))]
{
// HACK: the following operations are currently broken (at least on qemu-user):
// - aarch64's `AtomicI{8,16}::fetch_{max,min}` (release mode + lse)
// - armv5te's `Atomic{I,U}{8,16}::fetch_{max,min}`
// - mips's `AtomicI8::fetch_{max,min}` (release mode)
// - mipsel's `AtomicI{8,16}::fetch_{max,min}` (debug mode, at least)
// - mips64's `AtomicI8::fetch_{max,min}` (release mode)
// - mips64el's `AtomicI{8,16}::fetch_{max,min}` (debug mode, at least)
// - powerpc's `AtomicI{8,16}::fetch_{max,min}`
// - powerpc64's `AtomicI{8,16}::fetch_{max,min}` (debug mode, at least)
// - powerpc64le's `AtomicU{8,16}::fetch_{max,min}` (release mode + fat LTO)
// See also:
// https://github.com/llvm/llvm-project/issues/61880
// https://github.com/llvm/llvm-project/issues/61881
// https://github.com/llvm/llvm-project/issues/61882
// https://github.com/taiki-e/portable-atomic/issues/2
// https://github.com/rust-lang/rust/issues/100650
if core::mem::size_of::<$int_type>() <= 2 {
return self.fetch_update_(order, |x| core::cmp::min(x, val));
}
}
self.inner.fetch_min(val, order)
}
#[cfg(portable_atomic_no_atomic_min_max)]
{
self.fetch_update_(order, |x| core::cmp::min(x, val))
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_not(&self, order: Ordering) -> $int_type {
self.fetch_xor(!0, order)
}
// TODO: provide asm-based implementation on AArch64 without FEAT_LSE, Armv7, RISC-V, etc.
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_neg(&self, order: Ordering) -> $int_type {
self.fetch_update_(order, $int_type::wrapping_neg)
}
}
#[cfg(not(all(
any(target_arch = "x86", target_arch = "x86_64"),
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
)))]
items!({
impl_default_bit_opts!($atomic_type, $int_type);
impl $atomic_type {
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn not(&self, order: Ordering) {
self.fetch_not(order);
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn neg(&self, order: Ordering) {
self.fetch_neg(order);
}
}
});
});
impl core::ops::Deref for $atomic_type {
type Target = core::sync::atomic::$atomic_type;
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
fn deref(&self) -> &Self::Target {
&self.inner
}
}
};
}
atomic_int!(AtomicIsize, isize);
atomic_int!(AtomicUsize, usize);
#[cfg(not(portable_atomic_no_atomic_load_store))]
atomic_int!(AtomicI8, i8);
#[cfg(not(portable_atomic_no_atomic_load_store))]
atomic_int!(AtomicU8, u8);
#[cfg(not(portable_atomic_no_atomic_load_store))]
atomic_int!(AtomicI16, i16);
#[cfg(not(portable_atomic_no_atomic_load_store))]
atomic_int!(AtomicU16, u16);
#[cfg(not(portable_atomic_no_atomic_load_store))]
#[cfg(not(target_pointer_width = "16"))]
atomic_int!(AtomicI32, i32);
#[cfg(not(portable_atomic_no_atomic_load_store))]
#[cfg(not(target_pointer_width = "16"))]
atomic_int!(AtomicU32, u32);
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_64)))]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(any(
target_has_atomic = "64",
not(any(target_pointer_width = "16", target_pointer_width = "32")),
))
)]
items!({
atomic_int!(AtomicI64, i64);
atomic_int!(AtomicU64, u64);
});
vendor/portable-atomic/src/imp/detect/README.md
Vendored
+38
View File
@@ -0,0 +1,38 @@
# Run-time CPU feature detection
This module has run-time CPU feature detection implementations.
Here is the table of targets that support run-time CPU feature detection and the instruction or API used:
| target_arch | target_os/target_env | instruction/API | features | note |
| ----------- | -------------------- | --------------- | -------- | ---- |
| x86_64 | all (except for sgx) | cpuid | all | Enabled by default |
| aarch64 | linux (gnu/ohos/uclibc) | getauxval | all | Enabled by default (dlsym is used by default if needed for compatibility with older versions) |
| aarch64 | linux (musl) | getauxval | all | Only enabled by default when dynamic linking or `std` feature enabled (both disabled by default, see [auxv.rs](auxv.rs)) |
| aarch64 | android | getauxval | all | Enabled by default |
| aarch64 | freebsd | elf_aux_info | all | Enabled by default |
| aarch64 | netbsd | sysctlbyname | all | Enabled by default |
| aarch64 | openbsd | sysctl | all | Enabled by default |
| aarch64 | macos/ios/tvos/watchos/visionos | sysctlbyname | all | Enabled by default |
| aarch64 | illumos | getisax | lse, lse2 | Enabled by default |
| aarch64/arm64ec | windows | IsProcessorFeaturePresent | lse, lse2 | Enabled by default |
| aarch64 | fuchsia | zx_system_get_features | lse | Enabled by default |
| riscv32/riscv64 | linux/android | riscv_hwprobe | all | Enabled by default |
| powerpc64 | linux (gnu/ohos/uclibc) | getauxval | all | Enabled by default (dlsym is used by default if needed for compatibility with older versions) |
| powerpc64 | linux (musl) | getauxval | all | Only enabled by default when dynamic linking or `std` feature enabled (both disabled by default, see [auxv.rs](auxv.rs)) |
| powerpc64 | freebsd | elf_aux_info | all | Enabled by default (dlsym is used by default for compatibility with older versions) |
| powerpc64 | openbsd | elf_aux_info | all | Enabled by default (dlsym is used by default for compatibility with older versions) |
| powerpc64 | aix | getsystemcfg | all | Requires LLVM 20+. Disabled by default (see [powerpc64_aix.rs](powerpc64_aix.rs)) |
Run-time detection is enabled by default on most targets and can be disabled with `--cfg portable_atomic_no_outline_atomics`.
On some targets, run-time detection is disabled by default mainly for compatibility with incomplete build environments or support for it is experimental, and can be enabled by `--cfg portable_atomic_outline_atomics`. (When both cfg are enabled, `*_no_*` cfg is preferred.)
`dlsym` usually not working with static linking, so detection using implementations that use `dlsym` for compatibility will be disabled if static linking is enabled.
You can use `--cfg portable_atomic_outline_atomics` to force the use of non-`dlsym` implementations and enable run-time detection in such an environment.
For targets not included in the above table, run-time detection is always disabled and works the same as when `--cfg portable_atomic_no_outline_atomics` is set.
See [auxv.rs](auxv.rs) module-level comments for more details on Linux/Android/FreeBSD/OpenBSD.
See also [docs about `portable_atomic_no_outline_atomics` cfg](https://github.com/taiki-e/portable-atomic/blob/HEAD/README.md#optional-cfg-no-outline-atomics) in the top-level readme.
vendor/portable-atomic/src/imp/detect/aarch64_aa64reg.rs
Vendored
+492
View File
@@ -0,0 +1,492 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Run-time CPU feature detection on AArch64 Linux/Android/FreeBSD/NetBSD/OpenBSD by parsing system registers.
As of Rust 1.94, is_aarch64_feature_detected doesn't support run-time detection on NetBSD.
https://github.com/rust-lang/rust/blob/1.94.0/library/std_detect/src/detect/mod.rs
Run-time detection on OpenBSD by is_aarch64_feature_detected is supported on Rust 1.70+.
https://github.com/rust-lang/stdarch/pull/1374
Refs:
- https://developer.arm.com/documentation/ddi0601/2025-06/AArch64-Registers
- https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/arm64/cpu-feature-registers.rst
- https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/aarch64.rs
Supported platforms:
- Linux 4.11+ (emulate mrs instruction)
https://github.com/torvalds/linux/commit/77c97b4ee21290f5f083173d957843b615abbff2
- FreeBSD 12.0+ (emulate mrs instruction)
https://github.com/freebsd/freebsd-src/commit/398810619cb32abf349f8de23f29510b2ee0839b
- NetBSD 9.0+ (through sysctl/sysctlbyname)
https://github.com/NetBSD/src/commit/0e9d25528729f7fea53e78275d1bc5039dfe8ffb
sysctl/sysctlbyname returns an unsupported error if operation is not supported,
so we can safely use this on older versions.
- OpenBSD 7.1+ (through sysctl)
https://github.com/openbsd/src/commit/d335af936b9d7dd9cf655cae1ce19560c45de6c8
sysctl returns an unsupported error if operation is not supported,
so we can safely use this on older versions.
For now, this module is only used on NetBSD/OpenBSD.
On Linux/Android/FreeBSD, we use auxv.rs and this module is test-only because:
- On Linux/Android, this approach requires a higher kernel version than Rust supports,
and also does not work with qemu-user (as of QEMU 7.2) and Valgrind (as of Valgrind 3.24).
(Looking into HWCAP_CPUID in auxvec, it appears that Valgrind is setting it
to false correctly, but qemu-user is setting it to true.)
- qemu-user issue seem to be fixed as of QEMU 9.2.
- On FreeBSD, this approach does not work on FreeBSD 12 on QEMU (confirmed on
FreeBSD 12.{2,3,4}), and we got SIGILL (worked on FreeBSD 13 and 14).
*/
include!("common.rs");
#[cfg_attr(test, derive(Debug, PartialEq))]
struct AA64Reg {
aa64isar0: u64,
aa64isar1: u64,
#[cfg(test)]
aa64isar3: u64,
aa64mmfr2: u64,
}
#[cold]
#[must_use]
fn _detect(mut info: CpuInfo) -> CpuInfo {
let AA64Reg {
aa64isar0,
aa64isar1,
#[cfg(test)]
aa64isar3,
aa64mmfr2,
} = imp::aa64reg();
// ID_AA64ISAR0_EL1, AArch64 Instruction Set Attribute Register 0
// https://developer.arm.com/documentation/ddi0601/2025-06/AArch64-Registers/ID-AA64ISAR0-EL1--AArch64-Instruction-Set-Attribute-Register-0
// Atomic, bits [23:20]
// > FEAT_LSE implements the functionality identified by the value 0b0010.
// > FEAT_LSE128 implements the functionality identified by the value 0b0011.
// > From Armv8.1, the value 0b0000 is not permitted.
let atomic = extract(aa64isar0, 23, 20);
if atomic >= 0b0010 {
info.set(CpuInfoFlag::lse);
if atomic >= 0b0011 {
info.set(CpuInfoFlag::lse128);
}
}
// ID_AA64ISAR1_EL1, AArch64 Instruction Set Attribute Register 1
// https://developer.arm.com/documentation/ddi0601/2025-06/AArch64-Registers/ID-AA64ISAR1-EL1--AArch64-Instruction-Set-Attribute-Register-1
// LRCPC, bits [23:20]
// > FEAT_LRCPC implements the functionality identified by the value 0b0001.
// > FEAT_LRCPC2 implements the functionality identified by the value 0b0010.
// > FEAT_LRCPC3 implements the functionality identified by the value 0b0011.
// > From Armv8.3, the value 0b0000 is not permitted.
// > From Armv8.4, the value 0b0001 is not permitted.
let lrcpc = extract(aa64isar1, 23, 20);
if lrcpc >= 0b0011 {
info.set(CpuInfoFlag::rcpc3);
}
#[cfg(test)]
if lrcpc >= 0b0001 {
info.set(CpuInfoFlag::rcpc);
if lrcpc >= 0b0010 {
info.set(CpuInfoFlag::rcpc2);
}
}
// LS64, bits [63:60]
// > FEAT_LS64 implements the functionality identified by 0b0001.
// > FEAT_LS64_V implements the functionality identified by 0b0010.
// > FEAT_LS64_ACCDATA implements the functionality identified by 0b0011.
// > FEAT_LS64WB implements the functionality identified by 0b0100.
#[cfg(test)]
if extract(aa64isar1, 63, 60) >= 0b0100 {
info.set(CpuInfoFlag::ls64wb);
}
// ID_AA64ISAR3_EL1, AArch64 Instruction Set Attribute Register 3
// https://developer.arm.com/documentation/ddi0601/2025-06/AArch64-Registers/ID-AA64ISAR3-EL1--AArch64-Instruction-Set-Attribute-Register-3
// LSFE, bits [19:16]
// > FEAT_LSFE implements the functionality identified by the value 0b0001
#[cfg(test)]
if extract(aa64isar3, 19, 16) >= 0b0001 {
info.set(CpuInfoFlag::lsfe);
}
// ID_AA64MMFR2_EL1, AArch64 Memory Model Feature Register 2
// https://developer.arm.com/documentation/ddi0601/2025-06/AArch64-Registers/ID-AA64MMFR2-EL1--AArch64-Memory-Model-Feature-Register-2
// AT, bits [35:32]
// > FEAT_LSE2 implements the functionality identified by the value 0b0001.
// > From Armv8.4, the value 0b0000 is not permitted.
if extract(aa64mmfr2, 35, 32) >= 0b0001 {
info.set(CpuInfoFlag::lse2);
}
info
}
fn extract(x: u64, high: usize, low: usize) -> u64 {
(x >> low) & ((1 << (high - low + 1)) - 1)
}
#[cfg(not(any(target_os = "netbsd", target_os = "openbsd")))]
mod imp {
// This module is test-only. See parent module docs for details.
#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
use super::AA64Reg;
pub(super) fn aa64reg() -> AA64Reg {
// SAFETY: This is safe on FreeBSD 12.0+. FreeBSD 11 was EoL on 2021-09-30.
// Note that stdarch has been doing the same thing since before FreeBSD 11 was EoL.
// https://github.com/rust-lang/stdarch/pull/611
unsafe {
let aa64isar0: u64;
asm!(
"mrs {}, ID_AA64ISAR0_EL1",
out(reg) aa64isar0,
options(pure, nomem, nostack, preserves_flags),
);
let aa64isar1: u64;
asm!(
"mrs {}, ID_AA64ISAR1_EL1",
out(reg) aa64isar1,
options(pure, nomem, nostack, preserves_flags),
);
#[cfg(test)]
#[cfg(not(portable_atomic_pre_llvm_18))]
let aa64isar3: u64;
// ID_AA64ISAR3_EL1 is only recognized on LLVM 18+.
// https://github.com/llvm/llvm-project/commit/17baba9fa2728b1b1134f9dccb9318debd5a9a1b
#[cfg(test)]
#[cfg(not(portable_atomic_pre_llvm_18))]
asm!(
"mrs {}, ID_AA64ISAR3_EL1",
out(reg) aa64isar3,
options(pure, nomem, nostack, preserves_flags),
);
let aa64mmfr2: u64;
asm!(
"mrs {}, ID_AA64MMFR2_EL1",
out(reg) aa64mmfr2,
options(pure, nomem, nostack, preserves_flags),
);
AA64Reg {
aa64isar0,
aa64isar1,
#[cfg(test)]
#[cfg(not(portable_atomic_pre_llvm_18))]
aa64isar3,
#[cfg(test)]
#[cfg(portable_atomic_pre_llvm_18)]
aa64isar3: 0,
aa64mmfr2,
}
}
}
}
#[cfg(target_os = "netbsd")]
mod imp {
// NetBSD doesn't trap the mrs instruction, but exposes the system registers through sysctl.
// https://github.com/NetBSD/src/commit/0e9d25528729f7fea53e78275d1bc5039dfe8ffb
// https://github.com/golang/sys/commit/ef9fd89ba245e184bdd308f7f2b4f3c551fa5b0f
use core::{mem, ptr};
use super::AA64Reg;
// libc requires Rust 1.63
#[allow(non_camel_case_types)]
pub(super) mod ffi {
pub(crate) use crate::utils::ffi::{CStr, c_char, c_int, c_size_t, c_void};
sys_struct!({
// Defined in machine/armreg.h.
// https://github.com/NetBSD/src/blob/121914f187d0f46c2ce43f00531d2c500d8e81e5/sys/arch/aarch64/include/armreg.h#L1863
pub(crate) struct aarch64_sysctl_cpu_id {
// NetBSD 9.0+
// https://github.com/NetBSD/src/commit/0e9d25528729f7fea53e78275d1bc5039dfe8ffb
pub(crate) ac_midr: u64,
pub(crate) ac_revidr: u64,
pub(crate) ac_mpidr: u64,
pub(crate) ac_aa64dfr0: u64,
pub(crate) ac_aa64dfr1: u64,
pub(crate) ac_aa64isar0: u64,
pub(crate) ac_aa64isar1: u64,
pub(crate) ac_aa64mmfr0: u64,
pub(crate) ac_aa64mmfr1: u64,
pub(crate) ac_aa64mmfr2: u64,
pub(crate) ac_aa64pfr0: u64,
pub(crate) ac_aa64pfr1: u64,
pub(crate) ac_aa64zfr0: u64,
pub(crate) ac_mvfr0: u32,
pub(crate) ac_mvfr1: u32,
pub(crate) ac_mvfr2: u32,
// NetBSD 10.0+
// https://github.com/NetBSD/src/commit/0c7bdc13f0e332cccec56e307f023b4888638973
pub(crate) ac_pad: u32,
pub(crate) ac_clidr: u64,
pub(crate) ac_ctr: u64,
}
});
sys_fn!({
extern "C" {
// Defined in sys/sysctl.h.
// https://man.netbsd.org/sysctl.3
// https://github.com/NetBSD/src/blob/121914f187d0f46c2ce43f00531d2c500d8e81e5/sys/sys/sysctl.h
pub(crate) fn sysctlbyname(
name: *const c_char,
old_p: *mut c_void,
old_len_p: *mut c_size_t,
new_p: *const c_void,
new_len: c_size_t,
) -> c_int;
}
});
}
pub(super) fn sysctl_cpu_id(name: &ffi::CStr) -> Option<AA64Reg> {
const OUT_LEN: ffi::c_size_t =
mem::size_of::<ffi::aarch64_sysctl_cpu_id>() as ffi::c_size_t;
// SAFETY: all fields of aarch64_sysctl_cpu_id are zero-able and we use
// the result when machdep.cpuN.cpu_id sysctl was successful.
let mut buf: ffi::aarch64_sysctl_cpu_id = unsafe { mem::zeroed() };
let mut out_len = OUT_LEN;
// SAFETY:
// - `name` a valid C string.
// - `out_len` does not exceed the size of the value at `buf`.
// - `sysctlbyname` is thread-safe.
let res = unsafe {
ffi::sysctlbyname(
name.as_ptr(),
(&mut buf as *mut ffi::aarch64_sysctl_cpu_id).cast::<ffi::c_void>(),
&mut out_len,
ptr::null_mut(),
0,
)
};
if res != 0 {
return None;
}
Some(AA64Reg {
aa64isar0: buf.ac_aa64isar0,
aa64isar1: buf.ac_aa64isar1,
#[cfg(test)]
aa64isar3: 0,
aa64mmfr2: buf.ac_aa64mmfr2,
})
}
pub(super) fn aa64reg() -> AA64Reg {
// Get system registers for cpu0.
// If failed, returns default because machdep.cpuN.cpu_id sysctl is not available.
// machdep.cpuN.cpu_id sysctl was added in NetBSD 9.0 so it is not available on older versions.
// It is ok to check only cpu0, even if there are more CPUs.
// https://github.com/NetBSD/src/commit/bd9707e06ea7d21b5c24df6dfc14cb37c2819416
// https://github.com/golang/sys/commit/ef9fd89ba245e184bdd308f7f2b4f3c551fa5b0f
match sysctl_cpu_id(c!("machdep.cpu0.cpu_id")) {
Some(cpu_id) => cpu_id,
None => AA64Reg {
aa64isar0: 0,
aa64isar1: 0,
#[cfg(test)]
aa64isar3: 0,
aa64mmfr2: 0,
},
}
}
}
#[cfg(target_os = "openbsd")]
mod imp {
// OpenBSD doesn't trap the mrs instruction, but exposes the system registers through sysctl.
// https://github.com/openbsd/src/commit/d335af936b9d7dd9cf655cae1ce19560c45de6c8
// https://github.com/golang/go/commit/cd54ef1f61945459486e9eea2f016d99ef1da925
use core::{mem, ptr};
use super::AA64Reg;
// libc requires Rust 1.63
pub(super) mod ffi {
pub(crate) use crate::utils::ffi::{c_int, c_size_t, c_uint, c_void};
sys_const!({
// Defined in sys/sysctl.h.
// https://github.com/openbsd/src/blob/d2e81eb18e50973c87ae2c9c2ba348ff3ff89cf9/sys/sys/sysctl.h
pub(crate) const CTL_MACHDEP: c_int = 7;
// Defined in machine/cpu.h.
// https://github.com/openbsd/src/blob/d2e81eb18e50973c87ae2c9c2ba348ff3ff89cf9/sys/arch/arm64/include/cpu.h
// OpenBSD 7.1+
// https://github.com/openbsd/src/commit/d335af936b9d7dd9cf655cae1ce19560c45de6c8
pub(crate) const CPU_ID_AA64ISAR0: c_int = 2;
pub(crate) const CPU_ID_AA64ISAR1: c_int = 3;
// OpenBSD 7.3+
// https://github.com/openbsd/src/commit/c7654cd65262d532212f65123ee3905ba200365c
// However, on OpenBSD 7.3-7.5, querying CPU_ID_AA64MMFR2 always returns 0.
// https://github.com/openbsd/src/commit/e8331b74e5c20302d4bd948c9db722af688ccfc1
pub(crate) const CPU_ID_AA64MMFR2: c_int = 7;
});
sys_fn!({
extern "C" {
// Defined in sys/sysctl.h.
// https://man.openbsd.org/sysctl.2
// https://github.com/openbsd/src/blob/d2e81eb18e50973c87ae2c9c2ba348ff3ff89cf9/sys/sys/sysctl.h
pub(crate) fn sysctl(
name: *const c_int,
name_len: c_uint,
old_p: *mut c_void,
old_len_p: *mut c_size_t,
new_p: *mut c_void,
new_len: c_size_t,
) -> c_int;
}
});
}
// sysctl returns an unsupported error if operation is not supported,
// so we can safely use this function on older versions of OpenBSD.
pub(super) fn aa64reg() -> AA64Reg {
let aa64isar0 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64ISAR0]).unwrap_or(0);
let aa64isar1 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64ISAR1]).unwrap_or(0);
let aa64mmfr2 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64MMFR2]).unwrap_or(0);
AA64Reg {
aa64isar0,
aa64isar1,
#[cfg(test)]
aa64isar3: 0,
aa64mmfr2,
}
}
fn sysctl64(mib: &[ffi::c_int]) -> Option<u64> {
const OUT_LEN: ffi::c_size_t = mem::size_of::<u64>() as ffi::c_size_t;
let mut out = 0_u64;
let mut out_len = OUT_LEN;
#[allow(clippy::cast_possible_truncation)]
let mib_len = mib.len() as ffi::c_uint;
// SAFETY:
// - `mib_len` does not exceed the size of `mib`.
// - `out_len` does not exceed the size of `out`.
// - `sysctl` is thread-safe.
let res = unsafe {
ffi::sysctl(
mib.as_ptr(),
mib_len,
(&mut out as *mut u64).cast::<ffi::c_void>(),
&mut out_len,
ptr::null_mut(),
0,
)
};
// OpenBSD sysctl returns -1 on failure.
if res == -1 {
return None;
}
debug_assert_eq!(out_len, OUT_LEN);
Some(out)
}
}
#[allow(
clippy::alloc_instead_of_core,
clippy::std_instead_of_alloc,
clippy::std_instead_of_core,
clippy::undocumented_unsafe_blocks,
clippy::wildcard_imports
)]
#[cfg(test)]
mod tests {
use super::*;
#[test]
#[cfg_attr(portable_atomic_test_detect_false, ignore = "detection disabled")]
fn test_aa64reg() {
let AA64Reg { aa64isar0, aa64isar1, aa64isar3, aa64mmfr2 } = imp::aa64reg();
test_helper::eprintln_nocapture!(
"aa64isar0={},aa64isar1={},aa64isar3={},aa64mmfr2={}",
aa64isar0,
aa64isar1,
aa64isar3,
aa64mmfr2,
);
let atomic = extract(aa64isar0, 23, 20);
if detect().lse() {
if detect().lse128() {
assert_eq!(atomic, 0b0011);
} else {
assert_eq!(atomic, 0b0010);
}
} else {
assert_eq!(atomic, 0b0000);
}
let lrcpc = extract(aa64isar1, 23, 20);
if detect().rcpc() {
if detect().rcpc2() {
if detect().rcpc3() {
assert_eq!(lrcpc, 0b0011);
} else {
assert_eq!(lrcpc, 0b0010);
}
} else {
assert_eq!(lrcpc, 0b0001);
}
} else {
assert_eq!(lrcpc, 0b0000);
}
let ls64 = extract(aa64isar1, 63, 60);
if detect().ls64wb() {
assert_eq!(ls64, 0b0100);
} else {
assert!(ls64 < 0b0100, "{}", ls64);
}
let lsfe = extract(aa64isar3, 19, 16);
if detect().lsfe() {
assert_eq!(lsfe, 0b0001);
} else {
assert_eq!(lsfe, 0b0000);
}
let at = extract(aa64mmfr2, 35, 32);
if detect().lse2() {
assert_eq!(at, 0b0001);
} else {
assert_eq!(at, 0b0000);
}
}
#[cfg(target_os = "openbsd")]
#[test]
fn test_alternative() {
use std::{format, process::Command, string::String};
// Call sysctl command instead of libc API.
//
// This is used only for testing.
struct SysctlMachdepOutput(String);
impl SysctlMachdepOutput {
fn new() -> Self {
let output = Command::new("sysctl").arg("machdep").output().unwrap();
assert!(output.status.success());
let stdout = String::from_utf8(output.stdout).unwrap();
Self(stdout)
}
fn field(&self, name: &str) -> Option<u64> {
Some(
self.0
.lines()
.find_map(|s| s.strip_prefix(&format!("{}=", name)))?
.parse()
.unwrap(),
)
}
}
let AA64Reg { aa64isar0, aa64isar1, aa64isar3, aa64mmfr2 } = imp::aa64reg();
let sysctl_output = SysctlMachdepOutput::new();
assert_eq!(aa64isar0, sysctl_output.field("machdep.id_aa64isar0").unwrap_or(0));
assert_eq!(aa64isar1, sysctl_output.field("machdep.id_aa64isar1").unwrap_or(0));
assert_eq!(aa64isar3, sysctl_output.field("machdep.id_aa64isar3").unwrap_or(0));
assert_eq!(aa64mmfr2, sysctl_output.field("machdep.id_aa64mmfr2").unwrap_or(0));
}
}
vendor/portable-atomic/src/imp/detect/aarch64_apple.rs
Vendored
+217
View File
@@ -0,0 +1,217 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Run-time CPU feature detection on AArch64 Apple targets by using sysctlbyname.
On macOS, this module is currently only enabled on tests because there are no
instructions that were not available on the M1 but are now available on the
latest Apple hardware and this library currently wants to use:
```console
$ LC_ALL=C comm -23 <(rustc --print cfg --target aarch64-apple-darwin -C target-cpu=apple-m5 | grep -F target_feature) <(rustc --print cfg --target aarch64-apple-darwin | grep -F target_feature)
target_feature="bf16"
target_feature="bti"
target_feature="cssc"
target_feature="ecv"
target_feature="hbc"
target_feature="i8mm"
target_feature="mte"
target_feature="sme"
target_feature="sme-b16b16"
target_feature="sme-f16f16"
target_feature="sme-f64f64"
target_feature="sme-i16i64"
target_feature="sme2"
target_feature="sme2p1"
target_feature="sve-b16b16"
target_feature="v8.5a"
target_feature="v8.6a"
target_feature="v8.7a"
target_feature="wfxt"
```
Non-macOS targets doesn't always supports FEAT_LSE2, so we use this module on them.
As used in the example in https://developer.apple.com/documentation/xcode/writing-arm64-code-for-apple-platforms#Enable-DIT-for-constant-time-cryptographic-operations,
sysctlbyname is supported on all Apple platforms.
Refs: https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics
*/
include!("common.rs");
use core::{mem, ptr};
// libc requires Rust 1.63
mod ffi {
pub(crate) use crate::utils::ffi::{CStr, c_char, c_int, c_size_t, c_void};
#[cfg(test)] // test-only
sys_const!({
// https://github.com/apple-oss-distributions/xnu/blob/xnu-12377.121.6/osfmk/arm/cpu_capabilities_public.h
pub(crate) const CAP_BIT_FEAT_LSE: usize = 6;
pub(crate) const CAP_BIT_FEAT_LRCPC: usize = 15;
pub(crate) const CAP_BIT_FEAT_LRCPC2: usize = 16;
pub(crate) const CAP_BIT_FEAT_LSE2: usize = 30;
pub(crate) const CAP_BIT_NB: usize = 92;
});
sys_fn!({
extern "C" {
// https://developer.apple.com/documentation/kernel/1387446-sysctlbyname
// https://github.com/apple-oss-distributions/xnu/blob/xnu-12377.121.6/bsd/sys/sysctl.h
pub(crate) fn sysctlbyname(
name: *const c_char,
old_p: *mut c_void,
old_len_p: *mut c_size_t,
new_p: *mut c_void,
new_len: c_size_t,
) -> c_int;
}
});
}
fn sysctlbyname32(name: &ffi::CStr) -> Option<u32> {
const OUT_LEN: ffi::c_size_t = mem::size_of::<u32>() as ffi::c_size_t;
let mut out = 0_u32;
let mut out_len = OUT_LEN;
// SAFETY:
// - `name` a valid C string.
// - `out_len` does not exceed the size of `out`.
// - `sysctlbyname` is thread-safe.
let res = unsafe {
ffi::sysctlbyname(
name.as_ptr(),
(&mut out as *mut u32).cast::<ffi::c_void>(),
&mut out_len,
ptr::null_mut(),
0,
)
};
if res != 0 {
return None;
}
debug_assert_eq!(out_len, OUT_LEN);
Some(out)
}
#[cold]
#[must_use]
fn _detect(mut info: CpuInfo) -> CpuInfo {
macro_rules! check {
($flag:ident, $($name:tt) ||+) => {
if $(sysctlbyname32(c!($name)).unwrap_or(0) != 0) ||+ {
info.set(CpuInfoFlag::$flag);
}
};
}
// On macOS, AArch64 support was added in macOS 11,
// hw.optional.armv8_1_atomics is available on macOS 11+,
// hw.optional.arm.FEAT_* are only available on macOS 12+.
// Query both names in case future versions of macOS remove the old name.
// https://github.com/golang/go/commit/c15593197453b8bf90fc3a9080ba2afeaf7934ea
// https://github.com/google/boringssl/commit/91e0b11eba517d83b910b20fe3740eeb39ecb37e
// hw.optional.arm.caps is available on macOS 15+.
// https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc#L56
check!(lse, "hw.optional.arm.FEAT_LSE" || "hw.optional.armv8_1_atomics");
check!(lse2, "hw.optional.arm.FEAT_LSE2");
#[cfg(test)]
check!(rcpc, "hw.optional.arm.FEAT_LRCPC");
#[cfg(test)]
check!(rcpc2, "hw.optional.arm.FEAT_LRCPC2");
info
}
#[allow(
clippy::alloc_instead_of_core,
clippy::std_instead_of_alloc,
clippy::std_instead_of_core,
clippy::undocumented_unsafe_blocks,
clippy::wildcard_imports
)]
#[cfg(test)]
mod tests {
use std::{format, process::Command, str, string::String};
use super::*;
#[test]
fn test_alternative() {
use crate::utils::ffi::*;
const CAPS_LEN: usize = (ffi::CAP_BIT_NB + 63) / 64;
fn arm_caps(caps: &mut [u64; CAPS_LEN]) -> bool {
let mut caps_len = CAPS_LEN * 8;
// SAFETY:
// - name a valid C string.
// - `caps_len` does not exceed the size of `caps`.
// - `sysctlbyname` is thread-safe.
let res = unsafe {
ffi::sysctlbyname(
c!("hw.optional.arm.caps").as_ptr(),
caps.as_mut_ptr().cast::<ffi::c_void>(),
&mut caps_len,
ptr::null_mut(),
0,
)
};
res == 0
}
// Call sysctl command instead of libc API.
//
// This is used only for testing.
struct SysctlHwOptionalOutput(String);
impl SysctlHwOptionalOutput {
fn new() -> Self {
let output = Command::new("sysctl").arg("hw.optional").output().unwrap();
assert!(output.status.success());
let stdout = String::from_utf8(output.stdout).unwrap();
test_helper::eprintln_nocapture!("sysctl hw.optional:\n{}", stdout);
Self(stdout)
}
fn field(&self, name: &CStr) -> Option<u32> {
let name = name.to_bytes_with_nul();
let name = str::from_utf8(&name[..name.len() - 1]).unwrap();
let prefix = format!("{}: ", name);
Some(self.0.lines().find_map(|s| s.strip_prefix(&prefix))?.parse().unwrap())
}
}
let mut caps = [0_u64; CAPS_LEN];
let has_caps = arm_caps(&mut caps);
let sysctl_output = SysctlHwOptionalOutput::new();
for (name, expected_on_macos, cap_bit) in [
(c!("hw.optional.arm.FEAT_LSE"), Some(1), ffi::CAP_BIT_FEAT_LSE),
(c!("hw.optional.armv8_1_atomics"), Some(1), ffi::CAP_BIT_FEAT_LSE),
(c!("hw.optional.arm.FEAT_LSE2"), Some(1), ffi::CAP_BIT_FEAT_LSE2),
(c!("hw.optional.arm.FEAT_LSE128"), None, 0),
(c!("hw.optional.arm.FEAT_LSFE"), None, 0),
(c!("hw.optional.arm.FEAT_LRCPC"), Some(1), ffi::CAP_BIT_FEAT_LRCPC),
(c!("hw.optional.arm.FEAT_LRCPC2"), Some(1), ffi::CAP_BIT_FEAT_LRCPC2),
(c!("hw.optional.arm.FEAT_LRCPC3"), None, 0),
] {
let res = sysctlbyname32(name);
if res.is_none() {
assert_eq!(std::io::Error::last_os_error().kind(), std::io::ErrorKind::NotFound);
}
if cfg!(any(target_os = "macos", target_abi = "macabi")) {
assert_eq!(
res,
expected_on_macos,
"{}",
str::from_utf8(name.to_bytes_with_nul()).unwrap()
);
}
if let Some(res) = res {
assert_eq!(res, sysctl_output.field(name).unwrap());
if has_caps {
assert_eq!(caps[cap_bit / 64] & (1 << (cap_bit & 63)) != 0, res != 0);
}
} else {
assert!(sysctl_output.field(name).is_none());
}
}
}
}
vendor/portable-atomic/src/imp/detect/aarch64_fuchsia.rs
Vendored
+85
View File
@@ -0,0 +1,85 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Run-time CPU feature detection on AArch64 Fuchsia by using zx_system_get_features.
As of Rust 1.94, is_aarch64_feature_detected doesn't support run-time detection on Fuchsia.
https://github.com/rust-lang/rust/blob/1.94.0/library/std_detect/src/detect/mod.rs
Refs:
- https://fuchsia.dev/reference/syscalls/system_get_features
- https://github.com/llvm/llvm-project/commit/4e731abc55681751b5d736b613f7720e50eb1ad4
*/
include!("common.rs");
#[allow(non_camel_case_types)]
mod ffi {
sys_type!({
// https://fuchsia.googlesource.com/fuchsia/+/refs/heads/main/zircon/system/public/zircon/types.h
pub(crate) type zx_status_t = i32;
});
sys_const!({
// https://fuchsia.googlesource.com/fuchsia/+/refs/heads/main/zircon/system/public/zircon/errors.h
pub(crate) const ZX_OK: zx_status_t = 0;
// https://fuchsia.googlesource.com/fuchsia/+/refs/heads/main/zircon/system/public/zircon/features.h
pub(crate) const ZX_FEATURE_KIND_CPU: u32 = 0;
pub(crate) const ZX_ARM64_FEATURE_ISA_ATOMICS: u32 = 1 << 8;
});
// TODO: use sys_fn!
#[link(name = "zircon")]
extern "C" {
// https://fuchsia.dev/reference/syscalls/system_get_features
pub(crate) fn zx_system_get_features(kind: u32, features: *mut u32) -> zx_status_t;
}
}
fn zx_system_get_features(kind: u32) -> u32 {
let mut out = 0_u32;
// SAFETY: the pointer is valid because we got it from a reference.
let res = unsafe { ffi::zx_system_get_features(kind, &mut out) };
if res != ffi::ZX_OK {
return 0;
}
out
}
#[cold]
#[must_use]
fn _detect(mut info: CpuInfo) -> CpuInfo {
let features = zx_system_get_features(ffi::ZX_FEATURE_KIND_CPU);
macro_rules! check {
($flag:ident, $bit:ident) => {
if features & ffi::$bit != 0 {
info.set(CpuInfoFlag::$flag);
}
};
}
check!(lse, ZX_ARM64_FEATURE_ISA_ATOMICS);
info
}
#[allow(
clippy::alloc_instead_of_core,
clippy::std_instead_of_alloc,
clippy::std_instead_of_core,
clippy::undocumented_unsafe_blocks,
clippy::wildcard_imports
)]
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_fuchsia() {
let features = zx_system_get_features(ffi::ZX_FEATURE_KIND_CPU);
test_helper::eprintln_nocapture!(
"zx_system_get_features(ZX_FEATURE_KIND_CPU): {:b}",
features
);
assert_ne!(features, 0);
}
}
vendor/portable-atomic/src/imp/detect/aarch64_illumos.rs
Vendored
+62
View File
@@ -0,0 +1,62 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Run-time CPU feature detection on AArch64 illumos by using getisax.
As of Rust 1.94, is_aarch64_feature_detected doesn't support run-time detection on illumos.
https://github.com/rust-lang/rust/blob/1.94.0/library/std_detect/src/detect/mod.rs
*/
include!("common.rs");
// libc requires Rust 1.63
mod ffi {
pub(crate) use crate::utils::ffi::c_uint;
sys_const!({
// Defined in sys/auxv_aarch64.h.
// https://github.com/richlowe/illumos-gate/blob/arm64-gate/usr/src/uts/common/sys/auxv_aarch64.h
pub(crate) const AV_AARCH64_LSE: u32 = 1 << 15;
#[cfg(test)]
pub(crate) const AV_AARCH64_LRCPC: u32 = 1 << 28;
#[cfg(test)]
pub(crate) const AV_AARCH64_2_ILRCPC: u32 = 1 << 1;
pub(crate) const AV_AARCH64_2_LSE2: u32 = 1 << 2;
});
sys_fn!({
extern "C" {
// Defined in sys/auxv.h.
// https://illumos.org/man/2/getisax
// https://github.com/richlowe/illumos-gate/blob/arm64-gate/usr/src/uts/common/sys/auxv.h
pub(crate) fn getisax(array: *mut u32, n: c_uint) -> c_uint;
}
});
}
#[cold]
#[must_use]
fn _detect(mut info: CpuInfo) -> CpuInfo {
const OUT_LEN: ffi::c_uint = 2;
let mut out = [0_u32; OUT_LEN as usize];
// SAFETY: the pointer is valid because we got it from a reference.
unsafe {
ffi::getisax(out.as_mut_ptr(), OUT_LEN);
}
macro_rules! check {
($x:ident, $flag:ident, $bit:ident) => {
if $x & ffi::$bit != 0 {
info.set(CpuInfoFlag::$flag);
}
};
}
let v1 = out[0];
check!(v1, lse, AV_AARCH64_LSE);
#[cfg(test)]
check!(v1, rcpc, AV_AARCH64_LRCPC);
let v2 = out[1];
#[cfg(test)]
check!(v2, rcpc2, AV_AARCH64_2_ILRCPC);
check!(v2, lse2, AV_AARCH64_2_LSE2);
info
}
vendor/portable-atomic/src/imp/detect/aarch64_windows.rs
Vendored
+62
View File
@@ -0,0 +1,62 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Run-time CPU feature detection on AArch64 Windows by using IsProcessorFeaturePresent.
Run-time detection of FEAT_LSE on Windows by is_aarch64_feature_detected is supported on Rust 1.70+.
https://github.com/rust-lang/stdarch/pull/1373
Refs: https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-isprocessorfeaturepresent
*/
include!("common.rs");
// windows-sys requires Rust 1.71
#[allow(non_camel_case_types, clippy::upper_case_acronyms)]
mod ffi {
sys_type!({
pub(crate) type [Win32::System::Threading] PROCESSOR_FEATURE_ID = u32;
pub(crate) type [core] BOOL = i32;
});
sys_const!({
pub(crate) const [Win32::Foundation] FALSE: BOOL = 0;
// Defined in winnt.h of Windows SDK.
pub(crate) const [Win32::System::Threading]
PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE: PROCESSOR_FEATURE_ID = 34;
#[cfg(test)]
pub(crate) const [Win32::System::Threading]
PF_ARM_V83_LRCPC_INSTRUCTIONS_AVAILABLE: PROCESSOR_FEATURE_ID = 45;
});
// TODO: put in sys_const! once windows-sys updated.
pub(crate) const PF_ARM_LSE2_AVAILABLE: PROCESSOR_FEATURE_ID = 62;
sys_fn!({
extern "system" {
// https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-isprocessorfeaturepresent
pub(crate) fn [Win32::System::Threading] IsProcessorFeaturePresent(
ProcessorFeature: PROCESSOR_FEATURE_ID,
) -> BOOL;
}
});
}
#[cold]
#[must_use]
fn _detect(mut info: CpuInfo) -> CpuInfo {
macro_rules! check {
($flag:ident, $bit:ident) => {
// SAFETY: calling IsProcessorFeaturePresent is safe, and FALSE is also
// returned if the HAL does not support detection of the specified feature.
if unsafe { ffi::IsProcessorFeaturePresent(ffi::$bit) != ffi::FALSE } {
info.set(CpuInfoFlag::$flag);
}
};
}
check!(lse, PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE);
check!(lse2, PF_ARM_LSE2_AVAILABLE);
#[cfg(test)]
check!(rcpc, PF_ARM_V83_LRCPC_INSTRUCTIONS_AVAILABLE);
info
}
vendor/portable-atomic/src/imp/detect/auxv.rs
Vendored
+1083
View File
File diff suppressed because it is too large. Load diff
vendor/portable-atomic/src/imp/detect/common.rs
Vendored
+303
View File
@@ -0,0 +1,303 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
#[derive(Clone, Copy)]
#[repr(transparent)]
pub(crate) struct CpuInfo(u32);
impl CpuInfo {
#[inline]
fn set(&mut self, bit: CpuInfoFlag) {
self.0 = set(self.0, bit as u32);
}
#[inline]
#[must_use]
fn test(self, bit: CpuInfoFlag) -> bool {
test(self.0, bit as u32)
}
}
#[inline]
#[must_use]
fn set(x: u32, bit: u32) -> u32 {
x | (1 << bit)
}
#[inline]
#[must_use]
fn test(x: u32, bit: u32) -> bool {
x & (1 << bit) != 0
}
#[inline]
pub(crate) fn detect() -> CpuInfo {
use core::sync::atomic::{AtomicU32, Ordering};
static CACHE: AtomicU32 = AtomicU32::new(0);
let mut info = CpuInfo(CACHE.load(Ordering::Relaxed));
if info.0 != 0 {
return info;
}
info.set(CpuInfoFlag::Init);
// Note: detect_false cfg is intended to make it easy for developers to test
// cases where features usually available is not available, and is not a public API.
if !cfg!(portable_atomic_test_detect_false) {
info = _detect(info);
}
if cfg!(any(
all(
target_arch = "x86_64",
not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
),
all(
target_arch = "powerpc64",
not(any(
target_feature = "quadword-atomics",
portable_atomic_target_feature = "quadword-atomics",
)),
),
all(
any(target_arch = "riscv32", target_arch = "riscv64"),
not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")),
),
)) {
// Use CAS if the detection result affects lock-freeness.
match CACHE.compare_exchange(0, info.0, Ordering::Relaxed, Ordering::Relaxed) {
Ok(_) => info,
Err(info) => {
debug_assert_ne!(info, 0);
CpuInfo(info)
}
}
} else {
CACHE.store(info.0, Ordering::Relaxed);
info
}
}
macro_rules! flags {
($(
$(#[$attr:meta])*
$func:ident($name:literal, any($($cfg:ident),*)),
)*) => {
#[allow(dead_code, non_camel_case_types)]
#[derive(Clone, Copy)]
#[cfg_attr(test, derive(PartialEq, Eq, PartialOrd, Ord))]
#[repr(u32)]
enum CpuInfoFlag {
Init = 0,
$($func,)*
}
impl CpuInfo {
$(
$(#[$attr])*
#[cfg(any(test, not(any($($cfg = $name),*))))]
#[inline]
#[must_use]
pub(crate) fn $func(self) -> bool {
self.test(CpuInfoFlag::$func)
}
)*
#[cfg(test)] // for test
const ALL_FLAGS: &'static [(&'static str, CpuInfoFlag, bool)] = &[$(
($name, CpuInfoFlag::$func, cfg!(any($($cfg = $name),*))),
)*];
}
#[test]
#[cfg_attr(portable_atomic_test_detect_false, ignore = "detection disabled")]
fn test_detect() {$(
$(#[$attr])*
{
const _: u32 = 1_u32 << CpuInfoFlag::$func as u32;
assert_eq!($name.replace(|c: char| c == '-' || c == '.', "_"), stringify!($func));
if detect().$func() {
assert!(detect().test(CpuInfoFlag::$func));
} else {
assert!(!detect().test(CpuInfoFlag::$func));
}
}
)*}
};
}
// rustc definitions: https://github.com/rust-lang/rust/blob/ddaf12390d3ffb7d5ba74491a48f3cd528e5d777/compiler/rustc_target/src/target_features.rs
// LLVM definitions: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/AArch64/AArch64Features.td
#[cfg(any(target_arch = "aarch64", target_arch = "arm64ec"))]
flags! {
// The Armv8.1 architecture extension
// https://developer.arm.com/documentation/109697/2025_06/Feature-descriptions/The-Armv8-1-architecture-extension
// FEAT_LSE, Large System Extensions
// > This feature is supported in AArch64 state only.
// > FEAT_LSE is OPTIONAL from Armv8.0.
// > FEAT_LSE is mandatory from Armv8.1.
lse("lse", any(target_feature /* 1.61+ */, portable_atomic_target_feature)),
// The Armv8.3 architecture extension
// https://developer.arm.com/documentation/109697/2025_06/Feature-descriptions/The-Armv8-3-architecture-extension
// FEAT_LRCPC, Load-Acquire RCpc instructions
// > These instructions are added to the A64 instruction set only.
// > FEAT_LRCPC is OPTIONAL from Armv8.2.
// > FEAT_LRCPC is mandatory from Armv8.3.
#[cfg(test)] // test-only
rcpc("rcpc", any(target_feature /* 1.61+ */)),
// The Armv8.4 architecture extension
// https://developer.arm.com/documentation/109697/2025_06/Feature-descriptions/The-Armv8-4-architecture-extension
// FEAT_LSE2, Large System Extensions version 2
// > This feature is supported in AArch64 state only.
// > FEAT_LSE2 is OPTIONAL from Armv8.2.
// > FEAT_LSE2 is mandatory from Armv8.4.
#[cfg_attr(not(test), allow(dead_code))]
lse2("lse2", any(target_feature /* nightly */, portable_atomic_target_feature)),
// FEAT_LRCPC2, Load-Acquire RCpc instructions version 2
// > These instructions are added to the A64 instruction set only.
// > FEAT_LRCPC2 is OPTIONAL from Armv8.2.
// > FEAT_LRCPC2 is mandatory from Armv8.4.
// > If FEAT_LRCPC2 is implemented, then FEAT_LRCPC is implemented.
#[cfg(test)] // test-only
rcpc2("rcpc2", any(target_feature /* 1.61+ */)),
// The Armv8.9 architecture extension
// https://developer.arm.com/documentation/109697/2025_06/Feature-descriptions/The-Armv8-9-architecture-extension
// FEAT_LRCPC3, Load-Acquire RCpc instructions version 3
// > This feature is supported in AArch64 state only.
// > FEAT_LRCPC3 is OPTIONAL from Armv8.2.
// > If FEAT_LRCPC3 is implemented, then FEAT_LRCPC2 is implemented.
#[cfg_attr(not(test), allow(dead_code))]
rcpc3("rcpc3", any(target_feature /* nightly */, portable_atomic_target_feature)),
// The Armv9.4 architecture extension
// https://developer.arm.com/documentation/109697/2025_06/Feature-descriptions/The-Armv9-4-architecture-extension
// FEAT_LSE128, 128-bit Atomics
// > This feature is supported in AArch64 state only.
// > FEAT_LSE128 is OPTIONAL from Armv9.3.
// > If FEAT_LSE128 is implemented, then FEAT_LSE is implemented.
#[cfg_attr(not(test), allow(dead_code))]
lse128("lse128", any(target_feature /* nightly */, portable_atomic_target_feature)),
// The Armv9.6 architecture extension
// https://developer.arm.com/documentation/109697/2025_06/Feature-descriptions/The-Armv9-6-architecture-extension
// FEAT_LS64WB, LS64 for Write-back cacheable memory
// > This feature is supported in AArch64 state only.
// > FEAT_LS64WB is OPTIONAL from Armv9.2.
// > If FEAT_LS64WB is implemented, then FEAT_LS64_ACCDATA is implemented.
#[cfg(test)] // test-only
ls64wb("ls64wb", any(/* no corresponding target feature */)),
// FEAT_LSFE, Large System Float Extension
// > This feature is supported in AArch64 state only.
// > FEAT_LSFE is OPTIONAL from Armv9.3.
// > If FEAT_LSFE is implemented, then FEAT_FP is implemented.
#[cfg(test)] // test-only
lsfe("lsfe", any(target_feature /* N/A */, portable_atomic_target_feature)),
#[cfg(test)] // test-only
cpuid("cpuid", any(/* no corresponding target feature */)),
}
// LLVM definitions: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/ARM/ARMFeatures.td
#[cfg(target_arch = "arm")]
flags! {
#[cfg(test)] // test-only
lpae("lpae", any(/* no corresponding target feature */)),
}
// LLVM definitions: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/PowerPC/PPC.td
#[cfg(target_arch = "powerpc64")]
flags! {
// lqarx and stqcx.
quadword_atomics("quadword-atomics", any(target_feature /* nightly */, portable_atomic_target_feature)),
}
// LLVM definitions: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/RISCV/RISCVFeatures.td
#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
flags! {
// amocas.{w,d,q}
zacas("zacas", any(target_feature /* 1.94+ */, portable_atomic_target_feature)),
#[cfg(test)] // test-only
zabha("zabha", any(target_feature /* 1.94+ */, portable_atomic_target_feature)),
#[cfg(test)] // test-only
zalasr("zalasr", any(/* no corresponding target feature */)),
}
// LLVM definitions: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/X86/X86.td
#[cfg(target_arch = "x86_64")]
flags! {
// avx
#[cfg(target_feature = "sse")]
avx("avx", any(target_feature)),
// cmpxchg16b
cmpxchg16b("cmpxchg16b", any(target_feature /* 1.69+ */, portable_atomic_target_feature)),
}
#[allow(
clippy::alloc_instead_of_core,
clippy::std_instead_of_alloc,
clippy::std_instead_of_core,
clippy::undocumented_unsafe_blocks,
clippy::wildcard_imports
)]
#[cfg(test)]
mod tests_common {
use std::{collections::BTreeSet, vec};
use super::*;
#[test]
fn test_bit_flags() {
let mut flags = vec![("init", CpuInfoFlag::Init)];
flags.extend(CpuInfo::ALL_FLAGS.iter().map(|&(name, flag, _)| (name, flag)));
let flag_set = flags.iter().map(|(_, flag)| flag).collect::<BTreeSet<_>>();
let name_set = flags.iter().map(|(name, _)| name).collect::<BTreeSet<_>>();
if flag_set.len() != flags.len() {
panic!("CpuInfo flag values must be unique")
}
if name_set.len() != flags.len() {
panic!("CpuInfo flag names must be unique")
}
let mut x = CpuInfo(0);
for &(_, f) in &flags {
assert!(!x.test(f));
}
for i in 0..flags.len() {
x.set(flags[i].1);
for &(_, f) in &flags[..i + 1] {
assert!(x.test(f));
}
for &(_, f) in &flags[i + 1..] {
assert!(!x.test(f));
}
}
for &(_, f) in &flags {
assert!(x.test(f));
}
}
#[test]
fn print_features() {
use std::{fmt::Write as _, string::String};
let mut features = String::new();
features.push_str("\nfeatures:\n");
for &(name, flag, compile_time) in CpuInfo::ALL_FLAGS {
let run_time = detect().test(flag);
if run_time == compile_time {
let _ = writeln!(features, " {}: {}", name, run_time);
} else {
let _ = writeln!(
features,
" {}: {} (compile-time), {} (run-time)",
name, compile_time, run_time
);
}
}
test_helper::eprintln_nocapture!("{}", features);
}
// Static assertions for C type definitions.
// Assertions with core::ffi types are in crate::utils::ffi module.
#[cfg(not(any(windows, target_arch = "x86", target_arch = "x86_64")))]
const _: fn() = || {
use test_helper::sys;
let _: crate::utils::ffi::c_char = 0 as sys::c_char;
};
}
vendor/portable-atomic/src/imp/detect/powerpc64_aix.rs
Vendored
+56
View File
@@ -0,0 +1,56 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Run-time CPU feature detection on PowerPC64 AIX by using getsystemcfg.
Refs:
- https://github.com/golang/go/blob/go1.26.0/src/internal/cpu/cpu_ppc64x_aix.go
As of Rust 1.94, is_powerpc_feature_detected doesn't support run-time detection on AIX.
https://github.com/rust-lang/rust/blob/1.94.0/library/std_detect/src/detect/mod.rs
Run-time detection on PowerPC64 AIX is currently disabled by default as experimental
because we cannot run tests on the VM or real machine.
*/
include!("common.rs");
// libc requires Rust 1.63
mod ffi {
pub(crate) use crate::utils::ffi::{c_int, c_ulong};
sys_const!({
// https://github.com/rust-lang/libc/blob/0.2.158/src/unix/aix/mod.rs#L2058
// https://github.com/golang/go/blob/go1.26.0/src/internal/cpu/cpu_ppc64x_aix.go
pub(crate) const SC_IMPL: c_int = 2;
pub(crate) const POWER_8: c_ulong = 0x10000;
pub(crate) const POWER_9: c_ulong = 0x20000;
});
// TODO: use sys_const! once libc crate defined it.
pub(crate) const POWER_10: c_ulong = 0x40000;
sys_fn!({
extern "C" {
// https://www.ibm.com/docs/en/aix/7.3?topic=g-getsystemcfg-subroutine
// https://github.com/rust-lang/libc/blob/0.2.158/src/unix/aix/powerpc64.rs#L643
pub(crate) fn getsystemcfg(name: c_int) -> c_ulong;
}
});
}
#[cold]
#[must_use]
fn _detect(mut info: CpuInfo) -> CpuInfo {
// SAFETY: calling getsystemcfg is safe.
let impl_ = unsafe { ffi::getsystemcfg(ffi::SC_IMPL) };
if impl_ == ffi::c_ulong::MAX {
return info;
}
// Check both POWER_8 and later ISAs (which are superset of POWER_8) because
// AIX currently doesn't set POWER_8 when POWER_9 is set.
// https://github.com/golang/go/commit/51859ec2292d9c1d82a7054ec672ff551a0d7497
if impl_ & (ffi::POWER_8 | ffi::POWER_9 | ffi::POWER_10) != 0 {
info.set(CpuInfoFlag::quadword_atomics);
}
info
}
vendor/portable-atomic/src/imp/detect/riscv_linux.rs
Vendored
+195
View File
@@ -0,0 +1,195 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Run-time CPU feature detection on RISC-V Linux/Android by using riscv_hwprobe.
On RISC-V, detection using auxv only supports single-letter extensions.
So, we use riscv_hwprobe that supports multi-letter extensions.
Refs: https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/riscv/hwprobe.rst
*/
include!("common.rs");
use core::ptr;
// libc requires Rust 1.63
#[allow(non_camel_case_types, non_upper_case_globals)]
mod ffi {
pub(crate) use crate::utils::ffi::{c_long, c_size_t, c_uint, c_ulong};
sys_struct!({
// https://github.com/torvalds/linux/blob/v6.19/arch/riscv/include/uapi/asm/hwprobe.h
pub(crate) struct riscv_hwprobe {
pub(crate) key: i64,
pub(crate) value: u64,
}
});
sys_const!({
pub(crate) const __NR_riscv_hwprobe: c_long = 258;
// https://github.com/torvalds/linux/blob/v6.19/arch/riscv/include/uapi/asm/hwprobe.h
// Linux 6.4+
// https://github.com/torvalds/linux/commit/00e76e2c6a2bd3976d44d4a1fdd0b7a3c2566607
pub(crate) const RISCV_HWPROBE_KEY_BASE_BEHAVIOR: i64 = 3;
pub(crate) const RISCV_HWPROBE_BASE_BEHAVIOR_IMA: u64 = 1 << 0;
pub(crate) const RISCV_HWPROBE_KEY_IMA_EXT_0: i64 = 4;
// Linux 6.8+
// https://github.com/torvalds/linux/commit/154a3706122978eeb34d8223d49285ed4f3c61fa
pub(crate) const RISCV_HWPROBE_EXT_ZACAS: u64 = 1 << 34;
// Linux 6.16+
// https://github.com/torvalds/linux/commit/415a8c81da3dab0a585bd4f8d505a11ad5a171a7
#[cfg(test)]
pub(crate) const RISCV_HWPROBE_EXT_ZABHA: u64 = 1 << 58;
// Linux 6.19+
// https://github.com/torvalds/linux/commit/f4922b69165735e81752ee47d174f873e989a449
#[cfg(test)]
pub(crate) const RISCV_HWPROBE_EXT_ZALASR: u64 = 1 << 59;
});
cfg_sel!({
// Use asm-based syscall on Linux for compatibility with non-libc targets if possible.
// Do not use this on Android. See comments on syscall_helper module in src/utils.rs for details.
#[cfg(all(
target_os = "linux",
any(
target_arch = "riscv32",
all(target_arch = "riscv64", target_pointer_width = "64"),
),
not(portable_atomic_no_asm_syscall),
))]
{
use crate::utils::{RegISize, RegSize};
#[inline]
pub(crate) unsafe fn syscall5(
number: c_long,
arg1: *mut riscv_hwprobe,
arg2: c_size_t,
arg3: c_size_t,
arg4: *mut c_ulong,
arg5: c_uint,
) -> c_long {
#[allow(clippy::cast_possible_truncation)]
let number = number as RegISize;
let arg1 = ptr_reg!(arg1);
let arg2 = arg2 as RegSize;
let arg3 = arg3 as RegSize;
let arg4 = ptr_reg!(arg4);
let arg5 = arg5 as RegSize;
let r: RegISize;
// SAFETY: the caller must uphold the safety contract.
// we've extended the syscall number and arguments to the register size.
unsafe {
asm_syscall!(number, r, arg1, arg2, arg3, arg4, arg5);
}
#[allow(clippy::cast_possible_truncation)]
{
r as c_long
}
}
}
#[cfg(else)]
{
sys_fn!({
extern "C" {
// https://man7.org/linux/man-pages/man2/syscall.2.html
pub(crate) fn syscall(number: c_long, ...) -> c_long;
}
});
pub(crate) use self::syscall as syscall5;
}
});
// https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/riscv/hwprobe.rst
pub(crate) unsafe fn __riscv_hwprobe(
pairs: *mut riscv_hwprobe,
pair_count: c_size_t,
cpu_set_size: c_size_t,
cpus: *mut c_ulong,
flags: c_uint,
) -> c_long {
// SAFETY: the caller must uphold the safety contract.
unsafe { syscall5(__NR_riscv_hwprobe, pairs, pair_count, cpu_set_size, cpus, flags) }
}
}
// syscall returns an unsupported error if riscv_hwprobe is not supported,
// so we can safely use this function on older versions of Linux.
fn riscv_hwprobe(out: &mut [ffi::riscv_hwprobe]) -> bool {
let len = out.len();
// SAFETY: We've passed the valid pointer and length,
// passing null ptr for cpus is safe because cpu_set_size is zero.
unsafe { ffi::__riscv_hwprobe(out.as_mut_ptr(), len, 0, ptr::null_mut(), 0) == 0 }
}
#[cold]
#[must_use]
fn _detect(mut info: CpuInfo) -> CpuInfo {
let mut out = [
ffi::riscv_hwprobe { key: ffi::RISCV_HWPROBE_KEY_BASE_BEHAVIOR, value: 0 },
ffi::riscv_hwprobe { key: ffi::RISCV_HWPROBE_KEY_IMA_EXT_0, value: 0 },
];
if riscv_hwprobe(&mut out)
&& out[0].key != -1
&& out[0].value & ffi::RISCV_HWPROBE_BASE_BEHAVIOR_IMA != 0
&& out[1].key != -1
{
let value = out[1].value;
macro_rules! check {
($flag:ident, $bit:ident) => {
if value & ffi::$bit != 0 {
info.set(CpuInfoFlag::$flag);
}
};
}
check!(zacas, RISCV_HWPROBE_EXT_ZACAS);
#[cfg(test)]
check!(zabha, RISCV_HWPROBE_EXT_ZABHA);
#[cfg(test)]
check!(zalasr, RISCV_HWPROBE_EXT_ZALASR);
}
info
}
#[allow(
clippy::alloc_instead_of_core,
clippy::std_instead_of_alloc,
clippy::std_instead_of_core,
clippy::undocumented_unsafe_blocks,
clippy::wildcard_imports
)]
#[cfg(test)]
mod tests {
use super::*;
// We use asm-based syscall for compatibility with non-libc targets.
// This test tests that our ones and libc::syscall returns the same result.
#[test]
fn test_alternative() {
unsafe fn __riscv_hwprobe_libc(
pairs: *mut ffi::riscv_hwprobe,
pair_count: ffi::c_size_t,
cpu_set_size: ffi::c_size_t,
cpus: *mut ffi::c_ulong,
flags: ffi::c_uint,
) -> ffi::c_long {
// SAFETY: the caller must uphold the safety contract.
unsafe {
libc::syscall(ffi::__NR_riscv_hwprobe, pairs, pair_count, cpu_set_size, cpus, flags)
}
}
fn riscv_hwprobe_libc(out: &mut [ffi::riscv_hwprobe]) -> bool {
let len = out.len();
unsafe { __riscv_hwprobe_libc(out.as_mut_ptr(), len, 0, ptr::null_mut(), 0) == 0 }
}
let mut out = [
ffi::riscv_hwprobe { key: ffi::RISCV_HWPROBE_KEY_BASE_BEHAVIOR, value: 0 },
ffi::riscv_hwprobe { key: ffi::RISCV_HWPROBE_KEY_IMA_EXT_0, value: 0 },
];
let mut libc_out = out;
assert_eq!(riscv_hwprobe(&mut out), riscv_hwprobe_libc(&mut libc_out));
assert_eq!(out, libc_out);
}
}
vendor/portable-atomic/src/imp/detect/x86_64.rs
Vendored
+111
View File
@@ -0,0 +1,111 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Run-time CPU feature detection on x86_64 by using CPUID.
Adapted from https://github.com/rust-lang/rust/blob/1.92.0/library/std_detect/src/detect/os/x86.rs.
*/
#![cfg_attr(portable_atomic_sanitize_thread, allow(dead_code))]
// Miri doesn't support inline assembly used in __cpuid: https://github.com/rust-lang/miri/issues/932
// SGX doesn't support CPUID: https://github.com/rust-lang/rust/blob/1.92.0/library/std_detect/src/detect/os/x86.rs#L30-L33
#[cfg(any(target_env = "sgx", miri))]
compile_error!("internal error: this module is not supported on this environment");
include!("common.rs");
#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
use core::arch::x86_64::CpuidResult;
// Workaround for https://github.com/rust-lang/rust/issues/101346
// It is not clear if our use cases are affected, but we implement this just in case.
//
// Refs:
// - https://www.felixcloutier.com/x86/cpuid
// - https://en.wikipedia.org/wiki/CPUID
// - https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs
#[cfg(not(target_env = "sgx"))]
fn __cpuid(leaf: u32) -> CpuidResult {
let eax;
let mut ebx;
let ecx;
let edx;
// SAFETY: Calling `__cpuid` is safe on all x86_64 CPUs except for SGX,
// which doesn't support `cpuid`.
// https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs#L102-L109
unsafe {
asm!(
"mov r8, rbx", // save rbx which is reserved by LLVM
"cpuid",
"xchg r8, rbx", // restore rbx
out("r8") ebx,
inout("eax") leaf => eax,
inout("ecx") 0 => ecx,
out("edx") edx,
options(nostack, preserves_flags),
);
}
CpuidResult { eax, ebx, ecx, edx }
}
#[cold]
#[must_use]
fn _detect(mut info: CpuInfo) -> CpuInfo {
let CpuidResult { ecx: proc_info_ecx, .. } = __cpuid(1);
// https://github.com/rust-lang/rust/blob/1.92.0/library/std_detect/src/detect/os/x86.rs#L104
if test(proc_info_ecx, 13) {
info.set(CpuInfoFlag::cmpxchg16b);
}
// We only use VMOVDQA when SSE is enabled. See _atomic_load_vmovdqa() in atomic128/x86_64.rs for more.
#[cfg(target_feature = "sse")]
{
use core::arch::x86_64::_xgetbv;
// https://github.com/rust-lang/rust/blob/1.92.0/library/std_detect/src/detect/os/x86.rs#L166-L236
let cpu_xsave = test(proc_info_ecx, 26);
if cpu_xsave {
let cpu_osxsave = test(proc_info_ecx, 27);
if cpu_osxsave {
// SAFETY: Calling `_xgetbv` is safe because the CPU has `xsave` support
// and OS has set `osxsave`.
let xcr0 = unsafe { _xgetbv(0) };
let os_avx_support = xcr0 & 6 == 6;
if os_avx_support && test(proc_info_ecx, 28) {
info.set(CpuInfoFlag::avx);
}
}
}
}
info
}
#[allow(
clippy::alloc_instead_of_core,
clippy::std_instead_of_alloc,
clippy::std_instead_of_core,
clippy::undocumented_unsafe_blocks,
clippy::wildcard_imports
)]
#[cfg(test)]
mod tests {
use super::*;
#[test]
#[cfg_attr(portable_atomic_test_detect_false, ignore = "detection disabled")]
fn test_cpuid() {
// The recent Rosetta 2 unofficially implements AVX support.
// (The OS reports it as unsupported, likely due to poor performance.)
#[cfg(target_vendor = "apple")]
assert_eq!(
std::is_x86_feature_detected!("avx"),
detect().avx() || cfg!(target_feature = "avx")
);
#[cfg(not(target_vendor = "apple"))]
assert_eq!(std::is_x86_feature_detected!("avx"), detect().avx());
assert_eq!(std::is_x86_feature_detected!("cmpxchg16b"), detect().cmpxchg16b());
}
}
vendor/portable-atomic/src/imp/fallback/mod.rs
Vendored
+580
View File
@@ -0,0 +1,580 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Fallback implementation using global locks.
This implementation uses seqlock for global locks.
This is basically based on global locks in crossbeam-utils's `AtomicCell`,
but seqlock is implemented in a way that does not depend on UB
(see comments in optimistic_read method in atomic! macro for details).
Note that we cannot use a lock per atomic type, since the in-memory representation of the atomic
type and the value type must be the same.
*/
#![cfg_attr(
any(
all(
target_arch = "riscv32",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
not(portable_atomic_no_outline_atomics),
any(target_os = "linux", target_os = "android"),
),
all(
target_arch = "arm",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_os = "linux", target_os = "android"),
any(test, not(any(target_feature = "v6", portable_atomic_target_feature = "v6"))),
not(portable_atomic_no_outline_atomics),
),
),
allow(dead_code)
)]
// This module requires CAS and this crate only provides atomics up to 128-bit.
// I don't believe there are any 16-bit multi-core systems with CAS, and
// at least no such architecture is currently supported in Rust.
// 128-bit targets that lack atomic usize CAS also do not reach this module.
#[cfg(target_pointer_width = "16")]
compile_error!(
"internal error: unreachable since atomics for 16-bit targets can always be provided by disable interrupts"
);
#[cfg(target_pointer_width = "128")]
compile_error!(
"internal error: unreachable since 128-bit target either has atomic CAS for the pointer width or does not have CAS"
);
mod utils;
// Use "wide" sequence lock if the pointer width <= 32 for preventing its counter against wrap
// around.
//
// Some 64-bit architectures have ABI with 32-bit pointer width (e.g., x86_64 X32 ABI,
// AArch64 ILP32 ABI, mips64 N32 ABI). On those targets, AtomicU64 is available and fast,
// so use it to implement normal sequence lock and reduce chunks of byte-wise atomic memcpy.
cfg_has_fast_atomic_64!({
mod seq_lock;
type AtomicChunk = core::sync::atomic::AtomicU64;
type Chunk = u64;
});
cfg_no_fast_atomic_64!({
#[path = "seq_lock_wide.rs"]
mod seq_lock;
type AtomicChunk = core::sync::atomic::AtomicU32;
type Chunk = u32;
});
use core::{cell::UnsafeCell, mem, sync::atomic::Ordering};
use self::{
seq_lock::{SeqLock, SeqLockWriteGuard},
utils::CachePadded,
};
#[cfg(portable_atomic_no_strict_provenance)]
use crate::utils::ptr::PtrExt as _;
use crate::utils::unlikely;
// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/atomic/atomic_cell.rs#L970-L1010.
#[inline]
#[must_use]
fn lock(addr: usize) -> &'static SeqLock {
// The number of locks is a prime number because we want to make sure `addr % LEN` gets
// dispersed across all locks.
const LEN: usize = 67;
const L: CachePadded<SeqLock> = CachePadded::new(SeqLock::new());
static LOCKS: [CachePadded<SeqLock>; LEN] = [
L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L,
L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L,
L, L, L, L, L, L, L,
];
// If the modulus is a constant number, the compiler will use crazy math to transform this into
// a sequence of cheap arithmetic operations rather than using the slow modulo instruction.
&LOCKS[addr % LEN]
}
// Emit SeqCst fence to ensure SeqCst semantics when ordering is SeqCst.
#[must_use]
struct ScFenceGuard;
impl ScFenceGuard {
#[inline]
fn new(emit_sc_fence: bool) -> Option<Self> {
if unlikely(emit_sc_fence) {
crate::fence(Ordering::SeqCst);
Some(ScFenceGuard)
} else {
None
}
}
}
impl Drop for ScFenceGuard {
#[inline]
fn drop(&mut self) {
crate::fence(Ordering::SeqCst);
}
}
macro_rules! atomic {
($atomic_type:ident, $int_type:ident, $align:literal) => {
#[repr(C, align($align))]
pub(crate) struct $atomic_type {
v: UnsafeCell<$int_type>,
}
impl $atomic_type {
const LEN: usize = mem::size_of::<$int_type>() / mem::size_of::<Chunk>();
#[inline]
unsafe fn chunks(&self) -> &[AtomicChunk; Self::LEN] {
static_assert!($atomic_type::LEN > 1);
static_assert!(mem::size_of::<$int_type>() % mem::size_of::<Chunk>() == 0);
// SAFETY: the caller must uphold the safety contract for `chunks`.
unsafe { &*(self.v.get() as *const $int_type as *const [AtomicChunk; Self::LEN]) }
}
#[inline]
fn optimistic_read(&self) -> $int_type {
// Using `MaybeUninit<[usize; Self::LEN]>` here doesn't change codegen: https://godbolt.org/z/86f8s733M
let mut dst: [Chunk; Self::LEN] = [0; Self::LEN];
// SAFETY:
// - There are no threads that perform non-atomic concurrent write operations.
// - There is no writer that updates the value using atomic operations of different granularity.
//
// If the atomic operation is not used here, it will cause a data race
// when `write` performs concurrent write operation.
// Such a data race is sometimes considered virtually unproblematic
// in SeqLock implementations:
//
// - https://github.com/Amanieu/seqlock/issues/2
// - https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/atomic/atomic_cell.rs#L1063-L1069
// - https://rust-lang.zulipchat.com/#narrow/stream/136281-t-lang.2Fwg-unsafe-code-guidelines/topic/avoiding.20UB.20due.20to.20races.20by.20discarding.20result.3F
//
// However, in our use case, the implementation that loads/stores value as
// chunks of usize is enough fast and sound, so we use that implementation.
//
// See also atomic-memcpy crate, a generic implementation of this pattern:
// https://github.com/taiki-e/atomic-memcpy
let chunks = unsafe { self.chunks() };
for i in 0..Self::LEN {
dst[i] = chunks[i].load(Ordering::Relaxed);
}
// SAFETY: integers are plain old data types so we can always transmute to them.
unsafe { mem::transmute::<[Chunk; Self::LEN], $int_type>(dst) }
}
#[inline]
fn read(&self, _guard: &SeqLockWriteGuard<'static>) -> $int_type {
// This calls optimistic_read that can return teared value, but the resulting value
// is guaranteed not to be teared because we hold the lock to write.
self.optimistic_read()
}
#[inline]
fn write(&self, val: $int_type, _guard: &SeqLockWriteGuard<'static>) {
// SAFETY: integers are plain old data types so we can always transmute them to arrays of integers.
let val = unsafe { mem::transmute::<$int_type, [Chunk; Self::LEN]>(val) };
// SAFETY:
// - The guard guarantees that we hold the lock to write.
// - There are no threads that perform non-atomic concurrent read or write operations.
//
// See optimistic_read for the reason that atomic operations are used here.
let chunks = unsafe { self.chunks() };
for i in 0..Self::LEN {
chunks[i].store(val[i], Ordering::Relaxed);
}
}
}
// Send is implicitly implemented.
// SAFETY: any data races are prevented by the lock and atomic operation.
unsafe impl Sync for $atomic_type {}
#[cfg(any(
test,
not(any(
all(
target_arch = "x86_64",
not(all(
any(miri, portable_atomic_sanitize_thread),
portable_atomic_no_cmpxchg16b_intrinsic,
)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
not(portable_atomic_no_outline_atomics),
not(any(target_env = "sgx", miri)),
),
all(
target_arch = "powerpc64",
not(portable_atomic_no_asm),
not(portable_atomic_no_outline_atomics),
any(
all(
target_os = "linux",
any(
all(
target_env = "gnu",
any(target_endian = "little", not(target_feature = "crt-static")),
),
all(
target_env = "musl",
any(not(target_feature = "crt-static"), feature = "std"),
),
target_env = "ohos",
all(target_env = "uclibc", not(target_feature = "crt-static")),
portable_atomic_outline_atomics,
),
),
target_os = "android",
all(
target_os = "freebsd",
any(
target_endian = "little",
not(target_feature = "crt-static"),
portable_atomic_outline_atomics,
),
),
target_os = "openbsd",
all(
target_os = "aix",
not(portable_atomic_pre_llvm_20),
portable_atomic_outline_atomics, // TODO(aix): currently disabled by default
),
),
not(any(miri, portable_atomic_sanitize_thread)),
),
all(
target_arch = "riscv64",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
not(portable_atomic_no_outline_atomics),
any(target_os = "linux", target_os = "android"),
),
)),
))]
items!({
impl_default_no_fetch_ops!($atomic_type, $int_type);
impl_default_bit_opts!($atomic_type, $int_type);
impl $atomic_type {
#[inline]
pub(crate) const fn new(v: $int_type) -> Self {
Self { v: UnsafeCell::new(v) }
}
#[inline]
pub(crate) fn is_lock_free() -> bool {
Self::IS_ALWAYS_LOCK_FREE
}
pub(crate) const IS_ALWAYS_LOCK_FREE: bool = false;
#[inline]
#[cfg_attr(
all(debug_assertions, not(portable_atomic_no_track_caller)),
track_caller
)]
pub(crate) fn compare_exchange_weak(
&self,
current: $int_type,
new: $int_type,
success: Ordering,
failure: Ordering,
) -> Result<$int_type, $int_type> {
self.compare_exchange(current, new, success, failure)
}
#[inline]
pub(crate) fn not(&self, order: Ordering) {
self.fetch_not(order);
}
#[inline]
pub(crate) fn neg(&self, order: Ordering) {
self.fetch_neg(order);
}
#[inline]
pub(crate) const fn as_ptr(&self) -> *mut $int_type {
self.v.get()
}
}
});
#[cfg_attr(
any(
all(
target_arch = "x86_64",
not(all(
any(miri, portable_atomic_sanitize_thread),
portable_atomic_no_cmpxchg16b_intrinsic,
)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
not(portable_atomic_no_outline_atomics),
not(any(target_env = "sgx", miri)),
),
all(
target_arch = "powerpc64",
not(portable_atomic_no_asm),
not(portable_atomic_no_outline_atomics),
any(
all(
target_os = "linux",
any(
all(
target_env = "gnu",
any(target_endian = "little", not(target_feature = "crt-static")),
),
all(
target_env = "musl",
any(not(target_feature = "crt-static"), feature = "std"),
),
target_env = "ohos",
all(target_env = "uclibc", not(target_feature = "crt-static")),
portable_atomic_outline_atomics,
),
),
target_os = "android",
all(
target_os = "freebsd",
any(
target_endian = "little",
not(target_feature = "crt-static"),
portable_atomic_outline_atomics,
),
),
target_os = "openbsd",
all(
target_os = "aix",
not(portable_atomic_pre_llvm_20),
portable_atomic_outline_atomics, // TODO(aix): currently disabled by default
),
),
not(any(miri, portable_atomic_sanitize_thread)),
),
all(
target_arch = "riscv64",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
not(portable_atomic_no_outline_atomics),
any(target_os = "linux", target_os = "android"),
),
),
allow(dead_code)
)]
impl $atomic_type {
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub(crate) fn load(&self, order: Ordering) -> $int_type {
crate::utils::assert_load_ordering(order);
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
let lock = lock(self.v.get().addr());
// Try doing an optimistic read first.
if let Some(stamp) = lock.optimistic_read() {
let val = self.optimistic_read();
if lock.validate_read(stamp) {
return val;
}
}
// Grab a regular write lock so that writers don't starve this load.
let guard = lock.write();
let val = self.read(&guard);
// The value hasn't been changed. Drop the guard without incrementing the stamp.
guard.abort();
val
}
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub(crate) fn store(&self, val: $int_type, order: Ordering) {
crate::utils::assert_store_ordering(order);
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
let guard = lock(self.v.get().addr()).write();
self.write(val, &guard)
}
#[inline]
pub(crate) fn swap(&self, val: $int_type, order: Ordering) -> $int_type {
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
let guard = lock(self.v.get().addr()).write();
let prev = self.read(&guard);
self.write(val, &guard);
prev
}
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub(crate) fn compare_exchange(
&self,
current: $int_type,
new: $int_type,
success: Ordering,
failure: Ordering,
) -> Result<$int_type, $int_type> {
crate::utils::assert_compare_exchange_ordering(success, failure);
let _sc_fence = ScFenceGuard::new(
success == Ordering::SeqCst || failure == Ordering::SeqCst
);
let guard = lock(self.v.get().addr()).write();
let prev = self.read(&guard);
if prev == current {
self.write(new, &guard);
Ok(prev)
} else {
// The value hasn't been changed. Drop the guard without incrementing the stamp.
guard.abort();
Err(prev)
}
}
#[inline]
pub(crate) fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type {
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
let guard = lock(self.v.get().addr()).write();
let prev = self.read(&guard);
self.write(prev.wrapping_add(val), &guard);
prev
}
#[inline]
pub(crate) fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type {
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
let guard = lock(self.v.get().addr()).write();
let prev = self.read(&guard);
self.write(prev.wrapping_sub(val), &guard);
prev
}
#[inline]
pub(crate) fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type {
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
let guard = lock(self.v.get().addr()).write();
let prev = self.read(&guard);
self.write(prev & val, &guard);
prev
}
#[inline]
pub(crate) fn fetch_nand(&self, val: $int_type, order: Ordering) -> $int_type {
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
let guard = lock(self.v.get().addr()).write();
let prev = self.read(&guard);
self.write(!(prev & val), &guard);
prev
}
#[inline]
pub(crate) fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type {
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
let guard = lock(self.v.get().addr()).write();
let prev = self.read(&guard);
self.write(prev | val, &guard);
prev
}
#[inline]
pub(crate) fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type {
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
let guard = lock(self.v.get().addr()).write();
let prev = self.read(&guard);
self.write(prev ^ val, &guard);
prev
}
#[inline]
pub(crate) fn fetch_not(&self, order: Ordering) -> $int_type {
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
let guard = lock(self.v.get().addr()).write();
let prev = self.read(&guard);
self.write(!prev, &guard);
prev
}
#[inline]
pub(crate) fn fetch_neg(&self, order: Ordering) -> $int_type {
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
let guard = lock(self.v.get().addr()).write();
let prev = self.read(&guard);
self.write(prev.wrapping_neg(), &guard);
prev
}
}
impl $atomic_type {
#[inline]
pub(crate) fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type {
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
let guard = lock(self.v.get().addr()).write();
let prev = self.read(&guard);
self.write(core::cmp::max(prev, val), &guard);
prev
}
#[inline]
pub(crate) fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type {
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
let guard = lock(self.v.get().addr()).write();
let prev = self.read(&guard);
self.write(core::cmp::min(prev, val), &guard);
prev
}
}
};
}
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(any(
test,
not(any(
not(portable_atomic_no_atomic_64),
all(
target_arch = "riscv32",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
),
))
))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(any(
test,
not(any(
target_has_atomic = "64",
all(
target_arch = "riscv32",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
),
))
))
)]
cfg_no_fast_atomic_64!({
atomic!(AtomicI64, i64, 8);
atomic!(AtomicU64, u64, 8);
});
atomic!(AtomicI128, i128, 16);
atomic!(AtomicU128, u128, 16);
#[cfg(test)]
mod tests {
use super::*;
cfg_no_fast_atomic_64!({
test_atomic_int!(i64);
test_atomic_int!(u64);
});
test_atomic_int!(i128);
test_atomic_int!(u128);
// load/store/swap implementation is not affected by signedness, so it is
// enough to test only unsigned types.
cfg_no_fast_atomic_64!({
stress_test!(u64);
});
stress_test!(u128);
}
vendor/portable-atomic/src/imp/fallback/outline_atomics.rs
Vendored
+184
View File
@@ -0,0 +1,184 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Helper for outline-atomics.
On architectures where DW atomics are not supported on older CPUs, we use
fallback implementation when DW atomic instructions are not supported and
outline-atomics is enabled.
This module provides helpers to implement them.
*/
use core::sync::atomic::Ordering;
#[cfg(any(target_arch = "x86_64", target_arch = "powerpc64", target_arch = "riscv64"))]
pub(crate) type Udw = u128;
#[cfg(any(target_arch = "x86_64", target_arch = "powerpc64", target_arch = "riscv64"))]
pub(crate) type AtomicUdw = super::super::super::fallback::AtomicU128;
#[cfg(any(target_arch = "x86_64", target_arch = "powerpc64", target_arch = "riscv64"))]
pub(crate) type AtomicIdw = super::super::super::fallback::AtomicI128;
#[cfg(any(target_arch = "arm", target_arch = "riscv32"))]
pub(crate) type Udw = u64;
#[cfg(any(target_arch = "arm", target_arch = "riscv32"))]
pub(crate) type AtomicUdw = super::super::super::fallback::AtomicU64;
#[cfg(any(target_arch = "arm", target_arch = "riscv32"))]
pub(crate) type AtomicIdw = super::super::super::fallback::AtomicI64;
// Asserts that the function is called in the correct context.
macro_rules! debug_assert_outline_atomics {
() => {
#[cfg(target_arch = "x86_64")]
{
debug_assert!(!super::detect::detect().cmpxchg16b());
}
#[cfg(target_arch = "powerpc64")]
{
debug_assert!(!super::detect::detect().quadword_atomics());
}
#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
{
debug_assert!(!super::detect::detect().zacas());
}
#[cfg(target_arch = "arm")]
{
debug_assert!(!super::has_kuser_cmpxchg64());
}
};
}
#[cold]
pub(crate) unsafe fn atomic_load(src: *mut Udw, order: Ordering) -> Udw {
debug_assert_outline_atomics!();
#[allow(clippy::cast_ptr_alignment)]
// SAFETY: the caller must uphold the safety contract.
unsafe {
(*(src as *const AtomicUdw)).load(order)
}
}
fn_alias! {
#[cold]
pub(crate) unsafe fn(src: *mut Udw) -> Udw;
// fallback's atomic load has at least acquire semantics.
#[cfg(not(any(target_arch = "arm", target_arch = "x86_64")))]
atomic_load_non_seqcst = atomic_load(Ordering::Acquire);
atomic_load_seqcst = atomic_load(Ordering::SeqCst);
}
#[cfg(not(any(target_arch = "arm", target_arch = "riscv32", target_arch = "riscv64")))]
#[cold]
pub(crate) unsafe fn atomic_store(dst: *mut Udw, val: Udw, order: Ordering) {
debug_assert_outline_atomics!();
#[allow(clippy::cast_ptr_alignment)]
// SAFETY: the caller must uphold the safety contract.
unsafe {
(*(dst as *const AtomicUdw)).store(val, order);
}
}
#[cfg(not(any(target_arch = "arm", target_arch = "riscv32", target_arch = "riscv64")))]
fn_alias! {
#[cold]
pub(crate) unsafe fn(dst: *mut Udw, val: Udw);
// fallback's atomic store has at least release semantics.
atomic_store_non_seqcst = atomic_store(Ordering::Release);
atomic_store_seqcst = atomic_store(Ordering::SeqCst);
}
#[cold]
pub(crate) unsafe fn atomic_compare_exchange(
dst: *mut Udw,
old: Udw,
new: Udw,
success: Ordering,
failure: Ordering,
) -> (Udw, bool) {
debug_assert_outline_atomics!();
#[allow(clippy::cast_ptr_alignment)]
// SAFETY: the caller must uphold the safety contract.
unsafe {
match (*(dst as *const AtomicUdw)).compare_exchange(old, new, success, failure) {
Ok(v) => (v, true),
Err(v) => (v, false),
}
}
}
fn_alias! {
#[cold]
pub(crate) unsafe fn(dst: *mut Udw, old: Udw, new: Udw) -> (Udw, bool);
// fallback's atomic CAS has at least AcqRel semantics.
#[cfg(not(any(target_arch = "arm", target_arch = "x86_64")))]
atomic_compare_exchange_non_seqcst
= atomic_compare_exchange(Ordering::AcqRel, Ordering::Acquire);
atomic_compare_exchange_seqcst
= atomic_compare_exchange(Ordering::SeqCst, Ordering::SeqCst);
}
macro_rules! atomic_rmw_3 {
(
$name:ident($atomic_type:ident::$method_name:ident),
$non_seqcst_alias:ident, $seqcst_alias:ident
) => {
#[cold]
pub(crate) unsafe fn $name(dst: *mut Udw, val: Udw, order: Ordering) -> Udw {
debug_assert_outline_atomics!();
#[allow(
clippy::as_underscore,
clippy::cast_possible_wrap,
clippy::cast_ptr_alignment,
clippy::cast_sign_loss
)]
// SAFETY: the caller must uphold the safety contract.
unsafe {
(*(dst as *const $atomic_type)).$method_name(val as _, order) as Udw
}
}
fn_alias! {
#[cold]
pub(crate) unsafe fn(dst: *mut Udw, val: Udw) -> Udw;
// fallback's atomic RMW has at least AcqRel semantics.
#[cfg(not(any(target_arch = "arm", target_arch = "x86_64")))]
$non_seqcst_alias = $name(Ordering::AcqRel);
$seqcst_alias = $name(Ordering::SeqCst);
}
};
}
macro_rules! atomic_rmw_2 {
(
$name:ident($atomic_type:ident::$method_name:ident),
$non_seqcst_alias:ident, $seqcst_alias:ident
) => {
#[cold]
pub(crate) unsafe fn $name(dst: *mut Udw, order: Ordering) -> Udw {
debug_assert_outline_atomics!();
#[allow(clippy::cast_ptr_alignment)]
// SAFETY: the caller must uphold the safety contract.
unsafe {
(*(dst as *const $atomic_type)).$method_name(order) as Udw
}
}
fn_alias! {
#[cold]
pub(crate) unsafe fn(dst: *mut Udw) -> Udw;
// fallback's atomic RMW has at least AcqRel semantics.
#[cfg(not(any(target_arch = "arm", target_arch = "x86_64")))]
$non_seqcst_alias = $name(Ordering::AcqRel);
$seqcst_alias = $name(Ordering::SeqCst);
}
};
}
atomic_rmw_3!(atomic_swap(AtomicUdw::swap), atomic_swap_non_seqcst, atomic_swap_seqcst);
atomic_rmw_3!(atomic_add(AtomicUdw::fetch_add), atomic_add_non_seqcst, atomic_add_seqcst);
atomic_rmw_3!(atomic_sub(AtomicUdw::fetch_sub), atomic_sub_non_seqcst, atomic_sub_seqcst);
atomic_rmw_3!(atomic_and(AtomicUdw::fetch_and), atomic_and_non_seqcst, atomic_and_seqcst);
atomic_rmw_3!(atomic_nand(AtomicUdw::fetch_nand), atomic_nand_non_seqcst, atomic_nand_seqcst);
atomic_rmw_3!(atomic_or(AtomicUdw::fetch_or), atomic_or_non_seqcst, atomic_or_seqcst);
atomic_rmw_3!(atomic_xor(AtomicUdw::fetch_xor), atomic_xor_non_seqcst, atomic_xor_seqcst);
atomic_rmw_3!(atomic_max(AtomicIdw::fetch_max), atomic_max_non_seqcst, atomic_max_seqcst);
atomic_rmw_3!(atomic_umax(AtomicUdw::fetch_max), atomic_umax_non_seqcst, atomic_umax_seqcst);
atomic_rmw_3!(atomic_min(AtomicIdw::fetch_min), atomic_min_non_seqcst, atomic_min_seqcst);
atomic_rmw_3!(atomic_umin(AtomicUdw::fetch_min), atomic_umin_non_seqcst, atomic_umin_seqcst);
atomic_rmw_2!(atomic_not(AtomicUdw::fetch_not), atomic_not_non_seqcst, atomic_not_seqcst);
atomic_rmw_2!(atomic_neg(AtomicUdw::fetch_neg), atomic_neg_non_seqcst, atomic_neg_seqcst);
vendor/portable-atomic/src/imp/fallback/seq_lock.rs
Vendored
+56
View File
@@ -0,0 +1,56 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/atomic/seq_lock.rs.
include!("seq_lock_common.rs");
use crate::imp::core_atomic::AtomicU64;
pub(super) type State = u64;
/// A simple stamped lock.
pub(super) struct SeqLock {
/// The current state of the lock.
///
/// All bits except the least significant one hold the current stamp. When locked, the state
/// equals 1 and doesn't contain a valid stamp.
state: AtomicU64,
}
impl SeqLock {
#[inline]
pub(super) const fn new() -> Self {
Self { state: AtomicU64::new(0) }
}
/// If not locked, returns the current stamp.
///
/// This method should be called before optimistic reads.
#[inline]
pub(super) fn optimistic_read(&self) -> Option<State> {
// The acquire load from `state` synchronizes with the release stores in
// `SeqLockWriteGuard::drop` and `SeqLockWriteGuard::abort`.
let state = self.state.load(Ordering::Acquire);
if state == LOCKED { None } else { Some(state) }
}
/// Returns `true` if the current stamp is equal to `stamp`.
///
/// This method should be called after optimistic reads to check whether they are valid. The
/// argument `stamp` should correspond to the one returned by method `optimistic_read`.
#[inline]
pub(super) fn validate_read(&self, stamp: State) -> bool {
// Thanks to the fence, if we're noticing any modification to the data at the critical
// section of `stamp`, then the critical section's write of 1 to state
// should be visible.
crate::fence(Ordering::Acquire);
self.state.load(Ordering::Relaxed) == stamp
}
}
impl SeqLockWriteGuard<'_> {
#[inline]
fn next_stamp(&self) -> State {
self.state.wrapping_add(2)
}
}
vendor/portable-atomic/src/imp/fallback/seq_lock_common.rs
Vendored
+157
View File
@@ -0,0 +1,157 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/atomic/seq_lock.rs
// and https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/atomic/seq_lock_wide.rs.
use core::{mem::ManuallyDrop, sync::atomic::Ordering};
use super::utils::Backoff;
#[cfg(portable_atomic_unsafe_assume_privileged)]
use crate::imp::interrupt::arch as interrupt;
const LOCKED: State = 1;
impl SeqLock {
/// Grabs the lock for writing.
#[inline]
pub(super) fn write(&self) -> SeqLockWriteGuard<'_> {
// Get current interrupt state and disable interrupts when the user
// explicitly declares that privileged instructions are available.
#[cfg(portable_atomic_unsafe_assume_privileged)]
let interrupt_state = interrupt::disable();
let mut backoff = Backoff::new();
loop {
let previous = self.state.swap(LOCKED, Ordering::Acquire);
if previous != LOCKED {
// To synchronize with the acquire fence in `validate_read` via any modification to
// the data at the critical section of `previous` (seq_lock.rs) or `(state_hi, previous)`
// (seq_lock_wide.rs).
crate::fence(Ordering::Release);
return SeqLockWriteGuard {
lock: self,
state: previous,
#[cfg(portable_atomic_unsafe_assume_privileged)]
interrupt_state,
};
}
while self.state.load(Ordering::Relaxed) == LOCKED {
backoff.snooze();
}
}
}
}
/// An RAII guard that releases the lock and increments the stamp when dropped.
#[must_use]
pub(crate) struct SeqLockWriteGuard<'a> {
/// The parent lock.
lock: &'a SeqLock,
/// The stamp before locking.
state: State,
/// The interrupt state before disabling.
#[cfg(portable_atomic_unsafe_assume_privileged)]
interrupt_state: interrupt::State,
}
impl SeqLockWriteGuard<'_> {
/// Releases the lock without incrementing the stamp.
#[inline]
pub(crate) fn abort(self) {
// We specifically don't want to call drop(), since that's
// what increments the stamp.
let this = ManuallyDrop::new(self);
// Restore the stamp.
//
// Release ordering for synchronizing with `optimistic_read`.
this.lock.state.store(this.state, Ordering::Release);
// Restore interrupt state.
// SAFETY: the state was retrieved by the previous `disable`.
#[cfg(portable_atomic_unsafe_assume_privileged)]
unsafe {
interrupt::restore(this.interrupt_state);
}
}
}
impl Drop for SeqLockWriteGuard<'_> {
#[inline]
fn drop(&mut self) {
let state = self.next_stamp();
// Release the lock and increment the stamp.
//
// Release ordering for synchronizing with `optimistic_read`.
self.lock.state.store(state, Ordering::Release);
// Restore interrupt state.
// SAFETY: the state was retrieved by the previous `disable`.
#[cfg(portable_atomic_unsafe_assume_privileged)]
unsafe {
interrupt::restore(self.interrupt_state);
}
}
}
#[cfg(test)]
#[allow(clippy::items_after_test_module)]
mod tests {
use core::sync::atomic::Ordering;
use super::SeqLock;
#[test]
fn smoke() {
let lock = SeqLock::new();
let before = lock.optimistic_read().unwrap();
assert!(lock.validate_read(before));
{
let _guard = lock.write();
assert!(lock.optimistic_read().is_none());
}
assert!(!lock.validate_read(before));
let after = lock.optimistic_read().unwrap();
assert_ne!(before, after);
}
#[test]
fn test_wrap() {
let lock = SeqLock::new();
let zero = lock.optimistic_read().unwrap();
lock.state.store(0xFFFF_FFFE, Ordering::Relaxed); // u32::MAX & !1
let before = lock.optimistic_read().unwrap();
assert!(lock.validate_read(before));
{
let _guard = lock.write();
assert!(lock.optimistic_read().is_none());
}
assert!(!lock.validate_read(before));
let after = lock.optimistic_read().unwrap();
assert_ne!(before, after);
assert_ne!(after, zero);
#[allow(clippy::identity_op)]
{
assert_eq!(lock.state.load(Ordering::Relaxed) & 0xFFFF_FFFF, 0);
}
}
#[test]
fn test_abort() {
let lock = SeqLock::new();
let before = lock.optimistic_read().unwrap();
{
let guard = lock.write();
assert!(lock.optimistic_read().is_none());
guard.abort();
}
let after = lock.optimistic_read().unwrap();
assert_eq!(before, after, "aborted write does not update the stamp");
}
}
vendor/portable-atomic/src/imp/fallback/seq_lock_wide.rs
Vendored
+92
View File
@@ -0,0 +1,92 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/atomic/seq_lock_wide.rs.
include!("seq_lock_common.rs");
use crate::imp::core_atomic::AtomicU32;
pub(super) type State = u32;
/// A simple stamped lock.
///
/// The state is represented as two `AtomicU32`: `state_hi` for high bits and `state_lo` for low
/// bits.
pub(super) struct SeqLock {
/// The high bits of the current state of the lock.
state_hi: AtomicU32,
/// The low bits of the current state of the lock.
///
/// All bits except the least significant one hold the current stamp. When locked, the state_lo
/// equals 1 and doesn't contain a valid stamp.
state: AtomicU32,
}
impl SeqLock {
#[inline]
pub(super) const fn new() -> Self {
Self { state_hi: AtomicU32::new(0), state: AtomicU32::new(0) }
}
/// If not locked, returns the current stamp.
///
/// This method should be called before optimistic reads.
#[inline]
pub(super) fn optimistic_read(&self) -> Option<(State, State)> {
// The acquire loads from `state_hi` and `state_lo` synchronize with the release stores in
// `SeqLockWriteGuard::drop` and `SeqLockWriteGuard::abort`.
//
// As a consequence, we can make sure that (1) all writes within the era of `state_hi - 1`
// happens before now; and therefore, (2) if `state_lo` is even, all writes within the
// critical section of (`state_hi`, `state_lo`) happens before now.
let state_hi = self.state_hi.load(Ordering::Acquire);
let state_lo = self.state.load(Ordering::Acquire);
if state_lo == LOCKED { None } else { Some((state_hi, state_lo)) }
}
/// Returns `true` if the current stamp is equal to `stamp`.
///
/// This method should be called after optimistic reads to check whether they are valid. The
/// argument `stamp` should correspond to the one returned by method `optimistic_read`.
#[inline]
pub(super) fn validate_read(&self, stamp: (State, State)) -> bool {
// Thanks to the fence, if we're noticing any modification to the data at the critical
// section of `(stamp.0, stamp.1)`, then the critical section's write of 1 to state_lo
// should be visible.
crate::fence(Ordering::Acquire);
// So if `state_lo` coincides with `stamp.1`, then either (1) we're noticing no modification
// to the data after the critical section of `(stamp.0, stamp.1)`, or (2) `state_lo` wrapped
// around.
//
// If (2) is the case, the acquire ordering ensures we see the new value of `state_hi`.
let state_lo = self.state.load(Ordering::Acquire);
// If (2) is the case and `state_hi` coincides with `stamp.0`, then `state_hi` also wrapped
// around, which we give up to correctly validate the read.
let state_hi = self.state_hi.load(Ordering::Relaxed);
// Except for the case that both `state_hi` and `state_lo` wrapped around, the following
// condition implies that we're noticing no modification to the data after the critical
// section of `(stamp.0, stamp.1)`.
(state_hi, state_lo) == stamp
}
}
impl SeqLockWriteGuard<'_> {
#[inline]
fn next_stamp(&self) -> State {
let state_lo = self.state.wrapping_add(2);
// Increase the high bits if the low bits wrap around.
//
// Release ordering for synchronizing with `optimistic_read`.
if state_lo == 0 {
let state_hi = self.lock.state_hi.load(Ordering::Relaxed);
self.lock.state_hi.store(state_hi.wrapping_add(1), Ordering::Release);
}
state_lo
}
}
vendor/portable-atomic/src/imp/fallback/utils.rs
Vendored
+148
View File
@@ -0,0 +1,148 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
use core::ops;
// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/cache_padded.rs.
/// Pads and aligns a value to the length of a cache line.
// Starting from Intel's Sandy Bridge, spatial prefetcher is now pulling pairs of 64-byte cache
// lines at a time, so we have to align to 128 bytes rather than 64.
//
// Sources:
// - https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf
// - https://github.com/facebook/folly/blob/1b5288e6eea6df074758f877c849b6e73bbb9fbb/folly/lang/Align.h#L107
//
// aarch64/arm64ec's big.LITTLE architecture has asymmetric cores and "big" cores have 128-byte cache line size.
//
// Sources:
// - https://www.mono-project.com/news/2016/09/12/arm64-icache/
//
// powerpc64 has 128-byte cache line size.
//
// Sources:
// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_ppc64x.go#L9
// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/powerpc/include/asm/cache.h#L26
#[cfg_attr(
any(
target_arch = "x86_64",
target_arch = "aarch64",
target_arch = "arm64ec",
target_arch = "powerpc64",
),
repr(align(128))
)]
// arm, mips, mips64, sparc, and hexagon have 32-byte cache line size.
//
// Sources:
// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_arm.go#L7
// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips.go#L7
// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mipsle.go#L7
// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips64x.go#L9
// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/sparc/include/asm/cache.h#L17
// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/hexagon/include/asm/cache.h#L12
#[cfg_attr(
any(
target_arch = "arm",
target_arch = "mips",
target_arch = "mips32r6",
target_arch = "mips64",
target_arch = "mips64r6",
target_arch = "sparc",
target_arch = "hexagon",
),
repr(align(32))
)]
// m68k has 16-byte cache line size.
//
// Sources:
// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/m68k/include/asm/cache.h#L9
#[cfg_attr(target_arch = "m68k", repr(align(16)))]
// s390x has 256-byte cache line size.
//
// Sources:
// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_s390x.go#L7
// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/s390/include/asm/cache.h#L13
#[cfg_attr(target_arch = "s390x", repr(align(256)))]
// x86, wasm, riscv, and sparc64 have 64-byte cache line size.
//
// Sources:
// - https://github.com/golang/go/blob/dda2991c2ea0c5914714469c4defc2562a907230/src/internal/cpu/cpu_x86.go#L9
// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_wasm.go#L7
// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/riscv/include/asm/cache.h#L10
// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/sparc/include/asm/cache.h#L19
//
// All others are assumed to have 64-byte cache line size.
#[cfg_attr(
not(any(
target_arch = "x86_64",
target_arch = "aarch64",
target_arch = "arm64ec",
target_arch = "powerpc64",
target_arch = "arm",
target_arch = "mips",
target_arch = "mips32r6",
target_arch = "mips64",
target_arch = "mips64r6",
target_arch = "sparc",
target_arch = "hexagon",
target_arch = "m68k",
target_arch = "s390x",
)),
repr(align(64))
)]
#[repr(C)]
pub(crate) struct CachePadded<T> {
value: T,
}
impl<T> CachePadded<T> {
#[inline]
pub(crate) const fn new(value: T) -> Self {
Self { value }
}
}
impl<T> ops::Deref for CachePadded<T> {
type Target = T;
#[inline]
fn deref(&self) -> &Self::Target {
&self.value
}
}
// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/backoff.rs.
// Adjusted to reduce spinning.
/// Performs exponential backoff in spin loops.
pub(crate) struct Backoff {
step: u32,
}
// https://github.com/oneapi-src/oneTBB/blob/v2021.5.0/include/oneapi/tbb/detail/_utils.h#L46-L48
const SPIN_LIMIT: u32 = 4;
impl Backoff {
#[inline]
pub(crate) const fn new() -> Self {
Self { step: 0 }
}
#[inline]
pub(crate) fn snooze(&mut self) {
if self.step <= SPIN_LIMIT {
for _ in 0..1 << self.step {
#[allow(deprecated)]
core::sync::atomic::spin_loop_hint();
}
self.step += 1;
} else {
#[cfg(not(feature = "std"))]
for _ in 0..1 << self.step {
#[allow(deprecated)]
core::sync::atomic::spin_loop_hint();
}
#[cfg(feature = "std")]
std::thread::yield_now();
}
}
}
vendor/portable-atomic/src/imp/float/aarch64.rs
Vendored
+263
View File
@@ -0,0 +1,263 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Atomic float implementation based on AArch64 with FEAT_LSFE.
This module provides atomic float implementations using FEAT_LSFE instructions.
Generated asm:
- aarch64 (+lsfe) https://godbolt.org/z/7vaxeofv1
*/
#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
use core::sync::atomic::Ordering;
#[cfg(portable_atomic_unstable_f16)]
use super::int::AtomicF16;
#[cfg(portable_atomic_unstable_f128)]
use super::int::AtomicF128;
use super::int::{AtomicF32, AtomicF64};
// TODO: optimize no return cases:
// https://developer.arm.com/documentation/ddi0602/2025-06/SIMD-FP-Instructions/STFADD--STFADDL--Floating-point-atomic-add-in-memory--without-return-
// https://developer.arm.com/documentation/ddi0602/2025-06/SIMD-FP-Instructions/STFMAXNM--STFMAXNML--Floating-point-atomic-maximum-number-in-memory--without-return-
// https://developer.arm.com/documentation/ddi0602/2025-06/SIMD-FP-Instructions/STFMINNM--STFMINNML--Floating-point-atomic-minimum-number-in-memory--without-return-
#[cfg(not(portable_atomic_pre_llvm_20))]
macro_rules! start_lsfe {
() => {
".arch_extension lsfe"
};
}
#[cfg(not(portable_atomic_pre_llvm_20))]
macro_rules! atomic_rmw {
($op:ident, $order:ident) => {
atomic_rmw!($op, $order, write = $order)
};
($op:ident, $order:ident, write = $write:ident) => {
match $order {
Ordering::Relaxed => $op!("", "", ""),
Ordering::Acquire => $op!("a", "", ""),
Ordering::Release => $op!("", "l", ""),
Ordering::AcqRel => $op!("a", "l", ""),
// In MSVC environments, SeqCst stores/writes by non-LSE* instructions needs fences after writes.
// https://reviews.llvm.org/D141748
// https://github.com/llvm/llvm-project/commit/1ea201d73be2fdf03347e9c6be09ebed5f8e0e00
#[cfg(target_env = "msvc")]
Ordering::SeqCst if $write == Ordering::SeqCst => $op!("a", "l", "dmb ish"),
// AcqRel and SeqCst RMWs are equivalent in non-MSVC environments.
Ordering::SeqCst => $op!("a", "l", ""),
_ => unreachable!(),
}
};
}
#[cfg(portable_atomic_pre_llvm_20)]
macro_rules! atomic_rmw_inst {
($op:ident, $order:ident) => {
atomic_rmw_inst!($op, $order, write = $order)
};
($op:ident, $order:ident, write = $write:ident) => {
match $order {
Ordering::Relaxed => $op!("2", ""), // ""
Ordering::Acquire => $op!("a", ""), // "a"
Ordering::Release => $op!("6", ""), // "l"
Ordering::AcqRel => $op!("e", ""), // "al"
// In MSVC environments, SeqCst stores/writes by non-LSE* instructions needs fences after writes.
// https://reviews.llvm.org/D141748
// https://github.com/llvm/llvm-project/commit/1ea201d73be2fdf03347e9c6be09ebed5f8e0e00
#[cfg(target_env = "msvc")]
Ordering::SeqCst if $write == Ordering::SeqCst => $op!("e", "dmb ish"),
// AcqRel and SeqCst RMWs are equivalent in non-MSVC environments.
Ordering::SeqCst => $op!("e", ""),
_ => unreachable!(),
}
};
}
#[rustfmt::skip]
macro_rules! atomic_float {
($atomic_type:ident, $float_type:ident, $modifier:tt, $inst_modifier:tt) => {
impl $atomic_type {
#[inline]
pub(crate) fn fetch_add(&self, val: $float_type, order: Ordering) -> $float_type {
let dst = self.as_ptr();
let out;
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
//
// Refs: https://developer.arm.com/documentation/ddi0602/2025-06/SIMD-FP-Instructions/LDFADD--LDFADDA--LDFADDAL--LDFADDL--Floating-point-atomic-add-in-memory-
unsafe {
#[cfg(not(portable_atomic_pre_llvm_20))]
macro_rules! add {
($acquire:tt, $release:tt, $_msvc_fence:tt) => {
asm!(
start_lsfe!(),
concat!("ldfadd", $acquire, $release, " {val:", $modifier, "}, {out:", $modifier, "}, [{dst}]"), // atomic { _x = *dst; *dst = _x + val; out = _x }
dst = in(reg) ptr_reg!(dst),
val = in(vreg) val,
out = lateout(vreg) out,
options(nostack),
)
};
}
#[cfg(not(portable_atomic_pre_llvm_20))]
atomic_rmw!(add, order);
// LLVM supports FEAT_LSFE instructions on LLVM 20+, so use .inst directive on old LLVM.
// https://github.com/llvm/llvm-project/commit/67ff5ba9af9754261abe11d762af11532a816126
#[cfg(portable_atomic_pre_llvm_20)]
macro_rules! add {
($order:tt, $_msvc_fence:tt) => {
asm!(
// ldfadd{,a,l,al} {h,s,d}0, {h,s,d}1, [x2] // atomic { _x = *x2; *x2 = _x + {h,s,d}0; {h,s,d}1 = _x }
concat!(".inst 0x", $inst_modifier, "c", $order, "00041"),
in("x2") ptr_reg!(dst),
in("v0") val,
out("v1") out,
options(nostack),
)
};
}
#[cfg(portable_atomic_pre_llvm_20)]
atomic_rmw_inst!(add, order);
}
out
}
#[inline]
pub(crate) fn fetch_sub(&self, val: $float_type, order: Ordering) -> $float_type {
// There is no atomic sub instruction, so add `-val`.
self.fetch_add(-val, order)
}
#[inline]
pub(crate) fn fetch_max(&self, val: $float_type, order: Ordering) -> $float_type {
let dst = self.as_ptr();
let out;
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
//
// Refs: https://developer.arm.com/documentation/ddi0602/2025-06/SIMD-FP-Instructions/LDFMAXNM--LDFMAXNMA--LDFMAXNMAL--LDFMAXNML--Floating-point-atomic-maximum-number-in-memory-
unsafe {
#[cfg(not(portable_atomic_pre_llvm_20))]
macro_rules! max {
($acquire:tt, $release:tt, $_msvc_fence:tt) => {
asm!(
start_lsfe!(),
concat!("ldfmaxnm", $acquire, $release, " {val:", $modifier, "}, {out:", $modifier, "}, [{dst}]"), // atomic { _x = *dst; *dst = _x.max(val); out = _x }
dst = in(reg) ptr_reg!(dst),
val = in(vreg) val,
out = lateout(vreg) out,
options(nostack),
)
};
}
#[cfg(not(portable_atomic_pre_llvm_20))]
atomic_rmw!(max, order);
// LLVM supports FEAT_LSFE instructions on LLVM 20+, so use .inst directive on old LLVM.
// https://github.com/llvm/llvm-project/commit/67ff5ba9af9754261abe11d762af11532a816126
#[cfg(portable_atomic_pre_llvm_20)]
macro_rules! max {
($order:tt, $_msvc_fence:tt) => {
asm!(
// ldfmaxnm{,a,l,al} {h,s,d}0, {h,s,d}1, [x2] // atomic { _x = *x2; *x2 = _x.max({h,s,d}0); {h,s,d}1 = _x }
concat!(".inst 0x", $inst_modifier, "c", $order, "06041"),
in("x2") ptr_reg!(dst),
in("v0") val,
out("v1") out,
options(nostack),
)
};
}
#[cfg(portable_atomic_pre_llvm_20)]
atomic_rmw_inst!(max, order);
}
out
}
#[inline]
pub(crate) fn fetch_min(&self, val: $float_type, order: Ordering) -> $float_type {
let dst = self.as_ptr();
let out;
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
//
// Refs: https://developer.arm.com/documentation/ddi0602/2025-06/SIMD-FP-Instructions/LDFMINNM--LDFMINNMA--LDFMINNMAL--LDFMINNML--Floating-point-atomic-minimum-number-in-memory-
unsafe {
#[cfg(not(portable_atomic_pre_llvm_20))]
macro_rules! min {
($acquire:tt, $release:tt, $_msvc_fence:tt) => {
asm!(
start_lsfe!(),
concat!("ldfminnm", $acquire, $release, " {val:", $modifier, "}, {out:", $modifier, "}, [{dst}]"), // atomic { _x = *dst; *dst = _x.min(val); out = _x }
dst = in(reg) ptr_reg!(dst),
val = in(vreg) val,
out = lateout(vreg) out,
options(nostack),
)
};
}
#[cfg(not(portable_atomic_pre_llvm_20))]
atomic_rmw!(min, order);
// LLVM supports FEAT_LSFE instructions on LLVM 20+, so use .inst directive on old LLVM.
// https://github.com/llvm/llvm-project/commit/67ff5ba9af9754261abe11d762af11532a816126
#[cfg(portable_atomic_pre_llvm_20)]
macro_rules! min {
($order:tt, $_msvc_fence:tt) => {
asm!(
// ldfminnm{,a,l,al} {h,s,d}0, {h,s,d}1, [x2] // atomic { _x = *x2; *x2 = _x.min({h,s,d}0); {h,s,d}1 = _x }
concat!(".inst 0x", $inst_modifier, "c", $order, "07041"),
in("x2") ptr_reg!(dst),
in("v0") val,
out("v1") out,
options(nostack),
)
};
}
#[cfg(portable_atomic_pre_llvm_20)]
atomic_rmw_inst!(min, order);
}
out
}
}
};
}
#[cfg(portable_atomic_unstable_f16)]
atomic_float!(AtomicF16, f16, "h", "7");
atomic_float!(AtomicF32, f32, "s", "b");
atomic_float!(AtomicF64, f64, "d", "f");
#[cfg(portable_atomic_unstable_f128)]
impl AtomicF128 {
#[inline]
pub(crate) fn fetch_add(&self, val: f128, order: Ordering) -> f128 {
self.fetch_update_(order, |x| x + val)
}
#[inline]
pub(crate) fn fetch_sub(&self, val: f128, order: Ordering) -> f128 {
self.fetch_update_(order, |x| x - val)
}
#[inline]
pub(super) fn fetch_update_<F>(&self, order: Ordering, mut f: F) -> f128
where
F: FnMut(f128) -> f128,
{
// This is a private function and all instances of `f` only operate on the value
// loaded, so there is no need to synchronize the first load/failed CAS.
let mut prev = self.load(Ordering::Relaxed);
loop {
let next = f(prev);
match self.compare_exchange_weak(prev, next, order, Ordering::Relaxed) {
Ok(x) => return x,
Err(next_prev) => prev = next_prev,
}
}
}
#[inline]
pub(crate) fn fetch_max(&self, val: f128, order: Ordering) -> f128 {
self.fetch_update_(order, |x| x.max(val))
}
#[inline]
pub(crate) fn fetch_min(&self, val: f128, order: Ordering) -> f128 {
self.fetch_update_(order, |x| x.min(val))
}
}
vendor/portable-atomic/src/imp/float/int.rs
Vendored
+224
View File
@@ -0,0 +1,224 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Atomic float implementation based on atomic integer.
This module provides atomic float implementations using atomic integer.
Note that most of `fetch_*` operations of atomic floats are implemented using
CAS loops, which can be slower than equivalent operations of atomic integers.
AArch64 with FEAT_LSFE and GPU targets have atomic instructions for float.
See aarch64.rs for AArch64 with FEAT_LSFE.
GPU targets will also use architecture-specific implementations instead of this implementation in the
future: https://github.com/taiki-e/portable-atomic/issues/34 / https://github.com/taiki-e/portable-atomic/pull/45
*/
// TODO: fetch_{minimum,maximum}* https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2024/p3008r2.html / https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2024/p0493r5.pdf
#![cfg_attr(
all(target_pointer_width = "16", not(feature = "fallback")),
allow(unused_imports, unused_macros)
)]
use core::{cell::UnsafeCell, sync::atomic::Ordering};
macro_rules! atomic_float {
(
$atomic_type:ident, $float_type:ident, $atomic_int_type:ident, $int_type:ident,
$align:literal
) => {
#[repr(C, align($align))]
pub(crate) struct $atomic_type {
v: UnsafeCell<$float_type>,
}
// Send is implicitly implemented.
// SAFETY: any data races are prevented by atomic operations.
unsafe impl Sync for $atomic_type {}
impl $atomic_type {
#[inline]
pub(crate) const fn new(v: $float_type) -> Self {
Self { v: UnsafeCell::new(v) }
}
#[inline]
pub(crate) fn is_lock_free() -> bool {
crate::$atomic_int_type::is_lock_free()
}
pub(crate) const IS_ALWAYS_LOCK_FREE: bool =
crate::$atomic_int_type::is_always_lock_free();
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn load(&self, order: Ordering) -> $float_type {
$float_type::from_bits(self.as_bits().load(order))
}
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn store(&self, val: $float_type, order: Ordering) {
self.as_bits().store(val.to_bits(), order)
}
const_fn! {
const_if: #[cfg(not(portable_atomic_no_const_raw_ptr_deref))];
#[inline(always)]
pub(crate) const fn as_bits(&self) -> &crate::$atomic_int_type {
// SAFETY: $atomic_type and $atomic_int_type have the same layout,
// and there is no concurrent access to the value that does not go through this method.
unsafe { &*(self as *const Self as *const crate::$atomic_int_type) }
}
}
#[inline]
pub(crate) const fn as_ptr(&self) -> *mut $float_type {
self.v.get()
}
}
cfg_has_atomic_cas_or_amo32! {
impl $atomic_type {
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn swap(&self, val: $float_type, order: Ordering) -> $float_type {
$float_type::from_bits(self.as_bits().swap(val.to_bits(), order))
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_neg(&self, order: Ordering) -> $float_type {
const NEG_MASK: $int_type = !0 / 2 + 1;
$float_type::from_bits(self.as_bits().fetch_xor(NEG_MASK, order))
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_abs(&self, order: Ordering) -> $float_type {
const ABS_MASK: $int_type = !0 / 2;
$float_type::from_bits(self.as_bits().fetch_and(ABS_MASK, order))
}
}
cfg_has_atomic_cas! {
impl $atomic_type {
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn compare_exchange(
&self,
current: $float_type,
new: $float_type,
success: Ordering,
failure: Ordering,
) -> Result<$float_type, $float_type> {
match self.as_bits().compare_exchange(
current.to_bits(),
new.to_bits(),
success,
failure,
) {
Ok(v) => Ok($float_type::from_bits(v)),
Err(v) => Err($float_type::from_bits(v)),
}
}
#[inline]
#[cfg_attr(
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
track_caller
)]
pub(crate) fn compare_exchange_weak(
&self,
current: $float_type,
new: $float_type,
success: Ordering,
failure: Ordering,
) -> Result<$float_type, $float_type> {
match self.as_bits().compare_exchange_weak(
current.to_bits(),
new.to_bits(),
success,
failure,
) {
Ok(v) => Ok($float_type::from_bits(v)),
Err(v) => Err($float_type::from_bits(v)),
}
}
}
#[cfg(not(all(
any(target_arch = "aarch64", target_arch = "arm64ec"),
any(target_feature = "lsfe", portable_atomic_target_feature = "lsfe"),
target_feature = "neon", // for vreg
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
)))]
impl $atomic_type {
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_add(&self, val: $float_type, order: Ordering) -> $float_type {
self.fetch_update_(order, |x| x + val)
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_sub(&self, val: $float_type, order: Ordering) -> $float_type {
self.fetch_update_(order, |x| x - val)
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
fn fetch_update_<F>(&self, order: Ordering, mut f: F) -> $float_type
where
F: FnMut($float_type) -> $float_type,
{
// This is a private function and all instances of `f` only operate on the value
// loaded, so there is no need to synchronize the first load/failed CAS.
let mut prev = self.load(Ordering::Relaxed);
loop {
let next = f(prev);
match self.compare_exchange_weak(prev, next, order, Ordering::Relaxed) {
Ok(x) => return x,
Err(next_prev) => prev = next_prev,
}
}
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_max(&self, val: $float_type, order: Ordering) -> $float_type {
self.fetch_update_(order, |x| x.max(val))
}
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
pub(crate) fn fetch_min(&self, val: $float_type, order: Ordering) -> $float_type {
self.fetch_update_(order, |x| x.min(val))
}
}
} // cfg_has_atomic_cas!
} // cfg_has_atomic_cas_or_amo32!
};
}
#[cfg(portable_atomic_unstable_f16)]
cfg_has_atomic_16! {
atomic_float!(AtomicF16, f16, AtomicU16, u16, 2);
}
cfg_has_atomic_32! {
atomic_float!(AtomicF32, f32, AtomicU32, u32, 4);
}
cfg_has_atomic_64! {
atomic_float!(AtomicF64, f64, AtomicU64, u64, 8);
}
#[cfg(portable_atomic_unstable_f128)]
cfg_has_atomic_128! {
atomic_float!(AtomicF128, f128, AtomicU128, u128, 16);
}
vendor/portable-atomic/src/imp/float/mod.rs
Vendored
+33
View File
@@ -0,0 +1,33 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Atomic float implementations
*/
#![allow(clippy::float_arithmetic)]
mod int;
#[cfg(all(
any(target_arch = "aarch64", target_arch = "arm64ec"),
any(target_feature = "lsfe", portable_atomic_target_feature = "lsfe"),
target_feature = "neon", // for vreg
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
))]
mod aarch64;
#[cfg(portable_atomic_unstable_f16)]
cfg_has_atomic_16! {
pub(crate) use self::int::AtomicF16;
}
cfg_has_atomic_32! {
pub(crate) use self::int::AtomicF32;
}
cfg_has_atomic_64! {
pub(crate) use self::int::AtomicF64;
}
#[cfg(portable_atomic_unstable_f128)]
cfg_has_atomic_128! {
pub(crate) use self::int::AtomicF128;
}
vendor/portable-atomic/src/imp/interrupt/README.md
Vendored
+42
View File
@@ -0,0 +1,42 @@
# Fallback implementation based on disabling interrupts or critical-section
This module supports two different critical section implementations:
- Built-in "disable all interrupts".
- On MSP430 and AVR, they are always single-core and has no unprivileged mode, so this is enabled by default.
- On Armv6-M (thumbv6m), pre-v6 Arm (e.g., thumbv4t, thumbv5te), RISC-V without A-extension, and Xtensa, they could be multi-core or unprivileged mode, so this is enabled when the user explicitly declares that the system is single-core and that privileged instructions are available using `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg).
- Call into the `critical-section` crate (which allows the user to plug any implementation).
- This is enabled when the user asks for it with the [`critical-section` feature](../../../README.md#optional-features-critical-section).
`unsafe-assume-privileged` feature (`portable_atomic_unsafe_assume_privileged` cfg) also uses this module's interrupt disable implementation part.
The `unsafe-assume-single-core` implementation uses privileged instructions to disable interrupts, so it usually doesn't work on unprivileged mode.
Enabling this feature in an environment where privileged instructions are not available, or if the instructions used are not sufficient to disable interrupts in the system, it is also usually considered **unsound**, although the details are system-dependent.
Consider using the [`critical-section` feature](../../../README.md#optional-features-critical-section) for systems that cannot use the `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg).
For some targets, the implementation can be changed by explicitly enabling features.
- On Arm M-Profile architectures, this disables interrupts by modifying the PRIMASK register.
- On Arm (except for M-Profile architectures), this disables interrupts by modifying the I (IRQ mask) bit of the CPSR.
- On Arm (except for M-Profile architectures) with the `disable-fiq` feature (or `portable_atomic_disable_fiq` cfg), this disables interrupts by modifying the I (IRQ mask) bit and F (FIQ mask) bit of the CPSR.
- On RISC-V, this disables interrupts by modifying the MIE (Machine Interrupt Enable) bit of the `mstatus` register.
- On RISC-V with the `s-mode` feature (or `portable_atomic_s_mode` cfg), this disables interrupts by modifying the SIE (Supervisor Interrupt Enable) bit of the `sstatus` register.
- On RISC-V with the `zaamo` target feature (or `force-amo` feature or `portable_atomic_force_amo` cfg), this uses AMO instructions for RMWs that have corresponding AMO instructions even if A-extension is disabled. For other RMWs, this disables interrupts as usual.
- On MSP430, this disables interrupts by modifying the GIE (Global Interrupt Enable) bit of the status register (SR).
- On AVR, this disables interrupts by modifying the I (Global Interrupt Enable) bit of the status register (SREG).
- On Xtensa, this disables interrupts by modifying the PS special register.
<a name="no-disable-interrupts"></a>Some operations don't require disabling interrupts:
- On architectures except for AVR: loads and stores with pointer size or smaller
- On AVR: 8-bit loads and stores
- On AVR with `rmw` target feature additionally: 8-bit `swap`
- On MSP430 additionally: {8,16}-bit `add,sub,and,or,xor,not`
- On RISC-V with the `zaamo` target feature (or `portable_atomic_target_feature="zaamo"` cfg or `force-amo` feature or `portable_atomic_force_amo` cfg) additionally: 32-bit(RV32)/{32,64}-bit(RV64) `swap,fetch_{add,sub,and,or,xor,not,max,min},add,sub,and,or,xor,not`, {8,16}-bit `fetch_{and,or,xor,not},and,or,xor,not`[^1], and all operations of `AtomicBool`
However, when the `critical-section` feature is enabled, critical sections are taken for all atomic operations.
Feel free to submit an issue if your target is not supported yet.
[^1]: With the `zabha` target feature, {8,16}-bit `swap,fetch_{add,sub,max,min},add,sub` too.
vendor/portable-atomic/src/imp/interrupt/armv4t.rs
Vendored
+160
View File
@@ -0,0 +1,160 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Arm A-Profile Architectures, Arm R-Profile Architectures, Legacy Arm Architectures
Refs: https://developer.arm.com/documentation/ddi0406/cb/System-Level-Architecture/The-System-Level-Programmers--Model/ARM-processor-modes-and-ARM-core-registers/Program-Status-Registers--PSRs-
See tests/asm-test/asm/portable-atomic for generated assembly.
*/
#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
// - 0x80 - I (IRQ mask) bit (1 << 7)
// - 0x40 - F (FIQ mask) bit (1 << 6)
// We disable only IRQs by default. See also https://github.com/taiki-e/portable-atomic/pull/28#issuecomment-1214146912.
#[cfg(not(portable_atomic_disable_fiq))]
macro_rules! mask {
() => {
"0x80"
};
}
#[cfg(portable_atomic_disable_fiq)]
macro_rules! mask {
() => {
"0xC0" // 0x80 | 0x40
};
}
pub(crate) type State = u32;
/// Disables interrupts and returns the previous interrupt state.
#[inline]
#[cfg_attr(
not(any(target_feature = "v7", portable_atomic_target_feature = "v7")),
instruction_set(arm::a32)
)]
pub(crate) fn disable() -> State {
let cpsr: State;
// SAFETY: reading CPSR and disabling interrupts are safe.
// (see module-level comments of interrupt/mod.rs on the safety of using privileged instructions)
unsafe {
asm!(
"mrs {prev}, cpsr", // prev = CPSR
concat!("orr {new}, {prev}, ", mask!()), // new = prev | mask
"msr cpsr_c, {new}", // CPSR.{I,F,T,M} = new.{I,F,T,M}
prev = out(reg) cpsr,
new = out(reg) _,
// Do not use `nomem` and `readonly` because prevent subsequent memory accesses from being reordered before interrupts are disabled.
options(nostack, preserves_flags),
);
}
cpsr
}
/// Restores the previous interrupt state.
///
/// # Safety
///
/// The state must be the one retrieved by the previous `disable`.
#[inline]
#[cfg_attr(
not(any(target_feature = "v7", portable_atomic_target_feature = "v7")),
instruction_set(arm::a32)
)]
pub(crate) unsafe fn restore(prev_cpsr: State) {
// SAFETY: the caller must guarantee that the state was retrieved by the previous `disable`,
//
// This clobbers the control field mask byte of CPSR. See msp430.rs for safety on this.
// (preserves_flags is fine because we can clobber only the I, F, T, and M bits of CPSR.)
//
// Refs: https://developer.arm.com/documentation/dui0473/m/arm-and-thumb-instructions/msr--general-purpose-register-to-psr-
unsafe {
asm!(
"msr cpsr_c, {prev_cpsr}", // CPSR.{I,F,T,M} = prev_cpsr.{I,F,T,M}
prev_cpsr = in(reg) prev_cpsr,
// Do not use `nomem` and `readonly` because prevent preceding memory accesses from being reordered after interrupts are enabled.
options(nostack, preserves_flags),
);
}
}
// On pre-v6 Arm, we cannot use core::sync::atomic here because they call the
// `__sync_*` builtins for non-relaxed load/store (because pre-v6 Arm doesn't
// have Data Memory Barrier).
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(any(test, portable_atomic_no_atomic_cas)))]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(any(test, not(target_has_atomic = "ptr")))
)]
pub(super) mod atomic {
#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
use core::{cell::UnsafeCell, sync::atomic::Ordering};
macro_rules! atomic {
($([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $suffix:tt) => {
#[repr(transparent)]
pub(crate) struct $atomic_type $(<$($generics)*>)? {
v: UnsafeCell<$value_type>,
}
// Send is implicitly implemented for atomic integers, but not for atomic pointers.
// SAFETY: any data races are prevented by atomic operations.
unsafe impl $(<$($generics)*>)? Send for $atomic_type $(<$($generics)*>)? {}
// SAFETY: any data races are prevented by atomic operations.
unsafe impl $(<$($generics)*>)? Sync for $atomic_type $(<$($generics)*>)? {}
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
#[inline]
pub(crate) fn load(&self, _order: Ordering) -> $value_type {
let src = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
let out;
// inline asm without nomem/readonly implies compiler fence.
// And compiler fence is fine because the user explicitly declares that
// the system is single-core by using an unsafe cfg.
asm!(
concat!("ldr", $suffix, " {out}, [{src}]"), // atomic { out = *src }
src = in(reg) src,
out = lateout(reg) out,
options(nostack, preserves_flags),
);
out
}
}
#[inline]
pub(crate) fn store(&self, val: $value_type, _order: Ordering) {
let dst = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
// inline asm without nomem/readonly implies compiler fence.
// And compiler fence is fine because the user explicitly declares that
// the system is single-core by using an unsafe cfg.
asm!(
concat!("str", $suffix, " {val}, [{dst}]"), // atomic { *dst = val }
dst = in(reg) dst,
val = in(reg) val,
options(nostack, preserves_flags),
);
}
}
}
};
}
atomic!(AtomicI8, i8, "b");
atomic!(AtomicU8, u8, "b");
atomic!(AtomicI16, i16, "h");
atomic!(AtomicU16, u16, "h");
atomic!(AtomicI32, i32, "");
atomic!(AtomicU32, u32, "");
atomic!(AtomicIsize, isize, "");
atomic!(AtomicUsize, usize, "");
atomic!([T] AtomicPtr, *mut T, "");
}
vendor/portable-atomic/src/imp/interrupt/armv6m.rs
Vendored
+64
View File
@@ -0,0 +1,64 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Arm M-Profile Architectures
Adapted from https://github.com/rust-embedded/cortex-m.
Refs: https://developer.arm.com/documentation/ddi0419/c/System-Level-Architecture/System-Level-Programmers--Model/Registers/The-special-purpose-mask-register--PRIMASK
See tests/asm-test/asm/portable-atomic for generated assembly.
*/
#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(any(test, portable_atomic_no_atomic_cas))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(any(test, not(target_has_atomic = "ptr")))
)]
pub(super) use core::sync::atomic;
pub(crate) type State = u32;
/// Disables interrupts and returns the previous interrupt state.
#[inline(always)]
pub(crate) fn disable() -> State {
let primask: State;
// SAFETY: reading the priority mask register and disabling interrupts are safe.
// (see module-level comments of interrupt/mod.rs on the safety of using privileged instructions)
unsafe {
asm!(
"mrs {primask}, PRIMASK", // primask = PRIMASK
"cpsid i", // PRIMASK.PM = 1
primask = out(reg) primask,
// Do not use `nomem` and `readonly` because prevent subsequent memory accesses from being reordered before interrupts are disabled.
options(nostack, preserves_flags),
);
}
primask
}
/// Restores the previous interrupt state.
///
/// # Safety
///
/// The state must be the one retrieved by the previous `disable`.
#[inline(always)]
pub(crate) unsafe fn restore(prev_primask: State) {
// SAFETY: the caller must guarantee that the state was retrieved by the previous `disable`,
// and we've checked that interrupts were enabled before disabling interrupts.
//
// This clobbers the entire PRIMASK register. See msp430.rs for safety on this.
unsafe {
asm!(
"msr PRIMASK, {prev_primask}", // PRIMASK = prev_primask
prev_primask = in(reg) prev_primask,
// Do not use `nomem` and `readonly` because prevent preceding memory accesses from being reordered after interrupts are enabled.
options(nostack, preserves_flags),
);
}
}
vendor/portable-atomic/src/imp/interrupt/avr.rs
Vendored
+82
View File
@@ -0,0 +1,82 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Adapted from https://github.com/Rahix/avr-device.
See also src/imp/avr.rs.
Refs:
- AVR® Instruction Set Manual, Rev. DS40002198B
https://ww1.microchip.com/downloads/en/DeviceDoc/AVR-InstructionSet-Manual-DS40002198.pdf
See tests/asm-test/asm/portable-atomic for generated assembly.
*/
#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
#[cfg(not(portable_atomic_no_asm))]
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(any(test, portable_atomic_no_atomic_cas))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(any(test, not(target_has_atomic = "ptr")))
)]
pub(super) use super::super::avr as atomic;
pub(crate) type State = u8;
/// Disables interrupts and returns the previous interrupt state.
#[inline(always)]
pub(crate) fn disable() -> State {
let sreg: State;
// SAFETY: reading the status register (SREG) and disabling interrupts are safe.
// (see module-level comments of interrupt/mod.rs on the safety of using privileged instructions)
unsafe {
// Refs: https://ww1.microchip.com/downloads/en/DeviceDoc/AVR-InstructionSet-Manual-DS40002198.pdf#page=58
#[cfg(not(portable_atomic_no_asm))]
asm!(
"in {sreg}, 0x3F", // sreg = SREG
"cli", // SREG.I = 0
sreg = out(reg) sreg,
// Do not use `nomem` and `readonly` because prevent subsequent memory accesses from being reordered before interrupts are disabled.
// Do not use `preserves_flags` because CLI modifies the I bit of the status register (SREG).
options(nostack),
);
#[cfg(portable_atomic_no_asm)]
{
llvm_asm!("in $0, 0x3F" : "=r"(sreg) ::: "volatile");
llvm_asm!("cli" ::: "memory" : "volatile");
}
}
sreg
}
/// Restores the previous interrupt state.
///
/// # Safety
///
/// The state must be the one retrieved by the previous `disable`.
#[inline(always)]
pub(crate) unsafe fn restore(prev_sreg: State) {
// SAFETY: the caller must guarantee that the state was retrieved by the previous `disable`.
//
// This clobbers the entire status register, but all other flags are the arithmetic flags that
// are changed as a side effect of arithmetic operations, etc., which LLVM recognizes unless
// preserves_flags is set, so it is safe to clobber them here.
// See also the discussion at https://github.com/taiki-e/portable-atomic/pull/40.
unsafe {
#[cfg(not(portable_atomic_no_asm))]
asm!(
"out 0x3F, {prev_sreg}", // SREG = prev_sreg
prev_sreg = in(reg) prev_sreg,
// Do not use `nomem` and `readonly` because prevent preceding memory accesses from being reordered after interrupts are enabled.
// Do not use `preserves_flags` because OUT modifies the status register (SREG).
options(nostack),
);
#[cfg(portable_atomic_no_asm)]
llvm_asm!("out 0x3F, $0" :: "r"(prev_sreg) : "memory" : "volatile");
}
}
vendor/portable-atomic/src/imp/interrupt/mod.rs
Vendored
+696
View File
@@ -0,0 +1,696 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Fallback implementation based on disabling interrupts or critical-section
- mod.rs contains critical section based fallback implementations.
- Each architecture modules contain implementations of disabling interrupts.
See README.md of this directory for details.
*/
#[cfg(not(feature = "critical-section"))]
#[cfg_attr(
all(
target_arch = "arm",
any(target_feature = "mclass", portable_atomic_target_feature = "mclass"),
),
path = "armv6m.rs"
)]
#[cfg_attr(
all(
target_arch = "arm",
not(any(target_feature = "mclass", portable_atomic_target_feature = "mclass")),
),
path = "armv4t.rs"
)]
#[cfg_attr(target_arch = "avr", path = "avr.rs")]
#[cfg_attr(target_arch = "msp430", path = "msp430.rs")]
#[cfg_attr(any(target_arch = "riscv32", target_arch = "riscv64"), path = "riscv.rs")]
#[cfg_attr(target_arch = "xtensa", path = "xtensa.rs")]
pub(super) mod arch;
#[cfg(not(all(
portable_atomic_unsafe_assume_privileged,
not(portable_atomic_unsafe_assume_single_core),
)))]
items!({
use core::{cell::UnsafeCell, sync::atomic::Ordering};
// critical-section implementations might use locks internally.
#[cfg(feature = "critical-section")]
const IS_ALWAYS_LOCK_FREE: bool = false;
// Consider atomic operations based on disabling interrupts on single-core
// systems are lock-free. (We consider the pre-v6 Arm Linux's atomic operations
// provided in a similar way by the Linux kernel to be lock-free.)
#[cfg(not(feature = "critical-section"))]
const IS_ALWAYS_LOCK_FREE: bool = true;
// Put this in its own module to prevent guard creation.
use self::guard::disable;
mod guard {
// Note: The caller must NOT explicitly modify registers containing fields modified by disable/restore.
// (Fields modified as side effects of other operations are covered by the absence of preserves_flags,
// so they are fine -- see msp430.rs for more.)
#[inline(always)]
pub(super) fn disable() -> Guard {
Guard {
#[cfg(feature = "critical-section")]
// SAFETY: the state will be restored in the subsequent `release`.
state: unsafe { critical_section::acquire() },
#[cfg(not(feature = "critical-section"))]
// Get current interrupt state and disable interrupts.
state: super::arch::disable(),
}
}
pub(super) struct Guard {
#[cfg(feature = "critical-section")]
state: critical_section::RestoreState,
#[cfg(not(feature = "critical-section"))]
state: super::arch::State,
}
impl Drop for Guard {
#[inline(always)]
fn drop(&mut self) {
#[cfg(feature = "critical-section")]
// SAFETY: the state was retrieved by the previous `acquire`.
unsafe {
critical_section::release(self.state);
}
#[cfg(not(feature = "critical-section"))]
// Restore interrupt state.
// SAFETY: the state was retrieved by the previous `disable`.
unsafe {
super::arch::restore(self.state);
}
}
}
}
macro_rules! atomic_base {
(base, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $align:literal) => {
#[repr(C, align($align))]
pub(crate) struct $atomic_type $(<$($generics)*>)? {
v: UnsafeCell<$value_type>,
}
// Send is implicitly implemented for atomic integers, but not for atomic pointers.
// SAFETY: any data races are prevented by disabling interrupts (or
// atomic intrinsics) or critical-section (see module-level comments).
unsafe impl $(<$($generics)*>)? Send for $atomic_type $(<$($generics)*>)? {}
// SAFETY: any data races are prevented by disabling interrupts (or
// atomic intrinsics) or critical-section (see module-level comments).
unsafe impl $(<$($generics)*>)? Sync for $atomic_type $(<$($generics)*>)? {}
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
#[inline]
pub(crate) const fn new(v: $value_type) -> Self {
Self { v: UnsafeCell::new(v) }
}
#[inline]
pub(crate) fn is_lock_free() -> bool {
Self::IS_ALWAYS_LOCK_FREE
}
pub(crate) const IS_ALWAYS_LOCK_FREE: bool = IS_ALWAYS_LOCK_FREE;
#[inline]
fn read(&self, _guard: &guard::Guard) -> $value_type {
// SAFETY: any data races are prevented by disabling interrupts or critical-section (see
// module-level comments) and the raw pointer is valid because we got it
// from a reference.
unsafe { self.v.get().read() }
}
#[inline]
fn write(&self, val: $value_type, _guard: &guard::Guard) {
// SAFETY: any data races are prevented by disabling interrupts or critical-section (see
// module-level comments) and the raw pointer is valid because we got it
// from a reference.
unsafe { self.v.get().write(val) }
}
// As for CAS, there is no corresponding atomic operation on all architectures that use this code.
// (If the CAS instruction exists, all atomic operations can be implemented by it, so this code will not be used.)
#[inline]
#[cfg_attr(
all(debug_assertions, not(portable_atomic_no_track_caller)),
track_caller
)]
pub(crate) fn compare_exchange(
&self,
current: $value_type,
new: $value_type,
success: Ordering,
failure: Ordering,
) -> Result<$value_type, $value_type> {
crate::utils::assert_compare_exchange_ordering(success, failure);
let guard = disable();
let prev = self.read(&guard);
if prev == current {
self.write(new, &guard);
Ok(prev)
} else {
Err(prev)
}
}
#[inline]
#[cfg_attr(
all(debug_assertions, not(portable_atomic_no_track_caller)),
track_caller
)]
pub(crate) fn compare_exchange_weak(
&self,
current: $value_type,
new: $value_type,
success: Ordering,
failure: Ordering,
) -> Result<$value_type, $value_type> {
self.compare_exchange(current, new, success, failure)
}
#[inline]
pub(crate) const fn as_ptr(&self) -> *mut $value_type {
self.v.get()
}
}
};
(native_load_store, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty) => {
impl $(<$($generics)*>)? core::ops::Deref for $atomic_type $(<$($generics)*>)? {
type Target = atomic::$atomic_type $(<$($generics)*>)?;
#[inline(always)]
fn deref(&self) -> &Self::Target {
// SAFETY: $atomic_type and atomic::$atomic_type have the same layout and
// guarantee atomicity in a compatible way. (see module-level comments)
unsafe {
&*(self as *const Self as *const atomic::$atomic_type $(<$($generics)*>)?)
}
}
}
};
(emulate_load_store, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty) => {
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
#[inline]
#[cfg_attr(
all(debug_assertions, not(portable_atomic_no_track_caller)),
track_caller
)]
pub(crate) fn load(&self, order: Ordering) -> $value_type {
crate::utils::assert_load_ordering(order);
let guard = disable();
self.read(&guard)
}
#[inline]
#[cfg_attr(
all(debug_assertions, not(portable_atomic_no_track_caller)),
track_caller
)]
pub(crate) fn store(&self, val: $value_type, order: Ordering) {
crate::utils::assert_store_ordering(order);
let guard = disable();
self.write(val, &guard);
}
}
};
(emulate_swap, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty) => {
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
#[inline]
pub(crate) fn swap(&self, val: $value_type, _order: Ordering) -> $value_type {
let guard = disable();
let prev = self.read(&guard);
self.write(val, &guard);
prev
}
}
};
}
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(any(
test,
target_arch = "avr",
target_arch = "msp430",
portable_atomic_no_atomic_cas
))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(any(
test,
target_arch = "avr",
target_arch = "msp430",
not(target_has_atomic = "ptr")
))
)]
items!({
#[cfg(target_pointer_width = "16")]
atomic_base!(base, [T] AtomicPtr, *mut T, 2);
#[cfg(target_pointer_width = "32")]
atomic_base!(base, [T] AtomicPtr, *mut T, 4);
#[cfg(target_pointer_width = "64")]
atomic_base!(base, [T] AtomicPtr, *mut T, 8);
#[cfg(target_pointer_width = "128")]
atomic_base!(base, [T] AtomicPtr, *mut T, 16);
impl_default_bit_opts!(AtomicPtr, usize);
cfg_sel!({
#[cfg(any(target_arch = "avr", feature = "critical-section"))]
{
atomic_base!(emulate_load_store, [T] AtomicPtr, *mut T);
}
#[cfg(else)]
{
atomic_base!(native_load_store, [T] AtomicPtr, *mut T);
}
});
#[cfg(not(all(
any(target_arch = "riscv32", target_arch = "riscv64"),
not(feature = "critical-section"),
any(
portable_atomic_force_amo,
target_feature = "zaamo",
portable_atomic_target_feature = "zaamo",
),
)))]
items!({
atomic_base!(emulate_swap, [T] AtomicPtr, *mut T);
impl<T> AtomicPtr<T> {
#[inline]
pub(crate) fn fetch_byte_add(&self, val: usize, _order: Ordering) -> *mut T {
#[cfg(portable_atomic_no_strict_provenance)]
use crate::utils::ptr::PtrExt as _;
let guard = disable();
let prev = self.read(&guard);
self.write(prev.with_addr(prev.addr().wrapping_add(val)), &guard);
prev
}
#[inline]
pub(crate) fn fetch_byte_sub(&self, val: usize, _order: Ordering) -> *mut T {
#[cfg(portable_atomic_no_strict_provenance)]
use crate::utils::ptr::PtrExt as _;
let guard = disable();
let prev = self.read(&guard);
self.write(prev.with_addr(prev.addr().wrapping_sub(val)), &guard);
prev
}
#[inline]
pub(crate) fn fetch_and(&self, val: usize, _order: Ordering) -> *mut T {
#[cfg(portable_atomic_no_strict_provenance)]
use crate::utils::ptr::PtrExt as _;
let guard = disable();
let prev = self.read(&guard);
self.write(prev.with_addr(prev.addr() & val), &guard);
prev
}
#[inline]
pub(crate) fn fetch_or(&self, val: usize, _order: Ordering) -> *mut T {
#[cfg(portable_atomic_no_strict_provenance)]
use crate::utils::ptr::PtrExt as _;
let guard = disable();
let prev = self.read(&guard);
self.write(prev.with_addr(prev.addr() | val), &guard);
prev
}
#[inline]
pub(crate) fn fetch_xor(&self, val: usize, _order: Ordering) -> *mut T {
#[cfg(portable_atomic_no_strict_provenance)]
use crate::utils::ptr::PtrExt as _;
let guard = disable();
let prev = self.read(&guard);
self.write(prev.with_addr(prev.addr() ^ val), &guard);
prev
}
}
});
impl<T> AtomicPtr<T> {
#[cfg(test)]
#[inline]
fn fetch_ptr_add(&self, val: usize, order: Ordering) -> *mut T {
self.fetch_byte_add(val.wrapping_mul(core::mem::size_of::<T>()), order)
}
#[cfg(test)]
#[inline]
fn fetch_ptr_sub(&self, val: usize, order: Ordering) -> *mut T {
self.fetch_byte_sub(val.wrapping_mul(core::mem::size_of::<T>()), order)
}
}
});
macro_rules! atomic_int {
(base, $atomic_type:ident, $int_type:ty, $align:literal) => {
atomic_base!(base, $atomic_type, $int_type, $align);
// As for nand and neg, there is no corresponding atomic operation on all architectures that use this code.
impl $atomic_type {
#[inline]
pub(crate) fn fetch_nand(&self, val: $int_type, _order: Ordering) -> $int_type {
let guard = disable();
let prev = self.read(&guard);
self.write(!(prev & val), &guard);
prev
}
#[inline]
pub(crate) fn fetch_neg(&self, _order: Ordering) -> $int_type {
let guard = disable();
let prev = self.read(&guard);
self.write(prev.wrapping_neg(), &guard);
prev
}
#[inline]
pub(crate) fn neg(&self, order: Ordering) {
self.fetch_neg(order);
}
}
};
(load_store_atomic $([$kind:ident])?, $atomic_type:ident, $int_type:ty, $align:literal) => {
cfg_sel!({
#[cfg(feature = "critical-section")]
{
atomic_int!(all_critical_session, $atomic_type, $int_type, $align);
}
#[cfg(else)]
{
atomic_int!(base, $atomic_type, $int_type, $align);
impl_default_bit_opts!($atomic_type, $int_type);
// load/store
cfg_sel!({
// AVR with very old rustc
#[cfg(all(target_arch = "avr", portable_atomic_no_asm))]
{
atomic_base!(emulate_load_store, $atomic_type, $int_type);
}
#[cfg(else)]
{
atomic_base!(native_load_store, $atomic_type, $int_type);
}
});
// RMW
cfg_sel!({
// AVR 8-bit RMW with RMW instructions
#[cfg(all(
target_arch = "avr",
not(portable_atomic_no_asm),
any(target_feature = "rmw", portable_atomic_target_feature = "rmw"),
))]
{
atomic_int!(emulate_arithmetic, $atomic_type, $int_type);
}
// RISC-V RMW with Zaamo extension
#[cfg(all(
any(target_arch = "riscv32", target_arch = "riscv64"),
any(
portable_atomic_force_amo,
target_feature = "zaamo",
portable_atomic_target_feature = "zaamo",
),
))]
{
atomic_int!(cas $([$kind])?, $atomic_type, $int_type);
}
#[cfg(else)]
{
atomic_int!(cas[emulate], $atomic_type, $int_type);
}
});
// RMW (no-fetch)
#[cfg(not(target_arch = "msp430"))]
items!({
impl_default_no_fetch_ops!($atomic_type, $int_type);
impl $atomic_type {
#[inline]
pub(crate) fn not(&self, order: Ordering) {
self.fetch_not(order);
}
}
});
}
});
};
(all_critical_session, $atomic_type:ident, $int_type:ty, $align:literal) => {
atomic_int!(base, $atomic_type, $int_type, $align);
atomic_base!(emulate_load_store, $atomic_type, $int_type);
atomic_int!(cas[emulate], $atomic_type, $int_type);
impl_default_no_fetch_ops!($atomic_type, $int_type);
impl_default_bit_opts!($atomic_type, $int_type);
impl $atomic_type {
#[inline]
pub(crate) fn not(&self, order: Ordering) {
self.fetch_not(order);
}
}
};
(emulate_arithmetic, $atomic_type:ident, $int_type:ty) => {
impl $atomic_type {
#[inline]
pub(crate) fn fetch_add(&self, val: $int_type, _order: Ordering) -> $int_type {
let guard = disable();
let prev = self.read(&guard);
self.write(prev.wrapping_add(val), &guard);
prev
}
#[inline]
pub(crate) fn fetch_sub(&self, val: $int_type, _order: Ordering) -> $int_type {
let guard = disable();
let prev = self.read(&guard);
self.write(prev.wrapping_sub(val), &guard);
prev
}
#[inline]
pub(crate) fn fetch_max(&self, val: $int_type, _order: Ordering) -> $int_type {
let guard = disable();
let prev = self.read(&guard);
self.write(core::cmp::max(prev, val), &guard);
prev
}
#[inline]
pub(crate) fn fetch_min(&self, val: $int_type, _order: Ordering) -> $int_type {
let guard = disable();
let prev = self.read(&guard);
self.write(core::cmp::min(prev, val), &guard);
prev
}
}
};
(emulate_bit, $atomic_type:ident, $int_type:ty) => {
impl $atomic_type {
#[inline]
pub(crate) fn fetch_and(&self, val: $int_type, _order: Ordering) -> $int_type {
let guard = disable();
let prev = self.read(&guard);
self.write(prev & val, &guard);
prev
}
#[inline]
pub(crate) fn fetch_or(&self, val: $int_type, _order: Ordering) -> $int_type {
let guard = disable();
let prev = self.read(&guard);
self.write(prev | val, &guard);
prev
}
#[inline]
pub(crate) fn fetch_xor(&self, val: $int_type, _order: Ordering) -> $int_type {
let guard = disable();
let prev = self.read(&guard);
self.write(prev ^ val, &guard);
prev
}
#[inline]
pub(crate) fn fetch_not(&self, _order: Ordering) -> $int_type {
let guard = disable();
let prev = self.read(&guard);
self.write(!prev, &guard);
prev
}
}
};
(cas[emulate], $atomic_type:ident, $int_type:ty) => {
atomic_base!(emulate_swap, $atomic_type, $int_type);
atomic_int!(emulate_arithmetic, $atomic_type, $int_type);
atomic_int!(emulate_bit, $atomic_type, $int_type);
};
// RISC-V 32-bit(RV32)/{32,64}-bit(RV64) RMW with Zaamo extension
// RISC-V 8-bit/16-bit RMW with Zabha extension
(cas, $atomic_type:ident, $int_type:ty) => {};
// RISC-V 8-bit/16-bit RMW with Zaamo extension
(cas[sub_word], $atomic_type:ident, $int_type:ty) => {
// RISC-V 8-bit/16-bit RMW with Zaamo+Zabha extension
#[cfg(any(target_feature = "zabha", portable_atomic_target_feature = "zabha"))]
atomic_int!(cas, $atomic_type, $int_type);
// RISC-V 8-bit/16-bit RMW with Zaamo extension
#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
atomic_base!(emulate_swap, $atomic_type, $int_type);
#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
atomic_int!(emulate_arithmetic, $atomic_type, $int_type);
};
}
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(any(
test,
target_arch = "avr",
target_arch = "msp430",
portable_atomic_no_atomic_cas,
))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(any(
test,
target_arch = "avr",
target_arch = "msp430",
not(target_has_atomic = "ptr"),
))
)]
items!({
// On some platforms, atomic load/store can be implemented in a more efficient
// way than disabling interrupts. On MSP430, some RMWs that do not return the
// previous value can also be optimized.
//
// Note: On single-core systems, it is okay to use critical session-based
// CAS together with atomic load/store. The load/store will not be
// called while interrupts are disabled, and since the load/store is
// atomic, it is not affected by interrupts even if interrupts are enabled.
#[cfg(not(any(
all(target_arch = "avr", portable_atomic_no_asm),
feature = "critical-section",
)))]
use self::arch::atomic;
#[cfg(target_pointer_width = "16")]
#[cfg(not(target_arch = "avr"))]
atomic_int!(load_store_atomic, AtomicIsize, isize, 2);
#[cfg(target_pointer_width = "16")]
#[cfg(not(target_arch = "avr"))]
atomic_int!(load_store_atomic, AtomicUsize, usize, 2);
#[cfg(target_arch = "avr")]
atomic_int!(all_critical_session, AtomicIsize, isize, 2);
#[cfg(target_arch = "avr")]
atomic_int!(all_critical_session, AtomicUsize, usize, 2);
#[cfg(target_pointer_width = "32")]
atomic_int!(load_store_atomic, AtomicIsize, isize, 4);
#[cfg(target_pointer_width = "32")]
atomic_int!(load_store_atomic, AtomicUsize, usize, 4);
#[cfg(target_pointer_width = "64")]
atomic_int!(load_store_atomic, AtomicIsize, isize, 8);
#[cfg(target_pointer_width = "64")]
atomic_int!(load_store_atomic, AtomicUsize, usize, 8);
#[cfg(target_pointer_width = "128")]
atomic_int!(load_store_atomic, AtomicIsize, isize, 16);
#[cfg(target_pointer_width = "128")]
atomic_int!(load_store_atomic, AtomicUsize, usize, 16);
#[cfg(not(all(target_arch = "avr", portable_atomic_no_asm)))]
atomic_int!(load_store_atomic[sub_word], AtomicI8, i8, 1);
#[cfg(not(all(target_arch = "avr", portable_atomic_no_asm)))]
atomic_int!(load_store_atomic[sub_word], AtomicU8, u8, 1);
#[cfg(all(target_arch = "avr", portable_atomic_no_asm))]
atomic_int!(all_critical_session, AtomicI8, i8, 1);
#[cfg(all(target_arch = "avr", portable_atomic_no_asm))]
atomic_int!(all_critical_session, AtomicU8, u8, 1);
#[cfg(not(target_arch = "avr"))]
atomic_int!(load_store_atomic[sub_word], AtomicI16, i16, 2);
#[cfg(not(target_arch = "avr"))]
atomic_int!(load_store_atomic[sub_word], AtomicU16, u16, 2);
#[cfg(target_arch = "avr")]
atomic_int!(all_critical_session, AtomicI16, i16, 2);
#[cfg(target_arch = "avr")]
atomic_int!(all_critical_session, AtomicU16, u16, 2);
#[cfg(not(target_pointer_width = "16"))]
atomic_int!(load_store_atomic, AtomicI32, i32, 4);
#[cfg(not(target_pointer_width = "16"))]
atomic_int!(load_store_atomic, AtomicU32, u32, 4);
cfg_has_fast_atomic_64!({
atomic_int!(load_store_atomic, AtomicI64, i64, 8);
atomic_int!(load_store_atomic, AtomicU64, u64, 8);
});
});
// Double or more width atomics (require fallback feature for consistency with other situations).
#[cfg(target_pointer_width = "16")]
#[cfg(any(test, feature = "fallback"))]
items!({
atomic_int!(all_critical_session, AtomicI32, i32, 4);
atomic_int!(all_critical_session, AtomicU32, u32, 4);
});
#[cfg(any(
test,
all(
feature = "fallback",
not(all(
target_arch = "riscv32",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(
target_feature = "zacas",
portable_atomic_target_feature = "zacas",
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
any(target_os = "linux", target_os = "android"),
),
),
)),
),
))]
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(any(test, portable_atomic_no_atomic_64))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(any(test, not(target_has_atomic = "64")))
)]
cfg_no_fast_atomic_64!({
atomic_int!(all_critical_session, AtomicI64, i64, 8);
atomic_int!(all_critical_session, AtomicU64, u64, 8);
});
#[cfg(any(
test,
all(
feature = "fallback",
not(all(
target_arch = "riscv64",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(
target_feature = "zacas",
portable_atomic_target_feature = "zacas",
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
any(target_os = "linux", target_os = "android"),
),
),
)),
),
))]
items!({
atomic_int!(all_critical_session, AtomicI128, i128, 16);
atomic_int!(all_critical_session, AtomicU128, u128, 16);
});
});
#[cfg(test)]
mod tests {
use super::*;
test_atomic_ptr_single_thread!();
test_atomic_int_single_thread!(i8);
test_atomic_int_single_thread!(u8);
test_atomic_int_single_thread!(i16);
test_atomic_int_single_thread!(u16);
test_atomic_int_single_thread!(i32);
test_atomic_int_single_thread!(u32);
test_atomic_int_single_thread!(i64);
test_atomic_int_single_thread!(u64);
test_atomic_int_single_thread!(i128);
test_atomic_int_single_thread!(u128);
test_atomic_int_single_thread!(isize);
test_atomic_int_single_thread!(usize);
}
vendor/portable-atomic/src/imp/interrupt/msp430.rs
Vendored
+86
View File
@@ -0,0 +1,86 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Adapted from https://github.com/rust-embedded/msp430.
See also src/imp/msp430.rs.
Refs:
- MSP430x5xx and MSP430x6xx Family User's Guide, Rev. Q
https://www.ti.com/lit/ug/slau208q/slau208q.pdf
See tests/asm-test/asm/portable-atomic for generated assembly.
*/
#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(any(test, portable_atomic_no_atomic_cas))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(any(test, not(target_has_atomic = "ptr")))
)]
pub(super) use super::super::msp430 as atomic;
pub(crate) type State = u16;
/// Disables interrupts and returns the previous interrupt state.
#[inline(always)]
pub(crate) fn disable() -> State {
let sr: State;
// SAFETY: reading the status register and disabling interrupts are safe.
// (see module-level comments of interrupt/mod.rs on the safety of using privileged instructions)
//
// See "NOTE: Enable and Disable Interrupt" of User's Guide for NOP: https://www.ti.com/lit/ug/slau208q/slau208q.pdf#page=60
unsafe {
#[cfg(not(portable_atomic_no_asm))]
asm!(
"mov r2, {sr}", // sr = SR
"dint {{ nop", // SR.GIE = 0
sr = out(reg) sr,
// Do not use `nomem` and `readonly` because prevent subsequent memory accesses from being reordered before interrupts are disabled.
// Do not use `preserves_flags` because DINT modifies the GIE (global interrupt enable) bit of the status register.
options(nostack),
);
#[cfg(portable_atomic_no_asm)]
{
llvm_asm!("mov r2, $0" : "=r"(sr) ::: "volatile");
llvm_asm!("dint { nop" ::: "memory" : "volatile");
}
}
sr
}
/// Restores the previous interrupt state.
///
/// # Safety
///
/// The state must be the one retrieved by the previous `disable`.
#[inline(always)]
pub(crate) unsafe fn restore(prev_sr: State) {
// SAFETY: the caller must guarantee that the state was retrieved by the previous `disable`,
//
// This clobbers the entire status register, but we never explicitly modify
// flags within a critical session, and the only flags that may be changed
// within a critical session are the arithmetic flags that are changed as
// a side effect of arithmetic operations, etc., which LLVM recognizes unless
// preserves_flags is set, so it is safe to clobber them here.
// See also the discussion at https://github.com/taiki-e/portable-atomic/pull/40.
//
// See "NOTE: Enable and Disable Interrupt" of User's Guide for NOP: https://www.ti.com/lit/ug/slau208q/slau208q.pdf#page=60
unsafe {
#[cfg(not(portable_atomic_no_asm))]
asm!(
"nop {{ mov {prev_sr}, r2 {{ nop", // SR = prev_sr
prev_sr = in(reg) prev_sr,
// Do not use `nomem` and `readonly` because prevent preceding memory accesses from being reordered after interrupts are enabled.
// Do not use `preserves_flags` because MOV modifies the status register.
options(nostack),
);
#[cfg(portable_atomic_no_asm)]
llvm_asm!("nop { mov $0, r2 { nop" :: "r"(prev_sr) : "memory" : "volatile");
}
}
vendor/portable-atomic/src/imp/interrupt/riscv.rs
Vendored
+104
View File
@@ -0,0 +1,104 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Refs:
- RISC-V Instruction Set Manual
Machine Status (mstatus and mstatush) Registers
https://docs.riscv.org/reference/isa/v20260120/priv/machine.html#3-1-1-6-machine-status-mstatus-and-mstatush-registers
Supervisor Status (sstatus) Register
https://docs.riscv.org/reference/isa/v20260120/priv/supervisor.html#sstatus
See also src/imp/riscv.rs.
See tests/asm-test/asm/portable-atomic for generated assembly.
*/
#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(any(test, portable_atomic_no_atomic_cas))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(any(test, not(target_has_atomic = "ptr")))
)]
pub(super) use super::super::riscv as atomic;
cfg_sel!({
// Supervisor-mode (S-mode)
#[cfg(portable_atomic_s_mode)]
{
// Status register
macro_rules! status {
() => {
"sstatus"
};
}
// SIE (Supervisor Interrupt Enable) bit (1 << 1)
#[cfg(portable_atomic_s_mode)]
macro_rules! mask {
() => {
"0x2"
};
}
const MASK: State = 0x2;
}
// Machine-mode (M-mode)
#[cfg(else)]
{
// Status register
macro_rules! status {
() => {
"mstatus"
};
}
// MIE (Machine Interrupt Enable) bit (1 << 3)
macro_rules! mask {
() => {
"0x8"
};
}
const MASK: State = 0x8;
}
});
pub(crate) type State = crate::utils::RegSize;
/// Disables interrupts and returns the previous interrupt state.
#[inline(always)]
pub(crate) fn disable() -> State {
let status: State;
// SAFETY: reading mstatus/sstatus and disabling interrupts is safe.
// (see module-level comments of interrupt/mod.rs on the safety of using privileged instructions)
unsafe {
asm!(
concat!("csrrci {status}, ", status!(), ", ", mask!()), // atomic { status = status!(); status!() &= !mask!() }
status = out(reg) status,
// Do not use `nomem` and `readonly` because prevent subsequent memory accesses from being reordered before interrupts are disabled.
options(nostack, preserves_flags),
);
}
status
}
/// Restores the previous interrupt state.
///
/// # Safety
///
/// The state must be the one retrieved by the previous `disable`.
#[inline(always)]
pub(crate) unsafe fn restore(prev_status: State) {
// SAFETY: the caller must guarantee that the state was retrieved by the previous `disable`,
//
// This avoids clobbering other fields in the mstatus/sstatus register.
unsafe {
asm!(
concat!("csrs ", status!(), ", {mask}"), // atomic { status!() |= mask }
mask = in(reg) prev_status & MASK,
// Do not use `nomem` and `readonly` because prevent preceding memory accesses from being reordered after interrupts are enabled.
options(nostack, preserves_flags),
);
}
}
vendor/portable-atomic/src/imp/interrupt/xtensa.rs
Vendored
+68
View File
@@ -0,0 +1,68 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Refs:
- Xtensa Instruction Set Architecture (ISA) Summary for all Xtensa LX Processors
https://www.cadence.com/content/dam/cadence-www/global/en_US/documents/tools/silicon-solutions/compute-ip/isa-summary.pdf
- Linux kernel's Xtensa atomic implementation
https://github.com/torvalds/linux/blob/v6.19/arch/xtensa/include/asm/atomic.h
See tests/asm-test/asm/portable-atomic for generated assembly.
*/
use core::arch::asm;
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(any(test, portable_atomic_no_atomic_cas))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(any(test, not(target_has_atomic = "ptr")))
)]
pub(super) use core::sync::atomic;
pub(crate) type State = u32;
/// Disables interrupts and returns the previous interrupt state.
#[inline(always)]
pub(crate) fn disable() -> State {
let ps: State;
// SAFETY: reading the PS special register and disabling all interrupts is safe.
// (see module-level comments of interrupt/mod.rs on the safety of using privileged instructions)
//
// Interrupt level 15 to disable all interrupts.
// SYNC after RSIL is not required.
// Use a14 per https://github.com/torvalds/linux/blob/v6.16/arch/xtensa/include/asm/atomic.h#L33-L37.
unsafe {
asm!(
"rsil a14, 15", // a14 = PS; PS.INTLEVEL = 15
out("a14") ps,
// Do not use `nomem` and `readonly` because prevent subsequent memory accesses from being reordered before interrupts are disabled.
options(nostack),
);
}
ps
}
/// Restores the previous interrupt state.
///
/// # Safety
///
/// The state must be the one retrieved by the previous `disable`.
#[inline(always)]
pub(crate) unsafe fn restore(prev_ps: State) {
// SAFETY: the caller must guarantee that the state was retrieved by the previous `disable`,
// and we've checked that interrupts were enabled before disabling interrupts.
//
// SYNC after WSR is required to guarantee that subsequent RSIL read the written value.
// See also 3.8.10 Processor Control Instructions of Xtensa Instruction Set Architecture (ISA) Summary for all Xtensa LX Processors.
unsafe {
asm!(
"wsr.ps a14", // PS = a14
"rsync", // wait
in("a14") prev_ps,
// Do not use `nomem` and `readonly` because prevent preceding memory accesses from being reordered after interrupts are enabled.
options(nostack),
);
}
}
vendor/portable-atomic/src/imp/mod.rs
Vendored
+566
View File
@@ -0,0 +1,566 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
// -----------------------------------------------------------------------------
// Lock-free implementations
cfg_core_atomic!({
mod core_atomic;
});
// AVR
#[cfg(target_arch = "avr")]
#[cfg(not(portable_atomic_no_asm))]
#[cfg(not(feature = "critical-section"))]
mod avr;
// pre-v6 Arm Linux
#[cfg(all(
target_arch = "arm",
not(any(miri, portable_atomic_sanitize_thread)),
any(target_os = "linux", target_os = "android"),
not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
))]
pub(crate) mod arm_linux;
// MSP430
#[cfg(target_arch = "msp430")]
pub(crate) mod msp430;
// RISC-V without A-extension
#[cfg(any(test, not(feature = "critical-section")))]
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(any(
all(test, not(any(miri, portable_atomic_sanitize_thread))),
portable_atomic_no_atomic_cas,
))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(any(
all(test, not(any(miri, portable_atomic_sanitize_thread))),
not(target_has_atomic = "ptr"),
))
)]
#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
mod riscv;
// x86-specific optimizations
// Miri and Sanitizer do not support inline assembly.
#[cfg(all(
any(target_arch = "x86", target_arch = "x86_64"),
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
))]
pub(crate) mod x86;
// 64-bit atomic implementations on 32-bit architectures
#[cfg(any(target_arch = "arm", target_arch = "riscv32"))]
mod atomic64;
// 128-bit atomic implementations on 64-bit architectures
#[cfg(any(
target_arch = "aarch64",
target_arch = "arm64ec",
target_arch = "powerpc64",
target_arch = "riscv64",
target_arch = "s390x",
target_arch = "x86_64",
))]
mod atomic128;
// -----------------------------------------------------------------------------
// Lock-based fallback implementations
#[cfg(feature = "fallback")]
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
#[cfg_attr(
not(test),
cfg(not(any(
// ---------------
// Use other ways.
target_arch = "avr",
target_arch = "msp430",
portable_atomic_unsafe_assume_single_core,
// --------------------
// Has 128-bit atomics.
all(
target_arch = "aarch64",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
),
all(
target_arch = "arm64ec",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
not(portable_atomic_no_asm),
),
all(
target_arch = "x86_64",
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
),
all(
target_arch = "riscv64",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
),
all(
target_arch = "powerpc64",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
not(portable_atomic_no_asm),
any(
target_feature = "quadword-atomics",
portable_atomic_target_feature = "quadword-atomics",
),
),
all(
target_arch = "s390x",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
not(portable_atomic_no_asm),
),
))
))]
mod fallback;
// -----------------------------------------------------------------------------
// Fallback implementation based on disabling interrupts or critical-section
// On AVR, we always use critical section based fallback implementation.
// AVR can be safely assumed to be single-core, so this is sound.
// MSP430 as well.
// See the module-level comments of interrupt module for more.
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(any(
target_arch = "avr",
target_arch = "msp430",
all(feature = "critical-section", any(test, portable_atomic_no_atomic_cas)),
portable_atomic_unsafe_assume_single_core,
portable_atomic_unsafe_assume_privileged,
))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(any(
target_arch = "avr",
target_arch = "msp430",
all(feature = "critical-section", any(test, not(target_has_atomic = "ptr"))),
portable_atomic_unsafe_assume_single_core,
portable_atomic_unsafe_assume_privileged,
))
)]
#[cfg(any(
target_arch = "arm",
target_arch = "avr",
target_arch = "msp430",
target_arch = "riscv32",
target_arch = "riscv64",
target_arch = "xtensa",
feature = "critical-section",
))]
mod interrupt;
// -----------------------------------------------------------------------------
// Atomic float implementations
#[cfg(feature = "float")]
pub(crate) mod float;
// -----------------------------------------------------------------------------
// {8,16,32}-bit & ptr-sized atomics
cfg_sel!({
// has CAS | (has core atomic & !(avr | msp430 | critical section)) => core atomic
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(not(any(
target_arch = "avr",
target_arch = "msp430",
portable_atomic_no_atomic_load_store,
all(
any(
target_arch = "riscv32",
target_arch = "riscv64",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
),
portable_atomic_no_atomic_cas,
),
)))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(not(any(
target_arch = "avr",
target_arch = "msp430",
portable_atomic_no_atomic_load_store,
all(
any(
target_arch = "riscv32",
target_arch = "riscv64",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
),
not(target_has_atomic = "ptr"),
),
)))
)]
{
pub(crate) use self::core_atomic::{
AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, AtomicU32,
AtomicUsize,
};
}
// no native atomic CAS & (assume single core | critical section) => critical section based fallback
#[cfg(any(
target_arch = "avr",
target_arch = "msp430",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
))]
{
pub(crate) use self::interrupt::{
AtomicI8, AtomicI16, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, AtomicUsize,
};
#[cfg(any(not(target_pointer_width = "16"), feature = "fallback"))]
pub(crate) use self::interrupt::{AtomicI32, AtomicU32};
}
// RISC-V without A-extension & !(assume single core | critical section)
#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
{
pub(crate) use self::riscv::{
AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, AtomicU32,
AtomicUsize,
};
}
// bpf & !(critical section) => core atomic
#[cfg(target_arch = "bpf")]
{
pub(crate) use self::core_atomic::{AtomicIsize, AtomicPtr, AtomicUsize};
}
#[cfg(else)]
{
// no atomic
}
});
// 64-bit atomics
cfg_sel!({
// has CAS | (has core atomic & !(avr | msp430 | critical section)) => core atomic
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(all(
not(any(
target_arch = "avr",
target_arch = "msp430",
portable_atomic_no_atomic_load_store,
all(
any(
target_arch = "riscv32",
target_arch = "riscv64",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
),
portable_atomic_no_atomic_cas,
)
)),
any(
not(portable_atomic_no_atomic_64),
not(any(target_pointer_width = "16", target_pointer_width = "32")),
),
))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(all(
not(any(
target_arch = "avr",
target_arch = "msp430",
portable_atomic_no_atomic_load_store,
all(
any(
target_arch = "riscv32",
target_arch = "riscv64",
feature = "critical-section",
portable_atomic_unsafe_assume_single_core,
),
not(target_has_atomic = "ptr"),
)
)),
any(
target_has_atomic = "64",
not(any(target_pointer_width = "16", target_pointer_width = "32")),
),
))
)]
{
pub(crate) use self::core_atomic::{AtomicI64, AtomicU64};
}
// pre-v6 Arm Linux
#[cfg(all(
feature = "fallback",
target_arch = "arm",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_os = "linux", target_os = "android"),
not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
not(portable_atomic_no_outline_atomics),
))]
{
pub(crate) use self::atomic64::arm_linux::{AtomicI64, AtomicU64};
}
// riscv32 & (zacas | outline-atomics)
#[cfg(all(
target_arch = "riscv32",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(
target_feature = "zacas",
portable_atomic_target_feature = "zacas",
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
any(target_os = "linux", target_os = "android"),
),
),
))]
{
pub(crate) use self::atomic64::riscv32::{AtomicI64, AtomicU64};
}
// no native atomic CAS & (assume single core | critical section) => critical section based fallback
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(any(
target_arch = "avr",
target_arch = "msp430",
all(feature = "critical-section", portable_atomic_no_atomic_cas),
portable_atomic_unsafe_assume_single_core,
))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(any(
target_arch = "avr",
target_arch = "msp430",
all(feature = "critical-section", not(target_has_atomic = "ptr")),
portable_atomic_unsafe_assume_single_core,
))
)]
{
#[cfg(any(
not(any(target_pointer_width = "16", target_pointer_width = "32")),
feature = "fallback",
))]
pub(crate) use self::interrupt::{AtomicI64, AtomicU64};
}
// no core 64-bit atomic & has CAS => use lock-base fallback
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(all(feature = "fallback", not(portable_atomic_no_atomic_cas)))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(all(feature = "fallback", target_has_atomic = "ptr"))
)]
{
pub(crate) use self::fallback::{AtomicI64, AtomicU64};
}
// RISC-V without A-extension & !(assume single core | critical section)
#[cfg(target_arch = "riscv64")]
{
pub(crate) use self::riscv::{AtomicI64, AtomicU64};
}
// bpf & !(critical section) => core atomic
#[cfg(target_arch = "bpf")]
{
pub(crate) use self::core_atomic::{AtomicI64, AtomicU64};
}
#[cfg(else)]
{
// no atomic
}
});
// 128-bit atomics
cfg_sel!({
// AArch64
#[cfg(any(
all(
target_arch = "aarch64",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
),
all(
target_arch = "arm64ec",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
not(portable_atomic_no_asm),
),
))]
{
pub(crate) use self::atomic128::aarch64::{AtomicI128, AtomicU128};
}
// x86_64 & (cmpxchg16b | outline-atomics)
#[cfg(all(
target_arch = "x86_64",
not(all(
any(miri, portable_atomic_sanitize_thread),
portable_atomic_no_cmpxchg16b_intrinsic,
)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(
target_feature = "cmpxchg16b",
portable_atomic_target_feature = "cmpxchg16b",
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
not(any(target_env = "sgx", miri)),
),
),
))]
{
pub(crate) use self::atomic128::x86_64::{AtomicI128, AtomicU128};
}
// riscv64 & (zacas | outline-atomics)
#[cfg(all(
target_arch = "riscv64",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(
target_feature = "zacas",
portable_atomic_target_feature = "zacas",
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
any(target_os = "linux", target_os = "android"),
),
),
))]
{
pub(crate) use self::atomic128::riscv64::{AtomicI128, AtomicU128};
}
// powerpc64 & (pwr8 | outline-atomics)
#[cfg(all(
target_arch = "powerpc64",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
not(portable_atomic_no_asm),
any(
target_feature = "quadword-atomics",
portable_atomic_target_feature = "quadword-atomics",
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
any(
all(
target_os = "linux",
any(
all(
target_env = "gnu",
any(target_endian = "little", not(target_feature = "crt-static")),
),
all(
target_env = "musl",
any(not(target_feature = "crt-static"), feature = "std"),
),
target_env = "ohos",
all(target_env = "uclibc", not(target_feature = "crt-static")),
portable_atomic_outline_atomics,
),
),
target_os = "android",
all(
target_os = "freebsd",
any(
target_endian = "little",
not(target_feature = "crt-static"),
portable_atomic_outline_atomics,
),
),
target_os = "openbsd",
all(
target_os = "aix",
not(portable_atomic_pre_llvm_20),
portable_atomic_outline_atomics,
), // TODO(aix): currently disabled by default
),
not(any(miri, portable_atomic_sanitize_thread)),
),
),
))]
{
pub(crate) use self::atomic128::powerpc64::{AtomicI128, AtomicU128};
}
// s390x
#[cfg(all(
target_arch = "s390x",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
not(portable_atomic_no_asm),
))]
{
pub(crate) use self::atomic128::s390x::{AtomicI128, AtomicU128};
}
// no native atomic CAS & (assume single core | critical section) => critical section based fallback
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(any(
target_arch = "avr",
target_arch = "msp430",
all(feature = "critical-section", portable_atomic_no_atomic_cas),
portable_atomic_unsafe_assume_single_core,
))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(any(
target_arch = "avr",
target_arch = "msp430",
all(feature = "critical-section", not(target_has_atomic = "ptr")),
portable_atomic_unsafe_assume_single_core,
))
)]
{
#[cfg(feature = "fallback")]
pub(crate) use self::interrupt::{AtomicI128, AtomicU128};
}
// no core 128-bit atomic & has CAS => use lock-base fallback
#[cfg_attr(
portable_atomic_no_cfg_target_has_atomic,
cfg(all(feature = "fallback", not(portable_atomic_no_atomic_cas)))
)]
#[cfg_attr(
not(portable_atomic_no_cfg_target_has_atomic),
cfg(all(feature = "fallback", target_has_atomic = "ptr"))
)]
{
pub(crate) use self::fallback::{AtomicI128, AtomicU128};
}
#[cfg(else)]
{
// no atomic
}
});
vendor/portable-atomic/src/imp/msp430.rs
Vendored
+283
View File
@@ -0,0 +1,283 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Atomic implementation on MSP430.
Adapted from https://github.com/pftbest/msp430-atomic.
Operations not supported here are provided by disabling interrupts.
See also src/imp/interrupt/msp430.rs.
See "Atomic operation overview by architecture" in atomic-maybe-uninit for a more comprehensive and
detailed description of the atomic and synchronize instructions in this architecture:
https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#msp430
Note: Ordering is always SeqCst.
Refs:
- MSP430x5xx and MSP430x6xx Family User's Guide, Rev. Q
https://www.ti.com/lit/ug/slau208q/slau208q.pdf
- atomic-maybe-uninit
https://github.com/taiki-e/atomic-maybe-uninit
See tests/asm-test/asm/portable-atomic for generated assembly.
*/
#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
#[cfg(not(feature = "critical-section"))]
use core::cell::UnsafeCell;
use core::sync::atomic::Ordering;
/// An atomic fence.
///
/// # Panics
///
/// Panics if `order` is [`Relaxed`](Ordering::Relaxed).
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub fn fence(order: Ordering) {
match order {
Ordering::Relaxed => panic!("there is no such thing as a relaxed fence"),
// MSP430 is single-core and a compiler fence works as an atomic fence.
_ => compiler_fence(order),
}
}
/// A compiler memory fence.
///
/// # Panics
///
/// Panics if `order` is [`Relaxed`](Ordering::Relaxed).
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub fn compiler_fence(order: Ordering) {
match order {
Ordering::Relaxed => panic!("there is no such thing as a relaxed compiler fence"),
_ => {}
}
// SAFETY: using an empty asm is safe.
unsafe {
// Do not use `nomem` and `readonly` because prevent preceding and subsequent memory accesses from being reordered.
#[cfg(not(portable_atomic_no_asm))]
asm!("", options(nostack, preserves_flags));
#[cfg(portable_atomic_no_asm)]
llvm_asm!("" ::: "memory" : "volatile");
}
}
#[cfg(not(feature = "critical-section"))]
items!({
macro_rules! atomic {
(load_store, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $size:tt) => {
#[repr(transparent)]
pub(crate) struct $atomic_type $(<$($generics)*>)? {
v: UnsafeCell<$value_type>,
}
// Send is implicitly implemented for atomic integers, but not for atomic pointers.
// SAFETY: any data races are prevented by atomic operations.
unsafe impl $(<$($generics)*>)? Send for $atomic_type $(<$($generics)*>)? {}
// SAFETY: any data races are prevented by atomic operations.
unsafe impl $(<$($generics)*>)? Sync for $atomic_type $(<$($generics)*>)? {}
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
#[inline]
#[cfg_attr(
all(debug_assertions, not(portable_atomic_no_track_caller)),
track_caller
)]
pub(crate) fn load(&self, order: Ordering) -> $value_type {
crate::utils::assert_load_ordering(order);
let src = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
let out;
#[cfg(not(portable_atomic_no_asm))]
asm!(
concat!("mov.", $size, " @{src}, {out}"), // atomic { out = *src }
src = in(reg) src,
out = lateout(reg) out,
options(nostack, preserves_flags),
);
#[cfg(portable_atomic_no_asm)]
llvm_asm!(
concat!("mov.", $size, " $1, $0")
: "=r"(out) : "*m"(src) : "memory" : "volatile"
);
out
}
}
#[inline]
#[cfg_attr(
all(debug_assertions, not(portable_atomic_no_track_caller)),
track_caller
)]
pub(crate) fn store(&self, val: $value_type, order: Ordering) {
crate::utils::assert_store_ordering(order);
let dst = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
#[cfg(not(portable_atomic_no_asm))]
asm!(
concat!("mov.", $size, " {val}, 0({dst})"), // atomic { *dst = val }
dst = in(reg) dst,
val = in(reg) val,
options(nostack, preserves_flags),
);
#[cfg(portable_atomic_no_asm)]
llvm_asm!(
concat!("mov.", $size, " $1, $0")
:: "*m"(dst), "ir"(val) : "memory" : "volatile"
);
}
}
}
};
($([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $size:tt) => {
atomic!(load_store, $([$($generics)*])? $atomic_type, $value_type, $size);
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
#[inline]
pub(crate) fn add(&self, val: $value_type, _order: Ordering) {
let dst = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
#[cfg(not(portable_atomic_no_asm))]
asm!(
concat!("add.", $size, " {val}, 0({dst})"), // atomic { *dst += val }
dst = in(reg) dst,
val = in(reg) val,
// Do not use `preserves_flags` because ADD modifies the V, N, Z, and C bits of the status register.
options(nostack),
);
#[cfg(portable_atomic_no_asm)]
llvm_asm!(
concat!("add.", $size, " $1, $0")
:: "*m"(dst), "ir"(val) : "memory" : "volatile"
);
}
}
#[inline]
pub(crate) fn sub(&self, val: $value_type, _order: Ordering) {
let dst = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
#[cfg(not(portable_atomic_no_asm))]
asm!(
concat!("sub.", $size, " {val}, 0({dst})"), // atomic { *dst -= val }
dst = in(reg) dst,
val = in(reg) val,
// Do not use `preserves_flags` because SUB modifies the V, N, Z, and C bits of the status register.
options(nostack),
);
#[cfg(portable_atomic_no_asm)]
llvm_asm!(
concat!("sub.", $size, " $1, $0")
:: "*m"(dst), "ir"(val) : "memory" : "volatile"
);
}
}
#[inline]
pub(crate) fn and(&self, val: $value_type, _order: Ordering) {
let dst = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
#[cfg(not(portable_atomic_no_asm))]
asm!(
concat!("and.", $size, " {val}, 0({dst})"), // atomic { *dst &= val }
dst = in(reg) dst,
val = in(reg) val,
// Do not use `preserves_flags` because AND modifies the V, N, Z, and C bits of the status register.
options(nostack),
);
#[cfg(portable_atomic_no_asm)]
llvm_asm!(
concat!("and.", $size, " $1, $0")
:: "*m"(dst), "ir"(val) : "memory" : "volatile"
);
}
}
#[inline]
pub(crate) fn or(&self, val: $value_type, _order: Ordering) {
let dst = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
#[cfg(not(portable_atomic_no_asm))]
asm!(
concat!("bis.", $size, " {val}, 0({dst})"), // atomic { *dst |= val }
dst = in(reg) dst,
val = in(reg) val,
options(nostack, preserves_flags),
);
#[cfg(portable_atomic_no_asm)]
llvm_asm!(
concat!("bis.", $size, " $1, $0")
:: "*m"(dst), "ir"(val) : "memory" : "volatile"
);
}
}
#[inline]
pub(crate) fn xor(&self, val: $value_type, _order: Ordering) {
let dst = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
#[cfg(not(portable_atomic_no_asm))]
asm!(
concat!("xor.", $size, " {val}, 0({dst})"), // atomic { *dst ^= val }
dst = in(reg) dst,
val = in(reg) val,
// Do not use `preserves_flags` because XOR modifies the V, N, Z, and C bits of the status register.
options(nostack),
);
#[cfg(portable_atomic_no_asm)]
llvm_asm!(
concat!("xor.", $size, " $1, $0")
:: "*m"(dst), "ir"(val) : "memory" : "volatile"
);
}
}
#[inline]
pub(crate) fn not(&self, _order: Ordering) {
let dst = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
#[cfg(not(portable_atomic_no_asm))]
asm!(
concat!("inv.", $size, " 0({dst})"), // atomic { *dst = !*dst }
dst = in(reg) dst,
// Do not use `preserves_flags` because INV modifies the V, N, Z, and C bits of the status register.
options(nostack),
);
#[cfg(portable_atomic_no_asm)]
llvm_asm!(
concat!("inv.", $size, " $0")
:: "*m"(dst) : "memory" : "volatile"
);
}
}
}
};
}
atomic!(AtomicI8, i8, "b");
atomic!(AtomicU8, u8, "b");
atomic!(AtomicI16, i16, "w");
atomic!(AtomicU16, u16, "w");
atomic!(AtomicIsize, isize, "w");
atomic!(AtomicUsize, usize, "w");
atomic!(load_store, [T] AtomicPtr, *mut T, "w");
});
vendor/portable-atomic/src/imp/riscv.rs
Vendored
+947
View File
@@ -0,0 +1,947 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Atomic load/store implementation on RISC-V.
This is for RISC-V targets without A extension. (pre-1.76 rustc doesn't provide atomics
at all on such targets. https://github.com/rust-lang/rust/pull/114499)
Also, optionally provides RMW implementation when Zaamo extension or force-amo feature is enabled.
If A extension is not available but Zalrsc or Zacas extension is available, LLVM can implement all
atomic operations using it (https://github.com/llvm/llvm-project/commit/70f70390bb83a383fcb88ea843adbc4edf6ee34b),
so this module contains no code for these extensions.
See "Atomic operation overview by architecture" in atomic-maybe-uninit for a more comprehensive and
detailed description of the atomic and synchronize instructions in this architecture:
https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#risc-v
Refs:
- RISC-V Instruction Set Manual
"Zaamo" Extension for Atomic Memory Operations
https://docs.riscv.org/reference/isa/v20260120/unpriv/a-st-ext.html#sec:amo
"Zabha" Extension for Byte and Halfword Atomic Memory Operations
https://docs.riscv.org/reference/isa/v20260120/unpriv/zabha.html
- RISC-V Atomics ABI Specification
https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/draft-20250812-301374e92976e298e676e7129a6212926b2299ce/riscv-atomic.adoc
- atomic-maybe-uninit
https://github.com/taiki-e/atomic-maybe-uninit
See tests/asm-test/asm/portable-atomic for generated assembly.
*/
#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
use core::{cell::UnsafeCell, sync::atomic::Ordering};
#[cfg(any(
test,
portable_atomic_force_amo,
target_feature = "zaamo",
portable_atomic_target_feature = "zaamo",
))]
items!({
macro_rules! atomic_rmw_amo_ext {
// Use +a also for zaamo because `option arch +zaamo` requires LLVM 19.
// https://github.com/llvm/llvm-project/commit/8be079cdddfd628d356d9ddb5ab397ea95fb1030
("w") => {
"+a"
};
("d") => {
"+a"
};
("b") => {
"+a,+zabha"
};
("h") => {
"+a,+zabha"
};
}
macro_rules! atomic_rmw_amo {
($op:ident, $dst:ident, $val:ident, $order:ident, $size:tt) => {{
let out;
macro_rules! op {
($asm_order:tt) => {
// SAFETY: The user guaranteed that the AMO instruction is available in this
// system by setting the portable_atomic_force_amo/target_feature and
// portable_atomic_unsafe_assume_single_core.
// The caller of this macro must guarantee the validity of the pointer.
asm!(
".option push",
// https://github.com/riscv-non-isa/riscv-asm-manual/blob/v0.0.1/src/asm-manual.adoc#arch
// LLVM supports `.option arch` directive on LLVM 17+.
// https://github.com/llvm/llvm-project/commit/9e8ed3403c191ab9c4903e8eeb8f732ff8a43cb4
// Note that `.insn <value>` directive requires LLVM 19.
// https://github.com/llvm/llvm-project/commit/2a086dce691e3cc34a2fc27f4fb255bb2cbbfac9
concat!(".option arch, ", atomic_rmw_amo_ext!($size)),
concat!("amo", stringify!($op), ".", $size, $asm_order, " {out}, {val}, 0({dst})"), // atomic { _x = *dst; *dst = op(_x, val); out = _x }
".option pop",
dst = in(reg) ptr_reg!($dst),
val = in(reg) $val,
out = lateout(reg) out,
options(nostack, preserves_flags),
)
};
}
match $order {
Ordering::Relaxed => op!(""),
Ordering::Acquire => op!(".aq"),
Ordering::Release => op!(".rl"),
// AcqRel and SeqCst RMWs are equivalent.
Ordering::AcqRel | Ordering::SeqCst => op!(".aqrl"),
_ => unreachable!(),
}
out
}};
}
#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
items!({
#[cfg(target_arch = "riscv32")]
macro_rules! w {
() => {
""
};
}
#[cfg(target_arch = "riscv64")]
macro_rules! w {
() => {
"w"
};
}
// 32-bit val.wrapping_shl(shift) but no extra `& (u32::BITS - 1)`
#[inline(always)]
fn sllw(val: u32, shift: u32) -> u32 {
// SAFETY: Calling sll{,w} is safe.
unsafe {
let out;
asm!(
concat!("sll", w!(), " {out}, {val}, {shift}"), // out = val << shift & 31
out = lateout(reg) out,
val = in(reg) val,
shift = in(reg) shift,
options(pure, nomem, nostack, preserves_flags),
);
out
}
}
// 32-bit val.wrapping_shr(shift) but no extra `& (u32::BITS - 1)`
macro_rules! srlw {
($val:expr, $shift:expr) => {
// SAFETY: Calling srl{,w} is safe.
unsafe {
let val: u32 = $val;
let shift: u32 = $shift;
let out;
asm!(
concat!("srl", w!(), " {out}, {val}, {shift}"), // out = val >> shift & 31
out = lateout(reg) out,
val = in(reg) val,
shift = in(reg) shift,
options(pure, nomem, nostack, preserves_flags),
);
out
}
};
}
trait ZeroExtend: Copy {
/// Zero-extends `self` to `u32` if it is smaller than 32-bit.
fn zero_extend(self) -> u32;
}
macro_rules! zero_extend {
($int:ident, $uint:ident) => {
impl ZeroExtend for $uint {
#[inline(always)]
fn zero_extend(self) -> u32 {
self as u32
}
}
impl ZeroExtend for $int {
#[allow(clippy::cast_sign_loss)]
#[inline(always)]
fn zero_extend(self) -> u32 {
self as $uint as u32
}
}
};
}
zero_extend!(i8, u8);
zero_extend!(i16, u16);
});
});
macro_rules! atomic_load_store {
($([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $size:tt) => {
#[repr(transparent)]
pub(crate) struct $atomic_type $(<$($generics)*>)? {
v: UnsafeCell<$value_type>,
}
// Send is implicitly implemented for atomic integers, but not for atomic pointers.
// SAFETY: any data races are prevented by atomic operations.
unsafe impl $(<$($generics)*>)? Send for $atomic_type $(<$($generics)*>)? {}
// SAFETY: any data races are prevented by atomic operations.
unsafe impl $(<$($generics)*>)? Sync for $atomic_type $(<$($generics)*>)? {}
#[cfg(any(test, not(portable_atomic_unsafe_assume_single_core)))]
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
#[inline]
pub(crate) const fn new(v: $value_type) -> Self {
Self { v: UnsafeCell::new(v) }
}
#[inline]
pub(crate) fn is_lock_free() -> bool {
Self::IS_ALWAYS_LOCK_FREE
}
pub(crate) const IS_ALWAYS_LOCK_FREE: bool = true;
#[inline]
pub(crate) const fn as_ptr(&self) -> *mut $value_type {
self.v.get()
}
}
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub(crate) fn load(&self, order: Ordering) -> $value_type {
crate::utils::assert_load_ordering(order);
let src = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
let out;
macro_rules! atomic_load {
($acquire:tt, $release:tt) => {
asm!(
$release, // fence
concat!("l", $size, " {out}, 0({src})"), // atomic { out = *src }
$acquire, // fence
src = in(reg) ptr_reg!(src),
out = lateout(reg) out,
options(nostack, preserves_flags),
)
};
}
match order {
Ordering::Relaxed => atomic_load!("", ""),
Ordering::Acquire => atomic_load!("fence r, rw", ""),
Ordering::SeqCst => atomic_load!("fence r, rw", "fence rw, rw"),
_ => unreachable!(),
}
out
}
}
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub(crate) fn store(&self, val: $value_type, order: Ordering) {
crate::utils::assert_store_ordering(order);
let dst = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe {
macro_rules! atomic_store {
($acquire:tt, $release:tt) => {
asm!(
$release, // fence
concat!("s", $size, " {val}, 0({dst})"), // atomic { *dst = val }
$acquire, // fence
dst = in(reg) ptr_reg!(dst),
val = in(reg) val,
options(nostack, preserves_flags),
)
};
}
match order {
Ordering::Relaxed => atomic_store!("", ""),
Ordering::Release => atomic_store!("", "fence rw, w"),
// https://github.com/llvm/llvm-project/commit/3ea8f2526541884e03d5bd4f4e46f4eb190990b6
Ordering::SeqCst => atomic_store!("fence rw, rw", "fence rw, w"),
_ => unreachable!(),
}
}
}
}
};
}
macro_rules! atomic_base {
(
$([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $int_type:ty, $size:tt,
$fetch_add:ident, $fetch_sub:ident
) => {
atomic_load_store!($([$($generics)*])? $atomic_type, $value_type, $size);
#[cfg(any(
test,
portable_atomic_force_amo,
target_feature = "zaamo",
portable_atomic_target_feature = "zaamo",
))]
items!({
// There is no amo{sub,nand,neg}.
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
#[inline]
pub(crate) fn swap(&self, val: $value_type, order: Ordering) -> $value_type {
let dst = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe { atomic_rmw_amo!(swap, dst, val, order, $size) }
}
#[inline]
pub(crate) fn $fetch_add(&self, val: $int_type, order: Ordering) -> $value_type {
let dst = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe { atomic_rmw_amo!(add, dst, val, order, $size) }
}
#[inline]
pub(crate) fn $fetch_sub(&self, val: $int_type, order: Ordering) -> $value_type {
self.$fetch_add(val.wrapping_neg(), order)
}
#[inline]
pub(crate) fn fetch_and(&self, val: $int_type, order: Ordering) -> $value_type {
let dst = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe { atomic_rmw_amo!(and, dst, val, order, $size) }
}
#[inline]
pub(crate) fn fetch_or(&self, val: $int_type, order: Ordering) -> $value_type {
let dst = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe { atomic_rmw_amo!(or, dst, val, order, $size) }
}
#[inline]
pub(crate) fn fetch_xor(&self, val: $int_type, order: Ordering) -> $value_type {
let dst = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe { atomic_rmw_amo!(xor, dst, val, order, $size) }
}
}
#[cfg(not(any(portable_atomic_unsafe_assume_single_core, feature = "critical-section")))]
impl_default_bit_opts!($atomic_type, $int_type);
});
};
}
macro_rules! atomic_ptr {
($size:tt) => {
atomic_base!([T] AtomicPtr, *mut T, usize, $size, fetch_byte_add, fetch_byte_sub);
};
}
macro_rules! atomic {
($atomic_type:ident, $value_type:ty, $size:tt, $max:tt, $min:tt) => {
atomic_base!($atomic_type, $value_type, $value_type, $size, fetch_add, fetch_sub);
#[cfg(any(
test,
portable_atomic_force_amo,
target_feature = "zaamo",
portable_atomic_target_feature = "zaamo",
))]
items!({
// There is no amo{sub,nand,neg}.
impl $atomic_type {
#[inline]
pub(crate) fn fetch_not(&self, order: Ordering) -> $value_type {
let dst = self.v.get();
let val: crate::utils::RegSize = !0;
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe { atomic_rmw_amo!(xor, dst, val, order, $size) }
}
#[inline]
pub(crate) fn fetch_max(&self, val: $value_type, order: Ordering) -> $value_type {
let dst = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe { atomic_rmw_amo!($max, dst, val, order, $size) }
}
#[inline]
pub(crate) fn fetch_min(&self, val: $value_type, order: Ordering) -> $value_type {
let dst = self.v.get();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
unsafe { atomic_rmw_amo!($min, dst, val, order, $size) }
}
}
#[cfg(not(any(
portable_atomic_unsafe_assume_single_core,
feature = "critical-section",
)))]
items!({
impl_default_no_fetch_ops!($atomic_type, $value_type);
impl $atomic_type {
#[inline]
pub(crate) fn not(&self, order: Ordering) {
self.fetch_not(order);
}
}
});
});
};
}
macro_rules! atomic_sub_word {
($atomic_type:ident, $value_type:ty, $size:tt, $max:tt, $min:tt) => {
cfg_sel!({
#[cfg(any(target_feature = "zabha", portable_atomic_target_feature = "zabha"))]
{
atomic!($atomic_type, $value_type, $size, $max, $min);
}
#[cfg(else)]
{
atomic_load_store!($atomic_type, $value_type, $size);
#[cfg(any(
test,
portable_atomic_force_amo,
target_feature = "zaamo",
portable_atomic_target_feature = "zaamo",
))]
items!({
impl $atomic_type {
#[inline]
pub(crate) fn fetch_and(
&self,
val: $value_type,
order: Ordering,
) -> $value_type {
let dst = self.v.get();
let (dst, shift, mut mask) =
crate::utils::create_sub_word_mask_values(dst);
mask = !sllw(mask, shift);
let mut val = sllw(ZeroExtend::zero_extend(val), shift);
val |= mask;
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
let out: u32 = unsafe { atomic_rmw_amo!(and, dst, val, order, "w") };
srlw!(out, shift)
}
#[inline]
pub(crate) fn fetch_or(
&self,
val: $value_type,
order: Ordering,
) -> $value_type {
let dst = self.v.get();
let (dst, shift, _mask) =
crate::utils::create_sub_word_mask_values(dst);
let val = sllw(ZeroExtend::zero_extend(val), shift);
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
let out: u32 = unsafe { atomic_rmw_amo!(or, dst, val, order, "w") };
srlw!(out, shift)
}
#[inline]
pub(crate) fn fetch_xor(
&self,
val: $value_type,
order: Ordering,
) -> $value_type {
let dst = self.v.get();
let (dst, shift, _mask) =
crate::utils::create_sub_word_mask_values(dst);
let val = sllw(ZeroExtend::zero_extend(val), shift);
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
let out: u32 = unsafe { atomic_rmw_amo!(xor, dst, val, order, "w") };
srlw!(out, shift)
}
#[inline]
pub(crate) fn fetch_not(&self, order: Ordering) -> $value_type {
self.fetch_xor(!0, order)
}
}
#[cfg(not(any(
portable_atomic_unsafe_assume_single_core,
feature = "critical-section",
)))]
items!({
impl_default_bit_opts!($atomic_type, $value_type);
impl $atomic_type {
#[inline]
pub(crate) fn and(&self, val: $value_type, order: Ordering) {
self.fetch_and(val, order);
}
#[inline]
pub(crate) fn or(&self, val: $value_type, order: Ordering) {
self.fetch_or(val, order);
}
#[inline]
pub(crate) fn xor(&self, val: $value_type, order: Ordering) {
self.fetch_xor(val, order);
}
#[inline]
pub(crate) fn not(&self, order: Ordering) {
self.fetch_not(order);
}
}
});
});
}
});
};
}
atomic_sub_word!(AtomicI8, i8, "b", max, min);
atomic_sub_word!(AtomicU8, u8, "b", maxu, minu);
atomic_sub_word!(AtomicI16, i16, "h", max, min);
atomic_sub_word!(AtomicU16, u16, "h", maxu, minu);
atomic!(AtomicI32, i32, "w", max, min);
atomic!(AtomicU32, u32, "w", maxu, minu);
#[cfg(target_arch = "riscv64")]
atomic!(AtomicI64, i64, "d", max, min);
#[cfg(target_arch = "riscv64")]
atomic!(AtomicU64, u64, "d", maxu, minu);
#[cfg(target_pointer_width = "32")]
atomic!(AtomicIsize, isize, "w", max, min);
#[cfg(target_pointer_width = "32")]
atomic!(AtomicUsize, usize, "w", maxu, minu);
#[cfg(target_pointer_width = "32")]
atomic_ptr!("w");
#[cfg(target_pointer_width = "64")]
atomic!(AtomicIsize, isize, "d", max, min);
#[cfg(target_pointer_width = "64")]
atomic!(AtomicUsize, usize, "d", maxu, minu);
#[cfg(target_pointer_width = "64")]
atomic_ptr!("d");
#[cfg(test)]
mod tests {
use super::*;
test_atomic_ptr_load_store!();
test_atomic_int_load_store!(i8);
test_atomic_int_load_store!(u8);
test_atomic_int_load_store!(i16);
test_atomic_int_load_store!(u16);
test_atomic_int_load_store!(i32);
test_atomic_int_load_store!(u32);
#[cfg(target_arch = "riscv64")]
test_atomic_int_load_store!(i64);
#[cfg(target_arch = "riscv64")]
test_atomic_int_load_store!(u64);
test_atomic_int_load_store!(isize);
test_atomic_int_load_store!(usize);
macro_rules! test_atomic_ptr_amo {
() => {
#[allow(
clippy::alloc_instead_of_core,
clippy::std_instead_of_alloc,
clippy::std_instead_of_core,
clippy::undocumented_unsafe_blocks
)]
mod test_atomic_ptr_amo {
use super::*;
test_atomic_ptr_amo!(AtomicPtr<u8>);
}
};
($atomic_type:ty) => {
#[allow(unused_imports)]
use sptr::Strict as _;
use crate::tests::helper; // for old rustc
::quickcheck::quickcheck! {
fn quickcheck_swap(x: usize, y: usize) -> bool {
let x = sptr::invalid_mut(x);
let y = sptr::invalid_mut(y);
for &order in &helper::SWAP_ORDERINGS {
let a = <$atomic_type>::new(x);
assert_eq!(a.swap(y, order), x);
assert_eq!(a.swap(x, order), y);
}
true
}
fn quickcheck_fetch_byte_add(x: usize, y: usize) -> bool {
let x = sptr::invalid_mut(x);
let y = sptr::invalid_mut::<u8>(y);
for &order in &helper::SWAP_ORDERINGS {
let a = <$atomic_type>::new(x);
assert_eq!(a.fetch_byte_add(y.addr(), order), x);
assert_eq!(
a.load(Ordering::Relaxed).addr(), x.addr().wrapping_add(y.addr())
);
let a = <$atomic_type>::new(y);
assert_eq!(a.fetch_byte_add(x.addr(), order), y);
assert_eq!(
a.load(Ordering::Relaxed).addr(), y.addr().wrapping_add(x.addr())
);
}
true
}
fn quickcheck_fetch_byte_sub(x: usize, y: usize) -> bool {
let x = sptr::invalid_mut(x);
let y = sptr::invalid_mut::<u8>(y);
for &order in &helper::SWAP_ORDERINGS {
let a = <$atomic_type>::new(x);
assert_eq!(a.fetch_byte_sub(y.addr(), order), x);
assert_eq!(
a.load(Ordering::Relaxed).addr(), x.addr().wrapping_sub(y.addr())
);
let a = <$atomic_type>::new(y);
assert_eq!(a.fetch_byte_sub(x.addr(), order), y);
assert_eq!(
a.load(Ordering::Relaxed).addr(), y.addr().wrapping_sub(x.addr())
);
}
true
}
fn quickcheck_fetch_and(x: usize, y: usize) -> bool {
let x = sptr::invalid_mut(x);
let y = sptr::invalid_mut::<u8>(y);
for &order in &helper::SWAP_ORDERINGS {
let a = <$atomic_type>::new(x);
assert_eq!(a.fetch_and(y.addr(), order), x);
assert_eq!(a.load(Ordering::Relaxed).addr(), x.addr() & y.addr());
let a = <$atomic_type>::new(y);
assert_eq!(a.fetch_and(x.addr(), order), y);
assert_eq!(a.load(Ordering::Relaxed).addr(), y.addr() & x.addr());
}
true
}
fn quickcheck_fetch_or(x: usize, y: usize) -> bool {
let x = sptr::invalid_mut(x);
let y = sptr::invalid_mut::<u8>(y);
for &order in &helper::SWAP_ORDERINGS {
let a = <$atomic_type>::new(x);
assert_eq!(a.fetch_or(y.addr(), order), x);
assert_eq!(a.load(Ordering::Relaxed).addr(), x.addr() | y.addr());
let a = <$atomic_type>::new(y);
assert_eq!(a.fetch_or(x.addr(), order), y);
assert_eq!(a.load(Ordering::Relaxed).addr(), y.addr() | x.addr());
}
true
}
fn quickcheck_fetch_xor(x: usize, y: usize) -> bool {
let x = sptr::invalid_mut(x);
let y = sptr::invalid_mut::<u8>(y);
for &order in &helper::SWAP_ORDERINGS {
let a = <$atomic_type>::new(x);
assert_eq!(a.fetch_xor(y.addr(), order), x);
assert_eq!(a.load(Ordering::Relaxed).addr(), x.addr() ^ y.addr());
let a = <$atomic_type>::new(y);
assert_eq!(a.fetch_xor(x.addr(), order), y);
assert_eq!(a.load(Ordering::Relaxed).addr(), y.addr() ^ x.addr());
}
true
}
}
};
}
macro_rules! test_atomic_int_amo {
($int_type:ident) => {
paste::paste! {
#[allow(
clippy::alloc_instead_of_core,
clippy::std_instead_of_alloc,
clippy::std_instead_of_core,
clippy::undocumented_unsafe_blocks
)]
mod [<test_atomic_ $int_type _amo>] {
use super::*;
test_atomic_int_amo!([<Atomic $int_type:camel>], $int_type);
}
}
};
($atomic_type:ty, $int_type:ident) => {
use crate::tests::helper;
::quickcheck::quickcheck! {
fn quickcheck_swap(x: $int_type, y: $int_type) -> bool {
for &order in &helper::SWAP_ORDERINGS {
let a = <$atomic_type>::new(x);
assert_eq!(a.swap(y, order), x);
assert_eq!(a.swap(x, order), y);
}
true
}
fn quickcheck_fetch_add(x: $int_type, y: $int_type) -> bool {
for &order in &helper::SWAP_ORDERINGS {
let a = <$atomic_type>::new(x);
assert_eq!(a.fetch_add(y, order), x);
assert_eq!(a.load(Ordering::Relaxed), x.wrapping_add(y));
let a = <$atomic_type>::new(y);
assert_eq!(a.fetch_add(x, order), y);
assert_eq!(a.load(Ordering::Relaxed), y.wrapping_add(x));
}
true
}
fn quickcheck_fetch_sub(x: $int_type, y: $int_type) -> bool {
for &order in &helper::SWAP_ORDERINGS {
let a = <$atomic_type>::new(x);
assert_eq!(a.fetch_sub(y, order), x);
assert_eq!(a.load(Ordering::Relaxed), x.wrapping_sub(y));
let a = <$atomic_type>::new(y);
assert_eq!(a.fetch_sub(x, order), y);
assert_eq!(a.load(Ordering::Relaxed), y.wrapping_sub(x));
}
true
}
fn quickcheck_fetch_and(x: $int_type, y: $int_type) -> bool {
for &order in &helper::SWAP_ORDERINGS {
let a = <$atomic_type>::new(x);
assert_eq!(a.fetch_and(y, order), x);
assert_eq!(a.load(Ordering::Relaxed), x & y);
let a = <$atomic_type>::new(y);
assert_eq!(a.fetch_and(x, order), y);
assert_eq!(a.load(Ordering::Relaxed), y & x);
}
true
}
fn quickcheck_fetch_or(x: $int_type, y: $int_type) -> bool {
for &order in &helper::SWAP_ORDERINGS {
let a = <$atomic_type>::new(x);
assert_eq!(a.fetch_or(y, order), x);
assert_eq!(a.load(Ordering::Relaxed), x | y);
let a = <$atomic_type>::new(y);
assert_eq!(a.fetch_or(x, order), y);
assert_eq!(a.load(Ordering::Relaxed), y | x);
}
true
}
fn quickcheck_fetch_xor(x: $int_type, y: $int_type) -> bool {
for &order in &helper::SWAP_ORDERINGS {
let a = <$atomic_type>::new(x);
assert_eq!(a.fetch_xor(y, order), x);
assert_eq!(a.load(Ordering::Relaxed), x ^ y);
let a = <$atomic_type>::new(y);
assert_eq!(a.fetch_xor(x, order), y);
assert_eq!(a.load(Ordering::Relaxed), y ^ x);
}
true
}
fn quickcheck_fetch_max(x: $int_type, y: $int_type) -> bool {
for &order in &helper::SWAP_ORDERINGS {
let a = <$atomic_type>::new(x);
assert_eq!(a.fetch_max(y, order), x);
assert_eq!(a.load(Ordering::Relaxed), core::cmp::max(x, y));
let a = <$atomic_type>::new(y);
assert_eq!(a.fetch_max(x, order), y);
assert_eq!(a.load(Ordering::Relaxed), core::cmp::max(y, x));
}
true
}
fn quickcheck_fetch_min(x: $int_type, y: $int_type) -> bool {
for &order in &helper::SWAP_ORDERINGS {
let a = <$atomic_type>::new(x);
assert_eq!(a.fetch_min(y, order), x);
assert_eq!(a.load(Ordering::Relaxed), core::cmp::min(x, y));
let a = <$atomic_type>::new(y);
assert_eq!(a.fetch_min(x, order), y);
assert_eq!(a.load(Ordering::Relaxed), core::cmp::min(y, x));
}
true
}
fn quickcheck_fetch_not(x: $int_type) -> bool {
for &order in &helper::SWAP_ORDERINGS {
let a = <$atomic_type>::new(x);
assert_eq!(a.fetch_not(order), x);
assert_eq!(a.load(Ordering::Relaxed), !x);
assert_eq!(a.fetch_not(order), !x);
assert_eq!(a.load(Ordering::Relaxed), x);
}
true
}
}
};
}
macro_rules! test_atomic_int_amo_sub_word {
($int_type:ident) => {
paste::paste! {
#[allow(
clippy::alloc_instead_of_core,
clippy::std_instead_of_alloc,
clippy::std_instead_of_core,
clippy::undocumented_unsafe_blocks
)]
mod [<test_atomic_ $int_type _amo>] {
use super::*;
#[cfg(any(target_feature = "zabha", portable_atomic_target_feature = "zabha"))]
test_atomic_int_amo!([<Atomic $int_type:camel>], $int_type);
#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
test_atomic_int_amo_sub_word!([<Atomic $int_type:camel>], $int_type);
}
}
};
($atomic_type:ty, $int_type:ident) => {
use crate::tests::helper::{self, *};
::quickcheck::quickcheck! {
fn quickcheck_fetch_and(x: $int_type, y: $int_type) -> bool {
let mut rng = fastrand::Rng::new();
for &order in &helper::SWAP_ORDERINGS {
for base in [0, !0] {
let mut arr = Align16([
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
]);
let a_idx = rng.usize(3..=6);
arr.0[a_idx] = <$atomic_type>::new(x);
let a = &arr.0[a_idx];
assert_eq!(a.fetch_and(y, order), x);
assert_eq!(a.load(Ordering::Relaxed), x & y);
for i in 0..a_idx {
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
}
for i in a_idx + 1..arr.0.len() {
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
}
arr.0[a_idx] = <$atomic_type>::new(y);
let a = &arr.0[a_idx];
assert_eq!(a.fetch_and(x, order), y);
assert_eq!(a.load(Ordering::Relaxed), y & x);
for i in 0..a_idx {
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
}
for i in a_idx + 1..arr.0.len() {
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
}
}
}
true
}
fn quickcheck_fetch_or(x: $int_type, y: $int_type) -> bool {
let mut rng = fastrand::Rng::new();
for &order in &helper::SWAP_ORDERINGS {
for base in [0, !0] {
let mut arr = Align16([
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
]);
let a_idx = rng.usize(3..=6);
arr.0[a_idx] = <$atomic_type>::new(x);
let a = &arr.0[a_idx];
assert_eq!(a.fetch_or(y, order), x);
assert_eq!(a.load(Ordering::Relaxed), x | y);
for i in 0..a_idx {
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
}
for i in a_idx + 1..arr.0.len() {
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
}
arr.0[a_idx] = <$atomic_type>::new(y);
let a = &arr.0[a_idx];
assert_eq!(a.fetch_or(x, order), y);
assert_eq!(a.load(Ordering::Relaxed), y | x);
for i in 0..a_idx {
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
}
for i in a_idx + 1..arr.0.len() {
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
}
}
}
true
}
fn quickcheck_fetch_xor(x: $int_type, y: $int_type) -> bool {
let mut rng = fastrand::Rng::new();
for &order in &helper::SWAP_ORDERINGS {
for base in [0, !0] {
let mut arr = Align16([
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
]);
let a_idx = rng.usize(3..=6);
arr.0[a_idx] = <$atomic_type>::new(x);
let a = &arr.0[a_idx];
assert_eq!(a.fetch_xor(y, order), x);
assert_eq!(a.load(Ordering::Relaxed), x ^ y);
for i in 0..a_idx {
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
}
for i in a_idx + 1..arr.0.len() {
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
}
arr.0[a_idx] = <$atomic_type>::new(y);
let a = &arr.0[a_idx];
assert_eq!(a.fetch_xor(x, order), y);
assert_eq!(a.load(Ordering::Relaxed), y ^ x);
for i in 0..a_idx {
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
}
for i in a_idx + 1..arr.0.len() {
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
}
}
}
true
}
fn quickcheck_fetch_not(x: $int_type) -> bool {
let mut rng = fastrand::Rng::new();
for &order in &helper::SWAP_ORDERINGS {
for base in [0, !0] {
let mut arr = Align16([
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
<$atomic_type>::new(base),
]);
let a_idx = rng.usize(3..=6);
arr.0[a_idx] = <$atomic_type>::new(x);
let a = &arr.0[a_idx];
assert_eq!(a.fetch_not(order), x);
assert_eq!(a.load(Ordering::Relaxed), !x);
assert_eq!(a.fetch_not(order), !x);
assert_eq!(a.load(Ordering::Relaxed), x);
for i in 0..a_idx {
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
}
for i in a_idx + 1..arr.0.len() {
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
}
}
}
true
}
}
};
}
test_atomic_ptr_amo!();
test_atomic_int_amo_sub_word!(i8);
test_atomic_int_amo_sub_word!(u8);
test_atomic_int_amo_sub_word!(i16);
test_atomic_int_amo_sub_word!(u16);
test_atomic_int_amo!(i32);
test_atomic_int_amo!(u32);
#[cfg(target_arch = "riscv64")]
test_atomic_int_amo!(i64);
#[cfg(target_arch = "riscv64")]
test_atomic_int_amo!(u64);
test_atomic_int_amo!(isize);
test_atomic_int_amo!(usize);
}
vendor/portable-atomic/src/imp/x86.rs
Vendored
+283
View File
@@ -0,0 +1,283 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
/*
Atomic operations implementation on x86/x86_64.
This module provides atomic operations not supported by LLVM or optimizes
cases where LLVM code generation is not optimal.
Note: On Miri and ThreadSanitizer which do not support inline assembly, we don't use
this module and use CAS loop instead.
Refs:
- x86 and amd64 instruction reference https://www.felixcloutier.com/x86
See tests/asm-test/asm/portable-atomic for generated assembly.
*/
#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
use core::sync::atomic::Ordering;
#[cfg(target_pointer_width = "32")]
macro_rules! ptr_modifier {
() => {
":e"
};
}
#[cfg(target_pointer_width = "64")]
macro_rules! ptr_modifier {
() => {
""
};
}
/// An atomic fence.
///
/// # Panics
///
/// Panics if `order` is [`Relaxed`](Ordering::Relaxed).
#[cfg_attr(doc, allow(dead_code))]
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub fn fence(order: Ordering) {
if order == Ordering::SeqCst {
let p = core::cell::UnsafeCell::new(core::mem::MaybeUninit::<usize>::uninit());
// SAFETY: raw pointer passed in is valid because we got it from a reference.
unsafe {
// Equivalent to `mfence`, but is up to 3.1x faster on Coffee Lake and up to 2.4x faster on Raptor Lake-H at least in simple cases.
// - https://github.com/taiki-e/portable-atomic/pull/156
// - LLVM uses `lock or` https://godbolt.org/z/vv6rjzfYd
// - Windows uses `xchg` for x86_32 for MemoryBarrier https://learn.microsoft.com/en-us/windows/win32/api/winnt/nf-winnt-memorybarrier
// - MSVC STL uses `lock inc` https://github.com/microsoft/STL/pull/740
// - boost uses `lock or` https://github.com/boostorg/atomic/commit/559eba81af71386cedd99f170dc6101c6ad7bf22
#[cfg(target_pointer_width = "64")]
asm!(
concat!("xchg qword ptr [{p", ptr_modifier!(), "}], {tmp}"),
p = in(reg) p.get(),
tmp = out(reg) _,
options(nostack, preserves_flags),
);
#[cfg(target_pointer_width = "32")]
asm!(
concat!("xchg dword ptr [{p", ptr_modifier!(), "}], {tmp:e}"),
p = in(reg) p.get(),
tmp = out(reg) _,
options(nostack, preserves_flags),
);
}
} else {
core::sync::atomic::fence(order);
}
}
cfg_core_atomic!({
use super::core_atomic::{
AtomicI8, AtomicI16, AtomicI32, AtomicI64, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16,
AtomicU32, AtomicU64, AtomicUsize,
};
macro_rules! atomic_int {
($atomic_type:ident, $ptr_size:tt) => {
impl $atomic_type {
#[inline]
pub(crate) fn not(&self, _order: Ordering) {
let dst = self.as_ptr();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
//
// https://www.felixcloutier.com/x86/not
unsafe {
// atomic RMW is always SeqCst.
asm!(
concat!("lock not ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}]"),
dst = in(reg) dst,
options(nostack, preserves_flags),
);
}
}
#[inline]
pub(crate) fn neg(&self, _order: Ordering) {
let dst = self.as_ptr();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
//
// https://www.felixcloutier.com/x86/neg
unsafe {
// atomic RMW is always SeqCst.
asm!(
concat!("lock neg ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}]"),
dst = in(reg) dst,
// Do not use `preserves_flags` because NEG modifies the CF, OF, SF, ZF, AF, and PF flag.
options(nostack),
);
}
}
}
};
}
atomic_int!(AtomicI8, "byte");
atomic_int!(AtomicU8, "byte");
atomic_int!(AtomicI16, "word");
atomic_int!(AtomicU16, "word");
atomic_int!(AtomicI32, "dword");
atomic_int!(AtomicU32, "dword");
#[cfg(target_arch = "x86_64")]
atomic_int!(AtomicI64, "qword");
#[cfg(target_arch = "x86_64")]
atomic_int!(AtomicU64, "qword");
#[cfg(target_pointer_width = "32")]
atomic_int!(AtomicIsize, "dword");
#[cfg(target_pointer_width = "32")]
atomic_int!(AtomicUsize, "dword");
#[cfg(target_pointer_width = "64")]
atomic_int!(AtomicIsize, "qword");
#[cfg(target_pointer_width = "64")]
atomic_int!(AtomicUsize, "qword");
#[cfg(target_arch = "x86")]
impl AtomicI64 {
#[inline]
pub(crate) fn not(&self, order: Ordering) {
self.fetch_not(order);
}
#[inline]
pub(crate) fn neg(&self, order: Ordering) {
self.fetch_neg(order);
}
}
#[cfg(target_arch = "x86")]
impl AtomicU64 {
#[inline]
pub(crate) fn not(&self, order: Ordering) {
self.fetch_not(order);
}
#[inline]
pub(crate) fn neg(&self, order: Ordering) {
self.fetch_neg(order);
}
}
macro_rules! atomic_bit_opts {
(
$([$($generics:tt)*])? $atomic_type:ident, $int_type:ident, $val_modifier:tt,
$ptr_size:tt
) => {
// LLVM 14 and older don't support generating `lock bt{s,r,c}`.
// LLVM 15 only supports generating `lock bt{s,r,c}` for immediate bit offsets.
// LLVM 16+ can generate `lock bt{s,r,c}` for both immediate and register bit offsets.
// https://godbolt.org/z/TGhr5z4ds
// So, use fetch_* based implementations on LLVM 16+, otherwise use asm based implementations.
#[cfg(not(portable_atomic_pre_llvm_16))]
impl_default_bit_opts!($atomic_type, $int_type);
#[cfg(portable_atomic_pre_llvm_16)]
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
// `<integer>::BITS` requires Rust 1.53
const BITS: u32 = (core::mem::size_of::<$int_type>() * 8) as u32;
#[inline]
pub(crate) fn bit_set(&self, bit: u32, _order: Ordering) -> bool {
let dst = self.as_ptr();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
// the masking by the bit size of the type ensures that we do not shift
// out of bounds.
//
// https://www.felixcloutier.com/x86/bts
unsafe {
let r: u8;
// atomic RMW is always SeqCst.
asm!(
concat!("lock bts ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}], {bit", $val_modifier, "}"),
"setb {r}",
dst = in(reg) dst,
bit = in(reg) (bit & (Self::BITS - 1)) as $int_type,
r = out(reg_byte) r,
// Do not use `preserves_flags` because BTS modifies the CF flag.
options(nostack),
);
crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
r != 0
}
}
#[inline]
pub(crate) fn bit_clear(&self, bit: u32, _order: Ordering) -> bool {
let dst = self.as_ptr();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
// the masking by the bit size of the type ensures that we do not shift
// out of bounds.
//
// https://www.felixcloutier.com/x86/btr
unsafe {
let r: u8;
// atomic RMW is always SeqCst.
asm!(
concat!("lock btr ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}], {bit", $val_modifier, "}"),
"setb {r}",
dst = in(reg) dst,
bit = in(reg) (bit & (Self::BITS - 1)) as $int_type,
r = out(reg_byte) r,
// Do not use `preserves_flags` because BTR modifies the CF flag.
options(nostack),
);
crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
r != 0
}
}
#[inline]
pub(crate) fn bit_toggle(&self, bit: u32, _order: Ordering) -> bool {
let dst = self.as_ptr();
// SAFETY: any data races are prevented by atomic intrinsics and the raw
// pointer passed in is valid because we got it from a reference.
// the masking by the bit size of the type ensures that we do not shift
// out of bounds.
//
// https://www.felixcloutier.com/x86/btc
unsafe {
let r: u8;
// atomic RMW is always SeqCst.
asm!(
concat!("lock btc ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}], {bit", $val_modifier, "}"),
"setb {r}",
dst = in(reg) dst,
bit = in(reg) (bit & (Self::BITS - 1)) as $int_type,
r = out(reg_byte) r,
// Do not use `preserves_flags` because BTC modifies the CF flag.
options(nostack),
);
crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
r != 0
}
}
}
};
}
impl_default_bit_opts!(AtomicI8, i8);
impl_default_bit_opts!(AtomicU8, u8);
atomic_bit_opts!(AtomicI16, i16, ":x", "word");
atomic_bit_opts!(AtomicU16, u16, ":x", "word");
atomic_bit_opts!(AtomicI32, i32, ":e", "dword");
atomic_bit_opts!(AtomicU32, u32, ":e", "dword");
#[cfg(target_arch = "x86_64")]
atomic_bit_opts!(AtomicI64, i64, "", "qword");
#[cfg(target_arch = "x86_64")]
atomic_bit_opts!(AtomicU64, u64, "", "qword");
#[cfg(target_arch = "x86")]
impl_default_bit_opts!(AtomicI64, i64);
#[cfg(target_arch = "x86")]
impl_default_bit_opts!(AtomicU64, u64);
#[cfg(target_pointer_width = "32")]
atomic_bit_opts!(AtomicIsize, isize, ":e", "dword");
#[cfg(target_pointer_width = "32")]
atomic_bit_opts!(AtomicUsize, usize, ":e", "dword");
#[cfg(target_pointer_width = "32")]
atomic_bit_opts!([T] AtomicPtr, usize, ":e", "dword");
#[cfg(target_pointer_width = "64")]
atomic_bit_opts!(AtomicIsize, isize, "", "qword");
#[cfg(target_pointer_width = "64")]
atomic_bit_opts!(AtomicUsize, usize, "", "qword");
#[cfg(target_pointer_width = "64")]
atomic_bit_opts!([T] AtomicPtr, usize, "", "qword");
});
vendor/portable-atomic/src/lib.rs
Vendored
+5224
View File
File diff suppressed because it is too large. Load diff
vendor/portable-atomic/src/rustdoc.css
Vendored
+92
View File
@@ -0,0 +1,92 @@
/*
* Adapted from https://github.com/TheBevyFlock/bevy_cli/blob/d3e79396595f675798af5be8be56efbfd2737ba0/bevy_lint/assets/rustdoc.css.
* Adjusted to work well with our GFM (GitHub Flavored Markdown) to CommonMark converter.
*/
/*
* Extended theme configuration.
*
* These colors are based off Github's (blue for notes, green for tips, etc.), though the exact
* colors are taken from `rustdoc`'s themes.
*/
:root[data-theme="light"] {
--rustdoc-alert-note: #2196f3;
--rustdoc-alert-tip: #068000;
--rustdoc-alert-important: #6e4fc9;
--rustdoc-alert-warning: #ff8e00;
--rustdoc-alert-caution: #be2525;
}
:root[data-theme="dark"] {
--rustdoc-alert-note: #008dfd;
--rustdoc-alert-tip: #2bab63;
--rustdoc-alert-important: #b78cf2;
--rustdoc-alert-warning: #ff8e00;
--rustdoc-alert-caution: #d93d3d;
}
:root[data-theme="ayu"] {
--rustdoc-alert-note: #39afd7;
--rustdoc-alert-tip: #b8cc52;
--rustdoc-alert-important: #a37acc;
--rustdoc-alert-warning: #ff8e00;
--rustdoc-alert-caution: #df3600;
}
/*
* Alerts are siblings to the `.warning` class, and as such use many of the same rules.
*
* The first `<p>` in a alert must be the title, as it will be colored and styled differently.
*/
.rustdoc-alert {
border-left: 2px solid;
margin: 0 0 0.75em 0;
position: relative;
overflow-x: visible !important;
}
/* Remove the existing quote margin and setup some padding instead. */
.rustdoc-alert blockquote {
margin: 0;
padding: 14px;
}
/* Make the alert title the same sans-serif font and size as headings. */
.rustdoc-alert blockquote > p:first-child {
font-family: "Fira Sans", Arial, sans-serif;
font-size: 1.125rem;
}
/* Color styles for different kinds of alerts. */
.rustdoc-alert.rustdoc-alert-note {
border-left-color: var(--rustdoc-alert-note);
}
.rustdoc-alert.rustdoc-alert-note blockquote > p:first-child {
color: var(--rustdoc-alert-note);
}
.rustdoc-alert.rustdoc-alert-tip {
border-left-color: var(--rustdoc-alert-tip);
}
.rustdoc-alert.rustdoc-alert-tip blockquote > p:first-child {
color: var(--rustdoc-alert-tip);
}
.rustdoc-alert.rustdoc-alert-important {
border-left-color: var(--rustdoc-alert-important);
}
.rustdoc-alert.rustdoc-alert-important blockquote > p:first-child {
color: var(--rustdoc-alert-important);
}
.rustdoc-alert.rustdoc-alert-warning {
border-left-color: var(--rustdoc-alert-warning);
}
.rustdoc-alert.rustdoc-alert-warning blockquote > p:first-child {
color: var(--rustdoc-alert-warning);
}
.rustdoc-alert.rustdoc-alert-caution {
border-left-color: var(--rustdoc-alert-caution);
}
.rustdoc-alert.rustdoc-alert-caution blockquote > p:first-child {
color: var(--rustdoc-alert-caution);
}
vendor/portable-atomic/src/tests/helper.rs
Vendored
+3294
View File
File diff suppressed because it is too large. Load diff
vendor/portable-atomic/src/tests/mod.rs
Vendored
+499
View File
@@ -0,0 +1,499 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
#![allow(
clippy::alloc_instead_of_core,
clippy::std_instead_of_alloc,
clippy::std_instead_of_core,
clippy::undocumented_unsafe_blocks,
clippy::wildcard_imports
)]
#[macro_use]
pub(crate) mod helper;
#[allow(dead_code)]
#[path = "../../version.rs"]
mod version;
use test_helper as _; // For critical-section test
use super::*;
test_atomic_bool_pub!();
test_atomic_ptr_pub!();
test_atomic_int_pub!(isize);
test_atomic_int_pub!(usize);
test_atomic_int_pub!(i8);
test_atomic_int_pub!(u8);
test_atomic_int_pub!(i16);
test_atomic_int_pub!(u16);
test_atomic_int_pub!(i32);
test_atomic_int_pub!(u32);
test_atomic_int_pub!(i64);
test_atomic_int_pub!(u64);
#[cfg(not(all(valgrind, target_arch = "powerpc64")))] // TODO(powerpc64): Hang (as of Valgrind 3.26)
test_atomic_int_pub!(i128);
#[cfg(not(all(valgrind, target_arch = "powerpc64")))] // TODO(powerpc64): Hang (as of Valgrind 3.26)
test_atomic_int_pub!(u128);
#[cfg(all(feature = "float", portable_atomic_unstable_f16))]
test_atomic_float_pub!(f16);
#[cfg(feature = "float")]
test_atomic_float_pub!(f32);
#[cfg(feature = "float")]
// TODO: rustc bug: https://github.com/rust-lang/rust/issues/114479
#[cfg(not(all(not(debug_assertions), target_arch = "x86", not(target_feature = "sse2"))))]
test_atomic_float_pub!(f64);
#[cfg(all(feature = "float", portable_atomic_unstable_f128))]
test_atomic_float_pub!(f128);
#[deny(improper_ctypes)]
extern "C" {
fn _atomic_bool_ffi_safety(_: AtomicBool);
fn _atomic_ptr_ffi_safety(_: AtomicPtr<u8>);
fn _atomic_isize_ffi_safety(_: AtomicIsize);
fn _atomic_usize_ffi_safety(_: AtomicUsize);
fn _atomic_i8_ffi_safety(_: AtomicI8);
fn _atomic_u8_ffi_safety(_: AtomicU8);
fn _atomic_i16_ffi_safety(_: AtomicI16);
fn _atomic_u16_ffi_safety(_: AtomicU16);
fn _atomic_i32_ffi_safety(_: AtomicI32);
fn _atomic_u32_ffi_safety(_: AtomicU32);
fn _atomic_i64_ffi_safety(_: AtomicI64);
fn _atomic_u64_ffi_safety(_: AtomicU64);
#[rustversion::since(1.89)] // https://github.com/rust-lang/rust/pull/137306
fn _atomic_i128_ffi_safety(_: AtomicI128);
#[rustversion::since(1.89)] // https://github.com/rust-lang/rust/pull/137306
fn _atomic_u128_ffi_safety(_: AtomicU128);
#[cfg(all(feature = "float", portable_atomic_unstable_f16))]
fn _atomic_f16_ffi_safety(_: AtomicF16);
#[cfg(feature = "float")]
fn _atomic_f32_ffi_safety(_: AtomicF32);
#[cfg(feature = "float")]
fn _atomic_f64_ffi_safety(_: AtomicF64);
#[cfg(all(feature = "float", portable_atomic_unstable_f128))]
fn _atomic_f128_ffi_safety(_: AtomicF128);
}
#[test]
fn test_is_lock_free() {
assert!(AtomicI8::is_always_lock_free());
assert!(AtomicI8::is_lock_free());
assert!(AtomicU8::is_always_lock_free());
assert!(AtomicU8::is_lock_free());
assert!(AtomicI16::is_always_lock_free());
assert!(AtomicI16::is_lock_free());
assert!(AtomicU16::is_always_lock_free());
assert!(AtomicU16::is_lock_free());
#[cfg(all(feature = "float", portable_atomic_unstable_f16))]
assert!(AtomicF16::is_always_lock_free());
#[cfg(all(feature = "float", portable_atomic_unstable_f16))]
assert!(AtomicF16::is_lock_free());
assert!(AtomicI32::is_always_lock_free());
assert!(AtomicI32::is_lock_free());
assert!(AtomicU32::is_always_lock_free());
assert!(AtomicU32::is_lock_free());
#[cfg(feature = "float")]
assert!(AtomicF32::is_always_lock_free());
#[cfg(feature = "float")]
assert!(AtomicF32::is_lock_free());
#[cfg(not(portable_atomic_no_cfg_target_has_atomic))]
{
if cfg!(any(
target_has_atomic = "64",
all(
target_arch = "riscv32",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
),
)) {
assert!(AtomicI64::is_always_lock_free());
assert!(AtomicI64::is_lock_free());
assert!(AtomicU64::is_always_lock_free());
assert!(AtomicU64::is_lock_free());
#[cfg(feature = "float")]
assert!(AtomicF64::is_always_lock_free());
#[cfg(feature = "float")]
assert!(AtomicF64::is_lock_free());
} else if cfg!(all(
feature = "fallback",
target_arch = "arm",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_os = "linux", target_os = "android"),
not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
not(portable_atomic_no_outline_atomics),
not(target_has_atomic = "64"),
not(portable_atomic_test_detect_false),
)) {
assert!(!AtomicI64::is_always_lock_free());
assert!(!AtomicU64::is_always_lock_free());
#[cfg(feature = "float")]
assert!(!AtomicF64::is_always_lock_free());
assert!(AtomicI64::is_lock_free());
assert!(AtomicU64::is_lock_free());
#[cfg(feature = "float")]
assert!(AtomicF64::is_lock_free());
} else {
assert!(!AtomicI64::is_always_lock_free());
assert!(!AtomicU64::is_always_lock_free());
#[cfg(feature = "float")]
assert!(!AtomicF64::is_always_lock_free());
#[cfg(not(target_arch = "riscv32"))]
{
assert!(!AtomicI64::is_lock_free());
assert!(!AtomicU64::is_lock_free());
#[cfg(feature = "float")]
assert!(!AtomicF64::is_lock_free());
}
#[cfg(target_arch = "riscv32")]
{
// TODO(riscv): check detect.has_zacas
}
}
}
if cfg!(portable_atomic_no_asm) && cfg!(not(portable_atomic_unstable_asm)) {
assert!(!AtomicI128::is_always_lock_free());
assert!(!AtomicI128::is_lock_free());
assert!(!AtomicU128::is_always_lock_free());
assert!(!AtomicU128::is_lock_free());
#[cfg(all(feature = "float", portable_atomic_unstable_f128))]
assert!(!AtomicF128::is_always_lock_free());
#[cfg(all(feature = "float", portable_atomic_unstable_f128))]
assert!(!AtomicF128::is_lock_free());
} else if cfg!(any(
all(
target_arch = "aarch64",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
),
all(
target_arch = "arm64ec",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
not(portable_atomic_no_asm),
),
all(
target_arch = "x86_64",
any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
),
all(
target_arch = "riscv64",
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
),
all(
target_arch = "powerpc64",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
not(portable_atomic_no_asm),
any(
target_feature = "quadword-atomics",
portable_atomic_target_feature = "quadword-atomics",
),
),
all(
target_arch = "s390x",
not(all(
any(miri, portable_atomic_sanitize_thread),
not(portable_atomic_atomic_intrinsics),
)),
not(portable_atomic_no_asm),
),
)) {
assert!(AtomicI128::is_always_lock_free());
assert!(AtomicI128::is_lock_free());
assert!(AtomicU128::is_always_lock_free());
assert!(AtomicU128::is_lock_free());
#[cfg(all(feature = "float", portable_atomic_unstable_f128))]
assert!(AtomicF128::is_always_lock_free());
#[cfg(all(feature = "float", portable_atomic_unstable_f128))]
assert!(AtomicF128::is_lock_free());
} else {
assert!(!AtomicI128::is_always_lock_free());
assert!(!AtomicU128::is_always_lock_free());
#[cfg(all(feature = "float", portable_atomic_unstable_f128))]
assert!(!AtomicF128::is_always_lock_free());
#[cfg(not(any(
target_arch = "x86_64",
target_arch = "powerpc64",
target_arch = "riscv64",
)))]
{
assert!(!AtomicI128::is_lock_free());
assert!(!AtomicU128::is_lock_free());
#[cfg(all(feature = "float", portable_atomic_unstable_f128))]
assert!(!AtomicF128::is_lock_free());
}
#[cfg(target_arch = "x86_64")]
{
let has_cmpxchg16b = cfg!(all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
not(any(target_env = "sgx", miri)),
not(portable_atomic_test_detect_false),
)) && std::is_x86_feature_detected!("cmpxchg16b");
assert_eq!(AtomicI128::is_lock_free(), has_cmpxchg16b);
assert_eq!(AtomicU128::is_lock_free(), has_cmpxchg16b);
#[cfg(all(feature = "float", portable_atomic_unstable_f128))]
assert_eq!(AtomicF128::is_lock_free(), has_cmpxchg16b);
}
#[cfg(target_arch = "powerpc64")]
{
// TODO(powerpc64): is_powerpc_feature_detected is unstable
}
#[cfg(target_arch = "riscv64")]
{
// TODO(riscv): check detect.has_zacas
}
}
}
// test version parsing code used in the build script.
#[test]
fn test_rustc_version() {
use self::version::Version;
// rustc 1.34 (rustup)
let v = Version::parse(
"rustc 1.34.2 (6c2484dc3 2019-05-13)
binary: rustc
commit-hash: 6c2484dc3c532c052f159264e970278d8b77cdc9
commit-date: 2019-05-13
host: x86_64-apple-darwin
release: 1.34.2
LLVM version: 8.0",
)
.unwrap();
assert_eq!(v, Version::stable(34, 8));
// rustc 1.50 (rustup)
let v = Version::parse(
"rustc 1.50.0 (cb75ad5db 2021-02-10)
binary: rustc
commit-hash: cb75ad5db02783e8b0222fee363c5f63f7e2cf5b
commit-date: 2021-02-10
host: aarch64-unknown-linux-gnu
release: 1.50.0",
)
.unwrap();
assert_eq!(v, Version::stable(50, 0));
// rustc 1.67 (rustup)
let v = Version::parse(
"rustc 1.67.0 (fc594f156 2023-01-24)
binary: rustc
commit-hash: fc594f15669680fa70d255faec3ca3fb507c3405
commit-date: 2023-01-24
host: aarch64-apple-darwin
release: 1.67.0
LLVM version: 15.0.6",
)
.unwrap();
assert_eq!(v, Version::stable(67, 15));
// rustc 1.68-beta (rustup)
let v = Version::parse(
"rustc 1.68.0-beta.2 (10b73bf73 2023-02-01)
binary: rustc
commit-hash: 10b73bf73a6b770cd92ad8ff538173bc3298411c
commit-date: 2023-02-01
host: aarch64-apple-darwin
release: 1.68.0-beta.2
LLVM version: 15.0.6",
)
.unwrap();
// We do not distinguish between stable and beta because we are only
// interested in whether unstable features are potentially available.
assert_eq!(v, Version::stable(68, 15));
// rustc nightly-2019-01-27 (rustup)
let v = Version::parse(
"rustc 1.33.0-nightly (20c2cba61 2019-01-26)
binary: rustc
commit-hash: 20c2cba61dc83e612d25ed496025171caa3db30f
commit-date: 2019-01-26
host: x86_64-apple-darwin
release: 1.33.0-nightly
LLVM version: 8.0",
)
.unwrap();
assert_eq!(v.minor, 33);
assert!(v.nightly);
assert_eq!(v.llvm, 8);
assert_eq!(v.commit_date().year, 2019);
assert_eq!(v.commit_date().month, 1);
assert_eq!(v.commit_date().day, 26);
// rustc 1.69-nightly (rustup)
let v = Version::parse(
"rustc 1.69.0-nightly (bd39bbb4b 2023-02-07)
binary: rustc
commit-hash: bd39bbb4bb92df439bf6d85470e296cc6a47ffbd
commit-date: 2023-02-07
host: aarch64-apple-darwin
release: 1.69.0-nightly
LLVM version: 15.0.7",
)
.unwrap();
assert_eq!(v.minor, 69);
assert!(v.nightly);
assert_eq!(v.llvm, 15);
assert_eq!(v.commit_date().year, 2023);
assert_eq!(v.commit_date().month, 2);
assert_eq!(v.commit_date().day, 7);
// clippy-driver 1.69-nightly (rustup)
let v = Version::parse(
"rustc 1.69.0-nightly (bd39bbb4b 2023-02-07)
binary: rustc
commit-hash: bd39bbb4bb92df439bf6d85470e296cc6a47ffbd
commit-date: 2023-02-07
host: aarch64-apple-darwin
release: 1.69.0-nightly
LLVM version: 15.0.7",
)
.unwrap();
assert_eq!(v.minor, 69);
assert!(v.nightly);
assert_eq!(v.llvm, 15);
assert_eq!(v.commit_date().year, 2023);
assert_eq!(v.commit_date().month, 2);
assert_eq!(v.commit_date().day, 7);
// rustc 1.69-dev (from source: ./x.py build)
let v = Version::parse(
"rustc 1.69.0-dev
binary: rustc
commit-hash: unknown
commit-date: unknown
host: aarch64-unknown-linux-gnu
release: 1.69.0-dev
LLVM version: 16.0.0",
)
.unwrap();
assert_eq!(v.minor, 69);
assert!(v.nightly);
assert_eq!(v.llvm, 16);
assert_eq!(v.commit_date().year, 0);
assert_eq!(v.commit_date().month, 0);
assert_eq!(v.commit_date().day, 0);
// rustc 1.48 (debian 11: apt-get install cargo)
let v = Version::parse(
"rustc 1.48.0
binary: rustc
commit-hash: unknown
commit-date: unknown
host: aarch64-unknown-linux-gnu
release: 1.48.0
LLVM version: 11.0",
)
.unwrap();
assert_eq!(v, Version::stable(48, 11));
// rustc 1.67 (fedora: dnf install cargo)
let v = Version::parse(
"rustc 1.67.0 (fc594f156 2023-01-24) (Fedora 1.67.0-2.fc37)
binary: rustc
commit-hash: fc594f15669680fa70d255faec3ca3fb507c3405
commit-date: 2023-01-24
host: aarch64-unknown-linux-gnu
release: 1.67.0
LLVM version: 15.0.7",
)
.unwrap();
assert_eq!(v, Version::stable(67, 15));
// rustc 1.64 (alpine: apk add cargo)
let v = Version::parse(
"rustc 1.64.0
binary: rustc
commit-hash: unknown
commit-date: unknown
host: aarch64-alpine-linux-musl
release: 1.64.0
LLVM version: 15.0.3",
)
.unwrap();
assert_eq!(v, Version::stable(64, 15));
}
#[cfg(feature = "serde")]
#[test]
fn test_serde() {
use std::{eprint, eprintln, fmt};
use serde::{
de::{Deserialize, Deserializer},
ser::{Serialize, Serializer},
};
use serde_test::{Token, assert_tokens};
#[derive(Debug)]
struct DebugPartialEq<T>(T);
impl<T: fmt::Debug> PartialEq for DebugPartialEq<T> {
fn eq(&self, other: &Self) -> bool {
std::format!("{:?}", self) == std::format!("{:?}", other)
}
}
impl<T: Serialize> Serialize for DebugPartialEq<T> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
self.0.serialize(serializer)
}
}
impl<'de, T: Deserialize<'de>> Deserialize<'de> for DebugPartialEq<T> {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
T::deserialize(deserializer).map(Self)
}
}
macro_rules! t {
($atomic_type:ty, $value_type:ident $(as $token_value_type:ident)?, $token_type:ident) => {
eprint!("test_serde {} ... ", stringify!($value_type));
assert_tokens(&DebugPartialEq(<$atomic_type>::new($value_type::MAX)), &[
Token::$token_type($value_type::MAX $(as $token_value_type)?),
]);
assert_tokens(&DebugPartialEq(<$atomic_type>::new($value_type::MIN)), &[
Token::$token_type($value_type::MIN $(as $token_value_type)?),
]);
eprintln!("ok");
};
}
assert_tokens(&DebugPartialEq(AtomicBool::new(true)), &[Token::Bool(true)]);
assert_tokens(&DebugPartialEq(AtomicBool::new(false)), &[Token::Bool(false)]);
t!(AtomicIsize, isize as i64, I64);
t!(AtomicUsize, usize as u64, U64);
t!(AtomicI8, i8, I8);
t!(AtomicU8, u8, U8);
t!(AtomicI16, i16, I16);
t!(AtomicU16, u16, U16);
t!(AtomicI32, i32, I32);
t!(AtomicU32, u32, U32);
t!(AtomicI64, i64, I64);
t!(AtomicU64, u64, U64);
#[cfg(not(all(valgrind, target_arch = "powerpc64")))] // TODO(powerpc64): Hang (as of Valgrind 3.26)
t!(AtomicI128, i128, I128);
#[cfg(not(all(valgrind, target_arch = "powerpc64")))] // TODO(powerpc64): Hang (as of Valgrind 3.26)
t!(AtomicU128, u128, U128);
// TODO(f16_and_f128): Test f16 & f128 once stabilized.
#[cfg(feature = "float")]
t!(AtomicF32, f32, F32);
#[cfg(feature = "float")]
t!(AtomicF64, f64, F64);
}
vendor/portable-atomic/src/utils.rs
Vendored
+1289
View File
File diff suppressed because it is too large. Load diff