Vendor deps.
This commit is contained in:
jimmy committed
1 parent
294527085e
commit
91e18fde3a
1714 files changed
+556667
-7
No files matched your search
+1
@@ -0,0 +1 @@
|
||||
{"files":{".cargo_vcs_info.json":"3704de1736ccc44c6ef99d7563fa51cfafa77d135a2bdeff864fcfe0f3087ed0","CHANGELOG.md":"71861d11d0f90b2e2ec64986d4f99aa813b84b34f955a4a5d34f51dd3c378204","Cargo.lock":"a7572366edd2c39d5ee4b8440fa76c56060ddae97b7bbfff83aa803a06f3c4b8","Cargo.toml":"30e602bcf54f19dbdcf52315a262c88e8efbd12455ee4d4fe63f2880e4ac10e1","Cargo.toml.orig":"4fd62c038201d9efe897d2320af051c38e69cbb3cca9d77c589568e60cd83365","LICENSE-APACHE":"0d542e0c8804e39aa7f37eb00da5a762149dc682d7829451287e11b938e94594","LICENSE-MIT":"23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3","README.md":"a66e02c23633b74d19013225a52fe26742b59b9575a5cb0451f53e8e7468b622","build.rs":"9d0b1e071715580497aadb918d4b3b124ee366ba66f15ac05a3d8045f0bc2cf0","portable-atomic-util/README.md":"6a69c6fa501d2470e8b038160736c08897cdbbddd4318361c7a29cd20cf86261","src/cfgs.rs":"7a438600e742bca248c16c980c20c2b65be01bb786e6cde738cbd94f612344e0","src/gen/build.rs":"4ad2fa43f8f725631e9754338f39b80f64a2e61facdb3bd735efc215b95a6597","src/gen/utils.rs":"c44ee71296aaa90c8104339e470688f631a2380d7f10349c7766205c415050aa","src/imp/arm_linux.rs":"6f5ba9542ec89340444094ee4c96503dec7476c38d308abd1b09168785d672c5","src/imp/atomic128/README.md":"240d085f57eff9e5c92cb87498e0d1bb260a4f0fe41cd64eb86cb96e1d686c96","src/imp/atomic128/aarch64.rs":"8fe3bebbf1a7991b35e6ffeb796ff872bc15edcdbbdcf7a5835cb2b1ce149542","src/imp/atomic128/intrinsics.rs":"882b817a77ba6b90ad96585bdd5545c8526a9590473a93ca8dda96d91d1fcbcb","src/imp/atomic128/macros.rs":"31f63dd5f6523fdc80d752afdd7b13e90394e7210b8f14bfa38a4feab0822cd5","src/imp/atomic128/mod.rs":"4d12d39ed3c0e5ab46724ac9536598ccc0028147d258b739ed27212c4e4d5e7e","src/imp/atomic128/powerpc64.rs":"f1831b186b89049aba31be4c4cd37b5572554f2655944d5cfc04ab5654691715","src/imp/atomic128/riscv64.rs":"115c4d8bc0fed79973bc12462f506121c7e612f3cf14cc627a0d024c3a131387","src/imp/atomic128/s390x.rs":"b657e6f3e48d12b79dfaf7dec5268d2558967c14288e097bb37e40d49b9b6eba","src/imp/atomic128/x86_64.rs":"19c67a75d8ee1392aae67f49b88a3b69e78acce5004280c7a7be33051774c8ed","src/imp/atomic64/README.md":"56ef5c1f1750515a6a8a94b9347fd42faa889cddf5a9f77cd7d650008a4a8b7c","src/imp/atomic64/arm_linux.rs":"e4857c2c7240b73a3e3c9bb092b1251d9b71a44617cc43345ffd14bf11ee5f65","src/imp/atomic64/macros.rs":"5b6f0bfe40064885926cfd57a336f15da92f6964864c50d339d1bb9903802a9d","src/imp/atomic64/mod.rs":"2977468d184134a068179eb8361d73500865b8d1476b87590099d6b503db30e5","src/imp/atomic64/riscv32.rs":"c8e00a15c0524fc94b0df7946fcd84b6779eb01bfe48f06bf6f4f84039c00337","src/imp/avr.rs":"085cdcc03d7ffb85c491f3939e0bcc3ae1e78738acadf09472f547a0b9d7c8fc","src/imp/core_atomic.rs":"0ce4a6ca4fb50fd8e3c6fb2adf96535ab9b8593b951d25f16b5762ddf867b7bd","src/imp/detect/README.md":"24db78df52c56bb4bebc699ad0ce21272b214d9caed7c70fd92702f2331d6837","src/imp/detect/aarch64_aa64reg.rs":"77d6b3a2fafbdeb881c671379d0be0a330b46d78c8252d074c97ed44b1138123","src/imp/detect/aarch64_apple.rs":"149a79530b9bdb11b1521918a4d61ece2be2cb99ab4c86cb6765c3acde1c727a","src/imp/detect/aarch64_fuchsia.rs":"edb86bdc2630329b96396d3da23d3ee2ca55d3d45737df2de76ac2bfbce4964f","src/imp/detect/aarch64_illumos.rs":"50b95302831a5fe2837fc13b882d6d4e92f79e05b004c4c6cd2a6402d86eefc3","src/imp/detect/aarch64_windows.rs":"2cfc29df37c09aa8685b47e4973ab11fdb18f66ca9bbc2346e2713ef6898e1e1","src/imp/detect/auxv.rs":"4897e7d126b69bc7d8121bdc7516aca794c361ed767d6031e61b1e4456f9cae6","src/imp/detect/common.rs":"735925ba2717d649f49a7550d78fd523b002fb8c030cb7d52e542d982ada43a9","src/imp/detect/powerpc64_aix.rs":"b635538662e9add807cebd26c5742eae6dd764e11fd2c30fc6661baa03b69d01","src/imp/detect/riscv_linux.rs":"9e8b214013992dcf7ff5e0beb9228db4f2125aa134afdc573781788349f3a159","src/imp/detect/x86_64.rs":"8b315b61a409d3350b19c02e98a741f4c1c1e3861e6792594137267f4595eeb1","src/imp/fallback/mod.rs":"970934078d4cf02e424f2bc80ad7724e188bc8fe4c883898311ef32b4601d826","src/imp/fallback/outline_atomics.rs":"041d872affc4ad0b6b452f695a1373a1f76fe75d1b2377c24c3c0d199d89e383","src/imp/fallback/seq_lock.rs":"f099568551ecd866b8deff71655421f7eff21d045af598debe44d7a355c7bb39","src/imp/fallback/seq_lock_common.rs":"2c519835f7a9b20b64b9cc3428dcf9ed18ad7e56fd73791404d4f7f87975dd3e","src/imp/fallback/seq_lock_wide.rs":"b2870e179e7d903327ef4ca52e13471e771622ebefd8b06940a5584d8e83416c","src/imp/fallback/utils.rs":"4b04423031fa66eaf7619e4ddbb30689eb834f74e9dd876b8f4d163cf9c75f73","src/imp/float/aarch64.rs":"b3d69ef526252b8ee99662f2059898cfa6cb99e1c969828225a2be99de2d45df","src/imp/float/int.rs":"8801b336650a039bc6e056857c0f65c2f148a0774c83baa9bd9bf415618107e5","src/imp/float/mod.rs":"2c722182baee05df65f9f6e2b4de5d5474a3b2bf0c24803a65a771de78b76f95","src/imp/interrupt/README.md":"1f4ed0b78b42832800bc8e4db98576f17ef51e8e84b07375fa3e076addbf83e0","src/imp/interrupt/armv4t.rs":"6fb9e7d01f7fb483bc6bec07916fdcba8563b605b53bd87c5310662da04e55a4","src/imp/interrupt/armv6m.rs":"364a38f18ffb2b3cd836700dff6a390d9542f0f2a3eda07bc63bc8868dafaad1","src/imp/interrupt/avLine truncated
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"git": {
|
||||
"sha1": "e4595528211f1d53b7f4d93e528ef84b491fc871"
|
||||
},
|
||||
"path_in_vcs": ""
|
||||
}
|
||||
Vendored
+683
@@ -0,0 +1,683 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to this project will be documented in this file.
|
||||
|
||||
This project adheres to [Semantic Versioning](https://semver.org).
|
||||
|
||||
Releases may yanked if there is a security bug, a soundness bug, or a regression.
|
||||
|
||||
<!--
|
||||
Note: In this file, do not use the hard wrap in the middle of a sentence for compatibility with GitHub comment style markdown rendering.
|
||||
-->
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [1.15.0] - 2026-08-09
|
||||
|
||||
- Work around [`core::sync::atomic` bug](https://github.com/rust-lang/compiler-builtins/pull/1234) which may cause data races with non-relaxed load/CAS on pre-v6 ARM Linux/Android due to missing memory barrier. ([#112](https://github.com/taiki-e/portable-atomic/pull/112))
|
||||
|
||||
- Make {8,16,32}-bit non-relaxed load compatible with read-only memory on pre-v6 ARM Linux/Android. Previously only relaxed load was compatible with read-only memory. ([#112](https://github.com/taiki-e/portable-atomic/pull/112))
|
||||
|
||||
- Make `from_ptr` `const fn` on Rust 1.58+. Previously it was `const fn` only on Rust 1.83+. ([c09d0e6](https://github.com/taiki-e/portable-atomic/commit/c09d0e624f1416a01bbb11bba22c148b42dcb081))
|
||||
|
||||
- Optimize non-relaxed load/store on pre-v6 ARM Linux/Android. ([#112](https://github.com/taiki-e/portable-atomic/pull/112))
|
||||
|
||||
- Optimize fence on pre-v6 ARM Linux/Android. ([#112](https://github.com/taiki-e/portable-atomic/pull/112))
|
||||
|
||||
- Optimize 128-bit CAS/swap/min/max on s390x. This also includes removing the code complexity that caused the bug fixed in 1.14.0. ([33cd7e5](https://github.com/taiki-e/portable-atomic/commit/33cd7e55acfe1438c853b20b09ec786613228122), [6c8f764](https://github.com/taiki-e/portable-atomic/commit/6c8f764bee3c3ca570552e49a8117402b0186ead))
|
||||
|
||||
- Documentation improvements.
|
||||
|
||||
## [1.14.0] - 2026-07-17
|
||||
|
||||
- Guarantee that `is_lock_free` to always return the same result. ([e74f384](https://github.com/taiki-e/portable-atomic/commit/e74f384efab9eae540087a20ed140dddc0d06ce7))
|
||||
|
||||
- Fix bug in 128-bit `fetch_{min,max}` on pre-z196 s390x. ([a6c6c4b](https://github.com/taiki-e/portable-atomic/commit/a6c6c4bdb7c1b4f9d406ebbdcc54805e5e7eec10))
|
||||
|
||||
- Fix bug in atomic float `fetch_{add,sub,max,min}` on AArch64 when FEAT_LSFE is enabled. (There are currently no known CPUs that support FEAT_LSFE.) ([b105f9bc](https://github.com/taiki-e/portable-atomic/commit/b105f9bc612e07c6f6a07291fa2be8cba38fa5df))
|
||||
|
||||
- Fix panic in build script when custom target with non-standard name on non-nightly rustc. ([02ca737](https://github.com/taiki-e/portable-atomic/commit/02ca7371eecd005fa1b170c12f8a0bad5e77a7fb))
|
||||
|
||||
- Improve robustness of interrupt disable/restore on RISC-V and Xtensa. ([88cb3d5](https://github.com/taiki-e/portable-atomic/commit/88cb3d50c66569dab95cc93250bd1e4202508a65))
|
||||
|
||||
- Optimize SeqCst fence on x86. ([9127b31](https://github.com/taiki-e/portable-atomic/commit/9127b31caae0545424254b5ba6dcf5c91b7d59e6))
|
||||
|
||||
- Optimize 128-bit CAS when FEAT_LSE is not enabled. ([671b9a89](https://github.com/taiki-e/portable-atomic/commit/671b9a89d1e324547a3f29ae09a5a306b3b54a3b))
|
||||
|
||||
- Optimize SeqCst atomics on AArch64 Windows MSVC. ([671b9a89](https://github.com/taiki-e/portable-atomic/commit/671b9a89d1e324547a3f29ae09a5a306b3b54a3b))
|
||||
|
||||
- Optimize 8-bit fetch_and/fetch_or/fetch_xor/fetch_not on AVR when RMW instructions available. ([6f2f11f2](https://github.com/taiki-e/portable-atomic/commit/6f2f11f243673380e8732c5c5c1e18e6a5b553ff))
|
||||
|
||||
- Optimize 128-bit RMW on s390x. ([b8c502a9](https://github.com/taiki-e/portable-atomic/commit/b8c502a9238e9aed4e169f909b3d25c797f76cc4))
|
||||
|
||||
- Optimize run-time CPU feature detection. ([cae7032](https://github.com/taiki-e/portable-atomic/commit/cae703222aca87d2a014372a02afb14c0b80ed32))
|
||||
|
||||
- Enable run-time detection by default on AArch64 Apple targets. ([6c92015](https://github.com/taiki-e/portable-atomic/commit/6c920155e7645b859e6f667963c90f775e561213))
|
||||
|
||||
- Enable run-time detection by default on AArch64 illumos. ([59e3636b](https://github.com/taiki-e/portable-atomic/commit/59e3636bbe5515121c137d3b939e2cd39b50f1a4))
|
||||
|
||||
- aarch64: Support run-time detection of FEAT_LSE2 on Windows. ([16bc44f0](https://github.com/taiki-e/portable-atomic/commit/16bc44f07b0709828a9d2f100dccde7db5477794))
|
||||
|
||||
- Make `portable_atomic_no_outline_atomics` cfg no-op instead of compile error on unsupported architectures. ([6b8e3eb](https://github.com/taiki-e/portable-atomic/commit/6b8e3eb3b92bd64a0a6516bf4f72feb467ccbb3d))
|
||||
|
||||
- Improve compile-time detection of AVR target features. ([2b5a9a4f](https://github.com/taiki-e/portable-atomic/commit/2b5a9a4f6df1736f536aecade53181ca24446846))
|
||||
|
||||
- Documentation improvements.
|
||||
|
||||
## [1.13.1] - 2026-01-31
|
||||
|
||||
- Update to stabilized [PowerPC64](https://github.com/rust-lang/rust/pull/147996) inline assembly. ([92b02f8a](https://github.com/taiki-e/portable-atomic/commit/92b02f8a279327a1780cbe127d9effb2baae9b2f))
|
||||
|
||||
- Work around [rustc_codegen_gcc bugs on x86_64](https://github.com/rust-lang/rustc_codegen_gcc/issues/821#issuecomment-3793567607). ([ae4c501](https://github.com/taiki-e/portable-atomic/commit/ae4c501aec84a3537fe35ec57ceae94b3a05ade0))
|
||||
|
||||
- Optimize x86_64 128-bit atomics. ([a9d61eb](https://github.com/taiki-e/portable-atomic/commit/a9d61ebf8d7f466286a71a17f7d9063fcf07fce0), [90a17ca4](https://github.com/taiki-e/portable-atomic/commit/90a17ca40a8ff433d767c3b56264fb02ccdd71e1))
|
||||
|
||||
- Improve compile-time detection of RISC-V target features. ([535fced](https://github.com/taiki-e/portable-atomic/commit/535fced071ed095ee4d35b440ba55a0e2f533d80))
|
||||
|
||||
- Enable [release immutability](https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/immutable-releases).
|
||||
|
||||
## [1.13.0] - 2025-12-27
|
||||
|
||||
- Add `unsafe-assume-privileged` feature / `portable_atomic_unsafe_assume_privileged` cfg for safer lock-based fallback on multi-core privileged environments. ([b084ee1](https://github.com/taiki-e/portable-atomic/commit/b084ee1b6cba3e9d7158a6c9e5450e1fc8bbde36))
|
||||
|
||||
- Support `unsafe-assume-single-core`/`unsafe-assume-privileged` on all 32-bit Arm targets. Previously it was only M-profile and pre-v6 targets. ([7e07f5e](https://github.com/taiki-e/portable-atomic/commit/7e07f5e2bc8ad74287830522c02f960a8c8da59e))
|
||||
|
||||
- Make `AtomicPtr::fetch_*` strict-provenance compatible on all environments. Previously it was only strict-provenance compatible on `cfg(miri)` and otherwise permissive-provenance compatible. ([4306943](https://github.com/taiki-e/portable-atomic/commit/4306943fb09af3a4f763f1f8ff257fe752c7b7e3))
|
||||
|
||||
- Ensure sequential consistency in lock-based fallback when SeqCst is used. ([7e80742](https://github.com/taiki-e/portable-atomic/commit/7e80742eeed9fc4d1aa15455b862d70194f6f1bf))
|
||||
|
||||
- Support compile-time detection for x86_64 VMOVDQA. ([f7bb1aa](https://github.com/taiki-e/portable-atomic/commit/f7bb1aa246df0e13fa02fb707f8462d8dfe6b7e9))
|
||||
|
||||
- Improve compile-time detection of s390x miscellaneous-extensions-3. ([11045fe](https://github.com/taiki-e/portable-atomic/commit/11045fe513689a842e393324c05f2c5f169b59d4))
|
||||
|
||||
- Optimize AVR 8-bit swap when RMW instructions available. ([8cedb34](https://github.com/taiki-e/portable-atomic/commit/8cedb34a0b9f2ca1680d893a58020ba1e5d0a87b))
|
||||
|
||||
- Optimize interrupt restore on RISC-V. ([9b97a2a](https://github.com/taiki-e/portable-atomic/commit/9b97a2a18142c9a8d21f03de7fff30caea89d51e))
|
||||
|
||||
## [1.12.0] - 2025-12-19
|
||||
|
||||
- Fix build error on no-std pre-v6 Arm targets due to the [recent upstream change](https://github.com/rust-lang/rust/pull/149241). ([83f6f3e](https://github.com/taiki-e/portable-atomic/commit/83f6f3e4957833af6dd1bae054da1e8d51501a76))
|
||||
|
||||
- Support `unsafe-assume-single-core` feature / `portable_atomic_unsafe_assume_single_core` cfg for targets with CAS. ([38e9572](https://github.com/taiki-e/portable-atomic/commit/38e95722dde98e7a9b59d2acbff968450a0b09ea))
|
||||
|
||||
- Improve compile-time detection of s390x target feature. ([5ae0ef5](https://github.com/taiki-e/portable-atomic/commit/5ae0ef5ed7f9a0c9efe9e628ba2fbc5876487219))
|
||||
|
||||
- Documentation improvements. ([c84f720](https://github.com/taiki-e/portable-atomic/commit/c84f7203ae6e39a5c9966748123b95b90e8a127a))
|
||||
|
||||
## [1.11.1] - 2025-06-06
|
||||
|
||||
- Fix build error when building aarch64/arm64ec/powerpc64/s390x targets for Miri or ThreadSanitizer since nightly-2025-05-31.
|
||||
|
||||
- aarch64: Optimize atomic floats when FEAT_LSFE is enabled. ([#201](https://github.com/taiki-e/portable-atomic/pull/201))
|
||||
|
||||
- Improve compile-time detection of RISC-V Zacas extension. ([b7634e2](https://github.com/taiki-e/portable-atomic/commit/b7634e2cd808ea118266d12f99fd8877a92e3d31))
|
||||
|
||||
- Improve run-time detection on linux-musl. ([7fdad7f](https://github.com/taiki-e/portable-atomic/commit/7fdad7f7dd32e32ece7bd0eaf565db657b3406bb))
|
||||
|
||||
- Optimize interrupt restore on thumbv6m. ([dd2004a](https://github.com/taiki-e/portable-atomic/commit/dd2004aaa14d034b0db652eb9939b780d0d8221f))
|
||||
|
||||
## [1.11.0] - 2025-02-24
|
||||
|
||||
- Work around [nightly-2025-02-24 rustc regression causing "cannot use value of type `*mut T` for inline assembly" error](https://github.com/rust-lang/rust/issues/137512) on RISC-V without A extension, MSP430, and pre-v6 no-std Arm targets. ([eeb0235](https://github.com/taiki-e/portable-atomic/commit/eeb0235b9fda4c28a56ee5a9ffe0d7fb884a50ab))
|
||||
|
||||
- Support `AtomicF16` and `AtomicF128` for [unstable `f16` and `f128`](https://github.com/rust-lang/rust/issues/116909) under unstable cfgs. ([#200](https://github.com/taiki-e/portable-atomic/pull/200))
|
||||
|
||||
- RISC-V Zacas extension support is no longer experimental. ([#206](https://github.com/taiki-e/portable-atomic/pull/206))
|
||||
|
||||
- Improve support of run-time detection and outline-atomics:
|
||||
- riscv: Enable run-time detection of Zacas extension by default on Linux/Android. ([#207](https://github.com/taiki-e/portable-atomic/pull/207))
|
||||
- aarch64: Support run-time detection of FEAT_LRCPC3/FEAT_LSE128 on FreeBSD. ([6a5075d](https://github.com/taiki-e/portable-atomic/commit/6a5075d43543875cf38d6114f2951047e2e64f1a))
|
||||
- powerpc64: Support run-time detection of quadword-atomics on AIX (currently disabled by default because detection support for AIX is experimental). ([#102](https://github.com/taiki-e/portable-atomic/pull/102))
|
||||
|
||||
## [1.10.0] - 2024-11-23
|
||||
|
||||
- Update to stabilized [s390x](https://github.com/rust-lang/rust/pull/131258) and [Arm64EC](https://github.com/rust-lang/rust/pull/131781) inline assembly. ([97645c1](https://github.com/taiki-e/portable-atomic/commit/97645c1b2b938249f16eacb0fe696d4c7bb96754), [e1d1a97](https://github.com/taiki-e/portable-atomic/commit/e1d1a97cd1ab4bd04b45962c44ca1e9f0f9e1456))
|
||||
|
||||
- Make `get_mut` `const fn` on Rust 1.83+. ([0dea68c](https://github.com/taiki-e/portable-atomic/commit/0dea68c26e2bba3a83a849e2f137a2da445fc014))
|
||||
|
||||
- Make `from_ptr` `const fn` on Rust 1.83+. (align to the [std atomic change in Rust 1.84](https://github.com/rust-lang/rust/pull/131717)) ([50532d8](https://github.com/taiki-e/portable-atomic/commit/50532d8ce92408976307431a2f1a94000c630b23))
|
||||
|
||||
- Various optimizations:
|
||||
- RISC-V without A-extension: Optimize 16-bit fetch_not when Zabha enabled. ([a487a09](https://github.com/taiki-e/portable-atomic/commit/a487a094ab99fdf8ef3413b943c9c154a352d8c4))
|
||||
- s390x: Optimize 128-bit CAS/RMW. ([fba028d](https://github.com/taiki-e/portable-atomic/commit/fba028d7618e47b3de5b352c1789ea52f45298b4), [33ab2c1](https://github.com/taiki-e/portable-atomic/commit/33ab2c19719b29c0a30df8d011d0cd4003495dc8))
|
||||
- PowerPC64: Optimize 128-bit Acquire/AcqRel/SeqCst CAS/RMW and 128-bit CAS with Relaxed failure ordering. ([33ab2c1](https://github.com/taiki-e/portable-atomic/commit/33ab2c19719b29c0a30df8d011d0cd4003495dc8))
|
||||
- AVR: Optimize 8-bit load/store. ([33ab2c1](https://github.com/taiki-e/portable-atomic/commit/33ab2c19719b29c0a30df8d011d0cd4003495dc8))
|
||||
|
||||
- Improve support of run-time detection and outline-atomics:
|
||||
- Enable run-time detection by default on powerpc64 and aarch64 linux-uclibc. ([#193](https://github.com/taiki-e/portable-atomic/pull/193))
|
||||
- Improve run-time detection of powerpc64 quadword-atomics. ([1e3bfda](https://github.com/taiki-e/portable-atomic/commit/1e3bfda29bf6eb06a359f24554f19fc080ae57eb))
|
||||
- Improve run-time detection of Zhaoxin CPU. ([f283d2a](https://github.com/taiki-e/portable-atomic/commit/f283d2a1deb94e84676a2c1ee678439f507482db))
|
||||
|
||||
- Support RISC-V Zacas extension on pre-1.82 rustc. ([#194](https://github.com/taiki-e/portable-atomic/pull/194))
|
||||
|
||||
- Improve compile-time detection of RISC-V Zaamo/Zabha extensions. ([673137a](https://github.com/taiki-e/portable-atomic/commit/673137afb87b731ed19a1b0717059468712fad1d))
|
||||
|
||||
- Respect [`RUSTC_BOOTSTRAP=-1` recently added in nightly](https://github.com/rust-lang/rust/pull/132993) in rustc version detection. ([5b2847a](https://github.com/taiki-e/portable-atomic/commit/5b2847a8b99aa2a57a6c80f5a47327b2764f08cc))
|
||||
|
||||
## [1.9.0] - 2024-09-28
|
||||
|
||||
- RISC-V without A-extension: Support RMW when Zaamo extension enabled (even when `unsafe-assume-single-core` disabled). ([#185](https://github.com/taiki-e/portable-atomic/pull/185), [9983a8b](https://github.com/taiki-e/portable-atomic/commit/9983a8b9ad66efe4303b95678014369a56839aef))
|
||||
See "operations don't require disabling interrupts" list in [`interrupt` module's readme](https://github.com/taiki-e/portable-atomic/blob/HEAD/src/imp/interrupt/README.md) for the operations provided.
|
||||
|
||||
- Support run-time detection of RISC-V Zacas extension (currently disabled by default). ([#183](https://github.com/taiki-e/portable-atomic/pull/183))
|
||||
|
||||
- Support 128-bit atomics on Arm64EC (currently nightly-only) ([#184](https://github.com/taiki-e/portable-atomic/pull/184))
|
||||
|
||||
- Improve compile-time detection of powerpc64 quadword-atomics. ([3eb8507](https://github.com/taiki-e/portable-atomic/commit/3eb8507f91c1ab382d9f455b2311a8664a4c33ff))
|
||||
|
||||
## [1.8.0] - 2024-09-20
|
||||
|
||||
- Improve diagnostics when method that requires CAS is unavailable. ([#181](https://github.com/taiki-e/portable-atomic/pull/181))
|
||||
|
||||
Before:
|
||||
|
||||
```text
|
||||
error[E0599]: no method named `compare_exchange` found for struct `portable_atomic::AtomicUsize` in the current scope
|
||||
--> src/race.rs:60:24
|
||||
|
|
||||
60 | self.inner.compare_exchange(0, value.get(), Ordering::AcqRel, Ordering::Acquire);
|
||||
| ^^^^^^^^^^^^^^^^ method not found in `AtomicUsize`
|
||||
```
|
||||
|
||||
After:
|
||||
|
||||
```text
|
||||
error[E0277]: `compare_exchange` requires atomic CAS but not available on this target by default
|
||||
--> src/race.rs:60:24
|
||||
|
|
||||
60 | self.inner.compare_exchange(0, value.get(), Ordering::AcqRel, Ordering::Acquire);
|
||||
| ^^^^^^^^^^^^^^^^ this associated function is not available on this target by default
|
||||
|
|
||||
= help: the trait `HasCompareExchange` is not implemented for `&portable_atomic::AtomicUsize`
|
||||
= note: consider enabling one of the `unsafe-assume-single-core` or `critical-section` Cargo features
|
||||
= note: see <https://docs.rs/portable-atomic/latest/portable_atomic/#optional-features> for more.
|
||||
```
|
||||
|
||||
- Improve compile error messages for some other cases ([19716ac](https://github.com/taiki-e/portable-atomic/commit/19716ac1d3b6082a8cb838af532ccab871041249), [61dcaaa](https://github.com/taiki-e/portable-atomic/commit/61dcaaa320cb347ab799c6c4f4480600692de2ad))
|
||||
|
||||
- Various improvements to RISC-V.
|
||||
- riscv64: Support 128-bit atomics when Zacas extension enabled. ([173](https://github.com/taiki-e/portable-atomic/pull/173)) This is currently marked as experimental because LLVM marking the corresponding target feature as experimental.
|
||||
- riscv32: Support 64-bit atomics when Zacas extension enabled. ([173](https://github.com/taiki-e/portable-atomic/pull/173)) This is currently marked as experimental because LLVM marking the corresponding target feature as experimental.
|
||||
- Improvements for RISC-V without A-extension:
|
||||
- Support zaamo target feature. When building for single-core RISC-V without A-extension, this is equivalent to force-amo feature ([8abba4b](https://github.com/taiki-e/portable-atomic/commit/8abba4b0ea920d23799c2d7b6985617e6392d176))
|
||||
- Support zabha target feature. ([694364a](https://github.com/taiki-e/portable-atomic/commit/694364a179441f723e429516132436c46f5857b4))
|
||||
- Strengthen SeqCst store to improve compatibility with code that uses atomic instruction mapping that differs from LLVM and GCC. ([5b10b15](https://github.com/taiki-e/portable-atomic/commit/5b10b1516e056b16b851e498a271f751152feed9))
|
||||
|
||||
- Improve support of run-time detection and outline-atomics:
|
||||
- aarch64: Support run-time detection of FEAT_LRCPC3/FEAT_LSE128 for load/store. ([#174](https://github.com/taiki-e/portable-atomic/pull/174))
|
||||
- aarch64: Support run-time detection of FEAT_LSE2 on OpenBSD. ([4f8c735](https://github.com/taiki-e/portable-atomic/commit/4f8c7350fd80d005233d85191707e4329a5ef484))
|
||||
- aarch64: Support run-time detection of FEAT_LSE/FEAT_LSE2 on illumos (currently disabled by default because illumos AArch64 port is experimental). ([#175](https://github.com/taiki-e/portable-atomic/pull/175))
|
||||
- powerpc64: Support run-time detection on OpenBSD 7.6+ (currently disabled by default for compatibility with old versions). ([09a967b](https://github.com/taiki-e/portable-atomic/commit/09a967b59c217dc9ebb4d78ec114758ef9941fc8))
|
||||
|
||||
- Support AArch64 FEAT_LRCPC3/FEAT_LSE128 with pre-16 LLVM. ([#178](https://github.com/taiki-e/portable-atomic/pull/178))
|
||||
|
||||
- Improve compile-time detection of AArch64 FEAT_LSE2/FEAT_LRCPC3/FEAT_LSE128. ([10d47de](https://github.com/taiki-e/portable-atomic/commit/10d47def74b9c13fd864436d6118e902f118c028))
|
||||
|
||||
- Relax minimal version of `serde` (supported via optional feature) to 1.0.60.
|
||||
|
||||
## [1.7.0] - 2024-07-19
|
||||
|
||||
- Support run-time detection for cmpxchg16b on x86_64 on pre-1.69 rustc. ([#154](https://github.com/taiki-e/portable-atomic/pull/154))
|
||||
|
||||
- Make `into_inner` `const fn` on Rust 1.56+. (align to the [std atomic change in Rust 1.79](https://github.com/rust-lang/rust/pull/123522)) ([dee1f89](https://github.com/taiki-e/portable-atomic/commit/dee1f89739594271e4f5b5d3f122d2762fcbbd7d))
|
||||
|
||||
- Work around [rustc_codegen_gcc bug on x86_64](https://github.com/rust-lang/rustc_codegen_gcc/issues/485). ([d938f77](https://github.com/taiki-e/portable-atomic/commit/d938f77f3f2f353cbfb525ac23e63e880cd583df))
|
||||
|
||||
- Optimize x86_64 atomics.
|
||||
- Optimize 128-bit load/store on Zhaoxin CPU with AVX. ([86cee8f](https://github.com/taiki-e/portable-atomic/commit/86cee8fccf7deedb5b8cbcb3868ec2e43aca575f))
|
||||
- Optimize 128-bit SeqCst store on Intel/AMD/Zhaoxin CPU with AVX. ([#156](https://github.com/taiki-e/portable-atomic/pull/156), [0483042](https://github.com/taiki-e/portable-atomic/commit/0483042671fad4ccd0bdb691424a967fc2a5cb8e))
|
||||
- Remove needless test in CAS. ([573e025](https://github.com/taiki-e/portable-atomic/commit/573e02586f933a257c8e7383dcf64408b4708b85))
|
||||
|
||||
- Make rustc version detection robust for custom toolchains. ([f8ea85e](https://github.com/taiki-e/portable-atomic/commit/f8ea85e1aa46fa00bc865633fb40b05f8a0c823b))
|
||||
|
||||
- Respect `RUSTC_WRAPPER` in rustc version detection.
|
||||
|
||||
- Our build script is now less likely to be [re-run unnecessarily](https://github.com/taiki-e/portable-atomic/issues/151) in versions where the cargo bug fix is available (cargo 1.79+). ([52c277b](https://github.com/taiki-e/portable-atomic/commit/52c277bc9a9a8031175aaed54987260da1750af4))
|
||||
|
||||
## [1.6.0] - 2023-12-06
|
||||
|
||||
- Add `cfg_{has,no}_atomic_{8,16,32,64,128,ptr}` macros to enable code when the corresponding atomic implementation is available/unavailable.
|
||||
|
||||
- Add `cfg_{has,no}_atomic_cas` macros to enable code when atomic CAS/RMW implementation is available/unavailable.
|
||||
|
||||
- Improve support for RISC-V targets without atomic CAS.
|
||||
|
||||
## [1.5.1] - 2023-10-29
|
||||
|
||||
- Fix bug in `i{8,16}` `fetch_{or,xor}` on RISC-V without A-extension where `unsafe-assume-single-core` and `force-amo` are enabled.
|
||||
|
||||
- Optimize `swap` for targets that do not have native atomic CAS instructions.
|
||||
|
||||
## [1.5.0] - 2023-10-23
|
||||
|
||||
**Note:** This release has been yanked due to a bug fixed in 1.5.1.
|
||||
|
||||
- Add `from_ptr`.
|
||||
|
||||
- Add `force-amo` feature (`portable_atomic_force_amo` cfg) for single-core RISC-V without A-extension. ([#124](https://github.com/taiki-e/portable-atomic/pull/124))
|
||||
|
||||
- Support run-time detection on AArch64 on pre-1.61 rustc. ([#98](https://github.com/taiki-e/portable-atomic/pull/98))
|
||||
|
||||
This also solves [a compatibility issue with rustc_codegen_cranelift](https://github.com/rust-lang/rustc_codegen_cranelift/issues/1400).
|
||||
|
||||
- Support run-time detection of FEAT_LSE2. ([#126](https://github.com/taiki-e/portable-atomic/pull/126))
|
||||
|
||||
- Support run-time detection of FEAT_LSE on AArch64 NetBSD. ([#66](https://github.com/taiki-e/portable-atomic/pull/66))
|
||||
|
||||
- Acknowledge ESP-IDF targets' 64-bit atomics are not lock-free. See [#122](https://github.com/taiki-e/portable-atomic/issues/122) for more.
|
||||
|
||||
- Optimize 128-bit weak CAS on powerpc64.
|
||||
|
||||
- Optimize interrupt disable on no-std pre-v6 Arm where `unsafe-assume-single-core` and `disable-fiq` are enabled. ([771c45d](https://github.com/taiki-e/portable-atomic/commit/771c45da2d2afc4f83df033dd4bdf3f976d14a74))
|
||||
|
||||
- Improve detection of Apple hardware. ([5c3a43b](https://github.com/taiki-e/portable-atomic/commit/5c3a43b53f1c4188f9dd597599633bc1a315bf44))
|
||||
|
||||
- Improve compatibility with the future version of Miri.
|
||||
|
||||
## [1.4.3] - 2023-08-25
|
||||
|
||||
- Optimize AArch64 128-bit atomic store/swap/fetch_and/fetch_or when the `lse128` target feature is enabled at compile-time. ([#68](https://github.com/taiki-e/portable-atomic/pull/68))
|
||||
|
||||
- Optimize AArch64 128-bit atomic load/store when the `rcpc3` target feature is enabled at compile-time. ([#68](https://github.com/taiki-e/portable-atomic/pull/68))
|
||||
|
||||
- Optimize inline assemblies on Arm, AArch64, and MSP430.
|
||||
|
||||
## [1.4.2] - 2023-07-27
|
||||
|
||||
- Optimize `AtomicBool` on RISC-V/LoongArch64. This is the same as [rust-lang/rust#114034](https://github.com/rust-lang/rust/pull/114034), but is available for all rustc versions.
|
||||
|
||||
## [1.4.1] - 2023-07-15
|
||||
|
||||
- Improve compatibility with the future version of Miri.
|
||||
|
||||
## [1.4.0] - 2023-07-11
|
||||
|
||||
- Allow using embedded-related cfgs as Cargo features. ([#94](https://github.com/taiki-e/portable-atomic/pull/94), thanks @Dirbaio)
|
||||
|
||||
Originally, we were providing these as cfgs instead of features, but based on a strong request from the embedded ecosystem, we have agreed to provide them as features as well. See [#94](https://github.com/taiki-e/portable-atomic/pull/94) for more.
|
||||
|
||||
cfgs are kept and can be used as aliases for features.
|
||||
|
||||
- Acknowledge all x86_64 Apple targets support 128-bit atomics.
|
||||
|
||||
Our code already recognizes this via `cfg(target_feature)`, so this only affects docs and users using pre-1.69 stable rustc.
|
||||
|
||||
See also [rust-lang/rust#112150](https://github.com/rust-lang/rust/pull/112150).
|
||||
|
||||
- Optimize 128-bit atomics on AArch64/s390x.
|
||||
|
||||
## [1.3.3] - 2023-05-31
|
||||
|
||||
- Fix build error on AArch64 ILP32 ABI targets (tier 3).
|
||||
|
||||
- Optimize 128-bit atomics on s390x.
|
||||
|
||||
## [1.3.2] - 2023-05-09
|
||||
|
||||
- Fix bug in powerpc64/s390x 128-bit atomic RMWs on old nightly.
|
||||
|
||||
- Optimize 128-bit atomics on powerpc64/s390x.
|
||||
|
||||
## [1.3.1] - 2023-05-07
|
||||
|
||||
- Documentation improvements.
|
||||
|
||||
## [1.3.0] - 2023-05-06
|
||||
|
||||
- Add `require-cas` feature. ([#100](https://github.com/taiki-e/portable-atomic/pull/100))
|
||||
|
||||
If your crate supports no-std environment and requires atomic CAS, enabling this feature will allow the `portable-atomic` to display helpful error messages to users on targets requiring additional action on the user side to provide atomic CAS.
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
portable-atomic = { version = "1.3", default-features = false, features = ["require-cas"] }
|
||||
```
|
||||
|
||||
See [#100](https://github.com/taiki-e/portable-atomic/pull/100) for more.
|
||||
|
||||
- Support `portable_atomic_unsafe_assume_single_core` cfg on Xtensa targets without atomic CAS. ([#86](https://github.com/taiki-e/portable-atomic/pull/86))
|
||||
|
||||
- Fix bug in AArch64 128-bit SeqCst load when FEAT_LSE2 is enabled at compile-time. This is [the same bug that was fixed in the recently released GCC 13.1](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108891). LLVM also has the same bug, which had not yet been fixed when the patch was created; I will open a bug report if necessary after looking into the situation in LLVM. ([a29154b](https://github.com/taiki-e/portable-atomic/commit/a29154b21da270e90cb86f6865b591ab36eade7d))
|
||||
|
||||
- Fix compile error on `bpf{eb,el}-unknown-none` (tier 3) and `mipsel-sony-psx` (tier 3) when `critical-section` feature is disabled.
|
||||
|
||||
- Various optimizations
|
||||
- Optimize x86_64 128-bit outline-atomics. This improves performance by up to 15% in concurrent RMW/store for cases where the `cmpxchg16b` target feature is not available at compile-time. ([40c4cd4](https://github.com/taiki-e/portable-atomic/commit/40c4cd4f682f1cb153f18d4d6a88795bafaf5667))
|
||||
- Optimize x86_64 128-bit load that uses cmpxchg16b. ([40c4cd4](https://github.com/taiki-e/portable-atomic/commit/40c4cd4f682f1cb153f18d4d6a88795bafaf5667))
|
||||
- Optimize AArch64 128-bit load that uses FEAT_LSE. ([40c4cd4](https://github.com/taiki-e/portable-atomic/commit/40c4cd4f682f1cb153f18d4d6a88795bafaf5667))
|
||||
- Optimize pre-Armv6 Linux/Android 64-bit atomics. ([efacc89](https://github.com/taiki-e/portable-atomic/commit/efacc89c210d7a34ef5e879821112189da5d1901))
|
||||
- Support outline-atomics for powerpc64 128-bit atomics. This is currently disabled by default, and can be enabled by `--cfg portable_atomic_outline_atomics`. ([#90](https://github.com/taiki-e/portable-atomic/pull/90))
|
||||
- Optimize AArch64 outline-atomics on linux-musl. On linux-musl, outline-atomics is enabled by default only when dynamic linking is enabled. When static linking is enabled, this can be enabled by `--cfg portable_atomic_outline_atomics`. See the [`atomic128` module's readme](https://github.com/taiki-e/portable-atomic/blob/HEAD/src/imp/atomic128/README.md#run-time-feature-detection) for more. ([8418235](https://github.com/taiki-e/portable-atomic/commit/84182354e4a149074e28bda4683d538e5fb617ce), [31d0862](https://github.com/taiki-e/portable-atomic/commit/31d08623d4e21af207ff2343f5553b9b5a030452))
|
||||
|
||||
## [1.2.0] - 2023-03-25
|
||||
|
||||
- Make 64-bit atomics lock-free on Arm Linux/Android targets that do not have 64-bit atomics (e.g., armv5te-unknown-linux-gnueabi, arm-linux-androideabi, etc.) when the kernel version is 3.1 or later. ([#82](https://github.com/taiki-e/portable-atomic/pull/82))
|
||||
|
||||
- Fix AArch64 128-bit atomics performance regression on Apple hardware. ([#89](https://github.com/taiki-e/portable-atomic/pull/89))
|
||||
|
||||
- Optimize 128-bit atomics on AArch64, x86_64, powerpc64, and s390x.
|
||||
|
||||
## [1.1.0] - 2023-03-24
|
||||
|
||||
- Add `Atomic{I,U}*::bit_{set,clear,toggle}` and `AtomicPtr::bit_{set,clear,toggle}`. ([#72](https://github.com/taiki-e/portable-atomic/pull/72))
|
||||
|
||||
They correspond to x86's `lock bt{s,r,c}`, and the implementation calls them on x86/x86_64.
|
||||
|
||||
- Add `AtomicU*::{fetch_neg,neg}` methods. Previously it was only available on `AtomicI*` and `AtomicF*`.
|
||||
|
||||
- Add `as_ptr` method to all atomic types. ([#79](https://github.com/taiki-e/portable-atomic/pull/79))
|
||||
|
||||
- Make `AtomicF{32,64}::as_bits` `const fn` on Rust 1.58+. ([#79](https://github.com/taiki-e/portable-atomic/pull/79))
|
||||
|
||||
- Relax ordering in `Serialize` impl to reflect the [upstream change](https://github.com/serde-rs/serde/pull/2263).
|
||||
|
||||
- Optimize x86_64 outline-atomics for 128-bit atomics.
|
||||
- Support outline-atomics for cmpxchg16b on Rust 1.69+ (i.e., on Rust 1.69+, x86_64 128-bit atomics is lock-free on all Intel chips and almost all AMD chips, even if cmpxchg16b is not available at compile-time.). Previously it was only nightly. ([#80](https://github.com/taiki-e/portable-atomic/pull/80))
|
||||
- portable-atomic no longer enables outline-atomics on target where run-time CPU feature detection is not available. ([#80](https://github.com/taiki-e/portable-atomic/pull/80))
|
||||
|
||||
- Optimize AArch64 outline-atomics for 128-bit atomics.
|
||||
- Support more targets and improve performance. ([#63](https://github.com/taiki-e/portable-atomic/pull/63), [#64](https://github.com/taiki-e/portable-atomic/pull/64), [#67](https://github.com/taiki-e/portable-atomic/pull/67), [#69](https://github.com/taiki-e/portable-atomic/pull/69), [#75](https://github.com/taiki-e/portable-atomic/pull/75), [#76](https://github.com/taiki-e/portable-atomic/pull/76), [#77](https://github.com/taiki-e/portable-atomic/pull/77))
|
||||
See the [`atomic128` module's readme](https://github.com/taiki-e/portable-atomic/blob/HEAD/src/imp/atomic128/README.md#run-time-feature-detection) for a list of platforms that support outline-atomics.
|
||||
Most of these improvements have already been [submitted and accepted in rust-lang/stdarch](https://github.com/rust-lang/stdarch/pulls?q=is%3Apr+author%3Ataiki-e+std_detect) and will soon be available in `std::arch::is_aarch64_feature_detected`.
|
||||
- portable-atomic no longer enables outline-atomics on target where run-time CPU feature detection is not available.
|
||||
|
||||
- Performance improvements. ([#70](https://github.com/taiki-e/portable-atomic/pull/70), [#81](https://github.com/taiki-e/portable-atomic/pull/81), [6c189ae](https://github.com/taiki-e/portable-atomic/commit/6c189ae1792ce0c08b4f56b6e6c256c223475ce2), [13c92b0](https://github.com/taiki-e/portable-atomic/commit/13c92b015a8e8646a4b885229157547354d03b9e), etc.)
|
||||
|
||||
- Improve support for old nightly. ([#73](https://github.com/taiki-e/portable-atomic/pull/73), [872feb9](https://github.com/taiki-e/portable-atomic/commit/872feb9d7f3a4ca7cf9b63935265d46498fcae99))
|
||||
|
||||
- Documentation improvements.
|
||||
|
||||
## [1.0.1] - 2023-01-21
|
||||
|
||||
- Optimize `Atomic{I,U}*::{fetch_not,not}` methods. ([#62](https://github.com/taiki-e/portable-atomic/pull/62))
|
||||
|
||||
## [1.0.0] - 2023-01-15
|
||||
|
||||
- Add `critical-section` feature to use [critical-section](https://github.com/rust-embedded/critical-section) on targets where atomic CAS is not natively available. ([#51](https://github.com/taiki-e/portable-atomic/pull/51), thanks @Dirbaio)
|
||||
|
||||
This is useful to get atomic CAS when `--cfg portable_atomic_unsafe_assume_single_core` can't be used, such as multi-core targets, unprivileged code running under some RTOS, or environments where disabling interrupts needs extra care due to e.g. real-time requirements.
|
||||
|
||||
See [documentation](https://github.com/taiki-e/portable-atomic#optional-features-critical-section) for more.
|
||||
|
||||
- Remove `outline-atomics` feature. This was no-op since 0.3.19.
|
||||
|
||||
- Documentation improvements.
|
||||
|
||||
## [0.3.20] - 2023-05-07
|
||||
|
||||
The latest version of portable-atomic is 1.x. This release makes portable-atomic 0.3 is built on top of portable-atomic 1.x to make bug fixes and improvements such as [support for new targets](https://github.com/taiki-e/portable-atomic/pull/86) in 1.x available to the ecosystem that depends on older portable-atomic. portable-atomic 0.3 is still maintained passively, but upgrading to portable-atomic 1.x is recommended. (There are no breaking changes from 0.3, except that a deprecated no-op `outline-atomics` Cargo feature has been removed.) ([#99](https://github.com/taiki-e/portable-atomic/pull/99))
|
||||
|
||||
## [0.3.19] - 2022-12-25
|
||||
|
||||
- Add `AtomicI*::{fetch_neg,neg}` and `AtomicF*::fetch_neg` methods. ([#54](https://github.com/taiki-e/portable-atomic/pull/54))
|
||||
|
||||
`AtomicI*::neg` are equivalent to the corresponding `fetch_*` methods, but do not return the previous value. They are intended for optimization on platforms that have atomic instructions for the corresponding operation, such as x86's `lock neg`.
|
||||
|
||||
Currently, optimizations by these methods (`neg`) are only guaranteed for x86/x86_64.
|
||||
|
||||
- Add `Atomic{I,U}*::{fetch_not,not}` methods. ([#54](https://github.com/taiki-e/portable-atomic/pull/54))
|
||||
|
||||
`Atomic{I,U}*::not` are equivalent to the corresponding `fetch_*` methods, but do not return the previous value. They are intended for optimization on platforms that have atomic instructions for the corresponding operation, such as x86's `lock not`, MSP430's `inv`.
|
||||
|
||||
Currently, optimizations by these methods (`not`) are only guaranteed for x86/x86_64 and MSP430.
|
||||
|
||||
(Note: `AtomicBool` already has `fetch_not` and `not` methods.)
|
||||
|
||||
- Enable outline-atomics for 128-bit atomics by default. ([#57](https://github.com/taiki-e/portable-atomic/pull/57)) See [#57](https://github.com/taiki-e/portable-atomic/pull/57) for more.
|
||||
|
||||
- Improve support for old nightly compilers.
|
||||
|
||||
## [0.3.18] - 2022-12-15
|
||||
|
||||
- Fix build error when not using `portable_atomic_unsafe_assume_single_core` cfg on AVR and MSP430 custom targets. ([#50](https://github.com/taiki-e/portable-atomic/pull/50))
|
||||
|
||||
Since 0.3.11, atomic CAS was supported without the cfg on AVR and MSP430 builtin targets, but that change was not applied to custom targets.
|
||||
|
||||
## [0.3.17] - 2022-12-14
|
||||
|
||||
- Optimize x86_64 128-bit atomic load/store on AMD CPU with AVX. ([#49](https://github.com/taiki-e/portable-atomic/pull/49))
|
||||
|
||||
- Improve support for custom targets on old rustc.
|
||||
|
||||
## [0.3.16] - 2022-12-09
|
||||
|
||||
- Add `Atomic{I,U}*::{add,sub,and,or,xor}` and `AtomicBool::{and,or,xor}` methods. ([#47](https://github.com/taiki-e/portable-atomic/pull/47))
|
||||
|
||||
They are equivalent to the corresponding `fetch_*` methods, but do not return the previous value. They are intended for optimization on platforms that implement atomics using inline assembly, such as the MSP430.
|
||||
|
||||
Currently, optimizations by these methods (`add`,`sub`,`and`,`or`,`xor`) are only guaranteed for MSP430; on x86/x86_64, LLVM can optimize in most cases, so cases, where this would improve things, should be rare.
|
||||
|
||||
- Various improvements to `portable_atomic_unsafe_assume_single_core` cfg. ([#44](https://github.com/taiki-e/portable-atomic/pull/44), [#40](https://github.com/taiki-e/portable-atomic/pull/40))
|
||||
|
||||
- Support disabling FIQs on pre-v6 Arm under `portable_atomic_disable_fiq` cfg.
|
||||
- Support RISC-V supervisor mode under `portable_atomic_s_mode` cfg.
|
||||
- Optimize interrupt restore on AVR and MSP430. ([#40](https://github.com/taiki-e/portable-atomic/pull/40))
|
||||
- Documentation improvements.
|
||||
|
||||
See [#44](https://github.com/taiki-e/portable-atomic/pull/44) for more.
|
||||
|
||||
## [0.3.15] - 2022-09-09
|
||||
|
||||
- Implement workaround for std cpuid bug due to LLVM bug ([rust-lang/rust#101346](https://github.com/rust-lang/rust/issues/101346), [llvm/llvm-project#57550](https://github.com/llvm/llvm-project/issues/57550)).
|
||||
|
||||
- Our use case is likely not affected, but we implement this just in case.
|
||||
- We've confirmed that the uses of inline assembly in this crate are not affected by this LLVM bug.
|
||||
|
||||
## [0.3.14] - 2022-09-04
|
||||
|
||||
- Optimize atomic load/store on no-std pre-v6 Arm when `portable_atomic_unsafe_assume_single_core` cfg is used. ([#36](https://github.com/taiki-e/portable-atomic/pull/36))
|
||||
|
||||
- Support pre-power8 powerpc64le. powerpc64le's default cpu version is power8, but you can technically compile it for the old cpu using the unsafe `-C target-cpu` rustc flag.
|
||||
|
||||
## [0.3.13] - 2022-08-15
|
||||
|
||||
- Use track_caller when debug assertions are enabled on Rust 1.46+.
|
||||
|
||||
- Make powerpc64 128-bit atomics compatible with Miri and ThreadSanitizer on LLVM 15+.
|
||||
|
||||
- Document that 128-bit atomics are compatible with Miri and ThreadSanitizer on recent nightly.
|
||||
|
||||
## [0.3.12] - 2022-08-13
|
||||
|
||||
- Support atomic CAS on no-std pre-v6 Arm targets (e.g., thumbv4t-none-eabi) under unsafe cfg `portable_atomic_unsafe_assume_single_core`. ([#28](https://github.com/taiki-e/portable-atomic/pull/28))
|
||||
|
||||
## [0.3.11] - 2022-08-12
|
||||
|
||||
- Always provide atomic CAS for MSP430 and AVR. ([#31](https://github.com/taiki-e/portable-atomic/pull/31))
|
||||
|
||||
This previously required unsafe cfg `portable_atomic_unsafe_assume_single_core`, but since all MSP430 and AVR are single-core, we can safely provide atomic CAS based on disabling interrupts.
|
||||
|
||||
- Support `fence` and `compiler_fence` on MSP430. (On MSP430, the standard library's fences are currently unavailable due to LLVM errors.)
|
||||
|
||||
- Update safety requirements for unsafe cfg `portable_atomic_unsafe_assume_single_core` to mention use of privileged instructions to disable interrupts.
|
||||
|
||||
- Atomic operations based on disabling interrupts on single-core systems are now considered lock-free.
|
||||
|
||||
The previous behavior was inconsistent because we consider the pre-v6 Arm Linux's atomic operations provided in a similar way by the Linux kernel to be lock-free.
|
||||
|
||||
- Respect `-Z allow-features`.
|
||||
|
||||
## [0.3.10] - 2022-08-03
|
||||
|
||||
- Optimize AArch64 128-bit atomic load when the `lse` target feature is enabled at compile-time. ([#20](https://github.com/taiki-e/portable-atomic/pull/20))
|
||||
|
||||
## [0.3.9] - 2022-08-03
|
||||
|
||||
- Fix build error on old Miri.
|
||||
|
||||
- Documentation improvements.
|
||||
|
||||
## [0.3.8] - 2022-08-02
|
||||
|
||||
- Make AArch64 and s390x 128-bit atomics compatible with Miri and ThreadSanitizer.
|
||||
|
||||
## [0.3.7] - 2022-07-31
|
||||
|
||||
- Provide stable equivalent of [`#![feature(strict_provenance_atomic_ptr)]`](https://github.com/rust-lang/rust/issues/99108). ([#23](https://github.com/taiki-e/portable-atomic/pull/23))
|
||||
|
||||
- `AtomicPtr::fetch_ptr_{add,sub}`
|
||||
- `AtomicPtr::fetch_byte_{add,sub}`
|
||||
- `AtomicPtr::fetch_{or,and,xor}`
|
||||
|
||||
These APIs are compatible with strict-provenance on `cfg(miri)`. Otherwise, they are compatible with permissive-provenance.
|
||||
Once `#![feature(strict_provenance_atomic_ptr)]` is stabilized, these APIs will be strict-provenance compatible in all cases from the version in which it is stabilized.
|
||||
|
||||
- Provide stable equivalent of [`#![feature(atomic_bool_fetch_not)]`](https://github.com/rust-lang/rust/issues/98485). ([#24](https://github.com/taiki-e/portable-atomic/pull/24))
|
||||
|
||||
- `AtomicBool::fetch_not`
|
||||
|
||||
- Optimize x86_64 128-bit RMWs. ([#22](https://github.com/taiki-e/portable-atomic/pull/22))
|
||||
|
||||
- Optimize x86_64 outline-atomics.
|
||||
|
||||
- Optimize inline assemblies on Arm and AArch64.
|
||||
|
||||
- Revert [thumbv6m atomic load/store changes made in 0.3.5](https://github.com/taiki-e/portable-atomic/pull/18). This is because [rust-lang/rust#99595](https://github.com/rust-lang/rust/pull/99595) has been reverted, so this is no longer needed.
|
||||
|
||||
## [0.3.6] - 2022-07-26
|
||||
|
||||
- Fix build failure due to the existence of the `specs` directory.
|
||||
|
||||
- Documentation improvements.
|
||||
|
||||
- Optimize inline assemblies on x86_64, RISC-V, and MSP430.
|
||||
|
||||
## [0.3.5] - 2022-07-23
|
||||
|
||||
**Note:** This release has been yanked due to a bug fixed in 0.3.6.
|
||||
|
||||
- Provide thumbv6m atomic load/store which is planned to be removed from the standard library in [rust-lang/rust#99595](https://github.com/rust-lang/rust/pull/99595). ([#18](https://github.com/taiki-e/portable-atomic/pull/18))
|
||||
|
||||
- Optimize inline assemblies on AArch64, RISC-V, and powerpc64.
|
||||
|
||||
## [0.3.4] - 2022-06-25
|
||||
|
||||
- Optimize x86_64 128-bit atomic store.
|
||||
|
||||
## [0.3.3] - 2022-06-24
|
||||
|
||||
- Allow CAS failure ordering stronger than success ordering. ([#17](https://github.com/taiki-e/portable-atomic/pull/17))
|
||||
|
||||
## [0.3.2] - 2022-06-19
|
||||
|
||||
- Optimize x86_64 128-bit atomic load/store on Intel CPU with AVX. ([#16](https://github.com/taiki-e/portable-atomic/pull/16))
|
||||
|
||||
- Support native 128-bit atomic operations for powerpc64 (le or pwr8+, currently nightly-only).
|
||||
|
||||
- Fix behavior differences between stable and nightly. ([#15](https://github.com/taiki-e/portable-atomic/pull/15))
|
||||
|
||||
## [0.3.1] - 2022-06-16
|
||||
|
||||
- Optimize AArch64 128-bit atomic load/store when the `lse2` target feature is enabled at compile-time. ([#11](https://github.com/taiki-e/portable-atomic/pull/11))
|
||||
|
||||
- Relax ordering in `Debug` impl to reflect std changes. ([#12](https://github.com/taiki-e/portable-atomic/pull/12))
|
||||
|
||||
## [0.3.0] - 2022-03-25
|
||||
|
||||
- Support native 128-bit atomic operations for s390x (currently nightly-only).
|
||||
|
||||
- Add `AtomicF{32,64}::fetch_abs`.
|
||||
|
||||
- Add `#[must_use]` to constructors.
|
||||
|
||||
- Use 128-bit atomic operation mappings same as LLVM on AArch64.
|
||||
|
||||
- Remove `parking_lot` optional feature to allow the use of this crate within global allocators.
|
||||
|
||||
## [0.2.1] - 2022-03-17
|
||||
|
||||
- Implement AArch64 outline-atomics.
|
||||
|
||||
## [0.2.0] - 2022-03-10
|
||||
|
||||
- Remove `i128` feature. `Atomic{I,U}128` are now always enabled.
|
||||
|
||||
- Add `outline-atomics` feature. Currently, this is the same as the 0.1's `i128-dynamic`, except that `fallback` feature is not implicitly enabled.
|
||||
|
||||
- Remove `i128-dynamic` feature in favor of `outline-atomics` feature.
|
||||
|
||||
- Add `AtomicF{32,64}::as_bits`.
|
||||
|
||||
## [0.1.4] - 2022-03-02
|
||||
|
||||
- Support native 128-bit atomic operations for AArch64 at Rust 1.59+. This was previously supported only on nightly. ([#6](https://github.com/taiki-e/portable-atomic/pull/6))
|
||||
|
||||
## [0.1.3] - 2022-02-28
|
||||
|
||||
- Fix inline assembly for RISC-V without A-extension.
|
||||
|
||||
## [0.1.2] - 2022-02-26
|
||||
|
||||
**Note:** This release has been yanked due to a bug fixed in 0.1.3.
|
||||
|
||||
- Add `parking_lot` feature to use parking_lot in global locks of fallback implementation.
|
||||
|
||||
- Fix bug in cmpxchg16b support. ([#5](https://github.com/taiki-e/portable-atomic/pull/5))
|
||||
|
||||
## [0.1.1] - 2022-02-25
|
||||
|
||||
**Note:** This release has been yanked due to a bug fixed in 0.1.3.
|
||||
|
||||
- Fix doc cfg on `Atomic{I,U}128`.
|
||||
|
||||
## [0.1.0] - 2022-02-24
|
||||
|
||||
**Note:** This release has been yanked due to a bug fixed in 0.1.3.
|
||||
|
||||
Initial release
|
||||
|
||||
[Unreleased]: https://github.com/taiki-e/portable-atomic/compare/v1.15.0...HEAD
|
||||
[1.15.0]: https://github.com/taiki-e/portable-atomic/compare/v1.14.0...v1.15.0
|
||||
[1.14.0]: https://github.com/taiki-e/portable-atomic/compare/v1.13.1...v1.14.0
|
||||
[1.13.1]: https://github.com/taiki-e/portable-atomic/compare/v1.13.0...v1.13.1
|
||||
[1.13.0]: https://github.com/taiki-e/portable-atomic/compare/v1.12.0...v1.13.0
|
||||
[1.12.0]: https://github.com/taiki-e/portable-atomic/compare/v1.11.1...v1.12.0
|
||||
[1.11.1]: https://github.com/taiki-e/portable-atomic/compare/v1.11.0...v1.11.1
|
||||
[1.11.0]: https://github.com/taiki-e/portable-atomic/compare/v1.10.0...v1.11.0
|
||||
[1.10.0]: https://github.com/taiki-e/portable-atomic/compare/v1.9.0...v1.10.0
|
||||
[1.9.0]: https://github.com/taiki-e/portable-atomic/compare/v1.8.0...v1.9.0
|
||||
[1.8.0]: https://github.com/taiki-e/portable-atomic/compare/v1.7.0...v1.8.0
|
||||
[1.7.0]: https://github.com/taiki-e/portable-atomic/compare/v1.6.0...v1.7.0
|
||||
[1.6.0]: https://github.com/taiki-e/portable-atomic/compare/v1.5.1...v1.6.0
|
||||
[1.5.1]: https://github.com/taiki-e/portable-atomic/compare/v1.5.0...v1.5.1
|
||||
[1.5.0]: https://github.com/taiki-e/portable-atomic/compare/v1.4.3...v1.5.0
|
||||
[1.4.3]: https://github.com/taiki-e/portable-atomic/compare/v1.4.2...v1.4.3
|
||||
[1.4.2]: https://github.com/taiki-e/portable-atomic/compare/v1.4.1...v1.4.2
|
||||
[1.4.1]: https://github.com/taiki-e/portable-atomic/compare/v1.4.0...v1.4.1
|
||||
[1.4.0]: https://github.com/taiki-e/portable-atomic/compare/v1.3.3...v1.4.0
|
||||
[1.3.3]: https://github.com/taiki-e/portable-atomic/compare/v1.3.2...v1.3.3
|
||||
[1.3.2]: https://github.com/taiki-e/portable-atomic/compare/v1.3.1...v1.3.2
|
||||
[1.3.1]: https://github.com/taiki-e/portable-atomic/compare/v1.3.0...v1.3.1
|
||||
[1.3.0]: https://github.com/taiki-e/portable-atomic/compare/v1.2.0...v1.3.0
|
||||
[1.2.0]: https://github.com/taiki-e/portable-atomic/compare/v1.1.0...v1.2.0
|
||||
[1.1.0]: https://github.com/taiki-e/portable-atomic/compare/v1.0.1...v1.1.0
|
||||
[1.0.1]: https://github.com/taiki-e/portable-atomic/compare/v1.0.0...v1.0.1
|
||||
[1.0.0]: https://github.com/taiki-e/portable-atomic/compare/v0.3.19...v1.0.0
|
||||
[0.3.20]: https://github.com/taiki-e/portable-atomic/compare/v0.3.19...v0.3.20
|
||||
[0.3.19]: https://github.com/taiki-e/portable-atomic/compare/v0.3.18...v0.3.19
|
||||
[0.3.18]: https://github.com/taiki-e/portable-atomic/compare/v0.3.17...v0.3.18
|
||||
[0.3.17]: https://github.com/taiki-e/portable-atomic/compare/v0.3.16...v0.3.17
|
||||
[0.3.16]: https://github.com/taiki-e/portable-atomic/compare/v0.3.15...v0.3.16
|
||||
[0.3.15]: https://github.com/taiki-e/portable-atomic/compare/v0.3.14...v0.3.15
|
||||
[0.3.14]: https://github.com/taiki-e/portable-atomic/compare/v0.3.13...v0.3.14
|
||||
[0.3.13]: https://github.com/taiki-e/portable-atomic/compare/v0.3.12...v0.3.13
|
||||
[0.3.12]: https://github.com/taiki-e/portable-atomic/compare/v0.3.11...v0.3.12
|
||||
[0.3.11]: https://github.com/taiki-e/portable-atomic/compare/v0.3.10...v0.3.11
|
||||
[0.3.10]: https://github.com/taiki-e/portable-atomic/compare/v0.3.9...v0.3.10
|
||||
[0.3.9]: https://github.com/taiki-e/portable-atomic/compare/v0.3.8...v0.3.9
|
||||
[0.3.8]: https://github.com/taiki-e/portable-atomic/compare/v0.3.7...v0.3.8
|
||||
[0.3.7]: https://github.com/taiki-e/portable-atomic/compare/v0.3.6...v0.3.7
|
||||
[0.3.6]: https://github.com/taiki-e/portable-atomic/compare/v0.3.5...v0.3.6
|
||||
[0.3.5]: https://github.com/taiki-e/portable-atomic/compare/v0.3.4...v0.3.5
|
||||
[0.3.4]: https://github.com/taiki-e/portable-atomic/compare/v0.3.3...v0.3.4
|
||||
[0.3.3]: https://github.com/taiki-e/portable-atomic/compare/v0.3.2...v0.3.3
|
||||
[0.3.2]: https://github.com/taiki-e/portable-atomic/compare/v0.3.1...v0.3.2
|
||||
[0.3.1]: https://github.com/taiki-e/portable-atomic/compare/v0.3.0...v0.3.1
|
||||
[0.3.0]: https://github.com/taiki-e/portable-atomic/compare/v0.2.1...v0.3.0
|
||||
[0.2.1]: https://github.com/taiki-e/portable-atomic/compare/v0.2.0...v0.2.1
|
||||
[0.2.0]: https://github.com/taiki-e/portable-atomic/compare/v0.1.4...v0.2.0
|
||||
[0.1.4]: https://github.com/taiki-e/portable-atomic/compare/v0.1.3...v0.1.4
|
||||
[0.1.3]: https://github.com/taiki-e/portable-atomic/compare/v0.1.2...v0.1.3
|
||||
[0.1.2]: https://github.com/taiki-e/portable-atomic/compare/v0.1.1...v0.1.2
|
||||
[0.1.1]: https://github.com/taiki-e/portable-atomic/compare/v0.1.0...v0.1.1
|
||||
[0.1.0]: https://github.com/taiki-e/portable-atomic/releases/tag/v0.1.0
|
||||
+203
@@ -0,0 +1,203 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
[[package]]
|
||||
name = "build-context"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "critical-section"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.163 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"r-efi 6.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand_core 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.163"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "paste"
|
||||
version = "1.0.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic"
|
||||
version = "1.15.0"
|
||||
dependencies = [
|
||||
"build-context 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"critical-section 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.8.22 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"fastrand 2.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.163 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"paste 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"quickcheck 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rustversion 1.0.23 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.229 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"sptr 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"static_assertions 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"windows-sys 0.61.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.107"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"unicode-ident 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quickcheck"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"rand 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.107 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r-efi"
|
||||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"getrandom 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand_core 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "rustversion"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"serde_core 1.0.229 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"serde_derive 1.0.229 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.107 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"quote 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"syn 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sptr"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "static_assertions"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.107 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"quote 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"unicode-ident 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"windows-link 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[metadata]
|
||||
"checksum build-context 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ebb068a5ba5b65a7cdba869497ca2a352e88e6093c4909ddf4d12d9d3151ee9c"
|
||||
"checksum cfg-if 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
"checksum critical-section 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b"
|
||||
"checksum crossbeam-utils 0.8.22 (registry+https://github.com/rust-lang/crates.io-index)" = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
|
||||
"checksum fastrand 2.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
|
||||
"checksum getrandom 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
|
||||
"checksum libc 0.2.163 (registry+https://github.com/rust-lang/crates.io-index)" = "1fdaeca4cf44ed4ac623e86ef41f056e848dbeab7ec043ecb7326ba300b36fd0"
|
||||
"checksum paste 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)" = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
|
||||
"checksum proc-macro2 1.0.107 (registry+https://github.com/rust-lang/crates.io-index)" = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
||||
"checksum quickcheck 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "95c589f335db0f6aaa168a7cd27b1fc6920f5e1470c804f814d9cd6e62a0f70b"
|
||||
"checksum quote 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)" = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
||||
"checksum r-efi 6.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
||||
"checksum rand 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80"
|
||||
"checksum rand_core 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
|
||||
"checksum rustversion 1.0.23 (registry+https://github.com/rust-lang/crates.io-index)" = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
|
||||
"checksum serde 1.0.229 (registry+https://github.com/rust-lang/crates.io-index)" = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
||||
"checksum serde_core 1.0.229 (registry+https://github.com/rust-lang/crates.io-index)" = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
||||
"checksum serde_derive 1.0.229 (registry+https://github.com/rust-lang/crates.io-index)" = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
"checksum sptr 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a"
|
||||
"checksum static_assertions 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
|
||||
"checksum syn 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
|
||||
"checksum unicode-ident 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)" = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
"checksum windows-link 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
"checksum windows-sys 0.61.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
||||
Vendored
+266
@@ -0,0 +1,266 @@
|
||||
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
|
||||
#
|
||||
# When uploading crates to the registry Cargo will automatically
|
||||
# "normalize" Cargo.toml files for maximal compatibility
|
||||
# with all versions of Cargo and also rewrite `path` dependencies
|
||||
# to registry (e.g., crates.io) dependencies.
|
||||
#
|
||||
# If you are reading this file be aware that the original Cargo.toml
|
||||
# will likely look very different (and much more reasonable).
|
||||
# See Cargo.toml.orig for the original contents.
|
||||
|
||||
[package]
|
||||
edition = "2018"
|
||||
rust-version = "1.34"
|
||||
name = "portable-atomic"
|
||||
version = "1.15.0"
|
||||
build = "build.rs"
|
||||
exclude = [
|
||||
"/.*",
|
||||
"/tools",
|
||||
"/target-specs",
|
||||
"/DEVELOPMENT.md",
|
||||
]
|
||||
autolib = false
|
||||
autobins = false
|
||||
autoexamples = false
|
||||
autotests = false
|
||||
autobenches = false
|
||||
description = """
|
||||
Portable atomic types including support for 128-bit atomics, atomic float, etc.
|
||||
"""
|
||||
readme = "README.md"
|
||||
keywords = ["atomic"]
|
||||
categories = [
|
||||
"concurrency",
|
||||
"embedded",
|
||||
"hardware-support",
|
||||
"no-std",
|
||||
"no-std::no-alloc",
|
||||
]
|
||||
license = "Apache-2.0 OR MIT"
|
||||
repository = "https://github.com/taiki-e/portable-atomic"
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
features = [
|
||||
"float",
|
||||
"std",
|
||||
"serde",
|
||||
"critical-section",
|
||||
]
|
||||
rustdoc-args = [
|
||||
"--extend-css",
|
||||
"src/rustdoc.css",
|
||||
"--cfg",
|
||||
"portable_atomic_unstable_f16",
|
||||
"--cfg",
|
||||
"portable_atomic_unstable_f128",
|
||||
]
|
||||
targets = ["x86_64-unknown-linux-gnu"]
|
||||
|
||||
[package.metadata.cargo_check_external_types]
|
||||
allowed_external_types = ["serde_core::*"]
|
||||
|
||||
[features]
|
||||
default = ["fallback"]
|
||||
disable-fiq = []
|
||||
fallback = []
|
||||
float = []
|
||||
force-amo = []
|
||||
require-cas = []
|
||||
s-mode = []
|
||||
std = []
|
||||
unsafe-assume-privileged = []
|
||||
unsafe-assume-single-core = []
|
||||
|
||||
[lib]
|
||||
name = "portable_atomic"
|
||||
path = "src/lib.rs"
|
||||
doc-scrape-examples = false
|
||||
|
||||
[dependencies.critical-section]
|
||||
version = "1"
|
||||
optional = true
|
||||
|
||||
[dependencies.serde]
|
||||
version = "1.0.60"
|
||||
optional = true
|
||||
default-features = false
|
||||
|
||||
[dev-dependencies.build-context]
|
||||
version = "0.1"
|
||||
|
||||
[dev-dependencies.crossbeam-utils]
|
||||
version = "0.8"
|
||||
|
||||
[dev-dependencies.fastrand]
|
||||
version = "2"
|
||||
|
||||
[dev-dependencies.paste]
|
||||
version = "1"
|
||||
|
||||
[dev-dependencies.quickcheck]
|
||||
version = "1"
|
||||
default-features = false
|
||||
|
||||
[dev-dependencies.rustversion]
|
||||
version = "1"
|
||||
|
||||
[dev-dependencies.sptr]
|
||||
version = "0.3"
|
||||
|
||||
[dev-dependencies.static_assertions]
|
||||
version = "1"
|
||||
|
||||
[target."cfg(unix)".dev-dependencies.libc]
|
||||
version = "=0.2.163"
|
||||
|
||||
[target."cfg(windows)".dev-dependencies.windows-sys]
|
||||
version = "0.61"
|
||||
features = [
|
||||
"Win32_Foundation",
|
||||
"Win32_System_Threading",
|
||||
]
|
||||
|
||||
[lints.clippy]
|
||||
all = "warn"
|
||||
as_ptr_cast_mut = "warn"
|
||||
as_underscore = "warn"
|
||||
default_union_representation = "warn"
|
||||
inline_asm_x86_att_syntax = "warn"
|
||||
pedantic = "warn"
|
||||
trailing_empty_array = "warn"
|
||||
transmute_undefined_repr = "warn"
|
||||
undocumented_unsafe_blocks = "warn"
|
||||
unused_trait_names = "warn"
|
||||
|
||||
[lints.clippy.assert_is_empty]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.bool_assert_comparison]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.borrow_as_ptr]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.cast_lossless]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.collapsible_match]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.declare_interior_mutable_const]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.doc_markdown]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.float_cmp]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.incompatible_msrv]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.manual_assert]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.manual_range_contains]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.missing_errors_doc]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.module_name_repetitions]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.naive_bytecount]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.nonminimal_bool]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.range_plus_one]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.similar_names]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.single_match]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.single_match_else]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.struct_excessive_bools]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.struct_field_names]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.too_many_arguments]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.too_many_lines]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.clippy.type_complexity]
|
||||
level = "allow"
|
||||
priority = 1
|
||||
|
||||
[lints.rust]
|
||||
deprecated_safe = "warn"
|
||||
improper_ctypes = "warn"
|
||||
improper_ctypes_definitions = "warn"
|
||||
non_ascii_idents = "warn"
|
||||
rust_2018_idioms = "warn"
|
||||
single_use_lifetimes = "warn"
|
||||
unnameable_types = "warn"
|
||||
unreachable_pub = "warn"
|
||||
|
||||
[lints.rust.unexpected_cfgs]
|
||||
level = "warn"
|
||||
priority = 0
|
||||
check-cfg = [
|
||||
'cfg(target_arch,values("xtensa"))',
|
||||
'cfg(target_arch,values("amdgpu"))',
|
||||
'cfg(target_arch,values("loongarch32"))',
|
||||
'cfg(target_os,values("trusty"))',
|
||||
'cfg(target_os,values("psx"))',
|
||||
'cfg(target_os,values("zos"))',
|
||||
'cfg(target_env,values("psx"))',
|
||||
'cfg(target_feature,values("lse2","lse128","rcpc3"))',
|
||||
'cfg(target_feature,values("quadword-atomics"))',
|
||||
'cfg(target_feature,values("zaamo","zabha"))',
|
||||
'cfg(target_feature,values("zacas"))',
|
||||
'cfg(target_feature,values("ztso"))',
|
||||
'cfg(target_feature,values("miscellaneous-extensions-3"))',
|
||||
'cfg(target_pointer_width,values("128"))',
|
||||
"cfg(portable_atomic_no_outline_atomics,portable_atomic_outline_atomics,portable_atomic_unstable_f16,portable_atomic_unstable_f128)",
|
||||
"cfg(portable_atomic_test_detect_false,portable_atomic_test_no_std_static_assert_ffi,qemu,valgrind,valgrind_cross)",
|
||||
]
|
||||
|
||||
[profile.release]
|
||||
debug = 2
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
[package]
|
||||
name = "portable-atomic"
|
||||
version = "1.15.0" #publish:version
|
||||
edition = "2018"
|
||||
# NB: Sync with rust-version field in other Cargo.toml files and msrv badge in README.md
|
||||
rust-version = "1.34" # For Atomic{I,U}{8,16,32,64}
|
||||
license = "Apache-2.0 OR MIT"
|
||||
repository = "https://github.com/taiki-e/portable-atomic"
|
||||
keywords = ["atomic"]
|
||||
categories = ["concurrency", "embedded", "hardware-support", "no-std", "no-std::no-alloc"]
|
||||
exclude = ["/.*", "/tools", "/target-specs", "/DEVELOPMENT.md"]
|
||||
description = """
|
||||
Portable atomic types including support for 128-bit atomics, atomic float, etc.
|
||||
"""
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
# NB: sync with:
|
||||
# - env.TEST_FEATURES in .github/workflows/ci.yml.
|
||||
# - test_features list in tools/build.sh and tools/test.sh.
|
||||
features = ["float", "std", "serde", "critical-section"]
|
||||
rustdoc-args = ["--extend-css", "src/rustdoc.css", "--cfg", "portable_atomic_unstable_f16", "--cfg", "portable_atomic_unstable_f128"]
|
||||
targets = ["x86_64-unknown-linux-gnu"]
|
||||
|
||||
[package.metadata.cargo_check_external_types]
|
||||
# The following are external types that are allowed to be exposed in our public API.
|
||||
allowed_external_types = [
|
||||
"serde_core::*",
|
||||
]
|
||||
|
||||
[lib]
|
||||
doc-scrape-examples = false
|
||||
|
||||
# Please read the documentation before using optional features:
|
||||
# https://github.com/taiki-e/portable-atomic#optional-features
|
||||
[features]
|
||||
default = ["fallback"]
|
||||
fallback = []
|
||||
float = []
|
||||
std = []
|
||||
require-cas = []
|
||||
unsafe-assume-single-core = []
|
||||
unsafe-assume-privileged = []
|
||||
s-mode = []
|
||||
force-amo = []
|
||||
disable-fiq = []
|
||||
|
||||
# Note: serde and critical-section are public dependencies.
|
||||
[dependencies]
|
||||
# Please read the documentation before using optional features:
|
||||
# https://github.com/taiki-e/portable-atomic#optional-features
|
||||
serde = { version = "1.0.60", optional = true, default-features = false }
|
||||
critical-section = { version = "1", optional = true }
|
||||
|
||||
# [target.'cfg(portable_atomic_test_no_std_static_assert_ffi)'.dependencies] #build:static_assert_ffi
|
||||
# test-helper = { features = ["sys"], git = "https://github.com/taiki-e/test-helper.git", rev = "24b3ad70c19ea8de9cd4c87f2e549b930f1e3305" } #build:static_assert_ffi
|
||||
# [target.'cfg(all(portable_atomic_test_no_std_static_assert_ffi, target_os = "aix"))'.dependencies] #build:static_assert_ffi
|
||||
# libc = "=0.2.163" #build:static_assert_ffi
|
||||
|
||||
[dev-dependencies]
|
||||
build-context = "0.1"
|
||||
crossbeam-utils = "0.8"
|
||||
fastrand = "2"
|
||||
paste = "1"
|
||||
quickcheck = { version = "1", default-features = false, git = "https://github.com/taiki-e/quickcheck.git", rev = "83b1d5955557fc9dacc52d4a78098d8e6d7eda3b" } # https://github.com/BurntSushi/quickcheck/pull/304 + https://github.com/BurntSushi/quickcheck/pull/282 + https://github.com/BurntSushi/quickcheck/pull/296 + f16/f128 support + lower MSRV
|
||||
rustversion = "1"
|
||||
serde_test = { git = "https://github.com/taiki-e/serde_test.git", rev = "df513c55a97f0a72fa67bad5396ed09123e5676c" } # support {i,u}128
|
||||
sptr = "0.3"
|
||||
static_assertions = "1"
|
||||
test-helper = { features = ["sys", "critical-section-std", "valgrind"], git = "https://github.com/taiki-e/test-helper.git", rev = "24b3ad70c19ea8de9cd4c87f2e549b930f1e3305" }
|
||||
|
||||
[target.'cfg(unix)'.dev-dependencies]
|
||||
libc = "=0.2.163" # newer libc requires Rust 1.63
|
||||
|
||||
[target.'cfg(windows)'.dev-dependencies]
|
||||
windows-sys = { version = "0.61", features = [
|
||||
"Win32_Foundation",
|
||||
"Win32_System_Threading", # IsProcessorFeaturePresent
|
||||
] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[workspace]
|
||||
members = [
|
||||
"bench",
|
||||
"tests/api-test",
|
||||
]
|
||||
|
||||
# This table is shared by projects under github.com/taiki-e.
|
||||
# Expect for unexpected_cfgs.check-cfg, it is not intended for manual editing.
|
||||
[workspace.lints.rust]
|
||||
deprecated_safe = "warn"
|
||||
improper_ctypes = "warn"
|
||||
improper_ctypes_definitions = "warn"
|
||||
non_ascii_idents = "warn"
|
||||
rust_2018_idioms = "warn"
|
||||
single_use_lifetimes = "warn"
|
||||
unexpected_cfgs = { level = "warn", check-cfg = [
|
||||
'cfg(target_arch,values("xtensa"))', # 1.81+ https://github.com/rust-lang/rust/pull/125141
|
||||
'cfg(target_arch,values("amdgpu"))', # 1.86+ https://github.com/rust-lang/rust/pull/134740
|
||||
'cfg(target_arch,values("loongarch32"))', # 1.89+ https://github.com/rust-lang/rust/pull/142053
|
||||
'cfg(target_os,values("trusty"))', # 1.82+ https://github.com/rust-lang/rust/pull/129490
|
||||
'cfg(target_os,values("psx"))', # 1.84+ https://github.com/rust-lang/rust/pull/131168
|
||||
'cfg(target_os,values("zos"))', # https://github.com/rust-lang/compiler-team/issues/992
|
||||
'cfg(target_env,values("psx"))', # pre-1.84 https://github.com/rust-lang/rust/pull/131168
|
||||
'cfg(target_feature,values("lse2","lse128","rcpc3"))', # 1.82+ https://github.com/rust-lang/rust/pull/128192
|
||||
'cfg(target_feature,values("quadword-atomics"))', # 1.83+ https://github.com/rust-lang/rust/pull/130873
|
||||
'cfg(target_feature,values("zaamo","zabha"))', # 1.83+ https://github.com/rust-lang/rust/pull/130877
|
||||
'cfg(target_feature,values("zacas"))', # 1.87+ https://github.com/rust-lang/rust/pull/137417
|
||||
'cfg(target_feature,values("ztso"))', # 1.88+ https://github.com/rust-lang/rust/pull/139440
|
||||
'cfg(target_feature,values("miscellaneous-extensions-3"))', # 1.89+ https://github.com/rust-lang/rust/pull/141250
|
||||
'cfg(target_pointer_width,values("128"))',
|
||||
# Known custom cfgs, excluding those that may be set by build script.
|
||||
# Public APIs, considered unstable unless documented as stable in readme.
|
||||
'cfg(portable_atomic_no_outline_atomics,portable_atomic_outline_atomics,portable_atomic_unstable_f16,portable_atomic_unstable_f128)',
|
||||
# Not public API.
|
||||
'cfg(portable_atomic_test_detect_false,portable_atomic_test_no_std_static_assert_ffi,qemu,valgrind,valgrind_cross)',
|
||||
] }
|
||||
unnameable_types = "warn"
|
||||
unreachable_pub = "warn"
|
||||
# unsafe_op_in_unsafe_fn = "warn" # Set at crate-level instead since https://github.com/rust-lang/rust/pull/100081 merged in Rust 1.65 is not available on MSRV
|
||||
[workspace.lints.clippy]
|
||||
all = "warn" # Downgrade deny-by-default lints
|
||||
pedantic = "warn"
|
||||
as_ptr_cast_mut = "warn"
|
||||
as_underscore = "warn"
|
||||
default_union_representation = "warn"
|
||||
inline_asm_x86_att_syntax = "warn"
|
||||
trailing_empty_array = "warn"
|
||||
transmute_undefined_repr = "warn"
|
||||
undocumented_unsafe_blocks = "warn"
|
||||
unused_trait_names = "warn"
|
||||
# Suppress buggy or noisy clippy lints
|
||||
assert_is_empty = { level = "allow", priority = 1 } # completely useless in assert_ne case
|
||||
bool_assert_comparison = { level = "allow", priority = 1 }
|
||||
borrow_as_ptr = { level = "allow", priority = 1 } # https://github.com/rust-lang/rust-clippy/issues/8286
|
||||
cast_lossless = { level = "allow", priority = 1 } # suggested code has poor codegen with -C opt-level=0 https://godbolt.org/z/GzTxzbd9q
|
||||
collapsible_match = { level = "allow", priority = 1 }
|
||||
declare_interior_mutable_const = { level = "allow", priority = 1 } # https://github.com/rust-lang/rust-clippy/issues/7665
|
||||
doc_markdown = { level = "allow", priority = 1 }
|
||||
float_cmp = { level = "allow", priority = 1 } # https://github.com/rust-lang/rust-clippy/issues/7725
|
||||
incompatible_msrv = { level = "allow", priority = 1 } # buggy: doesn't consider cfg, https://github.com/rust-lang/rust-clippy/issues/12280, https://github.com/rust-lang/rust-clippy/issues/12257#issuecomment-2093667187
|
||||
manual_assert = { level = "allow", priority = 1 }
|
||||
manual_range_contains = { level = "allow", priority = 1 } # https://github.com/rust-lang/rust-clippy/issues/6455#issuecomment-1225966395
|
||||
missing_errors_doc = { level = "allow", priority = 1 }
|
||||
module_name_repetitions = { level = "allow", priority = 1 } # buggy: https://github.com/rust-lang/rust-clippy/issues?q=is%3Aissue+is%3Aopen+module_name_repetitions
|
||||
naive_bytecount = { level = "allow", priority = 1 }
|
||||
nonminimal_bool = { level = "allow", priority = 1 } # buggy: https://github.com/rust-lang/rust-clippy/issues?q=is%3Aissue+is%3Aopen+nonminimal_bool
|
||||
range_plus_one = { level = "allow", priority = 1 } # buggy: https://github.com/rust-lang/rust-clippy/issues?q=is%3Aissue+is%3Aopen+range_plus_one
|
||||
similar_names = { level = "allow", priority = 1 }
|
||||
single_match = { level = "allow", priority = 1 }
|
||||
single_match_else = { level = "allow", priority = 1 }
|
||||
struct_excessive_bools = { level = "allow", priority = 1 }
|
||||
struct_field_names = { level = "allow", priority = 1 }
|
||||
too_many_arguments = { level = "allow", priority = 1 }
|
||||
too_many_lines = { level = "allow", priority = 1 }
|
||||
type_complexity = { level = "allow", priority = 1 }
|
||||
|
||||
[profile.release]
|
||||
debug = true
|
||||
+177
@@ -0,0 +1,177 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
Vendored
+23
@@ -0,0 +1,23 @@
|
||||
Permission is hereby granted, free of charge, to any
|
||||
person obtaining a copy of this software and associated
|
||||
documentation files (the "Software"), to deal in the
|
||||
Software without restriction, including without
|
||||
limitation the rights to use, copy, modify, merge,
|
||||
publish, distribute, sublicense, and/or sell copies of
|
||||
the Software, and to permit persons to whom the Software
|
||||
is furnished to do so, subject to the following
|
||||
conditions:
|
||||
|
||||
The above copyright notice and this permission notice
|
||||
shall be included in all copies or substantial portions
|
||||
of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
|
||||
ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
|
||||
TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
|
||||
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
|
||||
SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
|
||||
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
Vendored
+250
@@ -0,0 +1,250 @@
|
||||
# portable-atomic
|
||||
|
||||
[](https://crates.io/crates/portable-atomic)
|
||||
[](https://docs.rs/portable-atomic)
|
||||
[](#license)
|
||||
[](https://www.rust-lang.org)
|
||||
[](https://github.com/taiki-e/portable-atomic/actions)
|
||||
|
||||
<!-- tidy:sync-markdown-to-rustdoc:start:src/lib.rs -->
|
||||
|
||||
Portable atomic types including support for 128-bit atomics, atomic float, etc.
|
||||
|
||||
- Provide all atomic integer types (`Atomic{I,U}{8,16,32,64}`) for all targets that can use atomic CAS. (i.e., all targets that can use `std`, and most no-std targets)
|
||||
- Provide `AtomicI128` and `AtomicU128`.
|
||||
- Provide `AtomicF32` and `AtomicF64`. ([optional, requires the `float` feature](#optional-features-float))
|
||||
- Provide `AtomicF16` and `AtomicF128` for [unstable `f16` and `f128`](https://github.com/rust-lang/rust/issues/116909). ([optional, requires the `float` feature and unstable cfgs](#optional-features-float))
|
||||
- Provide atomic load/store for targets where atomic is not available at all in the standard library. (RISC-V without A-extension, MSP430, AVR)
|
||||
- Provide atomic CAS for targets where atomic CAS is not available in the standard library. (thumbv6m, pre-v6 Arm, RISC-V without A-extension, MSP430, AVR, Xtensa, etc.) (always enabled for MSP430 and AVR, [optional](#optional-features-critical-section) otherwise)
|
||||
- Make features that require newer compilers, such as [`fetch_{max,min}`](https://doc.rust-lang.org/std/sync/atomic/struct.AtomicUsize.html#method.fetch_max), [`fetch_update`](https://doc.rust-lang.org/std/sync/atomic/struct.AtomicUsize.html#method.fetch_update), [`as_ptr`](https://doc.rust-lang.org/std/sync/atomic/struct.AtomicUsize.html#method.as_ptr), [`from_ptr`](https://doc.rust-lang.org/std/sync/atomic/struct.AtomicUsize.html#method.from_ptr), [`AtomicBool::fetch_not`](https://doc.rust-lang.org/std/sync/atomic/struct.AtomicBool.html#method.fetch_not), [`AtomicPtr::fetch_*`](https://doc.rust-lang.org/std/sync/atomic/struct.AtomicPtr.html#method.fetch_and), and [stronger CAS failure ordering](https://github.com/rust-lang/rust/pull/98383) available on Rust 1.34+.
|
||||
- Provide workaround for bugs in the standard library's atomic-related APIs, such as [rust-lang/compiler-builtins#1234], [rust-lang/rust#100650], `fence`/`compiler_fence` on MSP430 that cause LLVM error, etc.
|
||||
|
||||
<!-- TODO:
|
||||
- mention Atomic{I,U}*::fetch_neg, Atomic{I*,U*,Ptr}::bit_*, etc.
|
||||
- mention optimizations not available in the standard library's equivalents
|
||||
-->
|
||||
|
||||
portable-atomic version of `std::sync::Arc` and `std::task::Wake` is provided by the [portable-atomic-util] crate.
|
||||
|
||||
## Usage
|
||||
|
||||
Add this to your `Cargo.toml`:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
portable-atomic = "1"
|
||||
```
|
||||
|
||||
The default features are mainly for users who use atomics larger than the pointer width.
|
||||
If you don't need them, disabling the default features may reduce code size and compile time slightly.
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
portable-atomic = { version = "1", default-features = false }
|
||||
```
|
||||
|
||||
If your crate supports no-std environment and requires atomic CAS, enabling the `require-cas` feature will allow the portable-atomic to display a [helpful error message](https://github.com/taiki-e/portable-atomic/pull/100) to users on targets requiring additional action on the user side to provide atomic CAS.
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
portable-atomic = { version = "1.3", default-features = false, features = ["require-cas"] }
|
||||
```
|
||||
|
||||
(Since 1.8, portable-atomic can display a [helpful error message](https://github.com/taiki-e/portable-atomic/pull/181) even without the `require-cas` feature when the rustc version is 1.78+. However, the `require-cas` feature also allows rejecting builds at an earlier stage, we recommend enabling it unless enabling it causes [problems](https://github.com/matklad/once_cell/pull/267).)
|
||||
|
||||
## 128-bit atomics support
|
||||
|
||||
Native 128-bit atomic operations are available on x86_64 (Rust 1.59+), AArch64 (Rust 1.59+), riscv64 (Rust 1.59+), Arm64EC (Rust 1.84+), s390x (Rust 1.84+), and powerpc64 (Rust 1.95+), otherwise the fallback implementation is used.
|
||||
|
||||
On x86_64, even if `cmpxchg16b` is not available at compile-time (Note: `cmpxchg16b` target feature is enabled by default only on Apple, Windows (except Windows 7), and Fuchsia targets), run-time detection checks whether `cmpxchg16b` is available. If `cmpxchg16b` is not available at either compile-time or run-time detection, the fallback implementation is used. See also [`portable_atomic_no_outline_atomics`](#optional-cfg-no-outline-atomics) cfg.
|
||||
|
||||
They are usually implemented using inline assembly, and when using Miri or ThreadSanitizer that do not support inline assembly, core intrinsics are used instead of inline assembly if possible.
|
||||
|
||||
See the [`atomic128` module's readme](https://github.com/taiki-e/portable-atomic/blob/HEAD/src/imp/atomic128/README.md) for details.
|
||||
|
||||
## <a name="optional-features"></a><a name="optional-cfg"></a>Optional features/cfgs
|
||||
|
||||
portable-atomic provides features and cfgs to allow enabling specific APIs and customizing its behavior.
|
||||
|
||||
Some options have both a feature and a cfg. When both exist, it indicates that the feature does not follow Cargo's recommendation that [features should be additive](https://doc.rust-lang.org/nightly/cargo/reference/features.html#feature-unification). Therefore, the maintainer's recommendation is to use cfg instead of feature. However, in the embedded ecosystem, it is very common to use features in such places, so these options provide both so you can choose based on your preference.
|
||||
|
||||
<details>
|
||||
<summary>How to enable cfg (click to show)</summary>
|
||||
|
||||
One of the ways to enable cfg is to set [rustflags in the cargo config](https://doc.rust-lang.org/cargo/reference/config.html#targettriplerustflags):
|
||||
|
||||
```toml
|
||||
# .cargo/config.toml
|
||||
[target.<target>]
|
||||
rustflags = ["--cfg", "portable_atomic_unsafe_assume_single_core"]
|
||||
```
|
||||
|
||||
Or set environment variable:
|
||||
|
||||
```sh
|
||||
RUSTFLAGS="--cfg portable_atomic_unsafe_assume_single_core" cargo ...
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
- <a name="optional-features-fallback"></a>**`fallback` feature** *(enabled by default)*<br>
|
||||
Enable fallback implementations.
|
||||
|
||||
This enables atomic types with larger than the width supported by atomic instructions available on the current target. If the current target [supports 128-bit atomics](#128-bit-atomics-support), this is no-op.
|
||||
|
||||
This uses fallback implementation that using global locks by default. The following features/cfgs change this behavior:
|
||||
- [`unsafe-assume-single-core` feature / `portable_atomic_unsafe_assume_single_core` cfg](#optional-features-unsafe-assume-single-core): Use fallback implementations that disabling interrupts instead of using global locks.
|
||||
- If your target is single-core and calling interrupt disable instructions is safe, this is a safer and more efficient option.
|
||||
- [`unsafe-assume-privileged` feature / `portable_atomic_unsafe_assume_privileged` cfg](#optional-features-unsafe-assume-privileged): Use fallback implementations that using global locks with disabling interrupts.
|
||||
- If your target is multi-core and calling interrupt disable instructions is safe, this is a safer option.
|
||||
|
||||
- <a name="optional-features-float"></a>**`float` feature**<br>
|
||||
Provide `AtomicF{32,64}`.
|
||||
|
||||
If you want atomic types for unstable float types ([`f16` and `f128`](https://github.com/rust-lang/rust/issues/116909)), enable unstable cfg (`portable_atomic_unstable_f16` cfg for `AtomicF16`, `portable_atomic_unstable_f128` cfg for `AtomicF128`, [there is no possibility that both feature and cfg will be provided for unstable options.](https://github.com/taiki-e/portable-atomic/pull/200#issuecomment-2682252991)).
|
||||
|
||||
> [!NOTE]
|
||||
> - Atomic float's `fetch_{add,sub,min,max}` are usually implemented using CAS loops, which can be slower than equivalent operations of atomic integers. As an exception, AArch64 with FEAT_LSFE and GPU targets have atomic float instructions and we use them on AArch64 when `lsfe` target feature is available at compile-time. We [plan to use atomic float instructions for GPU targets as well in the future.](https://github.com/taiki-e/portable-atomic/issues/34)
|
||||
> - Unstable cfgs are outside of the normal semver guarantees and minor or patch versions of portable-atomic may make breaking changes to them at any time.
|
||||
|
||||
- <a name="optional-features-std"></a>**`std` feature**<br>
|
||||
Use `std`.
|
||||
|
||||
- <a name="optional-features-require-cas"></a>**`require-cas` feature**<br>
|
||||
Emit compile error if atomic CAS is not available. See [Usage](#usage) section for usage of this feature.
|
||||
|
||||
- <a name="optional-features-serde"></a>**`serde` feature**<br>
|
||||
Implement `serde::{Serialize,Deserialize}` for atomic types.
|
||||
|
||||
Note:
|
||||
- The MSRV when this feature is enabled depends on the MSRV of [serde].
|
||||
|
||||
- <a name="optional-features-critical-section"></a>**`critical-section` feature**<br>
|
||||
Use [critical-section] to provide atomic CAS for targets where atomic CAS is not available in the standard library.
|
||||
|
||||
`critical-section` support is useful to get atomic CAS when the [`unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)](#optional-features-unsafe-assume-single-core) can't be used,
|
||||
such as multi-core targets, unprivileged code running under some RTOS, or environments where disabling interrupts
|
||||
needs extra care due to e.g. real-time requirements.
|
||||
|
||||
> [!NOTE]
|
||||
> - When enabling this feature, you should provide a suitable critical section implementation for the current target, see the [critical-section] documentation for details on how to do so.
|
||||
> - With this feature, critical sections are taken for all atomic operations, while with `unsafe-assume-single-core` feature [some operations](https://github.com/taiki-e/portable-atomic/blob/HEAD/src/imp/interrupt/README.md#no-disable-interrupts) don't require disabling interrupts. Therefore, for better performance, if all the `critical-section` implementation for your target does is disable interrupts, prefer using `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg) instead.
|
||||
> - It is usually **discouraged** to always enable this feature in libraries that depend on `portable-atomic`.
|
||||
>
|
||||
> Enabling this feature will prevent the end user from having the chance to take advantage of other (potentially) efficient implementations (implementations provided by `unsafe-assume-single-core` feature mentioned above, implementation proposed in [#60], etc.). Also, targets that are currently unsupported may be supported in the future.
|
||||
>
|
||||
> The recommended approach for libraries is to leave it up to the end user whether or not to enable this feature. (However, it may make sense to enable this feature by default for libraries specific to a platform where other implementations are known not to work.)
|
||||
>
|
||||
> See also [this comment](https://github.com/matklad/once_cell/issues/264#issuecomment-2352654806).
|
||||
>
|
||||
> As an example, the end-user's `Cargo.toml` that uses a crate that provides a critical-section implementation and a crate that depends on portable-atomic as an option would be expected to look like this:
|
||||
>
|
||||
> ```toml
|
||||
> [dependencies]
|
||||
> portable-atomic = { version = "1", default-features = false, features = ["critical-section"] }
|
||||
> crate-provides-critical-section-impl = "..."
|
||||
> crate-uses-portable-atomic-as-feature = { version = "...", features = ["portable-atomic"] }
|
||||
> ```
|
||||
>
|
||||
> - Enabling both this feature and `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg) will result in a compile error.
|
||||
> - Enabling both this feature and `unsafe-assume-privileged` feature (or `portable_atomic_unsafe_assume_privileged` cfg) will result in a compile error.
|
||||
> - The MSRV when this feature is enabled depends on the MSRV of [critical-section].
|
||||
|
||||
- <a name="optional-features-unsafe-assume-single-core"></a><a name="optional-cfg-unsafe-assume-single-core"></a>**`unsafe-assume-single-core` feature / `portable_atomic_unsafe_assume_single_core` cfg**<br>
|
||||
Assume that the target is single-core and privileged instructions required to disable interrupts are available.
|
||||
|
||||
- When this feature/cfg is enabled, this crate provides atomic CAS for targets where atomic CAS is not available in the standard library by disabling interrupts.
|
||||
- When both this feature/cfg and enabled-by-default `fallback` feature is enabled, this crate provides atomic types with larger than the width supported by native instructions by disabling interrupts.
|
||||
|
||||
> [!WARNING]
|
||||
> This feature/cfg is `unsafe`, and note the following safety requirements:
|
||||
> - Enabling this feature/cfg for multi-core systems is always **unsound**.
|
||||
>
|
||||
> - This uses privileged instructions to disable interrupts, so it usually doesn't work on unprivileged mode.
|
||||
>
|
||||
> Enabling this feature/cfg in an environment where privileged instructions are not available, or if the instructions used are not sufficient to disable interrupts in the system, it is also usually considered **unsound**, although the details are system-dependent.
|
||||
>
|
||||
> The following are known cases:
|
||||
> - On Arm (except for M-Profile architectures), this disables only IRQs by default. For many systems (e.g., GBA) this is enough. If the system need to disable both IRQs and FIQs, you need to enable the `disable-fiq` feature (or `portable_atomic_disable_fiq` cfg) together.
|
||||
> - On RISC-V, this generates code for machine-mode (M-mode) by default. If you enable the `s-mode` feature (or `portable_atomic_s_mode` cfg) together, this generates code for supervisor-mode (S-mode). In particular, `qemu-system-riscv*` uses [OpenSBI](https://github.com/riscv-software-src/opensbi) as the default firmware.
|
||||
|
||||
Consider using the [`unsafe-assume-privileged` feature (or `portable_atomic_unsafe_assume_privileged` cfg)](#optional-features-unsafe-assume-privileged) for multi-core systems with atomic CAS.
|
||||
|
||||
Consider using the [`critical-section` feature](#optional-features-critical-section) for systems that cannot use this feature/cfg.
|
||||
|
||||
See also the [`interrupt` module's readme](https://github.com/taiki-e/portable-atomic/blob/HEAD/src/imp/interrupt/README.md).
|
||||
|
||||
> [!NOTE]
|
||||
> - It is **very strongly discouraged** to enable this feature/cfg in libraries that depend on `portable-atomic`.
|
||||
>
|
||||
> The recommended approach for libraries is to leave it up to the end user whether or not to enable this feature/cfg. (However, it may make sense to enable this feature/cfg by default for libraries specific to a platform where it is guaranteed to always be sound, for example in a hardware abstraction layer targeting a single-core chip.)
|
||||
> - Enabling this feature/cfg for unsupported architectures will result in a compile error.
|
||||
> - Arm, RISC-V, and Xtensa are currently supported. (Since all MSP430 and AVR are single-core, we always provide atomic CAS for them without this feature/cfg.)
|
||||
> - Feel free to [submit an issue](https://github.com/taiki-e/portable-atomic/issues/new) if your target is not supported yet.
|
||||
> - Enabling this feature/cfg for targets where privileged instructions are obviously unavailable (e.g., Linux) will result in a compile error.
|
||||
> - Feel free to [submit an issue](https://github.com/taiki-e/portable-atomic/issues/new) if your target supports privileged instructions but the build rejected.
|
||||
> - Enabling both this feature/cfg and `critical-section` feature will result in a compile error.
|
||||
> - When both this feature/cfg and `unsafe-assume-privileged` feature (or `portable_atomic_unsafe_assume_privileged` cfg) are enabled, this feature/cfg is preferred.
|
||||
|
||||
- <a name="optional-features-unsafe-assume-privileged"></a><a name="optional-cfg-unsafe-assume-privileged"></a>**`unsafe-assume-privileged` feature / `portable_atomic_unsafe_assume_privileged` cfg**<br>
|
||||
Similar to `unsafe-assume-single-core` feature / `portable_atomic_unsafe_assume_single_core` cfg, but only assumes about availability of privileged instructions required to disable interrupts.
|
||||
|
||||
- When both this feature/cfg and enabled-by-default `fallback` feature is enabled, this crate provides atomic types with larger than the width supported by native instructions by using global locks with disabling interrupts.
|
||||
|
||||
> [!WARNING]
|
||||
> This feature/cfg is `unsafe`, and except for being sound in multi-core systems, this has the same safety requirements as [`unsafe-assume-single-core` feature / `portable_atomic_unsafe_assume_single_core` cfg](#optional-features-unsafe-assume-single-core).
|
||||
|
||||
> [!NOTE]
|
||||
> - It is **very strongly discouraged** to enable this feature/cfg in libraries that depend on `portable-atomic`.
|
||||
>
|
||||
> The recommended approach for libraries is to leave it up to the end user whether or not to enable this feature/cfg. (However, it may make sense to enable this feature/cfg by default for libraries specific to a platform where it is guaranteed to always be sound, for example in a hardware abstraction layer.)
|
||||
> - Enabling this feature/cfg for unsupported targets will result in a compile error.
|
||||
> - This requires atomic CAS (`cfg(target_has_atomic = "ptr")` or `cfg_no_atomic_cas!`).
|
||||
> - Arm, RISC-V, and Xtensa are currently supported.
|
||||
> - Feel free to [submit an issue](https://github.com/taiki-e/portable-atomic/issues/new) if your target is not supported yet.
|
||||
> - Enabling this feature/cfg for targets where privileged instructions are obviously unavailable (e.g., Linux) will result in a compile error.
|
||||
> - Feel free to [submit an issue](https://github.com/taiki-e/portable-atomic/issues/new) if your target supports privileged instructions but the build rejected.
|
||||
> - Enabling both this feature/cfg and `critical-section` feature will result in a compile error.
|
||||
> - When both this feature/cfg and `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg) are enabled, `unsafe-assume-single-core` is preferred.
|
||||
|
||||
- <a name="optional-cfg-no-outline-atomics"></a>**`portable_atomic_no_outline_atomics` cfg**<br>
|
||||
Disable dynamic dispatching by run-time CPU feature detection.
|
||||
|
||||
Dynamic dispatching by run-time CPU feature detection allows maintaining support for older CPUs while using features that are not supported on older CPUs, such as CMPXCHG16B (x86_64) and FEAT_LSE/FEAT_LSE2 (AArch64).
|
||||
|
||||
See also the [`atomic128` module's readme](https://github.com/taiki-e/portable-atomic/blob/HEAD/src/imp/atomic128/README.md).
|
||||
|
||||
> [!NOTE]
|
||||
> - If the required target features are enabled at compile-time, dynamic dispatching is automatically disabled and the atomic operations are inlined.
|
||||
> - This is compatible with no-std (as with all features except `std`).
|
||||
> - On some targets, run-time detection is disabled by default mainly for compatibility with incomplete build environments or support for it is experimental, and can be enabled by `portable_atomic_outline_atomics` cfg. (When both cfg are enabled, `*_no_*` cfg is preferred.)
|
||||
> - Some AArch64 targets enable LLVM's `outline-atomics` target feature by default, so if you set this cfg, you may want to disable that as well. (However, portable-atomic's outline-atomics does not depend on the compiler-rt symbols, so even if you need to disable LLVM's outline-atomics, you may not need to disable portable-atomic's outline-atomics.)
|
||||
> - Dynamic detection is currently only supported in x86_64, AArch64, Arm, RISC-V, Arm64EC, and powerpc64. Enabling this cfg for unsupported architectures will be ignored.
|
||||
|
||||
## Related Projects
|
||||
|
||||
- [portable-atomic-util]: Synchronization primitives built with portable-atomic. This provides portable-atomic version of `std::sync::Arc` and `std::task::Wake`.
|
||||
- [atomic-maybe-uninit]: Atomic operations on potentially uninitialized integers.
|
||||
- [atomic-memcpy]: Byte-wise atomic memcpy.
|
||||
- [asmtest]: A library for tracking generated assemblies.
|
||||
|
||||
[#60]: https://github.com/taiki-e/portable-atomic/issues/60
|
||||
[asmtest]: https://github.com/taiki-e/asmtest
|
||||
[atomic-maybe-uninit]: https://github.com/taiki-e/atomic-maybe-uninit
|
||||
[atomic-memcpy]: https://github.com/taiki-e/atomic-memcpy
|
||||
[critical-section]: https://github.com/rust-embedded/critical-section
|
||||
[portable-atomic-util]: https://github.com/taiki-e/portable-atomic-util
|
||||
[rust-lang/compiler-builtins#1234]: https://github.com/rust-lang/compiler-builtins/pull/1234
|
||||
[rust-lang/rust#100650]: https://github.com/rust-lang/rust/issues/100650
|
||||
[serde]: https://github.com/serde-rs/serde
|
||||
|
||||
<!-- tidy:sync-markdown-to-rustdoc:end -->
|
||||
|
||||
## License
|
||||
|
||||
Licensed under either of [Apache License, Version 2.0](LICENSE-APACHE) or
|
||||
[MIT license](LICENSE-MIT) at your option.
|
||||
|
||||
Unless you explicitly state otherwise, any contribution intentionally submitted
|
||||
for inclusion in the work by you, as defined in the Apache-2.0 license, shall
|
||||
be dual licensed as above, without any additional terms or conditions.
|
||||
Vendored
+629
@@ -0,0 +1,629 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
// The rustc-cfg emitted by the build script are *not* public API.
|
||||
|
||||
#![allow(clippy::match_same_arms)] // https://github.com/rust-lang/rust-clippy/issues/12044
|
||||
|
||||
#[path = "src/gen/build.rs"]
|
||||
mod generated;
|
||||
#[path = "version.rs"]
|
||||
mod version;
|
||||
|
||||
use std::{env, str};
|
||||
|
||||
use self::version::{Version, rustc_version};
|
||||
|
||||
fn main() {
|
||||
println!("cargo:rerun-if-changed=build.rs");
|
||||
println!("cargo:rerun-if-changed=src/gen/build.rs");
|
||||
println!("cargo:rerun-if-changed=version.rs");
|
||||
|
||||
#[cfg(feature = "unsafe-assume-single-core")]
|
||||
println!("cargo:rustc-cfg=portable_atomic_unsafe_assume_single_core");
|
||||
#[cfg(feature = "unsafe-assume-privileged")]
|
||||
println!("cargo:rustc-cfg=portable_atomic_unsafe_assume_privileged");
|
||||
#[cfg(feature = "s-mode")]
|
||||
println!("cargo:rustc-cfg=portable_atomic_s_mode");
|
||||
#[cfg(feature = "force-amo")]
|
||||
println!("cargo:rustc-cfg=portable_atomic_force_amo");
|
||||
#[cfg(feature = "disable-fiq")]
|
||||
println!("cargo:rustc-cfg=portable_atomic_disable_fiq");
|
||||
|
||||
let target = &*env::var("TARGET").expect("TARGET not set");
|
||||
let target_arch = &*env::var("CARGO_CFG_TARGET_ARCH").expect("CARGO_CFG_TARGET_ARCH not set");
|
||||
let target_os = &*env::var("CARGO_CFG_TARGET_OS").expect("CARGO_CFG_TARGET_OS not set");
|
||||
|
||||
let version = match rustc_version() {
|
||||
Some(version) => version,
|
||||
None => {
|
||||
if env::var_os("PORTABLE_ATOMIC_DENY_WARNINGS").is_some() {
|
||||
panic!("unable to determine rustc version")
|
||||
}
|
||||
println!(
|
||||
"cargo:warning={}: unable to determine rustc version; assuming latest stable rustc (1.{})",
|
||||
env!("CARGO_PKG_NAME"),
|
||||
Version::LATEST.minor
|
||||
);
|
||||
Version::LATEST
|
||||
}
|
||||
};
|
||||
|
||||
if version.minor >= 80 {
|
||||
println!(
|
||||
r#"cargo:rustc-check-cfg=cfg(target_feature,values("lsfe","fast-serialization","load-store-on-cond","distinct-ops","rmw"))"#
|
||||
);
|
||||
|
||||
// Custom cfgs set by build script. Not public API.
|
||||
// grep -F 'cargo:rustc-cfg=' build.rs | grep -Ev '^ *//' | sed -E 's/^.*cargo:rustc-cfg=//; s/(=\\)?".*$//' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/'
|
||||
println!(
|
||||
"cargo:rustc-check-cfg=cfg(portable_atomic_atomic_intrinsics,portable_atomic_disable_fiq,portable_atomic_force_amo,portable_atomic_ll_sc_rmw,portable_atomic_no_asm,portable_atomic_no_asm_maybe_uninit,portable_atomic_no_asm_syscall,portable_atomic_no_atomic_64,portable_atomic_no_atomic_cas,portable_atomic_no_atomic_load_store,portable_atomic_no_atomic_min_max,portable_atomic_no_cfg_target_has_atomic,portable_atomic_no_cmpxchg16b_intrinsic,portable_atomic_no_cmpxchg16b_target_feature,portable_atomic_no_const_mut_refs,portable_atomic_no_const_raw_ptr_deref,portable_atomic_no_const_transmute,portable_atomic_no_core_unwind_safe,portable_atomic_no_diagnostic_namespace,portable_atomic_no_strict_provenance,portable_atomic_no_strict_provenance_atomic_ptr,portable_atomic_no_stronger_failure_ordering,portable_atomic_no_track_caller,portable_atomic_no_unsafe_op_in_unsafe_fn,portable_atomic_pre_llvm_16,portable_atomic_pre_llvm_18,portable_atomic_pre_llvm_20,portable_atomic_s_mode,portable_atomic_sanitize_thread,portable_atomic_target_feature,portable_atomic_unsafe_assume_privileged,portable_atomic_unsafe_assume_single_core,portable_atomic_unstable_asm,portable_atomic_unstable_asm_experimental_arch,portable_atomic_unstable_cfg_target_has_atomic,portable_atomic_unstable_isa_attribute)"
|
||||
);
|
||||
// TODO: handle multi-line target_feature_fallback
|
||||
// grep -F 'target_feature_fallback("' build.rs | grep -Ev '^ *//' | sed -E 's/^.*target_feature_fallback\(//; s/",.*$/"/' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/'
|
||||
println!(
|
||||
r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_feature,values("cmpxchg16b","distinct-ops","fast-serialization","load-store-on-cond","lse","lse128","lse2","lsfe","mclass","miscellaneous-extensions-3","quadword-atomics","rcpc3","rmw","v6","v7","zaamo","zabha","zacas"))"#
|
||||
);
|
||||
}
|
||||
|
||||
// https://github.com/rust-lang/rust/pull/123745 (includes https://github.com/rust-lang/cargo/pull/13560) merged in Rust 1.79 (nightly-2024-04-11).
|
||||
if !version.probe(79, 2024, 4, 10) {
|
||||
// HACK: If --target is specified, rustflags is not applied to the build
|
||||
// script itself, so the build script will not be recompiled when rustflags
|
||||
// is changed. That in itself is not a problem, but the old Cargo does
|
||||
// not rerun the build script as well, which can be problematic.
|
||||
// https://github.com/rust-lang/cargo/issues/13003
|
||||
// This problem has been fixed in 1.79 so only older versions need a workaround.
|
||||
println!("cargo:rerun-if-env-changed=CARGO_ENCODED_RUSTFLAGS");
|
||||
println!("cargo:rerun-if-env-changed=RUSTFLAGS");
|
||||
println!("cargo:rerun-if-env-changed=CARGO_BUILD_RUSTFLAGS");
|
||||
let mut target_upper = target.replace(|c: char| c == '-' || c == '.', "_");
|
||||
target_upper.make_ascii_uppercase();
|
||||
println!("cargo:rerun-if-env-changed=CARGO_TARGET_{}_RUSTFLAGS", target_upper);
|
||||
}
|
||||
|
||||
// Note that cfgs are `no_`*, not `has_*`. This allows treating as the latest
|
||||
// stable rustc is used when the build script doesn't run. This is useful
|
||||
// for non-cargo build systems that don't run the build script.
|
||||
|
||||
// atomic_min_max stabilized in Rust 1.45 (nightly-2020-05-30): https://github.com/rust-lang/rust/pull/72324
|
||||
if !version.probe(45, 2020, 5, 29) {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_atomic_min_max");
|
||||
}
|
||||
// track_caller stabilized in Rust 1.46 (nightly-2020-07-02): https://github.com/rust-lang/rust/pull/72445
|
||||
if !version.probe(46, 2020, 7, 1) {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_track_caller");
|
||||
}
|
||||
// unsafe_op_in_unsafe_fn stabilized in Rust 1.52 (nightly-2021-03-11): https://github.com/rust-lang/rust/pull/79208
|
||||
if !version.probe(52, 2021, 3, 10) {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_unsafe_op_in_unsafe_fn");
|
||||
}
|
||||
// const_transmute stabilized in Rust 1.56 (nightly-2021-07-29): https://github.com/rust-lang/rust/pull/85769
|
||||
if !version.probe(56, 2021, 7, 28) {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_const_transmute");
|
||||
}
|
||||
// https://github.com/rust-lang/rust/pull/84662 merged in Rust 1.56 (nightly-2021-08-02).
|
||||
if !version.probe(56, 2021, 8, 1) {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_core_unwind_safe");
|
||||
}
|
||||
// const_raw_ptr_deref stabilized in Rust 1.58 (nightly-2021-11-15): https://github.com/rust-lang/rust/pull/89551
|
||||
if !version.probe(58, 2021, 11, 14) {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_const_raw_ptr_deref");
|
||||
}
|
||||
// https://github.com/rust-lang/rust/pull/98383 merged in Rust 1.64 (nightly-2022-07-19).
|
||||
if !version.probe(64, 2022, 7, 18) {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_stronger_failure_ordering");
|
||||
}
|
||||
// https://github.com/rust-lang/rust/pull/114790 merged in nightly-2023-08-24
|
||||
if !version.probe(74, 2023, 8, 23) {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_asm_maybe_uninit");
|
||||
}
|
||||
// #[diagnostic] stabilized in Rust 1.78 (nightly-2024-03-09): https://github.com/rust-lang/rust/pull/119888
|
||||
if !version.probe(78, 2024, 3, 8) {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_diagnostic_namespace");
|
||||
}
|
||||
// const_mut_refs/const_refs_to_cell stabilized in Rust 1.83 (nightly-2024-09-16): https://github.com/rust-lang/rust/pull/129195
|
||||
if !version.probe(83, 2024, 9, 15) {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_const_mut_refs");
|
||||
}
|
||||
// strict_provenance/exposed_provenance APIs stabilized in Rust 1.84 (nightly-2024-10-22): https://github.com/rust-lang/rust/pull/130350
|
||||
if !version.probe(84, 2024, 10, 21) {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_strict_provenance");
|
||||
}
|
||||
// strict_provenance_atomic_ptr stabilized in Rust 1.91 (nightly-2024-10-22): https://github.com/rust-lang/rust/pull/145467
|
||||
if !version.probe(91, 2025, 8, 30) {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_strict_provenance_atomic_ptr");
|
||||
}
|
||||
|
||||
// For Miri and ThreadSanitizer. (aarch64, arm64ec, s390x, powerpc64)
|
||||
// https://github.com/rust-lang/rust/pull/97423 merged in Rust 1.64 (nightly-2022-06-30).
|
||||
// https://github.com/rust-lang/rust/pull/141507 merged in Rust 1.89 (nightly-2025-05-31).
|
||||
if version.nightly
|
||||
&& version.probe(64, 2022, 6, 29)
|
||||
&& !version.probe(89, 2025, 5, 30)
|
||||
&& (target_arch != "powerpc64" || version.llvm >= 15)
|
||||
{
|
||||
println!("cargo:rustc-cfg=portable_atomic_atomic_intrinsics");
|
||||
}
|
||||
|
||||
// asm! on AArch64, Arm, RISC-V, x86, and x86_64 stabilized in Rust 1.59 (nightly-2021-12-16): https://github.com/rust-lang/rust/pull/91728
|
||||
let no_asm = !version.probe(59, 2021, 12, 15);
|
||||
if no_asm {
|
||||
if version.nightly
|
||||
&& version.probe(46, 2020, 6, 20)
|
||||
&& ((target_arch != "x86" && target_arch != "x86_64") || version.llvm >= 10)
|
||||
&& is_allowed_feature("asm")
|
||||
{
|
||||
// This feature was added in Rust 1.45 (nightly-2020-05-20), but
|
||||
// concat! in asm! requires Rust 1.46 (nightly-2020-06-21).
|
||||
// x86 intel syntax requires LLVM 10 (since Rust 1.53, the minimum
|
||||
// external LLVM version is 10+: https://github.com/rust-lang/rust/pull/83387).
|
||||
// The part of this feature we use has not been changed since nightly-2020-06-21
|
||||
// until it was stabilized, so it can safely be enabled in nightly for that period.
|
||||
println!("cargo:rustc-cfg=portable_atomic_unstable_asm");
|
||||
if (target_arch == "riscv32" || target_arch == "riscv64")
|
||||
&& !version.probe(56, 2021, 8, 15)
|
||||
{
|
||||
// Clobbering vector status needed for asm syscall require Rust 1.56 (nightly-2021-08-16): https://github.com/rust-lang/rust/pull/87581
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_asm_syscall");
|
||||
}
|
||||
}
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_asm");
|
||||
} else {
|
||||
match target_arch {
|
||||
"arm64ec" | "s390x" => {
|
||||
// asm! on Arm64EC and s390x stabilized in Rust 1.84 (nightly-2024-11-11): https://github.com/rust-lang/rust/pull/131781, https://github.com/rust-lang/rust/pull/131258
|
||||
if !version.probe(84, 2024, 11, 10) {
|
||||
if version.nightly
|
||||
&& (target_arch != "s390x" || version.probe(71, 2023, 5, 8))
|
||||
&& is_allowed_feature("asm_experimental_arch")
|
||||
{
|
||||
// https://github.com/rust-lang/rust/pull/111331 merged in Rust 1.71 (nightly-2023-05-09).
|
||||
// The part of this feature we use has not been changed since nightly-2023-05-09
|
||||
// until it was stabilized, so it can safely be enabled in nightly for that period.
|
||||
println!("cargo:rustc-cfg=portable_atomic_unstable_asm_experimental_arch");
|
||||
} else {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_asm");
|
||||
}
|
||||
}
|
||||
}
|
||||
"powerpc64" => {
|
||||
// asm! on PowerPC stabilized in Rust 1.95 (nightly-2026-01-28): https://github.com/rust-lang/rust/pull/147996
|
||||
if !version.probe(95, 2026, 1, 27) {
|
||||
if version.nightly
|
||||
&& version.probe(60, 2022, 2, 12)
|
||||
&& is_allowed_feature("asm_experimental_arch")
|
||||
{
|
||||
// https://github.com/rust-lang/rust/pull/93868 merged in Rust 1.60 (nightly-2022-02-13).
|
||||
// The part of this feature we use has not been changed since nightly-2022-02-13
|
||||
// until it was stabilized, so it can safely be enabled in nightly for that period.
|
||||
println!("cargo:rustc-cfg=portable_atomic_unstable_asm_experimental_arch");
|
||||
} else {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_asm");
|
||||
}
|
||||
if !version.probe(92, 2025, 9, 22)
|
||||
|| env::var("CARGO_CFG_TARGET_ABI")
|
||||
.unwrap_or_default()
|
||||
.split(',')
|
||||
.any(|abi| abi == "spu")
|
||||
{
|
||||
// Clobbering ctr register needed for asm syscall require Rust 1.92 (nightly-2025-09-23): https://github.com/rust-lang/rust/pull/146831
|
||||
// Cell/B.E. Linux's SPU syscalls support a subset of powerpc64 syscalls with the exact semantics: https://github.com/torvalds/linux/blob/v7.1/arch/powerpc/platforms/cell/spu_callbacks.c#L17
|
||||
// but have a different syscall sequence: https://github.com/mirror/newlib-cygwin/blob/newlib-4.4.0/libgloss/spu/linux_syscalls.c#L37
|
||||
// This is currently used only for testing.
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_asm_syscall");
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
// feature(cfg_target_has_atomic) stabilized in Rust 1.60 (nightly-2022-02-11): https://github.com/rust-lang/rust/pull/93824
|
||||
if !version.probe(60, 2022, 2, 10) {
|
||||
if version.nightly
|
||||
&& version.probe(40, 2019, 10, 13)
|
||||
&& is_allowed_feature("cfg_target_has_atomic")
|
||||
{
|
||||
// The part of this feature we use has not been changed since nightly-2019-10-14
|
||||
// until it was stabilized, so it can safely be enabled in nightly for that period.
|
||||
println!("cargo:rustc-cfg=portable_atomic_unstable_cfg_target_has_atomic");
|
||||
} else {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_cfg_target_has_atomic");
|
||||
let target = &*convert_custom_linux_target(target);
|
||||
if generated::NO_ATOMIC_CAS.contains(&target) {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_atomic_cas");
|
||||
}
|
||||
if generated::NO_ATOMIC_64.contains(&target) {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_atomic_64");
|
||||
} else {
|
||||
// Otherwise, assuming `"max-atomic-width" == 64` or `"max-atomic-width" == 128`.
|
||||
}
|
||||
}
|
||||
}
|
||||
// We don't need to use convert_custom_linux_target here because all linux targets have atomics.
|
||||
if generated::NO_ATOMIC.contains(&target) {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_atomic_load_store");
|
||||
}
|
||||
|
||||
if version.llvm < 20 {
|
||||
println!("cargo:rustc-cfg=portable_atomic_pre_llvm_20");
|
||||
if version.llvm < 18 {
|
||||
println!("cargo:rustc-cfg=portable_atomic_pre_llvm_18");
|
||||
if version.llvm < 16 {
|
||||
println!("cargo:rustc-cfg=portable_atomic_pre_llvm_16");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if version.nightly {
|
||||
// `cfg(sanitize = "..")` is not stabilized.
|
||||
let sanitize = env::var("CARGO_CFG_SANITIZE").unwrap_or_default();
|
||||
if sanitize.contains("thread") {
|
||||
// Most kinds of sanitizers are not compatible with asm
|
||||
// (https://github.com/google/sanitizers/issues/192),
|
||||
// but it seems that ThreadSanitizer is the only one that can cause
|
||||
// false positives in our code.
|
||||
println!("cargo:rustc-cfg=portable_atomic_sanitize_thread");
|
||||
}
|
||||
}
|
||||
|
||||
match target_arch {
|
||||
"x86_64" => {
|
||||
// cmpxchg16b_target_feature stabilized in Rust 1.69 (nightly-2023-03-01): https://github.com/rust-lang/rust/pull/106774
|
||||
if !version.probe(69, 2023, 2, 28) {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_cmpxchg16b_target_feature");
|
||||
}
|
||||
// For Miri and ThreadSanitizer.
|
||||
// https://github.com/rust-lang/rust/pull/109359 (includes https://github.com/rust-lang/stdarch/pull/1358) merged in Rust 1.70 (nightly-2023-03-24).
|
||||
if version.nightly && !version.probe(70, 2023, 3, 23) {
|
||||
println!("cargo:rustc-cfg=portable_atomic_no_cmpxchg16b_intrinsic");
|
||||
}
|
||||
|
||||
// cmpxchg16b_target_feature stabilized in Rust 1.69.
|
||||
if needs_target_feature_fallback(&version, Some(69)) {
|
||||
// x86_64 Apple targets always support CMPXCHG16B:
|
||||
// https://github.com/rust-lang/rust/blob/1.68.0/compiler/rustc_target/src/spec/x86_64_apple_darwin.rs#L8
|
||||
// https://github.com/rust-lang/rust/blob/1.68.0/compiler/rustc_target/src/spec/apple_base.rs#L69-L70
|
||||
// (Windows (except Windows 7, since Rust 1.78) and Fuchsia (since Rust 1.87) targets
|
||||
// also enable CMPXCHG16B, but this branch is only used on pre-1.69 that
|
||||
// cmpxchg16b_target_feature is unstable.)
|
||||
// Script to get builtin targets that support CMPXCHG16B by default:
|
||||
// $ (for target in $(rustc -Z unstable-options --print all-target-specs-json | jq -r '. | to_entries[] | if .value.arch == "x86_64" then .key else empty end'); do rustc --print cfg --target "${target}" | grep -Fq '"cmpxchg16b"' && printf '%s\n' "${target}"; done)
|
||||
let is_apple = env::var("CARGO_CFG_TARGET_VENDOR").unwrap_or_default() == "apple";
|
||||
let cmpxchg16b = is_apple;
|
||||
// LLVM recognizes this also as cx16 target feature: https://godbolt.org/z/KM3jz616j
|
||||
// However, it is unlikely that rustc will support that name, so we ignore it.
|
||||
target_feature_fallback("cmpxchg16b", cmpxchg16b);
|
||||
}
|
||||
}
|
||||
"aarch64" | "arm64ec" => {
|
||||
// target_feature "lse2"/"lse128"/"rcpc3" is unstable and available on rustc side since nightly-2024-08-30: https://github.com/rust-lang/rust/pull/128192
|
||||
if !version.probe(82, 2024, 8, 29) || needs_target_feature_fallback(&version, None) {
|
||||
// FEAT_LSE2 doesn't imply FEAT_LSE. FEAT_LSE128 implies FEAT_LSE but not FEAT_LSE2.
|
||||
// AArch64 macOS always supports FEAT_LSE and FEAT_LSE2 because M1 is Armv8.4 with all features of Armv8.5 except FEAT_BTI:
|
||||
// https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/AArch64/AArch64Processors.td#L1558
|
||||
// https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/AArch64/AArch64Processors.td#L1180
|
||||
// Script to get builtin targets that support FEAT_LSE/FEAT_LSE2 by default:
|
||||
// $ (for target in $(rustc -Z unstable-options --print all-target-specs-json | jq -r '. | to_entries[] | if .value.arch == "aarch64" or .value.arch == "arm64ec" then .key else empty end'); do rustc --print cfg --target "${target}" | grep -Fq '"lse"' && printf '%s\n' "${target}"; done)
|
||||
// $ (for target in $(rustc -Z unstable-options --print all-target-specs-json | jq -r '. | to_entries[] | if .value.arch == "aarch64" or .value.arch == "arm64ec" then .key else empty end'); do rustc --print cfg --target "${target}" | grep -Fq '"lse2"' && printf '%s\n' "${target}"; done)
|
||||
let is_macos = target_os == "macos";
|
||||
let mut lse = is_macos;
|
||||
target_feature_fallback("lse2", is_macos);
|
||||
lse |= target_feature_fallback("lse128", false);
|
||||
target_feature_fallback("rcpc3", false);
|
||||
// aarch64_target_feature stabilized in Rust 1.61.
|
||||
if needs_target_feature_fallback(&version, Some(61)) {
|
||||
target_feature_fallback("lse", lse);
|
||||
}
|
||||
}
|
||||
// As of Rust 1.85, target_feature "lsfe" is not available on rustc side:
|
||||
// https://github.com/rust-lang/rust/blob/1.85.0/compiler/rustc_target/src/target_features.rs
|
||||
target_feature_fallback("lsfe", false);
|
||||
|
||||
// As of Apple M1/M1 Pro, on Apple hardware, CAS-loop-based RMW is much slower than
|
||||
// LL/SC-loop-based RMW: https://github.com/taiki-e/portable-atomic/pull/89
|
||||
let is_apple = env::var("CARGO_CFG_TARGET_VENDOR").unwrap_or_default() == "apple";
|
||||
if is_apple || target_cpu().map_or(false, |cpu| cpu.starts_with("apple-")) {
|
||||
println!("cargo:rustc-cfg=portable_atomic_ll_sc_rmw");
|
||||
}
|
||||
}
|
||||
"arm" => {
|
||||
// For non-Linux/Android pre-v6 Arm (tier 3) with unsafe_assume_single_core enabled.
|
||||
// feature(isa_attribute) stabilized in Rust 1.67 (nightly-2022-11-06): https://github.com/rust-lang/rust/pull/102458
|
||||
if version.nightly && !version.probe(67, 2022, 11, 5) {
|
||||
println!("cargo:rustc-cfg=portable_atomic_unstable_isa_attribute");
|
||||
}
|
||||
|
||||
if needs_target_feature_fallback(&version, None) {
|
||||
// #[cfg(target_feature = "v7")] and others don't work on stable.
|
||||
// armv7-unknown-linux-gnueabihf
|
||||
// ^^
|
||||
let mut mclass = false;
|
||||
let mut v6 = false;
|
||||
let mut v7 = false;
|
||||
if let Some(mut subarch) =
|
||||
strip_prefix(target, "arm").or_else(|| strip_prefix(target, "thumb"))
|
||||
{
|
||||
subarch = strip_prefix(subarch, "eb").unwrap_or(subarch); // ignore endianness
|
||||
subarch = subarch.split('-').next().unwrap(); // ignore vender/os/env
|
||||
let mut i = subarch.splitn(2, '.');
|
||||
subarch = i.next().unwrap();
|
||||
let suffix = i.next().unwrap_or_default(); // .base/.main suffix
|
||||
let mut known = true;
|
||||
// See https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/build.rs for details
|
||||
match subarch {
|
||||
"v7" | "v7a" | "v7neon" | "v7s" | "v7k" | "v8" | "v8a" | "v9" | "v9a" => {} // aclass
|
||||
"v7r" | "v8r" | "v9r" => {} // rclass
|
||||
"v6m" | "v7em" | "v7m" | "v8m" => mclass = true,
|
||||
// arm-linux-androideabi is v5te
|
||||
// https://github.com/rust-lang/rust/blob/1.84.0/compiler/rustc_target/src/spec/targets/arm_linux_androideabi.rs#L18
|
||||
_ if target == "arm-linux-androideabi" => subarch = "v5te",
|
||||
// armeb-unknown-linux-gnueabi is v8 & aclass
|
||||
// https://github.com/rust-lang/rust/blob/1.84.0/compiler/rustc_target/src/spec/targets/armeb_unknown_linux_gnueabi.rs#L18
|
||||
_ if target == "armeb-unknown-linux-gnueabi" => subarch = "v8",
|
||||
// Legacy Arm architectures (pre-v7 except v6m) don't have *class target feature.
|
||||
"" => subarch = "v6",
|
||||
"v4t" | "v5te" | "v6" | "v6k" => {}
|
||||
_ => {
|
||||
known = false;
|
||||
if env::var_os("PORTABLE_ATOMIC_DENY_WARNINGS").is_some() {
|
||||
panic!("unrecognized Arm subarch: {}", target)
|
||||
}
|
||||
println!(
|
||||
"cargo:warning={}: unrecognized Arm subarch: {}",
|
||||
env!("CARGO_PKG_NAME"),
|
||||
target
|
||||
);
|
||||
}
|
||||
}
|
||||
v6 = known && subarch.starts_with("v6");
|
||||
v7 = known && subarch.starts_with("v7");
|
||||
if known && (subarch.starts_with("v8") || subarch.starts_with("v9")) {
|
||||
// Armv8-M is not considered as v8 by LLVM.
|
||||
// https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/arm_shared/mod.rs
|
||||
if mclass {
|
||||
// Armv8-M Mainline is a superset of Armv7-M.
|
||||
// Armv8-M Baseline is a superset of Armv6-M.
|
||||
// That said, LLVM handles thumbv8m.main without v8m like v6m, not v7m: https://godbolt.org/z/Ph96v9zae
|
||||
// TODO: Armv9-M has not yet been released,
|
||||
// so it is not clear how it will be handled here.
|
||||
v6 = true;
|
||||
v7 = suffix == "main";
|
||||
} else {
|
||||
v7 = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
v6 |= target_feature_fallback("v7", v7);
|
||||
target_feature_fallback("v6", v6);
|
||||
target_feature_fallback("mclass", mclass);
|
||||
}
|
||||
}
|
||||
"riscv32" | "riscv64" => {
|
||||
// zabha and zacas imply zaamo in GCC, LLVM 20+, and Rust, but do not in LLVM 19.
|
||||
// However, enabling them without zaamo or a is not allowed in LLVM 19, so we can assume
|
||||
// zaamo is available when zabha is enabled).
|
||||
// https://github.com/llvm/llvm-project/commit/956361ca080a689a96b6552d28681aaf0ad2f494
|
||||
// https://github.com/gcc-mirror/gcc/commit/7b2b2e3d660edc8ef3a8cfbdfc2b0fd499459601
|
||||
// https://github.com/gcc-mirror/gcc/commit/11c2453a16b725b7fb67778e1ab4636a51a1217d
|
||||
// https://github.com/rust-lang/rust/pull/130877
|
||||
let mut zaamo = false;
|
||||
// target_feature "zacas" is available as unstable on rustc side
|
||||
// since nightly-2025-02-26 (https://github.com/rust-lang/rust/pull/137417),
|
||||
// and stabilized in Rust 1.94 (https://github.com/rust-lang/rust/pull/145948).
|
||||
if !version.probe(87, 2025, 2, 25) || needs_target_feature_fallback(&version, Some(94))
|
||||
{
|
||||
// amocas.{w,d,q} (and amocas.{b,h} if zabha is also available)
|
||||
// available as experimental since LLVM 17 https://github.com/llvm/llvm-project/commit/29f630a1ddcbb03caa31b5002f0cbc105ff3a869
|
||||
// available non-experimental since LLVM 20 https://github.com/llvm/llvm-project/commit/614aeda93b2225c6eb42b00ba189ba7ca2585c60
|
||||
zaamo |= target_feature_fallback("zacas", false);
|
||||
}
|
||||
// target_feature "zaamo"/"zabha" is available as unstable on rustc side
|
||||
// since nightly-2024-10-02 (https://github.com/rust-lang/rust/pull/130877),
|
||||
// and stabilized in Rust 1.94 (https://github.com/rust-lang/rust/pull/145948).
|
||||
if !version.probe(83, 2024, 10, 1) || needs_target_feature_fallback(&version, Some(94))
|
||||
{
|
||||
if version.llvm >= 19 {
|
||||
// amo*.{b,h}
|
||||
// available since LLVM 19 https://github.com/llvm/llvm-project/commit/89f87c387627150d342722b79c78cea2311cddf7 / https://github.com/llvm/llvm-project/commit/6b7444964a8d028989beee554a1f5c61d16a1cac
|
||||
zaamo |= target_feature_fallback("zabha", false);
|
||||
}
|
||||
// amo*.{w,d}
|
||||
target_feature_fallback("zaamo", zaamo);
|
||||
}
|
||||
}
|
||||
"powerpc64" => {
|
||||
// target_feature "quadword-atomics" is unstable and available on rustc side since nightly-2024-09-28: https://github.com/rust-lang/rust/pull/130873
|
||||
if !version.probe(83, 2024, 9, 27) || needs_target_feature_fallback(&version, None) {
|
||||
let mut pwr8_features = false;
|
||||
if let Some(cpu) = target_cpu() {
|
||||
if let Some(mut cpu_version) = strip_prefix(&cpu, "pwr") {
|
||||
cpu_version = strip_suffix(cpu_version, "x").unwrap_or(cpu_version); // for pwr5x and pwr6x
|
||||
if let Ok(cpu_version) = cpu_version.parse::<u32>() {
|
||||
pwr8_features = cpu_version >= 8;
|
||||
}
|
||||
} else {
|
||||
// https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/PowerPC/PPC.td#L789
|
||||
// https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/PowerPC/PPC.td#L557
|
||||
// On the minimum external LLVM version of the oldest rustc version which we can use asm_experimental_arch
|
||||
// on this target (see CI config for more), "future" is based on pwr10 features.
|
||||
// https://github.com/llvm/llvm-project/blob/llvmorg-12.0.0/llvm/lib/Target/PowerPC/PPC.td#L370
|
||||
pwr8_features = cpu == "future" || cpu == "ppc64le";
|
||||
}
|
||||
} else {
|
||||
// powerpc64le is pwr8 by default https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/PowerPC/PPC.td#L789
|
||||
// See also https://github.com/rust-lang/rust/issues/59932
|
||||
pwr8_features = env::var("CARGO_CFG_TARGET_ENDIAN")
|
||||
.expect("CARGO_CFG_TARGET_ENDIAN not set")
|
||||
== "little";
|
||||
}
|
||||
// power8 features: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/PowerPC/PPC.td#L484
|
||||
// lqarx and stqcx.
|
||||
target_feature_fallback("quadword-atomics", pwr8_features);
|
||||
}
|
||||
}
|
||||
"s390x" => {
|
||||
let mut arch9_features = false; // z196+
|
||||
let mut arch13_features = false; // z15+
|
||||
if let Some(cpu) = target_cpu() {
|
||||
// LLVM and GCC recognize the same names:
|
||||
// https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/SystemZ/SystemZProcessors.td
|
||||
// https://github.com/gcc-mirror/gcc/blob/releases/gcc-15.2.0/gcc/config/s390/s390.opt#L58-L128
|
||||
if let Some(arch_version) = strip_prefix(&cpu, "arch") {
|
||||
if let Ok(arch_version) = arch_version.parse::<u32>() {
|
||||
arch9_features = arch_version >= 9;
|
||||
arch13_features = arch_version >= 13;
|
||||
}
|
||||
} else {
|
||||
match &*cpu {
|
||||
"z196" | "zEC12" | "z13" | "z14" => arch9_features = true,
|
||||
"z15" | "z16" | "z17" => {
|
||||
arch9_features = true;
|
||||
arch13_features = true;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
// target_feature "miscellaneous-extensions-3" is available as unstable on rustc side
|
||||
// since nightly-2025-06-05 (https://github.com/rust-lang/rust/pull/141250),
|
||||
// and stabilized in Rust 1.93 (https://github.com/rust-lang/rust/pull/145656).
|
||||
if !version.probe(89, 2025, 6, 4) || needs_target_feature_fallback(&version, Some(93)) {
|
||||
// arch13 features: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/SystemZ/SystemZFeatures.td#L303
|
||||
// nand (nnr{,g}k), select (sel{,g}r), etc.
|
||||
target_feature_fallback("miscellaneous-extensions-3", arch13_features);
|
||||
}
|
||||
// As of Rust 1.84, target_feature "fast-serialization"/"load-store-on-cond"/"distinct-ops"/"miscellaneous-extensions-3" is not available on rustc side:
|
||||
// https://github.com/rust-lang/rust/blob/1.84.0/compiler/rustc_target/src/target_features.rs#L547
|
||||
// arch9 features: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/SystemZ/SystemZFeatures.td#L103
|
||||
// bcr 14,0
|
||||
target_feature_fallback("fast-serialization", arch9_features);
|
||||
// {l,st}oc{,g}{,r}
|
||||
target_feature_fallback("load-store-on-cond", arch9_features);
|
||||
// {al,sl,n,o,x}{,g}rk
|
||||
target_feature_fallback("distinct-ops", arch9_features);
|
||||
}
|
||||
"avr" => {
|
||||
// target_feature "rmw" is unstable and available on rustc side since nightly-2026-02-08: https://github.com/rust-lang/rust/pull/146900
|
||||
if !version.probe(95, 2026, 2, 7) || needs_target_feature_fallback(&version, None) {
|
||||
// https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/AVR/AVRDevices.td
|
||||
let mut xmegau = false; // FamilyXMEGAU
|
||||
if let Some(cpu) = target_cpu() {
|
||||
match &*cpu {
|
||||
"atxmega16a4u" | "atxmega16c4" | "atxmega32a4u" | "atxmega32c3"
|
||||
| "atxmega32c4" | "atxmega32e5" | "atxmega16e5" | "atxmega8e5"
|
||||
| "atxmega64a3u" | "atxmega64a4u" | "atxmega64b1" | "atxmega64b3"
|
||||
| "atxmega64c3" | "atxmega64a1u" | "atxmega128a3u" | "atxmega128b1"
|
||||
| "atxmega128b3" | "atxmega128c3" | "atxmega192a3u" | "atxmega192c3"
|
||||
| "atxmega256a3u" | "atxmega256a3bu" | "atxmega256c3" | "atxmega384c3"
|
||||
| "atxmega128a1u" | "atxmega128a4u" => xmegau = true,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
target_feature_fallback("rmw", xmegau);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
// HACK: Currently, it seems that the only way to handle unstable target
|
||||
// features on the stable is to parse the `-C target-feature` in RUSTFLAGS.
|
||||
//
|
||||
// - #[cfg(target_feature = "unstable_target_feature")] doesn't work on stable.
|
||||
// - CARGO_CFG_TARGET_FEATURE excludes unstable target features on stable.
|
||||
//
|
||||
// As mentioned in the [RFC2045], unstable target features are also passed to LLVM
|
||||
// (e.g., https://godbolt.org/z/4rr7rMcfG), so this hack works properly on stable.
|
||||
//
|
||||
// [RFC2045]: https://rust-lang.github.io/rfcs/2045-target-feature.html#backend-compilation-options
|
||||
fn needs_target_feature_fallback(version: &Version, stable: Option<u32>) -> bool {
|
||||
match stable {
|
||||
// In these cases, cfg(target_feature = "...") would work, so skip emitting our own fallback target_feature cfg.
|
||||
_ if version.nightly => false,
|
||||
Some(stabilized) if version.minor >= stabilized => false,
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
fn target_feature_fallback(name: &str, mut has_target_feature: bool) -> bool {
|
||||
if let Some(rustflags) = env::var_os("CARGO_ENCODED_RUSTFLAGS") {
|
||||
for mut flag in rustflags.to_string_lossy().split('\x1f') {
|
||||
flag = strip_prefix(flag, "-C").unwrap_or(flag);
|
||||
if let Some(flag) = strip_prefix(flag, "target-feature=") {
|
||||
for s in flag.split(',') {
|
||||
// TODO: Handles cases where a specific target feature
|
||||
// implicitly enables another target feature.
|
||||
match (s.as_bytes().first(), s.as_bytes().get(1..)) {
|
||||
(Some(b'+'), Some(f)) if f == name.as_bytes() => has_target_feature = true,
|
||||
(Some(b'-'), Some(f)) if f == name.as_bytes() => has_target_feature = false,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if has_target_feature {
|
||||
println!("cargo:rustc-cfg=portable_atomic_target_feature=\"{}\"", name);
|
||||
}
|
||||
has_target_feature
|
||||
}
|
||||
|
||||
fn target_cpu() -> Option<String> {
|
||||
let rustflags = env::var_os("CARGO_ENCODED_RUSTFLAGS")?;
|
||||
let rustflags = rustflags.to_string_lossy();
|
||||
let mut cpu = None;
|
||||
for mut flag in rustflags.split('\x1f') {
|
||||
flag = strip_prefix(flag, "-C").unwrap_or(flag);
|
||||
if let Some(flag) = strip_prefix(flag, "target-cpu=") {
|
||||
cpu = Some(flag);
|
||||
}
|
||||
}
|
||||
cpu.map(str::to_owned)
|
||||
}
|
||||
|
||||
fn is_allowed_feature(name: &str) -> bool {
|
||||
// https://github.com/dtolnay/thiserror/pull/248
|
||||
if env::var_os("RUSTC_STAGE").is_some() {
|
||||
return false;
|
||||
}
|
||||
|
||||
// allowed by default
|
||||
let mut allowed = true;
|
||||
if let Some(rustflags) = env::var_os("CARGO_ENCODED_RUSTFLAGS") {
|
||||
for mut flag in rustflags.to_string_lossy().split('\x1f') {
|
||||
flag = strip_prefix(flag, "-Z").unwrap_or(flag);
|
||||
if let Some(flag) = strip_prefix(flag, "allow-features=") {
|
||||
// If it is specified multiple times, the last value will be preferred.
|
||||
allowed = flag.split(',').any(|allowed| allowed == name);
|
||||
}
|
||||
}
|
||||
}
|
||||
allowed
|
||||
}
|
||||
|
||||
// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/build-common.rs.
|
||||
//
|
||||
// The target triplets have the form of 'arch-vendor-system'.
|
||||
//
|
||||
// When building for Linux (e.g. the 'system' part is
|
||||
// 'linux-something'), replace the vendor with 'unknown'
|
||||
// so that mapping to rust standard targets happens correctly.
|
||||
fn convert_custom_linux_target(target: &str) -> String {
|
||||
let mut parts: Vec<&str> = target.split('-').collect();
|
||||
let system = parts.get(2);
|
||||
if system == Some(&"linux") {
|
||||
parts[1] = "unknown";
|
||||
}
|
||||
parts.join("-")
|
||||
}
|
||||
|
||||
// str::strip_prefix requires Rust 1.45
|
||||
#[must_use]
|
||||
fn strip_prefix<'a>(s: &'a str, pat: &str) -> Option<&'a str> {
|
||||
if s.starts_with(pat) { Some(&s[pat.len()..]) } else { None }
|
||||
}
|
||||
// str::strip_suffix requires Rust 1.45
|
||||
#[must_use]
|
||||
fn strip_suffix<'a>(s: &'a str, pat: &str) -> Option<&'a str> {
|
||||
if s.ends_with(pat) { Some(&s[..s.len() - pat.len()]) } else { None }
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
# portable-atomic-util
|
||||
|
||||
portable-atomic-util crate has been moved to [its own repository](https://github.com/taiki-e/portable-atomic-util).
|
||||
Vendored
+631
@@ -0,0 +1,631 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
#![allow(missing_docs)]
|
||||
|
||||
#[cfg(not(all(
|
||||
portable_atomic_no_atomic_load_store,
|
||||
not(any(
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
target_arch = "riscv32",
|
||||
target_arch = "riscv64",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
)),
|
||||
)))]
|
||||
#[macro_use]
|
||||
mod atomic_8_16_macros {
|
||||
#[macro_export]
|
||||
macro_rules! cfg_has_atomic_8 {
|
||||
($($tt:tt)*) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
#[macro_export]
|
||||
macro_rules! cfg_no_atomic_8 {
|
||||
($($tt:tt)*) => {};
|
||||
}
|
||||
#[macro_export]
|
||||
macro_rules! cfg_has_atomic_16 {
|
||||
($($tt:tt)*) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
#[macro_export]
|
||||
macro_rules! cfg_no_atomic_16 {
|
||||
($($tt:tt)*) => {};
|
||||
}
|
||||
}
|
||||
#[cfg(all(
|
||||
portable_atomic_no_atomic_load_store,
|
||||
not(any(
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
target_arch = "riscv32",
|
||||
target_arch = "riscv64",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
)),
|
||||
))]
|
||||
#[macro_use]
|
||||
mod atomic_8_16_macros {
|
||||
#[macro_export]
|
||||
macro_rules! cfg_has_atomic_8 {
|
||||
($($tt:tt)*) => {};
|
||||
}
|
||||
#[macro_export]
|
||||
macro_rules! cfg_no_atomic_8 {
|
||||
($($tt:tt)*) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
#[macro_export]
|
||||
macro_rules! cfg_has_atomic_16 {
|
||||
($($tt:tt)*) => {};
|
||||
}
|
||||
#[macro_export]
|
||||
macro_rules! cfg_no_atomic_16 {
|
||||
($($tt:tt)*) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(
|
||||
any(not(target_pointer_width = "16"), feature = "fallback"),
|
||||
not(all(
|
||||
portable_atomic_no_atomic_load_store,
|
||||
not(any(
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
target_arch = "riscv32",
|
||||
target_arch = "riscv64",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
)),
|
||||
)),
|
||||
))]
|
||||
#[macro_use]
|
||||
mod atomic_32_macros {
|
||||
#[macro_export]
|
||||
macro_rules! cfg_has_atomic_32 {
|
||||
($($tt:tt)*) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
#[macro_export]
|
||||
macro_rules! cfg_no_atomic_32 {
|
||||
($($tt:tt)*) => {};
|
||||
}
|
||||
}
|
||||
#[cfg(not(all(
|
||||
any(not(target_pointer_width = "16"), feature = "fallback"),
|
||||
not(all(
|
||||
portable_atomic_no_atomic_load_store,
|
||||
not(any(
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
target_arch = "riscv32",
|
||||
target_arch = "riscv64",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
)),
|
||||
)),
|
||||
)))]
|
||||
#[macro_use]
|
||||
mod atomic_32_macros {
|
||||
#[macro_export]
|
||||
macro_rules! cfg_has_atomic_32 {
|
||||
($($tt:tt)*) => {};
|
||||
}
|
||||
#[macro_export]
|
||||
macro_rules! cfg_no_atomic_32 {
|
||||
($($tt:tt)*) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(any(
|
||||
all(
|
||||
feature = "fallback",
|
||||
any(
|
||||
not(portable_atomic_no_atomic_cas),
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
),
|
||||
),
|
||||
not(portable_atomic_no_atomic_64),
|
||||
not(any(target_pointer_width = "16", target_pointer_width = "32")),
|
||||
all(
|
||||
target_arch = "riscv32",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
|
||||
),
|
||||
))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(
|
||||
all(
|
||||
feature = "fallback",
|
||||
any(
|
||||
target_has_atomic = "ptr",
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
),
|
||||
),
|
||||
target_has_atomic = "64",
|
||||
not(any(target_pointer_width = "16", target_pointer_width = "32")),
|
||||
all(
|
||||
target_arch = "riscv32",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
|
||||
),
|
||||
))
|
||||
)]
|
||||
#[macro_use]
|
||||
mod atomic_64_macros {
|
||||
#[macro_export]
|
||||
macro_rules! cfg_has_atomic_64 {
|
||||
($($tt:tt)*) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
#[macro_export]
|
||||
macro_rules! cfg_no_atomic_64 {
|
||||
($($tt:tt)*) => {};
|
||||
}
|
||||
}
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(not(any(
|
||||
all(
|
||||
feature = "fallback",
|
||||
any(
|
||||
not(portable_atomic_no_atomic_cas),
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
),
|
||||
),
|
||||
not(portable_atomic_no_atomic_64),
|
||||
not(any(target_pointer_width = "16", target_pointer_width = "32")),
|
||||
all(
|
||||
target_arch = "riscv32",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
|
||||
),
|
||||
)))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(not(any(
|
||||
all(
|
||||
feature = "fallback",
|
||||
any(
|
||||
target_has_atomic = "ptr",
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
),
|
||||
),
|
||||
target_has_atomic = "64",
|
||||
not(any(target_pointer_width = "16", target_pointer_width = "32")),
|
||||
all(
|
||||
target_arch = "riscv32",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
|
||||
),
|
||||
)))
|
||||
)]
|
||||
#[macro_use]
|
||||
mod atomic_64_macros {
|
||||
#[macro_export]
|
||||
macro_rules! cfg_has_atomic_64 {
|
||||
($($tt:tt)*) => {};
|
||||
}
|
||||
#[macro_export]
|
||||
macro_rules! cfg_no_atomic_64 {
|
||||
($($tt:tt)*) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg_attr(
|
||||
not(feature = "fallback"),
|
||||
cfg(any(
|
||||
all(
|
||||
target_arch = "aarch64",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
),
|
||||
all(
|
||||
target_arch = "arm64ec",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
not(portable_atomic_no_asm),
|
||||
),
|
||||
all(
|
||||
target_arch = "x86_64",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
portable_atomic_no_cmpxchg16b_intrinsic,
|
||||
)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
|
||||
),
|
||||
all(
|
||||
target_arch = "riscv64",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
|
||||
),
|
||||
all(
|
||||
target_arch = "powerpc64",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
not(portable_atomic_no_asm),
|
||||
any(
|
||||
target_feature = "quadword-atomics",
|
||||
portable_atomic_target_feature = "quadword-atomics",
|
||||
),
|
||||
),
|
||||
all(
|
||||
target_arch = "s390x",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
not(portable_atomic_no_asm),
|
||||
),
|
||||
))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
all(feature = "fallback", portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(
|
||||
not(portable_atomic_no_atomic_cas),
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
all(feature = "fallback", not(portable_atomic_no_cfg_target_has_atomic)),
|
||||
cfg(any(
|
||||
target_has_atomic = "ptr",
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
))
|
||||
)]
|
||||
#[macro_use]
|
||||
mod atomic_128_macros {
|
||||
#[macro_export]
|
||||
macro_rules! cfg_has_atomic_128 {
|
||||
($($tt:tt)*) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
#[macro_export]
|
||||
macro_rules! cfg_no_atomic_128 {
|
||||
($($tt:tt)*) => {};
|
||||
}
|
||||
}
|
||||
#[cfg_attr(
|
||||
not(feature = "fallback"),
|
||||
cfg(not(any(
|
||||
all(
|
||||
target_arch = "aarch64",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
),
|
||||
all(
|
||||
target_arch = "arm64ec",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
not(portable_atomic_no_asm),
|
||||
),
|
||||
all(
|
||||
target_arch = "x86_64",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
portable_atomic_no_cmpxchg16b_intrinsic,
|
||||
)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
|
||||
),
|
||||
all(
|
||||
target_arch = "riscv64",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
|
||||
),
|
||||
all(
|
||||
target_arch = "powerpc64",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
not(portable_atomic_no_asm),
|
||||
any(
|
||||
target_feature = "quadword-atomics",
|
||||
portable_atomic_target_feature = "quadword-atomics",
|
||||
),
|
||||
),
|
||||
all(
|
||||
target_arch = "s390x",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
not(portable_atomic_no_asm),
|
||||
),
|
||||
)))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
all(feature = "fallback", portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(not(any(
|
||||
not(portable_atomic_no_atomic_cas),
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
)))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
all(feature = "fallback", not(portable_atomic_no_cfg_target_has_atomic)),
|
||||
cfg(not(any(
|
||||
target_has_atomic = "ptr",
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
)))
|
||||
)]
|
||||
#[macro_use]
|
||||
mod atomic_128_macros {
|
||||
#[macro_export]
|
||||
macro_rules! cfg_has_atomic_128 {
|
||||
($($tt:tt)*) => {};
|
||||
}
|
||||
#[macro_export]
|
||||
macro_rules! cfg_no_atomic_128 {
|
||||
($($tt:tt)*) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(any(
|
||||
not(portable_atomic_no_atomic_cas),
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(
|
||||
target_has_atomic = "ptr",
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
))
|
||||
)]
|
||||
#[macro_use]
|
||||
mod atomic_cas_macros {
|
||||
#[macro_export]
|
||||
macro_rules! cfg_has_atomic_cas {
|
||||
($($tt:tt)*) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
#[macro_export]
|
||||
macro_rules! cfg_no_atomic_cas {
|
||||
($($tt:tt)*) => {};
|
||||
}
|
||||
// private
|
||||
macro_rules! cfg_has_atomic_cas_or_amo32 {
|
||||
($($tt:tt)*) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
macro_rules! cfg_has_atomic_cas_or_amo8 {
|
||||
($($tt:tt)*) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
}
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(not(any(
|
||||
not(portable_atomic_no_atomic_cas),
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
)))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(not(any(
|
||||
target_has_atomic = "ptr",
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
)))
|
||||
)]
|
||||
#[macro_use]
|
||||
mod atomic_cas_macros {
|
||||
#[macro_export]
|
||||
macro_rules! cfg_has_atomic_cas {
|
||||
($($tt:tt)*) => {};
|
||||
}
|
||||
#[macro_export]
|
||||
macro_rules! cfg_no_atomic_cas {
|
||||
($($tt:tt)*) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
// private
|
||||
#[cfg_attr(
|
||||
any(target_arch = "riscv32", target_arch = "riscv64"),
|
||||
cfg(not(any(target_feature = "zaamo", portable_atomic_target_feature = "zaamo")))
|
||||
)]
|
||||
macro_rules! cfg_has_atomic_cas_or_amo32 {
|
||||
($($tt:tt)*) => {};
|
||||
}
|
||||
#[cfg_attr(
|
||||
any(target_arch = "riscv32", target_arch = "riscv64"),
|
||||
cfg(not(any(target_feature = "zaamo", portable_atomic_target_feature = "zaamo")))
|
||||
)]
|
||||
macro_rules! cfg_no_atomic_cas_or_amo32 {
|
||||
($($tt:tt)*) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
#[cfg(all(
|
||||
any(target_arch = "riscv32", target_arch = "riscv64"),
|
||||
any(target_feature = "zaamo", portable_atomic_target_feature = "zaamo"),
|
||||
))]
|
||||
macro_rules! cfg_has_atomic_cas_or_amo32 {
|
||||
($($tt:tt)*) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
#[cfg(all(
|
||||
any(target_arch = "riscv32", target_arch = "riscv64"),
|
||||
any(target_feature = "zaamo", portable_atomic_target_feature = "zaamo"),
|
||||
))]
|
||||
macro_rules! cfg_no_atomic_cas_or_amo32 {
|
||||
($($tt:tt)*) => {};
|
||||
}
|
||||
#[cfg_attr(
|
||||
any(target_arch = "riscv32", target_arch = "riscv64"),
|
||||
cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))
|
||||
)]
|
||||
#[allow(unused_macros)]
|
||||
macro_rules! cfg_has_atomic_cas_or_amo8 {
|
||||
($($tt:tt)*) => {};
|
||||
}
|
||||
#[cfg_attr(
|
||||
any(target_arch = "riscv32", target_arch = "riscv64"),
|
||||
cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))
|
||||
)]
|
||||
#[cfg_attr(target_arch = "bpf", allow(unused_macros))]
|
||||
macro_rules! cfg_no_atomic_cas_or_amo8 {
|
||||
($($tt:tt)*) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
#[cfg(all(
|
||||
any(target_arch = "riscv32", target_arch = "riscv64"),
|
||||
any(target_feature = "zabha", portable_atomic_target_feature = "zabha"),
|
||||
))]
|
||||
macro_rules! cfg_has_atomic_cas_or_amo8 {
|
||||
($($tt:tt)*) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
#[cfg(all(
|
||||
any(target_arch = "riscv32", target_arch = "riscv64"),
|
||||
any(target_feature = "zabha", portable_atomic_target_feature = "zabha"),
|
||||
))]
|
||||
macro_rules! cfg_no_atomic_cas_or_amo8 {
|
||||
($($tt:tt)*) => {};
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! cfg_core_atomic {
|
||||
($($tt:tt)*) => {
|
||||
#[cfg(not(any(
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
all(
|
||||
portable_atomic_no_atomic_load_store,
|
||||
not(all(target_arch = "bpf", not(feature = "critical-section"))),
|
||||
),
|
||||
)))]
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(not(all(
|
||||
any(
|
||||
target_arch = "riscv32",
|
||||
target_arch = "riscv64",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
),
|
||||
portable_atomic_no_atomic_cas,
|
||||
)))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(not(all(
|
||||
any(
|
||||
target_arch = "riscv32",
|
||||
target_arch = "riscv64",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
),
|
||||
not(target_has_atomic = "ptr"),
|
||||
)))
|
||||
)]
|
||||
items!($($tt)*);
|
||||
};
|
||||
}
|
||||
|
||||
// Check that all cfg_ macros work.
|
||||
mod check {
|
||||
crate::cfg_has_atomic_8! { type _Atomic8 = (); }
|
||||
crate::cfg_no_atomic_8! { type _Atomic8 = (); }
|
||||
crate::cfg_has_atomic_16! { type _Atomic16 = (); }
|
||||
crate::cfg_no_atomic_16! { type _Atomic16 = (); }
|
||||
crate::cfg_has_atomic_32! { type _Atomic32 = (); }
|
||||
crate::cfg_no_atomic_32! { type _Atomic32 = (); }
|
||||
crate::cfg_has_atomic_64! { type _Atomic64 = (); }
|
||||
crate::cfg_no_atomic_64! { type _Atomic64 = (); }
|
||||
crate::cfg_has_atomic_128! { type _Atomic128 = (); }
|
||||
crate::cfg_no_atomic_128! { type _Atomic128 = (); }
|
||||
crate::cfg_has_atomic_ptr! { type _AtomicPtr = (); }
|
||||
crate::cfg_no_atomic_ptr! { type _AtomicPtr = (); }
|
||||
crate::cfg_has_atomic_cas! { type __AtomicPtr = (); }
|
||||
crate::cfg_no_atomic_cas! { type __AtomicPtr = (); }
|
||||
#[allow(unused_imports)]
|
||||
use self::{
|
||||
__AtomicPtr as _, _Atomic8 as _, _Atomic16 as _, _Atomic32 as _, _Atomic64 as _,
|
||||
_Atomic128 as _, _AtomicPtr as _,
|
||||
};
|
||||
}
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
// This file is @generated by target_spec.sh.
|
||||
// It is not intended for manual editing.
|
||||
|
||||
// Note: This is the list as of nightly-2022-02-10. We don't refer to this in
|
||||
// nightly-2022-02-11+ because feature(cfg_target_has_atomic) stabilized.
|
||||
#[rustfmt::skip]
|
||||
pub(crate) static NO_ATOMIC_CAS: &[&str] = &[
|
||||
"avr-unknown-gnu-atmega328",
|
||||
"bpfeb-unknown-none",
|
||||
"bpfel-unknown-none",
|
||||
"msp430-none-elf",
|
||||
"riscv32i-unknown-none-elf",
|
||||
"riscv32imc-unknown-none-elf",
|
||||
"thumbv4t-none-eabi",
|
||||
"thumbv6m-none-eabi",
|
||||
];
|
||||
|
||||
// Note: This is the list as of nightly-2022-02-10. We don't refer to this in
|
||||
// nightly-2022-02-11+ because feature(cfg_target_has_atomic) stabilized.
|
||||
#[rustfmt::skip]
|
||||
pub(crate) static NO_ATOMIC_64: &[&str] = &[
|
||||
"arm-linux-androideabi",
|
||||
"armebv7r-none-eabi",
|
||||
"armebv7r-none-eabihf",
|
||||
"armv4t-unknown-linux-gnueabi",
|
||||
"armv5te-unknown-linux-gnueabi",
|
||||
"armv5te-unknown-linux-musleabi",
|
||||
"armv5te-unknown-linux-uclibceabi",
|
||||
"armv6k-nintendo-3ds",
|
||||
"armv7r-none-eabi",
|
||||
"armv7r-none-eabihf",
|
||||
"avr-unknown-gnu-atmega328",
|
||||
"hexagon-unknown-linux-musl",
|
||||
"m68k-unknown-linux-gnu",
|
||||
"mips-unknown-linux-gnu",
|
||||
"mips-unknown-linux-musl",
|
||||
"mips-unknown-linux-uclibc",
|
||||
"mipsel-sony-psp",
|
||||
"mipsel-unknown-linux-gnu",
|
||||
"mipsel-unknown-linux-musl",
|
||||
"mipsel-unknown-linux-uclibc",
|
||||
"mipsel-unknown-none",
|
||||
"mipsisa32r6-unknown-linux-gnu",
|
||||
"mipsisa32r6el-unknown-linux-gnu",
|
||||
"msp430-none-elf",
|
||||
"powerpc-unknown-freebsd",
|
||||
"powerpc-unknown-linux-gnu",
|
||||
"powerpc-unknown-linux-gnuspe",
|
||||
"powerpc-unknown-linux-musl",
|
||||
"powerpc-unknown-netbsd",
|
||||
"powerpc-unknown-openbsd",
|
||||
"powerpc-wrs-vxworks",
|
||||
"powerpc-wrs-vxworks-spe",
|
||||
"riscv32gc-unknown-linux-gnu",
|
||||
"riscv32gc-unknown-linux-musl",
|
||||
"riscv32i-unknown-none-elf",
|
||||
"riscv32imac-unknown-none-elf",
|
||||
"riscv32imc-esp-espidf",
|
||||
"riscv32imc-unknown-none-elf",
|
||||
"thumbv4t-none-eabi",
|
||||
"thumbv6m-none-eabi",
|
||||
"thumbv7em-none-eabi",
|
||||
"thumbv7em-none-eabihf",
|
||||
"thumbv7m-none-eabi",
|
||||
"thumbv8m.base-none-eabi",
|
||||
"thumbv8m.main-none-eabi",
|
||||
"thumbv8m.main-none-eabihf",
|
||||
];
|
||||
|
||||
#[rustfmt::skip]
|
||||
pub(crate) static NO_ATOMIC: &[&str] = &[
|
||||
"armv4t-none-eabi",
|
||||
"armv5te-none-eabi",
|
||||
"bpfeb-unknown-none",
|
||||
"bpfel-unknown-none",
|
||||
"mipsel-sony-psx",
|
||||
"thumbv4t-none-eabi",
|
||||
"thumbv5te-none-eabi",
|
||||
];
|
||||
+168
@@ -0,0 +1,168 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
// This file is @generated by target_spec.sh.
|
||||
// It is not intended for manual editing.
|
||||
|
||||
#![allow(dead_code, unused_macros)]
|
||||
|
||||
// On AArch64, the base register of memory-related instructions must be 64-bit.
|
||||
// Passing a 32-bit value to `in(reg)` on AArch64 results in the upper bits
|
||||
// having an undefined value, but to work correctly with ILP32 ABI, the upper
|
||||
// bits must be zero, which is handled here by casting to u64. Another way to
|
||||
// handle this is to pass it as a pointer and clear the upper bits inside asm,
|
||||
// but it is easier to overlook than cast, which can catch overlooks by
|
||||
// asm_sub_register lint.
|
||||
// See also https://github.com/ARM-software/abi-aa/blob/2025Q4/aapcs64/aapcs64.rst#pointers
|
||||
//
|
||||
// Except for x86_64, which can use 32-bit registers in the destination operand
|
||||
// (on x86_64, we use the ptr_modifier macro to handle this), we need to do the
|
||||
// same for ILP32 ABI on other 64-bit architectures. (At least, as far as I can
|
||||
// see from the assembly generated by LLVM, this is also required for MIPS64 N32
|
||||
// ABI. I don't know about other similar ABIs, but in any case, this should be
|
||||
// a safe default for such ABIs).
|
||||
//
|
||||
// Known architectures that have such ABI are x86_64 (X32), AArch64 (ILP32),
|
||||
// mips64/mips64r6 (N32), riscv64 (RV64ILP32*), PowerPC64 (Lv2). (As of
|
||||
// 2026-03-22, only the former two are supported by rustc.) However, we list
|
||||
// all known 64-bit architectures because similar ABIs may exist or future added
|
||||
// for other architectures.
|
||||
#[cfg(all(
|
||||
target_pointer_width = "32",
|
||||
any(
|
||||
target_arch = "aarch64",
|
||||
target_arch = "amdgpu",
|
||||
target_arch = "arm64ec",
|
||||
target_arch = "bpf",
|
||||
target_arch = "loongarch64",
|
||||
target_arch = "mips64",
|
||||
target_arch = "mips64r6",
|
||||
target_arch = "nvptx64",
|
||||
target_arch = "powerpc64",
|
||||
target_arch = "riscv64",
|
||||
target_arch = "s390x",
|
||||
target_arch = "sparc64",
|
||||
target_arch = "wasm64",
|
||||
target_arch = "x86_64",
|
||||
),
|
||||
))]
|
||||
#[macro_use]
|
||||
mod imp {
|
||||
#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
|
||||
macro_rules! ptr_reg {
|
||||
($ptr:ident) => {{
|
||||
let _: *const _ = $ptr; // ensure $ptr is a pointer (*mut _ or *const _)
|
||||
#[cfg(not(portable_atomic_no_asm_maybe_uninit))]
|
||||
#[allow(clippy::ptr_as_ptr)]
|
||||
{
|
||||
// If we cast to u64 here, the provenance will be lost,
|
||||
// so we convert to MaybeUninit<u64> via zero extend helper.
|
||||
crate::utils::zero_extend64_ptr($ptr as *mut ())
|
||||
}
|
||||
#[cfg(portable_atomic_no_asm_maybe_uninit)]
|
||||
{
|
||||
// Use cast on old rustc because it does not support MaybeUninit
|
||||
// registers. This is still permissive-provenance compatible and
|
||||
// is sound.
|
||||
$ptr as u64
|
||||
}
|
||||
}};
|
||||
}
|
||||
pub(crate) type RegISize = i64;
|
||||
pub(crate) type RegSize = u64;
|
||||
}
|
||||
#[cfg(not(all(
|
||||
target_pointer_width = "32",
|
||||
any(
|
||||
target_arch = "aarch64",
|
||||
target_arch = "amdgpu",
|
||||
target_arch = "arm64ec",
|
||||
target_arch = "bpf",
|
||||
target_arch = "loongarch64",
|
||||
target_arch = "mips64",
|
||||
target_arch = "mips64r6",
|
||||
target_arch = "nvptx64",
|
||||
target_arch = "powerpc64",
|
||||
target_arch = "riscv64",
|
||||
target_arch = "s390x",
|
||||
target_arch = "sparc64",
|
||||
target_arch = "wasm64",
|
||||
target_arch = "x86_64",
|
||||
),
|
||||
)))]
|
||||
#[macro_use]
|
||||
mod imp {
|
||||
#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
|
||||
macro_rules! ptr_reg {
|
||||
($ptr:ident) => {{
|
||||
let _: *const _ = $ptr; // ensure $ptr is a pointer (*mut _ or *const _)
|
||||
$ptr // cast is unnecessary here.
|
||||
}};
|
||||
}
|
||||
pub(crate) type RegISize = isize;
|
||||
pub(crate) type RegSize = usize;
|
||||
}
|
||||
pub(crate) use self::imp::{RegISize, RegSize};
|
||||
|
||||
// Some 64-bit architectures have ABI with 32-bit pointer width (e.g., x86_64 X32 ABI,
|
||||
// AArch64 ILP32 ABI, MIPS64 N32 ABI). On those targets, AtomicU64 is available
|
||||
// and fast, so use it to implement normal sequence lock.
|
||||
//
|
||||
// See ptr_reg macro for the reason why all known 64-bit architectures are listed.
|
||||
//
|
||||
// Note: These macros are cfg_*!({ }), not cfg_*! { }.
|
||||
// An extra brace is used in input to make contents rustfmt-able.
|
||||
#[cfg(any(
|
||||
not(any(target_pointer_width = "16", target_pointer_width = "32")), // i.e., 64-bit or greater
|
||||
target_arch = "aarch64",
|
||||
target_arch = "amdgpu",
|
||||
target_arch = "arm64ec",
|
||||
target_arch = "bpf",
|
||||
target_arch = "loongarch64",
|
||||
target_arch = "mips64",
|
||||
target_arch = "mips64r6",
|
||||
target_arch = "nvptx64",
|
||||
target_arch = "powerpc64",
|
||||
target_arch = "riscv64",
|
||||
target_arch = "s390x",
|
||||
target_arch = "sparc64",
|
||||
target_arch = "wasm64",
|
||||
target_arch = "x86_64",
|
||||
))]
|
||||
#[macro_use]
|
||||
mod fast_atomic_64_macros {
|
||||
macro_rules! cfg_has_fast_atomic_64 {
|
||||
({$($tt:tt)*}) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
macro_rules! cfg_no_fast_atomic_64 {
|
||||
({$($tt:tt)*}) => {};
|
||||
}
|
||||
}
|
||||
#[cfg(not(any(
|
||||
not(any(target_pointer_width = "16", target_pointer_width = "32")), // i.e., 64-bit or greater
|
||||
target_arch = "aarch64",
|
||||
target_arch = "amdgpu",
|
||||
target_arch = "arm64ec",
|
||||
target_arch = "bpf",
|
||||
target_arch = "loongarch64",
|
||||
target_arch = "mips64",
|
||||
target_arch = "mips64r6",
|
||||
target_arch = "nvptx64",
|
||||
target_arch = "powerpc64",
|
||||
target_arch = "riscv64",
|
||||
target_arch = "s390x",
|
||||
target_arch = "sparc64",
|
||||
target_arch = "wasm64",
|
||||
target_arch = "x86_64",
|
||||
)))]
|
||||
#[macro_use]
|
||||
mod fast_atomic_64_macros {
|
||||
macro_rules! cfg_has_fast_atomic_64 {
|
||||
({$($tt:tt)*}) => {};
|
||||
}
|
||||
macro_rules! cfg_no_fast_atomic_64 {
|
||||
({$($tt:tt)*}) => {
|
||||
$($tt)*
|
||||
};
|
||||
}
|
||||
}
|
||||
+257
@@ -0,0 +1,257 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Atomic operations implementation on pre-v6 ARM Linux/Android.
|
||||
|
||||
This module optimizes cases where LLVM code generation is not optimal,
|
||||
and provides workaround for core::sync::atomic bug (https://github.com/rust-lang/compiler-builtins/pull/1234).
|
||||
|
||||
See "Atomic operation overview by architecture" in atomic-maybe-uninit for a more comprehensive and
|
||||
detailed description of the atomic and synchronize instructions in this architecture:
|
||||
https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#arm
|
||||
|
||||
Refs:
|
||||
- https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/arm/kernel_user_helpers.rst
|
||||
- https://github.com/rust-lang/compiler-builtins/blob/compiler_builtins-v0.1.124/src/arm_linux.rs
|
||||
- https://github.com/taiki-e/atomic-maybe-uninit
|
||||
|
||||
Note: __kuser_memory_barrier is always SeqCst.
|
||||
https://github.com/taiki-e/atomic-maybe-uninit/blob/v0.3.21/src/arch/arm.rs#L215
|
||||
|
||||
Note: On Miri and ThreadSanitizer which do not go through compiler-builtins, we don't use
|
||||
this module and use normal core::sync::atomic instead.
|
||||
*/
|
||||
|
||||
use core::{mem, sync::atomic::Ordering};
|
||||
|
||||
// https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/arm/kernel_user_helpers.rst
|
||||
// __kuser_helper_version >= 3 (kernel version 2.6.15+)
|
||||
const KUSER_MEMORY_BARRIER: usize = 0xFFFF0FA0;
|
||||
#[inline]
|
||||
fn __kuser_memory_barrier() {
|
||||
// SAFETY: kernel docs specify a known address with the given signature.
|
||||
// core assumes __kuser_helper_version >= 3.
|
||||
unsafe {
|
||||
let f: extern "C" fn() =
|
||||
mem::transmute(crate::utils::ptr::with_exposed_provenance::<()>(KUSER_MEMORY_BARRIER));
|
||||
f();
|
||||
}
|
||||
}
|
||||
|
||||
/// An atomic fence.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `order` is [`Relaxed`](Ordering::Relaxed).
|
||||
#[cfg_attr(doc, allow(dead_code))]
|
||||
#[inline]
|
||||
#[cfg_attr(any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri), track_caller)]
|
||||
pub fn fence(order: Ordering) {
|
||||
// Call __kuser_memory_barrier without going through __sync_synchronize.
|
||||
match order {
|
||||
Ordering::Relaxed => panic!("there is no such thing as a relaxed fence"),
|
||||
_ => __kuser_memory_barrier(),
|
||||
}
|
||||
}
|
||||
|
||||
cfg_core_atomic!({
|
||||
use super::core_atomic::{
|
||||
AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, AtomicU32,
|
||||
AtomicUsize,
|
||||
};
|
||||
macro_rules! atomic {
|
||||
($([$($generics:tt)*])? $atomic_type:ident, $value_type:ty) => {
|
||||
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn load(&self, order: Ordering) -> $value_type {
|
||||
crate::utils::assert_load_ordering(order);
|
||||
match order {
|
||||
Ordering::Relaxed => self.inner.load(Ordering::Relaxed),
|
||||
// Acquire and SeqCst loads are equivalent.
|
||||
// This matches with LLVM, but GCC emits `dmb; ldr*; dmb` for SeqCst load.
|
||||
Ordering::Acquire | Ordering::SeqCst => {
|
||||
let out = self.inner.load(Ordering::Relaxed);
|
||||
__kuser_memory_barrier();
|
||||
out
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn store(&self, val: $value_type, order: Ordering) {
|
||||
crate::utils::assert_store_ordering(order);
|
||||
match order {
|
||||
Ordering::Relaxed => self.inner.store(val, Ordering::Relaxed),
|
||||
Ordering::Release => {
|
||||
__kuser_memory_barrier();
|
||||
self.inner.store(val, Ordering::Relaxed);
|
||||
}
|
||||
Ordering::SeqCst => {
|
||||
__kuser_memory_barrier();
|
||||
self.inner.store(val, Ordering::Relaxed);
|
||||
__kuser_memory_barrier();
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(not(portable_atomic_no_atomic_cas))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(target_has_atomic = "ptr")
|
||||
)]
|
||||
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn compare_exchange(
|
||||
&self,
|
||||
current: $value_type,
|
||||
new: $value_type,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<$value_type, $value_type> {
|
||||
crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)
|
||||
#[cfg(portable_atomic_no_stronger_failure_ordering)]
|
||||
let success = crate::utils::upgrade_success_ordering(success, failure);
|
||||
let res = self.inner.compare_exchange(current, new, success, failure);
|
||||
if failure != Ordering::Relaxed && res.is_err() {
|
||||
__kuser_memory_barrier();
|
||||
}
|
||||
res
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn compare_exchange_weak(
|
||||
&self,
|
||||
current: $value_type,
|
||||
new: $value_type,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<$value_type, $value_type> {
|
||||
crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)
|
||||
#[cfg(portable_atomic_no_stronger_failure_ordering)]
|
||||
let success = crate::utils::upgrade_success_ordering(success, failure);
|
||||
let res = self.inner.compare_exchange_weak(current, new, success, failure);
|
||||
if failure != Ordering::Relaxed && res.is_err() {
|
||||
__kuser_memory_barrier();
|
||||
}
|
||||
res
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
atomic!(AtomicI8, i8);
|
||||
atomic!(AtomicU8, u8);
|
||||
atomic!(AtomicI16, i16);
|
||||
atomic!(AtomicU16, u16);
|
||||
atomic!(AtomicI32, i32);
|
||||
atomic!(AtomicU32, u32);
|
||||
atomic!(AtomicIsize, isize);
|
||||
atomic!(AtomicUsize, usize);
|
||||
atomic!([T] AtomicPtr, *mut T);
|
||||
});
|
||||
|
||||
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_64)))]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(
|
||||
target_has_atomic = "64",
|
||||
not(any(target_pointer_width = "16", target_pointer_width = "32")),
|
||||
))
|
||||
)]
|
||||
items!({
|
||||
// This is not normally reached on this platform.
|
||||
// It is only reached if something strange occurs, such as using target-feature=-v6 on armv6 target.
|
||||
// Since 64-bit atomic operations differ from 32-bit or smaller ones on Arm, similar optimizations
|
||||
// or workarounds are useless here, so do the same as atomic_int! macro in core_atomic.rs.
|
||||
// NB: Sync with atomic_int! macro in core_atomic.rs
|
||||
use super::core_atomic::{AtomicI64, AtomicU64};
|
||||
macro_rules! atomic_int {
|
||||
($atomic_type:ident, $int_type:ident) => {
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn load(&self, order: Ordering) -> $int_type {
|
||||
crate::utils::assert_load_ordering(order); // for track_caller (compiler can omit double check)
|
||||
self.inner.load(order)
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn store(&self, val: $int_type, order: Ordering) {
|
||||
crate::utils::assert_store_ordering(order); // for track_caller (compiler can omit double check)
|
||||
self.inner.store(val, order);
|
||||
}
|
||||
}
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(not(portable_atomic_no_atomic_cas))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(target_has_atomic = "ptr")
|
||||
)]
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn compare_exchange(
|
||||
&self,
|
||||
current: $int_type,
|
||||
new: $int_type,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<$int_type, $int_type> {
|
||||
crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)
|
||||
#[cfg(portable_atomic_no_stronger_failure_ordering)]
|
||||
let success = crate::utils::upgrade_success_ordering(success, failure);
|
||||
self.inner.compare_exchange(current, new, success, failure)
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn compare_exchange_weak(
|
||||
&self,
|
||||
current: $int_type,
|
||||
new: $int_type,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<$int_type, $int_type> {
|
||||
crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)
|
||||
#[cfg(portable_atomic_no_stronger_failure_ordering)]
|
||||
let success = crate::utils::upgrade_success_ordering(success, failure);
|
||||
self.inner.compare_exchange_weak(current, new, success, failure)
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
atomic_int!(AtomicI64, i64);
|
||||
atomic_int!(AtomicU64, u64);
|
||||
});
|
||||
@@ -0,0 +1,39 @@
|
||||
# 128-bit atomic implementations on 64-bit architectures
|
||||
|
||||
(See the [`atomic64` module](../atomic64) for 64-bit atomic implementations on 32-bit architectures.)
|
||||
|
||||
## 128-bit atomic instructions
|
||||
|
||||
Here is the table of targets that support 128-bit atomics and the instructions used:
|
||||
|
||||
| target_arch | load | store | CAS | RMW | note |
|
||||
| ----------- | ---- | ----- | --- | --- | ---- |
|
||||
| x86_64 | cmpxchg16b or vmovdqa | cmpxchg16b or vmovdqa | cmpxchg16b | cmpxchg16b | Requires `cmpxchg16b` target feature (enabled by default on Apple, Windows (except Windows 7, since Rust 1.78), and Fuchsia (since Rust 1.87) targets). vmovdqa requires `avx` target feature. <br> Both compile-time and run-time detection are supported. <br> Requires Rust 1.59+ |
|
||||
| aarch64/arm64ec | ldxp/stxp or casp or ldp/ldiapp | ldxp/stxp or casp or stp/stilp/swpp | ldxp/stxp or casp | ldxp/stxp or casp/swpp/ldclrp/ldsetp | casp requires `lse` target feature, ldp/stp requires `lse2` target feature, ldiapp/stilp requires `lse2` and `rcpc3` target features, swpp/ldclrp/ldsetp requires `lse128` target feature. <br> Both compile-time and run-time detection are supported. <br> Requires Rust 1.59+ (aarch64) / 1.84+ (arm64ec) |
|
||||
| riscv64 | amocas.q | amocas.q | amocas.q | amocas.q | Requires `zacas` target feature. Both compile-time and run-time detection are supported. <br> Requires Rust 1.59+ |
|
||||
| powerpc64 | lq | stq | lqarx/stqcx. | lqarx/stqcx. | Requires `quadword-atomics` target feature (enabled by default on powerpc64le). Both compile-time and run-time detection are supported. <br> Requires Rust 1.95+ |
|
||||
| s390x | lpq | stpq | cdsg | cdsg | Requires Rust 1.84+ |
|
||||
| loongarch64 | sc.q | sc.q | sc.q | sc.q | Unimplemented. Requires `scq` target feature. |
|
||||
| mips64r6 | lldp | lldp/scdp | lldp/scdp | lldp/scdp | Unimplemented (unsupported in LLVM). Requires Release 6 Paired LL/SC family of instructions |
|
||||
| nvptx64 | ld.b128 | st.b128 | atom.cas.b128 | atom.exch.b128/atom.cas.b128 | Unimplemented. Requires `ptx83` and `sm_90`. |
|
||||
|
||||
On compiler versions or platforms where these are not supported, the fallback implementation is used.
|
||||
|
||||
See [aarch64.rs](aarch64.rs) module-level comments for more details on the instructions used on AArch64.
|
||||
|
||||
## Comparison with core::intrinsics::atomic_\* (core::sync::atomic::Atomic{I,U}128)
|
||||
|
||||
This directory has target-specific implementations with inline assembly ([x86_64.rs](x86_64.rs), [aarch64.rs](aarch64.rs), [riscv64.rs](riscv64.rs), [powerpc64.rs](powerpc64.rs), [s390x.rs](s390x.rs)) and an implementation without inline assembly ([intrinsics.rs](intrinsics.rs)). The latter currently always needs nightly compilers and is only used for Miri and ThreadSanitizer, which do not support inline assembly.
|
||||
|
||||
Implementations with inline assembly generate assemblies almost equivalent to the `core::intrinsics::atomic_*` (used in `core::sync::atomic::Atomic{I,U}128`) for many operations, but some operations may or may not generate more efficient code. For example:
|
||||
|
||||
- On x86_64 and AArch64, implementation with inline assembly contains additional optimizations (e.g., [#16](https://github.com/taiki-e/portable-atomic/pull/16), [#126](https://github.com/taiki-e/portable-atomic/pull/126)) and is much faster for some operations.
|
||||
- On AArch64, implementation with inline assembly supports outline-atomics on more operating systems, and may be faster in environments where outline-atomics can improve performance.
|
||||
- On powerpc64, LLVM does not support generating some 128-bit atomic operations (see [intrinsics.rs](intrinsics.rs) module-level comments), and we use CAS loop to implement them, so implementation with inline assembly may be faster for those operations.
|
||||
- In implementations without inline assembly, the compiler may reuse condition flags that have changed as a result of the operation, or use immediate values instead of registers, depending on the situation.
|
||||
|
||||
As 128-bit atomics-related APIs stabilize in the standard library, implementations with inline assembly are planned to be updated to get the benefits of both.
|
||||
|
||||
## Run-time CPU feature detection
|
||||
|
||||
See the [`detect` module's readme](../detect/README.md) for run-time CPU feature detection.
|
||||
+2239
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,508 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
128-bit atomic implementation without inline assembly.
|
||||
|
||||
Adapted from https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs.
|
||||
|
||||
Note: This module is currently only enabled on Miri and ThreadSanitizer which
|
||||
do not support inline assembly.
|
||||
|
||||
This uses `core::arch::x86_64::cmpxchg16b` on x86_64 and
|
||||
`core::intrinsics::atomic_*` on aarch64, powerpc64, and s390x.
|
||||
|
||||
See README.md of this directory for performance comparison with the
|
||||
implementation with inline assembly.
|
||||
|
||||
Note:
|
||||
- This currently needs Rust 1.70 on x86_64, otherwise nightly compilers.
|
||||
- On powerpc64, this requires LLVM 15+ and quadword-atomics target feature:
|
||||
https://github.com/llvm/llvm-project/commit/549e118e93c666914a1045fde38a2cac33e1e445
|
||||
- On s390x, old LLVM (pre-18) generates libcalls for operations other than load/store/cmpxchg:
|
||||
https://github.com/llvm/llvm-project/commit/c568927f3e2e7d9804ea74ecbf11c16c014ddcbc
|
||||
- On aarch64 big-endian, LLVM (as of 17) generates broken code. (wrong result in stress test)
|
||||
(on cfg(miri)/cfg(sanitize) it may be fine though)
|
||||
- On powerpc64, LLVM (as of 17) doesn't support 128-bit atomic min/max:
|
||||
https://github.com/llvm/llvm-project/issues/68390
|
||||
- On powerpc64le, LLVM (as of 17) generates broken code. (wrong result from fetch_add)
|
||||
- On riscv64, LLVM does not automatically use 128-bit atomic instructions even if zacas feature is
|
||||
enabled, because doing it changes the ABI. (If the ability to do that is provided by LLVM in the
|
||||
future, it should probably be controlled by another ABI feature similar to forced-atomics.)
|
||||
*/
|
||||
|
||||
include!("macros.rs");
|
||||
|
||||
#[allow(dead_code)] // we only use compare_exchange.
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
#[cfg(not(target_feature = "cmpxchg16b"))]
|
||||
#[path = "../fallback/outline_atomics.rs"]
|
||||
mod fallback;
|
||||
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
#[cfg(not(target_feature = "cmpxchg16b"))]
|
||||
#[path = "../detect/x86_64.rs"]
|
||||
mod detect;
|
||||
|
||||
#[cfg(not(target_arch = "x86_64"))]
|
||||
use core::intrinsics;
|
||||
use core::sync::atomic::Ordering::{self, AcqRel, Acquire, Relaxed, Release, SeqCst};
|
||||
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
#[inline]
|
||||
fn strongest_failure_ordering(order: Ordering) -> Ordering {
|
||||
match order {
|
||||
Release | Relaxed => Relaxed,
|
||||
SeqCst => SeqCst,
|
||||
Acquire | AcqRel => Acquire,
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 {
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
let fail_order = strongest_failure_ordering(order);
|
||||
match atomic_compare_exchange(src, 0, 0, order, fail_order) {
|
||||
Ok(v) | Err(v) => v,
|
||||
}
|
||||
}
|
||||
#[cfg(not(target_arch = "x86_64"))]
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
match order {
|
||||
Acquire => intrinsics::atomic_load_acquire(src),
|
||||
Relaxed => intrinsics::atomic_load_relaxed(src),
|
||||
SeqCst => intrinsics::atomic_load_seqcst(src),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
atomic_swap(dst, val, order);
|
||||
}
|
||||
#[cfg(not(target_arch = "x86_64"))]
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
match order {
|
||||
Release => intrinsics::atomic_store_release(dst, val),
|
||||
Relaxed => intrinsics::atomic_store_relaxed(dst, val),
|
||||
SeqCst => intrinsics::atomic_store_seqcst(dst, val),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_compare_exchange(
|
||||
dst: *mut u128,
|
||||
old: u128,
|
||||
new: u128,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<u128, u128> {
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
let (val, ok) = {
|
||||
#[target_feature(enable = "cmpxchg16b")]
|
||||
#[cfg_attr(target_feature = "cmpxchg16b", inline)]
|
||||
#[cfg_attr(not(target_feature = "cmpxchg16b"), inline(never))]
|
||||
unsafe fn cmpxchg16b(
|
||||
dst: *mut u128,
|
||||
old: u128,
|
||||
new: u128,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> (u128, bool) {
|
||||
#[cfg(portable_atomic_no_strict_provenance)]
|
||||
use crate::utils::ptr::PtrExt as _;
|
||||
debug_assert!(dst.addr() % 16 == 0);
|
||||
#[cfg(not(target_feature = "cmpxchg16b"))]
|
||||
{
|
||||
debug_assert!(detect::detect().cmpxchg16b());
|
||||
}
|
||||
// SAFETY: the caller must guarantee that `dst` is valid for both writes and
|
||||
// reads, 16-byte aligned (required by CMPXCHG16B), that there are no
|
||||
// concurrent non-atomic operations, and that the CPU supports CMPXCHG16B.
|
||||
let prev = unsafe { core::arch::x86_64::cmpxchg16b(dst, old, new, success, failure) };
|
||||
(prev, prev == old)
|
||||
}
|
||||
#[cfg(target_feature = "cmpxchg16b")]
|
||||
// SAFETY: the caller must guarantee that `dst` is valid for both writes and
|
||||
// reads, 16-byte aligned, that there are no concurrent non-atomic operations,
|
||||
// and cfg guarantees that CMPXCHG16B is available at compile-time.
|
||||
unsafe {
|
||||
cmpxchg16b(dst, old, new, success, failure)
|
||||
}
|
||||
#[cfg(not(target_feature = "cmpxchg16b"))]
|
||||
// SAFETY: the caller must guarantee that `dst` is valid for both writes and
|
||||
// reads, 16-byte aligned, and that there are no different kinds of concurrent accesses.
|
||||
unsafe {
|
||||
ifunc!(unsafe fn(
|
||||
dst: *mut u128, old: u128, new: u128, success: Ordering, failure: Ordering
|
||||
) -> (u128, bool) {
|
||||
if detect::detect().cmpxchg16b() {
|
||||
cmpxchg16b
|
||||
} else {
|
||||
fallback::atomic_compare_exchange
|
||||
}
|
||||
})
|
||||
}
|
||||
};
|
||||
#[cfg(not(target_arch = "x86_64"))]
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
let (val, ok) = unsafe {
|
||||
match (success, failure) {
|
||||
(Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed_relaxed(dst, old, new),
|
||||
(Relaxed, Acquire) => intrinsics::atomic_cxchg_relaxed_acquire(dst, old, new),
|
||||
(Relaxed, SeqCst) => intrinsics::atomic_cxchg_relaxed_seqcst(dst, old, new),
|
||||
(Acquire, Relaxed) => intrinsics::atomic_cxchg_acquire_relaxed(dst, old, new),
|
||||
(Acquire, Acquire) => intrinsics::atomic_cxchg_acquire_acquire(dst, old, new),
|
||||
(Acquire, SeqCst) => intrinsics::atomic_cxchg_acquire_seqcst(dst, old, new),
|
||||
(Release, Relaxed) => intrinsics::atomic_cxchg_release_relaxed(dst, old, new),
|
||||
(Release, Acquire) => intrinsics::atomic_cxchg_release_acquire(dst, old, new),
|
||||
(Release, SeqCst) => intrinsics::atomic_cxchg_release_seqcst(dst, old, new),
|
||||
(AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_relaxed(dst, old, new),
|
||||
(AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel_acquire(dst, old, new),
|
||||
(AcqRel, SeqCst) => intrinsics::atomic_cxchg_acqrel_seqcst(dst, old, new),
|
||||
(SeqCst, Relaxed) => intrinsics::atomic_cxchg_seqcst_relaxed(dst, old, new),
|
||||
(SeqCst, Acquire) => intrinsics::atomic_cxchg_seqcst_acquire(dst, old, new),
|
||||
(SeqCst, SeqCst) => intrinsics::atomic_cxchg_seqcst_seqcst(dst, old, new),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
};
|
||||
if ok { Ok(val) } else { Err(val) }
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
use self::atomic_compare_exchange as atomic_compare_exchange_weak;
|
||||
#[cfg(not(target_arch = "x86_64"))]
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_compare_exchange_weak(
|
||||
dst: *mut u128,
|
||||
old: u128,
|
||||
new: u128,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<u128, u128> {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
let (val, ok) = unsafe {
|
||||
match (success, failure) {
|
||||
(Relaxed, Relaxed) => intrinsics::atomic_cxchgweak_relaxed_relaxed(dst, old, new),
|
||||
(Relaxed, Acquire) => intrinsics::atomic_cxchgweak_relaxed_acquire(dst, old, new),
|
||||
(Relaxed, SeqCst) => intrinsics::atomic_cxchgweak_relaxed_seqcst(dst, old, new),
|
||||
(Acquire, Relaxed) => intrinsics::atomic_cxchgweak_acquire_relaxed(dst, old, new),
|
||||
(Acquire, Acquire) => intrinsics::atomic_cxchgweak_acquire_acquire(dst, old, new),
|
||||
(Acquire, SeqCst) => intrinsics::atomic_cxchgweak_acquire_seqcst(dst, old, new),
|
||||
(Release, Relaxed) => intrinsics::atomic_cxchgweak_release_relaxed(dst, old, new),
|
||||
(Release, Acquire) => intrinsics::atomic_cxchgweak_release_acquire(dst, old, new),
|
||||
(Release, SeqCst) => intrinsics::atomic_cxchgweak_release_seqcst(dst, old, new),
|
||||
(AcqRel, Relaxed) => intrinsics::atomic_cxchgweak_acqrel_relaxed(dst, old, new),
|
||||
(AcqRel, Acquire) => intrinsics::atomic_cxchgweak_acqrel_acquire(dst, old, new),
|
||||
(AcqRel, SeqCst) => intrinsics::atomic_cxchgweak_acqrel_seqcst(dst, old, new),
|
||||
(SeqCst, Relaxed) => intrinsics::atomic_cxchgweak_seqcst_relaxed(dst, old, new),
|
||||
(SeqCst, Acquire) => intrinsics::atomic_cxchgweak_seqcst_acquire(dst, old, new),
|
||||
(SeqCst, SeqCst) => intrinsics::atomic_cxchgweak_seqcst_seqcst(dst, old, new),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
};
|
||||
if ok { Ok(val) } else { Err(val) }
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_update<F>(dst: *mut u128, order: Ordering, mut f: F) -> u128
|
||||
where
|
||||
F: FnMut(u128) -> u128,
|
||||
{
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
// This is a private function and all instances of `f` only operate on the value
|
||||
// loaded, so there is no need to synchronize the first load/failed CAS.
|
||||
let mut prev = atomic_load(dst, Ordering::Relaxed);
|
||||
loop {
|
||||
let next = f(prev);
|
||||
match atomic_compare_exchange_weak(dst, prev, next, order, Ordering::Relaxed) {
|
||||
Ok(x) => return x,
|
||||
Err(x) => prev = x,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// On x86_64, we use core::arch::x86_64::cmpxchg16b instead of core::intrinsics.
|
||||
// - On s390x, old LLVM (pre-18) generates libcalls for operations other than load/store/cmpxchg (see also module-level comment).
|
||||
#[cfg(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18)))]
|
||||
atomic_rmw_by_atomic_update!();
|
||||
// On powerpc64, LLVM doesn't support 128-bit atomic min/max (see also module-level comment).
|
||||
#[cfg(target_arch = "powerpc64")]
|
||||
atomic_rmw_by_atomic_update!(cmp);
|
||||
|
||||
#[cfg(not(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18))))]
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_swap(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
match order {
|
||||
Acquire => intrinsics::atomic_xchg_acquire(dst, val),
|
||||
Release => intrinsics::atomic_xchg_release(dst, val),
|
||||
AcqRel => intrinsics::atomic_xchg_acqrel(dst, val),
|
||||
Relaxed => intrinsics::atomic_xchg_relaxed(dst, val),
|
||||
SeqCst => intrinsics::atomic_xchg_seqcst(dst, val),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18))))]
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_add(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
match order {
|
||||
Acquire => intrinsics::atomic_xadd_acquire(dst, val),
|
||||
Release => intrinsics::atomic_xadd_release(dst, val),
|
||||
AcqRel => intrinsics::atomic_xadd_acqrel(dst, val),
|
||||
Relaxed => intrinsics::atomic_xadd_relaxed(dst, val),
|
||||
SeqCst => intrinsics::atomic_xadd_seqcst(dst, val),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18))))]
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_sub(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
match order {
|
||||
Acquire => intrinsics::atomic_xsub_acquire(dst, val),
|
||||
Release => intrinsics::atomic_xsub_release(dst, val),
|
||||
AcqRel => intrinsics::atomic_xsub_acqrel(dst, val),
|
||||
Relaxed => intrinsics::atomic_xsub_relaxed(dst, val),
|
||||
SeqCst => intrinsics::atomic_xsub_seqcst(dst, val),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18))))]
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_and(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
match order {
|
||||
Acquire => intrinsics::atomic_and_acquire(dst, val),
|
||||
Release => intrinsics::atomic_and_release(dst, val),
|
||||
AcqRel => intrinsics::atomic_and_acqrel(dst, val),
|
||||
Relaxed => intrinsics::atomic_and_relaxed(dst, val),
|
||||
SeqCst => intrinsics::atomic_and_seqcst(dst, val),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18))))]
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_nand(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
match order {
|
||||
Acquire => intrinsics::atomic_nand_acquire(dst, val),
|
||||
Release => intrinsics::atomic_nand_release(dst, val),
|
||||
AcqRel => intrinsics::atomic_nand_acqrel(dst, val),
|
||||
Relaxed => intrinsics::atomic_nand_relaxed(dst, val),
|
||||
SeqCst => intrinsics::atomic_nand_seqcst(dst, val),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18))))]
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_or(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
match order {
|
||||
Acquire => intrinsics::atomic_or_acquire(dst, val),
|
||||
Release => intrinsics::atomic_or_release(dst, val),
|
||||
AcqRel => intrinsics::atomic_or_acqrel(dst, val),
|
||||
Relaxed => intrinsics::atomic_or_relaxed(dst, val),
|
||||
SeqCst => intrinsics::atomic_or_seqcst(dst, val),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18))))]
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_xor(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
match order {
|
||||
Acquire => intrinsics::atomic_xor_acquire(dst, val),
|
||||
Release => intrinsics::atomic_xor_release(dst, val),
|
||||
AcqRel => intrinsics::atomic_xor_acqrel(dst, val),
|
||||
Relaxed => intrinsics::atomic_xor_relaxed(dst, val),
|
||||
SeqCst => intrinsics::atomic_xor_seqcst(dst, val),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(any(
|
||||
target_arch = "x86_64",
|
||||
target_arch = "powerpc64",
|
||||
all(target_arch = "s390x", portable_atomic_pre_llvm_18),
|
||||
)))]
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_max(dst: *mut u128, val: u128, order: Ordering) -> i128 {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
match order {
|
||||
Acquire => intrinsics::atomic_max_acquire(dst.cast::<i128>(), val as i128),
|
||||
Release => intrinsics::atomic_max_release(dst.cast::<i128>(), val as i128),
|
||||
AcqRel => intrinsics::atomic_max_acqrel(dst.cast::<i128>(), val as i128),
|
||||
Relaxed => intrinsics::atomic_max_relaxed(dst.cast::<i128>(), val as i128),
|
||||
SeqCst => intrinsics::atomic_max_seqcst(dst.cast::<i128>(), val as i128),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(any(
|
||||
target_arch = "x86_64",
|
||||
target_arch = "powerpc64",
|
||||
all(target_arch = "s390x", portable_atomic_pre_llvm_18),
|
||||
)))]
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_min(dst: *mut u128, val: u128, order: Ordering) -> i128 {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
match order {
|
||||
Acquire => intrinsics::atomic_min_acquire(dst.cast::<i128>(), val as i128),
|
||||
Release => intrinsics::atomic_min_release(dst.cast::<i128>(), val as i128),
|
||||
AcqRel => intrinsics::atomic_min_acqrel(dst.cast::<i128>(), val as i128),
|
||||
Relaxed => intrinsics::atomic_min_relaxed(dst.cast::<i128>(), val as i128),
|
||||
SeqCst => intrinsics::atomic_min_seqcst(dst.cast::<i128>(), val as i128),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(any(
|
||||
target_arch = "x86_64",
|
||||
target_arch = "powerpc64",
|
||||
all(target_arch = "s390x", portable_atomic_pre_llvm_18),
|
||||
)))]
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_umax(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
match order {
|
||||
Acquire => intrinsics::atomic_umax_acquire(dst, val),
|
||||
Release => intrinsics::atomic_umax_release(dst, val),
|
||||
AcqRel => intrinsics::atomic_umax_acqrel(dst, val),
|
||||
Relaxed => intrinsics::atomic_umax_relaxed(dst, val),
|
||||
SeqCst => intrinsics::atomic_umax_seqcst(dst, val),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(any(
|
||||
target_arch = "x86_64",
|
||||
target_arch = "powerpc64",
|
||||
all(target_arch = "s390x", portable_atomic_pre_llvm_18),
|
||||
)))]
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_umin(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
match order {
|
||||
Acquire => intrinsics::atomic_umin_acquire(dst, val),
|
||||
Release => intrinsics::atomic_umin_release(dst, val),
|
||||
AcqRel => intrinsics::atomic_umin_acqrel(dst, val),
|
||||
Relaxed => intrinsics::atomic_umin_relaxed(dst, val),
|
||||
SeqCst => intrinsics::atomic_umin_seqcst(dst, val),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18))))]
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_not(dst: *mut u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe { atomic_xor(dst, !0, order) }
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_arch = "x86_64", all(target_arch = "s390x", portable_atomic_pre_llvm_18))))]
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_neg(dst: *mut u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe { atomic_update(dst, order, u128::wrapping_neg) }
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "x86_64"))]
|
||||
#[inline]
|
||||
const fn is_lock_free() -> bool {
|
||||
IS_ALWAYS_LOCK_FREE
|
||||
}
|
||||
#[cfg(not(target_arch = "x86_64"))]
|
||||
const IS_ALWAYS_LOCK_FREE: bool = true;
|
||||
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
#[inline]
|
||||
fn is_lock_free() -> bool {
|
||||
#[cfg(target_feature = "cmpxchg16b")]
|
||||
{
|
||||
// CMPXCHG16B is available at compile-time.
|
||||
true
|
||||
}
|
||||
#[cfg(not(target_feature = "cmpxchg16b"))]
|
||||
{
|
||||
detect::detect().cmpxchg16b()
|
||||
}
|
||||
}
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
const IS_ALWAYS_LOCK_FREE: bool = cfg!(target_feature = "cmpxchg16b");
|
||||
|
||||
atomic128!(AtomicI128, i128, atomic_max, atomic_min);
|
||||
atomic128!(AtomicU128, u128, atomic_umax, atomic_umin);
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
test_atomic_int!(i128);
|
||||
test_atomic_int!(u128);
|
||||
|
||||
// load/store/swap implementation is not affected by signedness, so it is
|
||||
// enough to test only unsigned types.
|
||||
stress_test!(u128);
|
||||
}
|
||||
+349
@@ -0,0 +1,349 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
macro_rules! atomic128 {
|
||||
($atomic_type:ident, $int_type:ident, $atomic_max:ident, $atomic_min:ident) => {
|
||||
#[repr(C, align(16))]
|
||||
pub(crate) struct $atomic_type {
|
||||
v: core::cell::UnsafeCell<$int_type>,
|
||||
}
|
||||
|
||||
// Send is implicitly implemented.
|
||||
// SAFETY: any data races are prevented by atomic intrinsics.
|
||||
unsafe impl Sync for $atomic_type {}
|
||||
|
||||
impl_default_no_fetch_ops!($atomic_type, $int_type);
|
||||
impl_default_bit_opts!($atomic_type, $int_type);
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
pub(crate) const fn new(v: $int_type) -> Self {
|
||||
Self { v: core::cell::UnsafeCell::new(v) }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn is_lock_free() -> bool {
|
||||
is_lock_free()
|
||||
}
|
||||
pub(crate) const IS_ALWAYS_LOCK_FREE: bool = IS_ALWAYS_LOCK_FREE;
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn load(&self, order: Ordering) -> $int_type {
|
||||
crate::utils::assert_load_ordering(order);
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_load(self.v.get().cast::<u128>(), order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn store(&self, val: $int_type, order: Ordering) {
|
||||
crate::utils::assert_store_ordering(order);
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_store(self.v.get().cast::<u128>(), val as u128, order)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn swap(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_swap(self.v.get().cast::<u128>(), val as u128, order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn compare_exchange(
|
||||
&self,
|
||||
current: $int_type,
|
||||
new: $int_type,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<$int_type, $int_type> {
|
||||
crate::utils::assert_compare_exchange_ordering(success, failure);
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
match atomic_compare_exchange(
|
||||
self.v.get().cast::<u128>(),
|
||||
current as u128,
|
||||
new as u128,
|
||||
success,
|
||||
failure,
|
||||
) {
|
||||
Ok(v) => Ok(v as $int_type),
|
||||
Err(v) => Err(v as $int_type),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn compare_exchange_weak(
|
||||
&self,
|
||||
current: $int_type,
|
||||
new: $int_type,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<$int_type, $int_type> {
|
||||
crate::utils::assert_compare_exchange_ordering(success, failure);
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
match atomic_compare_exchange_weak(
|
||||
self.v.get().cast::<u128>(),
|
||||
current as u128,
|
||||
new as u128,
|
||||
success,
|
||||
failure,
|
||||
) {
|
||||
Ok(v) => Ok(v as $int_type),
|
||||
Err(v) => Err(v as $int_type),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_add(self.v.get().cast::<u128>(), val as u128, order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_sub(self.v.get().cast::<u128>(), val as u128, order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_and(self.v.get().cast::<u128>(), val as u128, order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_nand(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_nand(self.v.get().cast::<u128>(), val as u128, order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_or(self.v.get().cast::<u128>(), val as u128, order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_xor(self.v.get().cast::<u128>(), val as u128, order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
$atomic_max(self.v.get().cast::<u128>(), val as u128, order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
$atomic_min(self.v.get().cast::<u128>(), val as u128, order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_not(&self, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_not(self.v.get().cast::<u128>(), order) as $int_type
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn not(&self, order: Ordering) {
|
||||
self.fetch_not(order);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_neg(&self, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_neg(self.v.get().cast::<u128>(), order) as $int_type
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn neg(&self, order: Ordering) {
|
||||
self.fetch_neg(order);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) const fn as_ptr(&self) -> *mut $int_type {
|
||||
self.v.get()
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(any(target_arch = "powerpc64", target_arch = "s390x", target_arch = "x86_64"))]
|
||||
#[allow(unused_macros)] // also used by intrinsics.rs
|
||||
macro_rules! atomic_rmw_by_atomic_update {
|
||||
() => {
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_swap(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe { atomic_update(dst, order, |_| val) }
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_add(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe { atomic_update(dst, order, |x| x.wrapping_add(val)) }
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_sub(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe { atomic_update(dst, order, |x| x.wrapping_sub(val)) }
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_and(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe { atomic_update(dst, order, |x| x & val) }
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_nand(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe { atomic_update(dst, order, |x| !(x & val)) }
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_or(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe { atomic_update(dst, order, |x| x | val) }
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_xor(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe { atomic_update(dst, order, |x| x ^ val) }
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_not(dst: *mut u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe { atomic_update(dst, order, |x| !x) }
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_neg(dst: *mut u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe { atomic_update(dst, order, u128::wrapping_neg) }
|
||||
}
|
||||
atomic_rmw_by_atomic_update!(cmp);
|
||||
};
|
||||
(cmp) => {
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_max(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
atomic_update(dst, order, |x| core::cmp::max(x as i128, val as i128) as u128)
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_umax(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe { atomic_update(dst, order, |x| core::cmp::max(x, val)) }
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_min(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
atomic_update(dst, order, |x| core::cmp::min(x as i128, val as i128) as u128)
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
unsafe fn atomic_umin(dst: *mut u128, val: u128, order: Ordering) -> u128 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe { atomic_update(dst, order, |x| core::cmp::min(x, val)) }
|
||||
}
|
||||
};
|
||||
}
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
128-bit atomic implementations on 64-bit architectures
|
||||
|
||||
See README.md for details.
|
||||
*/
|
||||
|
||||
// AArch64
|
||||
#[cfg(any(
|
||||
all(
|
||||
target_arch = "aarch64",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
),
|
||||
all(
|
||||
target_arch = "arm64ec",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
not(portable_atomic_no_asm),
|
||||
),
|
||||
))]
|
||||
// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.
|
||||
#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = "intrinsics.rs")]
|
||||
pub(super) mod aarch64;
|
||||
|
||||
// powerpc64
|
||||
#[cfg(all(
|
||||
target_arch = "powerpc64",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
not(portable_atomic_no_asm),
|
||||
any(
|
||||
target_feature = "quadword-atomics",
|
||||
portable_atomic_target_feature = "quadword-atomics",
|
||||
all(
|
||||
feature = "fallback",
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
any(
|
||||
all(
|
||||
target_os = "linux",
|
||||
any(
|
||||
all(
|
||||
target_env = "gnu",
|
||||
any(target_endian = "little", not(target_feature = "crt-static")),
|
||||
),
|
||||
all(
|
||||
target_env = "musl",
|
||||
any(not(target_feature = "crt-static"), feature = "std"),
|
||||
),
|
||||
target_env = "ohos",
|
||||
all(target_env = "uclibc", not(target_feature = "crt-static")),
|
||||
portable_atomic_outline_atomics,
|
||||
),
|
||||
),
|
||||
target_os = "android",
|
||||
all(
|
||||
target_os = "freebsd",
|
||||
any(
|
||||
target_endian = "little",
|
||||
not(target_feature = "crt-static"),
|
||||
portable_atomic_outline_atomics,
|
||||
),
|
||||
),
|
||||
target_os = "openbsd",
|
||||
all(
|
||||
target_os = "aix",
|
||||
not(portable_atomic_pre_llvm_20),
|
||||
any(test, portable_atomic_outline_atomics), // TODO(aix): currently disabled by default
|
||||
),
|
||||
),
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
),
|
||||
),
|
||||
))]
|
||||
// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.
|
||||
#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = "intrinsics.rs")]
|
||||
pub(super) mod powerpc64;
|
||||
|
||||
// riscv64
|
||||
#[cfg(all(
|
||||
target_arch = "riscv64",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(
|
||||
target_feature = "zacas",
|
||||
portable_atomic_target_feature = "zacas",
|
||||
all(
|
||||
feature = "fallback",
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
any(target_os = "linux", target_os = "android"),
|
||||
),
|
||||
),
|
||||
))]
|
||||
pub(super) mod riscv64;
|
||||
|
||||
// s390x
|
||||
#[cfg(all(
|
||||
target_arch = "s390x",
|
||||
not(all(any(miri, portable_atomic_sanitize_thread), not(portable_atomic_atomic_intrinsics))),
|
||||
not(portable_atomic_no_asm),
|
||||
))]
|
||||
// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.
|
||||
#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = "intrinsics.rs")]
|
||||
pub(super) mod s390x;
|
||||
|
||||
// x86_64
|
||||
#[cfg(all(
|
||||
target_arch = "x86_64",
|
||||
not(all(any(miri, portable_atomic_sanitize_thread), portable_atomic_no_cmpxchg16b_intrinsic)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(
|
||||
target_feature = "cmpxchg16b",
|
||||
portable_atomic_target_feature = "cmpxchg16b",
|
||||
all(
|
||||
feature = "fallback",
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
not(any(target_env = "sgx", miri)),
|
||||
),
|
||||
),
|
||||
))]
|
||||
// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.
|
||||
#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = "intrinsics.rs")]
|
||||
pub(super) mod x86_64;
|
||||
+1046
File diff suppressed because it is too large.
Load diff
+624
@@ -0,0 +1,624 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
128-bit atomic implementation on riscv64.
|
||||
|
||||
This architecture provides the following 128-bit atomic instructions:
|
||||
|
||||
- amocas.q: CAS (Zacas extension)
|
||||
|
||||
See "Atomic operation overview by architecture" in atomic-maybe-uninit for a more comprehensive and
|
||||
detailed description of the atomic and synchronize instructions in this architecture:
|
||||
https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#risc-v
|
||||
|
||||
Note: On Miri and ThreadSanitizer which do not support inline assembly, we don't use
|
||||
this module and use fallback implementation instead.
|
||||
|
||||
Refs:
|
||||
- RISC-V Instruction Set Manual
|
||||
"Zacas" Extension for Atomic Compare-and-Swap (CAS) Instructions
|
||||
https://docs.riscv.org/reference/isa/v20260120/unpriv/zacas.html
|
||||
- RISC-V Atomics ABI Specification
|
||||
https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/draft-20250812-301374e92976e298e676e7129a6212926b2299ce/riscv-atomic.adoc
|
||||
|
||||
See tests/asm-test/asm/portable-atomic for generated assembly.
|
||||
*/
|
||||
|
||||
include!("macros.rs");
|
||||
|
||||
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
|
||||
#[path = "../fallback/outline_atomics.rs"]
|
||||
mod fallback;
|
||||
|
||||
#[cfg(not(portable_atomic_no_outline_atomics))]
|
||||
#[cfg(any(test, not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))))]
|
||||
#[cfg(any(target_os = "linux", target_os = "android"))]
|
||||
#[path = "../detect/riscv_linux.rs"]
|
||||
mod detect;
|
||||
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
use core::arch::asm;
|
||||
use core::sync::atomic::Ordering;
|
||||
|
||||
#[cfg(portable_atomic_no_strict_provenance)]
|
||||
use crate::utils::ptr::PtrExt as _;
|
||||
use crate::utils::{Pair, U128};
|
||||
|
||||
macro_rules! debug_assert_zacas {
|
||||
() => {
|
||||
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
|
||||
{
|
||||
debug_assert!(detect::detect().zacas());
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// `.option arch, +zacas` directive requires LLVM 20, so we use .4byte directive for old LLVM.
|
||||
// Note that `.insn <value>` directive requires LLVM 19.
|
||||
// https://github.com/llvm/llvm-project/commit/2a086dce691e3cc34a2fc27f4fb255bb2cbbfac9
|
||||
// https://github.com/riscv-non-isa/riscv-asm-manual/blob/v0.0.1/src/asm-manual.adoc#arch
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
macro_rules! start_zacas {
|
||||
() => {
|
||||
".option push\n.option arch, +zacas"
|
||||
};
|
||||
}
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
macro_rules! end_zacas {
|
||||
() => {
|
||||
".option pop"
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
macro_rules! atomic_rmw_amocas_order {
|
||||
($op:ident, $order:ident) => {
|
||||
atomic_rmw_amocas_order!($op, $order, failure = $order)
|
||||
};
|
||||
($op:ident, $order:ident, failure = $failure:ident) => {
|
||||
match $order {
|
||||
Ordering::Relaxed => $op!("", ""),
|
||||
Ordering::Acquire => $op!("", ".aq"),
|
||||
Ordering::Release => $op!("", ".rl"),
|
||||
Ordering::AcqRel => $op!("", ".aqrl"),
|
||||
Ordering::SeqCst if $failure == Ordering::SeqCst => $op!("fence rw,rw", ".aqrl"),
|
||||
Ordering::SeqCst => $op!("", ".aqrl"),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
};
|
||||
}
|
||||
#[cfg(portable_atomic_pre_llvm_20)]
|
||||
macro_rules! atomic_rmw_amocas_order_insn {
|
||||
($op:ident, $order:ident) => {
|
||||
atomic_rmw_amocas_order_insn!($op, $order, failure = $order)
|
||||
};
|
||||
($op:ident, $order:ident, failure = $failure:ident) => {
|
||||
match $order {
|
||||
Ordering::Relaxed => $op!("", "8"),
|
||||
Ordering::Acquire => $op!("", "c"),
|
||||
Ordering::Release => $op!("", "a"),
|
||||
Ordering::AcqRel => $op!("", "e"),
|
||||
Ordering::SeqCst if $failure == Ordering::SeqCst => $op!("fence rw,rw", "e"),
|
||||
Ordering::SeqCst => $op!("", "e"),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// If zacas is available at compile-time, we can always use zacas_fn.
|
||||
#[cfg(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))]
|
||||
use self::atomic_load_zacas as atomic_load;
|
||||
// Otherwise, we need to do run-time detection and can use zacas_fn only if zacas is available.
|
||||
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
|
||||
#[inline]
|
||||
unsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 {
|
||||
fn_alias! {
|
||||
// inline(never) is just a hint and also not strictly necessary
|
||||
// because we use ifunc helper macro, but used for clarity.
|
||||
#[inline(never)]
|
||||
unsafe fn(src: *mut u128) -> u128;
|
||||
atomic_load_zacas_relaxed = atomic_load_zacas(Ordering::Relaxed);
|
||||
atomic_load_zacas_acquire = atomic_load_zacas(Ordering::Acquire);
|
||||
atomic_load_zacas_seqcst = atomic_load_zacas(Ordering::SeqCst);
|
||||
}
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
// we only calls atomic_load_zacas if zacas is available.
|
||||
unsafe {
|
||||
match order {
|
||||
Ordering::Relaxed => {
|
||||
ifunc!(unsafe fn(src: *mut u128) -> u128 {
|
||||
if detect::detect().zacas() {
|
||||
atomic_load_zacas_relaxed
|
||||
} else {
|
||||
fallback::atomic_load_non_seqcst
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::Acquire => {
|
||||
ifunc!(unsafe fn(src: *mut u128) -> u128 {
|
||||
if detect::detect().zacas() {
|
||||
atomic_load_zacas_acquire
|
||||
} else {
|
||||
fallback::atomic_load_non_seqcst
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::SeqCst => {
|
||||
ifunc!(unsafe fn(src: *mut u128) -> u128 {
|
||||
if detect::detect().zacas() {
|
||||
atomic_load_zacas_seqcst
|
||||
} else {
|
||||
fallback::atomic_load_seqcst
|
||||
}
|
||||
})
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
unsafe fn atomic_load_zacas(src: *mut u128, order: Ordering) -> u128 {
|
||||
debug_assert!(src.addr() % 16 == 0);
|
||||
debug_assert_zacas!();
|
||||
let (out_lo, out_hi);
|
||||
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
macro_rules! load {
|
||||
($fence:tt, $asm_order:tt) => {
|
||||
asm!(
|
||||
start_zacas!(),
|
||||
$fence, // fence
|
||||
concat!("amocas.q", $asm_order, " a2, a2, 0({src})"), // atomic { if *dst == a2:a3 { *dst = a2:a3 } else { a2:a3 = *dst } }
|
||||
end_zacas!(),
|
||||
src = in(reg) ptr_reg!(src),
|
||||
inout("a2") 0_u64 => out_lo,
|
||||
inout("a3") 0_u64 => out_hi,
|
||||
options(nostack, preserves_flags),
|
||||
)
|
||||
};
|
||||
}
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
atomic_rmw_amocas_order!(load, order);
|
||||
#[cfg(portable_atomic_pre_llvm_20)]
|
||||
macro_rules! load {
|
||||
($fence:tt, $insn_order:tt) => {
|
||||
asm!(
|
||||
$fence, // fence
|
||||
// amocas.q{,.aq,.rl,.aqrl} a2, a2, (a0) // atomic { if *a0 == a2:a3 { *a0 = a2:a3 } else { a2:a3 = *a0 } }
|
||||
concat!(".4byte 0x2", $insn_order, "c5462f"),
|
||||
in("a0") ptr_reg!(src),
|
||||
inout("a2") 0_u64 => out_lo,
|
||||
inout("a3") 0_u64 => out_hi,
|
||||
options(nostack, preserves_flags),
|
||||
)
|
||||
};
|
||||
}
|
||||
#[cfg(portable_atomic_pre_llvm_20)]
|
||||
atomic_rmw_amocas_order_insn!(load, order);
|
||||
U128 { pair: Pair { lo: out_lo, hi: out_hi } }.whole
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
atomic_swap(dst, val, order);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn atomic_compare_exchange(
|
||||
dst: *mut u128,
|
||||
old: u128,
|
||||
new: u128,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<u128, u128> {
|
||||
#[cfg(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))]
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
// cfg guarantees that zacas instructions are available at compile-time.
|
||||
let (prev, ok) = unsafe { atomic_compare_exchange_zacas(dst, old, new, success, failure) };
|
||||
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
|
||||
let (prev, ok) = {
|
||||
fn_alias! {
|
||||
// inline(never) is just a hint and also not strictly necessary
|
||||
// because we use ifunc helper macro, but used for clarity.
|
||||
#[inline(never)]
|
||||
unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool);
|
||||
zacas_relaxed_fn = atomic_compare_exchange_zacas(Ordering::Relaxed, Ordering::Relaxed);
|
||||
zacas_acquire_fn = atomic_compare_exchange_zacas(Ordering::Acquire, Ordering::Acquire);
|
||||
zacas_release_fn = atomic_compare_exchange_zacas(Ordering::Release, Ordering::Relaxed);
|
||||
zacas_acqrel_fn = atomic_compare_exchange_zacas(Ordering::AcqRel, Ordering::Acquire);
|
||||
zacas_seqcst_fn = atomic_compare_exchange_zacas(Ordering::SeqCst, Ordering::SeqCst);
|
||||
}
|
||||
let order = crate::utils::upgrade_success_ordering(success, failure);
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
// we only calls atomic_compare_exchange_zacas if zacas is available.
|
||||
unsafe {
|
||||
match order {
|
||||
Ordering::Relaxed => {
|
||||
ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {
|
||||
if detect::detect().zacas() {
|
||||
zacas_relaxed_fn
|
||||
} else {
|
||||
fallback::atomic_compare_exchange_non_seqcst
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::Acquire => {
|
||||
ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {
|
||||
if detect::detect().zacas() {
|
||||
zacas_acquire_fn
|
||||
} else {
|
||||
fallback::atomic_compare_exchange_non_seqcst
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::Release => {
|
||||
ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {
|
||||
if detect::detect().zacas() {
|
||||
zacas_release_fn
|
||||
} else {
|
||||
fallback::atomic_compare_exchange_non_seqcst
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::AcqRel => {
|
||||
ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {
|
||||
if detect::detect().zacas() {
|
||||
zacas_acqrel_fn
|
||||
} else {
|
||||
fallback::atomic_compare_exchange_non_seqcst
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::SeqCst => {
|
||||
ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {
|
||||
if detect::detect().zacas() {
|
||||
zacas_seqcst_fn
|
||||
} else {
|
||||
fallback::atomic_compare_exchange_seqcst
|
||||
}
|
||||
})
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
};
|
||||
if ok { Ok(prev) } else { Err(prev) }
|
||||
}
|
||||
#[inline]
|
||||
unsafe fn atomic_compare_exchange_zacas(
|
||||
dst: *mut u128,
|
||||
old: u128,
|
||||
new: u128,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> (u128, bool) {
|
||||
debug_assert!(dst.addr() % 16 == 0);
|
||||
debug_assert_zacas!();
|
||||
let order = crate::utils::upgrade_success_ordering(success, failure);
|
||||
let old = U128 { whole: old };
|
||||
let new = U128 { whole: new };
|
||||
let (prev_lo, prev_hi);
|
||||
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
macro_rules! cmpxchg {
|
||||
($fence:tt, $asm_order:tt) => {
|
||||
asm!(
|
||||
start_zacas!(),
|
||||
$fence, // fence
|
||||
concat!("amocas.q", $asm_order, " a4, a2, 0({dst})"), // atomic { if *dst == a4:a5 { *dst = a2:a3 } else { a4:a5 = *dst } }
|
||||
end_zacas!(),
|
||||
dst = in(reg) ptr_reg!(dst),
|
||||
// must be allocated to even/odd register pair
|
||||
inout("a4") old.pair.lo => prev_lo,
|
||||
inout("a5") old.pair.hi => prev_hi,
|
||||
// must be allocated to even/odd register pair
|
||||
in("a2") new.pair.lo,
|
||||
in("a3") new.pair.hi,
|
||||
options(nostack, preserves_flags),
|
||||
)
|
||||
};
|
||||
}
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
atomic_rmw_amocas_order!(cmpxchg, order, failure = failure);
|
||||
#[cfg(portable_atomic_pre_llvm_20)]
|
||||
macro_rules! cmpxchg {
|
||||
($fence:tt, $insn_order:tt) => {
|
||||
asm!(
|
||||
$fence, // fence
|
||||
// amocas.q{,.aq,.rl,.aqrl} a4, a2, (a0) // atomic { if *a0 == a4:a5 { *a0 = a2:a3 } else { a4:a5 = *a0 } }
|
||||
concat!(".4byte 0x2", $insn_order, "c5472f"),
|
||||
in("a0") ptr_reg!(dst),
|
||||
// must be allocated to even/odd register pair
|
||||
inout("a4") old.pair.lo => prev_lo,
|
||||
inout("a5") old.pair.hi => prev_hi,
|
||||
// must be allocated to even/odd register pair
|
||||
in("a2") new.pair.lo,
|
||||
in("a3") new.pair.hi,
|
||||
options(nostack, preserves_flags),
|
||||
)
|
||||
};
|
||||
}
|
||||
#[cfg(portable_atomic_pre_llvm_20)]
|
||||
atomic_rmw_amocas_order_insn!(cmpxchg, order, failure = failure);
|
||||
let prev = U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole;
|
||||
(prev, prev == old.whole)
|
||||
}
|
||||
}
|
||||
|
||||
// amocas is always strong.
|
||||
use self::atomic_compare_exchange as atomic_compare_exchange_weak;
|
||||
|
||||
// 128-bit atomic load by two 64-bit atomic loads.
|
||||
#[inline]
|
||||
unsafe fn byte_wise_atomic_load(src: *const u128) -> u128 {
|
||||
let (out_lo, out_hi);
|
||||
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
asm!(
|
||||
"ld {out_lo}, ({src})", // atomic { out_lo = *src }
|
||||
"ld {out_hi}, 8({src})", // atomic { out_hi = *src.byte_add(8) }
|
||||
src = in(reg) ptr_reg!(src),
|
||||
out_lo = out(reg) out_lo,
|
||||
out_hi = out(reg) out_hi,
|
||||
options(pure, nostack, preserves_flags, readonly),
|
||||
);
|
||||
U128 { pair: Pair { lo: out_lo, hi: out_hi } }.whole
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! select_atomic_rmw {
|
||||
(
|
||||
unsafe fn $name:ident(dst: *mut u128 $(, $($arg:tt)*)?) $(-> $ret_ty:ty)? {
|
||||
|$zacas_fn_binding:ident| $($zacas_fn_body:tt)*
|
||||
}
|
||||
zacas = $zacas_fn:ident;
|
||||
non_seqcst_fallback = $non_seqcst_fallback_fn:ident;
|
||||
seqcst_fallback = $seqcst_fallback_fn:ident;
|
||||
) => {
|
||||
#[inline]
|
||||
unsafe fn $zacas_fn(dst: *mut u128 $(, $($arg)*)?, order: Ordering) $(-> $ret_ty)? {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
// This is not single-copy atomic reads, but this is ok because subsequent
|
||||
// CAS will check for consistency.
|
||||
//
|
||||
// Note that the C++20 memory model does not allow mixed-sized atomic access,
|
||||
// so we must use inline assembly to implement byte_wise_atomic_load.
|
||||
// (i.e., byte-wise atomic based on the standard library's atomic types
|
||||
// cannot be used here).
|
||||
let mut prev = byte_wise_atomic_load(dst);
|
||||
loop {
|
||||
let next = {
|
||||
let $zacas_fn_binding = prev;
|
||||
$($zacas_fn_body)*
|
||||
};
|
||||
match atomic_compare_exchange_weak(dst, prev, next, order, Ordering::Relaxed) {
|
||||
Ok(x) => return x,
|
||||
Err(x) => prev = x,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// If zacas is available at compile-time, we can always use zacas_fn.
|
||||
#[cfg(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))]
|
||||
use self::$zacas_fn as $name;
|
||||
// Otherwise, we need to do run-time detection and can use zacas_fn only if zacas is available.
|
||||
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
|
||||
#[inline]
|
||||
unsafe fn $name(dst: *mut u128 $(, $($arg)*)?, order: Ordering) $(-> $ret_ty)? {
|
||||
fn_alias! {
|
||||
// inline(never) is just a hint and also not strictly necessary
|
||||
// because we use ifunc helper macro, but used for clarity.
|
||||
#[inline(never)]
|
||||
unsafe fn(dst: *mut u128 $(, $($arg)*)?) $(-> $ret_ty)?;
|
||||
zacas_relaxed_fn = $zacas_fn(Ordering::Relaxed);
|
||||
zacas_acquire_fn = $zacas_fn(Ordering::Acquire);
|
||||
zacas_release_fn = $zacas_fn(Ordering::Release);
|
||||
zacas_acqrel_fn = $zacas_fn(Ordering::AcqRel);
|
||||
zacas_seqcst_fn = $zacas_fn(Ordering::SeqCst);
|
||||
}
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
// we only calls zacas_fn if zacas is available.
|
||||
unsafe {
|
||||
match order {
|
||||
Ordering::Relaxed => {
|
||||
ifunc!(unsafe fn(dst: *mut u128 $(, $($arg)*)?) $(-> $ret_ty)? {
|
||||
if detect::detect().zacas() {
|
||||
zacas_relaxed_fn
|
||||
} else {
|
||||
fallback::$non_seqcst_fallback_fn
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::Acquire => {
|
||||
ifunc!(unsafe fn(dst: *mut u128 $(, $($arg)*)?) $(-> $ret_ty)? {
|
||||
if detect::detect().zacas() {
|
||||
zacas_acquire_fn
|
||||
} else {
|
||||
fallback::$non_seqcst_fallback_fn
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::Release => {
|
||||
ifunc!(unsafe fn(dst: *mut u128 $(, $($arg)*)?) $(-> $ret_ty)? {
|
||||
if detect::detect().zacas() {
|
||||
zacas_release_fn
|
||||
} else {
|
||||
fallback::$non_seqcst_fallback_fn
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::AcqRel => {
|
||||
ifunc!(unsafe fn(dst: *mut u128 $(, $($arg)*)?) $(-> $ret_ty)? {
|
||||
if detect::detect().zacas() {
|
||||
zacas_acqrel_fn
|
||||
} else {
|
||||
fallback::$non_seqcst_fallback_fn
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::SeqCst => {
|
||||
ifunc!(unsafe fn(dst: *mut u128 $(, $($arg)*)?) $(-> $ret_ty)? {
|
||||
if detect::detect().zacas() {
|
||||
zacas_seqcst_fn
|
||||
} else {
|
||||
fallback::$seqcst_fallback_fn
|
||||
}
|
||||
})
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_swap(dst: *mut u128, val: u128) -> u128 {
|
||||
|_x| val
|
||||
}
|
||||
zacas = atomic_swap_zacas;
|
||||
non_seqcst_fallback = atomic_swap_non_seqcst;
|
||||
seqcst_fallback = atomic_swap_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_add(dst: *mut u128, val: u128) -> u128 {
|
||||
|x| x.wrapping_add(val)
|
||||
}
|
||||
zacas = atomic_add_zacas;
|
||||
non_seqcst_fallback = atomic_add_non_seqcst;
|
||||
seqcst_fallback = atomic_add_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_sub(dst: *mut u128, val: u128) -> u128 {
|
||||
|x| x.wrapping_sub(val)
|
||||
}
|
||||
zacas = atomic_sub_zacas;
|
||||
non_seqcst_fallback = atomic_sub_non_seqcst;
|
||||
seqcst_fallback = atomic_sub_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_and(dst: *mut u128, val: u128) -> u128 {
|
||||
|x| x & val
|
||||
}
|
||||
zacas = atomic_and_zacas;
|
||||
non_seqcst_fallback = atomic_and_non_seqcst;
|
||||
seqcst_fallback = atomic_and_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_nand(dst: *mut u128, val: u128) -> u128 {
|
||||
|x| !(x & val)
|
||||
}
|
||||
zacas = atomic_nand_zacas;
|
||||
non_seqcst_fallback = atomic_nand_non_seqcst;
|
||||
seqcst_fallback = atomic_nand_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_or(dst: *mut u128, val: u128) -> u128 {
|
||||
|x| x | val
|
||||
}
|
||||
zacas = atomic_or_zacas;
|
||||
non_seqcst_fallback = atomic_or_non_seqcst;
|
||||
seqcst_fallback = atomic_or_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_xor(dst: *mut u128, val: u128) -> u128 {
|
||||
|x| x ^ val
|
||||
}
|
||||
zacas = atomic_xor_zacas;
|
||||
non_seqcst_fallback = atomic_xor_non_seqcst;
|
||||
seqcst_fallback = atomic_xor_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_max(dst: *mut u128, val: u128) -> u128 {
|
||||
|x| {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
{ core::cmp::max(x as i128, val as i128) as u128 }
|
||||
}
|
||||
}
|
||||
zacas = atomic_max_zacas;
|
||||
non_seqcst_fallback = atomic_max_non_seqcst;
|
||||
seqcst_fallback = atomic_max_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_umax(dst: *mut u128, val: u128) -> u128 {
|
||||
|x| core::cmp::max(x, val)
|
||||
}
|
||||
zacas = atomic_umax_zacas;
|
||||
non_seqcst_fallback = atomic_umax_non_seqcst;
|
||||
seqcst_fallback = atomic_umax_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_min(dst: *mut u128, val: u128) -> u128 {
|
||||
|x| {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
{ core::cmp::min(x as i128, val as i128) as u128 }
|
||||
}
|
||||
}
|
||||
zacas = atomic_min_zacas;
|
||||
non_seqcst_fallback = atomic_min_non_seqcst;
|
||||
seqcst_fallback = atomic_min_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_umin(dst: *mut u128, val: u128) -> u128 {
|
||||
|x| core::cmp::min(x, val)
|
||||
}
|
||||
zacas = atomic_umin_zacas;
|
||||
non_seqcst_fallback = atomic_umin_non_seqcst;
|
||||
seqcst_fallback = atomic_umin_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_not(dst: *mut u128) -> u128 {
|
||||
|x| !x
|
||||
}
|
||||
zacas = atomic_not_zacas;
|
||||
non_seqcst_fallback = atomic_not_non_seqcst;
|
||||
seqcst_fallback = atomic_not_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_neg(dst: *mut u128) -> u128 {
|
||||
|x| x.wrapping_neg()
|
||||
}
|
||||
zacas = atomic_neg_zacas;
|
||||
non_seqcst_fallback = atomic_neg_non_seqcst;
|
||||
seqcst_fallback = atomic_neg_seqcst;
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn is_lock_free() -> bool {
|
||||
#[cfg(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))]
|
||||
{
|
||||
// zacas is available at compile-time.
|
||||
true
|
||||
}
|
||||
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
|
||||
{
|
||||
detect::detect().zacas()
|
||||
}
|
||||
}
|
||||
const IS_ALWAYS_LOCK_FREE: bool =
|
||||
cfg!(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"));
|
||||
|
||||
atomic128!(AtomicI128, i128, atomic_max, atomic_min);
|
||||
atomic128!(AtomicU128, u128, atomic_umax, atomic_umin);
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
test_atomic_int!(i128);
|
||||
test_atomic_int!(u128);
|
||||
|
||||
// load/store/swap implementation is not affected by signedness, so it is
|
||||
// enough to test only unsigned types.
|
||||
stress_test!(u128);
|
||||
}
|
||||
+548
@@ -0,0 +1,548 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
128-bit atomic implementation on s390x.
|
||||
|
||||
This architecture provides the following 128-bit atomic instructions:
|
||||
|
||||
- LPQ/STPQ: load/store (arch1 or later, i.e., baseline)
|
||||
- CDSG: CAS (arch1 or later, i.e., baseline)
|
||||
|
||||
See "Atomic operation overview by architecture" in atomic-maybe-uninit for a more comprehensive and
|
||||
detailed description of the atomic and synchronize instructions in this architecture:
|
||||
https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#s390x
|
||||
|
||||
LLVM's minimal supported architecture level is arch8 (z10):
|
||||
https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/SystemZ/SystemZProcessors.td#L16-L17
|
||||
This does not appear to have changed since the current s390x backend was added in LLVM 3.3:
|
||||
https://github.com/llvm/llvm-project/commit/5f613dfd1f7edb0ae95d521b7107b582d9df5103#diff-cbaef692b3958312e80fd5507a7e2aff071f1acb086f10e8a96bc06a7bb289db
|
||||
|
||||
We could use asm byte_wise_atomic_load + asm atomic_compare_exchange pattern for RMW here, but using an
|
||||
inline assembly allows omitting the comparison of results and the storing/comparing of condition flags.
|
||||
|
||||
Note: On Miri and ThreadSanitizer which do not support inline assembly, we don't use
|
||||
this module and use intrinsics.rs instead.
|
||||
|
||||
Refs:
|
||||
- z/Architecture Principles of Operation, Fifteenth Edition (SA22-7832-14)
|
||||
https://www.ibm.com/docs/en/module_1678991624569/pdf/SA22-7832-14.pdf
|
||||
- atomic-maybe-uninit
|
||||
https://github.com/taiki-e/atomic-maybe-uninit
|
||||
|
||||
See tests/asm-test/asm/portable-atomic for generated assembly.
|
||||
*/
|
||||
|
||||
include!("macros.rs");
|
||||
|
||||
use core::{arch::asm, sync::atomic::Ordering};
|
||||
|
||||
#[cfg(portable_atomic_no_strict_provenance)]
|
||||
use crate::utils::ptr::PtrExt as _;
|
||||
use crate::utils::{Pair, U128};
|
||||
|
||||
// bcr 14,0 requires fast-BCR-serialization facility added in arch9 (z196).
|
||||
#[cfg(any(
|
||||
target_feature = "fast-serialization",
|
||||
portable_atomic_target_feature = "fast-serialization",
|
||||
))]
|
||||
macro_rules! serialization {
|
||||
() => {
|
||||
"bcr 14, 0"
|
||||
};
|
||||
}
|
||||
#[cfg(not(any(
|
||||
target_feature = "fast-serialization",
|
||||
portable_atomic_target_feature = "fast-serialization",
|
||||
)))]
|
||||
macro_rules! serialization {
|
||||
() => {
|
||||
"bcr 15, 0"
|
||||
};
|
||||
}
|
||||
|
||||
// Use distinct operands on z196 or later, otherwise split to lgr and $op.
|
||||
#[cfg(any(target_feature = "distinct-ops", portable_atomic_target_feature = "distinct-ops"))]
|
||||
macro_rules! distinct_op {
|
||||
($op:tt, $a0:tt, $a1:tt, $a2:tt) => {
|
||||
concat!($op, "k ", $a0, ", ", $a1, ", ", $a2)
|
||||
};
|
||||
}
|
||||
#[cfg(not(any(target_feature = "distinct-ops", portable_atomic_target_feature = "distinct-ops")))]
|
||||
macro_rules! distinct_op {
|
||||
($op:tt, $a0:tt, $a1:tt, $a2:tt) => {
|
||||
concat!("lgr ", $a0, ", ", $a1, "\n", $op, " ", $a0, ", ", $a2)
|
||||
};
|
||||
}
|
||||
|
||||
// Use selgr$cond on z15 or later, otherwise split to locgr$cond and $op.
|
||||
#[cfg(any(
|
||||
target_feature = "miscellaneous-extensions-3",
|
||||
portable_atomic_target_feature = "miscellaneous-extensions-3",
|
||||
))]
|
||||
#[cfg(any(
|
||||
target_feature = "load-store-on-cond",
|
||||
portable_atomic_target_feature = "load-store-on-cond",
|
||||
))]
|
||||
macro_rules! select_op {
|
||||
($cond:tt, $a0:tt, $a1:tt, $a2:tt) => {
|
||||
concat!("selgr", $cond, " ", $a0, ", ", $a1, ", ", $a2)
|
||||
};
|
||||
}
|
||||
#[cfg(not(any(
|
||||
target_feature = "miscellaneous-extensions-3",
|
||||
portable_atomic_target_feature = "miscellaneous-extensions-3",
|
||||
)))]
|
||||
#[cfg(any(
|
||||
target_feature = "load-store-on-cond",
|
||||
portable_atomic_target_feature = "load-store-on-cond",
|
||||
))]
|
||||
macro_rules! select_op {
|
||||
($cond:tt, $a0:tt, $a1:tt, $a2:tt) => {
|
||||
concat!("lgr ", $a0, ", ", $a2, "\n", "locgr", $cond, " ", $a0, ", ", $a1)
|
||||
};
|
||||
}
|
||||
|
||||
// Extracts and checks condition code.
|
||||
#[inline]
|
||||
fn extract_cc(r: i64) -> bool {
|
||||
r.wrapping_add(-268435456) & (1 << 31) != 0
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn atomic_load(src: *mut u128, _order: Ordering) -> u128 {
|
||||
debug_assert!(src.addr() % 16 == 0);
|
||||
let (out_hi, out_lo);
|
||||
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
// atomic load is always SeqCst.
|
||||
asm!(
|
||||
"lpq %r0, 0({src})", // atomic { r0:r1 = *src }
|
||||
src = in(reg) ptr_reg!(src),
|
||||
// Quadword atomic instructions work with even/odd pair of specified register and subsequent register.
|
||||
out("r0") out_hi,
|
||||
out("r1") out_lo,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
U128 { pair: Pair { hi: out_hi, lo: out_lo } }.whole
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {
|
||||
debug_assert!(dst.addr() % 16 == 0);
|
||||
let val = U128 { whole: val };
|
||||
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
macro_rules! atomic_store {
|
||||
($acquire:expr) => {
|
||||
asm!(
|
||||
"stpq %r0, 0({dst})", // atomic { *dst = r0:r1 }
|
||||
$acquire, // fence
|
||||
dst = in(reg) ptr_reg!(dst),
|
||||
// Quadword atomic instructions work with even/odd pair of specified register and subsequent register.
|
||||
in("r0") val.pair.hi,
|
||||
in("r1") val.pair.lo,
|
||||
options(nostack, preserves_flags),
|
||||
)
|
||||
};
|
||||
}
|
||||
match order {
|
||||
// Relaxed and Release stores are equivalent.
|
||||
Ordering::Relaxed | Ordering::Release => atomic_store!(""),
|
||||
Ordering::SeqCst => atomic_store!(serialization!()),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn atomic_compare_exchange(
|
||||
dst: *mut u128,
|
||||
old: u128,
|
||||
new: u128,
|
||||
_success: Ordering,
|
||||
_failure: Ordering,
|
||||
) -> Result<u128, u128> {
|
||||
debug_assert!(dst.addr() % 16 == 0);
|
||||
let old = U128 { whole: old };
|
||||
let new = U128 { whole: new };
|
||||
let (prev_hi, prev_lo);
|
||||
let r;
|
||||
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
// CDSG has SeqCst semantics.
|
||||
let prev = unsafe {
|
||||
macro_rules! cmpxchg {
|
||||
($new_hi:tt, $new_lo:tt) => {
|
||||
asm!(
|
||||
concat!("cdsg %r0, %", $new_hi, ", 0({dst})"), // atomic { if *dst == r0:r1 { cc = 0; *dst = new_hi:new_lo } else { cc = 1; r0:r1 = *dst } }
|
||||
"ipm {r}", // r[:] = cc
|
||||
dst = in(reg) ptr_reg!(dst),
|
||||
r = lateout(reg) r,
|
||||
// Quadword atomic instructions work with even/odd pair of specified register and subsequent register.
|
||||
inout("r0") old.pair.hi => prev_hi,
|
||||
inout("r1") old.pair.lo => prev_lo,
|
||||
in($new_hi) new.pair.hi,
|
||||
in($new_lo) new.pair.lo,
|
||||
// Do not use `preserves_flags` because CDSG modifies the condition code.
|
||||
options(nostack),
|
||||
)
|
||||
};
|
||||
}
|
||||
// r4 is stack pointer on z/OS.
|
||||
#[cfg(not(target_os = "zos"))]
|
||||
cmpxchg!("r4", "r5");
|
||||
#[cfg(target_os = "zos")]
|
||||
cmpxchg!("r12", "r13");
|
||||
U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole
|
||||
};
|
||||
if extract_cc(r) { Ok(prev) } else { Err(prev) }
|
||||
}
|
||||
|
||||
// cdsg is always strong.
|
||||
use self::atomic_compare_exchange as atomic_compare_exchange_weak;
|
||||
|
||||
#[inline]
|
||||
unsafe fn atomic_swap(dst: *mut u128, val: u128, _order: Ordering) -> u128 {
|
||||
debug_assert!(dst.addr() % 16 == 0);
|
||||
let val = U128 { whole: val };
|
||||
let (mut prev_hi, mut prev_lo);
|
||||
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
// CDSG has SeqCst semantics.
|
||||
//
|
||||
// Do not use atomic_rmw_cas_3 because it needs extra LGR to implement swap.
|
||||
unsafe {
|
||||
macro_rules! swap {
|
||||
($val_hi:tt, $val_lo:tt) => {
|
||||
asm!(
|
||||
"lg %r1, 8({dst})", // atomic { r1 = *dst.byte_add(8) }
|
||||
"lg %r0, 0({dst})", // atomic { r0 = *dst }
|
||||
"2:", // 'retry:
|
||||
concat!("cdsg %r0, %", $val_hi, ", 0({dst})"), // atomic { if *dst == r0:r1 { cc = 0; *dst = val_hi:val_lo } else { cc = 1; r0:r1 = *dst } }
|
||||
"jl 2b", // if cc == 1 { jump 'retry }
|
||||
dst = in(reg) ptr_reg!(dst),
|
||||
// Quadword atomic instructions work with even/odd pair of specified register and subsequent register.
|
||||
out("r0") prev_hi,
|
||||
out("r1") prev_lo,
|
||||
in($val_hi) val.pair.hi,
|
||||
in($val_lo) val.pair.lo,
|
||||
// Do not use `preserves_flags` because CDSG modifies the condition code.
|
||||
options(nostack),
|
||||
)
|
||||
};
|
||||
}
|
||||
// r4 is stack pointer on z/OS.
|
||||
#[cfg(not(target_os = "zos"))]
|
||||
swap!("r4", "r5");
|
||||
#[cfg(target_os = "zos")]
|
||||
swap!("r12", "r13");
|
||||
U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole
|
||||
}
|
||||
}
|
||||
|
||||
/// Atomic RMW by CAS loop (3 arguments)
|
||||
/// `unsafe fn(dst: *mut u128, val: u128, order: Ordering) -> u128;`
|
||||
///
|
||||
/// `$op` can use the following registers:
|
||||
/// - val_hi/val_lo pair: val argument (read-only for `$op`)
|
||||
/// - r0/r1 pair: previous value loaded (read-only for `$op`)
|
||||
/// - r12/r13 pair: new value that will be stored
|
||||
macro_rules! atomic_rmw_cas_3 {
|
||||
($name:ident, [$($reg:tt)*], $($op:tt)*) => {
|
||||
#[inline]
|
||||
unsafe fn $name(dst: *mut u128, val: u128, _order: Ordering) -> u128 {
|
||||
debug_assert!(dst.addr() % 16 == 0);
|
||||
let val = U128 { whole: val };
|
||||
let (mut prev_hi, mut prev_lo);
|
||||
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
// CDSG has SeqCst semantics.
|
||||
unsafe {
|
||||
asm!(
|
||||
"lg %r1, 8({dst})", // atomic { r1 = *dst.byte_add(8) }
|
||||
"lg %r0, 0({dst})", // atomic { r0 = *dst }
|
||||
"2:", // 'retry:
|
||||
$($op)*
|
||||
"cdsg %r0, %r12, 0({dst})", // atomic { if *dst == r0:r1 { cc = 0; *dst = r12:r13 } else { cc = 1; r0:r1 = *dst } }
|
||||
"jl 2b", // if cc == 1 { jump 'retry }
|
||||
dst = in(reg) ptr_reg!(dst),
|
||||
val_hi = in(reg) val.pair.hi,
|
||||
val_lo = in(reg) val.pair.lo,
|
||||
$($reg)*
|
||||
// Quadword atomic instructions work with even/odd pair of specified register and subsequent register.
|
||||
out("r0") prev_hi,
|
||||
out("r1") prev_lo,
|
||||
out("r12") _,
|
||||
out("r13") _,
|
||||
// Do not use `preserves_flags` because CDSG modifies the condition code.
|
||||
options(nostack),
|
||||
);
|
||||
U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
/// Atomic RMW by CAS loop (2 arguments)
|
||||
/// `unsafe fn(dst: *mut u128, order: Ordering) -> u128;`
|
||||
///
|
||||
/// `$op` can use the following registers:
|
||||
/// - r0/r1 pair: previous value loaded (read-only for `$op`)
|
||||
/// - r12/r13 pair: new value that will be stored
|
||||
macro_rules! atomic_rmw_cas_2 {
|
||||
($name:ident, [$($reg:tt)*], $($op:tt)*) => {
|
||||
#[inline]
|
||||
unsafe fn $name(dst: *mut u128, _order: Ordering) -> u128 {
|
||||
debug_assert!(dst.addr() % 16 == 0);
|
||||
let (mut prev_hi, mut prev_lo);
|
||||
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
// CDSG has SeqCst semantics.
|
||||
unsafe {
|
||||
asm!(
|
||||
"lg %r1, 8({dst})", // atomic { r1 = *dst.byte_add(8) }
|
||||
"lg %r0, 0({dst})", // atomic { r0 = *dst }
|
||||
"2:", // 'retry:
|
||||
$($op)*
|
||||
"cdsg %r0, %r12, 0({dst})", // atomic { if *dst == r0:r1 { cc = 0; *dst = r12:r13 } else { cc = 1; r0:r1 = *dst } }
|
||||
"jl 2b", // if cc == 1 { jump 'retry }
|
||||
dst = in(reg) ptr_reg!(dst),
|
||||
$($reg)*
|
||||
// Quadword atomic instructions work with even/odd pair of specified register and subsequent register.
|
||||
out("r0") prev_hi,
|
||||
out("r1") prev_lo,
|
||||
out("r12") _,
|
||||
out("r13") _,
|
||||
// Do not use `preserves_flags` because CDSG modifies the condition code.
|
||||
options(nostack),
|
||||
);
|
||||
U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
atomic_rmw_cas_3! {
|
||||
atomic_add, [],
|
||||
distinct_op!("algr", "%r13", "%r1", "{val_lo}"), // r13 = r1 + val_lo; cc = zero | carry
|
||||
"lgr %r12, %r0", // r12 = r0
|
||||
"alcgr %r12, {val_hi}", // r12 += val_hi + carry
|
||||
}
|
||||
atomic_rmw_cas_3! {
|
||||
atomic_sub, [],
|
||||
distinct_op!("slgr", "%r13", "%r1", "{val_lo}"), // r13 = r1 - val_lo; cc = zero | borrow
|
||||
"lgr %r12, %r0", // r12 = r0
|
||||
"slbgr %r12, {val_hi}", // r12 -= val_hi + borrow
|
||||
}
|
||||
atomic_rmw_cas_3! {
|
||||
atomic_and, [],
|
||||
distinct_op!("ngr", "%r13", "%r1", "{val_lo}"), // r13 = r1 & val_lo
|
||||
distinct_op!("ngr", "%r12", "%r0", "{val_hi}"), // r12 = r0 & val_hi
|
||||
}
|
||||
|
||||
// Use nngrk on z15 or later.
|
||||
#[cfg(any(
|
||||
target_feature = "miscellaneous-extensions-3",
|
||||
portable_atomic_target_feature = "miscellaneous-extensions-3",
|
||||
))]
|
||||
atomic_rmw_cas_3! {
|
||||
atomic_nand, [],
|
||||
"nngrk %r13, %r1, {val_lo}", // r13 = !(r1 & val_lo)
|
||||
"nngrk %r12, %r0, {val_hi}", // r12 = !(r0 & val_hi)
|
||||
}
|
||||
#[cfg(not(any(
|
||||
target_feature = "miscellaneous-extensions-3",
|
||||
portable_atomic_target_feature = "miscellaneous-extensions-3",
|
||||
)))]
|
||||
atomic_rmw_cas_3! {
|
||||
atomic_nand, [],
|
||||
distinct_op!("ngr", "%r13", "%r1", "{val_lo}"), // r13 = r1 & val_lo
|
||||
distinct_op!("ngr", "%r12", "%r0", "{val_hi}"), // r12 = r0 & val_hi
|
||||
"lcgr %r13, %r13", // r13 = !r13 + 1
|
||||
"aghi %r13, -1", // r13 -= 1
|
||||
"lcgr %r12, %r12", // r12 = !r12 + 1
|
||||
"aghi %r12, -1", // r12 -= 1
|
||||
}
|
||||
|
||||
atomic_rmw_cas_3! {
|
||||
atomic_or, [],
|
||||
distinct_op!("ogr", "%r13", "%r1", "{val_lo}"), // r13 = r1 | val_lo
|
||||
distinct_op!("ogr", "%r12", "%r0", "{val_hi}"), // r12 = r0 | val_hi
|
||||
}
|
||||
atomic_rmw_cas_3! {
|
||||
atomic_xor, [],
|
||||
distinct_op!("xgr", "%r13", "%r1", "{val_lo}"), // r13 = r1 ^ val_lo
|
||||
distinct_op!("xgr", "%r12", "%r0", "{val_hi}"), // r12 = r0 ^ val_hi
|
||||
}
|
||||
|
||||
cfg_sel!({
|
||||
#[cfg(any(
|
||||
target_feature = "load-store-on-cond",
|
||||
portable_atomic_target_feature = "load-store-on-cond",
|
||||
))]
|
||||
{
|
||||
atomic_rmw_cas_3! {
|
||||
atomic_max, [],
|
||||
"clgr %r1, {val_lo}", // if r1(u) < val_lo(u) { cc = 1 } else if r1(u) > val_lo(u) { cc = 2 } else { cc = 0 }
|
||||
select_op!("h", "%r12", "%r1", "{val_lo}"), // if cc == 2 { r12 = r1 } else { r12 = val_lo }
|
||||
"cgr %r0, {val_hi}", // if r0(i) < val_hi(i) { cc = 1 } else if r0(i) > val_hi(i) { cc = 2 } else { cc = 0 }
|
||||
select_op!("h", "%r13", "%r1", "{val_lo}"), // if cc == 2 { r13 = r1 } else { r13 = val_lo }
|
||||
"locgre %r13, %r12", // if cc == 0 { r13 = r12 }
|
||||
select_op!("h", "%r12", "%r0", "{val_hi}"), // if cc == 2 { r12 = r0 } else { r12 = val_hi }
|
||||
}
|
||||
atomic_rmw_cas_3! {
|
||||
atomic_umax, [tmp = out(reg) _,],
|
||||
"clgr %r1, {val_lo}", // if r1(u) < val_lo(u) { cc = 1 } else if r1(u) > val_lo(u) { cc = 2 } else { cc = 0 }
|
||||
select_op!("h", "{tmp}", "%r1", "{val_lo}"), // if cc == 2 { tmp = r1 } else { tmp = val_lo }
|
||||
"clgr %r0, {val_hi}", // if r0(u) < val_hi(u) { cc = 1 } else if r0(u) > val_hi(u) { cc = 2 } else { cc = 0 }
|
||||
select_op!("h", "%r12", "%r0", "{val_hi}"), // if cc == 2 { r12 = r0 } else { r12 = val_hi }
|
||||
select_op!("h", "%r13", "%r1", "{val_lo}"), // if cc == 2 { r13 = r1 } else { r13 = val_lo }
|
||||
"cgr %r0, {val_hi}", // if r0(i) < val_hi(i) { cc = 1 } else if r0(i) > val_hi(i) { cc = 2 } else { cc = 0 }
|
||||
"locgre %r13, {tmp}", // if cc == 0 { r13 = tmp }
|
||||
}
|
||||
atomic_rmw_cas_3! {
|
||||
atomic_min, [],
|
||||
"clgr %r1, {val_lo}", // if r1(u) < val_lo(u) { cc = 1 } else if r1(u) > val_lo(u) { cc = 2 } else { cc = 0 }
|
||||
select_op!("l", "%r12", "%r1", "{val_lo}"), // if cc == 1 { r12 = r1 } else { r12 = val_lo }
|
||||
"cgr %r0, {val_hi}", // if r0(i) < val_hi(i) { cc = 1 } else if r0(i) > val_hi(i) { cc = 2 } else { cc = 0 }
|
||||
select_op!("l", "%r13", "%r1", "{val_lo}"), // if cc == 1 { r13 = r1 } else { r13 = val_lo }
|
||||
"locgre %r13, %r12", // if cc == 0 { r13 = r12 }
|
||||
select_op!("l", "%r12", "%r0", "{val_hi}"), // if cc == 1 { r12 = r0 } else { r12 = val_hi }
|
||||
}
|
||||
atomic_rmw_cas_3! {
|
||||
atomic_umin, [tmp = out(reg) _,],
|
||||
"clgr %r1, {val_lo}", // if r1(u) < val_lo(u) { cc = 1 } else if r1(u) > val_lo(u) { cc = 2 } else { cc = 0 }
|
||||
select_op!("l", "{tmp}", "%r1", "{val_lo}"), // if cc == 1 { tmp = r1 } else { tmp = val_lo }
|
||||
"clgr %r0, {val_hi}", // if r0(u) < val_hi(u) { cc = 1 } else if r0(u) > val_hi(u) { cc = 2 } else { cc = 0 }
|
||||
select_op!("l", "%r12", "%r0", "{val_hi}"), // if cc == 1 { r12 = r0 } else { r12 = val_hi }
|
||||
select_op!("l", "%r13", "%r1", "{val_lo}"), // if cc == 1 { r13 = r1 } else { r13 = val_lo }
|
||||
"cgr %r0, {val_hi}", // if r0(i) < val_hi(i) { cc = 1 } else if r0(i) > val_hi(i) { cc = 2 } else { cc = 0 }
|
||||
"locgre %r13, {tmp}", // if cc == 0 { r13 = tmp }
|
||||
}
|
||||
}
|
||||
#[cfg(else)]
|
||||
{
|
||||
#[rustfmt::skip]
|
||||
macro_rules! atomic_cmp {
|
||||
($name:ident, $cmp_hi1:tt, $cmp_hi2:tt, $cmp_lo:tt,) => {
|
||||
#[inline]
|
||||
unsafe fn $name(dst: *mut u128, val: u128, _order: Ordering) -> u128 {
|
||||
debug_assert!(dst.addr() % 16 == 0);
|
||||
let val = U128 { whole: val };
|
||||
let (mut prev_hi, mut prev_lo);
|
||||
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
// CDSG has SeqCst semantics.
|
||||
unsafe {
|
||||
macro_rules! cmp {
|
||||
($val_hi:tt, $val_lo:tt) => {
|
||||
asm!(
|
||||
"lg %r1, 8({dst})", // atomic { r1 = *dst.byte_add(8) }
|
||||
"lg %r0, 0({dst})", // atomic { r0 = *dst }
|
||||
"2:", // 'retry:
|
||||
concat!($cmp_hi1, " %", $val_hi, ", %r0, 4f"), // if $cmp_hi1(val_hi, r0) { jump 'val }
|
||||
concat!($cmp_hi2, " %", $val_hi, ", %r0, 3f"), // if $cmp_hi2(val_hi, r0) { jump 'prev }
|
||||
concat!($cmp_lo, " %", $val_lo, ", %r1, 4f"), // if $cmp_lo(val_lo, r1) { jump 'val }
|
||||
"3:", // 'prev:
|
||||
"cdsg %r0, %r0, 0({dst})", // atomic { if *dst == r0:r1 { cc = 0; *dst = r0:r1 } else { cc = 1; r0:r1 = *dst } }
|
||||
"jl 2b", // if cc == 1 { jump 'retry }
|
||||
"j 5f",
|
||||
"4:", // 'val:
|
||||
concat!("cdsg %r0, %", $val_hi, ", 0({dst})"), // atomic { if *dst == r0:r1 { cc = 0; *dst = val_hi:val_lo } else { cc = 1; r0:r1 = *dst } }
|
||||
"jl 2b", // if cc == 1 { jump 'retry }
|
||||
"5:",
|
||||
dst = in(reg) ptr_reg!(dst),
|
||||
// Quadword atomic instructions work with even/odd pair of specified register and subsequent register.
|
||||
out("r0") prev_hi,
|
||||
out("r1") prev_lo,
|
||||
in($val_hi) val.pair.hi,
|
||||
in($val_lo) val.pair.lo,
|
||||
// Do not use `preserves_flags` because CDSG modifies the condition code.
|
||||
options(nostack),
|
||||
)
|
||||
};
|
||||
}
|
||||
// r4 is stack pointer on z/OS.
|
||||
#[cfg(not(target_os = "zos"))]
|
||||
cmp!("r4", "r5");
|
||||
#[cfg(target_os = "zos")]
|
||||
cmp!("r12", "r13");
|
||||
U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
// loc{,g}r requires z196 or later.
|
||||
atomic_cmp! {
|
||||
atomic_max,
|
||||
"cgrjh", // signed >
|
||||
"cgrjl", // signed <
|
||||
"clgrjhe", // unsigned >=
|
||||
}
|
||||
atomic_cmp! {
|
||||
atomic_umax,
|
||||
"clgrjh", // unsigned >
|
||||
"clgrjl", // unsigned <
|
||||
"clgrjhe", // unsigned >=
|
||||
}
|
||||
atomic_cmp! {
|
||||
atomic_min,
|
||||
"cgrjl", // signed <
|
||||
"cgrjh", // signed >
|
||||
"clgrjle", // unsigned <=
|
||||
}
|
||||
atomic_cmp! {
|
||||
atomic_umin,
|
||||
"clgrjl", // unsigned <
|
||||
"clgrjh", // unsigned >
|
||||
"clgrjle", // unsigned <=
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
atomic_rmw_cas_2! {
|
||||
atomic_not, [],
|
||||
"lcgr %r13, %r1", // r13 = !r1 + 1
|
||||
"aghi %r13, -1", // r13 -= 1
|
||||
"lcgr %r12, %r0", // r12 = !r0 + 1
|
||||
"aghi %r12, -1", // r12 -= 1
|
||||
}
|
||||
|
||||
#[cfg(any(target_feature = "distinct-ops", portable_atomic_target_feature = "distinct-ops"))]
|
||||
atomic_rmw_cas_2! {
|
||||
atomic_neg, [zero = in(reg) 0_u64,],
|
||||
"slgrk %r13, {zero}, %r1", // r13 = 0 - r1; cc = zero | borrow
|
||||
"lghi %r12, 0", // r12 = 0
|
||||
"slbgr %r12, %r0", // r12 -= r0 + borrow
|
||||
}
|
||||
#[cfg(not(any(target_feature = "distinct-ops", portable_atomic_target_feature = "distinct-ops")))]
|
||||
atomic_rmw_cas_2! {
|
||||
atomic_neg, [],
|
||||
"lghi %r13, 0", // r13 = 0
|
||||
"slgr %r13, %r1", // r13 -= r1; cc = zero | borrow
|
||||
"lghi %r12, 0", // r12 = 0
|
||||
"slbgr %r12, %r0", // r12 -= r0 + borrow
|
||||
}
|
||||
|
||||
#[inline]
|
||||
const fn is_lock_free() -> bool {
|
||||
IS_ALWAYS_LOCK_FREE
|
||||
}
|
||||
const IS_ALWAYS_LOCK_FREE: bool = true;
|
||||
|
||||
atomic128!(AtomicI128, i128, atomic_max, atomic_min);
|
||||
atomic128!(AtomicU128, u128, atomic_umax, atomic_umin);
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
test_atomic_int!(i128);
|
||||
test_atomic_int!(u128);
|
||||
|
||||
// load/store/swap implementation is not affected by signedness, so it is
|
||||
// enough to test only unsigned types.
|
||||
stress_test!(u128);
|
||||
}
|
||||
+1009
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,24 @@
|
||||
# 64-bit atomic implementations on 32-bit architectures
|
||||
|
||||
(See the [`atomic128` module](../atomic128) for 128-bit atomic implementations on 64-bit architectures.)
|
||||
|
||||
## 64-bit atomic instructions
|
||||
|
||||
Here is the table of targets that support 64-bit atomics and the instructions used:
|
||||
|
||||
| target_arch | load | store | CAS | RMW | note |
|
||||
| ----------- | ---- | ----- | --- | --- | ---- |
|
||||
| x86 | cmpxchg8b or fild or movlps or movq | cmpxchg8b or fistp or movlps | cmpxchg8b | cmpxchg8b | provided by `core::sync::atomic` |
|
||||
| arm | ldrexd | ldrexd/strexd | ldrexd/strexd | ldrexd/strexd | provided by `core::sync::atomic` for Armv6+, otherwise provided by us for Linux/Android using kuser_cmpxchg64 (see [arm_linux.rs](arm_linux.rs) for more) |
|
||||
| riscv32 | amocas.d | amocas.d | amocas.d | amocas.d | Requires `zacas` target feature. Both compile-time and run-time detection are supported. <br> Requires Rust 1.59+ |
|
||||
| hexagon | memd | memd | memd_locked | memd_locked | Unimplemented |
|
||||
| sparc | ldx | stx | casx | casx | Unimplemented (unsupported in LLVM). Requires `v8plus` and `v9` target feature (Linux is v8plus+v9 by default) |
|
||||
| m68k | cas2 | cas2 | cas2 | cas2 | Unimplemented (unsupported in LLVM). Requires M68020 or later (Linux is M68020 by default) |
|
||||
| mips32r6 | llwp | llwp/scwp | llwp/scwp | llwp/scwp | Unimplemented (unsupported in LLVM). Requires Release 6 Paired LL/SC family of instructions |
|
||||
|
||||
If `core::sync::atomic` provides 64-bit atomics, we use them.
|
||||
On compiler versions or platforms where these are not supported, the fallback implementation is used.
|
||||
|
||||
## Run-time CPU feature detection
|
||||
|
||||
See the [`detect` module's readme](../detect/README.md) for run-time CPU feature detection.
|
||||
@@ -0,0 +1,330 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
64-bit atomic implementation using kuser_cmpxchg64 on pre-v6 Arm Linux/Android.
|
||||
|
||||
See "Atomic operation overview by architecture" in atomic-maybe-uninit for a more comprehensive and
|
||||
detailed description of the atomic and synchronize instructions in this architecture:
|
||||
https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#arm
|
||||
|
||||
Refs:
|
||||
- https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/arm/kernel_user_helpers.rst
|
||||
- https://github.com/rust-lang/compiler-builtins/blob/compiler_builtins-v0.1.124/src/arm_linux.rs
|
||||
- https://github.com/taiki-e/atomic-maybe-uninit
|
||||
|
||||
Note: __kuser_cmpxchg64 is always SeqCst.
|
||||
https://github.com/taiki-e/atomic-maybe-uninit/blob/v0.3.21/src/arch/arm.rs#L230
|
||||
|
||||
Note: On Miri and ThreadSanitizer which do not support inline assembly, we don't use
|
||||
this module and use fallback implementation instead.
|
||||
*/
|
||||
|
||||
// TODO: Since Rust 1.64, the Linux kernel requirement for Rust when using std is 3.2+, so it should
|
||||
// be possible to omit the dynamic kernel version check if the std feature is enabled on Rust 1.64+.
|
||||
// https://blog.rust-lang.org/2022/08/01/Increasing-glibc-kernel-requirements
|
||||
|
||||
include!("macros.rs");
|
||||
|
||||
#[path = "../fallback/outline_atomics.rs"]
|
||||
mod fallback;
|
||||
|
||||
#[cfg(test)] // test-only (unused)
|
||||
#[cfg(not(portable_atomic_no_outline_atomics))]
|
||||
#[cfg(any(
|
||||
all(
|
||||
target_os = "linux",
|
||||
any(
|
||||
target_env = "gnu",
|
||||
target_env = "musl",
|
||||
target_env = "ohos",
|
||||
all(target_env = "uclibc", not(target_feature = "crt-static")),
|
||||
),
|
||||
),
|
||||
target_os = "android",
|
||||
target_os = "freebsd",
|
||||
target_os = "openbsd",
|
||||
))]
|
||||
#[path = "../detect/auxv.rs"]
|
||||
mod test_detect_auxv;
|
||||
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
use core::arch::asm;
|
||||
use core::{mem, sync::atomic::Ordering};
|
||||
|
||||
#[cfg(portable_atomic_no_strict_provenance)]
|
||||
use crate::utils::ptr::PtrExt as _;
|
||||
use crate::utils::{Pair, U64};
|
||||
|
||||
// https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/arm/kernel_user_helpers.rst
|
||||
const KUSER_HELPER_VERSION: usize = 0xFFFF0FFC;
|
||||
// __kuser_helper_version >= 5 (kernel version 3.1+)
|
||||
const KUSER_CMPXCHG64: usize = 0xFFFF0F60;
|
||||
#[inline]
|
||||
fn __kuser_helper_version() -> i32 {
|
||||
// SAFETY: core assumes that at least __kuser_memory_barrier (__kuser_helper_version >= 3) is
|
||||
// available on this platform. __kuser_helper_version is always available on such a platform.
|
||||
unsafe { crate::utils::ptr::with_exposed_provenance::<i32>(KUSER_HELPER_VERSION).read() }
|
||||
}
|
||||
#[inline]
|
||||
fn has_kuser_cmpxchg64() -> bool {
|
||||
// Note: detect_false cfg is intended to make it easy for developers to test
|
||||
// cases where features usually available is not available, and is not a public API.
|
||||
if cfg!(portable_atomic_test_detect_false) {
|
||||
return false;
|
||||
}
|
||||
__kuser_helper_version() >= 5
|
||||
}
|
||||
#[inline]
|
||||
unsafe fn __kuser_cmpxchg64(old_val: *const u64, new_val: *const u64, ptr: *mut u64) -> bool {
|
||||
// SAFETY: kernel docs specify a known address with the given signature.
|
||||
// And the caller must guarantee that the pointer is valid for read and write,
|
||||
// aligned to the element size, and __kuser_helper_version >= 5.
|
||||
unsafe {
|
||||
let f: extern "C" fn(*const u64, *const u64, *mut u64) -> u32 =
|
||||
mem::transmute(crate::utils::ptr::with_exposed_provenance::<()>(KUSER_CMPXCHG64));
|
||||
f(old_val, new_val, ptr) == 0
|
||||
}
|
||||
}
|
||||
|
||||
// 64-bit atomic load by two 32-bit atomic loads.
|
||||
#[inline]
|
||||
unsafe fn byte_wise_atomic_load(src: *const u64) -> u64 {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
let (out_lo, out_hi);
|
||||
asm!(
|
||||
"ldr {out_lo}, [{src}]", // atomic { out_lo = *src }
|
||||
"ldr {out_hi}, [{src}, #4]", // atomic { out_hi = *src.byte_add(4) }
|
||||
src = in(reg) src,
|
||||
out_lo = out(reg) out_lo,
|
||||
out_hi = out(reg) out_hi,
|
||||
options(pure, nostack, preserves_flags, readonly),
|
||||
);
|
||||
U64 { pair: Pair { lo: out_lo, hi: out_hi } }.whole
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! select_atomic {
|
||||
(
|
||||
unsafe fn $name:ident($dst:ident: *mut u64 $(, $($arg:tt)*)?) $(-> $ret_ty:ty)? {
|
||||
|$kuser_cmpxchg64_fn_binding:ident| $($kuser_cmpxchg64_fn_body:tt)*
|
||||
}
|
||||
fallback = $seqcst_fallback_fn:ident
|
||||
) => {
|
||||
#[inline]
|
||||
unsafe fn $name($dst: *mut u64 $(, $($arg)*)?, _: Ordering) $(-> $ret_ty)? {
|
||||
unsafe fn kuser_cmpxchg64_fn($dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {
|
||||
debug_assert!($dst.addr() % 8 == 0);
|
||||
debug_assert!(has_kuser_cmpxchg64());
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
loop {
|
||||
// This is not single-copy atomic reads, but this is ok because subsequent
|
||||
// CAS will check for consistency.
|
||||
//
|
||||
// Arm's memory model allow mixed-sized atomic access.
|
||||
// https://github.com/rust-lang/unsafe-code-guidelines/issues/345#issuecomment-1172891466
|
||||
//
|
||||
// Note that the C++20 memory model does not allow mixed-sized atomic access,
|
||||
// so we must use inline assembly to implement byte_wise_atomic_load.
|
||||
// (i.e., byte-wise atomic based on the standard library's atomic types
|
||||
// cannot be used here).
|
||||
let prev = byte_wise_atomic_load($dst);
|
||||
let next = {
|
||||
let $kuser_cmpxchg64_fn_binding = prev;
|
||||
$($kuser_cmpxchg64_fn_body)*
|
||||
};
|
||||
if __kuser_cmpxchg64(&prev, &next, $dst) {
|
||||
return prev;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
// we only calls __kuser_cmpxchg64 if it is available.
|
||||
unsafe {
|
||||
ifunc!(unsafe fn($dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {
|
||||
if has_kuser_cmpxchg64() {
|
||||
kuser_cmpxchg64_fn
|
||||
} else {
|
||||
// Use SeqCst because __kuser_cmpxchg64 is always SeqCst.
|
||||
fallback::$seqcst_fallback_fn
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
select_atomic! {
|
||||
unsafe fn atomic_load(src: *mut u64) -> u64 {
|
||||
|old| old
|
||||
}
|
||||
fallback = atomic_load_seqcst
|
||||
}
|
||||
#[inline]
|
||||
unsafe fn atomic_store(dst: *mut u64, val: u64, order: Ordering) {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
atomic_swap(dst, val, order);
|
||||
}
|
||||
}
|
||||
select_atomic! {
|
||||
unsafe fn atomic_swap(dst: *mut u64, val: u64) -> u64 {
|
||||
|_x| val
|
||||
}
|
||||
fallback = atomic_swap_seqcst
|
||||
}
|
||||
#[inline]
|
||||
unsafe fn atomic_compare_exchange(
|
||||
dst: *mut u64,
|
||||
old: u64,
|
||||
new: u64,
|
||||
_: Ordering,
|
||||
_: Ordering,
|
||||
) -> Result<u64, u64> {
|
||||
unsafe fn kuser_cmpxchg64_fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {
|
||||
debug_assert!(dst.addr() % 8 == 0);
|
||||
debug_assert!(has_kuser_cmpxchg64());
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
loop {
|
||||
// See select_atomic! for more.
|
||||
let prev = byte_wise_atomic_load(dst);
|
||||
let next = if prev == old { new } else { prev };
|
||||
if __kuser_cmpxchg64(&prev, &next, dst) {
|
||||
return (prev, prev == old);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
// we only calls __kuser_cmpxchg64 if it is available.
|
||||
let (prev, ok) = unsafe {
|
||||
ifunc!(unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {
|
||||
if has_kuser_cmpxchg64() {
|
||||
kuser_cmpxchg64_fn
|
||||
} else {
|
||||
// Use SeqCst because __kuser_cmpxchg64 is always SeqCst.
|
||||
fallback::atomic_compare_exchange_seqcst
|
||||
}
|
||||
})
|
||||
};
|
||||
if ok { Ok(prev) } else { Err(prev) }
|
||||
}
|
||||
use self::atomic_compare_exchange as atomic_compare_exchange_weak;
|
||||
select_atomic! {
|
||||
unsafe fn atomic_add(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| x.wrapping_add(val)
|
||||
}
|
||||
fallback = atomic_add_seqcst
|
||||
}
|
||||
select_atomic! {
|
||||
unsafe fn atomic_sub(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| x.wrapping_sub(val)
|
||||
}
|
||||
fallback = atomic_sub_seqcst
|
||||
}
|
||||
select_atomic! {
|
||||
unsafe fn atomic_and(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| x & val
|
||||
}
|
||||
fallback = atomic_and_seqcst
|
||||
}
|
||||
select_atomic! {
|
||||
unsafe fn atomic_nand(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| !(x & val)
|
||||
}
|
||||
fallback = atomic_nand_seqcst
|
||||
}
|
||||
select_atomic! {
|
||||
unsafe fn atomic_or(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| x | val
|
||||
}
|
||||
fallback = atomic_or_seqcst
|
||||
}
|
||||
select_atomic! {
|
||||
unsafe fn atomic_xor(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| x ^ val
|
||||
}
|
||||
fallback = atomic_xor_seqcst
|
||||
}
|
||||
select_atomic! {
|
||||
unsafe fn atomic_max(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
{ core::cmp::max(x as i64, val as i64) as u64 }
|
||||
}
|
||||
}
|
||||
fallback = atomic_max_seqcst
|
||||
}
|
||||
select_atomic! {
|
||||
unsafe fn atomic_umax(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| core::cmp::max(x, val)
|
||||
}
|
||||
fallback = atomic_umax_seqcst
|
||||
}
|
||||
select_atomic! {
|
||||
unsafe fn atomic_min(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
{ core::cmp::min(x as i64, val as i64) as u64 }
|
||||
}
|
||||
}
|
||||
fallback = atomic_min_seqcst
|
||||
}
|
||||
select_atomic! {
|
||||
unsafe fn atomic_umin(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| core::cmp::min(x, val)
|
||||
}
|
||||
fallback = atomic_umin_seqcst
|
||||
}
|
||||
select_atomic! {
|
||||
unsafe fn atomic_not(dst: *mut u64) -> u64 {
|
||||
|x| !x
|
||||
}
|
||||
fallback = atomic_not_seqcst
|
||||
}
|
||||
select_atomic! {
|
||||
unsafe fn atomic_neg(dst: *mut u64) -> u64 {
|
||||
|x| x.wrapping_neg()
|
||||
}
|
||||
fallback = atomic_neg_seqcst
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn is_lock_free() -> bool {
|
||||
has_kuser_cmpxchg64()
|
||||
}
|
||||
const IS_ALWAYS_LOCK_FREE: bool = false;
|
||||
|
||||
atomic64!(AtomicI64, i64, atomic_max, atomic_min);
|
||||
atomic64!(AtomicU64, u64, atomic_umax, atomic_umin);
|
||||
|
||||
#[allow(
|
||||
clippy::alloc_instead_of_core,
|
||||
clippy::std_instead_of_alloc,
|
||||
clippy::std_instead_of_core,
|
||||
clippy::undocumented_unsafe_blocks,
|
||||
clippy::wildcard_imports
|
||||
)]
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn kuser_helper_version() {
|
||||
let version = __kuser_helper_version();
|
||||
assert!(version >= 5, "{:?}", version);
|
||||
assert_eq!(version, unsafe {
|
||||
crate::utils::ptr::with_exposed_provenance::<i32>(KUSER_HELPER_VERSION).read()
|
||||
});
|
||||
}
|
||||
|
||||
test_atomic_int!(i64);
|
||||
test_atomic_int!(u64);
|
||||
|
||||
// load/store/swap implementation is not affected by signedness, so it is
|
||||
// enough to test only unsigned types.
|
||||
stress_test!(u64);
|
||||
}
|
||||
+230
@@ -0,0 +1,230 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
macro_rules! atomic64 {
|
||||
($atomic_type:ident, $int_type:ident, $atomic_max:ident, $atomic_min:ident) => {
|
||||
#[repr(C, align(8))]
|
||||
pub(crate) struct $atomic_type {
|
||||
v: core::cell::UnsafeCell<$int_type>,
|
||||
}
|
||||
|
||||
// Send is implicitly implemented.
|
||||
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock.
|
||||
unsafe impl Sync for $atomic_type {}
|
||||
|
||||
impl_default_no_fetch_ops!($atomic_type, $int_type);
|
||||
impl_default_bit_opts!($atomic_type, $int_type);
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
pub(crate) const fn new(v: $int_type) -> Self {
|
||||
Self { v: core::cell::UnsafeCell::new(v) }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn is_lock_free() -> bool {
|
||||
is_lock_free()
|
||||
}
|
||||
pub(crate) const IS_ALWAYS_LOCK_FREE: bool = IS_ALWAYS_LOCK_FREE;
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
|
||||
pub(crate) fn load(&self, order: Ordering) -> $int_type {
|
||||
crate::utils::assert_load_ordering(order);
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
|
||||
// and the raw pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_load(self.v.get().cast::<u64>(), order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
|
||||
pub(crate) fn store(&self, val: $int_type, order: Ordering) {
|
||||
crate::utils::assert_store_ordering(order);
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
|
||||
// and the raw pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_store(self.v.get().cast::<u64>(), val as u64, order)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn swap(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
|
||||
// and the raw pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_swap(self.v.get().cast::<u64>(), val as u64, order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
|
||||
pub(crate) fn compare_exchange(
|
||||
&self,
|
||||
current: $int_type,
|
||||
new: $int_type,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<$int_type, $int_type> {
|
||||
crate::utils::assert_compare_exchange_ordering(success, failure);
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
|
||||
// and the raw pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
match atomic_compare_exchange(
|
||||
self.v.get().cast::<u64>(),
|
||||
current as u64,
|
||||
new as u64,
|
||||
success,
|
||||
failure,
|
||||
) {
|
||||
Ok(v) => Ok(v as $int_type),
|
||||
Err(v) => Err(v as $int_type),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
|
||||
pub(crate) fn compare_exchange_weak(
|
||||
&self,
|
||||
current: $int_type,
|
||||
new: $int_type,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<$int_type, $int_type> {
|
||||
crate::utils::assert_compare_exchange_ordering(success, failure);
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
|
||||
// and the raw pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
match atomic_compare_exchange_weak(
|
||||
self.v.get().cast::<u64>(),
|
||||
current as u64,
|
||||
new as u64,
|
||||
success,
|
||||
failure,
|
||||
) {
|
||||
Ok(v) => Ok(v as $int_type),
|
||||
Err(v) => Err(v as $int_type),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
|
||||
// and the raw pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_add(self.v.get().cast::<u64>(), val as u64, order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
|
||||
// and the raw pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_sub(self.v.get().cast::<u64>(), val as u64, order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
|
||||
// and the raw pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_and(self.v.get().cast::<u64>(), val as u64, order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_nand(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
|
||||
// and the raw pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_nand(self.v.get().cast::<u64>(), val as u64, order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
|
||||
// and the raw pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_or(self.v.get().cast::<u64>(), val as u64, order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
|
||||
// and the raw pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_xor(self.v.get().cast::<u64>(), val as u64, order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
|
||||
// and the raw pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
$atomic_max(self.v.get().cast::<u64>(), val as u64, order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
|
||||
// and the raw pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
$atomic_min(self.v.get().cast::<u64>(), val as u64, order) as $int_type
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_not(&self, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
|
||||
// and the raw pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_not(self.v.get().cast::<u64>(), order) as $int_type
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn not(&self, order: Ordering) {
|
||||
self.fetch_not(order);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_neg(&self, order: Ordering) -> $int_type {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
// SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock
|
||||
// and the raw pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
atomic_neg(self.v.get().cast::<u64>(), order) as $int_type
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn neg(&self, order: Ordering) {
|
||||
self.fetch_neg(order);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) const fn as_ptr(&self) -> *mut $int_type {
|
||||
self.v.get()
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
64-bit atomic implementations on 32-bit architectures
|
||||
|
||||
See README.md for details.
|
||||
*/
|
||||
|
||||
// pre-v6 Arm Linux
|
||||
// Miri and Sanitizer do not support inline assembly.
|
||||
#[cfg(all(
|
||||
feature = "fallback",
|
||||
target_arch = "arm",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(target_os = "linux", target_os = "android"),
|
||||
any(test, not(any(target_feature = "v6", portable_atomic_target_feature = "v6"))),
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
))]
|
||||
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(any(test, portable_atomic_no_atomic_64)))]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(test, not(target_has_atomic = "64")))
|
||||
)]
|
||||
pub(super) mod arm_linux;
|
||||
|
||||
// riscv32
|
||||
// Miri and Sanitizer do not support inline assembly.
|
||||
#[cfg(all(
|
||||
target_arch = "riscv32",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(
|
||||
target_feature = "zacas",
|
||||
portable_atomic_target_feature = "zacas",
|
||||
all(
|
||||
feature = "fallback",
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
any(target_os = "linux", target_os = "android"),
|
||||
),
|
||||
),
|
||||
))]
|
||||
pub(super) mod riscv32;
|
||||
+626
@@ -0,0 +1,626 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
64-bit atomic implementation on riscv32.
|
||||
|
||||
This architecture provides the following 64-bit atomic instructions:
|
||||
|
||||
- amocas.d: CAS (Zacas extension)
|
||||
|
||||
See "Atomic operation overview by architecture" in atomic-maybe-uninit for a more comprehensive and
|
||||
detailed description of the atomic and synchronize instructions in this architecture:
|
||||
https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#risc-v
|
||||
|
||||
Note: On Miri and ThreadSanitizer which do not support inline assembly, we don't use
|
||||
this module and use fallback implementation instead.
|
||||
|
||||
Refs:
|
||||
- RISC-V Instruction Set Manual
|
||||
"Zacas" Extension for Atomic Compare-and-Swap (CAS) Instructions
|
||||
https://docs.riscv.org/reference/isa/v20260120/unpriv/zacas.html
|
||||
- RISC-V Atomics ABI Specification
|
||||
https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/draft-20250812-301374e92976e298e676e7129a6212926b2299ce/riscv-atomic.adoc
|
||||
|
||||
See tests/asm-test/asm/portable-atomic for generated assembly.
|
||||
*/
|
||||
|
||||
// TODO: merge duplicated code with atomic128/riscv64.rs
|
||||
|
||||
include!("macros.rs");
|
||||
|
||||
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
|
||||
#[path = "../fallback/outline_atomics.rs"]
|
||||
mod fallback;
|
||||
|
||||
#[cfg(not(portable_atomic_no_outline_atomics))]
|
||||
#[cfg(any(test, not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))))]
|
||||
#[cfg(any(target_os = "linux", target_os = "android"))]
|
||||
#[path = "../detect/riscv_linux.rs"]
|
||||
mod detect;
|
||||
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
use core::arch::asm;
|
||||
use core::sync::atomic::Ordering;
|
||||
|
||||
#[cfg(portable_atomic_no_strict_provenance)]
|
||||
use crate::utils::ptr::PtrExt as _;
|
||||
use crate::utils::{Pair, U64};
|
||||
|
||||
macro_rules! debug_assert_zacas {
|
||||
() => {
|
||||
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
|
||||
{
|
||||
debug_assert!(detect::detect().zacas());
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// `.option arch, +zacas` directive requires LLVM 20, so we use .4byte directive for old LLVM.
|
||||
// Note that `.insn <value>` directive requires LLVM 19.
|
||||
// https://github.com/llvm/llvm-project/commit/2a086dce691e3cc34a2fc27f4fb255bb2cbbfac9
|
||||
// https://github.com/riscv-non-isa/riscv-asm-manual/blob/v0.0.1/src/asm-manual.adoc#arch
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
macro_rules! start_zacas {
|
||||
() => {
|
||||
".option push\n.option arch, +zacas"
|
||||
};
|
||||
}
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
macro_rules! end_zacas {
|
||||
() => {
|
||||
".option pop"
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
macro_rules! atomic_rmw_amocas_order {
|
||||
($op:ident, $order:ident) => {
|
||||
atomic_rmw_amocas_order!($op, $order, failure = $order)
|
||||
};
|
||||
($op:ident, $order:ident, failure = $failure:ident) => {
|
||||
match $order {
|
||||
Ordering::Relaxed => $op!("", ""),
|
||||
Ordering::Acquire => $op!("", ".aq"),
|
||||
Ordering::Release => $op!("", ".rl"),
|
||||
Ordering::AcqRel => $op!("", ".aqrl"),
|
||||
Ordering::SeqCst if $failure == Ordering::SeqCst => $op!("fence rw,rw", ".aqrl"),
|
||||
Ordering::SeqCst => $op!("", ".aqrl"),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
};
|
||||
}
|
||||
#[cfg(portable_atomic_pre_llvm_20)]
|
||||
macro_rules! atomic_rmw_amocas_order_insn {
|
||||
($op:ident, $order:ident) => {
|
||||
atomic_rmw_amocas_order_insn!($op, $order, failure = $order)
|
||||
};
|
||||
($op:ident, $order:ident, failure = $failure:ident) => {
|
||||
match $order {
|
||||
Ordering::Relaxed => $op!("", "8"),
|
||||
Ordering::Acquire => $op!("", "c"),
|
||||
Ordering::Release => $op!("", "a"),
|
||||
Ordering::AcqRel => $op!("", "e"),
|
||||
Ordering::SeqCst if $failure == Ordering::SeqCst => $op!("fence rw,rw", "e"),
|
||||
Ordering::SeqCst => $op!("", "e"),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// If zacas is available at compile-time, we can always use zacas_fn.
|
||||
#[cfg(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))]
|
||||
use self::atomic_load_zacas as atomic_load;
|
||||
// Otherwise, we need to do run-time detection and can use zacas_fn only if zacas is available.
|
||||
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
|
||||
#[inline]
|
||||
unsafe fn atomic_load(src: *mut u64, order: Ordering) -> u64 {
|
||||
fn_alias! {
|
||||
// inline(never) is just a hint and also not strictly necessary
|
||||
// because we use ifunc helper macro, but used for clarity.
|
||||
#[inline(never)]
|
||||
unsafe fn(src: *mut u64) -> u64;
|
||||
atomic_load_zacas_relaxed = atomic_load_zacas(Ordering::Relaxed);
|
||||
atomic_load_zacas_acquire = atomic_load_zacas(Ordering::Acquire);
|
||||
atomic_load_zacas_seqcst = atomic_load_zacas(Ordering::SeqCst);
|
||||
}
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
// we only calls atomic_load_zacas if zacas is available.
|
||||
unsafe {
|
||||
match order {
|
||||
Ordering::Relaxed => {
|
||||
ifunc!(unsafe fn(src: *mut u64) -> u64 {
|
||||
if detect::detect().zacas() {
|
||||
atomic_load_zacas_relaxed
|
||||
} else {
|
||||
fallback::atomic_load_non_seqcst
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::Acquire => {
|
||||
ifunc!(unsafe fn(src: *mut u64) -> u64 {
|
||||
if detect::detect().zacas() {
|
||||
atomic_load_zacas_acquire
|
||||
} else {
|
||||
fallback::atomic_load_non_seqcst
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::SeqCst => {
|
||||
ifunc!(unsafe fn(src: *mut u64) -> u64 {
|
||||
if detect::detect().zacas() {
|
||||
atomic_load_zacas_seqcst
|
||||
} else {
|
||||
fallback::atomic_load_seqcst
|
||||
}
|
||||
})
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
unsafe fn atomic_load_zacas(src: *mut u64, order: Ordering) -> u64 {
|
||||
debug_assert!(src.addr() % 8 == 0);
|
||||
debug_assert_zacas!();
|
||||
let (out_lo, out_hi);
|
||||
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
macro_rules! load {
|
||||
($fence:tt, $asm_order:tt) => {
|
||||
asm!(
|
||||
start_zacas!(),
|
||||
$fence, // fence
|
||||
concat!("amocas.d", $asm_order, " a2, a2, 0({src})"), // atomic { if *dst == a2:a3 { *dst = a2:a3 } else { a2:a3 = *dst } }
|
||||
end_zacas!(),
|
||||
src = in(reg) ptr_reg!(src),
|
||||
inout("a2") 0_u32 => out_lo,
|
||||
inout("a3") 0_u32 => out_hi,
|
||||
options(nostack, preserves_flags),
|
||||
)
|
||||
};
|
||||
}
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
atomic_rmw_amocas_order!(load, order);
|
||||
#[cfg(portable_atomic_pre_llvm_20)]
|
||||
macro_rules! load {
|
||||
($fence:tt, $insn_order:tt) => {
|
||||
asm!(
|
||||
$fence, // fence
|
||||
// amocas.d{,.aq,.rl,.aqrl} a2, a2, (a0) // atomic { if *a0 == a2:a3 { *a0 = a2:a3 } else { a2:a3 = *a0 } }
|
||||
concat!(".4byte 0x2", $insn_order, "c5362f"),
|
||||
in("a0") ptr_reg!(src),
|
||||
inout("a2") 0_u32 => out_lo,
|
||||
inout("a3") 0_u32 => out_hi,
|
||||
options(nostack, preserves_flags),
|
||||
)
|
||||
};
|
||||
}
|
||||
#[cfg(portable_atomic_pre_llvm_20)]
|
||||
atomic_rmw_amocas_order_insn!(load, order);
|
||||
U64 { pair: Pair { lo: out_lo, hi: out_hi } }.whole
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn atomic_store(dst: *mut u64, val: u64, order: Ordering) {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
atomic_swap(dst, val, order);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn atomic_compare_exchange(
|
||||
dst: *mut u64,
|
||||
old: u64,
|
||||
new: u64,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<u64, u64> {
|
||||
#[cfg(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))]
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
// cfg guarantees that zacas instructions are available at compile-time.
|
||||
let (prev, ok) = unsafe { atomic_compare_exchange_zacas(dst, old, new, success, failure) };
|
||||
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
|
||||
let (prev, ok) = {
|
||||
fn_alias! {
|
||||
// inline(never) is just a hint and also not strictly necessary
|
||||
// because we use ifunc helper macro, but used for clarity.
|
||||
#[inline(never)]
|
||||
unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool);
|
||||
zacas_relaxed_fn = atomic_compare_exchange_zacas(Ordering::Relaxed, Ordering::Relaxed);
|
||||
zacas_acquire_fn = atomic_compare_exchange_zacas(Ordering::Acquire, Ordering::Acquire);
|
||||
zacas_release_fn = atomic_compare_exchange_zacas(Ordering::Release, Ordering::Relaxed);
|
||||
zacas_acqrel_fn = atomic_compare_exchange_zacas(Ordering::AcqRel, Ordering::Acquire);
|
||||
zacas_seqcst_fn = atomic_compare_exchange_zacas(Ordering::SeqCst, Ordering::SeqCst);
|
||||
}
|
||||
let order = crate::utils::upgrade_success_ordering(success, failure);
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
// we only calls atomic_compare_exchange_zacas if zacas is available.
|
||||
unsafe {
|
||||
match order {
|
||||
Ordering::Relaxed => {
|
||||
ifunc!(unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {
|
||||
if detect::detect().zacas() {
|
||||
zacas_relaxed_fn
|
||||
} else {
|
||||
fallback::atomic_compare_exchange_non_seqcst
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::Acquire => {
|
||||
ifunc!(unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {
|
||||
if detect::detect().zacas() {
|
||||
zacas_acquire_fn
|
||||
} else {
|
||||
fallback::atomic_compare_exchange_non_seqcst
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::Release => {
|
||||
ifunc!(unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {
|
||||
if detect::detect().zacas() {
|
||||
zacas_release_fn
|
||||
} else {
|
||||
fallback::atomic_compare_exchange_non_seqcst
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::AcqRel => {
|
||||
ifunc!(unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {
|
||||
if detect::detect().zacas() {
|
||||
zacas_acqrel_fn
|
||||
} else {
|
||||
fallback::atomic_compare_exchange_non_seqcst
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::SeqCst => {
|
||||
ifunc!(unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {
|
||||
if detect::detect().zacas() {
|
||||
zacas_seqcst_fn
|
||||
} else {
|
||||
fallback::atomic_compare_exchange_seqcst
|
||||
}
|
||||
})
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
};
|
||||
if ok { Ok(prev) } else { Err(prev) }
|
||||
}
|
||||
#[inline]
|
||||
unsafe fn atomic_compare_exchange_zacas(
|
||||
dst: *mut u64,
|
||||
old: u64,
|
||||
new: u64,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> (u64, bool) {
|
||||
debug_assert!(dst.addr() % 8 == 0);
|
||||
debug_assert_zacas!();
|
||||
let order = crate::utils::upgrade_success_ordering(success, failure);
|
||||
let old = U64 { whole: old };
|
||||
let new = U64 { whole: new };
|
||||
let (prev_lo, prev_hi);
|
||||
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
macro_rules! cmpxchg {
|
||||
($fence:tt, $asm_order:tt) => {
|
||||
asm!(
|
||||
start_zacas!(),
|
||||
$fence, // fence
|
||||
concat!("amocas.d", $asm_order, " a4, a2, 0({dst})"), // atomic { if *dst == a4:a5 { *dst = a2:a3 } else { a4:a5 = *dst } }
|
||||
end_zacas!(),
|
||||
dst = in(reg) ptr_reg!(dst),
|
||||
// must be allocated to even/odd register pair
|
||||
inout("a4") old.pair.lo => prev_lo,
|
||||
inout("a5") old.pair.hi => prev_hi,
|
||||
// must be allocated to even/odd register pair
|
||||
in("a2") new.pair.lo,
|
||||
in("a3") new.pair.hi,
|
||||
options(nostack, preserves_flags),
|
||||
)
|
||||
};
|
||||
}
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
atomic_rmw_amocas_order!(cmpxchg, order, failure = failure);
|
||||
#[cfg(portable_atomic_pre_llvm_20)]
|
||||
macro_rules! cmpxchg {
|
||||
($fence:tt, $insn_order:tt) => {
|
||||
asm!(
|
||||
$fence, // fence
|
||||
// amocas.d{,.aq,.rl,.aqrl} a4, a2, (a0) // atomic { if *a0 == a4:a5 { *a0 = a2:a3 } else { a4:a5 = *a0 } }
|
||||
concat!(".4byte 0x2", $insn_order, "c5372f"),
|
||||
in("a0") ptr_reg!(dst),
|
||||
// must be allocated to even/odd register pair
|
||||
inout("a4") old.pair.lo => prev_lo,
|
||||
inout("a5") old.pair.hi => prev_hi,
|
||||
// must be allocated to even/odd register pair
|
||||
in("a2") new.pair.lo,
|
||||
in("a3") new.pair.hi,
|
||||
options(nostack, preserves_flags),
|
||||
)
|
||||
};
|
||||
}
|
||||
#[cfg(portable_atomic_pre_llvm_20)]
|
||||
atomic_rmw_amocas_order_insn!(cmpxchg, order, failure = failure);
|
||||
let prev = U64 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole;
|
||||
(prev, prev == old.whole)
|
||||
}
|
||||
}
|
||||
|
||||
// amocas is always strong.
|
||||
use self::atomic_compare_exchange as atomic_compare_exchange_weak;
|
||||
|
||||
// 64-bit atomic load by two 32-bit atomic loads.
|
||||
#[inline]
|
||||
unsafe fn byte_wise_atomic_load(src: *const u64) -> u64 {
|
||||
let (out_lo, out_hi);
|
||||
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
asm!(
|
||||
"lw {out_lo}, ({src})", // atomic { out_lo = *src }
|
||||
"lw {out_hi}, 4({src})", // atomic { out_hi = *src.byte_add(4) }
|
||||
src = in(reg) ptr_reg!(src),
|
||||
out_lo = out(reg) out_lo,
|
||||
out_hi = out(reg) out_hi,
|
||||
options(pure, nostack, preserves_flags, readonly),
|
||||
);
|
||||
U64 { pair: Pair { lo: out_lo, hi: out_hi } }.whole
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! select_atomic_rmw {
|
||||
(
|
||||
unsafe fn $name:ident(dst: *mut u64 $(, $($arg:tt)*)?) $(-> $ret_ty:ty)? {
|
||||
|$zacas_fn_binding:ident| $($zacas_fn_body:tt)*
|
||||
}
|
||||
zacas = $zacas_fn:ident;
|
||||
non_seqcst_fallback = $non_seqcst_fallback_fn:ident;
|
||||
seqcst_fallback = $seqcst_fallback_fn:ident;
|
||||
) => {
|
||||
#[inline]
|
||||
unsafe fn $zacas_fn(dst: *mut u64 $(, $($arg)*)?, order: Ordering) $(-> $ret_ty)? {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
// This is not single-copy atomic reads, but this is ok because subsequent
|
||||
// CAS will check for consistency.
|
||||
//
|
||||
// Note that the C++20 memory model does not allow mixed-sized atomic access,
|
||||
// so we must use inline assembly to implement byte_wise_atomic_load.
|
||||
// (i.e., byte-wise atomic based on the standard library's atomic types
|
||||
// cannot be used here).
|
||||
let mut prev = byte_wise_atomic_load(dst);
|
||||
loop {
|
||||
let next = {
|
||||
let $zacas_fn_binding = prev;
|
||||
$($zacas_fn_body)*
|
||||
};
|
||||
match atomic_compare_exchange_weak(dst, prev, next, order, Ordering::Relaxed) {
|
||||
Ok(x) => return x,
|
||||
Err(x) => prev = x,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// If zacas is available at compile-time, we can always use zacas_fn.
|
||||
#[cfg(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))]
|
||||
use self::$zacas_fn as $name;
|
||||
// Otherwise, we need to do run-time detection and can use zacas_fn only if zacas is available.
|
||||
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
|
||||
#[inline]
|
||||
unsafe fn $name(dst: *mut u64 $(, $($arg)*)?, order: Ordering) $(-> $ret_ty)? {
|
||||
fn_alias! {
|
||||
// inline(never) is just a hint and also not strictly necessary
|
||||
// because we use ifunc helper macro, but used for clarity.
|
||||
#[inline(never)]
|
||||
unsafe fn(dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)?;
|
||||
zacas_relaxed_fn = $zacas_fn(Ordering::Relaxed);
|
||||
zacas_acquire_fn = $zacas_fn(Ordering::Acquire);
|
||||
zacas_release_fn = $zacas_fn(Ordering::Release);
|
||||
zacas_acqrel_fn = $zacas_fn(Ordering::AcqRel);
|
||||
zacas_seqcst_fn = $zacas_fn(Ordering::SeqCst);
|
||||
}
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
// we only calls zacas_fn if zacas is available.
|
||||
unsafe {
|
||||
match order {
|
||||
Ordering::Relaxed => {
|
||||
ifunc!(unsafe fn(dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {
|
||||
if detect::detect().zacas() {
|
||||
zacas_relaxed_fn
|
||||
} else {
|
||||
fallback::$non_seqcst_fallback_fn
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::Acquire => {
|
||||
ifunc!(unsafe fn(dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {
|
||||
if detect::detect().zacas() {
|
||||
zacas_acquire_fn
|
||||
} else {
|
||||
fallback::$non_seqcst_fallback_fn
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::Release => {
|
||||
ifunc!(unsafe fn(dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {
|
||||
if detect::detect().zacas() {
|
||||
zacas_release_fn
|
||||
} else {
|
||||
fallback::$non_seqcst_fallback_fn
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::AcqRel => {
|
||||
ifunc!(unsafe fn(dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {
|
||||
if detect::detect().zacas() {
|
||||
zacas_acqrel_fn
|
||||
} else {
|
||||
fallback::$non_seqcst_fallback_fn
|
||||
}
|
||||
})
|
||||
}
|
||||
Ordering::SeqCst => {
|
||||
ifunc!(unsafe fn(dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {
|
||||
if detect::detect().zacas() {
|
||||
zacas_seqcst_fn
|
||||
} else {
|
||||
fallback::$seqcst_fallback_fn
|
||||
}
|
||||
})
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_swap(dst: *mut u64, val: u64) -> u64 {
|
||||
|_x| val
|
||||
}
|
||||
zacas = atomic_swap_zacas;
|
||||
non_seqcst_fallback = atomic_swap_non_seqcst;
|
||||
seqcst_fallback = atomic_swap_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_add(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| x.wrapping_add(val)
|
||||
}
|
||||
zacas = atomic_add_zacas;
|
||||
non_seqcst_fallback = atomic_add_non_seqcst;
|
||||
seqcst_fallback = atomic_add_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_sub(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| x.wrapping_sub(val)
|
||||
}
|
||||
zacas = atomic_sub_zacas;
|
||||
non_seqcst_fallback = atomic_sub_non_seqcst;
|
||||
seqcst_fallback = atomic_sub_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_and(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| x & val
|
||||
}
|
||||
zacas = atomic_and_zacas;
|
||||
non_seqcst_fallback = atomic_and_non_seqcst;
|
||||
seqcst_fallback = atomic_and_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_nand(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| !(x & val)
|
||||
}
|
||||
zacas = atomic_nand_zacas;
|
||||
non_seqcst_fallback = atomic_nand_non_seqcst;
|
||||
seqcst_fallback = atomic_nand_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_or(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| x | val
|
||||
}
|
||||
zacas = atomic_or_zacas;
|
||||
non_seqcst_fallback = atomic_or_non_seqcst;
|
||||
seqcst_fallback = atomic_or_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_xor(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| x ^ val
|
||||
}
|
||||
zacas = atomic_xor_zacas;
|
||||
non_seqcst_fallback = atomic_xor_non_seqcst;
|
||||
seqcst_fallback = atomic_xor_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_max(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
{ core::cmp::max(x as i64, val as i64) as u64 }
|
||||
}
|
||||
}
|
||||
zacas = atomic_max_zacas;
|
||||
non_seqcst_fallback = atomic_max_non_seqcst;
|
||||
seqcst_fallback = atomic_max_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_umax(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| core::cmp::max(x, val)
|
||||
}
|
||||
zacas = atomic_umax_zacas;
|
||||
non_seqcst_fallback = atomic_umax_non_seqcst;
|
||||
seqcst_fallback = atomic_umax_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_min(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| {
|
||||
#[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
|
||||
{ core::cmp::min(x as i64, val as i64) as u64 }
|
||||
}
|
||||
}
|
||||
zacas = atomic_min_zacas;
|
||||
non_seqcst_fallback = atomic_min_non_seqcst;
|
||||
seqcst_fallback = atomic_min_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_umin(dst: *mut u64, val: u64) -> u64 {
|
||||
|x| core::cmp::min(x, val)
|
||||
}
|
||||
zacas = atomic_umin_zacas;
|
||||
non_seqcst_fallback = atomic_umin_non_seqcst;
|
||||
seqcst_fallback = atomic_umin_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_not(dst: *mut u64) -> u64 {
|
||||
|x| !x
|
||||
}
|
||||
zacas = atomic_not_zacas;
|
||||
non_seqcst_fallback = atomic_not_non_seqcst;
|
||||
seqcst_fallback = atomic_not_seqcst;
|
||||
}
|
||||
select_atomic_rmw! {
|
||||
unsafe fn atomic_neg(dst: *mut u64) -> u64 {
|
||||
|x| x.wrapping_neg()
|
||||
}
|
||||
zacas = atomic_neg_zacas;
|
||||
non_seqcst_fallback = atomic_neg_non_seqcst;
|
||||
seqcst_fallback = atomic_neg_seqcst;
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn is_lock_free() -> bool {
|
||||
#[cfg(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"))]
|
||||
{
|
||||
// zacas is available at compile-time.
|
||||
true
|
||||
}
|
||||
#[cfg(not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")))]
|
||||
{
|
||||
detect::detect().zacas()
|
||||
}
|
||||
}
|
||||
const IS_ALWAYS_LOCK_FREE: bool =
|
||||
cfg!(any(target_feature = "zacas", portable_atomic_target_feature = "zacas"));
|
||||
|
||||
atomic64!(AtomicI64, i64, atomic_max, atomic_min);
|
||||
atomic64!(AtomicU64, u64, atomic_umax, atomic_umin);
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
test_atomic_int!(i64);
|
||||
test_atomic_int!(u64);
|
||||
|
||||
// load/store/swap implementation is not affected by signedness, so it is
|
||||
// enough to test only unsigned types.
|
||||
stress_test!(u64);
|
||||
}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Atomic load/store implementation on AVR.
|
||||
|
||||
Operations not supported here are provided by disabling interrupts.
|
||||
See also src/imp/interrupt/avr.rs.
|
||||
|
||||
See "Atomic operation overview by architecture" in atomic-maybe-uninit for a more comprehensive and
|
||||
detailed description of the atomic and synchronize instructions in this architecture:
|
||||
https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#avr
|
||||
|
||||
Note: Ordering is always SeqCst.
|
||||
|
||||
Refs:
|
||||
- AVR® Instruction Set Manual, Rev. DS40002198B
|
||||
https://ww1.microchip.com/downloads/en/DeviceDoc/AVR-InstructionSet-Manual-DS40002198.pdf
|
||||
- atomic-maybe-uninit
|
||||
https://github.com/taiki-e/atomic-maybe-uninit
|
||||
|
||||
See tests/asm-test/asm/portable-atomic for generated assembly.
|
||||
*/
|
||||
|
||||
use core::{arch::asm, cell::UnsafeCell, sync::atomic::Ordering};
|
||||
|
||||
macro_rules! atomic8 {
|
||||
($atomic_type:ident, $value_type:ty) => {
|
||||
#[repr(transparent)]
|
||||
pub(crate) struct $atomic_type {
|
||||
v: UnsafeCell<$value_type>,
|
||||
}
|
||||
|
||||
// Send is implicitly implemented for atomic integers, but not for atomic pointers.
|
||||
// SAFETY: any data races are prevented by atomic operations.
|
||||
unsafe impl Send for $atomic_type {}
|
||||
// SAFETY: any data races are prevented by atomic operations.
|
||||
unsafe impl Sync for $atomic_type {}
|
||||
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
|
||||
pub(crate) fn load(&self, order: Ordering) -> $value_type {
|
||||
crate::utils::assert_load_ordering(order);
|
||||
let src = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
let out;
|
||||
asm!(
|
||||
"ld {out}, Z", // atomic { out = *Z }
|
||||
out = out(reg) out,
|
||||
in("Z") src,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
|
||||
pub(crate) fn store(&self, val: $value_type, order: Ordering) {
|
||||
crate::utils::assert_store_ordering(order);
|
||||
let dst = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
asm!(
|
||||
"st Z, {val}", // atomic { *Z = val }
|
||||
val = in(reg) val,
|
||||
in("Z") dst,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(target_feature = "rmw", portable_atomic_target_feature = "rmw"))]
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
pub(crate) fn swap(&self, val: $value_type, _order: Ordering) -> $value_type {
|
||||
let dst = self.v.get();
|
||||
let out;
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
// cfg guarantee that the CPU supports RMW instructions.
|
||||
unsafe {
|
||||
asm!(
|
||||
"xch Z, {val}", // atomic { _x = *Z; *Z = val; val = _x }
|
||||
val = inout(reg) val => out,
|
||||
in("Z") dst,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_and(&self, val: $value_type, _order: Ordering) -> $value_type {
|
||||
let dst = self.v.get();
|
||||
let out;
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
// cfg guarantee that the CPU supports RMW instructions.
|
||||
unsafe {
|
||||
asm!(
|
||||
"lac Z, {val}", // atomic { _x = *Z; *Z &= !val; val = _x }
|
||||
val = inout(reg) !val => out,
|
||||
in("Z") dst,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_or(&self, val: $value_type, _order: Ordering) -> $value_type {
|
||||
let dst = self.v.get();
|
||||
let out;
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
// cfg guarantee that the CPU supports RMW instructions.
|
||||
unsafe {
|
||||
asm!(
|
||||
"las Z, {val}", // atomic { _x = *Z; *Z |= val; val = _x }
|
||||
val = inout(reg) val => out,
|
||||
in("Z") dst,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_xor(&self, val: $value_type, _order: Ordering) -> $value_type {
|
||||
let dst = self.v.get();
|
||||
let out;
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
// cfg guarantee that the CPU supports RMW instructions.
|
||||
unsafe {
|
||||
asm!(
|
||||
"lat Z, {val}", // atomic { _x = *Z; *Z ^= val; val = _x }
|
||||
val = inout(reg) val => out,
|
||||
in("Z") dst,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_not(&self, order: Ordering) -> $value_type {
|
||||
self.fetch_xor(!0, order)
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
atomic8!(AtomicI8, i8);
|
||||
atomic8!(AtomicU8, u8);
|
||||
+574
@@ -0,0 +1,574 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Wrap the standard library's atomic types in newtype.
|
||||
|
||||
This is not a reexport, because we want to backport changes like
|
||||
https://github.com/rust-lang/rust/pull/98383 to old compilers.
|
||||
*/
|
||||
|
||||
use core::{cell::UnsafeCell, marker::PhantomData, sync::atomic::Ordering};
|
||||
|
||||
// core::panic::RefUnwindSafe is only available on Rust 1.56+, so on pre-1.56
|
||||
// Rust, we implement RefUnwindSafe when "std" feature is enabled.
|
||||
// However, on pre-1.56 Rust, the standard library's atomic types implement
|
||||
// RefUnwindSafe when "linked to std", and that's behavior that our other atomic
|
||||
// implementations can't emulate, so use PhantomData<NotRefUnwindSafe> to match
|
||||
// conditions where our other atomic implementations implement RefUnwindSafe.
|
||||
//
|
||||
// If we do not do this, for example, downstream that is only tested on x86_64
|
||||
// may incorrectly assume that AtomicU64 always implements RefUnwindSafe even on
|
||||
// older rustc, and may be broken on platforms where std AtomicU64 is not available.
|
||||
struct NotRefUnwindSafe(UnsafeCell<()>);
|
||||
// SAFETY: this is a marker type and we'll never access the value.
|
||||
unsafe impl Sync for NotRefUnwindSafe {}
|
||||
|
||||
#[repr(transparent)]
|
||||
pub(crate) struct AtomicPtr<T> {
|
||||
pub(super) inner: core::sync::atomic::AtomicPtr<T>,
|
||||
// Prevent RefUnwindSafe from being propagated from the std atomic type. See NotRefUnwindSafe for more.
|
||||
_not_ref_unwind_safe: PhantomData<NotRefUnwindSafe>,
|
||||
}
|
||||
impl<T> AtomicPtr<T> {
|
||||
#[inline]
|
||||
pub(crate) const fn new(v: *mut T) -> Self {
|
||||
Self { inner: core::sync::atomic::AtomicPtr::new(v), _not_ref_unwind_safe: PhantomData }
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn is_lock_free() -> bool {
|
||||
Self::IS_ALWAYS_LOCK_FREE
|
||||
}
|
||||
pub(crate) const IS_ALWAYS_LOCK_FREE: bool = true;
|
||||
const_fn! {
|
||||
const_if: #[cfg(not(portable_atomic_no_const_raw_ptr_deref))];
|
||||
#[inline]
|
||||
pub(crate) const fn as_ptr(&self) -> *mut *mut T {
|
||||
// SAFETY: Self is #[repr(C)] and internally UnsafeCell<*mut T>.
|
||||
// See also https://github.com/rust-lang/rust/pull/66705 and
|
||||
// https://github.com/rust-lang/rust/issues/66136#issuecomment-557867116.
|
||||
unsafe { (*(self as *const Self as *const UnsafeCell<*mut T>)).get() }
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(not(all(
|
||||
target_arch = "arm",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(target_os = "linux", target_os = "android"),
|
||||
not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
|
||||
)))]
|
||||
impl<T> AtomicPtr<T> {
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn load(&self, order: Ordering) -> *mut T {
|
||||
crate::utils::assert_load_ordering(order); // for track_caller (compiler can omit double check)
|
||||
self.inner.load(order)
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn store(&self, ptr: *mut T, order: Ordering) {
|
||||
crate::utils::assert_store_ordering(order); // for track_caller (compiler can omit double check)
|
||||
self.inner.store(ptr, order);
|
||||
}
|
||||
}
|
||||
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
|
||||
#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
|
||||
items!({
|
||||
#[cfg(not(all(
|
||||
target_arch = "arm",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(target_os = "linux", target_os = "android"),
|
||||
not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
|
||||
)))]
|
||||
impl<T> AtomicPtr<T> {
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn compare_exchange(
|
||||
&self,
|
||||
current: *mut T,
|
||||
new: *mut T,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<*mut T, *mut T> {
|
||||
crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)
|
||||
#[cfg(portable_atomic_no_stronger_failure_ordering)]
|
||||
let success = crate::utils::upgrade_success_ordering(success, failure);
|
||||
self.inner.compare_exchange(current, new, success, failure)
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn compare_exchange_weak(
|
||||
&self,
|
||||
current: *mut T,
|
||||
new: *mut T,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<*mut T, *mut T> {
|
||||
crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)
|
||||
#[cfg(portable_atomic_no_stronger_failure_ordering)]
|
||||
let success = crate::utils::upgrade_success_ordering(success, failure);
|
||||
self.inner.compare_exchange_weak(current, new, success, failure)
|
||||
}
|
||||
}
|
||||
// Ideally, we would always use AtomicPtr::fetch_* since it is strict-provenance
|
||||
// compatible, but it requires 1.91+. So, for now emulate it only on cfg(miri).
|
||||
// Code using AtomicUsize::fetch_* via casts is still permissive-provenance
|
||||
// compatible and is sound.
|
||||
#[cfg(portable_atomic_no_strict_provenance_atomic_ptr)]
|
||||
impl<T> AtomicPtr<T> {
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_byte_add(&self, val: usize, order: Ordering) -> *mut T {
|
||||
#[cfg(miri)]
|
||||
{
|
||||
#[cfg(portable_atomic_no_strict_provenance)]
|
||||
use crate::utils::ptr::PtrExt as _;
|
||||
self.fetch_update_(order, |x| x.with_addr(x.addr().wrapping_add(val)))
|
||||
}
|
||||
#[cfg(not(miri))]
|
||||
{
|
||||
crate::utils::ptr::with_exposed_provenance_mut(
|
||||
self.as_atomic_usize().fetch_add(val, order),
|
||||
)
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_byte_sub(&self, val: usize, order: Ordering) -> *mut T {
|
||||
#[cfg(miri)]
|
||||
{
|
||||
#[cfg(portable_atomic_no_strict_provenance)]
|
||||
use crate::utils::ptr::PtrExt as _;
|
||||
self.fetch_update_(order, |x| x.with_addr(x.addr().wrapping_sub(val)))
|
||||
}
|
||||
#[cfg(not(miri))]
|
||||
{
|
||||
crate::utils::ptr::with_exposed_provenance_mut(
|
||||
self.as_atomic_usize().fetch_sub(val, order),
|
||||
)
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_or(&self, val: usize, order: Ordering) -> *mut T {
|
||||
#[cfg(miri)]
|
||||
{
|
||||
#[cfg(portable_atomic_no_strict_provenance)]
|
||||
use crate::utils::ptr::PtrExt as _;
|
||||
self.fetch_update_(order, |x| x.with_addr(x.addr() | val))
|
||||
}
|
||||
#[cfg(not(miri))]
|
||||
{
|
||||
crate::utils::ptr::with_exposed_provenance_mut(
|
||||
self.as_atomic_usize().fetch_or(val, order),
|
||||
)
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_and(&self, val: usize, order: Ordering) -> *mut T {
|
||||
#[cfg(miri)]
|
||||
{
|
||||
#[cfg(portable_atomic_no_strict_provenance)]
|
||||
use crate::utils::ptr::PtrExt as _;
|
||||
self.fetch_update_(order, |x| x.with_addr(x.addr() & val))
|
||||
}
|
||||
#[cfg(not(miri))]
|
||||
{
|
||||
crate::utils::ptr::with_exposed_provenance_mut(
|
||||
self.as_atomic_usize().fetch_and(val, order),
|
||||
)
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_xor(&self, val: usize, order: Ordering) -> *mut T {
|
||||
#[cfg(miri)]
|
||||
{
|
||||
#[cfg(portable_atomic_no_strict_provenance)]
|
||||
use crate::utils::ptr::PtrExt as _;
|
||||
self.fetch_update_(order, |x| x.with_addr(x.addr() ^ val))
|
||||
}
|
||||
#[cfg(not(miri))]
|
||||
{
|
||||
crate::utils::ptr::with_exposed_provenance_mut(
|
||||
self.as_atomic_usize().fetch_xor(val, order),
|
||||
)
|
||||
}
|
||||
}
|
||||
#[cfg(miri)]
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
fn fetch_update_<F>(&self, order: Ordering, mut f: F) -> *mut T
|
||||
where
|
||||
F: FnMut(*mut T) -> *mut T,
|
||||
{
|
||||
// This is a private function and all instances of `f` only operate on the value
|
||||
// loaded, so there is no need to synchronize the first load/failed CAS.
|
||||
let mut prev = self.load(Ordering::Relaxed);
|
||||
loop {
|
||||
let next = f(prev);
|
||||
match self.compare_exchange_weak(prev, next, order, Ordering::Relaxed) {
|
||||
Ok(x) => return x,
|
||||
Err(next_prev) => prev = next_prev,
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(not(miri))]
|
||||
#[inline(always)]
|
||||
fn as_atomic_usize(&self) -> &AtomicUsize {
|
||||
static_assert!(
|
||||
core::mem::size_of::<AtomicPtr<()>>() == core::mem::size_of::<AtomicUsize>()
|
||||
);
|
||||
static_assert!(
|
||||
core::mem::align_of::<AtomicPtr<()>>() == core::mem::align_of::<AtomicUsize>()
|
||||
);
|
||||
// SAFETY: AtomicPtr and AtomicUsize have the same layout,
|
||||
// and both access data in the same way.
|
||||
unsafe { &*(self as *const Self as *const AtomicUsize) }
|
||||
}
|
||||
}
|
||||
#[cfg(not(all(
|
||||
any(target_arch = "x86", target_arch = "x86_64"),
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
)))]
|
||||
impl_default_bit_opts!(AtomicPtr, usize);
|
||||
});
|
||||
impl<T> core::ops::Deref for AtomicPtr<T> {
|
||||
type Target = core::sync::atomic::AtomicPtr<T>;
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.inner
|
||||
}
|
||||
}
|
||||
|
||||
// NB: Sync with atomic_int! macro in arm_linux.rs
|
||||
macro_rules! atomic_int {
|
||||
($atomic_type:ident, $int_type:ident) => {
|
||||
#[repr(transparent)]
|
||||
pub(crate) struct $atomic_type {
|
||||
pub(super) inner: core::sync::atomic::$atomic_type,
|
||||
// Prevent RefUnwindSafe from being propagated from the std atomic type. See NotRefUnwindSafe for more.
|
||||
_not_ref_unwind_safe: PhantomData<NotRefUnwindSafe>,
|
||||
}
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
pub(crate) const fn new(v: $int_type) -> Self {
|
||||
Self {
|
||||
inner: core::sync::atomic::$atomic_type::new(v),
|
||||
_not_ref_unwind_safe: PhantomData,
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn is_lock_free() -> bool {
|
||||
Self::IS_ALWAYS_LOCK_FREE
|
||||
}
|
||||
// ESP-IDF targets' 64-bit atomics are not lock-free.
|
||||
// https://github.com/rust-lang/rust/pull/115577#issuecomment-1732259297
|
||||
pub(crate) const IS_ALWAYS_LOCK_FREE: bool = cfg!(not(all(
|
||||
any(target_arch = "riscv32", target_arch = "xtensa"),
|
||||
target_os = "espidf",
|
||||
))) | (core::mem::size_of::<$int_type>()
|
||||
< 8);
|
||||
const_fn! {
|
||||
const_if: #[cfg(not(portable_atomic_no_const_raw_ptr_deref))];
|
||||
#[inline]
|
||||
pub(crate) const fn as_ptr(&self) -> *mut $int_type {
|
||||
// SAFETY: Self is #[repr(C)] and internally UnsafeCell<$int_type>.
|
||||
// See also https://github.com/rust-lang/rust/pull/66705 and
|
||||
// https://github.com/rust-lang/rust/issues/66136#issuecomment-557867116.
|
||||
unsafe {
|
||||
(*(self as *const Self as *const UnsafeCell<$int_type>)).get()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(not(all(
|
||||
target_arch = "arm",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(target_os = "linux", target_os = "android"),
|
||||
not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
|
||||
)))]
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn load(&self, order: Ordering) -> $int_type {
|
||||
crate::utils::assert_load_ordering(order); // for track_caller (compiler can omit double check)
|
||||
self.inner.load(order)
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn store(&self, val: $int_type, order: Ordering) {
|
||||
crate::utils::assert_store_ordering(order); // for track_caller (compiler can omit double check)
|
||||
self.inner.store(val, order);
|
||||
}
|
||||
}
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(not(portable_atomic_no_atomic_cas))
|
||||
)]
|
||||
#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
|
||||
items!({
|
||||
impl_default_no_fetch_ops!($atomic_type, $int_type);
|
||||
#[cfg(not(all(
|
||||
target_arch = "arm",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(target_os = "linux", target_os = "android"),
|
||||
not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
|
||||
)))]
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn compare_exchange(
|
||||
&self,
|
||||
current: $int_type,
|
||||
new: $int_type,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<$int_type, $int_type> {
|
||||
crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)
|
||||
#[cfg(portable_atomic_no_stronger_failure_ordering)]
|
||||
let success = crate::utils::upgrade_success_ordering(success, failure);
|
||||
self.inner.compare_exchange(current, new, success, failure)
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn compare_exchange_weak(
|
||||
&self,
|
||||
current: $int_type,
|
||||
new: $int_type,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<$int_type, $int_type> {
|
||||
crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)
|
||||
#[cfg(portable_atomic_no_stronger_failure_ordering)]
|
||||
let success = crate::utils::upgrade_success_ordering(success, failure);
|
||||
self.inner.compare_exchange_weak(current, new, success, failure)
|
||||
}
|
||||
}
|
||||
impl $atomic_type {
|
||||
#[allow(dead_code)]
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
fn fetch_update_<F>(&self, order: Ordering, mut f: F) -> $int_type
|
||||
where
|
||||
F: FnMut($int_type) -> $int_type,
|
||||
{
|
||||
// This is a private function and all instances of `f` only operate on the value
|
||||
// loaded, so there is no need to synchronize the first load/failed CAS.
|
||||
let mut prev = self.load(Ordering::Relaxed);
|
||||
loop {
|
||||
let next = f(prev);
|
||||
match self.compare_exchange_weak(prev, next, order, Ordering::Relaxed) {
|
||||
Ok(x) => return x,
|
||||
Err(next_prev) => prev = next_prev,
|
||||
}
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[cfg(not(portable_atomic_no_atomic_min_max))]
|
||||
{
|
||||
#[cfg(any(
|
||||
all(
|
||||
any(target_arch = "aarch64", target_arch = "arm64ec"),
|
||||
any(
|
||||
target_feature = "lse",
|
||||
portable_atomic_target_feature = "lse",
|
||||
),
|
||||
),
|
||||
all(
|
||||
target_arch = "arm",
|
||||
not(any(
|
||||
target_feature = "v6",
|
||||
portable_atomic_target_feature = "v6",
|
||||
)),
|
||||
),
|
||||
target_arch = "mips",
|
||||
target_arch = "mips32r6",
|
||||
target_arch = "mips64",
|
||||
target_arch = "mips64r6",
|
||||
target_arch = "powerpc",
|
||||
target_arch = "powerpc64",
|
||||
))]
|
||||
{
|
||||
// HACK: the following operations are currently broken (at least on qemu-user):
|
||||
// - aarch64's `AtomicI{8,16}::fetch_{max,min}` (release mode + lse)
|
||||
// - armv5te's `Atomic{I,U}{8,16}::fetch_{max,min}`
|
||||
// - mips's `AtomicI8::fetch_{max,min}` (release mode)
|
||||
// - mipsel's `AtomicI{8,16}::fetch_{max,min}` (debug mode, at least)
|
||||
// - mips64's `AtomicI8::fetch_{max,min}` (release mode)
|
||||
// - mips64el's `AtomicI{8,16}::fetch_{max,min}` (debug mode, at least)
|
||||
// - powerpc's `AtomicI{8,16}::fetch_{max,min}`
|
||||
// - powerpc64's `AtomicI{8,16}::fetch_{max,min}` (debug mode, at least)
|
||||
// - powerpc64le's `AtomicU{8,16}::fetch_{max,min}` (release mode + fat LTO)
|
||||
// See also:
|
||||
// https://github.com/llvm/llvm-project/issues/61880
|
||||
// https://github.com/llvm/llvm-project/issues/61881
|
||||
// https://github.com/llvm/llvm-project/issues/61882
|
||||
// https://github.com/taiki-e/portable-atomic/issues/2
|
||||
// https://github.com/rust-lang/rust/issues/100650
|
||||
if core::mem::size_of::<$int_type>() <= 2 {
|
||||
return self.fetch_update_(order, |x| core::cmp::max(x, val));
|
||||
}
|
||||
}
|
||||
self.inner.fetch_max(val, order)
|
||||
}
|
||||
#[cfg(portable_atomic_no_atomic_min_max)]
|
||||
{
|
||||
self.fetch_update_(order, |x| core::cmp::max(x, val))
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
#[cfg(not(portable_atomic_no_atomic_min_max))]
|
||||
{
|
||||
#[cfg(any(
|
||||
all(
|
||||
any(target_arch = "aarch64", target_arch = "arm64ec"),
|
||||
any(
|
||||
target_feature = "lse",
|
||||
portable_atomic_target_feature = "lse",
|
||||
),
|
||||
),
|
||||
all(
|
||||
target_arch = "arm",
|
||||
not(any(
|
||||
target_feature = "v6",
|
||||
portable_atomic_target_feature = "v6",
|
||||
)),
|
||||
),
|
||||
target_arch = "mips",
|
||||
target_arch = "mips32r6",
|
||||
target_arch = "mips64",
|
||||
target_arch = "mips64r6",
|
||||
target_arch = "powerpc",
|
||||
target_arch = "powerpc64",
|
||||
))]
|
||||
{
|
||||
// HACK: the following operations are currently broken (at least on qemu-user):
|
||||
// - aarch64's `AtomicI{8,16}::fetch_{max,min}` (release mode + lse)
|
||||
// - armv5te's `Atomic{I,U}{8,16}::fetch_{max,min}`
|
||||
// - mips's `AtomicI8::fetch_{max,min}` (release mode)
|
||||
// - mipsel's `AtomicI{8,16}::fetch_{max,min}` (debug mode, at least)
|
||||
// - mips64's `AtomicI8::fetch_{max,min}` (release mode)
|
||||
// - mips64el's `AtomicI{8,16}::fetch_{max,min}` (debug mode, at least)
|
||||
// - powerpc's `AtomicI{8,16}::fetch_{max,min}`
|
||||
// - powerpc64's `AtomicI{8,16}::fetch_{max,min}` (debug mode, at least)
|
||||
// - powerpc64le's `AtomicU{8,16}::fetch_{max,min}` (release mode + fat LTO)
|
||||
// See also:
|
||||
// https://github.com/llvm/llvm-project/issues/61880
|
||||
// https://github.com/llvm/llvm-project/issues/61881
|
||||
// https://github.com/llvm/llvm-project/issues/61882
|
||||
// https://github.com/taiki-e/portable-atomic/issues/2
|
||||
// https://github.com/rust-lang/rust/issues/100650
|
||||
if core::mem::size_of::<$int_type>() <= 2 {
|
||||
return self.fetch_update_(order, |x| core::cmp::min(x, val));
|
||||
}
|
||||
}
|
||||
self.inner.fetch_min(val, order)
|
||||
}
|
||||
#[cfg(portable_atomic_no_atomic_min_max)]
|
||||
{
|
||||
self.fetch_update_(order, |x| core::cmp::min(x, val))
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_not(&self, order: Ordering) -> $int_type {
|
||||
self.fetch_xor(!0, order)
|
||||
}
|
||||
// TODO: provide asm-based implementation on AArch64 without FEAT_LSE, Armv7, RISC-V, etc.
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_neg(&self, order: Ordering) -> $int_type {
|
||||
self.fetch_update_(order, $int_type::wrapping_neg)
|
||||
}
|
||||
}
|
||||
#[cfg(not(all(
|
||||
any(target_arch = "x86", target_arch = "x86_64"),
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
)))]
|
||||
items!({
|
||||
impl_default_bit_opts!($atomic_type, $int_type);
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn not(&self, order: Ordering) {
|
||||
self.fetch_not(order);
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn neg(&self, order: Ordering) {
|
||||
self.fetch_neg(order);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
impl core::ops::Deref for $atomic_type {
|
||||
type Target = core::sync::atomic::$atomic_type;
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.inner
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
atomic_int!(AtomicIsize, isize);
|
||||
atomic_int!(AtomicUsize, usize);
|
||||
#[cfg(not(portable_atomic_no_atomic_load_store))]
|
||||
atomic_int!(AtomicI8, i8);
|
||||
#[cfg(not(portable_atomic_no_atomic_load_store))]
|
||||
atomic_int!(AtomicU8, u8);
|
||||
#[cfg(not(portable_atomic_no_atomic_load_store))]
|
||||
atomic_int!(AtomicI16, i16);
|
||||
#[cfg(not(portable_atomic_no_atomic_load_store))]
|
||||
atomic_int!(AtomicU16, u16);
|
||||
#[cfg(not(portable_atomic_no_atomic_load_store))]
|
||||
#[cfg(not(target_pointer_width = "16"))]
|
||||
atomic_int!(AtomicI32, i32);
|
||||
#[cfg(not(portable_atomic_no_atomic_load_store))]
|
||||
#[cfg(not(target_pointer_width = "16"))]
|
||||
atomic_int!(AtomicU32, u32);
|
||||
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_64)))]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(
|
||||
target_has_atomic = "64",
|
||||
not(any(target_pointer_width = "16", target_pointer_width = "32")),
|
||||
))
|
||||
)]
|
||||
items!({
|
||||
atomic_int!(AtomicI64, i64);
|
||||
atomic_int!(AtomicU64, u64);
|
||||
});
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
# Run-time CPU feature detection
|
||||
|
||||
This module has run-time CPU feature detection implementations.
|
||||
|
||||
Here is the table of targets that support run-time CPU feature detection and the instruction or API used:
|
||||
|
||||
| target_arch | target_os/target_env | instruction/API | features | note |
|
||||
| ----------- | -------------------- | --------------- | -------- | ---- |
|
||||
| x86_64 | all (except for sgx) | cpuid | all | Enabled by default |
|
||||
| aarch64 | linux (gnu/ohos/uclibc) | getauxval | all | Enabled by default (dlsym is used by default if needed for compatibility with older versions) |
|
||||
| aarch64 | linux (musl) | getauxval | all | Only enabled by default when dynamic linking or `std` feature enabled (both disabled by default, see [auxv.rs](auxv.rs)) |
|
||||
| aarch64 | android | getauxval | all | Enabled by default |
|
||||
| aarch64 | freebsd | elf_aux_info | all | Enabled by default |
|
||||
| aarch64 | netbsd | sysctlbyname | all | Enabled by default |
|
||||
| aarch64 | openbsd | sysctl | all | Enabled by default |
|
||||
| aarch64 | macos/ios/tvos/watchos/visionos | sysctlbyname | all | Enabled by default |
|
||||
| aarch64 | illumos | getisax | lse, lse2 | Enabled by default |
|
||||
| aarch64/arm64ec | windows | IsProcessorFeaturePresent | lse, lse2 | Enabled by default |
|
||||
| aarch64 | fuchsia | zx_system_get_features | lse | Enabled by default |
|
||||
| riscv32/riscv64 | linux/android | riscv_hwprobe | all | Enabled by default |
|
||||
| powerpc64 | linux (gnu/ohos/uclibc) | getauxval | all | Enabled by default (dlsym is used by default if needed for compatibility with older versions) |
|
||||
| powerpc64 | linux (musl) | getauxval | all | Only enabled by default when dynamic linking or `std` feature enabled (both disabled by default, see [auxv.rs](auxv.rs)) |
|
||||
| powerpc64 | freebsd | elf_aux_info | all | Enabled by default (dlsym is used by default for compatibility with older versions) |
|
||||
| powerpc64 | openbsd | elf_aux_info | all | Enabled by default (dlsym is used by default for compatibility with older versions) |
|
||||
| powerpc64 | aix | getsystemcfg | all | Requires LLVM 20+. Disabled by default (see [powerpc64_aix.rs](powerpc64_aix.rs)) |
|
||||
|
||||
Run-time detection is enabled by default on most targets and can be disabled with `--cfg portable_atomic_no_outline_atomics`.
|
||||
|
||||
On some targets, run-time detection is disabled by default mainly for compatibility with incomplete build environments or support for it is experimental, and can be enabled by `--cfg portable_atomic_outline_atomics`. (When both cfg are enabled, `*_no_*` cfg is preferred.)
|
||||
|
||||
`dlsym` usually not working with static linking, so detection using implementations that use `dlsym` for compatibility will be disabled if static linking is enabled.
|
||||
You can use `--cfg portable_atomic_outline_atomics` to force the use of non-`dlsym` implementations and enable run-time detection in such an environment.
|
||||
|
||||
For targets not included in the above table, run-time detection is always disabled and works the same as when `--cfg portable_atomic_no_outline_atomics` is set.
|
||||
|
||||
See [auxv.rs](auxv.rs) module-level comments for more details on Linux/Android/FreeBSD/OpenBSD.
|
||||
|
||||
See also [docs about `portable_atomic_no_outline_atomics` cfg](https://github.com/taiki-e/portable-atomic/blob/HEAD/README.md#optional-cfg-no-outline-atomics) in the top-level readme.
|
||||
@@ -0,0 +1,492 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Run-time CPU feature detection on AArch64 Linux/Android/FreeBSD/NetBSD/OpenBSD by parsing system registers.
|
||||
|
||||
As of Rust 1.94, is_aarch64_feature_detected doesn't support run-time detection on NetBSD.
|
||||
https://github.com/rust-lang/rust/blob/1.94.0/library/std_detect/src/detect/mod.rs
|
||||
Run-time detection on OpenBSD by is_aarch64_feature_detected is supported on Rust 1.70+.
|
||||
https://github.com/rust-lang/stdarch/pull/1374
|
||||
|
||||
Refs:
|
||||
- https://developer.arm.com/documentation/ddi0601/2025-06/AArch64-Registers
|
||||
- https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/arm64/cpu-feature-registers.rst
|
||||
- https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/aarch64.rs
|
||||
|
||||
Supported platforms:
|
||||
- Linux 4.11+ (emulate mrs instruction)
|
||||
https://github.com/torvalds/linux/commit/77c97b4ee21290f5f083173d957843b615abbff2
|
||||
- FreeBSD 12.0+ (emulate mrs instruction)
|
||||
https://github.com/freebsd/freebsd-src/commit/398810619cb32abf349f8de23f29510b2ee0839b
|
||||
- NetBSD 9.0+ (through sysctl/sysctlbyname)
|
||||
https://github.com/NetBSD/src/commit/0e9d25528729f7fea53e78275d1bc5039dfe8ffb
|
||||
sysctl/sysctlbyname returns an unsupported error if operation is not supported,
|
||||
so we can safely use this on older versions.
|
||||
- OpenBSD 7.1+ (through sysctl)
|
||||
https://github.com/openbsd/src/commit/d335af936b9d7dd9cf655cae1ce19560c45de6c8
|
||||
sysctl returns an unsupported error if operation is not supported,
|
||||
so we can safely use this on older versions.
|
||||
|
||||
For now, this module is only used on NetBSD/OpenBSD.
|
||||
|
||||
On Linux/Android/FreeBSD, we use auxv.rs and this module is test-only because:
|
||||
- On Linux/Android, this approach requires a higher kernel version than Rust supports,
|
||||
and also does not work with qemu-user (as of QEMU 7.2) and Valgrind (as of Valgrind 3.24).
|
||||
(Looking into HWCAP_CPUID in auxvec, it appears that Valgrind is setting it
|
||||
to false correctly, but qemu-user is setting it to true.)
|
||||
- qemu-user issue seem to be fixed as of QEMU 9.2.
|
||||
- On FreeBSD, this approach does not work on FreeBSD 12 on QEMU (confirmed on
|
||||
FreeBSD 12.{2,3,4}), and we got SIGILL (worked on FreeBSD 13 and 14).
|
||||
*/
|
||||
|
||||
include!("common.rs");
|
||||
|
||||
#[cfg_attr(test, derive(Debug, PartialEq))]
|
||||
struct AA64Reg {
|
||||
aa64isar0: u64,
|
||||
aa64isar1: u64,
|
||||
#[cfg(test)]
|
||||
aa64isar3: u64,
|
||||
aa64mmfr2: u64,
|
||||
}
|
||||
|
||||
#[cold]
|
||||
#[must_use]
|
||||
fn _detect(mut info: CpuInfo) -> CpuInfo {
|
||||
let AA64Reg {
|
||||
aa64isar0,
|
||||
aa64isar1,
|
||||
#[cfg(test)]
|
||||
aa64isar3,
|
||||
aa64mmfr2,
|
||||
} = imp::aa64reg();
|
||||
|
||||
// ID_AA64ISAR0_EL1, AArch64 Instruction Set Attribute Register 0
|
||||
// https://developer.arm.com/documentation/ddi0601/2025-06/AArch64-Registers/ID-AA64ISAR0-EL1--AArch64-Instruction-Set-Attribute-Register-0
|
||||
// Atomic, bits [23:20]
|
||||
// > FEAT_LSE implements the functionality identified by the value 0b0010.
|
||||
// > FEAT_LSE128 implements the functionality identified by the value 0b0011.
|
||||
// > From Armv8.1, the value 0b0000 is not permitted.
|
||||
let atomic = extract(aa64isar0, 23, 20);
|
||||
if atomic >= 0b0010 {
|
||||
info.set(CpuInfoFlag::lse);
|
||||
if atomic >= 0b0011 {
|
||||
info.set(CpuInfoFlag::lse128);
|
||||
}
|
||||
}
|
||||
// ID_AA64ISAR1_EL1, AArch64 Instruction Set Attribute Register 1
|
||||
// https://developer.arm.com/documentation/ddi0601/2025-06/AArch64-Registers/ID-AA64ISAR1-EL1--AArch64-Instruction-Set-Attribute-Register-1
|
||||
// LRCPC, bits [23:20]
|
||||
// > FEAT_LRCPC implements the functionality identified by the value 0b0001.
|
||||
// > FEAT_LRCPC2 implements the functionality identified by the value 0b0010.
|
||||
// > FEAT_LRCPC3 implements the functionality identified by the value 0b0011.
|
||||
// > From Armv8.3, the value 0b0000 is not permitted.
|
||||
// > From Armv8.4, the value 0b0001 is not permitted.
|
||||
let lrcpc = extract(aa64isar1, 23, 20);
|
||||
if lrcpc >= 0b0011 {
|
||||
info.set(CpuInfoFlag::rcpc3);
|
||||
}
|
||||
#[cfg(test)]
|
||||
if lrcpc >= 0b0001 {
|
||||
info.set(CpuInfoFlag::rcpc);
|
||||
if lrcpc >= 0b0010 {
|
||||
info.set(CpuInfoFlag::rcpc2);
|
||||
}
|
||||
}
|
||||
// LS64, bits [63:60]
|
||||
// > FEAT_LS64 implements the functionality identified by 0b0001.
|
||||
// > FEAT_LS64_V implements the functionality identified by 0b0010.
|
||||
// > FEAT_LS64_ACCDATA implements the functionality identified by 0b0011.
|
||||
// > FEAT_LS64WB implements the functionality identified by 0b0100.
|
||||
#[cfg(test)]
|
||||
if extract(aa64isar1, 63, 60) >= 0b0100 {
|
||||
info.set(CpuInfoFlag::ls64wb);
|
||||
}
|
||||
// ID_AA64ISAR3_EL1, AArch64 Instruction Set Attribute Register 3
|
||||
// https://developer.arm.com/documentation/ddi0601/2025-06/AArch64-Registers/ID-AA64ISAR3-EL1--AArch64-Instruction-Set-Attribute-Register-3
|
||||
// LSFE, bits [19:16]
|
||||
// > FEAT_LSFE implements the functionality identified by the value 0b0001
|
||||
#[cfg(test)]
|
||||
if extract(aa64isar3, 19, 16) >= 0b0001 {
|
||||
info.set(CpuInfoFlag::lsfe);
|
||||
}
|
||||
// ID_AA64MMFR2_EL1, AArch64 Memory Model Feature Register 2
|
||||
// https://developer.arm.com/documentation/ddi0601/2025-06/AArch64-Registers/ID-AA64MMFR2-EL1--AArch64-Memory-Model-Feature-Register-2
|
||||
// AT, bits [35:32]
|
||||
// > FEAT_LSE2 implements the functionality identified by the value 0b0001.
|
||||
// > From Armv8.4, the value 0b0000 is not permitted.
|
||||
if extract(aa64mmfr2, 35, 32) >= 0b0001 {
|
||||
info.set(CpuInfoFlag::lse2);
|
||||
}
|
||||
info
|
||||
}
|
||||
|
||||
fn extract(x: u64, high: usize, low: usize) -> u64 {
|
||||
(x >> low) & ((1 << (high - low + 1)) - 1)
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "netbsd", target_os = "openbsd")))]
|
||||
mod imp {
|
||||
// This module is test-only. See parent module docs for details.
|
||||
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
use core::arch::asm;
|
||||
|
||||
use super::AA64Reg;
|
||||
|
||||
pub(super) fn aa64reg() -> AA64Reg {
|
||||
// SAFETY: This is safe on FreeBSD 12.0+. FreeBSD 11 was EoL on 2021-09-30.
|
||||
// Note that stdarch has been doing the same thing since before FreeBSD 11 was EoL.
|
||||
// https://github.com/rust-lang/stdarch/pull/611
|
||||
unsafe {
|
||||
let aa64isar0: u64;
|
||||
asm!(
|
||||
"mrs {}, ID_AA64ISAR0_EL1",
|
||||
out(reg) aa64isar0,
|
||||
options(pure, nomem, nostack, preserves_flags),
|
||||
);
|
||||
let aa64isar1: u64;
|
||||
asm!(
|
||||
"mrs {}, ID_AA64ISAR1_EL1",
|
||||
out(reg) aa64isar1,
|
||||
options(pure, nomem, nostack, preserves_flags),
|
||||
);
|
||||
#[cfg(test)]
|
||||
#[cfg(not(portable_atomic_pre_llvm_18))]
|
||||
let aa64isar3: u64;
|
||||
// ID_AA64ISAR3_EL1 is only recognized on LLVM 18+.
|
||||
// https://github.com/llvm/llvm-project/commit/17baba9fa2728b1b1134f9dccb9318debd5a9a1b
|
||||
#[cfg(test)]
|
||||
#[cfg(not(portable_atomic_pre_llvm_18))]
|
||||
asm!(
|
||||
"mrs {}, ID_AA64ISAR3_EL1",
|
||||
out(reg) aa64isar3,
|
||||
options(pure, nomem, nostack, preserves_flags),
|
||||
);
|
||||
let aa64mmfr2: u64;
|
||||
asm!(
|
||||
"mrs {}, ID_AA64MMFR2_EL1",
|
||||
out(reg) aa64mmfr2,
|
||||
options(pure, nomem, nostack, preserves_flags),
|
||||
);
|
||||
AA64Reg {
|
||||
aa64isar0,
|
||||
aa64isar1,
|
||||
#[cfg(test)]
|
||||
#[cfg(not(portable_atomic_pre_llvm_18))]
|
||||
aa64isar3,
|
||||
#[cfg(test)]
|
||||
#[cfg(portable_atomic_pre_llvm_18)]
|
||||
aa64isar3: 0,
|
||||
aa64mmfr2,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "netbsd")]
|
||||
mod imp {
|
||||
// NetBSD doesn't trap the mrs instruction, but exposes the system registers through sysctl.
|
||||
// https://github.com/NetBSD/src/commit/0e9d25528729f7fea53e78275d1bc5039dfe8ffb
|
||||
// https://github.com/golang/sys/commit/ef9fd89ba245e184bdd308f7f2b4f3c551fa5b0f
|
||||
|
||||
use core::{mem, ptr};
|
||||
|
||||
use super::AA64Reg;
|
||||
|
||||
// libc requires Rust 1.63
|
||||
#[allow(non_camel_case_types)]
|
||||
pub(super) mod ffi {
|
||||
pub(crate) use crate::utils::ffi::{CStr, c_char, c_int, c_size_t, c_void};
|
||||
|
||||
sys_struct!({
|
||||
// Defined in machine/armreg.h.
|
||||
// https://github.com/NetBSD/src/blob/121914f187d0f46c2ce43f00531d2c500d8e81e5/sys/arch/aarch64/include/armreg.h#L1863
|
||||
pub(crate) struct aarch64_sysctl_cpu_id {
|
||||
// NetBSD 9.0+
|
||||
// https://github.com/NetBSD/src/commit/0e9d25528729f7fea53e78275d1bc5039dfe8ffb
|
||||
pub(crate) ac_midr: u64,
|
||||
pub(crate) ac_revidr: u64,
|
||||
pub(crate) ac_mpidr: u64,
|
||||
pub(crate) ac_aa64dfr0: u64,
|
||||
pub(crate) ac_aa64dfr1: u64,
|
||||
pub(crate) ac_aa64isar0: u64,
|
||||
pub(crate) ac_aa64isar1: u64,
|
||||
pub(crate) ac_aa64mmfr0: u64,
|
||||
pub(crate) ac_aa64mmfr1: u64,
|
||||
pub(crate) ac_aa64mmfr2: u64,
|
||||
pub(crate) ac_aa64pfr0: u64,
|
||||
pub(crate) ac_aa64pfr1: u64,
|
||||
pub(crate) ac_aa64zfr0: u64,
|
||||
pub(crate) ac_mvfr0: u32,
|
||||
pub(crate) ac_mvfr1: u32,
|
||||
pub(crate) ac_mvfr2: u32,
|
||||
// NetBSD 10.0+
|
||||
// https://github.com/NetBSD/src/commit/0c7bdc13f0e332cccec56e307f023b4888638973
|
||||
pub(crate) ac_pad: u32,
|
||||
pub(crate) ac_clidr: u64,
|
||||
pub(crate) ac_ctr: u64,
|
||||
}
|
||||
});
|
||||
|
||||
sys_fn!({
|
||||
extern "C" {
|
||||
// Defined in sys/sysctl.h.
|
||||
// https://man.netbsd.org/sysctl.3
|
||||
// https://github.com/NetBSD/src/blob/121914f187d0f46c2ce43f00531d2c500d8e81e5/sys/sys/sysctl.h
|
||||
pub(crate) fn sysctlbyname(
|
||||
name: *const c_char,
|
||||
old_p: *mut c_void,
|
||||
old_len_p: *mut c_size_t,
|
||||
new_p: *const c_void,
|
||||
new_len: c_size_t,
|
||||
) -> c_int;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn sysctl_cpu_id(name: &ffi::CStr) -> Option<AA64Reg> {
|
||||
const OUT_LEN: ffi::c_size_t =
|
||||
mem::size_of::<ffi::aarch64_sysctl_cpu_id>() as ffi::c_size_t;
|
||||
|
||||
// SAFETY: all fields of aarch64_sysctl_cpu_id are zero-able and we use
|
||||
// the result when machdep.cpuN.cpu_id sysctl was successful.
|
||||
let mut buf: ffi::aarch64_sysctl_cpu_id = unsafe { mem::zeroed() };
|
||||
let mut out_len = OUT_LEN;
|
||||
// SAFETY:
|
||||
// - `name` a valid C string.
|
||||
// - `out_len` does not exceed the size of the value at `buf`.
|
||||
// - `sysctlbyname` is thread-safe.
|
||||
let res = unsafe {
|
||||
ffi::sysctlbyname(
|
||||
name.as_ptr(),
|
||||
(&mut buf as *mut ffi::aarch64_sysctl_cpu_id).cast::<ffi::c_void>(),
|
||||
&mut out_len,
|
||||
ptr::null_mut(),
|
||||
0,
|
||||
)
|
||||
};
|
||||
if res != 0 {
|
||||
return None;
|
||||
}
|
||||
Some(AA64Reg {
|
||||
aa64isar0: buf.ac_aa64isar0,
|
||||
aa64isar1: buf.ac_aa64isar1,
|
||||
#[cfg(test)]
|
||||
aa64isar3: 0,
|
||||
aa64mmfr2: buf.ac_aa64mmfr2,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn aa64reg() -> AA64Reg {
|
||||
// Get system registers for cpu0.
|
||||
// If failed, returns default because machdep.cpuN.cpu_id sysctl is not available.
|
||||
// machdep.cpuN.cpu_id sysctl was added in NetBSD 9.0 so it is not available on older versions.
|
||||
// It is ok to check only cpu0, even if there are more CPUs.
|
||||
// https://github.com/NetBSD/src/commit/bd9707e06ea7d21b5c24df6dfc14cb37c2819416
|
||||
// https://github.com/golang/sys/commit/ef9fd89ba245e184bdd308f7f2b4f3c551fa5b0f
|
||||
match sysctl_cpu_id(c!("machdep.cpu0.cpu_id")) {
|
||||
Some(cpu_id) => cpu_id,
|
||||
None => AA64Reg {
|
||||
aa64isar0: 0,
|
||||
aa64isar1: 0,
|
||||
#[cfg(test)]
|
||||
aa64isar3: 0,
|
||||
aa64mmfr2: 0,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "openbsd")]
|
||||
mod imp {
|
||||
// OpenBSD doesn't trap the mrs instruction, but exposes the system registers through sysctl.
|
||||
// https://github.com/openbsd/src/commit/d335af936b9d7dd9cf655cae1ce19560c45de6c8
|
||||
// https://github.com/golang/go/commit/cd54ef1f61945459486e9eea2f016d99ef1da925
|
||||
|
||||
use core::{mem, ptr};
|
||||
|
||||
use super::AA64Reg;
|
||||
|
||||
// libc requires Rust 1.63
|
||||
pub(super) mod ffi {
|
||||
pub(crate) use crate::utils::ffi::{c_int, c_size_t, c_uint, c_void};
|
||||
|
||||
sys_const!({
|
||||
// Defined in sys/sysctl.h.
|
||||
// https://github.com/openbsd/src/blob/d2e81eb18e50973c87ae2c9c2ba348ff3ff89cf9/sys/sys/sysctl.h
|
||||
pub(crate) const CTL_MACHDEP: c_int = 7;
|
||||
|
||||
// Defined in machine/cpu.h.
|
||||
// https://github.com/openbsd/src/blob/d2e81eb18e50973c87ae2c9c2ba348ff3ff89cf9/sys/arch/arm64/include/cpu.h
|
||||
// OpenBSD 7.1+
|
||||
// https://github.com/openbsd/src/commit/d335af936b9d7dd9cf655cae1ce19560c45de6c8
|
||||
pub(crate) const CPU_ID_AA64ISAR0: c_int = 2;
|
||||
pub(crate) const CPU_ID_AA64ISAR1: c_int = 3;
|
||||
// OpenBSD 7.3+
|
||||
// https://github.com/openbsd/src/commit/c7654cd65262d532212f65123ee3905ba200365c
|
||||
// However, on OpenBSD 7.3-7.5, querying CPU_ID_AA64MMFR2 always returns 0.
|
||||
// https://github.com/openbsd/src/commit/e8331b74e5c20302d4bd948c9db722af688ccfc1
|
||||
pub(crate) const CPU_ID_AA64MMFR2: c_int = 7;
|
||||
});
|
||||
|
||||
sys_fn!({
|
||||
extern "C" {
|
||||
// Defined in sys/sysctl.h.
|
||||
// https://man.openbsd.org/sysctl.2
|
||||
// https://github.com/openbsd/src/blob/d2e81eb18e50973c87ae2c9c2ba348ff3ff89cf9/sys/sys/sysctl.h
|
||||
pub(crate) fn sysctl(
|
||||
name: *const c_int,
|
||||
name_len: c_uint,
|
||||
old_p: *mut c_void,
|
||||
old_len_p: *mut c_size_t,
|
||||
new_p: *mut c_void,
|
||||
new_len: c_size_t,
|
||||
) -> c_int;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// sysctl returns an unsupported error if operation is not supported,
|
||||
// so we can safely use this function on older versions of OpenBSD.
|
||||
pub(super) fn aa64reg() -> AA64Reg {
|
||||
let aa64isar0 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64ISAR0]).unwrap_or(0);
|
||||
let aa64isar1 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64ISAR1]).unwrap_or(0);
|
||||
let aa64mmfr2 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64MMFR2]).unwrap_or(0);
|
||||
AA64Reg {
|
||||
aa64isar0,
|
||||
aa64isar1,
|
||||
#[cfg(test)]
|
||||
aa64isar3: 0,
|
||||
aa64mmfr2,
|
||||
}
|
||||
}
|
||||
|
||||
fn sysctl64(mib: &[ffi::c_int]) -> Option<u64> {
|
||||
const OUT_LEN: ffi::c_size_t = mem::size_of::<u64>() as ffi::c_size_t;
|
||||
let mut out = 0_u64;
|
||||
let mut out_len = OUT_LEN;
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let mib_len = mib.len() as ffi::c_uint;
|
||||
// SAFETY:
|
||||
// - `mib_len` does not exceed the size of `mib`.
|
||||
// - `out_len` does not exceed the size of `out`.
|
||||
// - `sysctl` is thread-safe.
|
||||
let res = unsafe {
|
||||
ffi::sysctl(
|
||||
mib.as_ptr(),
|
||||
mib_len,
|
||||
(&mut out as *mut u64).cast::<ffi::c_void>(),
|
||||
&mut out_len,
|
||||
ptr::null_mut(),
|
||||
0,
|
||||
)
|
||||
};
|
||||
// OpenBSD sysctl returns -1 on failure.
|
||||
if res == -1 {
|
||||
return None;
|
||||
}
|
||||
debug_assert_eq!(out_len, OUT_LEN);
|
||||
Some(out)
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
clippy::alloc_instead_of_core,
|
||||
clippy::std_instead_of_alloc,
|
||||
clippy::std_instead_of_core,
|
||||
clippy::undocumented_unsafe_blocks,
|
||||
clippy::wildcard_imports
|
||||
)]
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
#[cfg_attr(portable_atomic_test_detect_false, ignore = "detection disabled")]
|
||||
fn test_aa64reg() {
|
||||
let AA64Reg { aa64isar0, aa64isar1, aa64isar3, aa64mmfr2 } = imp::aa64reg();
|
||||
test_helper::eprintln_nocapture!(
|
||||
"aa64isar0={},aa64isar1={},aa64isar3={},aa64mmfr2={}",
|
||||
aa64isar0,
|
||||
aa64isar1,
|
||||
aa64isar3,
|
||||
aa64mmfr2,
|
||||
);
|
||||
let atomic = extract(aa64isar0, 23, 20);
|
||||
if detect().lse() {
|
||||
if detect().lse128() {
|
||||
assert_eq!(atomic, 0b0011);
|
||||
} else {
|
||||
assert_eq!(atomic, 0b0010);
|
||||
}
|
||||
} else {
|
||||
assert_eq!(atomic, 0b0000);
|
||||
}
|
||||
let lrcpc = extract(aa64isar1, 23, 20);
|
||||
if detect().rcpc() {
|
||||
if detect().rcpc2() {
|
||||
if detect().rcpc3() {
|
||||
assert_eq!(lrcpc, 0b0011);
|
||||
} else {
|
||||
assert_eq!(lrcpc, 0b0010);
|
||||
}
|
||||
} else {
|
||||
assert_eq!(lrcpc, 0b0001);
|
||||
}
|
||||
} else {
|
||||
assert_eq!(lrcpc, 0b0000);
|
||||
}
|
||||
let ls64 = extract(aa64isar1, 63, 60);
|
||||
if detect().ls64wb() {
|
||||
assert_eq!(ls64, 0b0100);
|
||||
} else {
|
||||
assert!(ls64 < 0b0100, "{}", ls64);
|
||||
}
|
||||
let lsfe = extract(aa64isar3, 19, 16);
|
||||
if detect().lsfe() {
|
||||
assert_eq!(lsfe, 0b0001);
|
||||
} else {
|
||||
assert_eq!(lsfe, 0b0000);
|
||||
}
|
||||
let at = extract(aa64mmfr2, 35, 32);
|
||||
if detect().lse2() {
|
||||
assert_eq!(at, 0b0001);
|
||||
} else {
|
||||
assert_eq!(at, 0b0000);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "openbsd")]
|
||||
#[test]
|
||||
fn test_alternative() {
|
||||
use std::{format, process::Command, string::String};
|
||||
|
||||
// Call sysctl command instead of libc API.
|
||||
//
|
||||
// This is used only for testing.
|
||||
struct SysctlMachdepOutput(String);
|
||||
impl SysctlMachdepOutput {
|
||||
fn new() -> Self {
|
||||
let output = Command::new("sysctl").arg("machdep").output().unwrap();
|
||||
assert!(output.status.success());
|
||||
let stdout = String::from_utf8(output.stdout).unwrap();
|
||||
Self(stdout)
|
||||
}
|
||||
fn field(&self, name: &str) -> Option<u64> {
|
||||
Some(
|
||||
self.0
|
||||
.lines()
|
||||
.find_map(|s| s.strip_prefix(&format!("{}=", name)))?
|
||||
.parse()
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
let AA64Reg { aa64isar0, aa64isar1, aa64isar3, aa64mmfr2 } = imp::aa64reg();
|
||||
let sysctl_output = SysctlMachdepOutput::new();
|
||||
assert_eq!(aa64isar0, sysctl_output.field("machdep.id_aa64isar0").unwrap_or(0));
|
||||
assert_eq!(aa64isar1, sysctl_output.field("machdep.id_aa64isar1").unwrap_or(0));
|
||||
assert_eq!(aa64isar3, sysctl_output.field("machdep.id_aa64isar3").unwrap_or(0));
|
||||
assert_eq!(aa64mmfr2, sysctl_output.field("machdep.id_aa64mmfr2").unwrap_or(0));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Run-time CPU feature detection on AArch64 Apple targets by using sysctlbyname.
|
||||
|
||||
On macOS, this module is currently only enabled on tests because there are no
|
||||
instructions that were not available on the M1 but are now available on the
|
||||
latest Apple hardware and this library currently wants to use:
|
||||
|
||||
```console
|
||||
$ LC_ALL=C comm -23 <(rustc --print cfg --target aarch64-apple-darwin -C target-cpu=apple-m5 | grep -F target_feature) <(rustc --print cfg --target aarch64-apple-darwin | grep -F target_feature)
|
||||
target_feature="bf16"
|
||||
target_feature="bti"
|
||||
target_feature="cssc"
|
||||
target_feature="ecv"
|
||||
target_feature="hbc"
|
||||
target_feature="i8mm"
|
||||
target_feature="mte"
|
||||
target_feature="sme"
|
||||
target_feature="sme-b16b16"
|
||||
target_feature="sme-f16f16"
|
||||
target_feature="sme-f64f64"
|
||||
target_feature="sme-i16i64"
|
||||
target_feature="sme2"
|
||||
target_feature="sme2p1"
|
||||
target_feature="sve-b16b16"
|
||||
target_feature="v8.5a"
|
||||
target_feature="v8.6a"
|
||||
target_feature="v8.7a"
|
||||
target_feature="wfxt"
|
||||
```
|
||||
|
||||
Non-macOS targets doesn't always supports FEAT_LSE2, so we use this module on them.
|
||||
As used in the example in https://developer.apple.com/documentation/xcode/writing-arm64-code-for-apple-platforms#Enable-DIT-for-constant-time-cryptographic-operations,
|
||||
sysctlbyname is supported on all Apple platforms.
|
||||
|
||||
Refs: https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics
|
||||
*/
|
||||
|
||||
include!("common.rs");
|
||||
|
||||
use core::{mem, ptr};
|
||||
|
||||
// libc requires Rust 1.63
|
||||
mod ffi {
|
||||
pub(crate) use crate::utils::ffi::{CStr, c_char, c_int, c_size_t, c_void};
|
||||
|
||||
#[cfg(test)] // test-only
|
||||
sys_const!({
|
||||
// https://github.com/apple-oss-distributions/xnu/blob/xnu-12377.121.6/osfmk/arm/cpu_capabilities_public.h
|
||||
pub(crate) const CAP_BIT_FEAT_LSE: usize = 6;
|
||||
pub(crate) const CAP_BIT_FEAT_LRCPC: usize = 15;
|
||||
pub(crate) const CAP_BIT_FEAT_LRCPC2: usize = 16;
|
||||
pub(crate) const CAP_BIT_FEAT_LSE2: usize = 30;
|
||||
pub(crate) const CAP_BIT_NB: usize = 92;
|
||||
});
|
||||
|
||||
sys_fn!({
|
||||
extern "C" {
|
||||
// https://developer.apple.com/documentation/kernel/1387446-sysctlbyname
|
||||
// https://github.com/apple-oss-distributions/xnu/blob/xnu-12377.121.6/bsd/sys/sysctl.h
|
||||
pub(crate) fn sysctlbyname(
|
||||
name: *const c_char,
|
||||
old_p: *mut c_void,
|
||||
old_len_p: *mut c_size_t,
|
||||
new_p: *mut c_void,
|
||||
new_len: c_size_t,
|
||||
) -> c_int;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn sysctlbyname32(name: &ffi::CStr) -> Option<u32> {
|
||||
const OUT_LEN: ffi::c_size_t = mem::size_of::<u32>() as ffi::c_size_t;
|
||||
|
||||
let mut out = 0_u32;
|
||||
let mut out_len = OUT_LEN;
|
||||
// SAFETY:
|
||||
// - `name` a valid C string.
|
||||
// - `out_len` does not exceed the size of `out`.
|
||||
// - `sysctlbyname` is thread-safe.
|
||||
let res = unsafe {
|
||||
ffi::sysctlbyname(
|
||||
name.as_ptr(),
|
||||
(&mut out as *mut u32).cast::<ffi::c_void>(),
|
||||
&mut out_len,
|
||||
ptr::null_mut(),
|
||||
0,
|
||||
)
|
||||
};
|
||||
if res != 0 {
|
||||
return None;
|
||||
}
|
||||
debug_assert_eq!(out_len, OUT_LEN);
|
||||
Some(out)
|
||||
}
|
||||
|
||||
#[cold]
|
||||
#[must_use]
|
||||
fn _detect(mut info: CpuInfo) -> CpuInfo {
|
||||
macro_rules! check {
|
||||
($flag:ident, $($name:tt) ||+) => {
|
||||
if $(sysctlbyname32(c!($name)).unwrap_or(0) != 0) ||+ {
|
||||
info.set(CpuInfoFlag::$flag);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// On macOS, AArch64 support was added in macOS 11,
|
||||
// hw.optional.armv8_1_atomics is available on macOS 11+,
|
||||
// hw.optional.arm.FEAT_* are only available on macOS 12+.
|
||||
// Query both names in case future versions of macOS remove the old name.
|
||||
// https://github.com/golang/go/commit/c15593197453b8bf90fc3a9080ba2afeaf7934ea
|
||||
// https://github.com/google/boringssl/commit/91e0b11eba517d83b910b20fe3740eeb39ecb37e
|
||||
// hw.optional.arm.caps is available on macOS 15+.
|
||||
// https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc#L56
|
||||
check!(lse, "hw.optional.arm.FEAT_LSE" || "hw.optional.armv8_1_atomics");
|
||||
check!(lse2, "hw.optional.arm.FEAT_LSE2");
|
||||
#[cfg(test)]
|
||||
check!(rcpc, "hw.optional.arm.FEAT_LRCPC");
|
||||
#[cfg(test)]
|
||||
check!(rcpc2, "hw.optional.arm.FEAT_LRCPC2");
|
||||
info
|
||||
}
|
||||
|
||||
#[allow(
|
||||
clippy::alloc_instead_of_core,
|
||||
clippy::std_instead_of_alloc,
|
||||
clippy::std_instead_of_core,
|
||||
clippy::undocumented_unsafe_blocks,
|
||||
clippy::wildcard_imports
|
||||
)]
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::{format, process::Command, str, string::String};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_alternative() {
|
||||
use crate::utils::ffi::*;
|
||||
|
||||
const CAPS_LEN: usize = (ffi::CAP_BIT_NB + 63) / 64;
|
||||
fn arm_caps(caps: &mut [u64; CAPS_LEN]) -> bool {
|
||||
let mut caps_len = CAPS_LEN * 8;
|
||||
// SAFETY:
|
||||
// - name a valid C string.
|
||||
// - `caps_len` does not exceed the size of `caps`.
|
||||
// - `sysctlbyname` is thread-safe.
|
||||
let res = unsafe {
|
||||
ffi::sysctlbyname(
|
||||
c!("hw.optional.arm.caps").as_ptr(),
|
||||
caps.as_mut_ptr().cast::<ffi::c_void>(),
|
||||
&mut caps_len,
|
||||
ptr::null_mut(),
|
||||
0,
|
||||
)
|
||||
};
|
||||
res == 0
|
||||
}
|
||||
|
||||
// Call sysctl command instead of libc API.
|
||||
//
|
||||
// This is used only for testing.
|
||||
struct SysctlHwOptionalOutput(String);
|
||||
impl SysctlHwOptionalOutput {
|
||||
fn new() -> Self {
|
||||
let output = Command::new("sysctl").arg("hw.optional").output().unwrap();
|
||||
assert!(output.status.success());
|
||||
let stdout = String::from_utf8(output.stdout).unwrap();
|
||||
test_helper::eprintln_nocapture!("sysctl hw.optional:\n{}", stdout);
|
||||
Self(stdout)
|
||||
}
|
||||
fn field(&self, name: &CStr) -> Option<u32> {
|
||||
let name = name.to_bytes_with_nul();
|
||||
let name = str::from_utf8(&name[..name.len() - 1]).unwrap();
|
||||
let prefix = format!("{}: ", name);
|
||||
Some(self.0.lines().find_map(|s| s.strip_prefix(&prefix))?.parse().unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
let mut caps = [0_u64; CAPS_LEN];
|
||||
let has_caps = arm_caps(&mut caps);
|
||||
let sysctl_output = SysctlHwOptionalOutput::new();
|
||||
for (name, expected_on_macos, cap_bit) in [
|
||||
(c!("hw.optional.arm.FEAT_LSE"), Some(1), ffi::CAP_BIT_FEAT_LSE),
|
||||
(c!("hw.optional.armv8_1_atomics"), Some(1), ffi::CAP_BIT_FEAT_LSE),
|
||||
(c!("hw.optional.arm.FEAT_LSE2"), Some(1), ffi::CAP_BIT_FEAT_LSE2),
|
||||
(c!("hw.optional.arm.FEAT_LSE128"), None, 0),
|
||||
(c!("hw.optional.arm.FEAT_LSFE"), None, 0),
|
||||
(c!("hw.optional.arm.FEAT_LRCPC"), Some(1), ffi::CAP_BIT_FEAT_LRCPC),
|
||||
(c!("hw.optional.arm.FEAT_LRCPC2"), Some(1), ffi::CAP_BIT_FEAT_LRCPC2),
|
||||
(c!("hw.optional.arm.FEAT_LRCPC3"), None, 0),
|
||||
] {
|
||||
let res = sysctlbyname32(name);
|
||||
if res.is_none() {
|
||||
assert_eq!(std::io::Error::last_os_error().kind(), std::io::ErrorKind::NotFound);
|
||||
}
|
||||
if cfg!(any(target_os = "macos", target_abi = "macabi")) {
|
||||
assert_eq!(
|
||||
res,
|
||||
expected_on_macos,
|
||||
"{}",
|
||||
str::from_utf8(name.to_bytes_with_nul()).unwrap()
|
||||
);
|
||||
}
|
||||
if let Some(res) = res {
|
||||
assert_eq!(res, sysctl_output.field(name).unwrap());
|
||||
if has_caps {
|
||||
assert_eq!(caps[cap_bit / 64] & (1 << (cap_bit & 63)) != 0, res != 0);
|
||||
}
|
||||
} else {
|
||||
assert!(sysctl_output.field(name).is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Run-time CPU feature detection on AArch64 Fuchsia by using zx_system_get_features.
|
||||
|
||||
As of Rust 1.94, is_aarch64_feature_detected doesn't support run-time detection on Fuchsia.
|
||||
https://github.com/rust-lang/rust/blob/1.94.0/library/std_detect/src/detect/mod.rs
|
||||
|
||||
Refs:
|
||||
- https://fuchsia.dev/reference/syscalls/system_get_features
|
||||
- https://github.com/llvm/llvm-project/commit/4e731abc55681751b5d736b613f7720e50eb1ad4
|
||||
*/
|
||||
|
||||
include!("common.rs");
|
||||
|
||||
#[allow(non_camel_case_types)]
|
||||
mod ffi {
|
||||
sys_type!({
|
||||
// https://fuchsia.googlesource.com/fuchsia/+/refs/heads/main/zircon/system/public/zircon/types.h
|
||||
pub(crate) type zx_status_t = i32;
|
||||
});
|
||||
|
||||
sys_const!({
|
||||
// https://fuchsia.googlesource.com/fuchsia/+/refs/heads/main/zircon/system/public/zircon/errors.h
|
||||
pub(crate) const ZX_OK: zx_status_t = 0;
|
||||
|
||||
// https://fuchsia.googlesource.com/fuchsia/+/refs/heads/main/zircon/system/public/zircon/features.h
|
||||
pub(crate) const ZX_FEATURE_KIND_CPU: u32 = 0;
|
||||
pub(crate) const ZX_ARM64_FEATURE_ISA_ATOMICS: u32 = 1 << 8;
|
||||
});
|
||||
|
||||
// TODO: use sys_fn!
|
||||
#[link(name = "zircon")]
|
||||
extern "C" {
|
||||
// https://fuchsia.dev/reference/syscalls/system_get_features
|
||||
pub(crate) fn zx_system_get_features(kind: u32, features: *mut u32) -> zx_status_t;
|
||||
}
|
||||
}
|
||||
|
||||
fn zx_system_get_features(kind: u32) -> u32 {
|
||||
let mut out = 0_u32;
|
||||
// SAFETY: the pointer is valid because we got it from a reference.
|
||||
let res = unsafe { ffi::zx_system_get_features(kind, &mut out) };
|
||||
if res != ffi::ZX_OK {
|
||||
return 0;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cold]
|
||||
#[must_use]
|
||||
fn _detect(mut info: CpuInfo) -> CpuInfo {
|
||||
let features = zx_system_get_features(ffi::ZX_FEATURE_KIND_CPU);
|
||||
macro_rules! check {
|
||||
($flag:ident, $bit:ident) => {
|
||||
if features & ffi::$bit != 0 {
|
||||
info.set(CpuInfoFlag::$flag);
|
||||
}
|
||||
};
|
||||
}
|
||||
check!(lse, ZX_ARM64_FEATURE_ISA_ATOMICS);
|
||||
info
|
||||
}
|
||||
|
||||
#[allow(
|
||||
clippy::alloc_instead_of_core,
|
||||
clippy::std_instead_of_alloc,
|
||||
clippy::std_instead_of_core,
|
||||
clippy::undocumented_unsafe_blocks,
|
||||
clippy::wildcard_imports
|
||||
)]
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_fuchsia() {
|
||||
let features = zx_system_get_features(ffi::ZX_FEATURE_KIND_CPU);
|
||||
test_helper::eprintln_nocapture!(
|
||||
"zx_system_get_features(ZX_FEATURE_KIND_CPU): {:b}",
|
||||
features
|
||||
);
|
||||
assert_ne!(features, 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Run-time CPU feature detection on AArch64 illumos by using getisax.
|
||||
|
||||
As of Rust 1.94, is_aarch64_feature_detected doesn't support run-time detection on illumos.
|
||||
https://github.com/rust-lang/rust/blob/1.94.0/library/std_detect/src/detect/mod.rs
|
||||
*/
|
||||
|
||||
include!("common.rs");
|
||||
|
||||
// libc requires Rust 1.63
|
||||
mod ffi {
|
||||
pub(crate) use crate::utils::ffi::c_uint;
|
||||
|
||||
sys_const!({
|
||||
// Defined in sys/auxv_aarch64.h.
|
||||
// https://github.com/richlowe/illumos-gate/blob/arm64-gate/usr/src/uts/common/sys/auxv_aarch64.h
|
||||
pub(crate) const AV_AARCH64_LSE: u32 = 1 << 15;
|
||||
#[cfg(test)]
|
||||
pub(crate) const AV_AARCH64_LRCPC: u32 = 1 << 28;
|
||||
#[cfg(test)]
|
||||
pub(crate) const AV_AARCH64_2_ILRCPC: u32 = 1 << 1;
|
||||
pub(crate) const AV_AARCH64_2_LSE2: u32 = 1 << 2;
|
||||
});
|
||||
|
||||
sys_fn!({
|
||||
extern "C" {
|
||||
// Defined in sys/auxv.h.
|
||||
// https://illumos.org/man/2/getisax
|
||||
// https://github.com/richlowe/illumos-gate/blob/arm64-gate/usr/src/uts/common/sys/auxv.h
|
||||
pub(crate) fn getisax(array: *mut u32, n: c_uint) -> c_uint;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[cold]
|
||||
#[must_use]
|
||||
fn _detect(mut info: CpuInfo) -> CpuInfo {
|
||||
const OUT_LEN: ffi::c_uint = 2;
|
||||
let mut out = [0_u32; OUT_LEN as usize];
|
||||
// SAFETY: the pointer is valid because we got it from a reference.
|
||||
unsafe {
|
||||
ffi::getisax(out.as_mut_ptr(), OUT_LEN);
|
||||
}
|
||||
macro_rules! check {
|
||||
($x:ident, $flag:ident, $bit:ident) => {
|
||||
if $x & ffi::$bit != 0 {
|
||||
info.set(CpuInfoFlag::$flag);
|
||||
}
|
||||
};
|
||||
}
|
||||
let v1 = out[0];
|
||||
check!(v1, lse, AV_AARCH64_LSE);
|
||||
#[cfg(test)]
|
||||
check!(v1, rcpc, AV_AARCH64_LRCPC);
|
||||
let v2 = out[1];
|
||||
#[cfg(test)]
|
||||
check!(v2, rcpc2, AV_AARCH64_2_ILRCPC);
|
||||
check!(v2, lse2, AV_AARCH64_2_LSE2);
|
||||
info
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Run-time CPU feature detection on AArch64 Windows by using IsProcessorFeaturePresent.
|
||||
|
||||
Run-time detection of FEAT_LSE on Windows by is_aarch64_feature_detected is supported on Rust 1.70+.
|
||||
https://github.com/rust-lang/stdarch/pull/1373
|
||||
|
||||
Refs: https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-isprocessorfeaturepresent
|
||||
*/
|
||||
|
||||
include!("common.rs");
|
||||
|
||||
// windows-sys requires Rust 1.71
|
||||
#[allow(non_camel_case_types, clippy::upper_case_acronyms)]
|
||||
mod ffi {
|
||||
sys_type!({
|
||||
pub(crate) type [Win32::System::Threading] PROCESSOR_FEATURE_ID = u32;
|
||||
pub(crate) type [core] BOOL = i32;
|
||||
});
|
||||
|
||||
sys_const!({
|
||||
pub(crate) const [Win32::Foundation] FALSE: BOOL = 0;
|
||||
|
||||
// Defined in winnt.h of Windows SDK.
|
||||
pub(crate) const [Win32::System::Threading]
|
||||
PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE: PROCESSOR_FEATURE_ID = 34;
|
||||
#[cfg(test)]
|
||||
pub(crate) const [Win32::System::Threading]
|
||||
PF_ARM_V83_LRCPC_INSTRUCTIONS_AVAILABLE: PROCESSOR_FEATURE_ID = 45;
|
||||
});
|
||||
// TODO: put in sys_const! once windows-sys updated.
|
||||
pub(crate) const PF_ARM_LSE2_AVAILABLE: PROCESSOR_FEATURE_ID = 62;
|
||||
|
||||
sys_fn!({
|
||||
extern "system" {
|
||||
// https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-isprocessorfeaturepresent
|
||||
pub(crate) fn [Win32::System::Threading] IsProcessorFeaturePresent(
|
||||
ProcessorFeature: PROCESSOR_FEATURE_ID,
|
||||
) -> BOOL;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[cold]
|
||||
#[must_use]
|
||||
fn _detect(mut info: CpuInfo) -> CpuInfo {
|
||||
macro_rules! check {
|
||||
($flag:ident, $bit:ident) => {
|
||||
// SAFETY: calling IsProcessorFeaturePresent is safe, and FALSE is also
|
||||
// returned if the HAL does not support detection of the specified feature.
|
||||
if unsafe { ffi::IsProcessorFeaturePresent(ffi::$bit) != ffi::FALSE } {
|
||||
info.set(CpuInfoFlag::$flag);
|
||||
}
|
||||
};
|
||||
}
|
||||
check!(lse, PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE);
|
||||
check!(lse2, PF_ARM_LSE2_AVAILABLE);
|
||||
#[cfg(test)]
|
||||
check!(rcpc, PF_ARM_V83_LRCPC_INSTRUCTIONS_AVAILABLE);
|
||||
info
|
||||
}
|
||||
+1083
File diff suppressed because it is too large.
Load diff
+303
@@ -0,0 +1,303 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
#[repr(transparent)]
|
||||
pub(crate) struct CpuInfo(u32);
|
||||
|
||||
impl CpuInfo {
|
||||
#[inline]
|
||||
fn set(&mut self, bit: CpuInfoFlag) {
|
||||
self.0 = set(self.0, bit as u32);
|
||||
}
|
||||
#[inline]
|
||||
#[must_use]
|
||||
fn test(self, bit: CpuInfoFlag) -> bool {
|
||||
test(self.0, bit as u32)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[must_use]
|
||||
fn set(x: u32, bit: u32) -> u32 {
|
||||
x | (1 << bit)
|
||||
}
|
||||
#[inline]
|
||||
#[must_use]
|
||||
fn test(x: u32, bit: u32) -> bool {
|
||||
x & (1 << bit) != 0
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn detect() -> CpuInfo {
|
||||
use core::sync::atomic::{AtomicU32, Ordering};
|
||||
|
||||
static CACHE: AtomicU32 = AtomicU32::new(0);
|
||||
let mut info = CpuInfo(CACHE.load(Ordering::Relaxed));
|
||||
if info.0 != 0 {
|
||||
return info;
|
||||
}
|
||||
info.set(CpuInfoFlag::Init);
|
||||
// Note: detect_false cfg is intended to make it easy for developers to test
|
||||
// cases where features usually available is not available, and is not a public API.
|
||||
if !cfg!(portable_atomic_test_detect_false) {
|
||||
info = _detect(info);
|
||||
}
|
||||
if cfg!(any(
|
||||
all(
|
||||
target_arch = "x86_64",
|
||||
not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
|
||||
),
|
||||
all(
|
||||
target_arch = "powerpc64",
|
||||
not(any(
|
||||
target_feature = "quadword-atomics",
|
||||
portable_atomic_target_feature = "quadword-atomics",
|
||||
)),
|
||||
),
|
||||
all(
|
||||
any(target_arch = "riscv32", target_arch = "riscv64"),
|
||||
not(any(target_feature = "zacas", portable_atomic_target_feature = "zacas")),
|
||||
),
|
||||
)) {
|
||||
// Use CAS if the detection result affects lock-freeness.
|
||||
match CACHE.compare_exchange(0, info.0, Ordering::Relaxed, Ordering::Relaxed) {
|
||||
Ok(_) => info,
|
||||
Err(info) => {
|
||||
debug_assert_ne!(info, 0);
|
||||
CpuInfo(info)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
CACHE.store(info.0, Ordering::Relaxed);
|
||||
info
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! flags {
|
||||
($(
|
||||
$(#[$attr:meta])*
|
||||
$func:ident($name:literal, any($($cfg:ident),*)),
|
||||
)*) => {
|
||||
#[allow(dead_code, non_camel_case_types)]
|
||||
#[derive(Clone, Copy)]
|
||||
#[cfg_attr(test, derive(PartialEq, Eq, PartialOrd, Ord))]
|
||||
#[repr(u32)]
|
||||
enum CpuInfoFlag {
|
||||
Init = 0,
|
||||
$($func,)*
|
||||
}
|
||||
impl CpuInfo {
|
||||
$(
|
||||
$(#[$attr])*
|
||||
#[cfg(any(test, not(any($($cfg = $name),*))))]
|
||||
#[inline]
|
||||
#[must_use]
|
||||
pub(crate) fn $func(self) -> bool {
|
||||
self.test(CpuInfoFlag::$func)
|
||||
}
|
||||
)*
|
||||
#[cfg(test)] // for test
|
||||
const ALL_FLAGS: &'static [(&'static str, CpuInfoFlag, bool)] = &[$(
|
||||
($name, CpuInfoFlag::$func, cfg!(any($($cfg = $name),*))),
|
||||
)*];
|
||||
}
|
||||
#[test]
|
||||
#[cfg_attr(portable_atomic_test_detect_false, ignore = "detection disabled")]
|
||||
fn test_detect() {$(
|
||||
$(#[$attr])*
|
||||
{
|
||||
const _: u32 = 1_u32 << CpuInfoFlag::$func as u32;
|
||||
assert_eq!($name.replace(|c: char| c == '-' || c == '.', "_"), stringify!($func));
|
||||
if detect().$func() {
|
||||
assert!(detect().test(CpuInfoFlag::$func));
|
||||
} else {
|
||||
assert!(!detect().test(CpuInfoFlag::$func));
|
||||
}
|
||||
}
|
||||
)*}
|
||||
};
|
||||
}
|
||||
|
||||
// rustc definitions: https://github.com/rust-lang/rust/blob/ddaf12390d3ffb7d5ba74491a48f3cd528e5d777/compiler/rustc_target/src/target_features.rs
|
||||
|
||||
// LLVM definitions: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/AArch64/AArch64Features.td
|
||||
#[cfg(any(target_arch = "aarch64", target_arch = "arm64ec"))]
|
||||
flags! {
|
||||
// The Armv8.1 architecture extension
|
||||
// https://developer.arm.com/documentation/109697/2025_06/Feature-descriptions/The-Armv8-1-architecture-extension
|
||||
// FEAT_LSE, Large System Extensions
|
||||
// > This feature is supported in AArch64 state only.
|
||||
// > FEAT_LSE is OPTIONAL from Armv8.0.
|
||||
// > FEAT_LSE is mandatory from Armv8.1.
|
||||
lse("lse", any(target_feature /* 1.61+ */, portable_atomic_target_feature)),
|
||||
|
||||
// The Armv8.3 architecture extension
|
||||
// https://developer.arm.com/documentation/109697/2025_06/Feature-descriptions/The-Armv8-3-architecture-extension
|
||||
// FEAT_LRCPC, Load-Acquire RCpc instructions
|
||||
// > These instructions are added to the A64 instruction set only.
|
||||
// > FEAT_LRCPC is OPTIONAL from Armv8.2.
|
||||
// > FEAT_LRCPC is mandatory from Armv8.3.
|
||||
#[cfg(test)] // test-only
|
||||
rcpc("rcpc", any(target_feature /* 1.61+ */)),
|
||||
|
||||
// The Armv8.4 architecture extension
|
||||
// https://developer.arm.com/documentation/109697/2025_06/Feature-descriptions/The-Armv8-4-architecture-extension
|
||||
// FEAT_LSE2, Large System Extensions version 2
|
||||
// > This feature is supported in AArch64 state only.
|
||||
// > FEAT_LSE2 is OPTIONAL from Armv8.2.
|
||||
// > FEAT_LSE2 is mandatory from Armv8.4.
|
||||
#[cfg_attr(not(test), allow(dead_code))]
|
||||
lse2("lse2", any(target_feature /* nightly */, portable_atomic_target_feature)),
|
||||
// FEAT_LRCPC2, Load-Acquire RCpc instructions version 2
|
||||
// > These instructions are added to the A64 instruction set only.
|
||||
// > FEAT_LRCPC2 is OPTIONAL from Armv8.2.
|
||||
// > FEAT_LRCPC2 is mandatory from Armv8.4.
|
||||
// > If FEAT_LRCPC2 is implemented, then FEAT_LRCPC is implemented.
|
||||
#[cfg(test)] // test-only
|
||||
rcpc2("rcpc2", any(target_feature /* 1.61+ */)),
|
||||
|
||||
// The Armv8.9 architecture extension
|
||||
// https://developer.arm.com/documentation/109697/2025_06/Feature-descriptions/The-Armv8-9-architecture-extension
|
||||
// FEAT_LRCPC3, Load-Acquire RCpc instructions version 3
|
||||
// > This feature is supported in AArch64 state only.
|
||||
// > FEAT_LRCPC3 is OPTIONAL from Armv8.2.
|
||||
// > If FEAT_LRCPC3 is implemented, then FEAT_LRCPC2 is implemented.
|
||||
#[cfg_attr(not(test), allow(dead_code))]
|
||||
rcpc3("rcpc3", any(target_feature /* nightly */, portable_atomic_target_feature)),
|
||||
|
||||
// The Armv9.4 architecture extension
|
||||
// https://developer.arm.com/documentation/109697/2025_06/Feature-descriptions/The-Armv9-4-architecture-extension
|
||||
// FEAT_LSE128, 128-bit Atomics
|
||||
// > This feature is supported in AArch64 state only.
|
||||
// > FEAT_LSE128 is OPTIONAL from Armv9.3.
|
||||
// > If FEAT_LSE128 is implemented, then FEAT_LSE is implemented.
|
||||
#[cfg_attr(not(test), allow(dead_code))]
|
||||
lse128("lse128", any(target_feature /* nightly */, portable_atomic_target_feature)),
|
||||
|
||||
// The Armv9.6 architecture extension
|
||||
// https://developer.arm.com/documentation/109697/2025_06/Feature-descriptions/The-Armv9-6-architecture-extension
|
||||
// FEAT_LS64WB, LS64 for Write-back cacheable memory
|
||||
// > This feature is supported in AArch64 state only.
|
||||
// > FEAT_LS64WB is OPTIONAL from Armv9.2.
|
||||
// > If FEAT_LS64WB is implemented, then FEAT_LS64_ACCDATA is implemented.
|
||||
#[cfg(test)] // test-only
|
||||
ls64wb("ls64wb", any(/* no corresponding target feature */)),
|
||||
// FEAT_LSFE, Large System Float Extension
|
||||
// > This feature is supported in AArch64 state only.
|
||||
// > FEAT_LSFE is OPTIONAL from Armv9.3.
|
||||
// > If FEAT_LSFE is implemented, then FEAT_FP is implemented.
|
||||
#[cfg(test)] // test-only
|
||||
lsfe("lsfe", any(target_feature /* N/A */, portable_atomic_target_feature)),
|
||||
|
||||
#[cfg(test)] // test-only
|
||||
cpuid("cpuid", any(/* no corresponding target feature */)),
|
||||
}
|
||||
|
||||
// LLVM definitions: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/ARM/ARMFeatures.td
|
||||
#[cfg(target_arch = "arm")]
|
||||
flags! {
|
||||
#[cfg(test)] // test-only
|
||||
lpae("lpae", any(/* no corresponding target feature */)),
|
||||
}
|
||||
|
||||
// LLVM definitions: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/PowerPC/PPC.td
|
||||
#[cfg(target_arch = "powerpc64")]
|
||||
flags! {
|
||||
// lqarx and stqcx.
|
||||
quadword_atomics("quadword-atomics", any(target_feature /* nightly */, portable_atomic_target_feature)),
|
||||
}
|
||||
|
||||
// LLVM definitions: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/RISCV/RISCVFeatures.td
|
||||
#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
|
||||
flags! {
|
||||
// amocas.{w,d,q}
|
||||
zacas("zacas", any(target_feature /* 1.94+ */, portable_atomic_target_feature)),
|
||||
#[cfg(test)] // test-only
|
||||
zabha("zabha", any(target_feature /* 1.94+ */, portable_atomic_target_feature)),
|
||||
#[cfg(test)] // test-only
|
||||
zalasr("zalasr", any(/* no corresponding target feature */)),
|
||||
}
|
||||
|
||||
// LLVM definitions: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/X86/X86.td
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
flags! {
|
||||
// avx
|
||||
#[cfg(target_feature = "sse")]
|
||||
avx("avx", any(target_feature)),
|
||||
// cmpxchg16b
|
||||
cmpxchg16b("cmpxchg16b", any(target_feature /* 1.69+ */, portable_atomic_target_feature)),
|
||||
}
|
||||
|
||||
#[allow(
|
||||
clippy::alloc_instead_of_core,
|
||||
clippy::std_instead_of_alloc,
|
||||
clippy::std_instead_of_core,
|
||||
clippy::undocumented_unsafe_blocks,
|
||||
clippy::wildcard_imports
|
||||
)]
|
||||
#[cfg(test)]
|
||||
mod tests_common {
|
||||
use std::{collections::BTreeSet, vec};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_bit_flags() {
|
||||
let mut flags = vec![("init", CpuInfoFlag::Init)];
|
||||
flags.extend(CpuInfo::ALL_FLAGS.iter().map(|&(name, flag, _)| (name, flag)));
|
||||
let flag_set = flags.iter().map(|(_, flag)| flag).collect::<BTreeSet<_>>();
|
||||
let name_set = flags.iter().map(|(name, _)| name).collect::<BTreeSet<_>>();
|
||||
if flag_set.len() != flags.len() {
|
||||
panic!("CpuInfo flag values must be unique")
|
||||
}
|
||||
if name_set.len() != flags.len() {
|
||||
panic!("CpuInfo flag names must be unique")
|
||||
}
|
||||
|
||||
let mut x = CpuInfo(0);
|
||||
for &(_, f) in &flags {
|
||||
assert!(!x.test(f));
|
||||
}
|
||||
for i in 0..flags.len() {
|
||||
x.set(flags[i].1);
|
||||
for &(_, f) in &flags[..i + 1] {
|
||||
assert!(x.test(f));
|
||||
}
|
||||
for &(_, f) in &flags[i + 1..] {
|
||||
assert!(!x.test(f));
|
||||
}
|
||||
}
|
||||
for &(_, f) in &flags {
|
||||
assert!(x.test(f));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn print_features() {
|
||||
use std::{fmt::Write as _, string::String};
|
||||
|
||||
let mut features = String::new();
|
||||
features.push_str("\nfeatures:\n");
|
||||
for &(name, flag, compile_time) in CpuInfo::ALL_FLAGS {
|
||||
let run_time = detect().test(flag);
|
||||
if run_time == compile_time {
|
||||
let _ = writeln!(features, " {}: {}", name, run_time);
|
||||
} else {
|
||||
let _ = writeln!(
|
||||
features,
|
||||
" {}: {} (compile-time), {} (run-time)",
|
||||
name, compile_time, run_time
|
||||
);
|
||||
}
|
||||
}
|
||||
test_helper::eprintln_nocapture!("{}", features);
|
||||
}
|
||||
|
||||
// Static assertions for C type definitions.
|
||||
// Assertions with core::ffi types are in crate::utils::ffi module.
|
||||
#[cfg(not(any(windows, target_arch = "x86", target_arch = "x86_64")))]
|
||||
const _: fn() = || {
|
||||
use test_helper::sys;
|
||||
let _: crate::utils::ffi::c_char = 0 as sys::c_char;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Run-time CPU feature detection on PowerPC64 AIX by using getsystemcfg.
|
||||
|
||||
Refs:
|
||||
- https://github.com/golang/go/blob/go1.26.0/src/internal/cpu/cpu_ppc64x_aix.go
|
||||
|
||||
As of Rust 1.94, is_powerpc_feature_detected doesn't support run-time detection on AIX.
|
||||
https://github.com/rust-lang/rust/blob/1.94.0/library/std_detect/src/detect/mod.rs
|
||||
|
||||
Run-time detection on PowerPC64 AIX is currently disabled by default as experimental
|
||||
because we cannot run tests on the VM or real machine.
|
||||
*/
|
||||
|
||||
include!("common.rs");
|
||||
|
||||
// libc requires Rust 1.63
|
||||
mod ffi {
|
||||
pub(crate) use crate::utils::ffi::{c_int, c_ulong};
|
||||
|
||||
sys_const!({
|
||||
// https://github.com/rust-lang/libc/blob/0.2.158/src/unix/aix/mod.rs#L2058
|
||||
// https://github.com/golang/go/blob/go1.26.0/src/internal/cpu/cpu_ppc64x_aix.go
|
||||
pub(crate) const SC_IMPL: c_int = 2;
|
||||
pub(crate) const POWER_8: c_ulong = 0x10000;
|
||||
pub(crate) const POWER_9: c_ulong = 0x20000;
|
||||
});
|
||||
// TODO: use sys_const! once libc crate defined it.
|
||||
pub(crate) const POWER_10: c_ulong = 0x40000;
|
||||
|
||||
sys_fn!({
|
||||
extern "C" {
|
||||
// https://www.ibm.com/docs/en/aix/7.3?topic=g-getsystemcfg-subroutine
|
||||
// https://github.com/rust-lang/libc/blob/0.2.158/src/unix/aix/powerpc64.rs#L643
|
||||
pub(crate) fn getsystemcfg(name: c_int) -> c_ulong;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[cold]
|
||||
#[must_use]
|
||||
fn _detect(mut info: CpuInfo) -> CpuInfo {
|
||||
// SAFETY: calling getsystemcfg is safe.
|
||||
let impl_ = unsafe { ffi::getsystemcfg(ffi::SC_IMPL) };
|
||||
if impl_ == ffi::c_ulong::MAX {
|
||||
return info;
|
||||
}
|
||||
// Check both POWER_8 and later ISAs (which are superset of POWER_8) because
|
||||
// AIX currently doesn't set POWER_8 when POWER_9 is set.
|
||||
// https://github.com/golang/go/commit/51859ec2292d9c1d82a7054ec672ff551a0d7497
|
||||
if impl_ & (ffi::POWER_8 | ffi::POWER_9 | ffi::POWER_10) != 0 {
|
||||
info.set(CpuInfoFlag::quadword_atomics);
|
||||
}
|
||||
info
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Run-time CPU feature detection on RISC-V Linux/Android by using riscv_hwprobe.
|
||||
|
||||
On RISC-V, detection using auxv only supports single-letter extensions.
|
||||
So, we use riscv_hwprobe that supports multi-letter extensions.
|
||||
|
||||
Refs: https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/riscv/hwprobe.rst
|
||||
*/
|
||||
|
||||
include!("common.rs");
|
||||
|
||||
use core::ptr;
|
||||
|
||||
// libc requires Rust 1.63
|
||||
#[allow(non_camel_case_types, non_upper_case_globals)]
|
||||
mod ffi {
|
||||
pub(crate) use crate::utils::ffi::{c_long, c_size_t, c_uint, c_ulong};
|
||||
|
||||
sys_struct!({
|
||||
// https://github.com/torvalds/linux/blob/v6.19/arch/riscv/include/uapi/asm/hwprobe.h
|
||||
pub(crate) struct riscv_hwprobe {
|
||||
pub(crate) key: i64,
|
||||
pub(crate) value: u64,
|
||||
}
|
||||
});
|
||||
|
||||
sys_const!({
|
||||
pub(crate) const __NR_riscv_hwprobe: c_long = 258;
|
||||
|
||||
// https://github.com/torvalds/linux/blob/v6.19/arch/riscv/include/uapi/asm/hwprobe.h
|
||||
// Linux 6.4+
|
||||
// https://github.com/torvalds/linux/commit/00e76e2c6a2bd3976d44d4a1fdd0b7a3c2566607
|
||||
pub(crate) const RISCV_HWPROBE_KEY_BASE_BEHAVIOR: i64 = 3;
|
||||
pub(crate) const RISCV_HWPROBE_BASE_BEHAVIOR_IMA: u64 = 1 << 0;
|
||||
pub(crate) const RISCV_HWPROBE_KEY_IMA_EXT_0: i64 = 4;
|
||||
// Linux 6.8+
|
||||
// https://github.com/torvalds/linux/commit/154a3706122978eeb34d8223d49285ed4f3c61fa
|
||||
pub(crate) const RISCV_HWPROBE_EXT_ZACAS: u64 = 1 << 34;
|
||||
// Linux 6.16+
|
||||
// https://github.com/torvalds/linux/commit/415a8c81da3dab0a585bd4f8d505a11ad5a171a7
|
||||
#[cfg(test)]
|
||||
pub(crate) const RISCV_HWPROBE_EXT_ZABHA: u64 = 1 << 58;
|
||||
// Linux 6.19+
|
||||
// https://github.com/torvalds/linux/commit/f4922b69165735e81752ee47d174f873e989a449
|
||||
#[cfg(test)]
|
||||
pub(crate) const RISCV_HWPROBE_EXT_ZALASR: u64 = 1 << 59;
|
||||
});
|
||||
|
||||
cfg_sel!({
|
||||
// Use asm-based syscall on Linux for compatibility with non-libc targets if possible.
|
||||
// Do not use this on Android. See comments on syscall_helper module in src/utils.rs for details.
|
||||
#[cfg(all(
|
||||
target_os = "linux",
|
||||
any(
|
||||
target_arch = "riscv32",
|
||||
all(target_arch = "riscv64", target_pointer_width = "64"),
|
||||
),
|
||||
not(portable_atomic_no_asm_syscall),
|
||||
))]
|
||||
{
|
||||
use crate::utils::{RegISize, RegSize};
|
||||
|
||||
#[inline]
|
||||
pub(crate) unsafe fn syscall5(
|
||||
number: c_long,
|
||||
arg1: *mut riscv_hwprobe,
|
||||
arg2: c_size_t,
|
||||
arg3: c_size_t,
|
||||
arg4: *mut c_ulong,
|
||||
arg5: c_uint,
|
||||
) -> c_long {
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let number = number as RegISize;
|
||||
let arg1 = ptr_reg!(arg1);
|
||||
let arg2 = arg2 as RegSize;
|
||||
let arg3 = arg3 as RegSize;
|
||||
let arg4 = ptr_reg!(arg4);
|
||||
let arg5 = arg5 as RegSize;
|
||||
let r: RegISize;
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
// we've extended the syscall number and arguments to the register size.
|
||||
unsafe {
|
||||
asm_syscall!(number, r, arg1, arg2, arg3, arg4, arg5);
|
||||
}
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
{
|
||||
r as c_long
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(else)]
|
||||
{
|
||||
sys_fn!({
|
||||
extern "C" {
|
||||
// https://man7.org/linux/man-pages/man2/syscall.2.html
|
||||
pub(crate) fn syscall(number: c_long, ...) -> c_long;
|
||||
}
|
||||
});
|
||||
pub(crate) use self::syscall as syscall5;
|
||||
}
|
||||
});
|
||||
|
||||
// https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/riscv/hwprobe.rst
|
||||
pub(crate) unsafe fn __riscv_hwprobe(
|
||||
pairs: *mut riscv_hwprobe,
|
||||
pair_count: c_size_t,
|
||||
cpu_set_size: c_size_t,
|
||||
cpus: *mut c_ulong,
|
||||
flags: c_uint,
|
||||
) -> c_long {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe { syscall5(__NR_riscv_hwprobe, pairs, pair_count, cpu_set_size, cpus, flags) }
|
||||
}
|
||||
}
|
||||
|
||||
// syscall returns an unsupported error if riscv_hwprobe is not supported,
|
||||
// so we can safely use this function on older versions of Linux.
|
||||
fn riscv_hwprobe(out: &mut [ffi::riscv_hwprobe]) -> bool {
|
||||
let len = out.len();
|
||||
// SAFETY: We've passed the valid pointer and length,
|
||||
// passing null ptr for cpus is safe because cpu_set_size is zero.
|
||||
unsafe { ffi::__riscv_hwprobe(out.as_mut_ptr(), len, 0, ptr::null_mut(), 0) == 0 }
|
||||
}
|
||||
|
||||
#[cold]
|
||||
#[must_use]
|
||||
fn _detect(mut info: CpuInfo) -> CpuInfo {
|
||||
let mut out = [
|
||||
ffi::riscv_hwprobe { key: ffi::RISCV_HWPROBE_KEY_BASE_BEHAVIOR, value: 0 },
|
||||
ffi::riscv_hwprobe { key: ffi::RISCV_HWPROBE_KEY_IMA_EXT_0, value: 0 },
|
||||
];
|
||||
if riscv_hwprobe(&mut out)
|
||||
&& out[0].key != -1
|
||||
&& out[0].value & ffi::RISCV_HWPROBE_BASE_BEHAVIOR_IMA != 0
|
||||
&& out[1].key != -1
|
||||
{
|
||||
let value = out[1].value;
|
||||
macro_rules! check {
|
||||
($flag:ident, $bit:ident) => {
|
||||
if value & ffi::$bit != 0 {
|
||||
info.set(CpuInfoFlag::$flag);
|
||||
}
|
||||
};
|
||||
}
|
||||
check!(zacas, RISCV_HWPROBE_EXT_ZACAS);
|
||||
#[cfg(test)]
|
||||
check!(zabha, RISCV_HWPROBE_EXT_ZABHA);
|
||||
#[cfg(test)]
|
||||
check!(zalasr, RISCV_HWPROBE_EXT_ZALASR);
|
||||
}
|
||||
info
|
||||
}
|
||||
|
||||
#[allow(
|
||||
clippy::alloc_instead_of_core,
|
||||
clippy::std_instead_of_alloc,
|
||||
clippy::std_instead_of_core,
|
||||
clippy::undocumented_unsafe_blocks,
|
||||
clippy::wildcard_imports
|
||||
)]
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// We use asm-based syscall for compatibility with non-libc targets.
|
||||
// This test tests that our ones and libc::syscall returns the same result.
|
||||
#[test]
|
||||
fn test_alternative() {
|
||||
unsafe fn __riscv_hwprobe_libc(
|
||||
pairs: *mut ffi::riscv_hwprobe,
|
||||
pair_count: ffi::c_size_t,
|
||||
cpu_set_size: ffi::c_size_t,
|
||||
cpus: *mut ffi::c_ulong,
|
||||
flags: ffi::c_uint,
|
||||
) -> ffi::c_long {
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
libc::syscall(ffi::__NR_riscv_hwprobe, pairs, pair_count, cpu_set_size, cpus, flags)
|
||||
}
|
||||
}
|
||||
fn riscv_hwprobe_libc(out: &mut [ffi::riscv_hwprobe]) -> bool {
|
||||
let len = out.len();
|
||||
unsafe { __riscv_hwprobe_libc(out.as_mut_ptr(), len, 0, ptr::null_mut(), 0) == 0 }
|
||||
}
|
||||
let mut out = [
|
||||
ffi::riscv_hwprobe { key: ffi::RISCV_HWPROBE_KEY_BASE_BEHAVIOR, value: 0 },
|
||||
ffi::riscv_hwprobe { key: ffi::RISCV_HWPROBE_KEY_IMA_EXT_0, value: 0 },
|
||||
];
|
||||
let mut libc_out = out;
|
||||
assert_eq!(riscv_hwprobe(&mut out), riscv_hwprobe_libc(&mut libc_out));
|
||||
assert_eq!(out, libc_out);
|
||||
}
|
||||
}
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Run-time CPU feature detection on x86_64 by using CPUID.
|
||||
|
||||
Adapted from https://github.com/rust-lang/rust/blob/1.92.0/library/std_detect/src/detect/os/x86.rs.
|
||||
*/
|
||||
|
||||
#![cfg_attr(portable_atomic_sanitize_thread, allow(dead_code))]
|
||||
|
||||
// Miri doesn't support inline assembly used in __cpuid: https://github.com/rust-lang/miri/issues/932
|
||||
// SGX doesn't support CPUID: https://github.com/rust-lang/rust/blob/1.92.0/library/std_detect/src/detect/os/x86.rs#L30-L33
|
||||
#[cfg(any(target_env = "sgx", miri))]
|
||||
compile_error!("internal error: this module is not supported on this environment");
|
||||
|
||||
include!("common.rs");
|
||||
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
use core::arch::asm;
|
||||
use core::arch::x86_64::CpuidResult;
|
||||
|
||||
// Workaround for https://github.com/rust-lang/rust/issues/101346
|
||||
// It is not clear if our use cases are affected, but we implement this just in case.
|
||||
//
|
||||
// Refs:
|
||||
// - https://www.felixcloutier.com/x86/cpuid
|
||||
// - https://en.wikipedia.org/wiki/CPUID
|
||||
// - https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs
|
||||
#[cfg(not(target_env = "sgx"))]
|
||||
fn __cpuid(leaf: u32) -> CpuidResult {
|
||||
let eax;
|
||||
let mut ebx;
|
||||
let ecx;
|
||||
let edx;
|
||||
// SAFETY: Calling `__cpuid` is safe on all x86_64 CPUs except for SGX,
|
||||
// which doesn't support `cpuid`.
|
||||
// https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs#L102-L109
|
||||
unsafe {
|
||||
asm!(
|
||||
"mov r8, rbx", // save rbx which is reserved by LLVM
|
||||
"cpuid",
|
||||
"xchg r8, rbx", // restore rbx
|
||||
out("r8") ebx,
|
||||
inout("eax") leaf => eax,
|
||||
inout("ecx") 0 => ecx,
|
||||
out("edx") edx,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
}
|
||||
CpuidResult { eax, ebx, ecx, edx }
|
||||
}
|
||||
|
||||
#[cold]
|
||||
#[must_use]
|
||||
fn _detect(mut info: CpuInfo) -> CpuInfo {
|
||||
let CpuidResult { ecx: proc_info_ecx, .. } = __cpuid(1);
|
||||
|
||||
// https://github.com/rust-lang/rust/blob/1.92.0/library/std_detect/src/detect/os/x86.rs#L104
|
||||
if test(proc_info_ecx, 13) {
|
||||
info.set(CpuInfoFlag::cmpxchg16b);
|
||||
}
|
||||
|
||||
// We only use VMOVDQA when SSE is enabled. See _atomic_load_vmovdqa() in atomic128/x86_64.rs for more.
|
||||
#[cfg(target_feature = "sse")]
|
||||
{
|
||||
use core::arch::x86_64::_xgetbv;
|
||||
|
||||
// https://github.com/rust-lang/rust/blob/1.92.0/library/std_detect/src/detect/os/x86.rs#L166-L236
|
||||
let cpu_xsave = test(proc_info_ecx, 26);
|
||||
if cpu_xsave {
|
||||
let cpu_osxsave = test(proc_info_ecx, 27);
|
||||
if cpu_osxsave {
|
||||
// SAFETY: Calling `_xgetbv` is safe because the CPU has `xsave` support
|
||||
// and OS has set `osxsave`.
|
||||
let xcr0 = unsafe { _xgetbv(0) };
|
||||
let os_avx_support = xcr0 & 6 == 6;
|
||||
if os_avx_support && test(proc_info_ecx, 28) {
|
||||
info.set(CpuInfoFlag::avx);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
info
|
||||
}
|
||||
|
||||
#[allow(
|
||||
clippy::alloc_instead_of_core,
|
||||
clippy::std_instead_of_alloc,
|
||||
clippy::std_instead_of_core,
|
||||
clippy::undocumented_unsafe_blocks,
|
||||
clippy::wildcard_imports
|
||||
)]
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
#[cfg_attr(portable_atomic_test_detect_false, ignore = "detection disabled")]
|
||||
fn test_cpuid() {
|
||||
// The recent Rosetta 2 unofficially implements AVX support.
|
||||
// (The OS reports it as unsupported, likely due to poor performance.)
|
||||
#[cfg(target_vendor = "apple")]
|
||||
assert_eq!(
|
||||
std::is_x86_feature_detected!("avx"),
|
||||
detect().avx() || cfg!(target_feature = "avx")
|
||||
);
|
||||
#[cfg(not(target_vendor = "apple"))]
|
||||
assert_eq!(std::is_x86_feature_detected!("avx"), detect().avx());
|
||||
assert_eq!(std::is_x86_feature_detected!("cmpxchg16b"), detect().cmpxchg16b());
|
||||
}
|
||||
}
|
||||
+580
@@ -0,0 +1,580 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Fallback implementation using global locks.
|
||||
|
||||
This implementation uses seqlock for global locks.
|
||||
|
||||
This is basically based on global locks in crossbeam-utils's `AtomicCell`,
|
||||
but seqlock is implemented in a way that does not depend on UB
|
||||
(see comments in optimistic_read method in atomic! macro for details).
|
||||
|
||||
Note that we cannot use a lock per atomic type, since the in-memory representation of the atomic
|
||||
type and the value type must be the same.
|
||||
*/
|
||||
|
||||
#![cfg_attr(
|
||||
any(
|
||||
all(
|
||||
target_arch = "riscv32",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
any(target_os = "linux", target_os = "android"),
|
||||
),
|
||||
all(
|
||||
target_arch = "arm",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(target_os = "linux", target_os = "android"),
|
||||
any(test, not(any(target_feature = "v6", portable_atomic_target_feature = "v6"))),
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
),
|
||||
),
|
||||
allow(dead_code)
|
||||
)]
|
||||
|
||||
// This module requires CAS and this crate only provides atomics up to 128-bit.
|
||||
// I don't believe there are any 16-bit multi-core systems with CAS, and
|
||||
// at least no such architecture is currently supported in Rust.
|
||||
// 128-bit targets that lack atomic usize CAS also do not reach this module.
|
||||
#[cfg(target_pointer_width = "16")]
|
||||
compile_error!(
|
||||
"internal error: unreachable since atomics for 16-bit targets can always be provided by disable interrupts"
|
||||
);
|
||||
#[cfg(target_pointer_width = "128")]
|
||||
compile_error!(
|
||||
"internal error: unreachable since 128-bit target either has atomic CAS for the pointer width or does not have CAS"
|
||||
);
|
||||
|
||||
mod utils;
|
||||
|
||||
// Use "wide" sequence lock if the pointer width <= 32 for preventing its counter against wrap
|
||||
// around.
|
||||
//
|
||||
// Some 64-bit architectures have ABI with 32-bit pointer width (e.g., x86_64 X32 ABI,
|
||||
// AArch64 ILP32 ABI, mips64 N32 ABI). On those targets, AtomicU64 is available and fast,
|
||||
// so use it to implement normal sequence lock and reduce chunks of byte-wise atomic memcpy.
|
||||
cfg_has_fast_atomic_64!({
|
||||
mod seq_lock;
|
||||
type AtomicChunk = core::sync::atomic::AtomicU64;
|
||||
type Chunk = u64;
|
||||
});
|
||||
cfg_no_fast_atomic_64!({
|
||||
#[path = "seq_lock_wide.rs"]
|
||||
mod seq_lock;
|
||||
type AtomicChunk = core::sync::atomic::AtomicU32;
|
||||
type Chunk = u32;
|
||||
});
|
||||
|
||||
use core::{cell::UnsafeCell, mem, sync::atomic::Ordering};
|
||||
|
||||
use self::{
|
||||
seq_lock::{SeqLock, SeqLockWriteGuard},
|
||||
utils::CachePadded,
|
||||
};
|
||||
#[cfg(portable_atomic_no_strict_provenance)]
|
||||
use crate::utils::ptr::PtrExt as _;
|
||||
use crate::utils::unlikely;
|
||||
|
||||
// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/atomic/atomic_cell.rs#L970-L1010.
|
||||
#[inline]
|
||||
#[must_use]
|
||||
fn lock(addr: usize) -> &'static SeqLock {
|
||||
// The number of locks is a prime number because we want to make sure `addr % LEN` gets
|
||||
// dispersed across all locks.
|
||||
const LEN: usize = 67;
|
||||
const L: CachePadded<SeqLock> = CachePadded::new(SeqLock::new());
|
||||
static LOCKS: [CachePadded<SeqLock>; LEN] = [
|
||||
L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L,
|
||||
L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L,
|
||||
L, L, L, L, L, L, L,
|
||||
];
|
||||
|
||||
// If the modulus is a constant number, the compiler will use crazy math to transform this into
|
||||
// a sequence of cheap arithmetic operations rather than using the slow modulo instruction.
|
||||
&LOCKS[addr % LEN]
|
||||
}
|
||||
|
||||
// Emit SeqCst fence to ensure SeqCst semantics when ordering is SeqCst.
|
||||
#[must_use]
|
||||
struct ScFenceGuard;
|
||||
impl ScFenceGuard {
|
||||
#[inline]
|
||||
fn new(emit_sc_fence: bool) -> Option<Self> {
|
||||
if unlikely(emit_sc_fence) {
|
||||
crate::fence(Ordering::SeqCst);
|
||||
Some(ScFenceGuard)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Drop for ScFenceGuard {
|
||||
#[inline]
|
||||
fn drop(&mut self) {
|
||||
crate::fence(Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! atomic {
|
||||
($atomic_type:ident, $int_type:ident, $align:literal) => {
|
||||
#[repr(C, align($align))]
|
||||
pub(crate) struct $atomic_type {
|
||||
v: UnsafeCell<$int_type>,
|
||||
}
|
||||
|
||||
impl $atomic_type {
|
||||
const LEN: usize = mem::size_of::<$int_type>() / mem::size_of::<Chunk>();
|
||||
|
||||
#[inline]
|
||||
unsafe fn chunks(&self) -> &[AtomicChunk; Self::LEN] {
|
||||
static_assert!($atomic_type::LEN > 1);
|
||||
static_assert!(mem::size_of::<$int_type>() % mem::size_of::<Chunk>() == 0);
|
||||
|
||||
// SAFETY: the caller must uphold the safety contract for `chunks`.
|
||||
unsafe { &*(self.v.get() as *const $int_type as *const [AtomicChunk; Self::LEN]) }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn optimistic_read(&self) -> $int_type {
|
||||
// Using `MaybeUninit<[usize; Self::LEN]>` here doesn't change codegen: https://godbolt.org/z/86f8s733M
|
||||
let mut dst: [Chunk; Self::LEN] = [0; Self::LEN];
|
||||
// SAFETY:
|
||||
// - There are no threads that perform non-atomic concurrent write operations.
|
||||
// - There is no writer that updates the value using atomic operations of different granularity.
|
||||
//
|
||||
// If the atomic operation is not used here, it will cause a data race
|
||||
// when `write` performs concurrent write operation.
|
||||
// Such a data race is sometimes considered virtually unproblematic
|
||||
// in SeqLock implementations:
|
||||
//
|
||||
// - https://github.com/Amanieu/seqlock/issues/2
|
||||
// - https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/atomic/atomic_cell.rs#L1063-L1069
|
||||
// - https://rust-lang.zulipchat.com/#narrow/stream/136281-t-lang.2Fwg-unsafe-code-guidelines/topic/avoiding.20UB.20due.20to.20races.20by.20discarding.20result.3F
|
||||
//
|
||||
// However, in our use case, the implementation that loads/stores value as
|
||||
// chunks of usize is enough fast and sound, so we use that implementation.
|
||||
//
|
||||
// See also atomic-memcpy crate, a generic implementation of this pattern:
|
||||
// https://github.com/taiki-e/atomic-memcpy
|
||||
let chunks = unsafe { self.chunks() };
|
||||
for i in 0..Self::LEN {
|
||||
dst[i] = chunks[i].load(Ordering::Relaxed);
|
||||
}
|
||||
// SAFETY: integers are plain old data types so we can always transmute to them.
|
||||
unsafe { mem::transmute::<[Chunk; Self::LEN], $int_type>(dst) }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn read(&self, _guard: &SeqLockWriteGuard<'static>) -> $int_type {
|
||||
// This calls optimistic_read that can return teared value, but the resulting value
|
||||
// is guaranteed not to be teared because we hold the lock to write.
|
||||
self.optimistic_read()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn write(&self, val: $int_type, _guard: &SeqLockWriteGuard<'static>) {
|
||||
// SAFETY: integers are plain old data types so we can always transmute them to arrays of integers.
|
||||
let val = unsafe { mem::transmute::<$int_type, [Chunk; Self::LEN]>(val) };
|
||||
// SAFETY:
|
||||
// - The guard guarantees that we hold the lock to write.
|
||||
// - There are no threads that perform non-atomic concurrent read or write operations.
|
||||
//
|
||||
// See optimistic_read for the reason that atomic operations are used here.
|
||||
let chunks = unsafe { self.chunks() };
|
||||
for i in 0..Self::LEN {
|
||||
chunks[i].store(val[i], Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Send is implicitly implemented.
|
||||
// SAFETY: any data races are prevented by the lock and atomic operation.
|
||||
unsafe impl Sync for $atomic_type {}
|
||||
|
||||
#[cfg(any(
|
||||
test,
|
||||
not(any(
|
||||
all(
|
||||
target_arch = "x86_64",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
portable_atomic_no_cmpxchg16b_intrinsic,
|
||||
)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
not(any(target_env = "sgx", miri)),
|
||||
),
|
||||
all(
|
||||
target_arch = "powerpc64",
|
||||
not(portable_atomic_no_asm),
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
any(
|
||||
all(
|
||||
target_os = "linux",
|
||||
any(
|
||||
all(
|
||||
target_env = "gnu",
|
||||
any(target_endian = "little", not(target_feature = "crt-static")),
|
||||
),
|
||||
all(
|
||||
target_env = "musl",
|
||||
any(not(target_feature = "crt-static"), feature = "std"),
|
||||
),
|
||||
target_env = "ohos",
|
||||
all(target_env = "uclibc", not(target_feature = "crt-static")),
|
||||
portable_atomic_outline_atomics,
|
||||
),
|
||||
),
|
||||
target_os = "android",
|
||||
all(
|
||||
target_os = "freebsd",
|
||||
any(
|
||||
target_endian = "little",
|
||||
not(target_feature = "crt-static"),
|
||||
portable_atomic_outline_atomics,
|
||||
),
|
||||
),
|
||||
target_os = "openbsd",
|
||||
all(
|
||||
target_os = "aix",
|
||||
not(portable_atomic_pre_llvm_20),
|
||||
portable_atomic_outline_atomics, // TODO(aix): currently disabled by default
|
||||
),
|
||||
),
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
),
|
||||
all(
|
||||
target_arch = "riscv64",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
any(target_os = "linux", target_os = "android"),
|
||||
),
|
||||
)),
|
||||
))]
|
||||
items!({
|
||||
impl_default_no_fetch_ops!($atomic_type, $int_type);
|
||||
impl_default_bit_opts!($atomic_type, $int_type);
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
pub(crate) const fn new(v: $int_type) -> Self {
|
||||
Self { v: UnsafeCell::new(v) }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn is_lock_free() -> bool {
|
||||
Self::IS_ALWAYS_LOCK_FREE
|
||||
}
|
||||
pub(crate) const IS_ALWAYS_LOCK_FREE: bool = false;
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
all(debug_assertions, not(portable_atomic_no_track_caller)),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn compare_exchange_weak(
|
||||
&self,
|
||||
current: $int_type,
|
||||
new: $int_type,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<$int_type, $int_type> {
|
||||
self.compare_exchange(current, new, success, failure)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn not(&self, order: Ordering) {
|
||||
self.fetch_not(order);
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn neg(&self, order: Ordering) {
|
||||
self.fetch_neg(order);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) const fn as_ptr(&self) -> *mut $int_type {
|
||||
self.v.get()
|
||||
}
|
||||
}
|
||||
});
|
||||
#[cfg_attr(
|
||||
any(
|
||||
all(
|
||||
target_arch = "x86_64",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
portable_atomic_no_cmpxchg16b_intrinsic,
|
||||
)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
not(any(target_env = "sgx", miri)),
|
||||
),
|
||||
all(
|
||||
target_arch = "powerpc64",
|
||||
not(portable_atomic_no_asm),
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
any(
|
||||
all(
|
||||
target_os = "linux",
|
||||
any(
|
||||
all(
|
||||
target_env = "gnu",
|
||||
any(target_endian = "little", not(target_feature = "crt-static")),
|
||||
),
|
||||
all(
|
||||
target_env = "musl",
|
||||
any(not(target_feature = "crt-static"), feature = "std"),
|
||||
),
|
||||
target_env = "ohos",
|
||||
all(target_env = "uclibc", not(target_feature = "crt-static")),
|
||||
portable_atomic_outline_atomics,
|
||||
),
|
||||
),
|
||||
target_os = "android",
|
||||
all(
|
||||
target_os = "freebsd",
|
||||
any(
|
||||
target_endian = "little",
|
||||
not(target_feature = "crt-static"),
|
||||
portable_atomic_outline_atomics,
|
||||
),
|
||||
),
|
||||
target_os = "openbsd",
|
||||
all(
|
||||
target_os = "aix",
|
||||
not(portable_atomic_pre_llvm_20),
|
||||
portable_atomic_outline_atomics, // TODO(aix): currently disabled by default
|
||||
),
|
||||
),
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
),
|
||||
all(
|
||||
target_arch = "riscv64",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
any(target_os = "linux", target_os = "android"),
|
||||
),
|
||||
),
|
||||
allow(dead_code)
|
||||
)]
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
|
||||
pub(crate) fn load(&self, order: Ordering) -> $int_type {
|
||||
crate::utils::assert_load_ordering(order);
|
||||
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
|
||||
let lock = lock(self.v.get().addr());
|
||||
|
||||
// Try doing an optimistic read first.
|
||||
if let Some(stamp) = lock.optimistic_read() {
|
||||
let val = self.optimistic_read();
|
||||
|
||||
if lock.validate_read(stamp) {
|
||||
return val;
|
||||
}
|
||||
}
|
||||
|
||||
// Grab a regular write lock so that writers don't starve this load.
|
||||
let guard = lock.write();
|
||||
let val = self.read(&guard);
|
||||
// The value hasn't been changed. Drop the guard without incrementing the stamp.
|
||||
guard.abort();
|
||||
val
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
|
||||
pub(crate) fn store(&self, val: $int_type, order: Ordering) {
|
||||
crate::utils::assert_store_ordering(order);
|
||||
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
|
||||
let guard = lock(self.v.get().addr()).write();
|
||||
self.write(val, &guard)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn swap(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
|
||||
let guard = lock(self.v.get().addr()).write();
|
||||
let prev = self.read(&guard);
|
||||
self.write(val, &guard);
|
||||
prev
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
|
||||
pub(crate) fn compare_exchange(
|
||||
&self,
|
||||
current: $int_type,
|
||||
new: $int_type,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<$int_type, $int_type> {
|
||||
crate::utils::assert_compare_exchange_ordering(success, failure);
|
||||
let _sc_fence = ScFenceGuard::new(
|
||||
success == Ordering::SeqCst || failure == Ordering::SeqCst
|
||||
);
|
||||
let guard = lock(self.v.get().addr()).write();
|
||||
let prev = self.read(&guard);
|
||||
if prev == current {
|
||||
self.write(new, &guard);
|
||||
Ok(prev)
|
||||
} else {
|
||||
// The value hasn't been changed. Drop the guard without incrementing the stamp.
|
||||
guard.abort();
|
||||
Err(prev)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
|
||||
let guard = lock(self.v.get().addr()).write();
|
||||
let prev = self.read(&guard);
|
||||
self.write(prev.wrapping_add(val), &guard);
|
||||
prev
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
|
||||
let guard = lock(self.v.get().addr()).write();
|
||||
let prev = self.read(&guard);
|
||||
self.write(prev.wrapping_sub(val), &guard);
|
||||
prev
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
|
||||
let guard = lock(self.v.get().addr()).write();
|
||||
let prev = self.read(&guard);
|
||||
self.write(prev & val, &guard);
|
||||
prev
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_nand(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
|
||||
let guard = lock(self.v.get().addr()).write();
|
||||
let prev = self.read(&guard);
|
||||
self.write(!(prev & val), &guard);
|
||||
prev
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
|
||||
let guard = lock(self.v.get().addr()).write();
|
||||
let prev = self.read(&guard);
|
||||
self.write(prev | val, &guard);
|
||||
prev
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
|
||||
let guard = lock(self.v.get().addr()).write();
|
||||
let prev = self.read(&guard);
|
||||
self.write(prev ^ val, &guard);
|
||||
prev
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_not(&self, order: Ordering) -> $int_type {
|
||||
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
|
||||
let guard = lock(self.v.get().addr()).write();
|
||||
let prev = self.read(&guard);
|
||||
self.write(!prev, &guard);
|
||||
prev
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_neg(&self, order: Ordering) -> $int_type {
|
||||
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
|
||||
let guard = lock(self.v.get().addr()).write();
|
||||
let prev = self.read(&guard);
|
||||
self.write(prev.wrapping_neg(), &guard);
|
||||
prev
|
||||
}
|
||||
}
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
pub(crate) fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
|
||||
let guard = lock(self.v.get().addr()).write();
|
||||
let prev = self.read(&guard);
|
||||
self.write(core::cmp::max(prev, val), &guard);
|
||||
prev
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type {
|
||||
let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);
|
||||
let guard = lock(self.v.get().addr()).write();
|
||||
let prev = self.read(&guard);
|
||||
self.write(core::cmp::min(prev, val), &guard);
|
||||
prev
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(any(
|
||||
test,
|
||||
not(any(
|
||||
not(portable_atomic_no_atomic_64),
|
||||
all(
|
||||
target_arch = "riscv32",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
|
||||
),
|
||||
))
|
||||
))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(
|
||||
test,
|
||||
not(any(
|
||||
target_has_atomic = "64",
|
||||
all(
|
||||
target_arch = "riscv32",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
|
||||
),
|
||||
))
|
||||
))
|
||||
)]
|
||||
cfg_no_fast_atomic_64!({
|
||||
atomic!(AtomicI64, i64, 8);
|
||||
atomic!(AtomicU64, u64, 8);
|
||||
});
|
||||
|
||||
atomic!(AtomicI128, i128, 16);
|
||||
atomic!(AtomicU128, u128, 16);
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
cfg_no_fast_atomic_64!({
|
||||
test_atomic_int!(i64);
|
||||
test_atomic_int!(u64);
|
||||
});
|
||||
test_atomic_int!(i128);
|
||||
test_atomic_int!(u128);
|
||||
|
||||
// load/store/swap implementation is not affected by signedness, so it is
|
||||
// enough to test only unsigned types.
|
||||
cfg_no_fast_atomic_64!({
|
||||
stress_test!(u64);
|
||||
});
|
||||
stress_test!(u128);
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Helper for outline-atomics.
|
||||
|
||||
On architectures where DW atomics are not supported on older CPUs, we use
|
||||
fallback implementation when DW atomic instructions are not supported and
|
||||
outline-atomics is enabled.
|
||||
|
||||
This module provides helpers to implement them.
|
||||
*/
|
||||
|
||||
use core::sync::atomic::Ordering;
|
||||
|
||||
#[cfg(any(target_arch = "x86_64", target_arch = "powerpc64", target_arch = "riscv64"))]
|
||||
pub(crate) type Udw = u128;
|
||||
#[cfg(any(target_arch = "x86_64", target_arch = "powerpc64", target_arch = "riscv64"))]
|
||||
pub(crate) type AtomicUdw = super::super::super::fallback::AtomicU128;
|
||||
#[cfg(any(target_arch = "x86_64", target_arch = "powerpc64", target_arch = "riscv64"))]
|
||||
pub(crate) type AtomicIdw = super::super::super::fallback::AtomicI128;
|
||||
|
||||
#[cfg(any(target_arch = "arm", target_arch = "riscv32"))]
|
||||
pub(crate) type Udw = u64;
|
||||
#[cfg(any(target_arch = "arm", target_arch = "riscv32"))]
|
||||
pub(crate) type AtomicUdw = super::super::super::fallback::AtomicU64;
|
||||
#[cfg(any(target_arch = "arm", target_arch = "riscv32"))]
|
||||
pub(crate) type AtomicIdw = super::super::super::fallback::AtomicI64;
|
||||
|
||||
// Asserts that the function is called in the correct context.
|
||||
macro_rules! debug_assert_outline_atomics {
|
||||
() => {
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
{
|
||||
debug_assert!(!super::detect::detect().cmpxchg16b());
|
||||
}
|
||||
#[cfg(target_arch = "powerpc64")]
|
||||
{
|
||||
debug_assert!(!super::detect::detect().quadword_atomics());
|
||||
}
|
||||
#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
|
||||
{
|
||||
debug_assert!(!super::detect::detect().zacas());
|
||||
}
|
||||
#[cfg(target_arch = "arm")]
|
||||
{
|
||||
debug_assert!(!super::has_kuser_cmpxchg64());
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
#[cold]
|
||||
pub(crate) unsafe fn atomic_load(src: *mut Udw, order: Ordering) -> Udw {
|
||||
debug_assert_outline_atomics!();
|
||||
#[allow(clippy::cast_ptr_alignment)]
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
(*(src as *const AtomicUdw)).load(order)
|
||||
}
|
||||
}
|
||||
fn_alias! {
|
||||
#[cold]
|
||||
pub(crate) unsafe fn(src: *mut Udw) -> Udw;
|
||||
// fallback's atomic load has at least acquire semantics.
|
||||
#[cfg(not(any(target_arch = "arm", target_arch = "x86_64")))]
|
||||
atomic_load_non_seqcst = atomic_load(Ordering::Acquire);
|
||||
atomic_load_seqcst = atomic_load(Ordering::SeqCst);
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_arch = "arm", target_arch = "riscv32", target_arch = "riscv64")))]
|
||||
#[cold]
|
||||
pub(crate) unsafe fn atomic_store(dst: *mut Udw, val: Udw, order: Ordering) {
|
||||
debug_assert_outline_atomics!();
|
||||
#[allow(clippy::cast_ptr_alignment)]
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
(*(dst as *const AtomicUdw)).store(val, order);
|
||||
}
|
||||
}
|
||||
#[cfg(not(any(target_arch = "arm", target_arch = "riscv32", target_arch = "riscv64")))]
|
||||
fn_alias! {
|
||||
#[cold]
|
||||
pub(crate) unsafe fn(dst: *mut Udw, val: Udw);
|
||||
// fallback's atomic store has at least release semantics.
|
||||
atomic_store_non_seqcst = atomic_store(Ordering::Release);
|
||||
atomic_store_seqcst = atomic_store(Ordering::SeqCst);
|
||||
}
|
||||
|
||||
#[cold]
|
||||
pub(crate) unsafe fn atomic_compare_exchange(
|
||||
dst: *mut Udw,
|
||||
old: Udw,
|
||||
new: Udw,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> (Udw, bool) {
|
||||
debug_assert_outline_atomics!();
|
||||
#[allow(clippy::cast_ptr_alignment)]
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
match (*(dst as *const AtomicUdw)).compare_exchange(old, new, success, failure) {
|
||||
Ok(v) => (v, true),
|
||||
Err(v) => (v, false),
|
||||
}
|
||||
}
|
||||
}
|
||||
fn_alias! {
|
||||
#[cold]
|
||||
pub(crate) unsafe fn(dst: *mut Udw, old: Udw, new: Udw) -> (Udw, bool);
|
||||
// fallback's atomic CAS has at least AcqRel semantics.
|
||||
#[cfg(not(any(target_arch = "arm", target_arch = "x86_64")))]
|
||||
atomic_compare_exchange_non_seqcst
|
||||
= atomic_compare_exchange(Ordering::AcqRel, Ordering::Acquire);
|
||||
atomic_compare_exchange_seqcst
|
||||
= atomic_compare_exchange(Ordering::SeqCst, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
macro_rules! atomic_rmw_3 {
|
||||
(
|
||||
$name:ident($atomic_type:ident::$method_name:ident),
|
||||
$non_seqcst_alias:ident, $seqcst_alias:ident
|
||||
) => {
|
||||
#[cold]
|
||||
pub(crate) unsafe fn $name(dst: *mut Udw, val: Udw, order: Ordering) -> Udw {
|
||||
debug_assert_outline_atomics!();
|
||||
#[allow(
|
||||
clippy::as_underscore,
|
||||
clippy::cast_possible_wrap,
|
||||
clippy::cast_ptr_alignment,
|
||||
clippy::cast_sign_loss
|
||||
)]
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
(*(dst as *const $atomic_type)).$method_name(val as _, order) as Udw
|
||||
}
|
||||
}
|
||||
fn_alias! {
|
||||
#[cold]
|
||||
pub(crate) unsafe fn(dst: *mut Udw, val: Udw) -> Udw;
|
||||
// fallback's atomic RMW has at least AcqRel semantics.
|
||||
#[cfg(not(any(target_arch = "arm", target_arch = "x86_64")))]
|
||||
$non_seqcst_alias = $name(Ordering::AcqRel);
|
||||
$seqcst_alias = $name(Ordering::SeqCst);
|
||||
}
|
||||
};
|
||||
}
|
||||
macro_rules! atomic_rmw_2 {
|
||||
(
|
||||
$name:ident($atomic_type:ident::$method_name:ident),
|
||||
$non_seqcst_alias:ident, $seqcst_alias:ident
|
||||
) => {
|
||||
#[cold]
|
||||
pub(crate) unsafe fn $name(dst: *mut Udw, order: Ordering) -> Udw {
|
||||
debug_assert_outline_atomics!();
|
||||
#[allow(clippy::cast_ptr_alignment)]
|
||||
// SAFETY: the caller must uphold the safety contract.
|
||||
unsafe {
|
||||
(*(dst as *const $atomic_type)).$method_name(order) as Udw
|
||||
}
|
||||
}
|
||||
fn_alias! {
|
||||
#[cold]
|
||||
pub(crate) unsafe fn(dst: *mut Udw) -> Udw;
|
||||
// fallback's atomic RMW has at least AcqRel semantics.
|
||||
#[cfg(not(any(target_arch = "arm", target_arch = "x86_64")))]
|
||||
$non_seqcst_alias = $name(Ordering::AcqRel);
|
||||
$seqcst_alias = $name(Ordering::SeqCst);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
atomic_rmw_3!(atomic_swap(AtomicUdw::swap), atomic_swap_non_seqcst, atomic_swap_seqcst);
|
||||
atomic_rmw_3!(atomic_add(AtomicUdw::fetch_add), atomic_add_non_seqcst, atomic_add_seqcst);
|
||||
atomic_rmw_3!(atomic_sub(AtomicUdw::fetch_sub), atomic_sub_non_seqcst, atomic_sub_seqcst);
|
||||
atomic_rmw_3!(atomic_and(AtomicUdw::fetch_and), atomic_and_non_seqcst, atomic_and_seqcst);
|
||||
atomic_rmw_3!(atomic_nand(AtomicUdw::fetch_nand), atomic_nand_non_seqcst, atomic_nand_seqcst);
|
||||
atomic_rmw_3!(atomic_or(AtomicUdw::fetch_or), atomic_or_non_seqcst, atomic_or_seqcst);
|
||||
atomic_rmw_3!(atomic_xor(AtomicUdw::fetch_xor), atomic_xor_non_seqcst, atomic_xor_seqcst);
|
||||
atomic_rmw_3!(atomic_max(AtomicIdw::fetch_max), atomic_max_non_seqcst, atomic_max_seqcst);
|
||||
atomic_rmw_3!(atomic_umax(AtomicUdw::fetch_max), atomic_umax_non_seqcst, atomic_umax_seqcst);
|
||||
atomic_rmw_3!(atomic_min(AtomicIdw::fetch_min), atomic_min_non_seqcst, atomic_min_seqcst);
|
||||
atomic_rmw_3!(atomic_umin(AtomicUdw::fetch_min), atomic_umin_non_seqcst, atomic_umin_seqcst);
|
||||
|
||||
atomic_rmw_2!(atomic_not(AtomicUdw::fetch_not), atomic_not_non_seqcst, atomic_not_seqcst);
|
||||
atomic_rmw_2!(atomic_neg(AtomicUdw::fetch_neg), atomic_neg_non_seqcst, atomic_neg_seqcst);
|
||||
@@ -0,0 +1,56 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/atomic/seq_lock.rs.
|
||||
|
||||
include!("seq_lock_common.rs");
|
||||
|
||||
use crate::imp::core_atomic::AtomicU64;
|
||||
|
||||
pub(super) type State = u64;
|
||||
|
||||
/// A simple stamped lock.
|
||||
pub(super) struct SeqLock {
|
||||
/// The current state of the lock.
|
||||
///
|
||||
/// All bits except the least significant one hold the current stamp. When locked, the state
|
||||
/// equals 1 and doesn't contain a valid stamp.
|
||||
state: AtomicU64,
|
||||
}
|
||||
|
||||
impl SeqLock {
|
||||
#[inline]
|
||||
pub(super) const fn new() -> Self {
|
||||
Self { state: AtomicU64::new(0) }
|
||||
}
|
||||
|
||||
/// If not locked, returns the current stamp.
|
||||
///
|
||||
/// This method should be called before optimistic reads.
|
||||
#[inline]
|
||||
pub(super) fn optimistic_read(&self) -> Option<State> {
|
||||
// The acquire load from `state` synchronizes with the release stores in
|
||||
// `SeqLockWriteGuard::drop` and `SeqLockWriteGuard::abort`.
|
||||
let state = self.state.load(Ordering::Acquire);
|
||||
if state == LOCKED { None } else { Some(state) }
|
||||
}
|
||||
|
||||
/// Returns `true` if the current stamp is equal to `stamp`.
|
||||
///
|
||||
/// This method should be called after optimistic reads to check whether they are valid. The
|
||||
/// argument `stamp` should correspond to the one returned by method `optimistic_read`.
|
||||
#[inline]
|
||||
pub(super) fn validate_read(&self, stamp: State) -> bool {
|
||||
// Thanks to the fence, if we're noticing any modification to the data at the critical
|
||||
// section of `stamp`, then the critical section's write of 1 to state
|
||||
// should be visible.
|
||||
crate::fence(Ordering::Acquire);
|
||||
self.state.load(Ordering::Relaxed) == stamp
|
||||
}
|
||||
}
|
||||
|
||||
impl SeqLockWriteGuard<'_> {
|
||||
#[inline]
|
||||
fn next_stamp(&self) -> State {
|
||||
self.state.wrapping_add(2)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/atomic/seq_lock.rs
|
||||
// and https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/atomic/seq_lock_wide.rs.
|
||||
|
||||
use core::{mem::ManuallyDrop, sync::atomic::Ordering};
|
||||
|
||||
use super::utils::Backoff;
|
||||
#[cfg(portable_atomic_unsafe_assume_privileged)]
|
||||
use crate::imp::interrupt::arch as interrupt;
|
||||
|
||||
const LOCKED: State = 1;
|
||||
|
||||
impl SeqLock {
|
||||
/// Grabs the lock for writing.
|
||||
#[inline]
|
||||
pub(super) fn write(&self) -> SeqLockWriteGuard<'_> {
|
||||
// Get current interrupt state and disable interrupts when the user
|
||||
// explicitly declares that privileged instructions are available.
|
||||
#[cfg(portable_atomic_unsafe_assume_privileged)]
|
||||
let interrupt_state = interrupt::disable();
|
||||
|
||||
let mut backoff = Backoff::new();
|
||||
loop {
|
||||
let previous = self.state.swap(LOCKED, Ordering::Acquire);
|
||||
|
||||
if previous != LOCKED {
|
||||
// To synchronize with the acquire fence in `validate_read` via any modification to
|
||||
// the data at the critical section of `previous` (seq_lock.rs) or `(state_hi, previous)`
|
||||
// (seq_lock_wide.rs).
|
||||
crate::fence(Ordering::Release);
|
||||
|
||||
return SeqLockWriteGuard {
|
||||
lock: self,
|
||||
state: previous,
|
||||
#[cfg(portable_atomic_unsafe_assume_privileged)]
|
||||
interrupt_state,
|
||||
};
|
||||
}
|
||||
|
||||
while self.state.load(Ordering::Relaxed) == LOCKED {
|
||||
backoff.snooze();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// An RAII guard that releases the lock and increments the stamp when dropped.
|
||||
#[must_use]
|
||||
pub(crate) struct SeqLockWriteGuard<'a> {
|
||||
/// The parent lock.
|
||||
lock: &'a SeqLock,
|
||||
|
||||
/// The stamp before locking.
|
||||
state: State,
|
||||
|
||||
/// The interrupt state before disabling.
|
||||
#[cfg(portable_atomic_unsafe_assume_privileged)]
|
||||
interrupt_state: interrupt::State,
|
||||
}
|
||||
|
||||
impl SeqLockWriteGuard<'_> {
|
||||
/// Releases the lock without incrementing the stamp.
|
||||
#[inline]
|
||||
pub(crate) fn abort(self) {
|
||||
// We specifically don't want to call drop(), since that's
|
||||
// what increments the stamp.
|
||||
let this = ManuallyDrop::new(self);
|
||||
|
||||
// Restore the stamp.
|
||||
//
|
||||
// Release ordering for synchronizing with `optimistic_read`.
|
||||
this.lock.state.store(this.state, Ordering::Release);
|
||||
|
||||
// Restore interrupt state.
|
||||
// SAFETY: the state was retrieved by the previous `disable`.
|
||||
#[cfg(portable_atomic_unsafe_assume_privileged)]
|
||||
unsafe {
|
||||
interrupt::restore(this.interrupt_state);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SeqLockWriteGuard<'_> {
|
||||
#[inline]
|
||||
fn drop(&mut self) {
|
||||
let state = self.next_stamp();
|
||||
|
||||
// Release the lock and increment the stamp.
|
||||
//
|
||||
// Release ordering for synchronizing with `optimistic_read`.
|
||||
self.lock.state.store(state, Ordering::Release);
|
||||
|
||||
// Restore interrupt state.
|
||||
// SAFETY: the state was retrieved by the previous `disable`.
|
||||
#[cfg(portable_atomic_unsafe_assume_privileged)]
|
||||
unsafe {
|
||||
interrupt::restore(self.interrupt_state);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::items_after_test_module)]
|
||||
mod tests {
|
||||
use core::sync::atomic::Ordering;
|
||||
|
||||
use super::SeqLock;
|
||||
|
||||
#[test]
|
||||
fn smoke() {
|
||||
let lock = SeqLock::new();
|
||||
let before = lock.optimistic_read().unwrap();
|
||||
assert!(lock.validate_read(before));
|
||||
{
|
||||
let _guard = lock.write();
|
||||
assert!(lock.optimistic_read().is_none());
|
||||
}
|
||||
assert!(!lock.validate_read(before));
|
||||
let after = lock.optimistic_read().unwrap();
|
||||
assert_ne!(before, after);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wrap() {
|
||||
let lock = SeqLock::new();
|
||||
let zero = lock.optimistic_read().unwrap();
|
||||
lock.state.store(0xFFFF_FFFE, Ordering::Relaxed); // u32::MAX & !1
|
||||
let before = lock.optimistic_read().unwrap();
|
||||
assert!(lock.validate_read(before));
|
||||
{
|
||||
let _guard = lock.write();
|
||||
assert!(lock.optimistic_read().is_none());
|
||||
}
|
||||
assert!(!lock.validate_read(before));
|
||||
let after = lock.optimistic_read().unwrap();
|
||||
assert_ne!(before, after);
|
||||
assert_ne!(after, zero);
|
||||
#[allow(clippy::identity_op)]
|
||||
{
|
||||
assert_eq!(lock.state.load(Ordering::Relaxed) & 0xFFFF_FFFF, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_abort() {
|
||||
let lock = SeqLock::new();
|
||||
let before = lock.optimistic_read().unwrap();
|
||||
{
|
||||
let guard = lock.write();
|
||||
assert!(lock.optimistic_read().is_none());
|
||||
guard.abort();
|
||||
}
|
||||
let after = lock.optimistic_read().unwrap();
|
||||
assert_eq!(before, after, "aborted write does not update the stamp");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/atomic/seq_lock_wide.rs.
|
||||
|
||||
include!("seq_lock_common.rs");
|
||||
|
||||
use crate::imp::core_atomic::AtomicU32;
|
||||
|
||||
pub(super) type State = u32;
|
||||
|
||||
/// A simple stamped lock.
|
||||
///
|
||||
/// The state is represented as two `AtomicU32`: `state_hi` for high bits and `state_lo` for low
|
||||
/// bits.
|
||||
pub(super) struct SeqLock {
|
||||
/// The high bits of the current state of the lock.
|
||||
state_hi: AtomicU32,
|
||||
|
||||
/// The low bits of the current state of the lock.
|
||||
///
|
||||
/// All bits except the least significant one hold the current stamp. When locked, the state_lo
|
||||
/// equals 1 and doesn't contain a valid stamp.
|
||||
state: AtomicU32,
|
||||
}
|
||||
|
||||
impl SeqLock {
|
||||
#[inline]
|
||||
pub(super) const fn new() -> Self {
|
||||
Self { state_hi: AtomicU32::new(0), state: AtomicU32::new(0) }
|
||||
}
|
||||
|
||||
/// If not locked, returns the current stamp.
|
||||
///
|
||||
/// This method should be called before optimistic reads.
|
||||
#[inline]
|
||||
pub(super) fn optimistic_read(&self) -> Option<(State, State)> {
|
||||
// The acquire loads from `state_hi` and `state_lo` synchronize with the release stores in
|
||||
// `SeqLockWriteGuard::drop` and `SeqLockWriteGuard::abort`.
|
||||
//
|
||||
// As a consequence, we can make sure that (1) all writes within the era of `state_hi - 1`
|
||||
// happens before now; and therefore, (2) if `state_lo` is even, all writes within the
|
||||
// critical section of (`state_hi`, `state_lo`) happens before now.
|
||||
let state_hi = self.state_hi.load(Ordering::Acquire);
|
||||
let state_lo = self.state.load(Ordering::Acquire);
|
||||
if state_lo == LOCKED { None } else { Some((state_hi, state_lo)) }
|
||||
}
|
||||
|
||||
/// Returns `true` if the current stamp is equal to `stamp`.
|
||||
///
|
||||
/// This method should be called after optimistic reads to check whether they are valid. The
|
||||
/// argument `stamp` should correspond to the one returned by method `optimistic_read`.
|
||||
#[inline]
|
||||
pub(super) fn validate_read(&self, stamp: (State, State)) -> bool {
|
||||
// Thanks to the fence, if we're noticing any modification to the data at the critical
|
||||
// section of `(stamp.0, stamp.1)`, then the critical section's write of 1 to state_lo
|
||||
// should be visible.
|
||||
crate::fence(Ordering::Acquire);
|
||||
|
||||
// So if `state_lo` coincides with `stamp.1`, then either (1) we're noticing no modification
|
||||
// to the data after the critical section of `(stamp.0, stamp.1)`, or (2) `state_lo` wrapped
|
||||
// around.
|
||||
//
|
||||
// If (2) is the case, the acquire ordering ensures we see the new value of `state_hi`.
|
||||
let state_lo = self.state.load(Ordering::Acquire);
|
||||
|
||||
// If (2) is the case and `state_hi` coincides with `stamp.0`, then `state_hi` also wrapped
|
||||
// around, which we give up to correctly validate the read.
|
||||
let state_hi = self.state_hi.load(Ordering::Relaxed);
|
||||
|
||||
// Except for the case that both `state_hi` and `state_lo` wrapped around, the following
|
||||
// condition implies that we're noticing no modification to the data after the critical
|
||||
// section of `(stamp.0, stamp.1)`.
|
||||
(state_hi, state_lo) == stamp
|
||||
}
|
||||
}
|
||||
|
||||
impl SeqLockWriteGuard<'_> {
|
||||
#[inline]
|
||||
fn next_stamp(&self) -> State {
|
||||
let state_lo = self.state.wrapping_add(2);
|
||||
|
||||
// Increase the high bits if the low bits wrap around.
|
||||
//
|
||||
// Release ordering for synchronizing with `optimistic_read`.
|
||||
if state_lo == 0 {
|
||||
let state_hi = self.lock.state_hi.load(Ordering::Relaxed);
|
||||
self.lock.state_hi.store(state_hi.wrapping_add(1), Ordering::Release);
|
||||
}
|
||||
|
||||
state_lo
|
||||
}
|
||||
}
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
use core::ops;
|
||||
|
||||
// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/cache_padded.rs.
|
||||
/// Pads and aligns a value to the length of a cache line.
|
||||
// Starting from Intel's Sandy Bridge, spatial prefetcher is now pulling pairs of 64-byte cache
|
||||
// lines at a time, so we have to align to 128 bytes rather than 64.
|
||||
//
|
||||
// Sources:
|
||||
// - https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf
|
||||
// - https://github.com/facebook/folly/blob/1b5288e6eea6df074758f877c849b6e73bbb9fbb/folly/lang/Align.h#L107
|
||||
//
|
||||
// aarch64/arm64ec's big.LITTLE architecture has asymmetric cores and "big" cores have 128-byte cache line size.
|
||||
//
|
||||
// Sources:
|
||||
// - https://www.mono-project.com/news/2016/09/12/arm64-icache/
|
||||
//
|
||||
// powerpc64 has 128-byte cache line size.
|
||||
//
|
||||
// Sources:
|
||||
// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_ppc64x.go#L9
|
||||
// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/powerpc/include/asm/cache.h#L26
|
||||
#[cfg_attr(
|
||||
any(
|
||||
target_arch = "x86_64",
|
||||
target_arch = "aarch64",
|
||||
target_arch = "arm64ec",
|
||||
target_arch = "powerpc64",
|
||||
),
|
||||
repr(align(128))
|
||||
)]
|
||||
// arm, mips, mips64, sparc, and hexagon have 32-byte cache line size.
|
||||
//
|
||||
// Sources:
|
||||
// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_arm.go#L7
|
||||
// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips.go#L7
|
||||
// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mipsle.go#L7
|
||||
// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips64x.go#L9
|
||||
// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/sparc/include/asm/cache.h#L17
|
||||
// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/hexagon/include/asm/cache.h#L12
|
||||
#[cfg_attr(
|
||||
any(
|
||||
target_arch = "arm",
|
||||
target_arch = "mips",
|
||||
target_arch = "mips32r6",
|
||||
target_arch = "mips64",
|
||||
target_arch = "mips64r6",
|
||||
target_arch = "sparc",
|
||||
target_arch = "hexagon",
|
||||
),
|
||||
repr(align(32))
|
||||
)]
|
||||
// m68k has 16-byte cache line size.
|
||||
//
|
||||
// Sources:
|
||||
// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/m68k/include/asm/cache.h#L9
|
||||
#[cfg_attr(target_arch = "m68k", repr(align(16)))]
|
||||
// s390x has 256-byte cache line size.
|
||||
//
|
||||
// Sources:
|
||||
// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_s390x.go#L7
|
||||
// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/s390/include/asm/cache.h#L13
|
||||
#[cfg_attr(target_arch = "s390x", repr(align(256)))]
|
||||
// x86, wasm, riscv, and sparc64 have 64-byte cache line size.
|
||||
//
|
||||
// Sources:
|
||||
// - https://github.com/golang/go/blob/dda2991c2ea0c5914714469c4defc2562a907230/src/internal/cpu/cpu_x86.go#L9
|
||||
// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_wasm.go#L7
|
||||
// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/riscv/include/asm/cache.h#L10
|
||||
// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/sparc/include/asm/cache.h#L19
|
||||
//
|
||||
// All others are assumed to have 64-byte cache line size.
|
||||
#[cfg_attr(
|
||||
not(any(
|
||||
target_arch = "x86_64",
|
||||
target_arch = "aarch64",
|
||||
target_arch = "arm64ec",
|
||||
target_arch = "powerpc64",
|
||||
target_arch = "arm",
|
||||
target_arch = "mips",
|
||||
target_arch = "mips32r6",
|
||||
target_arch = "mips64",
|
||||
target_arch = "mips64r6",
|
||||
target_arch = "sparc",
|
||||
target_arch = "hexagon",
|
||||
target_arch = "m68k",
|
||||
target_arch = "s390x",
|
||||
)),
|
||||
repr(align(64))
|
||||
)]
|
||||
#[repr(C)]
|
||||
pub(crate) struct CachePadded<T> {
|
||||
value: T,
|
||||
}
|
||||
|
||||
impl<T> CachePadded<T> {
|
||||
#[inline]
|
||||
pub(crate) const fn new(value: T) -> Self {
|
||||
Self { value }
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> ops::Deref for CachePadded<T> {
|
||||
type Target = T;
|
||||
|
||||
#[inline]
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.value
|
||||
}
|
||||
}
|
||||
|
||||
// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/backoff.rs.
|
||||
// Adjusted to reduce spinning.
|
||||
/// Performs exponential backoff in spin loops.
|
||||
pub(crate) struct Backoff {
|
||||
step: u32,
|
||||
}
|
||||
|
||||
// https://github.com/oneapi-src/oneTBB/blob/v2021.5.0/include/oneapi/tbb/detail/_utils.h#L46-L48
|
||||
const SPIN_LIMIT: u32 = 4;
|
||||
|
||||
impl Backoff {
|
||||
#[inline]
|
||||
pub(crate) const fn new() -> Self {
|
||||
Self { step: 0 }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn snooze(&mut self) {
|
||||
if self.step <= SPIN_LIMIT {
|
||||
for _ in 0..1 << self.step {
|
||||
#[allow(deprecated)]
|
||||
core::sync::atomic::spin_loop_hint();
|
||||
}
|
||||
self.step += 1;
|
||||
} else {
|
||||
#[cfg(not(feature = "std"))]
|
||||
for _ in 0..1 << self.step {
|
||||
#[allow(deprecated)]
|
||||
core::sync::atomic::spin_loop_hint();
|
||||
}
|
||||
|
||||
#[cfg(feature = "std")]
|
||||
std::thread::yield_now();
|
||||
}
|
||||
}
|
||||
}
|
||||
+263
@@ -0,0 +1,263 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Atomic float implementation based on AArch64 with FEAT_LSFE.
|
||||
|
||||
This module provides atomic float implementations using FEAT_LSFE instructions.
|
||||
|
||||
Generated asm:
|
||||
- aarch64 (+lsfe) https://godbolt.org/z/7vaxeofv1
|
||||
*/
|
||||
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
use core::arch::asm;
|
||||
use core::sync::atomic::Ordering;
|
||||
|
||||
#[cfg(portable_atomic_unstable_f16)]
|
||||
use super::int::AtomicF16;
|
||||
#[cfg(portable_atomic_unstable_f128)]
|
||||
use super::int::AtomicF128;
|
||||
use super::int::{AtomicF32, AtomicF64};
|
||||
|
||||
// TODO: optimize no return cases:
|
||||
// https://developer.arm.com/documentation/ddi0602/2025-06/SIMD-FP-Instructions/STFADD--STFADDL--Floating-point-atomic-add-in-memory--without-return-
|
||||
// https://developer.arm.com/documentation/ddi0602/2025-06/SIMD-FP-Instructions/STFMAXNM--STFMAXNML--Floating-point-atomic-maximum-number-in-memory--without-return-
|
||||
// https://developer.arm.com/documentation/ddi0602/2025-06/SIMD-FP-Instructions/STFMINNM--STFMINNML--Floating-point-atomic-minimum-number-in-memory--without-return-
|
||||
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
macro_rules! start_lsfe {
|
||||
() => {
|
||||
".arch_extension lsfe"
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
macro_rules! atomic_rmw {
|
||||
($op:ident, $order:ident) => {
|
||||
atomic_rmw!($op, $order, write = $order)
|
||||
};
|
||||
($op:ident, $order:ident, write = $write:ident) => {
|
||||
match $order {
|
||||
Ordering::Relaxed => $op!("", "", ""),
|
||||
Ordering::Acquire => $op!("a", "", ""),
|
||||
Ordering::Release => $op!("", "l", ""),
|
||||
Ordering::AcqRel => $op!("a", "l", ""),
|
||||
// In MSVC environments, SeqCst stores/writes by non-LSE* instructions needs fences after writes.
|
||||
// https://reviews.llvm.org/D141748
|
||||
// https://github.com/llvm/llvm-project/commit/1ea201d73be2fdf03347e9c6be09ebed5f8e0e00
|
||||
#[cfg(target_env = "msvc")]
|
||||
Ordering::SeqCst if $write == Ordering::SeqCst => $op!("a", "l", "dmb ish"),
|
||||
// AcqRel and SeqCst RMWs are equivalent in non-MSVC environments.
|
||||
Ordering::SeqCst => $op!("a", "l", ""),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
};
|
||||
}
|
||||
#[cfg(portable_atomic_pre_llvm_20)]
|
||||
macro_rules! atomic_rmw_inst {
|
||||
($op:ident, $order:ident) => {
|
||||
atomic_rmw_inst!($op, $order, write = $order)
|
||||
};
|
||||
($op:ident, $order:ident, write = $write:ident) => {
|
||||
match $order {
|
||||
Ordering::Relaxed => $op!("2", ""), // ""
|
||||
Ordering::Acquire => $op!("a", ""), // "a"
|
||||
Ordering::Release => $op!("6", ""), // "l"
|
||||
Ordering::AcqRel => $op!("e", ""), // "al"
|
||||
// In MSVC environments, SeqCst stores/writes by non-LSE* instructions needs fences after writes.
|
||||
// https://reviews.llvm.org/D141748
|
||||
// https://github.com/llvm/llvm-project/commit/1ea201d73be2fdf03347e9c6be09ebed5f8e0e00
|
||||
#[cfg(target_env = "msvc")]
|
||||
Ordering::SeqCst if $write == Ordering::SeqCst => $op!("e", "dmb ish"),
|
||||
// AcqRel and SeqCst RMWs are equivalent in non-MSVC environments.
|
||||
Ordering::SeqCst => $op!("e", ""),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
#[rustfmt::skip]
|
||||
macro_rules! atomic_float {
|
||||
($atomic_type:ident, $float_type:ident, $modifier:tt, $inst_modifier:tt) => {
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
pub(crate) fn fetch_add(&self, val: $float_type, order: Ordering) -> $float_type {
|
||||
let dst = self.as_ptr();
|
||||
let out;
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
//
|
||||
// Refs: https://developer.arm.com/documentation/ddi0602/2025-06/SIMD-FP-Instructions/LDFADD--LDFADDA--LDFADDAL--LDFADDL--Floating-point-atomic-add-in-memory-
|
||||
unsafe {
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
macro_rules! add {
|
||||
($acquire:tt, $release:tt, $_msvc_fence:tt) => {
|
||||
asm!(
|
||||
start_lsfe!(),
|
||||
concat!("ldfadd", $acquire, $release, " {val:", $modifier, "}, {out:", $modifier, "}, [{dst}]"), // atomic { _x = *dst; *dst = _x + val; out = _x }
|
||||
dst = in(reg) ptr_reg!(dst),
|
||||
val = in(vreg) val,
|
||||
out = lateout(vreg) out,
|
||||
options(nostack),
|
||||
)
|
||||
};
|
||||
}
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
atomic_rmw!(add, order);
|
||||
// LLVM supports FEAT_LSFE instructions on LLVM 20+, so use .inst directive on old LLVM.
|
||||
// https://github.com/llvm/llvm-project/commit/67ff5ba9af9754261abe11d762af11532a816126
|
||||
#[cfg(portable_atomic_pre_llvm_20)]
|
||||
macro_rules! add {
|
||||
($order:tt, $_msvc_fence:tt) => {
|
||||
asm!(
|
||||
// ldfadd{,a,l,al} {h,s,d}0, {h,s,d}1, [x2] // atomic { _x = *x2; *x2 = _x + {h,s,d}0; {h,s,d}1 = _x }
|
||||
concat!(".inst 0x", $inst_modifier, "c", $order, "00041"),
|
||||
in("x2") ptr_reg!(dst),
|
||||
in("v0") val,
|
||||
out("v1") out,
|
||||
options(nostack),
|
||||
)
|
||||
};
|
||||
}
|
||||
#[cfg(portable_atomic_pre_llvm_20)]
|
||||
atomic_rmw_inst!(add, order);
|
||||
}
|
||||
out
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_sub(&self, val: $float_type, order: Ordering) -> $float_type {
|
||||
// There is no atomic sub instruction, so add `-val`.
|
||||
self.fetch_add(-val, order)
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_max(&self, val: $float_type, order: Ordering) -> $float_type {
|
||||
let dst = self.as_ptr();
|
||||
let out;
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
//
|
||||
// Refs: https://developer.arm.com/documentation/ddi0602/2025-06/SIMD-FP-Instructions/LDFMAXNM--LDFMAXNMA--LDFMAXNMAL--LDFMAXNML--Floating-point-atomic-maximum-number-in-memory-
|
||||
unsafe {
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
macro_rules! max {
|
||||
($acquire:tt, $release:tt, $_msvc_fence:tt) => {
|
||||
asm!(
|
||||
start_lsfe!(),
|
||||
concat!("ldfmaxnm", $acquire, $release, " {val:", $modifier, "}, {out:", $modifier, "}, [{dst}]"), // atomic { _x = *dst; *dst = _x.max(val); out = _x }
|
||||
dst = in(reg) ptr_reg!(dst),
|
||||
val = in(vreg) val,
|
||||
out = lateout(vreg) out,
|
||||
options(nostack),
|
||||
)
|
||||
};
|
||||
}
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
atomic_rmw!(max, order);
|
||||
// LLVM supports FEAT_LSFE instructions on LLVM 20+, so use .inst directive on old LLVM.
|
||||
// https://github.com/llvm/llvm-project/commit/67ff5ba9af9754261abe11d762af11532a816126
|
||||
#[cfg(portable_atomic_pre_llvm_20)]
|
||||
macro_rules! max {
|
||||
($order:tt, $_msvc_fence:tt) => {
|
||||
asm!(
|
||||
// ldfmaxnm{,a,l,al} {h,s,d}0, {h,s,d}1, [x2] // atomic { _x = *x2; *x2 = _x.max({h,s,d}0); {h,s,d}1 = _x }
|
||||
concat!(".inst 0x", $inst_modifier, "c", $order, "06041"),
|
||||
in("x2") ptr_reg!(dst),
|
||||
in("v0") val,
|
||||
out("v1") out,
|
||||
options(nostack),
|
||||
)
|
||||
};
|
||||
}
|
||||
#[cfg(portable_atomic_pre_llvm_20)]
|
||||
atomic_rmw_inst!(max, order);
|
||||
}
|
||||
out
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_min(&self, val: $float_type, order: Ordering) -> $float_type {
|
||||
let dst = self.as_ptr();
|
||||
let out;
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
//
|
||||
// Refs: https://developer.arm.com/documentation/ddi0602/2025-06/SIMD-FP-Instructions/LDFMINNM--LDFMINNMA--LDFMINNMAL--LDFMINNML--Floating-point-atomic-minimum-number-in-memory-
|
||||
unsafe {
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
macro_rules! min {
|
||||
($acquire:tt, $release:tt, $_msvc_fence:tt) => {
|
||||
asm!(
|
||||
start_lsfe!(),
|
||||
concat!("ldfminnm", $acquire, $release, " {val:", $modifier, "}, {out:", $modifier, "}, [{dst}]"), // atomic { _x = *dst; *dst = _x.min(val); out = _x }
|
||||
dst = in(reg) ptr_reg!(dst),
|
||||
val = in(vreg) val,
|
||||
out = lateout(vreg) out,
|
||||
options(nostack),
|
||||
)
|
||||
};
|
||||
}
|
||||
#[cfg(not(portable_atomic_pre_llvm_20))]
|
||||
atomic_rmw!(min, order);
|
||||
// LLVM supports FEAT_LSFE instructions on LLVM 20+, so use .inst directive on old LLVM.
|
||||
// https://github.com/llvm/llvm-project/commit/67ff5ba9af9754261abe11d762af11532a816126
|
||||
#[cfg(portable_atomic_pre_llvm_20)]
|
||||
macro_rules! min {
|
||||
($order:tt, $_msvc_fence:tt) => {
|
||||
asm!(
|
||||
// ldfminnm{,a,l,al} {h,s,d}0, {h,s,d}1, [x2] // atomic { _x = *x2; *x2 = _x.min({h,s,d}0); {h,s,d}1 = _x }
|
||||
concat!(".inst 0x", $inst_modifier, "c", $order, "07041"),
|
||||
in("x2") ptr_reg!(dst),
|
||||
in("v0") val,
|
||||
out("v1") out,
|
||||
options(nostack),
|
||||
)
|
||||
};
|
||||
}
|
||||
#[cfg(portable_atomic_pre_llvm_20)]
|
||||
atomic_rmw_inst!(min, order);
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(portable_atomic_unstable_f16)]
|
||||
atomic_float!(AtomicF16, f16, "h", "7");
|
||||
atomic_float!(AtomicF32, f32, "s", "b");
|
||||
atomic_float!(AtomicF64, f64, "d", "f");
|
||||
|
||||
#[cfg(portable_atomic_unstable_f128)]
|
||||
impl AtomicF128 {
|
||||
#[inline]
|
||||
pub(crate) fn fetch_add(&self, val: f128, order: Ordering) -> f128 {
|
||||
self.fetch_update_(order, |x| x + val)
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_sub(&self, val: f128, order: Ordering) -> f128 {
|
||||
self.fetch_update_(order, |x| x - val)
|
||||
}
|
||||
#[inline]
|
||||
pub(super) fn fetch_update_<F>(&self, order: Ordering, mut f: F) -> f128
|
||||
where
|
||||
F: FnMut(f128) -> f128,
|
||||
{
|
||||
// This is a private function and all instances of `f` only operate on the value
|
||||
// loaded, so there is no need to synchronize the first load/failed CAS.
|
||||
let mut prev = self.load(Ordering::Relaxed);
|
||||
loop {
|
||||
let next = f(prev);
|
||||
match self.compare_exchange_weak(prev, next, order, Ordering::Relaxed) {
|
||||
Ok(x) => return x,
|
||||
Err(next_prev) => prev = next_prev,
|
||||
}
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_max(&self, val: f128, order: Ordering) -> f128 {
|
||||
self.fetch_update_(order, |x| x.max(val))
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_min(&self, val: f128, order: Ordering) -> f128 {
|
||||
self.fetch_update_(order, |x| x.min(val))
|
||||
}
|
||||
}
|
||||
+224
@@ -0,0 +1,224 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Atomic float implementation based on atomic integer.
|
||||
|
||||
This module provides atomic float implementations using atomic integer.
|
||||
|
||||
Note that most of `fetch_*` operations of atomic floats are implemented using
|
||||
CAS loops, which can be slower than equivalent operations of atomic integers.
|
||||
|
||||
AArch64 with FEAT_LSFE and GPU targets have atomic instructions for float.
|
||||
See aarch64.rs for AArch64 with FEAT_LSFE.
|
||||
GPU targets will also use architecture-specific implementations instead of this implementation in the
|
||||
future: https://github.com/taiki-e/portable-atomic/issues/34 / https://github.com/taiki-e/portable-atomic/pull/45
|
||||
*/
|
||||
|
||||
// TODO: fetch_{minimum,maximum}* https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2024/p3008r2.html / https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2024/p0493r5.pdf
|
||||
|
||||
#![cfg_attr(
|
||||
all(target_pointer_width = "16", not(feature = "fallback")),
|
||||
allow(unused_imports, unused_macros)
|
||||
)]
|
||||
|
||||
use core::{cell::UnsafeCell, sync::atomic::Ordering};
|
||||
|
||||
macro_rules! atomic_float {
|
||||
(
|
||||
$atomic_type:ident, $float_type:ident, $atomic_int_type:ident, $int_type:ident,
|
||||
$align:literal
|
||||
) => {
|
||||
#[repr(C, align($align))]
|
||||
pub(crate) struct $atomic_type {
|
||||
v: UnsafeCell<$float_type>,
|
||||
}
|
||||
|
||||
// Send is implicitly implemented.
|
||||
// SAFETY: any data races are prevented by atomic operations.
|
||||
unsafe impl Sync for $atomic_type {}
|
||||
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
pub(crate) const fn new(v: $float_type) -> Self {
|
||||
Self { v: UnsafeCell::new(v) }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn is_lock_free() -> bool {
|
||||
crate::$atomic_int_type::is_lock_free()
|
||||
}
|
||||
pub(crate) const IS_ALWAYS_LOCK_FREE: bool =
|
||||
crate::$atomic_int_type::is_always_lock_free();
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn load(&self, order: Ordering) -> $float_type {
|
||||
$float_type::from_bits(self.as_bits().load(order))
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn store(&self, val: $float_type, order: Ordering) {
|
||||
self.as_bits().store(val.to_bits(), order)
|
||||
}
|
||||
|
||||
const_fn! {
|
||||
const_if: #[cfg(not(portable_atomic_no_const_raw_ptr_deref))];
|
||||
#[inline(always)]
|
||||
pub(crate) const fn as_bits(&self) -> &crate::$atomic_int_type {
|
||||
// SAFETY: $atomic_type and $atomic_int_type have the same layout,
|
||||
// and there is no concurrent access to the value that does not go through this method.
|
||||
unsafe { &*(self as *const Self as *const crate::$atomic_int_type) }
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) const fn as_ptr(&self) -> *mut $float_type {
|
||||
self.v.get()
|
||||
}
|
||||
}
|
||||
|
||||
cfg_has_atomic_cas_or_amo32! {
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn swap(&self, val: $float_type, order: Ordering) -> $float_type {
|
||||
$float_type::from_bits(self.as_bits().swap(val.to_bits(), order))
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_neg(&self, order: Ordering) -> $float_type {
|
||||
const NEG_MASK: $int_type = !0 / 2 + 1;
|
||||
$float_type::from_bits(self.as_bits().fetch_xor(NEG_MASK, order))
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_abs(&self, order: Ordering) -> $float_type {
|
||||
const ABS_MASK: $int_type = !0 / 2;
|
||||
$float_type::from_bits(self.as_bits().fetch_and(ABS_MASK, order))
|
||||
}
|
||||
}
|
||||
cfg_has_atomic_cas! {
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn compare_exchange(
|
||||
&self,
|
||||
current: $float_type,
|
||||
new: $float_type,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<$float_type, $float_type> {
|
||||
match self.as_bits().compare_exchange(
|
||||
current.to_bits(),
|
||||
new.to_bits(),
|
||||
success,
|
||||
failure,
|
||||
) {
|
||||
Ok(v) => Ok($float_type::from_bits(v)),
|
||||
Err(v) => Err($float_type::from_bits(v)),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn compare_exchange_weak(
|
||||
&self,
|
||||
current: $float_type,
|
||||
new: $float_type,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<$float_type, $float_type> {
|
||||
match self.as_bits().compare_exchange_weak(
|
||||
current.to_bits(),
|
||||
new.to_bits(),
|
||||
success,
|
||||
failure,
|
||||
) {
|
||||
Ok(v) => Ok($float_type::from_bits(v)),
|
||||
Err(v) => Err($float_type::from_bits(v)),
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(not(all(
|
||||
any(target_arch = "aarch64", target_arch = "arm64ec"),
|
||||
any(target_feature = "lsfe", portable_atomic_target_feature = "lsfe"),
|
||||
target_feature = "neon", // for vreg
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
)))]
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_add(&self, val: $float_type, order: Ordering) -> $float_type {
|
||||
self.fetch_update_(order, |x| x + val)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_sub(&self, val: $float_type, order: Ordering) -> $float_type {
|
||||
self.fetch_update_(order, |x| x - val)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
fn fetch_update_<F>(&self, order: Ordering, mut f: F) -> $float_type
|
||||
where
|
||||
F: FnMut($float_type) -> $float_type,
|
||||
{
|
||||
// This is a private function and all instances of `f` only operate on the value
|
||||
// loaded, so there is no need to synchronize the first load/failed CAS.
|
||||
let mut prev = self.load(Ordering::Relaxed);
|
||||
loop {
|
||||
let next = f(prev);
|
||||
match self.compare_exchange_weak(prev, next, order, Ordering::Relaxed) {
|
||||
Ok(x) => return x,
|
||||
Err(next_prev) => prev = next_prev,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_max(&self, val: $float_type, order: Ordering) -> $float_type {
|
||||
self.fetch_update_(order, |x| x.max(val))
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
|
||||
pub(crate) fn fetch_min(&self, val: $float_type, order: Ordering) -> $float_type {
|
||||
self.fetch_update_(order, |x| x.min(val))
|
||||
}
|
||||
}
|
||||
} // cfg_has_atomic_cas!
|
||||
} // cfg_has_atomic_cas_or_amo32!
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(portable_atomic_unstable_f16)]
|
||||
cfg_has_atomic_16! {
|
||||
atomic_float!(AtomicF16, f16, AtomicU16, u16, 2);
|
||||
}
|
||||
cfg_has_atomic_32! {
|
||||
atomic_float!(AtomicF32, f32, AtomicU32, u32, 4);
|
||||
}
|
||||
cfg_has_atomic_64! {
|
||||
atomic_float!(AtomicF64, f64, AtomicU64, u64, 8);
|
||||
}
|
||||
#[cfg(portable_atomic_unstable_f128)]
|
||||
cfg_has_atomic_128! {
|
||||
atomic_float!(AtomicF128, f128, AtomicU128, u128, 16);
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Atomic float implementations
|
||||
*/
|
||||
|
||||
#![allow(clippy::float_arithmetic)]
|
||||
|
||||
mod int;
|
||||
|
||||
#[cfg(all(
|
||||
any(target_arch = "aarch64", target_arch = "arm64ec"),
|
||||
any(target_feature = "lsfe", portable_atomic_target_feature = "lsfe"),
|
||||
target_feature = "neon", // for vreg
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
))]
|
||||
mod aarch64;
|
||||
|
||||
#[cfg(portable_atomic_unstable_f16)]
|
||||
cfg_has_atomic_16! {
|
||||
pub(crate) use self::int::AtomicF16;
|
||||
}
|
||||
cfg_has_atomic_32! {
|
||||
pub(crate) use self::int::AtomicF32;
|
||||
}
|
||||
cfg_has_atomic_64! {
|
||||
pub(crate) use self::int::AtomicF64;
|
||||
}
|
||||
#[cfg(portable_atomic_unstable_f128)]
|
||||
cfg_has_atomic_128! {
|
||||
pub(crate) use self::int::AtomicF128;
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
# Fallback implementation based on disabling interrupts or critical-section
|
||||
|
||||
This module supports two different critical section implementations:
|
||||
|
||||
- Built-in "disable all interrupts".
|
||||
- On MSP430 and AVR, they are always single-core and has no unprivileged mode, so this is enabled by default.
|
||||
- On Armv6-M (thumbv6m), pre-v6 Arm (e.g., thumbv4t, thumbv5te), RISC-V without A-extension, and Xtensa, they could be multi-core or unprivileged mode, so this is enabled when the user explicitly declares that the system is single-core and that privileged instructions are available using `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg).
|
||||
- Call into the `critical-section` crate (which allows the user to plug any implementation).
|
||||
- This is enabled when the user asks for it with the [`critical-section` feature](../../../README.md#optional-features-critical-section).
|
||||
|
||||
`unsafe-assume-privileged` feature (`portable_atomic_unsafe_assume_privileged` cfg) also uses this module's interrupt disable implementation part.
|
||||
|
||||
The `unsafe-assume-single-core` implementation uses privileged instructions to disable interrupts, so it usually doesn't work on unprivileged mode.
|
||||
Enabling this feature in an environment where privileged instructions are not available, or if the instructions used are not sufficient to disable interrupts in the system, it is also usually considered **unsound**, although the details are system-dependent.
|
||||
|
||||
Consider using the [`critical-section` feature](../../../README.md#optional-features-critical-section) for systems that cannot use the `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg).
|
||||
|
||||
For some targets, the implementation can be changed by explicitly enabling features.
|
||||
|
||||
- On Arm M-Profile architectures, this disables interrupts by modifying the PRIMASK register.
|
||||
- On Arm (except for M-Profile architectures), this disables interrupts by modifying the I (IRQ mask) bit of the CPSR.
|
||||
- On Arm (except for M-Profile architectures) with the `disable-fiq` feature (or `portable_atomic_disable_fiq` cfg), this disables interrupts by modifying the I (IRQ mask) bit and F (FIQ mask) bit of the CPSR.
|
||||
- On RISC-V, this disables interrupts by modifying the MIE (Machine Interrupt Enable) bit of the `mstatus` register.
|
||||
- On RISC-V with the `s-mode` feature (or `portable_atomic_s_mode` cfg), this disables interrupts by modifying the SIE (Supervisor Interrupt Enable) bit of the `sstatus` register.
|
||||
- On RISC-V with the `zaamo` target feature (or `force-amo` feature or `portable_atomic_force_amo` cfg), this uses AMO instructions for RMWs that have corresponding AMO instructions even if A-extension is disabled. For other RMWs, this disables interrupts as usual.
|
||||
- On MSP430, this disables interrupts by modifying the GIE (Global Interrupt Enable) bit of the status register (SR).
|
||||
- On AVR, this disables interrupts by modifying the I (Global Interrupt Enable) bit of the status register (SREG).
|
||||
- On Xtensa, this disables interrupts by modifying the PS special register.
|
||||
|
||||
<a name="no-disable-interrupts"></a>Some operations don't require disabling interrupts:
|
||||
|
||||
- On architectures except for AVR: loads and stores with pointer size or smaller
|
||||
- On AVR: 8-bit loads and stores
|
||||
- On AVR with `rmw` target feature additionally: 8-bit `swap`
|
||||
- On MSP430 additionally: {8,16}-bit `add,sub,and,or,xor,not`
|
||||
- On RISC-V with the `zaamo` target feature (or `portable_atomic_target_feature="zaamo"` cfg or `force-amo` feature or `portable_atomic_force_amo` cfg) additionally: 32-bit(RV32)/{32,64}-bit(RV64) `swap,fetch_{add,sub,and,or,xor,not,max,min},add,sub,and,or,xor,not`, {8,16}-bit `fetch_{and,or,xor,not},and,or,xor,not`[^1], and all operations of `AtomicBool`
|
||||
|
||||
However, when the `critical-section` feature is enabled, critical sections are taken for all atomic operations.
|
||||
|
||||
Feel free to submit an issue if your target is not supported yet.
|
||||
|
||||
[^1]: With the `zabha` target feature, {8,16}-bit `swap,fetch_{add,sub,max,min},add,sub` too.
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Arm A-Profile Architectures, Arm R-Profile Architectures, Legacy Arm Architectures
|
||||
|
||||
Refs: https://developer.arm.com/documentation/ddi0406/cb/System-Level-Architecture/The-System-Level-Programmers--Model/ARM-processor-modes-and-ARM-core-registers/Program-Status-Registers--PSRs-
|
||||
|
||||
See tests/asm-test/asm/portable-atomic for generated assembly.
|
||||
*/
|
||||
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
use core::arch::asm;
|
||||
|
||||
// - 0x80 - I (IRQ mask) bit (1 << 7)
|
||||
// - 0x40 - F (FIQ mask) bit (1 << 6)
|
||||
// We disable only IRQs by default. See also https://github.com/taiki-e/portable-atomic/pull/28#issuecomment-1214146912.
|
||||
#[cfg(not(portable_atomic_disable_fiq))]
|
||||
macro_rules! mask {
|
||||
() => {
|
||||
"0x80"
|
||||
};
|
||||
}
|
||||
#[cfg(portable_atomic_disable_fiq)]
|
||||
macro_rules! mask {
|
||||
() => {
|
||||
"0xC0" // 0x80 | 0x40
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) type State = u32;
|
||||
|
||||
/// Disables interrupts and returns the previous interrupt state.
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
not(any(target_feature = "v7", portable_atomic_target_feature = "v7")),
|
||||
instruction_set(arm::a32)
|
||||
)]
|
||||
pub(crate) fn disable() -> State {
|
||||
let cpsr: State;
|
||||
// SAFETY: reading CPSR and disabling interrupts are safe.
|
||||
// (see module-level comments of interrupt/mod.rs on the safety of using privileged instructions)
|
||||
unsafe {
|
||||
asm!(
|
||||
"mrs {prev}, cpsr", // prev = CPSR
|
||||
concat!("orr {new}, {prev}, ", mask!()), // new = prev | mask
|
||||
"msr cpsr_c, {new}", // CPSR.{I,F,T,M} = new.{I,F,T,M}
|
||||
prev = out(reg) cpsr,
|
||||
new = out(reg) _,
|
||||
// Do not use `nomem` and `readonly` because prevent subsequent memory accesses from being reordered before interrupts are disabled.
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
}
|
||||
cpsr
|
||||
}
|
||||
|
||||
/// Restores the previous interrupt state.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// The state must be the one retrieved by the previous `disable`.
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
not(any(target_feature = "v7", portable_atomic_target_feature = "v7")),
|
||||
instruction_set(arm::a32)
|
||||
)]
|
||||
pub(crate) unsafe fn restore(prev_cpsr: State) {
|
||||
// SAFETY: the caller must guarantee that the state was retrieved by the previous `disable`,
|
||||
//
|
||||
// This clobbers the control field mask byte of CPSR. See msp430.rs for safety on this.
|
||||
// (preserves_flags is fine because we can clobber only the I, F, T, and M bits of CPSR.)
|
||||
//
|
||||
// Refs: https://developer.arm.com/documentation/dui0473/m/arm-and-thumb-instructions/msr--general-purpose-register-to-psr-
|
||||
unsafe {
|
||||
asm!(
|
||||
"msr cpsr_c, {prev_cpsr}", // CPSR.{I,F,T,M} = prev_cpsr.{I,F,T,M}
|
||||
prev_cpsr = in(reg) prev_cpsr,
|
||||
// Do not use `nomem` and `readonly` because prevent preceding memory accesses from being reordered after interrupts are enabled.
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// On pre-v6 Arm, we cannot use core::sync::atomic here because they call the
|
||||
// `__sync_*` builtins for non-relaxed load/store (because pre-v6 Arm doesn't
|
||||
// have Data Memory Barrier).
|
||||
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(any(test, portable_atomic_no_atomic_cas)))]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(test, not(target_has_atomic = "ptr")))
|
||||
)]
|
||||
pub(super) mod atomic {
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
use core::arch::asm;
|
||||
use core::{cell::UnsafeCell, sync::atomic::Ordering};
|
||||
|
||||
macro_rules! atomic {
|
||||
($([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $suffix:tt) => {
|
||||
#[repr(transparent)]
|
||||
pub(crate) struct $atomic_type $(<$($generics)*>)? {
|
||||
v: UnsafeCell<$value_type>,
|
||||
}
|
||||
|
||||
// Send is implicitly implemented for atomic integers, but not for atomic pointers.
|
||||
// SAFETY: any data races are prevented by atomic operations.
|
||||
unsafe impl $(<$($generics)*>)? Send for $atomic_type $(<$($generics)*>)? {}
|
||||
// SAFETY: any data races are prevented by atomic operations.
|
||||
unsafe impl $(<$($generics)*>)? Sync for $atomic_type $(<$($generics)*>)? {}
|
||||
|
||||
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
|
||||
#[inline]
|
||||
pub(crate) fn load(&self, _order: Ordering) -> $value_type {
|
||||
let src = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
let out;
|
||||
// inline asm without nomem/readonly implies compiler fence.
|
||||
// And compiler fence is fine because the user explicitly declares that
|
||||
// the system is single-core by using an unsafe cfg.
|
||||
asm!(
|
||||
concat!("ldr", $suffix, " {out}, [{src}]"), // atomic { out = *src }
|
||||
src = in(reg) src,
|
||||
out = lateout(reg) out,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn store(&self, val: $value_type, _order: Ordering) {
|
||||
let dst = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
// inline asm without nomem/readonly implies compiler fence.
|
||||
// And compiler fence is fine because the user explicitly declares that
|
||||
// the system is single-core by using an unsafe cfg.
|
||||
asm!(
|
||||
concat!("str", $suffix, " {val}, [{dst}]"), // atomic { *dst = val }
|
||||
dst = in(reg) dst,
|
||||
val = in(reg) val,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
atomic!(AtomicI8, i8, "b");
|
||||
atomic!(AtomicU8, u8, "b");
|
||||
atomic!(AtomicI16, i16, "h");
|
||||
atomic!(AtomicU16, u16, "h");
|
||||
atomic!(AtomicI32, i32, "");
|
||||
atomic!(AtomicU32, u32, "");
|
||||
atomic!(AtomicIsize, isize, "");
|
||||
atomic!(AtomicUsize, usize, "");
|
||||
atomic!([T] AtomicPtr, *mut T, "");
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Arm M-Profile Architectures
|
||||
|
||||
Adapted from https://github.com/rust-embedded/cortex-m.
|
||||
|
||||
Refs: https://developer.arm.com/documentation/ddi0419/c/System-Level-Architecture/System-Level-Programmers--Model/Registers/The-special-purpose-mask-register--PRIMASK
|
||||
|
||||
See tests/asm-test/asm/portable-atomic for generated assembly.
|
||||
*/
|
||||
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
use core::arch::asm;
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(any(test, portable_atomic_no_atomic_cas))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(test, not(target_has_atomic = "ptr")))
|
||||
)]
|
||||
pub(super) use core::sync::atomic;
|
||||
|
||||
pub(crate) type State = u32;
|
||||
|
||||
/// Disables interrupts and returns the previous interrupt state.
|
||||
#[inline(always)]
|
||||
pub(crate) fn disable() -> State {
|
||||
let primask: State;
|
||||
// SAFETY: reading the priority mask register and disabling interrupts are safe.
|
||||
// (see module-level comments of interrupt/mod.rs on the safety of using privileged instructions)
|
||||
unsafe {
|
||||
asm!(
|
||||
"mrs {primask}, PRIMASK", // primask = PRIMASK
|
||||
"cpsid i", // PRIMASK.PM = 1
|
||||
primask = out(reg) primask,
|
||||
// Do not use `nomem` and `readonly` because prevent subsequent memory accesses from being reordered before interrupts are disabled.
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
}
|
||||
primask
|
||||
}
|
||||
|
||||
/// Restores the previous interrupt state.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// The state must be the one retrieved by the previous `disable`.
|
||||
#[inline(always)]
|
||||
pub(crate) unsafe fn restore(prev_primask: State) {
|
||||
// SAFETY: the caller must guarantee that the state was retrieved by the previous `disable`,
|
||||
// and we've checked that interrupts were enabled before disabling interrupts.
|
||||
//
|
||||
// This clobbers the entire PRIMASK register. See msp430.rs for safety on this.
|
||||
unsafe {
|
||||
asm!(
|
||||
"msr PRIMASK, {prev_primask}", // PRIMASK = prev_primask
|
||||
prev_primask = in(reg) prev_primask,
|
||||
// Do not use `nomem` and `readonly` because prevent preceding memory accesses from being reordered after interrupts are enabled.
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Adapted from https://github.com/Rahix/avr-device.
|
||||
|
||||
See also src/imp/avr.rs.
|
||||
|
||||
Refs:
|
||||
- AVR® Instruction Set Manual, Rev. DS40002198B
|
||||
https://ww1.microchip.com/downloads/en/DeviceDoc/AVR-InstructionSet-Manual-DS40002198.pdf
|
||||
|
||||
See tests/asm-test/asm/portable-atomic for generated assembly.
|
||||
*/
|
||||
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
use core::arch::asm;
|
||||
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(any(test, portable_atomic_no_atomic_cas))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(test, not(target_has_atomic = "ptr")))
|
||||
)]
|
||||
pub(super) use super::super::avr as atomic;
|
||||
|
||||
pub(crate) type State = u8;
|
||||
|
||||
/// Disables interrupts and returns the previous interrupt state.
|
||||
#[inline(always)]
|
||||
pub(crate) fn disable() -> State {
|
||||
let sreg: State;
|
||||
// SAFETY: reading the status register (SREG) and disabling interrupts are safe.
|
||||
// (see module-level comments of interrupt/mod.rs on the safety of using privileged instructions)
|
||||
unsafe {
|
||||
// Refs: https://ww1.microchip.com/downloads/en/DeviceDoc/AVR-InstructionSet-Manual-DS40002198.pdf#page=58
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
asm!(
|
||||
"in {sreg}, 0x3F", // sreg = SREG
|
||||
"cli", // SREG.I = 0
|
||||
sreg = out(reg) sreg,
|
||||
// Do not use `nomem` and `readonly` because prevent subsequent memory accesses from being reordered before interrupts are disabled.
|
||||
// Do not use `preserves_flags` because CLI modifies the I bit of the status register (SREG).
|
||||
options(nostack),
|
||||
);
|
||||
#[cfg(portable_atomic_no_asm)]
|
||||
{
|
||||
llvm_asm!("in $0, 0x3F" : "=r"(sreg) ::: "volatile");
|
||||
llvm_asm!("cli" ::: "memory" : "volatile");
|
||||
}
|
||||
}
|
||||
sreg
|
||||
}
|
||||
|
||||
/// Restores the previous interrupt state.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// The state must be the one retrieved by the previous `disable`.
|
||||
#[inline(always)]
|
||||
pub(crate) unsafe fn restore(prev_sreg: State) {
|
||||
// SAFETY: the caller must guarantee that the state was retrieved by the previous `disable`.
|
||||
//
|
||||
// This clobbers the entire status register, but all other flags are the arithmetic flags that
|
||||
// are changed as a side effect of arithmetic operations, etc., which LLVM recognizes unless
|
||||
// preserves_flags is set, so it is safe to clobber them here.
|
||||
// See also the discussion at https://github.com/taiki-e/portable-atomic/pull/40.
|
||||
unsafe {
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
asm!(
|
||||
"out 0x3F, {prev_sreg}", // SREG = prev_sreg
|
||||
prev_sreg = in(reg) prev_sreg,
|
||||
// Do not use `nomem` and `readonly` because prevent preceding memory accesses from being reordered after interrupts are enabled.
|
||||
// Do not use `preserves_flags` because OUT modifies the status register (SREG).
|
||||
options(nostack),
|
||||
);
|
||||
#[cfg(portable_atomic_no_asm)]
|
||||
llvm_asm!("out 0x3F, $0" :: "r"(prev_sreg) : "memory" : "volatile");
|
||||
}
|
||||
}
|
||||
+696
@@ -0,0 +1,696 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Fallback implementation based on disabling interrupts or critical-section
|
||||
|
||||
- mod.rs contains critical section based fallback implementations.
|
||||
- Each architecture modules contain implementations of disabling interrupts.
|
||||
|
||||
See README.md of this directory for details.
|
||||
*/
|
||||
|
||||
#[cfg(not(feature = "critical-section"))]
|
||||
#[cfg_attr(
|
||||
all(
|
||||
target_arch = "arm",
|
||||
any(target_feature = "mclass", portable_atomic_target_feature = "mclass"),
|
||||
),
|
||||
path = "armv6m.rs"
|
||||
)]
|
||||
#[cfg_attr(
|
||||
all(
|
||||
target_arch = "arm",
|
||||
not(any(target_feature = "mclass", portable_atomic_target_feature = "mclass")),
|
||||
),
|
||||
path = "armv4t.rs"
|
||||
)]
|
||||
#[cfg_attr(target_arch = "avr", path = "avr.rs")]
|
||||
#[cfg_attr(target_arch = "msp430", path = "msp430.rs")]
|
||||
#[cfg_attr(any(target_arch = "riscv32", target_arch = "riscv64"), path = "riscv.rs")]
|
||||
#[cfg_attr(target_arch = "xtensa", path = "xtensa.rs")]
|
||||
pub(super) mod arch;
|
||||
|
||||
#[cfg(not(all(
|
||||
portable_atomic_unsafe_assume_privileged,
|
||||
not(portable_atomic_unsafe_assume_single_core),
|
||||
)))]
|
||||
items!({
|
||||
use core::{cell::UnsafeCell, sync::atomic::Ordering};
|
||||
|
||||
// critical-section implementations might use locks internally.
|
||||
#[cfg(feature = "critical-section")]
|
||||
const IS_ALWAYS_LOCK_FREE: bool = false;
|
||||
// Consider atomic operations based on disabling interrupts on single-core
|
||||
// systems are lock-free. (We consider the pre-v6 Arm Linux's atomic operations
|
||||
// provided in a similar way by the Linux kernel to be lock-free.)
|
||||
#[cfg(not(feature = "critical-section"))]
|
||||
const IS_ALWAYS_LOCK_FREE: bool = true;
|
||||
|
||||
// Put this in its own module to prevent guard creation.
|
||||
use self::guard::disable;
|
||||
mod guard {
|
||||
// Note: The caller must NOT explicitly modify registers containing fields modified by disable/restore.
|
||||
// (Fields modified as side effects of other operations are covered by the absence of preserves_flags,
|
||||
// so they are fine -- see msp430.rs for more.)
|
||||
#[inline(always)]
|
||||
pub(super) fn disable() -> Guard {
|
||||
Guard {
|
||||
#[cfg(feature = "critical-section")]
|
||||
// SAFETY: the state will be restored in the subsequent `release`.
|
||||
state: unsafe { critical_section::acquire() },
|
||||
#[cfg(not(feature = "critical-section"))]
|
||||
// Get current interrupt state and disable interrupts.
|
||||
state: super::arch::disable(),
|
||||
}
|
||||
}
|
||||
pub(super) struct Guard {
|
||||
#[cfg(feature = "critical-section")]
|
||||
state: critical_section::RestoreState,
|
||||
#[cfg(not(feature = "critical-section"))]
|
||||
state: super::arch::State,
|
||||
}
|
||||
impl Drop for Guard {
|
||||
#[inline(always)]
|
||||
fn drop(&mut self) {
|
||||
#[cfg(feature = "critical-section")]
|
||||
// SAFETY: the state was retrieved by the previous `acquire`.
|
||||
unsafe {
|
||||
critical_section::release(self.state);
|
||||
}
|
||||
#[cfg(not(feature = "critical-section"))]
|
||||
// Restore interrupt state.
|
||||
// SAFETY: the state was retrieved by the previous `disable`.
|
||||
unsafe {
|
||||
super::arch::restore(self.state);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! atomic_base {
|
||||
(base, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $align:literal) => {
|
||||
#[repr(C, align($align))]
|
||||
pub(crate) struct $atomic_type $(<$($generics)*>)? {
|
||||
v: UnsafeCell<$value_type>,
|
||||
}
|
||||
|
||||
// Send is implicitly implemented for atomic integers, but not for atomic pointers.
|
||||
// SAFETY: any data races are prevented by disabling interrupts (or
|
||||
// atomic intrinsics) or critical-section (see module-level comments).
|
||||
unsafe impl $(<$($generics)*>)? Send for $atomic_type $(<$($generics)*>)? {}
|
||||
// SAFETY: any data races are prevented by disabling interrupts (or
|
||||
// atomic intrinsics) or critical-section (see module-level comments).
|
||||
unsafe impl $(<$($generics)*>)? Sync for $atomic_type $(<$($generics)*>)? {}
|
||||
|
||||
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
|
||||
#[inline]
|
||||
pub(crate) const fn new(v: $value_type) -> Self {
|
||||
Self { v: UnsafeCell::new(v) }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn is_lock_free() -> bool {
|
||||
Self::IS_ALWAYS_LOCK_FREE
|
||||
}
|
||||
pub(crate) const IS_ALWAYS_LOCK_FREE: bool = IS_ALWAYS_LOCK_FREE;
|
||||
|
||||
#[inline]
|
||||
fn read(&self, _guard: &guard::Guard) -> $value_type {
|
||||
// SAFETY: any data races are prevented by disabling interrupts or critical-section (see
|
||||
// module-level comments) and the raw pointer is valid because we got it
|
||||
// from a reference.
|
||||
unsafe { self.v.get().read() }
|
||||
}
|
||||
#[inline]
|
||||
fn write(&self, val: $value_type, _guard: &guard::Guard) {
|
||||
// SAFETY: any data races are prevented by disabling interrupts or critical-section (see
|
||||
// module-level comments) and the raw pointer is valid because we got it
|
||||
// from a reference.
|
||||
unsafe { self.v.get().write(val) }
|
||||
}
|
||||
|
||||
// As for CAS, there is no corresponding atomic operation on all architectures that use this code.
|
||||
// (If the CAS instruction exists, all atomic operations can be implemented by it, so this code will not be used.)
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
all(debug_assertions, not(portable_atomic_no_track_caller)),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn compare_exchange(
|
||||
&self,
|
||||
current: $value_type,
|
||||
new: $value_type,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<$value_type, $value_type> {
|
||||
crate::utils::assert_compare_exchange_ordering(success, failure);
|
||||
let guard = disable();
|
||||
let prev = self.read(&guard);
|
||||
if prev == current {
|
||||
self.write(new, &guard);
|
||||
Ok(prev)
|
||||
} else {
|
||||
Err(prev)
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
all(debug_assertions, not(portable_atomic_no_track_caller)),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn compare_exchange_weak(
|
||||
&self,
|
||||
current: $value_type,
|
||||
new: $value_type,
|
||||
success: Ordering,
|
||||
failure: Ordering,
|
||||
) -> Result<$value_type, $value_type> {
|
||||
self.compare_exchange(current, new, success, failure)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) const fn as_ptr(&self) -> *mut $value_type {
|
||||
self.v.get()
|
||||
}
|
||||
}
|
||||
};
|
||||
(native_load_store, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty) => {
|
||||
impl $(<$($generics)*>)? core::ops::Deref for $atomic_type $(<$($generics)*>)? {
|
||||
type Target = atomic::$atomic_type $(<$($generics)*>)?;
|
||||
#[inline(always)]
|
||||
fn deref(&self) -> &Self::Target {
|
||||
// SAFETY: $atomic_type and atomic::$atomic_type have the same layout and
|
||||
// guarantee atomicity in a compatible way. (see module-level comments)
|
||||
unsafe {
|
||||
&*(self as *const Self as *const atomic::$atomic_type $(<$($generics)*>)?)
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
(emulate_load_store, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty) => {
|
||||
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
all(debug_assertions, not(portable_atomic_no_track_caller)),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn load(&self, order: Ordering) -> $value_type {
|
||||
crate::utils::assert_load_ordering(order);
|
||||
let guard = disable();
|
||||
self.read(&guard)
|
||||
}
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
all(debug_assertions, not(portable_atomic_no_track_caller)),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn store(&self, val: $value_type, order: Ordering) {
|
||||
crate::utils::assert_store_ordering(order);
|
||||
let guard = disable();
|
||||
self.write(val, &guard);
|
||||
}
|
||||
}
|
||||
};
|
||||
(emulate_swap, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty) => {
|
||||
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
|
||||
#[inline]
|
||||
pub(crate) fn swap(&self, val: $value_type, _order: Ordering) -> $value_type {
|
||||
let guard = disable();
|
||||
let prev = self.read(&guard);
|
||||
self.write(val, &guard);
|
||||
prev
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(any(
|
||||
test,
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
portable_atomic_no_atomic_cas
|
||||
))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(
|
||||
test,
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
not(target_has_atomic = "ptr")
|
||||
))
|
||||
)]
|
||||
items!({
|
||||
#[cfg(target_pointer_width = "16")]
|
||||
atomic_base!(base, [T] AtomicPtr, *mut T, 2);
|
||||
#[cfg(target_pointer_width = "32")]
|
||||
atomic_base!(base, [T] AtomicPtr, *mut T, 4);
|
||||
#[cfg(target_pointer_width = "64")]
|
||||
atomic_base!(base, [T] AtomicPtr, *mut T, 8);
|
||||
#[cfg(target_pointer_width = "128")]
|
||||
atomic_base!(base, [T] AtomicPtr, *mut T, 16);
|
||||
|
||||
impl_default_bit_opts!(AtomicPtr, usize);
|
||||
|
||||
cfg_sel!({
|
||||
#[cfg(any(target_arch = "avr", feature = "critical-section"))]
|
||||
{
|
||||
atomic_base!(emulate_load_store, [T] AtomicPtr, *mut T);
|
||||
}
|
||||
#[cfg(else)]
|
||||
{
|
||||
atomic_base!(native_load_store, [T] AtomicPtr, *mut T);
|
||||
}
|
||||
});
|
||||
|
||||
#[cfg(not(all(
|
||||
any(target_arch = "riscv32", target_arch = "riscv64"),
|
||||
not(feature = "critical-section"),
|
||||
any(
|
||||
portable_atomic_force_amo,
|
||||
target_feature = "zaamo",
|
||||
portable_atomic_target_feature = "zaamo",
|
||||
),
|
||||
)))]
|
||||
items!({
|
||||
atomic_base!(emulate_swap, [T] AtomicPtr, *mut T);
|
||||
impl<T> AtomicPtr<T> {
|
||||
#[inline]
|
||||
pub(crate) fn fetch_byte_add(&self, val: usize, _order: Ordering) -> *mut T {
|
||||
#[cfg(portable_atomic_no_strict_provenance)]
|
||||
use crate::utils::ptr::PtrExt as _;
|
||||
let guard = disable();
|
||||
let prev = self.read(&guard);
|
||||
self.write(prev.with_addr(prev.addr().wrapping_add(val)), &guard);
|
||||
prev
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_byte_sub(&self, val: usize, _order: Ordering) -> *mut T {
|
||||
#[cfg(portable_atomic_no_strict_provenance)]
|
||||
use crate::utils::ptr::PtrExt as _;
|
||||
let guard = disable();
|
||||
let prev = self.read(&guard);
|
||||
self.write(prev.with_addr(prev.addr().wrapping_sub(val)), &guard);
|
||||
prev
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_and(&self, val: usize, _order: Ordering) -> *mut T {
|
||||
#[cfg(portable_atomic_no_strict_provenance)]
|
||||
use crate::utils::ptr::PtrExt as _;
|
||||
let guard = disable();
|
||||
let prev = self.read(&guard);
|
||||
self.write(prev.with_addr(prev.addr() & val), &guard);
|
||||
prev
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_or(&self, val: usize, _order: Ordering) -> *mut T {
|
||||
#[cfg(portable_atomic_no_strict_provenance)]
|
||||
use crate::utils::ptr::PtrExt as _;
|
||||
let guard = disable();
|
||||
let prev = self.read(&guard);
|
||||
self.write(prev.with_addr(prev.addr() | val), &guard);
|
||||
prev
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_xor(&self, val: usize, _order: Ordering) -> *mut T {
|
||||
#[cfg(portable_atomic_no_strict_provenance)]
|
||||
use crate::utils::ptr::PtrExt as _;
|
||||
let guard = disable();
|
||||
let prev = self.read(&guard);
|
||||
self.write(prev.with_addr(prev.addr() ^ val), &guard);
|
||||
prev
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
impl<T> AtomicPtr<T> {
|
||||
#[cfg(test)]
|
||||
#[inline]
|
||||
fn fetch_ptr_add(&self, val: usize, order: Ordering) -> *mut T {
|
||||
self.fetch_byte_add(val.wrapping_mul(core::mem::size_of::<T>()), order)
|
||||
}
|
||||
#[cfg(test)]
|
||||
#[inline]
|
||||
fn fetch_ptr_sub(&self, val: usize, order: Ordering) -> *mut T {
|
||||
self.fetch_byte_sub(val.wrapping_mul(core::mem::size_of::<T>()), order)
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
macro_rules! atomic_int {
|
||||
(base, $atomic_type:ident, $int_type:ty, $align:literal) => {
|
||||
atomic_base!(base, $atomic_type, $int_type, $align);
|
||||
// As for nand and neg, there is no corresponding atomic operation on all architectures that use this code.
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
pub(crate) fn fetch_nand(&self, val: $int_type, _order: Ordering) -> $int_type {
|
||||
let guard = disable();
|
||||
let prev = self.read(&guard);
|
||||
self.write(!(prev & val), &guard);
|
||||
prev
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_neg(&self, _order: Ordering) -> $int_type {
|
||||
let guard = disable();
|
||||
let prev = self.read(&guard);
|
||||
self.write(prev.wrapping_neg(), &guard);
|
||||
prev
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn neg(&self, order: Ordering) {
|
||||
self.fetch_neg(order);
|
||||
}
|
||||
}
|
||||
};
|
||||
(load_store_atomic $([$kind:ident])?, $atomic_type:ident, $int_type:ty, $align:literal) => {
|
||||
cfg_sel!({
|
||||
#[cfg(feature = "critical-section")]
|
||||
{
|
||||
atomic_int!(all_critical_session, $atomic_type, $int_type, $align);
|
||||
}
|
||||
#[cfg(else)]
|
||||
{
|
||||
atomic_int!(base, $atomic_type, $int_type, $align);
|
||||
impl_default_bit_opts!($atomic_type, $int_type);
|
||||
// load/store
|
||||
cfg_sel!({
|
||||
// AVR with very old rustc
|
||||
#[cfg(all(target_arch = "avr", portable_atomic_no_asm))]
|
||||
{
|
||||
atomic_base!(emulate_load_store, $atomic_type, $int_type);
|
||||
}
|
||||
#[cfg(else)]
|
||||
{
|
||||
atomic_base!(native_load_store, $atomic_type, $int_type);
|
||||
}
|
||||
});
|
||||
// RMW
|
||||
cfg_sel!({
|
||||
// AVR 8-bit RMW with RMW instructions
|
||||
#[cfg(all(
|
||||
target_arch = "avr",
|
||||
not(portable_atomic_no_asm),
|
||||
any(target_feature = "rmw", portable_atomic_target_feature = "rmw"),
|
||||
))]
|
||||
{
|
||||
atomic_int!(emulate_arithmetic, $atomic_type, $int_type);
|
||||
}
|
||||
// RISC-V RMW with Zaamo extension
|
||||
#[cfg(all(
|
||||
any(target_arch = "riscv32", target_arch = "riscv64"),
|
||||
any(
|
||||
portable_atomic_force_amo,
|
||||
target_feature = "zaamo",
|
||||
portable_atomic_target_feature = "zaamo",
|
||||
),
|
||||
))]
|
||||
{
|
||||
atomic_int!(cas $([$kind])?, $atomic_type, $int_type);
|
||||
}
|
||||
#[cfg(else)]
|
||||
{
|
||||
atomic_int!(cas[emulate], $atomic_type, $int_type);
|
||||
}
|
||||
});
|
||||
// RMW (no-fetch)
|
||||
#[cfg(not(target_arch = "msp430"))]
|
||||
items!({
|
||||
impl_default_no_fetch_ops!($atomic_type, $int_type);
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
pub(crate) fn not(&self, order: Ordering) {
|
||||
self.fetch_not(order);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
};
|
||||
(all_critical_session, $atomic_type:ident, $int_type:ty, $align:literal) => {
|
||||
atomic_int!(base, $atomic_type, $int_type, $align);
|
||||
atomic_base!(emulate_load_store, $atomic_type, $int_type);
|
||||
atomic_int!(cas[emulate], $atomic_type, $int_type);
|
||||
impl_default_no_fetch_ops!($atomic_type, $int_type);
|
||||
impl_default_bit_opts!($atomic_type, $int_type);
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
pub(crate) fn not(&self, order: Ordering) {
|
||||
self.fetch_not(order);
|
||||
}
|
||||
}
|
||||
};
|
||||
(emulate_arithmetic, $atomic_type:ident, $int_type:ty) => {
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
pub(crate) fn fetch_add(&self, val: $int_type, _order: Ordering) -> $int_type {
|
||||
let guard = disable();
|
||||
let prev = self.read(&guard);
|
||||
self.write(prev.wrapping_add(val), &guard);
|
||||
prev
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_sub(&self, val: $int_type, _order: Ordering) -> $int_type {
|
||||
let guard = disable();
|
||||
let prev = self.read(&guard);
|
||||
self.write(prev.wrapping_sub(val), &guard);
|
||||
prev
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_max(&self, val: $int_type, _order: Ordering) -> $int_type {
|
||||
let guard = disable();
|
||||
let prev = self.read(&guard);
|
||||
self.write(core::cmp::max(prev, val), &guard);
|
||||
prev
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_min(&self, val: $int_type, _order: Ordering) -> $int_type {
|
||||
let guard = disable();
|
||||
let prev = self.read(&guard);
|
||||
self.write(core::cmp::min(prev, val), &guard);
|
||||
prev
|
||||
}
|
||||
}
|
||||
};
|
||||
(emulate_bit, $atomic_type:ident, $int_type:ty) => {
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
pub(crate) fn fetch_and(&self, val: $int_type, _order: Ordering) -> $int_type {
|
||||
let guard = disable();
|
||||
let prev = self.read(&guard);
|
||||
self.write(prev & val, &guard);
|
||||
prev
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_or(&self, val: $int_type, _order: Ordering) -> $int_type {
|
||||
let guard = disable();
|
||||
let prev = self.read(&guard);
|
||||
self.write(prev | val, &guard);
|
||||
prev
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_xor(&self, val: $int_type, _order: Ordering) -> $int_type {
|
||||
let guard = disable();
|
||||
let prev = self.read(&guard);
|
||||
self.write(prev ^ val, &guard);
|
||||
prev
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_not(&self, _order: Ordering) -> $int_type {
|
||||
let guard = disable();
|
||||
let prev = self.read(&guard);
|
||||
self.write(!prev, &guard);
|
||||
prev
|
||||
}
|
||||
}
|
||||
};
|
||||
(cas[emulate], $atomic_type:ident, $int_type:ty) => {
|
||||
atomic_base!(emulate_swap, $atomic_type, $int_type);
|
||||
atomic_int!(emulate_arithmetic, $atomic_type, $int_type);
|
||||
atomic_int!(emulate_bit, $atomic_type, $int_type);
|
||||
};
|
||||
// RISC-V 32-bit(RV32)/{32,64}-bit(RV64) RMW with Zaamo extension
|
||||
// RISC-V 8-bit/16-bit RMW with Zabha extension
|
||||
(cas, $atomic_type:ident, $int_type:ty) => {};
|
||||
// RISC-V 8-bit/16-bit RMW with Zaamo extension
|
||||
(cas[sub_word], $atomic_type:ident, $int_type:ty) => {
|
||||
// RISC-V 8-bit/16-bit RMW with Zaamo+Zabha extension
|
||||
#[cfg(any(target_feature = "zabha", portable_atomic_target_feature = "zabha"))]
|
||||
atomic_int!(cas, $atomic_type, $int_type);
|
||||
|
||||
// RISC-V 8-bit/16-bit RMW with Zaamo extension
|
||||
#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
|
||||
atomic_base!(emulate_swap, $atomic_type, $int_type);
|
||||
#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
|
||||
atomic_int!(emulate_arithmetic, $atomic_type, $int_type);
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(any(
|
||||
test,
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
portable_atomic_no_atomic_cas,
|
||||
))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(
|
||||
test,
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
not(target_has_atomic = "ptr"),
|
||||
))
|
||||
)]
|
||||
items!({
|
||||
// On some platforms, atomic load/store can be implemented in a more efficient
|
||||
// way than disabling interrupts. On MSP430, some RMWs that do not return the
|
||||
// previous value can also be optimized.
|
||||
//
|
||||
// Note: On single-core systems, it is okay to use critical session-based
|
||||
// CAS together with atomic load/store. The load/store will not be
|
||||
// called while interrupts are disabled, and since the load/store is
|
||||
// atomic, it is not affected by interrupts even if interrupts are enabled.
|
||||
#[cfg(not(any(
|
||||
all(target_arch = "avr", portable_atomic_no_asm),
|
||||
feature = "critical-section",
|
||||
)))]
|
||||
use self::arch::atomic;
|
||||
|
||||
#[cfg(target_pointer_width = "16")]
|
||||
#[cfg(not(target_arch = "avr"))]
|
||||
atomic_int!(load_store_atomic, AtomicIsize, isize, 2);
|
||||
#[cfg(target_pointer_width = "16")]
|
||||
#[cfg(not(target_arch = "avr"))]
|
||||
atomic_int!(load_store_atomic, AtomicUsize, usize, 2);
|
||||
#[cfg(target_arch = "avr")]
|
||||
atomic_int!(all_critical_session, AtomicIsize, isize, 2);
|
||||
#[cfg(target_arch = "avr")]
|
||||
atomic_int!(all_critical_session, AtomicUsize, usize, 2);
|
||||
#[cfg(target_pointer_width = "32")]
|
||||
atomic_int!(load_store_atomic, AtomicIsize, isize, 4);
|
||||
#[cfg(target_pointer_width = "32")]
|
||||
atomic_int!(load_store_atomic, AtomicUsize, usize, 4);
|
||||
#[cfg(target_pointer_width = "64")]
|
||||
atomic_int!(load_store_atomic, AtomicIsize, isize, 8);
|
||||
#[cfg(target_pointer_width = "64")]
|
||||
atomic_int!(load_store_atomic, AtomicUsize, usize, 8);
|
||||
#[cfg(target_pointer_width = "128")]
|
||||
atomic_int!(load_store_atomic, AtomicIsize, isize, 16);
|
||||
#[cfg(target_pointer_width = "128")]
|
||||
atomic_int!(load_store_atomic, AtomicUsize, usize, 16);
|
||||
|
||||
#[cfg(not(all(target_arch = "avr", portable_atomic_no_asm)))]
|
||||
atomic_int!(load_store_atomic[sub_word], AtomicI8, i8, 1);
|
||||
#[cfg(not(all(target_arch = "avr", portable_atomic_no_asm)))]
|
||||
atomic_int!(load_store_atomic[sub_word], AtomicU8, u8, 1);
|
||||
#[cfg(all(target_arch = "avr", portable_atomic_no_asm))]
|
||||
atomic_int!(all_critical_session, AtomicI8, i8, 1);
|
||||
#[cfg(all(target_arch = "avr", portable_atomic_no_asm))]
|
||||
atomic_int!(all_critical_session, AtomicU8, u8, 1);
|
||||
#[cfg(not(target_arch = "avr"))]
|
||||
atomic_int!(load_store_atomic[sub_word], AtomicI16, i16, 2);
|
||||
#[cfg(not(target_arch = "avr"))]
|
||||
atomic_int!(load_store_atomic[sub_word], AtomicU16, u16, 2);
|
||||
#[cfg(target_arch = "avr")]
|
||||
atomic_int!(all_critical_session, AtomicI16, i16, 2);
|
||||
#[cfg(target_arch = "avr")]
|
||||
atomic_int!(all_critical_session, AtomicU16, u16, 2);
|
||||
|
||||
#[cfg(not(target_pointer_width = "16"))]
|
||||
atomic_int!(load_store_atomic, AtomicI32, i32, 4);
|
||||
#[cfg(not(target_pointer_width = "16"))]
|
||||
atomic_int!(load_store_atomic, AtomicU32, u32, 4);
|
||||
|
||||
cfg_has_fast_atomic_64!({
|
||||
atomic_int!(load_store_atomic, AtomicI64, i64, 8);
|
||||
atomic_int!(load_store_atomic, AtomicU64, u64, 8);
|
||||
});
|
||||
});
|
||||
|
||||
// Double or more width atomics (require fallback feature for consistency with other situations).
|
||||
#[cfg(target_pointer_width = "16")]
|
||||
#[cfg(any(test, feature = "fallback"))]
|
||||
items!({
|
||||
atomic_int!(all_critical_session, AtomicI32, i32, 4);
|
||||
atomic_int!(all_critical_session, AtomicU32, u32, 4);
|
||||
});
|
||||
#[cfg(any(
|
||||
test,
|
||||
all(
|
||||
feature = "fallback",
|
||||
not(all(
|
||||
target_arch = "riscv32",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(
|
||||
target_feature = "zacas",
|
||||
portable_atomic_target_feature = "zacas",
|
||||
all(
|
||||
feature = "fallback",
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
any(target_os = "linux", target_os = "android"),
|
||||
),
|
||||
),
|
||||
)),
|
||||
),
|
||||
))]
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(any(test, portable_atomic_no_atomic_64))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(test, not(target_has_atomic = "64")))
|
||||
)]
|
||||
cfg_no_fast_atomic_64!({
|
||||
atomic_int!(all_critical_session, AtomicI64, i64, 8);
|
||||
atomic_int!(all_critical_session, AtomicU64, u64, 8);
|
||||
});
|
||||
#[cfg(any(
|
||||
test,
|
||||
all(
|
||||
feature = "fallback",
|
||||
not(all(
|
||||
target_arch = "riscv64",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(
|
||||
target_feature = "zacas",
|
||||
portable_atomic_target_feature = "zacas",
|
||||
all(
|
||||
feature = "fallback",
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
any(target_os = "linux", target_os = "android"),
|
||||
),
|
||||
),
|
||||
)),
|
||||
),
|
||||
))]
|
||||
items!({
|
||||
atomic_int!(all_critical_session, AtomicI128, i128, 16);
|
||||
atomic_int!(all_critical_session, AtomicU128, u128, 16);
|
||||
});
|
||||
});
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
test_atomic_ptr_single_thread!();
|
||||
test_atomic_int_single_thread!(i8);
|
||||
test_atomic_int_single_thread!(u8);
|
||||
test_atomic_int_single_thread!(i16);
|
||||
test_atomic_int_single_thread!(u16);
|
||||
test_atomic_int_single_thread!(i32);
|
||||
test_atomic_int_single_thread!(u32);
|
||||
test_atomic_int_single_thread!(i64);
|
||||
test_atomic_int_single_thread!(u64);
|
||||
test_atomic_int_single_thread!(i128);
|
||||
test_atomic_int_single_thread!(u128);
|
||||
test_atomic_int_single_thread!(isize);
|
||||
test_atomic_int_single_thread!(usize);
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Adapted from https://github.com/rust-embedded/msp430.
|
||||
|
||||
See also src/imp/msp430.rs.
|
||||
|
||||
Refs:
|
||||
- MSP430x5xx and MSP430x6xx Family User's Guide, Rev. Q
|
||||
https://www.ti.com/lit/ug/slau208q/slau208q.pdf
|
||||
|
||||
See tests/asm-test/asm/portable-atomic for generated assembly.
|
||||
*/
|
||||
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
use core::arch::asm;
|
||||
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(any(test, portable_atomic_no_atomic_cas))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(test, not(target_has_atomic = "ptr")))
|
||||
)]
|
||||
pub(super) use super::super::msp430 as atomic;
|
||||
|
||||
pub(crate) type State = u16;
|
||||
|
||||
/// Disables interrupts and returns the previous interrupt state.
|
||||
#[inline(always)]
|
||||
pub(crate) fn disable() -> State {
|
||||
let sr: State;
|
||||
// SAFETY: reading the status register and disabling interrupts are safe.
|
||||
// (see module-level comments of interrupt/mod.rs on the safety of using privileged instructions)
|
||||
//
|
||||
// See "NOTE: Enable and Disable Interrupt" of User's Guide for NOP: https://www.ti.com/lit/ug/slau208q/slau208q.pdf#page=60
|
||||
unsafe {
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
asm!(
|
||||
"mov r2, {sr}", // sr = SR
|
||||
"dint {{ nop", // SR.GIE = 0
|
||||
sr = out(reg) sr,
|
||||
// Do not use `nomem` and `readonly` because prevent subsequent memory accesses from being reordered before interrupts are disabled.
|
||||
// Do not use `preserves_flags` because DINT modifies the GIE (global interrupt enable) bit of the status register.
|
||||
options(nostack),
|
||||
);
|
||||
#[cfg(portable_atomic_no_asm)]
|
||||
{
|
||||
llvm_asm!("mov r2, $0" : "=r"(sr) ::: "volatile");
|
||||
llvm_asm!("dint { nop" ::: "memory" : "volatile");
|
||||
}
|
||||
}
|
||||
sr
|
||||
}
|
||||
|
||||
/// Restores the previous interrupt state.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// The state must be the one retrieved by the previous `disable`.
|
||||
#[inline(always)]
|
||||
pub(crate) unsafe fn restore(prev_sr: State) {
|
||||
// SAFETY: the caller must guarantee that the state was retrieved by the previous `disable`,
|
||||
//
|
||||
// This clobbers the entire status register, but we never explicitly modify
|
||||
// flags within a critical session, and the only flags that may be changed
|
||||
// within a critical session are the arithmetic flags that are changed as
|
||||
// a side effect of arithmetic operations, etc., which LLVM recognizes unless
|
||||
// preserves_flags is set, so it is safe to clobber them here.
|
||||
// See also the discussion at https://github.com/taiki-e/portable-atomic/pull/40.
|
||||
//
|
||||
// See "NOTE: Enable and Disable Interrupt" of User's Guide for NOP: https://www.ti.com/lit/ug/slau208q/slau208q.pdf#page=60
|
||||
unsafe {
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
asm!(
|
||||
"nop {{ mov {prev_sr}, r2 {{ nop", // SR = prev_sr
|
||||
prev_sr = in(reg) prev_sr,
|
||||
// Do not use `nomem` and `readonly` because prevent preceding memory accesses from being reordered after interrupts are enabled.
|
||||
// Do not use `preserves_flags` because MOV modifies the status register.
|
||||
options(nostack),
|
||||
);
|
||||
#[cfg(portable_atomic_no_asm)]
|
||||
llvm_asm!("nop { mov $0, r2 { nop" :: "r"(prev_sr) : "memory" : "volatile");
|
||||
}
|
||||
}
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Refs:
|
||||
- RISC-V Instruction Set Manual
|
||||
Machine Status (mstatus and mstatush) Registers
|
||||
https://docs.riscv.org/reference/isa/v20260120/priv/machine.html#3-1-1-6-machine-status-mstatus-and-mstatush-registers
|
||||
Supervisor Status (sstatus) Register
|
||||
https://docs.riscv.org/reference/isa/v20260120/priv/supervisor.html#sstatus
|
||||
|
||||
See also src/imp/riscv.rs.
|
||||
|
||||
See tests/asm-test/asm/portable-atomic for generated assembly.
|
||||
*/
|
||||
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
use core::arch::asm;
|
||||
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(any(test, portable_atomic_no_atomic_cas))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(test, not(target_has_atomic = "ptr")))
|
||||
)]
|
||||
pub(super) use super::super::riscv as atomic;
|
||||
|
||||
cfg_sel!({
|
||||
// Supervisor-mode (S-mode)
|
||||
#[cfg(portable_atomic_s_mode)]
|
||||
{
|
||||
// Status register
|
||||
macro_rules! status {
|
||||
() => {
|
||||
"sstatus"
|
||||
};
|
||||
}
|
||||
// SIE (Supervisor Interrupt Enable) bit (1 << 1)
|
||||
#[cfg(portable_atomic_s_mode)]
|
||||
macro_rules! mask {
|
||||
() => {
|
||||
"0x2"
|
||||
};
|
||||
}
|
||||
const MASK: State = 0x2;
|
||||
}
|
||||
// Machine-mode (M-mode)
|
||||
#[cfg(else)]
|
||||
{
|
||||
// Status register
|
||||
macro_rules! status {
|
||||
() => {
|
||||
"mstatus"
|
||||
};
|
||||
}
|
||||
// MIE (Machine Interrupt Enable) bit (1 << 3)
|
||||
macro_rules! mask {
|
||||
() => {
|
||||
"0x8"
|
||||
};
|
||||
}
|
||||
const MASK: State = 0x8;
|
||||
}
|
||||
});
|
||||
|
||||
pub(crate) type State = crate::utils::RegSize;
|
||||
|
||||
/// Disables interrupts and returns the previous interrupt state.
|
||||
#[inline(always)]
|
||||
pub(crate) fn disable() -> State {
|
||||
let status: State;
|
||||
// SAFETY: reading mstatus/sstatus and disabling interrupts is safe.
|
||||
// (see module-level comments of interrupt/mod.rs on the safety of using privileged instructions)
|
||||
unsafe {
|
||||
asm!(
|
||||
concat!("csrrci {status}, ", status!(), ", ", mask!()), // atomic { status = status!(); status!() &= !mask!() }
|
||||
status = out(reg) status,
|
||||
// Do not use `nomem` and `readonly` because prevent subsequent memory accesses from being reordered before interrupts are disabled.
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
}
|
||||
status
|
||||
}
|
||||
|
||||
/// Restores the previous interrupt state.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// The state must be the one retrieved by the previous `disable`.
|
||||
#[inline(always)]
|
||||
pub(crate) unsafe fn restore(prev_status: State) {
|
||||
// SAFETY: the caller must guarantee that the state was retrieved by the previous `disable`,
|
||||
//
|
||||
// This avoids clobbering other fields in the mstatus/sstatus register.
|
||||
unsafe {
|
||||
asm!(
|
||||
concat!("csrs ", status!(), ", {mask}"), // atomic { status!() |= mask }
|
||||
mask = in(reg) prev_status & MASK,
|
||||
// Do not use `nomem` and `readonly` because prevent preceding memory accesses from being reordered after interrupts are enabled.
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Refs:
|
||||
- Xtensa Instruction Set Architecture (ISA) Summary for all Xtensa LX Processors
|
||||
https://www.cadence.com/content/dam/cadence-www/global/en_US/documents/tools/silicon-solutions/compute-ip/isa-summary.pdf
|
||||
- Linux kernel's Xtensa atomic implementation
|
||||
https://github.com/torvalds/linux/blob/v6.19/arch/xtensa/include/asm/atomic.h
|
||||
|
||||
See tests/asm-test/asm/portable-atomic for generated assembly.
|
||||
*/
|
||||
|
||||
use core::arch::asm;
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(any(test, portable_atomic_no_atomic_cas))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(test, not(target_has_atomic = "ptr")))
|
||||
)]
|
||||
pub(super) use core::sync::atomic;
|
||||
|
||||
pub(crate) type State = u32;
|
||||
|
||||
/// Disables interrupts and returns the previous interrupt state.
|
||||
#[inline(always)]
|
||||
pub(crate) fn disable() -> State {
|
||||
let ps: State;
|
||||
// SAFETY: reading the PS special register and disabling all interrupts is safe.
|
||||
// (see module-level comments of interrupt/mod.rs on the safety of using privileged instructions)
|
||||
//
|
||||
// Interrupt level 15 to disable all interrupts.
|
||||
// SYNC after RSIL is not required.
|
||||
// Use a14 per https://github.com/torvalds/linux/blob/v6.16/arch/xtensa/include/asm/atomic.h#L33-L37.
|
||||
unsafe {
|
||||
asm!(
|
||||
"rsil a14, 15", // a14 = PS; PS.INTLEVEL = 15
|
||||
out("a14") ps,
|
||||
// Do not use `nomem` and `readonly` because prevent subsequent memory accesses from being reordered before interrupts are disabled.
|
||||
options(nostack),
|
||||
);
|
||||
}
|
||||
ps
|
||||
}
|
||||
|
||||
/// Restores the previous interrupt state.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// The state must be the one retrieved by the previous `disable`.
|
||||
#[inline(always)]
|
||||
pub(crate) unsafe fn restore(prev_ps: State) {
|
||||
// SAFETY: the caller must guarantee that the state was retrieved by the previous `disable`,
|
||||
// and we've checked that interrupts were enabled before disabling interrupts.
|
||||
//
|
||||
// SYNC after WSR is required to guarantee that subsequent RSIL read the written value.
|
||||
// See also 3.8.10 Processor Control Instructions of Xtensa Instruction Set Architecture (ISA) Summary for all Xtensa LX Processors.
|
||||
unsafe {
|
||||
asm!(
|
||||
"wsr.ps a14", // PS = a14
|
||||
"rsync", // wait
|
||||
in("a14") prev_ps,
|
||||
// Do not use `nomem` and `readonly` because prevent preceding memory accesses from being reordered after interrupts are enabled.
|
||||
options(nostack),
|
||||
);
|
||||
}
|
||||
}
|
||||
+566
@@ -0,0 +1,566 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Lock-free implementations
|
||||
|
||||
cfg_core_atomic!({
|
||||
mod core_atomic;
|
||||
});
|
||||
|
||||
// AVR
|
||||
#[cfg(target_arch = "avr")]
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
#[cfg(not(feature = "critical-section"))]
|
||||
mod avr;
|
||||
|
||||
// pre-v6 Arm Linux
|
||||
#[cfg(all(
|
||||
target_arch = "arm",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(target_os = "linux", target_os = "android"),
|
||||
not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
|
||||
))]
|
||||
pub(crate) mod arm_linux;
|
||||
|
||||
// MSP430
|
||||
#[cfg(target_arch = "msp430")]
|
||||
pub(crate) mod msp430;
|
||||
|
||||
// RISC-V without A-extension
|
||||
#[cfg(any(test, not(feature = "critical-section")))]
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(any(
|
||||
all(test, not(any(miri, portable_atomic_sanitize_thread))),
|
||||
portable_atomic_no_atomic_cas,
|
||||
))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(
|
||||
all(test, not(any(miri, portable_atomic_sanitize_thread))),
|
||||
not(target_has_atomic = "ptr"),
|
||||
))
|
||||
)]
|
||||
#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
|
||||
mod riscv;
|
||||
|
||||
// x86-specific optimizations
|
||||
// Miri and Sanitizer do not support inline assembly.
|
||||
#[cfg(all(
|
||||
any(target_arch = "x86", target_arch = "x86_64"),
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
))]
|
||||
pub(crate) mod x86;
|
||||
|
||||
// 64-bit atomic implementations on 32-bit architectures
|
||||
#[cfg(any(target_arch = "arm", target_arch = "riscv32"))]
|
||||
mod atomic64;
|
||||
|
||||
// 128-bit atomic implementations on 64-bit architectures
|
||||
#[cfg(any(
|
||||
target_arch = "aarch64",
|
||||
target_arch = "arm64ec",
|
||||
target_arch = "powerpc64",
|
||||
target_arch = "riscv64",
|
||||
target_arch = "s390x",
|
||||
target_arch = "x86_64",
|
||||
))]
|
||||
mod atomic128;
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Lock-based fallback implementations
|
||||
|
||||
#[cfg(feature = "fallback")]
|
||||
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
|
||||
#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
|
||||
#[cfg_attr(
|
||||
not(test),
|
||||
cfg(not(any(
|
||||
// ---------------
|
||||
// Use other ways.
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
// --------------------
|
||||
// Has 128-bit atomics.
|
||||
all(
|
||||
target_arch = "aarch64",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
),
|
||||
all(
|
||||
target_arch = "arm64ec",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
not(portable_atomic_no_asm),
|
||||
),
|
||||
all(
|
||||
target_arch = "x86_64",
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
|
||||
),
|
||||
all(
|
||||
target_arch = "riscv64",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
|
||||
),
|
||||
all(
|
||||
target_arch = "powerpc64",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
not(portable_atomic_no_asm),
|
||||
any(
|
||||
target_feature = "quadword-atomics",
|
||||
portable_atomic_target_feature = "quadword-atomics",
|
||||
),
|
||||
),
|
||||
all(
|
||||
target_arch = "s390x",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
not(portable_atomic_no_asm),
|
||||
),
|
||||
))
|
||||
))]
|
||||
mod fallback;
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Fallback implementation based on disabling interrupts or critical-section
|
||||
|
||||
// On AVR, we always use critical section based fallback implementation.
|
||||
// AVR can be safely assumed to be single-core, so this is sound.
|
||||
// MSP430 as well.
|
||||
// See the module-level comments of interrupt module for more.
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(any(
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
all(feature = "critical-section", any(test, portable_atomic_no_atomic_cas)),
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
portable_atomic_unsafe_assume_privileged,
|
||||
))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
all(feature = "critical-section", any(test, not(target_has_atomic = "ptr"))),
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
portable_atomic_unsafe_assume_privileged,
|
||||
))
|
||||
)]
|
||||
#[cfg(any(
|
||||
target_arch = "arm",
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
target_arch = "riscv32",
|
||||
target_arch = "riscv64",
|
||||
target_arch = "xtensa",
|
||||
feature = "critical-section",
|
||||
))]
|
||||
mod interrupt;
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Atomic float implementations
|
||||
|
||||
#[cfg(feature = "float")]
|
||||
pub(crate) mod float;
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
// {8,16,32}-bit & ptr-sized atomics
|
||||
cfg_sel!({
|
||||
// has CAS | (has core atomic & !(avr | msp430 | critical section)) => core atomic
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(not(any(
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
portable_atomic_no_atomic_load_store,
|
||||
all(
|
||||
any(
|
||||
target_arch = "riscv32",
|
||||
target_arch = "riscv64",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
),
|
||||
portable_atomic_no_atomic_cas,
|
||||
),
|
||||
)))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(not(any(
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
portable_atomic_no_atomic_load_store,
|
||||
all(
|
||||
any(
|
||||
target_arch = "riscv32",
|
||||
target_arch = "riscv64",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
),
|
||||
not(target_has_atomic = "ptr"),
|
||||
),
|
||||
)))
|
||||
)]
|
||||
{
|
||||
pub(crate) use self::core_atomic::{
|
||||
AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, AtomicU32,
|
||||
AtomicUsize,
|
||||
};
|
||||
}
|
||||
// no native atomic CAS & (assume single core | critical section) => critical section based fallback
|
||||
#[cfg(any(
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
))]
|
||||
{
|
||||
pub(crate) use self::interrupt::{
|
||||
AtomicI8, AtomicI16, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, AtomicUsize,
|
||||
};
|
||||
#[cfg(any(not(target_pointer_width = "16"), feature = "fallback"))]
|
||||
pub(crate) use self::interrupt::{AtomicI32, AtomicU32};
|
||||
}
|
||||
// RISC-V without A-extension & !(assume single core | critical section)
|
||||
#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
|
||||
{
|
||||
pub(crate) use self::riscv::{
|
||||
AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, AtomicU32,
|
||||
AtomicUsize,
|
||||
};
|
||||
}
|
||||
// bpf & !(critical section) => core atomic
|
||||
#[cfg(target_arch = "bpf")]
|
||||
{
|
||||
pub(crate) use self::core_atomic::{AtomicIsize, AtomicPtr, AtomicUsize};
|
||||
}
|
||||
#[cfg(else)]
|
||||
{
|
||||
// no atomic
|
||||
}
|
||||
});
|
||||
|
||||
// 64-bit atomics
|
||||
cfg_sel!({
|
||||
// has CAS | (has core atomic & !(avr | msp430 | critical section)) => core atomic
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(all(
|
||||
not(any(
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
portable_atomic_no_atomic_load_store,
|
||||
all(
|
||||
any(
|
||||
target_arch = "riscv32",
|
||||
target_arch = "riscv64",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
),
|
||||
portable_atomic_no_atomic_cas,
|
||||
)
|
||||
)),
|
||||
any(
|
||||
not(portable_atomic_no_atomic_64),
|
||||
not(any(target_pointer_width = "16", target_pointer_width = "32")),
|
||||
),
|
||||
))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(all(
|
||||
not(any(
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
portable_atomic_no_atomic_load_store,
|
||||
all(
|
||||
any(
|
||||
target_arch = "riscv32",
|
||||
target_arch = "riscv64",
|
||||
feature = "critical-section",
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
),
|
||||
not(target_has_atomic = "ptr"),
|
||||
)
|
||||
)),
|
||||
any(
|
||||
target_has_atomic = "64",
|
||||
not(any(target_pointer_width = "16", target_pointer_width = "32")),
|
||||
),
|
||||
))
|
||||
)]
|
||||
{
|
||||
pub(crate) use self::core_atomic::{AtomicI64, AtomicU64};
|
||||
}
|
||||
// pre-v6 Arm Linux
|
||||
#[cfg(all(
|
||||
feature = "fallback",
|
||||
target_arch = "arm",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(target_os = "linux", target_os = "android"),
|
||||
not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
))]
|
||||
{
|
||||
pub(crate) use self::atomic64::arm_linux::{AtomicI64, AtomicU64};
|
||||
}
|
||||
// riscv32 & (zacas | outline-atomics)
|
||||
#[cfg(all(
|
||||
target_arch = "riscv32",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(
|
||||
target_feature = "zacas",
|
||||
portable_atomic_target_feature = "zacas",
|
||||
all(
|
||||
feature = "fallback",
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
any(target_os = "linux", target_os = "android"),
|
||||
),
|
||||
),
|
||||
))]
|
||||
{
|
||||
pub(crate) use self::atomic64::riscv32::{AtomicI64, AtomicU64};
|
||||
}
|
||||
// no native atomic CAS & (assume single core | critical section) => critical section based fallback
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(any(
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
all(feature = "critical-section", portable_atomic_no_atomic_cas),
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
all(feature = "critical-section", not(target_has_atomic = "ptr")),
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
))
|
||||
)]
|
||||
{
|
||||
#[cfg(any(
|
||||
not(any(target_pointer_width = "16", target_pointer_width = "32")),
|
||||
feature = "fallback",
|
||||
))]
|
||||
pub(crate) use self::interrupt::{AtomicI64, AtomicU64};
|
||||
}
|
||||
// no core 64-bit atomic & has CAS => use lock-base fallback
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(all(feature = "fallback", not(portable_atomic_no_atomic_cas)))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(all(feature = "fallback", target_has_atomic = "ptr"))
|
||||
)]
|
||||
{
|
||||
pub(crate) use self::fallback::{AtomicI64, AtomicU64};
|
||||
}
|
||||
// RISC-V without A-extension & !(assume single core | critical section)
|
||||
#[cfg(target_arch = "riscv64")]
|
||||
{
|
||||
pub(crate) use self::riscv::{AtomicI64, AtomicU64};
|
||||
}
|
||||
// bpf & !(critical section) => core atomic
|
||||
#[cfg(target_arch = "bpf")]
|
||||
{
|
||||
pub(crate) use self::core_atomic::{AtomicI64, AtomicU64};
|
||||
}
|
||||
#[cfg(else)]
|
||||
{
|
||||
// no atomic
|
||||
}
|
||||
});
|
||||
|
||||
// 128-bit atomics
|
||||
cfg_sel!({
|
||||
// AArch64
|
||||
#[cfg(any(
|
||||
all(
|
||||
target_arch = "aarch64",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
),
|
||||
all(
|
||||
target_arch = "arm64ec",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
not(portable_atomic_no_asm),
|
||||
),
|
||||
))]
|
||||
{
|
||||
pub(crate) use self::atomic128::aarch64::{AtomicI128, AtomicU128};
|
||||
}
|
||||
// x86_64 & (cmpxchg16b | outline-atomics)
|
||||
#[cfg(all(
|
||||
target_arch = "x86_64",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
portable_atomic_no_cmpxchg16b_intrinsic,
|
||||
)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(
|
||||
target_feature = "cmpxchg16b",
|
||||
portable_atomic_target_feature = "cmpxchg16b",
|
||||
all(
|
||||
feature = "fallback",
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
not(any(target_env = "sgx", miri)),
|
||||
),
|
||||
),
|
||||
))]
|
||||
{
|
||||
pub(crate) use self::atomic128::x86_64::{AtomicI128, AtomicU128};
|
||||
}
|
||||
// riscv64 & (zacas | outline-atomics)
|
||||
#[cfg(all(
|
||||
target_arch = "riscv64",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(
|
||||
target_feature = "zacas",
|
||||
portable_atomic_target_feature = "zacas",
|
||||
all(
|
||||
feature = "fallback",
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
any(target_os = "linux", target_os = "android"),
|
||||
),
|
||||
),
|
||||
))]
|
||||
{
|
||||
pub(crate) use self::atomic128::riscv64::{AtomicI128, AtomicU128};
|
||||
}
|
||||
// powerpc64 & (pwr8 | outline-atomics)
|
||||
#[cfg(all(
|
||||
target_arch = "powerpc64",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
not(portable_atomic_no_asm),
|
||||
any(
|
||||
target_feature = "quadword-atomics",
|
||||
portable_atomic_target_feature = "quadword-atomics",
|
||||
all(
|
||||
feature = "fallback",
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
any(
|
||||
all(
|
||||
target_os = "linux",
|
||||
any(
|
||||
all(
|
||||
target_env = "gnu",
|
||||
any(target_endian = "little", not(target_feature = "crt-static")),
|
||||
),
|
||||
all(
|
||||
target_env = "musl",
|
||||
any(not(target_feature = "crt-static"), feature = "std"),
|
||||
),
|
||||
target_env = "ohos",
|
||||
all(target_env = "uclibc", not(target_feature = "crt-static")),
|
||||
portable_atomic_outline_atomics,
|
||||
),
|
||||
),
|
||||
target_os = "android",
|
||||
all(
|
||||
target_os = "freebsd",
|
||||
any(
|
||||
target_endian = "little",
|
||||
not(target_feature = "crt-static"),
|
||||
portable_atomic_outline_atomics,
|
||||
),
|
||||
),
|
||||
target_os = "openbsd",
|
||||
all(
|
||||
target_os = "aix",
|
||||
not(portable_atomic_pre_llvm_20),
|
||||
portable_atomic_outline_atomics,
|
||||
), // TODO(aix): currently disabled by default
|
||||
),
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
),
|
||||
),
|
||||
))]
|
||||
{
|
||||
pub(crate) use self::atomic128::powerpc64::{AtomicI128, AtomicU128};
|
||||
}
|
||||
// s390x
|
||||
#[cfg(all(
|
||||
target_arch = "s390x",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
not(portable_atomic_no_asm),
|
||||
))]
|
||||
{
|
||||
pub(crate) use self::atomic128::s390x::{AtomicI128, AtomicU128};
|
||||
}
|
||||
// no native atomic CAS & (assume single core | critical section) => critical section based fallback
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(any(
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
all(feature = "critical-section", portable_atomic_no_atomic_cas),
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(any(
|
||||
target_arch = "avr",
|
||||
target_arch = "msp430",
|
||||
all(feature = "critical-section", not(target_has_atomic = "ptr")),
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
))
|
||||
)]
|
||||
{
|
||||
#[cfg(feature = "fallback")]
|
||||
pub(crate) use self::interrupt::{AtomicI128, AtomicU128};
|
||||
}
|
||||
// no core 128-bit atomic & has CAS => use lock-base fallback
|
||||
#[cfg_attr(
|
||||
portable_atomic_no_cfg_target_has_atomic,
|
||||
cfg(all(feature = "fallback", not(portable_atomic_no_atomic_cas)))
|
||||
)]
|
||||
#[cfg_attr(
|
||||
not(portable_atomic_no_cfg_target_has_atomic),
|
||||
cfg(all(feature = "fallback", target_has_atomic = "ptr"))
|
||||
)]
|
||||
{
|
||||
pub(crate) use self::fallback::{AtomicI128, AtomicU128};
|
||||
}
|
||||
#[cfg(else)]
|
||||
{
|
||||
// no atomic
|
||||
}
|
||||
});
|
||||
+283
@@ -0,0 +1,283 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Atomic implementation on MSP430.
|
||||
|
||||
Adapted from https://github.com/pftbest/msp430-atomic.
|
||||
|
||||
Operations not supported here are provided by disabling interrupts.
|
||||
See also src/imp/interrupt/msp430.rs.
|
||||
|
||||
See "Atomic operation overview by architecture" in atomic-maybe-uninit for a more comprehensive and
|
||||
detailed description of the atomic and synchronize instructions in this architecture:
|
||||
https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#msp430
|
||||
|
||||
Note: Ordering is always SeqCst.
|
||||
|
||||
Refs:
|
||||
- MSP430x5xx and MSP430x6xx Family User's Guide, Rev. Q
|
||||
https://www.ti.com/lit/ug/slau208q/slau208q.pdf
|
||||
- atomic-maybe-uninit
|
||||
https://github.com/taiki-e/atomic-maybe-uninit
|
||||
|
||||
See tests/asm-test/asm/portable-atomic for generated assembly.
|
||||
*/
|
||||
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
use core::arch::asm;
|
||||
#[cfg(not(feature = "critical-section"))]
|
||||
use core::cell::UnsafeCell;
|
||||
use core::sync::atomic::Ordering;
|
||||
|
||||
/// An atomic fence.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `order` is [`Relaxed`](Ordering::Relaxed).
|
||||
#[inline]
|
||||
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
|
||||
pub fn fence(order: Ordering) {
|
||||
match order {
|
||||
Ordering::Relaxed => panic!("there is no such thing as a relaxed fence"),
|
||||
// MSP430 is single-core and a compiler fence works as an atomic fence.
|
||||
_ => compiler_fence(order),
|
||||
}
|
||||
}
|
||||
|
||||
/// A compiler memory fence.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `order` is [`Relaxed`](Ordering::Relaxed).
|
||||
#[inline]
|
||||
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
|
||||
pub fn compiler_fence(order: Ordering) {
|
||||
match order {
|
||||
Ordering::Relaxed => panic!("there is no such thing as a relaxed compiler fence"),
|
||||
_ => {}
|
||||
}
|
||||
// SAFETY: using an empty asm is safe.
|
||||
unsafe {
|
||||
// Do not use `nomem` and `readonly` because prevent preceding and subsequent memory accesses from being reordered.
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
asm!("", options(nostack, preserves_flags));
|
||||
#[cfg(portable_atomic_no_asm)]
|
||||
llvm_asm!("" ::: "memory" : "volatile");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "critical-section"))]
|
||||
items!({
|
||||
macro_rules! atomic {
|
||||
(load_store, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $size:tt) => {
|
||||
#[repr(transparent)]
|
||||
pub(crate) struct $atomic_type $(<$($generics)*>)? {
|
||||
v: UnsafeCell<$value_type>,
|
||||
}
|
||||
|
||||
// Send is implicitly implemented for atomic integers, but not for atomic pointers.
|
||||
// SAFETY: any data races are prevented by atomic operations.
|
||||
unsafe impl $(<$($generics)*>)? Send for $atomic_type $(<$($generics)*>)? {}
|
||||
// SAFETY: any data races are prevented by atomic operations.
|
||||
unsafe impl $(<$($generics)*>)? Sync for $atomic_type $(<$($generics)*>)? {}
|
||||
|
||||
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
all(debug_assertions, not(portable_atomic_no_track_caller)),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn load(&self, order: Ordering) -> $value_type {
|
||||
crate::utils::assert_load_ordering(order);
|
||||
let src = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
let out;
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
asm!(
|
||||
concat!("mov.", $size, " @{src}, {out}"), // atomic { out = *src }
|
||||
src = in(reg) src,
|
||||
out = lateout(reg) out,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
#[cfg(portable_atomic_no_asm)]
|
||||
llvm_asm!(
|
||||
concat!("mov.", $size, " $1, $0")
|
||||
: "=r"(out) : "*m"(src) : "memory" : "volatile"
|
||||
);
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(
|
||||
all(debug_assertions, not(portable_atomic_no_track_caller)),
|
||||
track_caller
|
||||
)]
|
||||
pub(crate) fn store(&self, val: $value_type, order: Ordering) {
|
||||
crate::utils::assert_store_ordering(order);
|
||||
let dst = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
asm!(
|
||||
concat!("mov.", $size, " {val}, 0({dst})"), // atomic { *dst = val }
|
||||
dst = in(reg) dst,
|
||||
val = in(reg) val,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
#[cfg(portable_atomic_no_asm)]
|
||||
llvm_asm!(
|
||||
concat!("mov.", $size, " $1, $0")
|
||||
:: "*m"(dst), "ir"(val) : "memory" : "volatile"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
($([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $size:tt) => {
|
||||
atomic!(load_store, $([$($generics)*])? $atomic_type, $value_type, $size);
|
||||
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
|
||||
#[inline]
|
||||
pub(crate) fn add(&self, val: $value_type, _order: Ordering) {
|
||||
let dst = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
asm!(
|
||||
concat!("add.", $size, " {val}, 0({dst})"), // atomic { *dst += val }
|
||||
dst = in(reg) dst,
|
||||
val = in(reg) val,
|
||||
// Do not use `preserves_flags` because ADD modifies the V, N, Z, and C bits of the status register.
|
||||
options(nostack),
|
||||
);
|
||||
#[cfg(portable_atomic_no_asm)]
|
||||
llvm_asm!(
|
||||
concat!("add.", $size, " $1, $0")
|
||||
:: "*m"(dst), "ir"(val) : "memory" : "volatile"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn sub(&self, val: $value_type, _order: Ordering) {
|
||||
let dst = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
asm!(
|
||||
concat!("sub.", $size, " {val}, 0({dst})"), // atomic { *dst -= val }
|
||||
dst = in(reg) dst,
|
||||
val = in(reg) val,
|
||||
// Do not use `preserves_flags` because SUB modifies the V, N, Z, and C bits of the status register.
|
||||
options(nostack),
|
||||
);
|
||||
#[cfg(portable_atomic_no_asm)]
|
||||
llvm_asm!(
|
||||
concat!("sub.", $size, " $1, $0")
|
||||
:: "*m"(dst), "ir"(val) : "memory" : "volatile"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn and(&self, val: $value_type, _order: Ordering) {
|
||||
let dst = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
asm!(
|
||||
concat!("and.", $size, " {val}, 0({dst})"), // atomic { *dst &= val }
|
||||
dst = in(reg) dst,
|
||||
val = in(reg) val,
|
||||
// Do not use `preserves_flags` because AND modifies the V, N, Z, and C bits of the status register.
|
||||
options(nostack),
|
||||
);
|
||||
#[cfg(portable_atomic_no_asm)]
|
||||
llvm_asm!(
|
||||
concat!("and.", $size, " $1, $0")
|
||||
:: "*m"(dst), "ir"(val) : "memory" : "volatile"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn or(&self, val: $value_type, _order: Ordering) {
|
||||
let dst = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
asm!(
|
||||
concat!("bis.", $size, " {val}, 0({dst})"), // atomic { *dst |= val }
|
||||
dst = in(reg) dst,
|
||||
val = in(reg) val,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
#[cfg(portable_atomic_no_asm)]
|
||||
llvm_asm!(
|
||||
concat!("bis.", $size, " $1, $0")
|
||||
:: "*m"(dst), "ir"(val) : "memory" : "volatile"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn xor(&self, val: $value_type, _order: Ordering) {
|
||||
let dst = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
asm!(
|
||||
concat!("xor.", $size, " {val}, 0({dst})"), // atomic { *dst ^= val }
|
||||
dst = in(reg) dst,
|
||||
val = in(reg) val,
|
||||
// Do not use `preserves_flags` because XOR modifies the V, N, Z, and C bits of the status register.
|
||||
options(nostack),
|
||||
);
|
||||
#[cfg(portable_atomic_no_asm)]
|
||||
llvm_asm!(
|
||||
concat!("xor.", $size, " $1, $0")
|
||||
:: "*m"(dst), "ir"(val) : "memory" : "volatile"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn not(&self, _order: Ordering) {
|
||||
let dst = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
asm!(
|
||||
concat!("inv.", $size, " 0({dst})"), // atomic { *dst = !*dst }
|
||||
dst = in(reg) dst,
|
||||
// Do not use `preserves_flags` because INV modifies the V, N, Z, and C bits of the status register.
|
||||
options(nostack),
|
||||
);
|
||||
#[cfg(portable_atomic_no_asm)]
|
||||
llvm_asm!(
|
||||
concat!("inv.", $size, " $0")
|
||||
:: "*m"(dst) : "memory" : "volatile"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
atomic!(AtomicI8, i8, "b");
|
||||
atomic!(AtomicU8, u8, "b");
|
||||
atomic!(AtomicI16, i16, "w");
|
||||
atomic!(AtomicU16, u16, "w");
|
||||
atomic!(AtomicIsize, isize, "w");
|
||||
atomic!(AtomicUsize, usize, "w");
|
||||
atomic!(load_store, [T] AtomicPtr, *mut T, "w");
|
||||
});
|
||||
+947
@@ -0,0 +1,947 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Atomic load/store implementation on RISC-V.
|
||||
|
||||
This is for RISC-V targets without A extension. (pre-1.76 rustc doesn't provide atomics
|
||||
at all on such targets. https://github.com/rust-lang/rust/pull/114499)
|
||||
|
||||
Also, optionally provides RMW implementation when Zaamo extension or force-amo feature is enabled.
|
||||
|
||||
If A extension is not available but Zalrsc or Zacas extension is available, LLVM can implement all
|
||||
atomic operations using it (https://github.com/llvm/llvm-project/commit/70f70390bb83a383fcb88ea843adbc4edf6ee34b),
|
||||
so this module contains no code for these extensions.
|
||||
|
||||
See "Atomic operation overview by architecture" in atomic-maybe-uninit for a more comprehensive and
|
||||
detailed description of the atomic and synchronize instructions in this architecture:
|
||||
https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#risc-v
|
||||
|
||||
Refs:
|
||||
- RISC-V Instruction Set Manual
|
||||
"Zaamo" Extension for Atomic Memory Operations
|
||||
https://docs.riscv.org/reference/isa/v20260120/unpriv/a-st-ext.html#sec:amo
|
||||
"Zabha" Extension for Byte and Halfword Atomic Memory Operations
|
||||
https://docs.riscv.org/reference/isa/v20260120/unpriv/zabha.html
|
||||
- RISC-V Atomics ABI Specification
|
||||
https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/draft-20250812-301374e92976e298e676e7129a6212926b2299ce/riscv-atomic.adoc
|
||||
- atomic-maybe-uninit
|
||||
https://github.com/taiki-e/atomic-maybe-uninit
|
||||
|
||||
See tests/asm-test/asm/portable-atomic for generated assembly.
|
||||
*/
|
||||
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
use core::arch::asm;
|
||||
use core::{cell::UnsafeCell, sync::atomic::Ordering};
|
||||
|
||||
#[cfg(any(
|
||||
test,
|
||||
portable_atomic_force_amo,
|
||||
target_feature = "zaamo",
|
||||
portable_atomic_target_feature = "zaamo",
|
||||
))]
|
||||
items!({
|
||||
macro_rules! atomic_rmw_amo_ext {
|
||||
// Use +a also for zaamo because `option arch +zaamo` requires LLVM 19.
|
||||
// https://github.com/llvm/llvm-project/commit/8be079cdddfd628d356d9ddb5ab397ea95fb1030
|
||||
("w") => {
|
||||
"+a"
|
||||
};
|
||||
("d") => {
|
||||
"+a"
|
||||
};
|
||||
("b") => {
|
||||
"+a,+zabha"
|
||||
};
|
||||
("h") => {
|
||||
"+a,+zabha"
|
||||
};
|
||||
}
|
||||
macro_rules! atomic_rmw_amo {
|
||||
($op:ident, $dst:ident, $val:ident, $order:ident, $size:tt) => {{
|
||||
let out;
|
||||
macro_rules! op {
|
||||
($asm_order:tt) => {
|
||||
// SAFETY: The user guaranteed that the AMO instruction is available in this
|
||||
// system by setting the portable_atomic_force_amo/target_feature and
|
||||
// portable_atomic_unsafe_assume_single_core.
|
||||
// The caller of this macro must guarantee the validity of the pointer.
|
||||
asm!(
|
||||
".option push",
|
||||
// https://github.com/riscv-non-isa/riscv-asm-manual/blob/v0.0.1/src/asm-manual.adoc#arch
|
||||
// LLVM supports `.option arch` directive on LLVM 17+.
|
||||
// https://github.com/llvm/llvm-project/commit/9e8ed3403c191ab9c4903e8eeb8f732ff8a43cb4
|
||||
// Note that `.insn <value>` directive requires LLVM 19.
|
||||
// https://github.com/llvm/llvm-project/commit/2a086dce691e3cc34a2fc27f4fb255bb2cbbfac9
|
||||
concat!(".option arch, ", atomic_rmw_amo_ext!($size)),
|
||||
concat!("amo", stringify!($op), ".", $size, $asm_order, " {out}, {val}, 0({dst})"), // atomic { _x = *dst; *dst = op(_x, val); out = _x }
|
||||
".option pop",
|
||||
dst = in(reg) ptr_reg!($dst),
|
||||
val = in(reg) $val,
|
||||
out = lateout(reg) out,
|
||||
options(nostack, preserves_flags),
|
||||
)
|
||||
};
|
||||
}
|
||||
match $order {
|
||||
Ordering::Relaxed => op!(""),
|
||||
Ordering::Acquire => op!(".aq"),
|
||||
Ordering::Release => op!(".rl"),
|
||||
// AcqRel and SeqCst RMWs are equivalent.
|
||||
Ordering::AcqRel | Ordering::SeqCst => op!(".aqrl"),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
out
|
||||
}};
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
|
||||
items!({
|
||||
#[cfg(target_arch = "riscv32")]
|
||||
macro_rules! w {
|
||||
() => {
|
||||
""
|
||||
};
|
||||
}
|
||||
#[cfg(target_arch = "riscv64")]
|
||||
macro_rules! w {
|
||||
() => {
|
||||
"w"
|
||||
};
|
||||
}
|
||||
|
||||
// 32-bit val.wrapping_shl(shift) but no extra `& (u32::BITS - 1)`
|
||||
#[inline(always)]
|
||||
fn sllw(val: u32, shift: u32) -> u32 {
|
||||
// SAFETY: Calling sll{,w} is safe.
|
||||
unsafe {
|
||||
let out;
|
||||
asm!(
|
||||
concat!("sll", w!(), " {out}, {val}, {shift}"), // out = val << shift & 31
|
||||
out = lateout(reg) out,
|
||||
val = in(reg) val,
|
||||
shift = in(reg) shift,
|
||||
options(pure, nomem, nostack, preserves_flags),
|
||||
);
|
||||
out
|
||||
}
|
||||
}
|
||||
// 32-bit val.wrapping_shr(shift) but no extra `& (u32::BITS - 1)`
|
||||
macro_rules! srlw {
|
||||
($val:expr, $shift:expr) => {
|
||||
// SAFETY: Calling srl{,w} is safe.
|
||||
unsafe {
|
||||
let val: u32 = $val;
|
||||
let shift: u32 = $shift;
|
||||
let out;
|
||||
asm!(
|
||||
concat!("srl", w!(), " {out}, {val}, {shift}"), // out = val >> shift & 31
|
||||
out = lateout(reg) out,
|
||||
val = in(reg) val,
|
||||
shift = in(reg) shift,
|
||||
options(pure, nomem, nostack, preserves_flags),
|
||||
);
|
||||
out
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
trait ZeroExtend: Copy {
|
||||
/// Zero-extends `self` to `u32` if it is smaller than 32-bit.
|
||||
fn zero_extend(self) -> u32;
|
||||
}
|
||||
macro_rules! zero_extend {
|
||||
($int:ident, $uint:ident) => {
|
||||
impl ZeroExtend for $uint {
|
||||
#[inline(always)]
|
||||
fn zero_extend(self) -> u32 {
|
||||
self as u32
|
||||
}
|
||||
}
|
||||
impl ZeroExtend for $int {
|
||||
#[allow(clippy::cast_sign_loss)]
|
||||
#[inline(always)]
|
||||
fn zero_extend(self) -> u32 {
|
||||
self as $uint as u32
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
zero_extend!(i8, u8);
|
||||
zero_extend!(i16, u16);
|
||||
});
|
||||
});
|
||||
|
||||
macro_rules! atomic_load_store {
|
||||
($([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $size:tt) => {
|
||||
#[repr(transparent)]
|
||||
pub(crate) struct $atomic_type $(<$($generics)*>)? {
|
||||
v: UnsafeCell<$value_type>,
|
||||
}
|
||||
|
||||
// Send is implicitly implemented for atomic integers, but not for atomic pointers.
|
||||
// SAFETY: any data races are prevented by atomic operations.
|
||||
unsafe impl $(<$($generics)*>)? Send for $atomic_type $(<$($generics)*>)? {}
|
||||
// SAFETY: any data races are prevented by atomic operations.
|
||||
unsafe impl $(<$($generics)*>)? Sync for $atomic_type $(<$($generics)*>)? {}
|
||||
|
||||
#[cfg(any(test, not(portable_atomic_unsafe_assume_single_core)))]
|
||||
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
|
||||
#[inline]
|
||||
pub(crate) const fn new(v: $value_type) -> Self {
|
||||
Self { v: UnsafeCell::new(v) }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn is_lock_free() -> bool {
|
||||
Self::IS_ALWAYS_LOCK_FREE
|
||||
}
|
||||
pub(crate) const IS_ALWAYS_LOCK_FREE: bool = true;
|
||||
|
||||
#[inline]
|
||||
pub(crate) const fn as_ptr(&self) -> *mut $value_type {
|
||||
self.v.get()
|
||||
}
|
||||
}
|
||||
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
|
||||
#[inline]
|
||||
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
|
||||
pub(crate) fn load(&self, order: Ordering) -> $value_type {
|
||||
crate::utils::assert_load_ordering(order);
|
||||
let src = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
let out;
|
||||
macro_rules! atomic_load {
|
||||
($acquire:tt, $release:tt) => {
|
||||
asm!(
|
||||
$release, // fence
|
||||
concat!("l", $size, " {out}, 0({src})"), // atomic { out = *src }
|
||||
$acquire, // fence
|
||||
src = in(reg) ptr_reg!(src),
|
||||
out = lateout(reg) out,
|
||||
options(nostack, preserves_flags),
|
||||
)
|
||||
};
|
||||
}
|
||||
match order {
|
||||
Ordering::Relaxed => atomic_load!("", ""),
|
||||
Ordering::Acquire => atomic_load!("fence r, rw", ""),
|
||||
Ordering::SeqCst => atomic_load!("fence r, rw", "fence rw, rw"),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
|
||||
pub(crate) fn store(&self, val: $value_type, order: Ordering) {
|
||||
crate::utils::assert_store_ordering(order);
|
||||
let dst = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
macro_rules! atomic_store {
|
||||
($acquire:tt, $release:tt) => {
|
||||
asm!(
|
||||
$release, // fence
|
||||
concat!("s", $size, " {val}, 0({dst})"), // atomic { *dst = val }
|
||||
$acquire, // fence
|
||||
dst = in(reg) ptr_reg!(dst),
|
||||
val = in(reg) val,
|
||||
options(nostack, preserves_flags),
|
||||
)
|
||||
};
|
||||
}
|
||||
match order {
|
||||
Ordering::Relaxed => atomic_store!("", ""),
|
||||
Ordering::Release => atomic_store!("", "fence rw, w"),
|
||||
// https://github.com/llvm/llvm-project/commit/3ea8f2526541884e03d5bd4f4e46f4eb190990b6
|
||||
Ordering::SeqCst => atomic_store!("fence rw, rw", "fence rw, w"),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! atomic_base {
|
||||
(
|
||||
$([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $int_type:ty, $size:tt,
|
||||
$fetch_add:ident, $fetch_sub:ident
|
||||
) => {
|
||||
atomic_load_store!($([$($generics)*])? $atomic_type, $value_type, $size);
|
||||
#[cfg(any(
|
||||
test,
|
||||
portable_atomic_force_amo,
|
||||
target_feature = "zaamo",
|
||||
portable_atomic_target_feature = "zaamo",
|
||||
))]
|
||||
items!({
|
||||
// There is no amo{sub,nand,neg}.
|
||||
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
|
||||
#[inline]
|
||||
pub(crate) fn swap(&self, val: $value_type, order: Ordering) -> $value_type {
|
||||
let dst = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe { atomic_rmw_amo!(swap, dst, val, order, $size) }
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn $fetch_add(&self, val: $int_type, order: Ordering) -> $value_type {
|
||||
let dst = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe { atomic_rmw_amo!(add, dst, val, order, $size) }
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn $fetch_sub(&self, val: $int_type, order: Ordering) -> $value_type {
|
||||
self.$fetch_add(val.wrapping_neg(), order)
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_and(&self, val: $int_type, order: Ordering) -> $value_type {
|
||||
let dst = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe { atomic_rmw_amo!(and, dst, val, order, $size) }
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_or(&self, val: $int_type, order: Ordering) -> $value_type {
|
||||
let dst = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe { atomic_rmw_amo!(or, dst, val, order, $size) }
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_xor(&self, val: $int_type, order: Ordering) -> $value_type {
|
||||
let dst = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe { atomic_rmw_amo!(xor, dst, val, order, $size) }
|
||||
}
|
||||
}
|
||||
#[cfg(not(any(portable_atomic_unsafe_assume_single_core, feature = "critical-section")))]
|
||||
impl_default_bit_opts!($atomic_type, $int_type);
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! atomic_ptr {
|
||||
($size:tt) => {
|
||||
atomic_base!([T] AtomicPtr, *mut T, usize, $size, fetch_byte_add, fetch_byte_sub);
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! atomic {
|
||||
($atomic_type:ident, $value_type:ty, $size:tt, $max:tt, $min:tt) => {
|
||||
atomic_base!($atomic_type, $value_type, $value_type, $size, fetch_add, fetch_sub);
|
||||
#[cfg(any(
|
||||
test,
|
||||
portable_atomic_force_amo,
|
||||
target_feature = "zaamo",
|
||||
portable_atomic_target_feature = "zaamo",
|
||||
))]
|
||||
items!({
|
||||
// There is no amo{sub,nand,neg}.
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
pub(crate) fn fetch_not(&self, order: Ordering) -> $value_type {
|
||||
let dst = self.v.get();
|
||||
let val: crate::utils::RegSize = !0;
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe { atomic_rmw_amo!(xor, dst, val, order, $size) }
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_max(&self, val: $value_type, order: Ordering) -> $value_type {
|
||||
let dst = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe { atomic_rmw_amo!($max, dst, val, order, $size) }
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_min(&self, val: $value_type, order: Ordering) -> $value_type {
|
||||
let dst = self.v.get();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
unsafe { atomic_rmw_amo!($min, dst, val, order, $size) }
|
||||
}
|
||||
}
|
||||
#[cfg(not(any(
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
feature = "critical-section",
|
||||
)))]
|
||||
items!({
|
||||
impl_default_no_fetch_ops!($atomic_type, $value_type);
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
pub(crate) fn not(&self, order: Ordering) {
|
||||
self.fetch_not(order);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! atomic_sub_word {
|
||||
($atomic_type:ident, $value_type:ty, $size:tt, $max:tt, $min:tt) => {
|
||||
cfg_sel!({
|
||||
#[cfg(any(target_feature = "zabha", portable_atomic_target_feature = "zabha"))]
|
||||
{
|
||||
atomic!($atomic_type, $value_type, $size, $max, $min);
|
||||
}
|
||||
#[cfg(else)]
|
||||
{
|
||||
atomic_load_store!($atomic_type, $value_type, $size);
|
||||
#[cfg(any(
|
||||
test,
|
||||
portable_atomic_force_amo,
|
||||
target_feature = "zaamo",
|
||||
portable_atomic_target_feature = "zaamo",
|
||||
))]
|
||||
items!({
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
pub(crate) fn fetch_and(
|
||||
&self,
|
||||
val: $value_type,
|
||||
order: Ordering,
|
||||
) -> $value_type {
|
||||
let dst = self.v.get();
|
||||
let (dst, shift, mut mask) =
|
||||
crate::utils::create_sub_word_mask_values(dst);
|
||||
mask = !sllw(mask, shift);
|
||||
let mut val = sllw(ZeroExtend::zero_extend(val), shift);
|
||||
val |= mask;
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
let out: u32 = unsafe { atomic_rmw_amo!(and, dst, val, order, "w") };
|
||||
srlw!(out, shift)
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_or(
|
||||
&self,
|
||||
val: $value_type,
|
||||
order: Ordering,
|
||||
) -> $value_type {
|
||||
let dst = self.v.get();
|
||||
let (dst, shift, _mask) =
|
||||
crate::utils::create_sub_word_mask_values(dst);
|
||||
let val = sllw(ZeroExtend::zero_extend(val), shift);
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
let out: u32 = unsafe { atomic_rmw_amo!(or, dst, val, order, "w") };
|
||||
srlw!(out, shift)
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_xor(
|
||||
&self,
|
||||
val: $value_type,
|
||||
order: Ordering,
|
||||
) -> $value_type {
|
||||
let dst = self.v.get();
|
||||
let (dst, shift, _mask) =
|
||||
crate::utils::create_sub_word_mask_values(dst);
|
||||
let val = sllw(ZeroExtend::zero_extend(val), shift);
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
let out: u32 = unsafe { atomic_rmw_amo!(xor, dst, val, order, "w") };
|
||||
srlw!(out, shift)
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn fetch_not(&self, order: Ordering) -> $value_type {
|
||||
self.fetch_xor(!0, order)
|
||||
}
|
||||
}
|
||||
#[cfg(not(any(
|
||||
portable_atomic_unsafe_assume_single_core,
|
||||
feature = "critical-section",
|
||||
)))]
|
||||
items!({
|
||||
impl_default_bit_opts!($atomic_type, $value_type);
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
pub(crate) fn and(&self, val: $value_type, order: Ordering) {
|
||||
self.fetch_and(val, order);
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn or(&self, val: $value_type, order: Ordering) {
|
||||
self.fetch_or(val, order);
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn xor(&self, val: $value_type, order: Ordering) {
|
||||
self.fetch_xor(val, order);
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn not(&self, order: Ordering) {
|
||||
self.fetch_not(order);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
atomic_sub_word!(AtomicI8, i8, "b", max, min);
|
||||
atomic_sub_word!(AtomicU8, u8, "b", maxu, minu);
|
||||
atomic_sub_word!(AtomicI16, i16, "h", max, min);
|
||||
atomic_sub_word!(AtomicU16, u16, "h", maxu, minu);
|
||||
atomic!(AtomicI32, i32, "w", max, min);
|
||||
atomic!(AtomicU32, u32, "w", maxu, minu);
|
||||
#[cfg(target_arch = "riscv64")]
|
||||
atomic!(AtomicI64, i64, "d", max, min);
|
||||
#[cfg(target_arch = "riscv64")]
|
||||
atomic!(AtomicU64, u64, "d", maxu, minu);
|
||||
#[cfg(target_pointer_width = "32")]
|
||||
atomic!(AtomicIsize, isize, "w", max, min);
|
||||
#[cfg(target_pointer_width = "32")]
|
||||
atomic!(AtomicUsize, usize, "w", maxu, minu);
|
||||
#[cfg(target_pointer_width = "32")]
|
||||
atomic_ptr!("w");
|
||||
#[cfg(target_pointer_width = "64")]
|
||||
atomic!(AtomicIsize, isize, "d", max, min);
|
||||
#[cfg(target_pointer_width = "64")]
|
||||
atomic!(AtomicUsize, usize, "d", maxu, minu);
|
||||
#[cfg(target_pointer_width = "64")]
|
||||
atomic_ptr!("d");
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
test_atomic_ptr_load_store!();
|
||||
test_atomic_int_load_store!(i8);
|
||||
test_atomic_int_load_store!(u8);
|
||||
test_atomic_int_load_store!(i16);
|
||||
test_atomic_int_load_store!(u16);
|
||||
test_atomic_int_load_store!(i32);
|
||||
test_atomic_int_load_store!(u32);
|
||||
#[cfg(target_arch = "riscv64")]
|
||||
test_atomic_int_load_store!(i64);
|
||||
#[cfg(target_arch = "riscv64")]
|
||||
test_atomic_int_load_store!(u64);
|
||||
test_atomic_int_load_store!(isize);
|
||||
test_atomic_int_load_store!(usize);
|
||||
|
||||
macro_rules! test_atomic_ptr_amo {
|
||||
() => {
|
||||
#[allow(
|
||||
clippy::alloc_instead_of_core,
|
||||
clippy::std_instead_of_alloc,
|
||||
clippy::std_instead_of_core,
|
||||
clippy::undocumented_unsafe_blocks
|
||||
)]
|
||||
mod test_atomic_ptr_amo {
|
||||
use super::*;
|
||||
test_atomic_ptr_amo!(AtomicPtr<u8>);
|
||||
}
|
||||
};
|
||||
($atomic_type:ty) => {
|
||||
#[allow(unused_imports)]
|
||||
use sptr::Strict as _;
|
||||
|
||||
use crate::tests::helper; // for old rustc
|
||||
|
||||
::quickcheck::quickcheck! {
|
||||
fn quickcheck_swap(x: usize, y: usize) -> bool {
|
||||
let x = sptr::invalid_mut(x);
|
||||
let y = sptr::invalid_mut(y);
|
||||
for &order in &helper::SWAP_ORDERINGS {
|
||||
let a = <$atomic_type>::new(x);
|
||||
assert_eq!(a.swap(y, order), x);
|
||||
assert_eq!(a.swap(x, order), y);
|
||||
}
|
||||
true
|
||||
}
|
||||
fn quickcheck_fetch_byte_add(x: usize, y: usize) -> bool {
|
||||
let x = sptr::invalid_mut(x);
|
||||
let y = sptr::invalid_mut::<u8>(y);
|
||||
for &order in &helper::SWAP_ORDERINGS {
|
||||
let a = <$atomic_type>::new(x);
|
||||
assert_eq!(a.fetch_byte_add(y.addr(), order), x);
|
||||
assert_eq!(
|
||||
a.load(Ordering::Relaxed).addr(), x.addr().wrapping_add(y.addr())
|
||||
);
|
||||
let a = <$atomic_type>::new(y);
|
||||
assert_eq!(a.fetch_byte_add(x.addr(), order), y);
|
||||
assert_eq!(
|
||||
a.load(Ordering::Relaxed).addr(), y.addr().wrapping_add(x.addr())
|
||||
);
|
||||
}
|
||||
true
|
||||
}
|
||||
fn quickcheck_fetch_byte_sub(x: usize, y: usize) -> bool {
|
||||
let x = sptr::invalid_mut(x);
|
||||
let y = sptr::invalid_mut::<u8>(y);
|
||||
for &order in &helper::SWAP_ORDERINGS {
|
||||
let a = <$atomic_type>::new(x);
|
||||
assert_eq!(a.fetch_byte_sub(y.addr(), order), x);
|
||||
assert_eq!(
|
||||
a.load(Ordering::Relaxed).addr(), x.addr().wrapping_sub(y.addr())
|
||||
);
|
||||
let a = <$atomic_type>::new(y);
|
||||
assert_eq!(a.fetch_byte_sub(x.addr(), order), y);
|
||||
assert_eq!(
|
||||
a.load(Ordering::Relaxed).addr(), y.addr().wrapping_sub(x.addr())
|
||||
);
|
||||
}
|
||||
true
|
||||
}
|
||||
fn quickcheck_fetch_and(x: usize, y: usize) -> bool {
|
||||
let x = sptr::invalid_mut(x);
|
||||
let y = sptr::invalid_mut::<u8>(y);
|
||||
for &order in &helper::SWAP_ORDERINGS {
|
||||
let a = <$atomic_type>::new(x);
|
||||
assert_eq!(a.fetch_and(y.addr(), order), x);
|
||||
assert_eq!(a.load(Ordering::Relaxed).addr(), x.addr() & y.addr());
|
||||
let a = <$atomic_type>::new(y);
|
||||
assert_eq!(a.fetch_and(x.addr(), order), y);
|
||||
assert_eq!(a.load(Ordering::Relaxed).addr(), y.addr() & x.addr());
|
||||
}
|
||||
true
|
||||
}
|
||||
fn quickcheck_fetch_or(x: usize, y: usize) -> bool {
|
||||
let x = sptr::invalid_mut(x);
|
||||
let y = sptr::invalid_mut::<u8>(y);
|
||||
for &order in &helper::SWAP_ORDERINGS {
|
||||
let a = <$atomic_type>::new(x);
|
||||
assert_eq!(a.fetch_or(y.addr(), order), x);
|
||||
assert_eq!(a.load(Ordering::Relaxed).addr(), x.addr() | y.addr());
|
||||
let a = <$atomic_type>::new(y);
|
||||
assert_eq!(a.fetch_or(x.addr(), order), y);
|
||||
assert_eq!(a.load(Ordering::Relaxed).addr(), y.addr() | x.addr());
|
||||
}
|
||||
true
|
||||
}
|
||||
fn quickcheck_fetch_xor(x: usize, y: usize) -> bool {
|
||||
let x = sptr::invalid_mut(x);
|
||||
let y = sptr::invalid_mut::<u8>(y);
|
||||
for &order in &helper::SWAP_ORDERINGS {
|
||||
let a = <$atomic_type>::new(x);
|
||||
assert_eq!(a.fetch_xor(y.addr(), order), x);
|
||||
assert_eq!(a.load(Ordering::Relaxed).addr(), x.addr() ^ y.addr());
|
||||
let a = <$atomic_type>::new(y);
|
||||
assert_eq!(a.fetch_xor(x.addr(), order), y);
|
||||
assert_eq!(a.load(Ordering::Relaxed).addr(), y.addr() ^ x.addr());
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
macro_rules! test_atomic_int_amo {
|
||||
($int_type:ident) => {
|
||||
paste::paste! {
|
||||
#[allow(
|
||||
clippy::alloc_instead_of_core,
|
||||
clippy::std_instead_of_alloc,
|
||||
clippy::std_instead_of_core,
|
||||
clippy::undocumented_unsafe_blocks
|
||||
)]
|
||||
mod [<test_atomic_ $int_type _amo>] {
|
||||
use super::*;
|
||||
test_atomic_int_amo!([<Atomic $int_type:camel>], $int_type);
|
||||
}
|
||||
}
|
||||
};
|
||||
($atomic_type:ty, $int_type:ident) => {
|
||||
use crate::tests::helper;
|
||||
::quickcheck::quickcheck! {
|
||||
fn quickcheck_swap(x: $int_type, y: $int_type) -> bool {
|
||||
for &order in &helper::SWAP_ORDERINGS {
|
||||
let a = <$atomic_type>::new(x);
|
||||
assert_eq!(a.swap(y, order), x);
|
||||
assert_eq!(a.swap(x, order), y);
|
||||
}
|
||||
true
|
||||
}
|
||||
fn quickcheck_fetch_add(x: $int_type, y: $int_type) -> bool {
|
||||
for &order in &helper::SWAP_ORDERINGS {
|
||||
let a = <$atomic_type>::new(x);
|
||||
assert_eq!(a.fetch_add(y, order), x);
|
||||
assert_eq!(a.load(Ordering::Relaxed), x.wrapping_add(y));
|
||||
let a = <$atomic_type>::new(y);
|
||||
assert_eq!(a.fetch_add(x, order), y);
|
||||
assert_eq!(a.load(Ordering::Relaxed), y.wrapping_add(x));
|
||||
}
|
||||
true
|
||||
}
|
||||
fn quickcheck_fetch_sub(x: $int_type, y: $int_type) -> bool {
|
||||
for &order in &helper::SWAP_ORDERINGS {
|
||||
let a = <$atomic_type>::new(x);
|
||||
assert_eq!(a.fetch_sub(y, order), x);
|
||||
assert_eq!(a.load(Ordering::Relaxed), x.wrapping_sub(y));
|
||||
let a = <$atomic_type>::new(y);
|
||||
assert_eq!(a.fetch_sub(x, order), y);
|
||||
assert_eq!(a.load(Ordering::Relaxed), y.wrapping_sub(x));
|
||||
}
|
||||
true
|
||||
}
|
||||
fn quickcheck_fetch_and(x: $int_type, y: $int_type) -> bool {
|
||||
for &order in &helper::SWAP_ORDERINGS {
|
||||
let a = <$atomic_type>::new(x);
|
||||
assert_eq!(a.fetch_and(y, order), x);
|
||||
assert_eq!(a.load(Ordering::Relaxed), x & y);
|
||||
let a = <$atomic_type>::new(y);
|
||||
assert_eq!(a.fetch_and(x, order), y);
|
||||
assert_eq!(a.load(Ordering::Relaxed), y & x);
|
||||
}
|
||||
true
|
||||
}
|
||||
fn quickcheck_fetch_or(x: $int_type, y: $int_type) -> bool {
|
||||
for &order in &helper::SWAP_ORDERINGS {
|
||||
let a = <$atomic_type>::new(x);
|
||||
assert_eq!(a.fetch_or(y, order), x);
|
||||
assert_eq!(a.load(Ordering::Relaxed), x | y);
|
||||
let a = <$atomic_type>::new(y);
|
||||
assert_eq!(a.fetch_or(x, order), y);
|
||||
assert_eq!(a.load(Ordering::Relaxed), y | x);
|
||||
}
|
||||
true
|
||||
}
|
||||
fn quickcheck_fetch_xor(x: $int_type, y: $int_type) -> bool {
|
||||
for &order in &helper::SWAP_ORDERINGS {
|
||||
let a = <$atomic_type>::new(x);
|
||||
assert_eq!(a.fetch_xor(y, order), x);
|
||||
assert_eq!(a.load(Ordering::Relaxed), x ^ y);
|
||||
let a = <$atomic_type>::new(y);
|
||||
assert_eq!(a.fetch_xor(x, order), y);
|
||||
assert_eq!(a.load(Ordering::Relaxed), y ^ x);
|
||||
}
|
||||
true
|
||||
}
|
||||
fn quickcheck_fetch_max(x: $int_type, y: $int_type) -> bool {
|
||||
for &order in &helper::SWAP_ORDERINGS {
|
||||
let a = <$atomic_type>::new(x);
|
||||
assert_eq!(a.fetch_max(y, order), x);
|
||||
assert_eq!(a.load(Ordering::Relaxed), core::cmp::max(x, y));
|
||||
let a = <$atomic_type>::new(y);
|
||||
assert_eq!(a.fetch_max(x, order), y);
|
||||
assert_eq!(a.load(Ordering::Relaxed), core::cmp::max(y, x));
|
||||
}
|
||||
true
|
||||
}
|
||||
fn quickcheck_fetch_min(x: $int_type, y: $int_type) -> bool {
|
||||
for &order in &helper::SWAP_ORDERINGS {
|
||||
let a = <$atomic_type>::new(x);
|
||||
assert_eq!(a.fetch_min(y, order), x);
|
||||
assert_eq!(a.load(Ordering::Relaxed), core::cmp::min(x, y));
|
||||
let a = <$atomic_type>::new(y);
|
||||
assert_eq!(a.fetch_min(x, order), y);
|
||||
assert_eq!(a.load(Ordering::Relaxed), core::cmp::min(y, x));
|
||||
}
|
||||
true
|
||||
}
|
||||
fn quickcheck_fetch_not(x: $int_type) -> bool {
|
||||
for &order in &helper::SWAP_ORDERINGS {
|
||||
let a = <$atomic_type>::new(x);
|
||||
assert_eq!(a.fetch_not(order), x);
|
||||
assert_eq!(a.load(Ordering::Relaxed), !x);
|
||||
assert_eq!(a.fetch_not(order), !x);
|
||||
assert_eq!(a.load(Ordering::Relaxed), x);
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
macro_rules! test_atomic_int_amo_sub_word {
|
||||
($int_type:ident) => {
|
||||
paste::paste! {
|
||||
#[allow(
|
||||
clippy::alloc_instead_of_core,
|
||||
clippy::std_instead_of_alloc,
|
||||
clippy::std_instead_of_core,
|
||||
clippy::undocumented_unsafe_blocks
|
||||
)]
|
||||
mod [<test_atomic_ $int_type _amo>] {
|
||||
use super::*;
|
||||
#[cfg(any(target_feature = "zabha", portable_atomic_target_feature = "zabha"))]
|
||||
test_atomic_int_amo!([<Atomic $int_type:camel>], $int_type);
|
||||
#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
|
||||
test_atomic_int_amo_sub_word!([<Atomic $int_type:camel>], $int_type);
|
||||
}
|
||||
}
|
||||
};
|
||||
($atomic_type:ty, $int_type:ident) => {
|
||||
use crate::tests::helper::{self, *};
|
||||
::quickcheck::quickcheck! {
|
||||
fn quickcheck_fetch_and(x: $int_type, y: $int_type) -> bool {
|
||||
let mut rng = fastrand::Rng::new();
|
||||
for &order in &helper::SWAP_ORDERINGS {
|
||||
for base in [0, !0] {
|
||||
let mut arr = Align16([
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
]);
|
||||
let a_idx = rng.usize(3..=6);
|
||||
arr.0[a_idx] = <$atomic_type>::new(x);
|
||||
let a = &arr.0[a_idx];
|
||||
assert_eq!(a.fetch_and(y, order), x);
|
||||
assert_eq!(a.load(Ordering::Relaxed), x & y);
|
||||
for i in 0..a_idx {
|
||||
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
|
||||
}
|
||||
for i in a_idx + 1..arr.0.len() {
|
||||
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
|
||||
}
|
||||
arr.0[a_idx] = <$atomic_type>::new(y);
|
||||
let a = &arr.0[a_idx];
|
||||
assert_eq!(a.fetch_and(x, order), y);
|
||||
assert_eq!(a.load(Ordering::Relaxed), y & x);
|
||||
for i in 0..a_idx {
|
||||
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
|
||||
}
|
||||
for i in a_idx + 1..arr.0.len() {
|
||||
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
|
||||
}
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
fn quickcheck_fetch_or(x: $int_type, y: $int_type) -> bool {
|
||||
let mut rng = fastrand::Rng::new();
|
||||
for &order in &helper::SWAP_ORDERINGS {
|
||||
for base in [0, !0] {
|
||||
let mut arr = Align16([
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
]);
|
||||
let a_idx = rng.usize(3..=6);
|
||||
arr.0[a_idx] = <$atomic_type>::new(x);
|
||||
let a = &arr.0[a_idx];
|
||||
assert_eq!(a.fetch_or(y, order), x);
|
||||
assert_eq!(a.load(Ordering::Relaxed), x | y);
|
||||
for i in 0..a_idx {
|
||||
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
|
||||
}
|
||||
for i in a_idx + 1..arr.0.len() {
|
||||
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
|
||||
}
|
||||
arr.0[a_idx] = <$atomic_type>::new(y);
|
||||
let a = &arr.0[a_idx];
|
||||
assert_eq!(a.fetch_or(x, order), y);
|
||||
assert_eq!(a.load(Ordering::Relaxed), y | x);
|
||||
for i in 0..a_idx {
|
||||
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
|
||||
}
|
||||
for i in a_idx + 1..arr.0.len() {
|
||||
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
|
||||
}
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
fn quickcheck_fetch_xor(x: $int_type, y: $int_type) -> bool {
|
||||
let mut rng = fastrand::Rng::new();
|
||||
for &order in &helper::SWAP_ORDERINGS {
|
||||
for base in [0, !0] {
|
||||
let mut arr = Align16([
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
]);
|
||||
let a_idx = rng.usize(3..=6);
|
||||
arr.0[a_idx] = <$atomic_type>::new(x);
|
||||
let a = &arr.0[a_idx];
|
||||
assert_eq!(a.fetch_xor(y, order), x);
|
||||
assert_eq!(a.load(Ordering::Relaxed), x ^ y);
|
||||
for i in 0..a_idx {
|
||||
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
|
||||
}
|
||||
for i in a_idx + 1..arr.0.len() {
|
||||
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
|
||||
}
|
||||
arr.0[a_idx] = <$atomic_type>::new(y);
|
||||
let a = &arr.0[a_idx];
|
||||
assert_eq!(a.fetch_xor(x, order), y);
|
||||
assert_eq!(a.load(Ordering::Relaxed), y ^ x);
|
||||
for i in 0..a_idx {
|
||||
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
|
||||
}
|
||||
for i in a_idx + 1..arr.0.len() {
|
||||
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
|
||||
}
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
fn quickcheck_fetch_not(x: $int_type) -> bool {
|
||||
let mut rng = fastrand::Rng::new();
|
||||
for &order in &helper::SWAP_ORDERINGS {
|
||||
for base in [0, !0] {
|
||||
let mut arr = Align16([
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
<$atomic_type>::new(base),
|
||||
]);
|
||||
let a_idx = rng.usize(3..=6);
|
||||
arr.0[a_idx] = <$atomic_type>::new(x);
|
||||
let a = &arr.0[a_idx];
|
||||
assert_eq!(a.fetch_not(order), x);
|
||||
assert_eq!(a.load(Ordering::Relaxed), !x);
|
||||
assert_eq!(a.fetch_not(order), !x);
|
||||
assert_eq!(a.load(Ordering::Relaxed), x);
|
||||
for i in 0..a_idx {
|
||||
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
|
||||
}
|
||||
for i in a_idx + 1..arr.0.len() {
|
||||
assert_eq!(arr.0[i].load(Ordering::Relaxed), base, "invalid value written");
|
||||
}
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
test_atomic_ptr_amo!();
|
||||
test_atomic_int_amo_sub_word!(i8);
|
||||
test_atomic_int_amo_sub_word!(u8);
|
||||
test_atomic_int_amo_sub_word!(i16);
|
||||
test_atomic_int_amo_sub_word!(u16);
|
||||
test_atomic_int_amo!(i32);
|
||||
test_atomic_int_amo!(u32);
|
||||
#[cfg(target_arch = "riscv64")]
|
||||
test_atomic_int_amo!(i64);
|
||||
#[cfg(target_arch = "riscv64")]
|
||||
test_atomic_int_amo!(u64);
|
||||
test_atomic_int_amo!(isize);
|
||||
test_atomic_int_amo!(usize);
|
||||
}
|
||||
+283
@@ -0,0 +1,283 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
/*
|
||||
Atomic operations implementation on x86/x86_64.
|
||||
|
||||
This module provides atomic operations not supported by LLVM or optimizes
|
||||
cases where LLVM code generation is not optimal.
|
||||
|
||||
Note: On Miri and ThreadSanitizer which do not support inline assembly, we don't use
|
||||
this module and use CAS loop instead.
|
||||
|
||||
Refs:
|
||||
- x86 and amd64 instruction reference https://www.felixcloutier.com/x86
|
||||
|
||||
See tests/asm-test/asm/portable-atomic for generated assembly.
|
||||
*/
|
||||
|
||||
#[cfg(not(portable_atomic_no_asm))]
|
||||
use core::arch::asm;
|
||||
use core::sync::atomic::Ordering;
|
||||
|
||||
#[cfg(target_pointer_width = "32")]
|
||||
macro_rules! ptr_modifier {
|
||||
() => {
|
||||
":e"
|
||||
};
|
||||
}
|
||||
#[cfg(target_pointer_width = "64")]
|
||||
macro_rules! ptr_modifier {
|
||||
() => {
|
||||
""
|
||||
};
|
||||
}
|
||||
|
||||
/// An atomic fence.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `order` is [`Relaxed`](Ordering::Relaxed).
|
||||
#[cfg_attr(doc, allow(dead_code))]
|
||||
#[inline]
|
||||
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
|
||||
pub fn fence(order: Ordering) {
|
||||
if order == Ordering::SeqCst {
|
||||
let p = core::cell::UnsafeCell::new(core::mem::MaybeUninit::<usize>::uninit());
|
||||
// SAFETY: raw pointer passed in is valid because we got it from a reference.
|
||||
unsafe {
|
||||
// Equivalent to `mfence`, but is up to 3.1x faster on Coffee Lake and up to 2.4x faster on Raptor Lake-H at least in simple cases.
|
||||
// - https://github.com/taiki-e/portable-atomic/pull/156
|
||||
// - LLVM uses `lock or` https://godbolt.org/z/vv6rjzfYd
|
||||
// - Windows uses `xchg` for x86_32 for MemoryBarrier https://learn.microsoft.com/en-us/windows/win32/api/winnt/nf-winnt-memorybarrier
|
||||
// - MSVC STL uses `lock inc` https://github.com/microsoft/STL/pull/740
|
||||
// - boost uses `lock or` https://github.com/boostorg/atomic/commit/559eba81af71386cedd99f170dc6101c6ad7bf22
|
||||
#[cfg(target_pointer_width = "64")]
|
||||
asm!(
|
||||
concat!("xchg qword ptr [{p", ptr_modifier!(), "}], {tmp}"),
|
||||
p = in(reg) p.get(),
|
||||
tmp = out(reg) _,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
#[cfg(target_pointer_width = "32")]
|
||||
asm!(
|
||||
concat!("xchg dword ptr [{p", ptr_modifier!(), "}], {tmp:e}"),
|
||||
p = in(reg) p.get(),
|
||||
tmp = out(reg) _,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
}
|
||||
} else {
|
||||
core::sync::atomic::fence(order);
|
||||
}
|
||||
}
|
||||
|
||||
cfg_core_atomic!({
|
||||
use super::core_atomic::{
|
||||
AtomicI8, AtomicI16, AtomicI32, AtomicI64, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16,
|
||||
AtomicU32, AtomicU64, AtomicUsize,
|
||||
};
|
||||
|
||||
macro_rules! atomic_int {
|
||||
($atomic_type:ident, $ptr_size:tt) => {
|
||||
impl $atomic_type {
|
||||
#[inline]
|
||||
pub(crate) fn not(&self, _order: Ordering) {
|
||||
let dst = self.as_ptr();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
//
|
||||
// https://www.felixcloutier.com/x86/not
|
||||
unsafe {
|
||||
// atomic RMW is always SeqCst.
|
||||
asm!(
|
||||
concat!("lock not ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}]"),
|
||||
dst = in(reg) dst,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn neg(&self, _order: Ordering) {
|
||||
let dst = self.as_ptr();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
//
|
||||
// https://www.felixcloutier.com/x86/neg
|
||||
unsafe {
|
||||
// atomic RMW is always SeqCst.
|
||||
asm!(
|
||||
concat!("lock neg ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}]"),
|
||||
dst = in(reg) dst,
|
||||
// Do not use `preserves_flags` because NEG modifies the CF, OF, SF, ZF, AF, and PF flag.
|
||||
options(nostack),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
atomic_int!(AtomicI8, "byte");
|
||||
atomic_int!(AtomicU8, "byte");
|
||||
atomic_int!(AtomicI16, "word");
|
||||
atomic_int!(AtomicU16, "word");
|
||||
atomic_int!(AtomicI32, "dword");
|
||||
atomic_int!(AtomicU32, "dword");
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
atomic_int!(AtomicI64, "qword");
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
atomic_int!(AtomicU64, "qword");
|
||||
#[cfg(target_pointer_width = "32")]
|
||||
atomic_int!(AtomicIsize, "dword");
|
||||
#[cfg(target_pointer_width = "32")]
|
||||
atomic_int!(AtomicUsize, "dword");
|
||||
#[cfg(target_pointer_width = "64")]
|
||||
atomic_int!(AtomicIsize, "qword");
|
||||
#[cfg(target_pointer_width = "64")]
|
||||
atomic_int!(AtomicUsize, "qword");
|
||||
|
||||
#[cfg(target_arch = "x86")]
|
||||
impl AtomicI64 {
|
||||
#[inline]
|
||||
pub(crate) fn not(&self, order: Ordering) {
|
||||
self.fetch_not(order);
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn neg(&self, order: Ordering) {
|
||||
self.fetch_neg(order);
|
||||
}
|
||||
}
|
||||
#[cfg(target_arch = "x86")]
|
||||
impl AtomicU64 {
|
||||
#[inline]
|
||||
pub(crate) fn not(&self, order: Ordering) {
|
||||
self.fetch_not(order);
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn neg(&self, order: Ordering) {
|
||||
self.fetch_neg(order);
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! atomic_bit_opts {
|
||||
(
|
||||
$([$($generics:tt)*])? $atomic_type:ident, $int_type:ident, $val_modifier:tt,
|
||||
$ptr_size:tt
|
||||
) => {
|
||||
// LLVM 14 and older don't support generating `lock bt{s,r,c}`.
|
||||
// LLVM 15 only supports generating `lock bt{s,r,c}` for immediate bit offsets.
|
||||
// LLVM 16+ can generate `lock bt{s,r,c}` for both immediate and register bit offsets.
|
||||
// https://godbolt.org/z/TGhr5z4ds
|
||||
// So, use fetch_* based implementations on LLVM 16+, otherwise use asm based implementations.
|
||||
#[cfg(not(portable_atomic_pre_llvm_16))]
|
||||
impl_default_bit_opts!($atomic_type, $int_type);
|
||||
#[cfg(portable_atomic_pre_llvm_16)]
|
||||
impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {
|
||||
// `<integer>::BITS` requires Rust 1.53
|
||||
const BITS: u32 = (core::mem::size_of::<$int_type>() * 8) as u32;
|
||||
#[inline]
|
||||
pub(crate) fn bit_set(&self, bit: u32, _order: Ordering) -> bool {
|
||||
let dst = self.as_ptr();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
// the masking by the bit size of the type ensures that we do not shift
|
||||
// out of bounds.
|
||||
//
|
||||
// https://www.felixcloutier.com/x86/bts
|
||||
unsafe {
|
||||
let r: u8;
|
||||
// atomic RMW is always SeqCst.
|
||||
asm!(
|
||||
concat!("lock bts ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}], {bit", $val_modifier, "}"),
|
||||
"setb {r}",
|
||||
dst = in(reg) dst,
|
||||
bit = in(reg) (bit & (Self::BITS - 1)) as $int_type,
|
||||
r = out(reg_byte) r,
|
||||
// Do not use `preserves_flags` because BTS modifies the CF flag.
|
||||
options(nostack),
|
||||
);
|
||||
crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
|
||||
r != 0
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn bit_clear(&self, bit: u32, _order: Ordering) -> bool {
|
||||
let dst = self.as_ptr();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
// the masking by the bit size of the type ensures that we do not shift
|
||||
// out of bounds.
|
||||
//
|
||||
// https://www.felixcloutier.com/x86/btr
|
||||
unsafe {
|
||||
let r: u8;
|
||||
// atomic RMW is always SeqCst.
|
||||
asm!(
|
||||
concat!("lock btr ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}], {bit", $val_modifier, "}"),
|
||||
"setb {r}",
|
||||
dst = in(reg) dst,
|
||||
bit = in(reg) (bit & (Self::BITS - 1)) as $int_type,
|
||||
r = out(reg_byte) r,
|
||||
// Do not use `preserves_flags` because BTR modifies the CF flag.
|
||||
options(nostack),
|
||||
);
|
||||
crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
|
||||
r != 0
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
pub(crate) fn bit_toggle(&self, bit: u32, _order: Ordering) -> bool {
|
||||
let dst = self.as_ptr();
|
||||
// SAFETY: any data races are prevented by atomic intrinsics and the raw
|
||||
// pointer passed in is valid because we got it from a reference.
|
||||
// the masking by the bit size of the type ensures that we do not shift
|
||||
// out of bounds.
|
||||
//
|
||||
// https://www.felixcloutier.com/x86/btc
|
||||
unsafe {
|
||||
let r: u8;
|
||||
// atomic RMW is always SeqCst.
|
||||
asm!(
|
||||
concat!("lock btc ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}], {bit", $val_modifier, "}"),
|
||||
"setb {r}",
|
||||
dst = in(reg) dst,
|
||||
bit = in(reg) (bit & (Self::BITS - 1)) as $int_type,
|
||||
r = out(reg_byte) r,
|
||||
// Do not use `preserves_flags` because BTC modifies the CF flag.
|
||||
options(nostack),
|
||||
);
|
||||
crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
|
||||
r != 0
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
impl_default_bit_opts!(AtomicI8, i8);
|
||||
impl_default_bit_opts!(AtomicU8, u8);
|
||||
atomic_bit_opts!(AtomicI16, i16, ":x", "word");
|
||||
atomic_bit_opts!(AtomicU16, u16, ":x", "word");
|
||||
atomic_bit_opts!(AtomicI32, i32, ":e", "dword");
|
||||
atomic_bit_opts!(AtomicU32, u32, ":e", "dword");
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
atomic_bit_opts!(AtomicI64, i64, "", "qword");
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
atomic_bit_opts!(AtomicU64, u64, "", "qword");
|
||||
#[cfg(target_arch = "x86")]
|
||||
impl_default_bit_opts!(AtomicI64, i64);
|
||||
#[cfg(target_arch = "x86")]
|
||||
impl_default_bit_opts!(AtomicU64, u64);
|
||||
#[cfg(target_pointer_width = "32")]
|
||||
atomic_bit_opts!(AtomicIsize, isize, ":e", "dword");
|
||||
#[cfg(target_pointer_width = "32")]
|
||||
atomic_bit_opts!(AtomicUsize, usize, ":e", "dword");
|
||||
#[cfg(target_pointer_width = "32")]
|
||||
atomic_bit_opts!([T] AtomicPtr, usize, ":e", "dword");
|
||||
#[cfg(target_pointer_width = "64")]
|
||||
atomic_bit_opts!(AtomicIsize, isize, "", "qword");
|
||||
#[cfg(target_pointer_width = "64")]
|
||||
atomic_bit_opts!(AtomicUsize, usize, "", "qword");
|
||||
#[cfg(target_pointer_width = "64")]
|
||||
atomic_bit_opts!([T] AtomicPtr, usize, "", "qword");
|
||||
});
|
||||
Vendored
+5224
File diff suppressed because it is too large.
Load diff
+92
@@ -0,0 +1,92 @@
|
||||
/*
|
||||
* Adapted from https://github.com/TheBevyFlock/bevy_cli/blob/d3e79396595f675798af5be8be56efbfd2737ba0/bevy_lint/assets/rustdoc.css.
|
||||
* Adjusted to work well with our GFM (GitHub Flavored Markdown) to CommonMark converter.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Extended theme configuration.
|
||||
*
|
||||
* These colors are based off Github's (blue for notes, green for tips, etc.), though the exact
|
||||
* colors are taken from `rustdoc`'s themes.
|
||||
*/
|
||||
:root[data-theme="light"] {
|
||||
--rustdoc-alert-note: #2196f3;
|
||||
--rustdoc-alert-tip: #068000;
|
||||
--rustdoc-alert-important: #6e4fc9;
|
||||
--rustdoc-alert-warning: #ff8e00;
|
||||
--rustdoc-alert-caution: #be2525;
|
||||
}
|
||||
:root[data-theme="dark"] {
|
||||
--rustdoc-alert-note: #008dfd;
|
||||
--rustdoc-alert-tip: #2bab63;
|
||||
--rustdoc-alert-important: #b78cf2;
|
||||
--rustdoc-alert-warning: #ff8e00;
|
||||
--rustdoc-alert-caution: #d93d3d;
|
||||
}
|
||||
:root[data-theme="ayu"] {
|
||||
--rustdoc-alert-note: #39afd7;
|
||||
--rustdoc-alert-tip: #b8cc52;
|
||||
--rustdoc-alert-important: #a37acc;
|
||||
--rustdoc-alert-warning: #ff8e00;
|
||||
--rustdoc-alert-caution: #df3600;
|
||||
}
|
||||
|
||||
/*
|
||||
* Alerts are siblings to the `.warning` class, and as such use many of the same rules.
|
||||
*
|
||||
* The first `<p>` in a alert must be the title, as it will be colored and styled differently.
|
||||
*/
|
||||
.rustdoc-alert {
|
||||
border-left: 2px solid;
|
||||
margin: 0 0 0.75em 0;
|
||||
position: relative;
|
||||
overflow-x: visible !important;
|
||||
}
|
||||
|
||||
/* Remove the existing quote margin and setup some padding instead. */
|
||||
.rustdoc-alert blockquote {
|
||||
margin: 0;
|
||||
padding: 14px;
|
||||
}
|
||||
|
||||
/* Make the alert title the same sans-serif font and size as headings. */
|
||||
.rustdoc-alert blockquote > p:first-child {
|
||||
font-family: "Fira Sans", Arial, sans-serif;
|
||||
font-size: 1.125rem;
|
||||
}
|
||||
|
||||
/* Color styles for different kinds of alerts. */
|
||||
.rustdoc-alert.rustdoc-alert-note {
|
||||
border-left-color: var(--rustdoc-alert-note);
|
||||
}
|
||||
.rustdoc-alert.rustdoc-alert-note blockquote > p:first-child {
|
||||
color: var(--rustdoc-alert-note);
|
||||
}
|
||||
|
||||
.rustdoc-alert.rustdoc-alert-tip {
|
||||
border-left-color: var(--rustdoc-alert-tip);
|
||||
}
|
||||
.rustdoc-alert.rustdoc-alert-tip blockquote > p:first-child {
|
||||
color: var(--rustdoc-alert-tip);
|
||||
}
|
||||
|
||||
.rustdoc-alert.rustdoc-alert-important {
|
||||
border-left-color: var(--rustdoc-alert-important);
|
||||
}
|
||||
.rustdoc-alert.rustdoc-alert-important blockquote > p:first-child {
|
||||
color: var(--rustdoc-alert-important);
|
||||
}
|
||||
|
||||
.rustdoc-alert.rustdoc-alert-warning {
|
||||
border-left-color: var(--rustdoc-alert-warning);
|
||||
}
|
||||
.rustdoc-alert.rustdoc-alert-warning blockquote > p:first-child {
|
||||
color: var(--rustdoc-alert-warning);
|
||||
}
|
||||
|
||||
.rustdoc-alert.rustdoc-alert-caution {
|
||||
border-left-color: var(--rustdoc-alert-caution);
|
||||
}
|
||||
.rustdoc-alert.rustdoc-alert-caution blockquote > p:first-child {
|
||||
color: var(--rustdoc-alert-caution);
|
||||
}
|
||||
+3294
File diff suppressed because it is too large.
Load diff
+499
@@ -0,0 +1,499 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
#![allow(
|
||||
clippy::alloc_instead_of_core,
|
||||
clippy::std_instead_of_alloc,
|
||||
clippy::std_instead_of_core,
|
||||
clippy::undocumented_unsafe_blocks,
|
||||
clippy::wildcard_imports
|
||||
)]
|
||||
|
||||
#[macro_use]
|
||||
pub(crate) mod helper;
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[path = "../../version.rs"]
|
||||
mod version;
|
||||
|
||||
use test_helper as _; // For critical-section test
|
||||
|
||||
use super::*;
|
||||
|
||||
test_atomic_bool_pub!();
|
||||
test_atomic_ptr_pub!();
|
||||
|
||||
test_atomic_int_pub!(isize);
|
||||
test_atomic_int_pub!(usize);
|
||||
test_atomic_int_pub!(i8);
|
||||
test_atomic_int_pub!(u8);
|
||||
test_atomic_int_pub!(i16);
|
||||
test_atomic_int_pub!(u16);
|
||||
test_atomic_int_pub!(i32);
|
||||
test_atomic_int_pub!(u32);
|
||||
test_atomic_int_pub!(i64);
|
||||
test_atomic_int_pub!(u64);
|
||||
#[cfg(not(all(valgrind, target_arch = "powerpc64")))] // TODO(powerpc64): Hang (as of Valgrind 3.26)
|
||||
test_atomic_int_pub!(i128);
|
||||
#[cfg(not(all(valgrind, target_arch = "powerpc64")))] // TODO(powerpc64): Hang (as of Valgrind 3.26)
|
||||
test_atomic_int_pub!(u128);
|
||||
|
||||
#[cfg(all(feature = "float", portable_atomic_unstable_f16))]
|
||||
test_atomic_float_pub!(f16);
|
||||
#[cfg(feature = "float")]
|
||||
test_atomic_float_pub!(f32);
|
||||
#[cfg(feature = "float")]
|
||||
// TODO: rustc bug: https://github.com/rust-lang/rust/issues/114479
|
||||
#[cfg(not(all(not(debug_assertions), target_arch = "x86", not(target_feature = "sse2"))))]
|
||||
test_atomic_float_pub!(f64);
|
||||
#[cfg(all(feature = "float", portable_atomic_unstable_f128))]
|
||||
test_atomic_float_pub!(f128);
|
||||
|
||||
#[deny(improper_ctypes)]
|
||||
extern "C" {
|
||||
fn _atomic_bool_ffi_safety(_: AtomicBool);
|
||||
fn _atomic_ptr_ffi_safety(_: AtomicPtr<u8>);
|
||||
fn _atomic_isize_ffi_safety(_: AtomicIsize);
|
||||
fn _atomic_usize_ffi_safety(_: AtomicUsize);
|
||||
fn _atomic_i8_ffi_safety(_: AtomicI8);
|
||||
fn _atomic_u8_ffi_safety(_: AtomicU8);
|
||||
fn _atomic_i16_ffi_safety(_: AtomicI16);
|
||||
fn _atomic_u16_ffi_safety(_: AtomicU16);
|
||||
fn _atomic_i32_ffi_safety(_: AtomicI32);
|
||||
fn _atomic_u32_ffi_safety(_: AtomicU32);
|
||||
fn _atomic_i64_ffi_safety(_: AtomicI64);
|
||||
fn _atomic_u64_ffi_safety(_: AtomicU64);
|
||||
#[rustversion::since(1.89)] // https://github.com/rust-lang/rust/pull/137306
|
||||
fn _atomic_i128_ffi_safety(_: AtomicI128);
|
||||
#[rustversion::since(1.89)] // https://github.com/rust-lang/rust/pull/137306
|
||||
fn _atomic_u128_ffi_safety(_: AtomicU128);
|
||||
#[cfg(all(feature = "float", portable_atomic_unstable_f16))]
|
||||
fn _atomic_f16_ffi_safety(_: AtomicF16);
|
||||
#[cfg(feature = "float")]
|
||||
fn _atomic_f32_ffi_safety(_: AtomicF32);
|
||||
#[cfg(feature = "float")]
|
||||
fn _atomic_f64_ffi_safety(_: AtomicF64);
|
||||
#[cfg(all(feature = "float", portable_atomic_unstable_f128))]
|
||||
fn _atomic_f128_ffi_safety(_: AtomicF128);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_lock_free() {
|
||||
assert!(AtomicI8::is_always_lock_free());
|
||||
assert!(AtomicI8::is_lock_free());
|
||||
assert!(AtomicU8::is_always_lock_free());
|
||||
assert!(AtomicU8::is_lock_free());
|
||||
assert!(AtomicI16::is_always_lock_free());
|
||||
assert!(AtomicI16::is_lock_free());
|
||||
assert!(AtomicU16::is_always_lock_free());
|
||||
assert!(AtomicU16::is_lock_free());
|
||||
#[cfg(all(feature = "float", portable_atomic_unstable_f16))]
|
||||
assert!(AtomicF16::is_always_lock_free());
|
||||
#[cfg(all(feature = "float", portable_atomic_unstable_f16))]
|
||||
assert!(AtomicF16::is_lock_free());
|
||||
assert!(AtomicI32::is_always_lock_free());
|
||||
assert!(AtomicI32::is_lock_free());
|
||||
assert!(AtomicU32::is_always_lock_free());
|
||||
assert!(AtomicU32::is_lock_free());
|
||||
#[cfg(feature = "float")]
|
||||
assert!(AtomicF32::is_always_lock_free());
|
||||
#[cfg(feature = "float")]
|
||||
assert!(AtomicF32::is_lock_free());
|
||||
#[cfg(not(portable_atomic_no_cfg_target_has_atomic))]
|
||||
{
|
||||
if cfg!(any(
|
||||
target_has_atomic = "64",
|
||||
all(
|
||||
target_arch = "riscv32",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
|
||||
),
|
||||
)) {
|
||||
assert!(AtomicI64::is_always_lock_free());
|
||||
assert!(AtomicI64::is_lock_free());
|
||||
assert!(AtomicU64::is_always_lock_free());
|
||||
assert!(AtomicU64::is_lock_free());
|
||||
#[cfg(feature = "float")]
|
||||
assert!(AtomicF64::is_always_lock_free());
|
||||
#[cfg(feature = "float")]
|
||||
assert!(AtomicF64::is_lock_free());
|
||||
} else if cfg!(all(
|
||||
feature = "fallback",
|
||||
target_arch = "arm",
|
||||
not(any(miri, portable_atomic_sanitize_thread)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
any(target_os = "linux", target_os = "android"),
|
||||
not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
not(target_has_atomic = "64"),
|
||||
not(portable_atomic_test_detect_false),
|
||||
)) {
|
||||
assert!(!AtomicI64::is_always_lock_free());
|
||||
assert!(!AtomicU64::is_always_lock_free());
|
||||
#[cfg(feature = "float")]
|
||||
assert!(!AtomicF64::is_always_lock_free());
|
||||
assert!(AtomicI64::is_lock_free());
|
||||
assert!(AtomicU64::is_lock_free());
|
||||
#[cfg(feature = "float")]
|
||||
assert!(AtomicF64::is_lock_free());
|
||||
} else {
|
||||
assert!(!AtomicI64::is_always_lock_free());
|
||||
assert!(!AtomicU64::is_always_lock_free());
|
||||
#[cfg(feature = "float")]
|
||||
assert!(!AtomicF64::is_always_lock_free());
|
||||
#[cfg(not(target_arch = "riscv32"))]
|
||||
{
|
||||
assert!(!AtomicI64::is_lock_free());
|
||||
assert!(!AtomicU64::is_lock_free());
|
||||
#[cfg(feature = "float")]
|
||||
assert!(!AtomicF64::is_lock_free());
|
||||
}
|
||||
#[cfg(target_arch = "riscv32")]
|
||||
{
|
||||
// TODO(riscv): check detect.has_zacas
|
||||
}
|
||||
}
|
||||
}
|
||||
if cfg!(portable_atomic_no_asm) && cfg!(not(portable_atomic_unstable_asm)) {
|
||||
assert!(!AtomicI128::is_always_lock_free());
|
||||
assert!(!AtomicI128::is_lock_free());
|
||||
assert!(!AtomicU128::is_always_lock_free());
|
||||
assert!(!AtomicU128::is_lock_free());
|
||||
#[cfg(all(feature = "float", portable_atomic_unstable_f128))]
|
||||
assert!(!AtomicF128::is_always_lock_free());
|
||||
#[cfg(all(feature = "float", portable_atomic_unstable_f128))]
|
||||
assert!(!AtomicF128::is_lock_free());
|
||||
} else if cfg!(any(
|
||||
all(
|
||||
target_arch = "aarch64",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
|
||||
),
|
||||
all(
|
||||
target_arch = "arm64ec",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
not(portable_atomic_no_asm),
|
||||
),
|
||||
all(
|
||||
target_arch = "x86_64",
|
||||
any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
|
||||
),
|
||||
all(
|
||||
target_arch = "riscv64",
|
||||
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
|
||||
),
|
||||
all(
|
||||
target_arch = "powerpc64",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
not(portable_atomic_no_asm),
|
||||
any(
|
||||
target_feature = "quadword-atomics",
|
||||
portable_atomic_target_feature = "quadword-atomics",
|
||||
),
|
||||
),
|
||||
all(
|
||||
target_arch = "s390x",
|
||||
not(all(
|
||||
any(miri, portable_atomic_sanitize_thread),
|
||||
not(portable_atomic_atomic_intrinsics),
|
||||
)),
|
||||
not(portable_atomic_no_asm),
|
||||
),
|
||||
)) {
|
||||
assert!(AtomicI128::is_always_lock_free());
|
||||
assert!(AtomicI128::is_lock_free());
|
||||
assert!(AtomicU128::is_always_lock_free());
|
||||
assert!(AtomicU128::is_lock_free());
|
||||
#[cfg(all(feature = "float", portable_atomic_unstable_f128))]
|
||||
assert!(AtomicF128::is_always_lock_free());
|
||||
#[cfg(all(feature = "float", portable_atomic_unstable_f128))]
|
||||
assert!(AtomicF128::is_lock_free());
|
||||
} else {
|
||||
assert!(!AtomicI128::is_always_lock_free());
|
||||
assert!(!AtomicU128::is_always_lock_free());
|
||||
#[cfg(all(feature = "float", portable_atomic_unstable_f128))]
|
||||
assert!(!AtomicF128::is_always_lock_free());
|
||||
#[cfg(not(any(
|
||||
target_arch = "x86_64",
|
||||
target_arch = "powerpc64",
|
||||
target_arch = "riscv64",
|
||||
)))]
|
||||
{
|
||||
assert!(!AtomicI128::is_lock_free());
|
||||
assert!(!AtomicU128::is_lock_free());
|
||||
#[cfg(all(feature = "float", portable_atomic_unstable_f128))]
|
||||
assert!(!AtomicF128::is_lock_free());
|
||||
}
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
{
|
||||
let has_cmpxchg16b = cfg!(all(
|
||||
feature = "fallback",
|
||||
not(portable_atomic_no_outline_atomics),
|
||||
not(any(target_env = "sgx", miri)),
|
||||
not(portable_atomic_test_detect_false),
|
||||
)) && std::is_x86_feature_detected!("cmpxchg16b");
|
||||
assert_eq!(AtomicI128::is_lock_free(), has_cmpxchg16b);
|
||||
assert_eq!(AtomicU128::is_lock_free(), has_cmpxchg16b);
|
||||
#[cfg(all(feature = "float", portable_atomic_unstable_f128))]
|
||||
assert_eq!(AtomicF128::is_lock_free(), has_cmpxchg16b);
|
||||
}
|
||||
#[cfg(target_arch = "powerpc64")]
|
||||
{
|
||||
// TODO(powerpc64): is_powerpc_feature_detected is unstable
|
||||
}
|
||||
#[cfg(target_arch = "riscv64")]
|
||||
{
|
||||
// TODO(riscv): check detect.has_zacas
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// test version parsing code used in the build script.
|
||||
#[test]
|
||||
fn test_rustc_version() {
|
||||
use self::version::Version;
|
||||
|
||||
// rustc 1.34 (rustup)
|
||||
let v = Version::parse(
|
||||
"rustc 1.34.2 (6c2484dc3 2019-05-13)
|
||||
binary: rustc
|
||||
commit-hash: 6c2484dc3c532c052f159264e970278d8b77cdc9
|
||||
commit-date: 2019-05-13
|
||||
host: x86_64-apple-darwin
|
||||
release: 1.34.2
|
||||
LLVM version: 8.0",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(v, Version::stable(34, 8));
|
||||
|
||||
// rustc 1.50 (rustup)
|
||||
let v = Version::parse(
|
||||
"rustc 1.50.0 (cb75ad5db 2021-02-10)
|
||||
binary: rustc
|
||||
commit-hash: cb75ad5db02783e8b0222fee363c5f63f7e2cf5b
|
||||
commit-date: 2021-02-10
|
||||
host: aarch64-unknown-linux-gnu
|
||||
release: 1.50.0",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(v, Version::stable(50, 0));
|
||||
|
||||
// rustc 1.67 (rustup)
|
||||
let v = Version::parse(
|
||||
"rustc 1.67.0 (fc594f156 2023-01-24)
|
||||
binary: rustc
|
||||
commit-hash: fc594f15669680fa70d255faec3ca3fb507c3405
|
||||
commit-date: 2023-01-24
|
||||
host: aarch64-apple-darwin
|
||||
release: 1.67.0
|
||||
LLVM version: 15.0.6",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(v, Version::stable(67, 15));
|
||||
|
||||
// rustc 1.68-beta (rustup)
|
||||
let v = Version::parse(
|
||||
"rustc 1.68.0-beta.2 (10b73bf73 2023-02-01)
|
||||
binary: rustc
|
||||
commit-hash: 10b73bf73a6b770cd92ad8ff538173bc3298411c
|
||||
commit-date: 2023-02-01
|
||||
host: aarch64-apple-darwin
|
||||
release: 1.68.0-beta.2
|
||||
LLVM version: 15.0.6",
|
||||
)
|
||||
.unwrap();
|
||||
// We do not distinguish between stable and beta because we are only
|
||||
// interested in whether unstable features are potentially available.
|
||||
assert_eq!(v, Version::stable(68, 15));
|
||||
|
||||
// rustc nightly-2019-01-27 (rustup)
|
||||
let v = Version::parse(
|
||||
"rustc 1.33.0-nightly (20c2cba61 2019-01-26)
|
||||
binary: rustc
|
||||
commit-hash: 20c2cba61dc83e612d25ed496025171caa3db30f
|
||||
commit-date: 2019-01-26
|
||||
host: x86_64-apple-darwin
|
||||
release: 1.33.0-nightly
|
||||
LLVM version: 8.0",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(v.minor, 33);
|
||||
assert!(v.nightly);
|
||||
assert_eq!(v.llvm, 8);
|
||||
assert_eq!(v.commit_date().year, 2019);
|
||||
assert_eq!(v.commit_date().month, 1);
|
||||
assert_eq!(v.commit_date().day, 26);
|
||||
|
||||
// rustc 1.69-nightly (rustup)
|
||||
let v = Version::parse(
|
||||
"rustc 1.69.0-nightly (bd39bbb4b 2023-02-07)
|
||||
binary: rustc
|
||||
commit-hash: bd39bbb4bb92df439bf6d85470e296cc6a47ffbd
|
||||
commit-date: 2023-02-07
|
||||
host: aarch64-apple-darwin
|
||||
release: 1.69.0-nightly
|
||||
LLVM version: 15.0.7",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(v.minor, 69);
|
||||
assert!(v.nightly);
|
||||
assert_eq!(v.llvm, 15);
|
||||
assert_eq!(v.commit_date().year, 2023);
|
||||
assert_eq!(v.commit_date().month, 2);
|
||||
assert_eq!(v.commit_date().day, 7);
|
||||
|
||||
// clippy-driver 1.69-nightly (rustup)
|
||||
let v = Version::parse(
|
||||
"rustc 1.69.0-nightly (bd39bbb4b 2023-02-07)
|
||||
binary: rustc
|
||||
commit-hash: bd39bbb4bb92df439bf6d85470e296cc6a47ffbd
|
||||
commit-date: 2023-02-07
|
||||
host: aarch64-apple-darwin
|
||||
release: 1.69.0-nightly
|
||||
LLVM version: 15.0.7",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(v.minor, 69);
|
||||
assert!(v.nightly);
|
||||
assert_eq!(v.llvm, 15);
|
||||
assert_eq!(v.commit_date().year, 2023);
|
||||
assert_eq!(v.commit_date().month, 2);
|
||||
assert_eq!(v.commit_date().day, 7);
|
||||
|
||||
// rustc 1.69-dev (from source: ./x.py build)
|
||||
let v = Version::parse(
|
||||
"rustc 1.69.0-dev
|
||||
binary: rustc
|
||||
commit-hash: unknown
|
||||
commit-date: unknown
|
||||
host: aarch64-unknown-linux-gnu
|
||||
release: 1.69.0-dev
|
||||
LLVM version: 16.0.0",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(v.minor, 69);
|
||||
assert!(v.nightly);
|
||||
assert_eq!(v.llvm, 16);
|
||||
assert_eq!(v.commit_date().year, 0);
|
||||
assert_eq!(v.commit_date().month, 0);
|
||||
assert_eq!(v.commit_date().day, 0);
|
||||
|
||||
// rustc 1.48 (debian 11: apt-get install cargo)
|
||||
let v = Version::parse(
|
||||
"rustc 1.48.0
|
||||
binary: rustc
|
||||
commit-hash: unknown
|
||||
commit-date: unknown
|
||||
host: aarch64-unknown-linux-gnu
|
||||
release: 1.48.0
|
||||
LLVM version: 11.0",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(v, Version::stable(48, 11));
|
||||
|
||||
// rustc 1.67 (fedora: dnf install cargo)
|
||||
let v = Version::parse(
|
||||
"rustc 1.67.0 (fc594f156 2023-01-24) (Fedora 1.67.0-2.fc37)
|
||||
binary: rustc
|
||||
commit-hash: fc594f15669680fa70d255faec3ca3fb507c3405
|
||||
commit-date: 2023-01-24
|
||||
host: aarch64-unknown-linux-gnu
|
||||
release: 1.67.0
|
||||
LLVM version: 15.0.7",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(v, Version::stable(67, 15));
|
||||
|
||||
// rustc 1.64 (alpine: apk add cargo)
|
||||
let v = Version::parse(
|
||||
"rustc 1.64.0
|
||||
binary: rustc
|
||||
commit-hash: unknown
|
||||
commit-date: unknown
|
||||
host: aarch64-alpine-linux-musl
|
||||
release: 1.64.0
|
||||
LLVM version: 15.0.3",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(v, Version::stable(64, 15));
|
||||
}
|
||||
|
||||
#[cfg(feature = "serde")]
|
||||
#[test]
|
||||
fn test_serde() {
|
||||
use std::{eprint, eprintln, fmt};
|
||||
|
||||
use serde::{
|
||||
de::{Deserialize, Deserializer},
|
||||
ser::{Serialize, Serializer},
|
||||
};
|
||||
use serde_test::{Token, assert_tokens};
|
||||
|
||||
#[derive(Debug)]
|
||||
struct DebugPartialEq<T>(T);
|
||||
impl<T: fmt::Debug> PartialEq for DebugPartialEq<T> {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
std::format!("{:?}", self) == std::format!("{:?}", other)
|
||||
}
|
||||
}
|
||||
impl<T: Serialize> Serialize for DebugPartialEq<T> {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
self.0.serialize(serializer)
|
||||
}
|
||||
}
|
||||
impl<'de, T: Deserialize<'de>> Deserialize<'de> for DebugPartialEq<T> {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
T::deserialize(deserializer).map(Self)
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! t {
|
||||
($atomic_type:ty, $value_type:ident $(as $token_value_type:ident)?, $token_type:ident) => {
|
||||
eprint!("test_serde {} ... ", stringify!($value_type));
|
||||
assert_tokens(&DebugPartialEq(<$atomic_type>::new($value_type::MAX)), &[
|
||||
Token::$token_type($value_type::MAX $(as $token_value_type)?),
|
||||
]);
|
||||
assert_tokens(&DebugPartialEq(<$atomic_type>::new($value_type::MIN)), &[
|
||||
Token::$token_type($value_type::MIN $(as $token_value_type)?),
|
||||
]);
|
||||
eprintln!("ok");
|
||||
};
|
||||
}
|
||||
|
||||
assert_tokens(&DebugPartialEq(AtomicBool::new(true)), &[Token::Bool(true)]);
|
||||
assert_tokens(&DebugPartialEq(AtomicBool::new(false)), &[Token::Bool(false)]);
|
||||
t!(AtomicIsize, isize as i64, I64);
|
||||
t!(AtomicUsize, usize as u64, U64);
|
||||
t!(AtomicI8, i8, I8);
|
||||
t!(AtomicU8, u8, U8);
|
||||
t!(AtomicI16, i16, I16);
|
||||
t!(AtomicU16, u16, U16);
|
||||
t!(AtomicI32, i32, I32);
|
||||
t!(AtomicU32, u32, U32);
|
||||
t!(AtomicI64, i64, I64);
|
||||
t!(AtomicU64, u64, U64);
|
||||
#[cfg(not(all(valgrind, target_arch = "powerpc64")))] // TODO(powerpc64): Hang (as of Valgrind 3.26)
|
||||
t!(AtomicI128, i128, I128);
|
||||
#[cfg(not(all(valgrind, target_arch = "powerpc64")))] // TODO(powerpc64): Hang (as of Valgrind 3.26)
|
||||
t!(AtomicU128, u128, U128);
|
||||
// TODO(f16_and_f128): Test f16 & f128 once stabilized.
|
||||
#[cfg(feature = "float")]
|
||||
t!(AtomicF32, f32, F32);
|
||||
#[cfg(feature = "float")]
|
||||
t!(AtomicF64, f64, F64);
|
||||
}
|
||||
Vendored
+1289
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1 @@
|
||||
!rust-toolchain.toml
|
||||
Vendored
+137
@@ -0,0 +1,137 @@
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
use std::{env, iter, process::Command, str};
|
||||
|
||||
pub(crate) fn rustc_version() -> Option<Version> {
|
||||
let rustc = env::var_os("RUSTC")?;
|
||||
let rustc_wrapper = if env::var_os("CARGO_ENCODED_RUSTFLAGS").is_some() {
|
||||
env::var_os("RUSTC_WRAPPER").filter(|v| !v.is_empty())
|
||||
} else {
|
||||
// Cargo sets environment variables for wrappers correctly only since https://github.com/rust-lang/cargo/pull/9601.
|
||||
None
|
||||
};
|
||||
// Do not apply RUSTC_WORKSPACE_WRAPPER: https://github.com/cuviper/autocfg/issues/58#issuecomment-2067625980
|
||||
let mut rustc = rustc_wrapper.into_iter().chain(iter::once(rustc));
|
||||
let mut cmd = Command::new(rustc.next().unwrap());
|
||||
cmd.args(rustc);
|
||||
// Use verbose version output because the packagers add extra strings to the normal version output.
|
||||
// Do not use long flags (--version --verbose) because clippy-deriver doesn't handle them properly.
|
||||
// -vV is also matched with that cargo internally uses: https://github.com/rust-lang/cargo/blob/0.80.0/src/cargo/util/rustc.rs#L65
|
||||
let output = cmd.arg("-vV").output().ok()?;
|
||||
let verbose_version = str::from_utf8(&output.stdout).ok()?;
|
||||
Version::parse(verbose_version)
|
||||
}
|
||||
|
||||
#[cfg_attr(test, derive(Debug, PartialEq))]
|
||||
pub(crate) struct Version {
|
||||
pub(crate) minor: u32,
|
||||
pub(crate) nightly: bool,
|
||||
commit_date: Date,
|
||||
pub(crate) llvm: u32,
|
||||
}
|
||||
|
||||
impl Version {
|
||||
// The known latest stable version. If we unable to determine
|
||||
// the rustc version, we assume this is the current version.
|
||||
// It is no problem if this is older than the actual latest stable.
|
||||
// LLVM version is assumed to be the minimum external LLVM version:
|
||||
// https://github.com/rust-lang/rust/blob/1.97.0/src/bootstrap/src/core/build_steps/llvm.rs#L638
|
||||
pub(crate) const LATEST: Self = Self::stable(97, 21);
|
||||
|
||||
pub(crate) const fn stable(rustc_minor: u32, llvm_major: u32) -> Self {
|
||||
Self { minor: rustc_minor, nightly: false, commit_date: Date::UNKNOWN, llvm: llvm_major }
|
||||
}
|
||||
|
||||
pub(crate) fn probe(&self, minor: u32, year: u16, month: u8, day: u8) -> bool {
|
||||
if self.nightly {
|
||||
self.minor > minor
|
||||
|| self.minor == minor && self.commit_date >= Date::new(year, month, day)
|
||||
} else {
|
||||
self.minor >= minor
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn commit_date(&self) -> &Date {
|
||||
&self.commit_date
|
||||
}
|
||||
|
||||
pub(crate) fn parse(verbose_version: &str) -> Option<Self> {
|
||||
let mut release = verbose_version
|
||||
.lines()
|
||||
.find(|line| line.starts_with("release: "))
|
||||
.map(|line| &line["release: ".len()..])?
|
||||
.splitn(2, '-');
|
||||
let version = release.next().unwrap();
|
||||
let channel = release.next().unwrap_or_default();
|
||||
let mut digits = version.splitn(3, '.');
|
||||
let major = digits.next()?;
|
||||
if major != "1" {
|
||||
return None;
|
||||
}
|
||||
let minor = digits.next()?.parse::<u32>().ok()?;
|
||||
let _patch = digits.next().unwrap_or("0").parse::<u32>().ok()?;
|
||||
let nightly = match env::var_os("RUSTC_BOOTSTRAP") {
|
||||
// When -1 is passed rustc works like stable, e.g., cfg(target_feature = "unstable_target_feature") will never be set. https://github.com/rust-lang/rust/pull/132993
|
||||
Some(ref v) if v == "-1" => false,
|
||||
// When 1 is passed stable rustc works like nightly, but we ignore it for now.
|
||||
_ => channel == "nightly" || channel == "dev",
|
||||
};
|
||||
|
||||
// Note that rustc 1.49-1.50 (and 1.13 or older) don't print LLVM version.
|
||||
let llvm_major = (|| {
|
||||
let version = verbose_version
|
||||
.lines()
|
||||
.find(|line| line.starts_with("LLVM version: "))
|
||||
.map(|line| &line["LLVM version: ".len()..])?;
|
||||
let mut digits = version.splitn(3, '.');
|
||||
let major = digits.next()?.parse::<u32>().ok()?;
|
||||
let _minor = digits.next()?.parse::<u32>().ok()?;
|
||||
let _patch = digits.next().unwrap_or("0").parse::<u32>().ok()?;
|
||||
Some(major)
|
||||
})()
|
||||
.unwrap_or(0);
|
||||
|
||||
// we don't refer commit date on stable/beta.
|
||||
if nightly {
|
||||
let commit_date = (|| {
|
||||
let mut commit_date = verbose_version
|
||||
.lines()
|
||||
.find(|line| line.starts_with("commit-date: "))
|
||||
.map(|line| &line["commit-date: ".len()..])?
|
||||
.splitn(3, '-');
|
||||
let year = commit_date.next()?.parse::<u16>().ok()?;
|
||||
let month = commit_date.next()?.parse::<u8>().ok()?;
|
||||
let day = commit_date.next()?.parse::<u8>().ok()?;
|
||||
if month > 12 || day > 31 {
|
||||
return None;
|
||||
}
|
||||
Some(Date::new(year, month, day))
|
||||
})();
|
||||
Some(Self {
|
||||
minor,
|
||||
nightly,
|
||||
commit_date: commit_date.unwrap_or(Date::UNKNOWN),
|
||||
llvm: llvm_major,
|
||||
})
|
||||
} else {
|
||||
Some(Self::stable(minor, llvm_major))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(PartialEq, PartialOrd)]
|
||||
#[cfg_attr(test, derive(Debug))]
|
||||
pub(crate) struct Date {
|
||||
pub(crate) year: u16,
|
||||
pub(crate) month: u8,
|
||||
pub(crate) day: u8,
|
||||
}
|
||||
|
||||
impl Date {
|
||||
const UNKNOWN: Self = Self::new(0, 0, 0);
|
||||
|
||||
const fn new(year: u16, month: u8, day: u8) -> Self {
|
||||
Self { year, month, day }
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user