Skip to content

riscv: Zacas extension is no longer experimental in LLVM #206

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
taiki-e merged 1 commit into from
Feb 22, 2025
Merged

riscv: Zacas extension is no longer experimental in LLVM #206

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -651,11 +651,11 @@ jobs:
RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+zabha
QEMU_CPU: max
if: startsWith(matrix.target, 'riscv')
# riscv +experimental-zacas
# riscv +zacas
- run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
env:
RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+experimental-zacas
RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+experimental-zacas
RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+zacas
RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+zacas
if: startsWith(matrix.target, 'riscv')
# s390x z196 (arch9)
- run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
Expand Down
36 changes: 16 additions & 20 deletions build.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,18 +48,18 @@ fn main() {

if version.minor >= 80 {
println!(
r#"cargo:rustc-check-cfg=cfg(target_feature,values("experimental-zacas","fast-serialization","load-store-on-cond","distinct-ops","miscellaneous-extensions-3"))"#
r#"cargo:rustc-check-cfg=cfg(target_feature,values("zacas","fast-serialization","load-store-on-cond","distinct-ops","miscellaneous-extensions-3"))"#
);

// Custom cfgs set by build script. Not public API.
// grep -F 'cargo:rustc-cfg=' build.rs | grep -Ev '^ *//' | sed -E 's/^.*cargo:rustc-cfg=//; s/(=\\)?".*$//' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/'
println!(
"cargo:rustc-check-cfg=cfg(portable_atomic_disable_fiq,portable_atomic_force_amo,portable_atomic_ll_sc_rmw,portable_atomic_new_atomic_intrinsics,portable_atomic_no_asm,portable_atomic_no_asm_maybe_uninit,portable_atomic_no_atomic_64,portable_atomic_no_atomic_cas,portable_atomic_no_atomic_load_store,portable_atomic_no_atomic_min_max,portable_atomic_no_cfg_target_has_atomic,portable_atomic_no_cmpxchg16b_intrinsic,portable_atomic_no_cmpxchg16b_target_feature,portable_atomic_no_const_mut_refs,portable_atomic_no_const_raw_ptr_deref,portable_atomic_no_const_transmute,portable_atomic_no_core_unwind_safe,portable_atomic_no_diagnostic_namespace,portable_atomic_no_offset_of,portable_atomic_no_strict_provenance,portable_atomic_no_stronger_failure_ordering,portable_atomic_no_track_caller,portable_atomic_no_unsafe_op_in_unsafe_fn,portable_atomic_pre_llvm_15,portable_atomic_pre_llvm_16,portable_atomic_pre_llvm_18,portable_atomic_s_mode,portable_atomic_sanitize_thread,portable_atomic_target_feature,portable_atomic_unsafe_assume_single_core,portable_atomic_unstable_asm,portable_atomic_unstable_asm_experimental_arch,portable_atomic_unstable_cfg_target_has_atomic,portable_atomic_unstable_isa_attribute)"
"cargo:rustc-check-cfg=cfg(portable_atomic_disable_fiq,portable_atomic_force_amo,portable_atomic_ll_sc_rmw,portable_atomic_new_atomic_intrinsics,portable_atomic_no_asm,portable_atomic_no_asm_maybe_uninit,portable_atomic_no_atomic_64,portable_atomic_no_atomic_cas,portable_atomic_no_atomic_load_store,portable_atomic_no_atomic_min_max,portable_atomic_no_cfg_target_has_atomic,portable_atomic_no_cmpxchg16b_intrinsic,portable_atomic_no_cmpxchg16b_target_feature,portable_atomic_no_const_mut_refs,portable_atomic_no_const_raw_ptr_deref,portable_atomic_no_const_transmute,portable_atomic_no_core_unwind_safe,portable_atomic_no_diagnostic_namespace,portable_atomic_no_offset_of,portable_atomic_no_strict_provenance,portable_atomic_no_stronger_failure_ordering,portable_atomic_no_track_caller,portable_atomic_no_unsafe_op_in_unsafe_fn,portable_atomic_pre_llvm_15,portable_atomic_pre_llvm_16,portable_atomic_pre_llvm_18,portable_atomic_pre_llvm_20,portable_atomic_s_mode,portable_atomic_sanitize_thread,portable_atomic_target_feature,portable_atomic_unsafe_assume_single_core,portable_atomic_unstable_asm,portable_atomic_unstable_asm_experimental_arch,portable_atomic_unstable_cfg_target_has_atomic,portable_atomic_unstable_isa_attribute)"
);
// TODO: handle multi-line target_feature_fallback
// grep -F 'target_feature_fallback("' build.rs | grep -Ev '^ *//' | sed -E 's/^.*target_feature_fallback\(//; s/",.*$/"/' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/'
println!(
r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_feature,values("cmpxchg16b","distinct-ops","experimental-zacas","fast-serialization","load-store-on-cond","lse","lse128","lse2","mclass","miscellaneous-extensions-3","quadword-atomics","rcpc3","v6","zaamo","zabha"))"#
r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_feature,values("cmpxchg16b","distinct-ops","fast-serialization","load-store-on-cond","lse","lse128","lse2","mclass","miscellaneous-extensions-3","quadword-atomics","rcpc3","v6","zaamo","zabha","zacas"))"#
);
}

Expand Down Expand Up @@ -207,12 +207,15 @@ fn main() {
println!("cargo:rustc-cfg=portable_atomic_no_atomic_load_store");
}

if version.llvm < 18 {
println!("cargo:rustc-cfg=portable_atomic_pre_llvm_18");
if version.llvm < 16 {
println!("cargo:rustc-cfg=portable_atomic_pre_llvm_16");
if version.llvm < 15 {
println!("cargo:rustc-cfg=portable_atomic_pre_llvm_15");
if version.llvm < 20 {
println!("cargo:rustc-cfg=portable_atomic_pre_llvm_20");
if version.llvm < 18 {
println!("cargo:rustc-cfg=portable_atomic_pre_llvm_18");
if version.llvm < 16 {
println!("cargo:rustc-cfg=portable_atomic_pre_llvm_16");
if version.llvm < 15 {
println!("cargo:rustc-cfg=portable_atomic_pre_llvm_15");
}
}
}
}
Expand Down Expand Up @@ -355,17 +358,10 @@ fn main() {
let mut zaamo = false;
// As of rustc 1.84, target_feature "zacas" is not available on rustc side:
// https://github.com/rust-lang/rust/blob/1.84.0/compiler/rustc_target/src/target_features.rs#L425
if version.llvm == 19 {
// amocas.{w,d,q} (and amocas.{b,h} if zabha is also available)
// available as experimental since LLVM 17 https://github.com/llvm/llvm-project/commit/29f630a1ddcbb03caa31b5002f0cbc105ff3a869
// attempted to make non-experimental in LLVM 19 https://github.com/llvm/llvm-project/commit/95aab69c109adf29e183090c25dc95c773215746
// but reverted in https://github.com/llvm/llvm-project/commit/70e7d26e560173c8b9db4c75ab4a3004cd5f021a
// check == 19 instead of range 17..=19 because it is more experimental in LLVM 17/18.
// check == 19 instead of >= 19 because "experimental-zacas" feature
// may no longer exist when it is marked as non-experimental in LLVM 20.
// https://github.com/llvm/llvm-project/commit/614aeda93b2225c6eb42b00ba189ba7ca2585c60
zaamo |= target_feature_fallback("experimental-zacas", false);
}
// amocas.{w,d,q} (and amocas.{b,h} if zabha is also available)
// available as experimental since LLVM 17 https://github.com/llvm/llvm-project/commit/29f630a1ddcbb03caa31b5002f0cbc105ff3a869
// available non-experimental since LLVM 20 https://github.com/llvm/llvm-project/commit/614aeda93b2225c6eb42b00ba189ba7ca2585c60
zaamo |= target_feature_fallback("zacas", false);
// target_feature "zaamo"/"zabha" is unstable and available on rustc side since nightly-2024-10-02: https://github.com/rust-lang/rust/pull/130877
if !version.probe(83, 2024, 10, 1) || needs_target_feature_fallback(&version, None) {
if version.llvm >= 19 {
Expand Down
136 changes: 24 additions & 112 deletions src/cfgs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,12 @@ mod atomic_32_macros {
),
not(portable_atomic_no_atomic_64),
not(any(target_pointer_width = "16", target_pointer_width = "32")),
all(
target_arch = "riscv32",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
),
))
)]
#[cfg_attr(
Expand All @@ -157,17 +163,8 @@ mod atomic_32_macros {
all(
target_arch = "riscv32",
not(any(miri, portable_atomic_sanitize_thread)),
not(portable_atomic_no_asm),
any(
target_feature = "experimental-zacas",
portable_atomic_target_feature = "experimental-zacas",
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
any(target_os = "linux", target_os = "android"),
),
),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
),
))
)]
Expand Down Expand Up @@ -199,6 +196,12 @@ mod atomic_64_macros {
),
not(portable_atomic_no_atomic_64),
not(any(target_pointer_width = "16", target_pointer_width = "32")),
all(
target_arch = "riscv32",
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
),
)))
)]
#[cfg_attr(
Expand All @@ -219,17 +222,8 @@ mod atomic_64_macros {
all(
target_arch = "riscv32",
not(any(miri, portable_atomic_sanitize_thread)),
not(portable_atomic_no_asm),
any(
target_feature = "experimental-zacas",
portable_atomic_target_feature = "experimental-zacas",
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
any(target_os = "linux", target_os = "android"),
),
),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
),
)))
)]
Expand Down Expand Up @@ -262,61 +256,20 @@ mod atomic_64_macros {
portable_atomic_no_cmpxchg16b_intrinsic,
)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(
target_feature = "cmpxchg16b",
portable_atomic_target_feature = "cmpxchg16b",
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
not(any(target_env = "sgx", miri)),
),
),
any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
),
all(
target_arch = "riscv64",
not(portable_atomic_no_asm),
any(
target_feature = "experimental-zacas",
portable_atomic_target_feature = "experimental-zacas",
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
any(target_os = "linux", target_os = "android"),
not(any(miri, portable_atomic_sanitize_thread)),
),
),
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
),
all(
target_arch = "powerpc64",
portable_atomic_unstable_asm_experimental_arch,
any(
target_feature = "quadword-atomics",
portable_atomic_target_feature = "quadword-atomics",
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
any(
all(
target_os = "linux",
any(
all(
target_env = "gnu",
any(target_endian = "little", not(target_feature = "crt-static")),
),
all(
any(target_env = "musl", target_env = "ohos", target_env = "uclibc"),
not(target_feature = "crt-static"),
),
portable_atomic_outline_atomics,
),
),
target_os = "android",
target_os = "freebsd",
target_os = "openbsd",
),
not(any(miri, portable_atomic_sanitize_thread)),
),
),
),
all(target_arch = "s390x", not(portable_atomic_no_asm)),
Expand Down Expand Up @@ -370,61 +323,20 @@ mod atomic_128_macros {
portable_atomic_no_cmpxchg16b_intrinsic,
)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(
target_feature = "cmpxchg16b",
portable_atomic_target_feature = "cmpxchg16b",
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
not(any(target_env = "sgx", miri)),
),
),
any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
),
all(
target_arch = "riscv64",
not(portable_atomic_no_asm),
any(
target_feature = "experimental-zacas",
portable_atomic_target_feature = "experimental-zacas",
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
any(target_os = "linux", target_os = "android"),
not(any(miri, portable_atomic_sanitize_thread)),
),
),
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
),
all(
target_arch = "powerpc64",
portable_atomic_unstable_asm_experimental_arch,
any(
target_feature = "quadword-atomics",
portable_atomic_target_feature = "quadword-atomics",
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
any(
all(
target_os = "linux",
any(
all(
target_env = "gnu",
any(target_endian = "little", not(target_feature = "crt-static")),
),
all(
any(target_env = "musl", target_env = "ohos", target_env = "uclibc"),
not(target_feature = "crt-static"),
),
portable_atomic_outline_atomics,
),
),
target_os = "android",
target_os = "freebsd",
target_os = "openbsd",
),
not(any(miri, portable_atomic_sanitize_thread)),
),
),
),
all(target_arch = "s390x", not(portable_atomic_no_asm)),
Expand Down
2 changes: 1 addition & 1 deletion src/imp/atomic128/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ Here is the table of targets that support 128-bit atomics and the instructions u
| ----------- | ---- | ----- | --- | --- | ---- |
| x86_64 | cmpxchg16b or vmovdqa | cmpxchg16b or vmovdqa | cmpxchg16b | cmpxchg16b | Requires `cmpxchg16b` target feature (enabled by default on Apple and Windows (except Windows 7) targets). vmovdqa requires Intel, AMD, or Zhaoxin CPU with AVX. <br> Both compile-time and run-time detection are supported for cmpxchg16b. vmovdqa is currently run-time detection only. <br> Requires rustc 1.59+ |
| aarch64/arm64ec | ldxp/stxp or casp or ldp/ldiapp | ldxp/stxp or casp or stp/stilp/swpp | ldxp/stxp or casp | ldxp/stxp or casp/swpp/ldclrp/ldsetp | casp requires `lse` target feature, ldp/stp requires `lse2` target feature, ldiapp/stilp requires `lse2` and `rcpc3` target features, swpp/ldclrp/ldsetp requires `lse128` target feature. <br> Both compile-time and run-time detection are supported. <br> Requires rustc 1.59+ (aarch64) / 1.84+ (arm64ec) |
| riscv64 | amocas.q | amocas.q | amocas.q | amocas.q | Experimental because LLVM marking the corresponding target feature as experimental. Requires `experimental-zacas` target feature. Both compile-time and run-time detection are supported (run-time detection is currently disabled by default). <br> Requires rustc 1.59+ |
| riscv64 | amocas.q | amocas.q | amocas.q | amocas.q | Requires `zacas` target feature. Both compile-time and run-time detection are supported (run-time detection is currently disabled by default). <br> Requires rustc 1.59+ |
| powerpc64 | lq | stq | lqarx/stqcx. | lqarx/stqcx. | Requires `quadword-atomics` target feature (enabled by default on powerpc64le). Both compile-time and run-time detection are supported. <br> Requires nightly |
| s390x | lpq | stpq | cdsg | cdsg | Requires rustc 1.84+ |
| mips64r6 | lldp | lldp/scdp | lldp/scdp | lldp/scdp | Unimplemented (unsupported in LLVM). Requires Release 6 Paired LL/SC family of instructions |
Expand Down
3 changes: 3 additions & 0 deletions src/imp/atomic128/intrinsics.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ Note:
- On powerpc64, LLVM (as of 17) doesn't support 128-bit atomic min/max:
https://github.com/llvm/llvm-project/issues/68390
- On powerpc64le, LLVM (as of 17) generates broken code. (wrong result from fetch_add)
- On riscv64, LLVM does not automatically use 128-bit atomic instructions even if zacas feature is
enabled, because doing it changes the ABI. (If the ability to do that is provided by LLVM in the
future, it should probably be controlled by another ABI feature similar to forced-atomics.)
*/

include!("macros.rs");
Expand Down
10 changes: 4 additions & 6 deletions src/imp/atomic128/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,21 +61,19 @@ pub(super) mod powerpc64;
// riscv64
#[cfg(all(
target_arch = "riscv64",
not(portable_atomic_no_asm),
not(any(miri, portable_atomic_sanitize_thread)),
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(
target_feature = "experimental-zacas",
portable_atomic_target_feature = "experimental-zacas",
target_feature = "zacas",
portable_atomic_target_feature = "zacas",
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
any(target_os = "linux", target_os = "android"),
not(any(miri, portable_atomic_sanitize_thread)),
),
),
))]
// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.
#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = "intrinsics.rs")]
pub(super) mod riscv64;

// s390x
Expand Down
Loading