Skip to content

Commit 343d10c

Browse files
taiki-etaiki-e
committed
riscv32: Support 64-bit atomics (Zacas extension)
1 parent 5b503be commit 343d10c

File tree

13 files changed

+859
-38
lines changed

13 files changed

+859
-38
lines changed

‎.github/.cspell/project-dictionary.txt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ DWCAS
4646
elems
4747
espup
4848
exynos
49+
fild
50+
fistp
4951
getauxval
5052
getisax
5153
getpid
@@ -70,6 +72,7 @@ ldclrpa
7072
ldclrpal
7173
ldclrpl
7274
ldiapp
75+
ldrexd
7376
ldsetp
7477
ldsetpa
7578
ldsetpal
@@ -101,6 +104,8 @@ minu
101104
mipsn
102105
miscompiles
103106
mmfr
107+
movlps
108+
movq
104109
mpidr
105110
mstatus
106111
mvfr
@@ -150,6 +155,7 @@ stilp
150155
stlxp
151156
stpq
152157
stqcx
158+
strexd
153159
stxp
154160
subarch
155161
subc

‎.github/workflows/ci.yml

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -259,8 +259,6 @@ jobs:
259259
target: riscv64gc-unknown-linux-gnu
260260
- rust: '1.59' # LLVM 13
261261
target: riscv64gc-unknown-linux-gnu
262-
- rust: '1.73' # LLVM 17 (oldest version we can use experimental-zacas on this target)
263-
target: riscv64gc-unknown-linux-gnu
264262
- rust: stable
265263
target: riscv64gc-unknown-linux-gnu
266264
- rust: nightly
@@ -317,6 +315,9 @@ jobs:
317315
# TODO: LLVM bug: Undefined temporary symbol error when building std.
318316
- run: printf 'RELEASE=--release\n' >>"${GITHUB_ENV}"
319317
if: startsWith(matrix.target, 'mips-') || startsWith(matrix.target, 'mipsel-')
318+
# for serde
319+
- run: printf '%s\n' "RUSTFLAGS=${RUSTFLAGS} --cfg no_diagnostic_namespace" >>"${GITHUB_ENV}"
320+
if: matrix.rust == 'nightly-2024-02-13'
320321

321322
- run: tools/test.sh -vv ${TARGET:-} ${DOCTEST_XCOMPILE:-} ${BUILD_STD:-} ${RELEASE:-}
322323
# We test doctest only once with the default build conditions because doctest is slow. Both api-test
@@ -388,21 +389,21 @@ jobs:
388389
RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-cpu=pwr8
389390
RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-cpu=pwr8
390391
if: startsWith(matrix.target, 'powerpc64-')
391-
# riscv64 +zabha
392+
# riscv +zabha
392393
- run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
393394
env:
394395
RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+zabha
395396
RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+zabha
396397
QEMU_CPU: max
397398
# TODO: cranelift doesn't support cfg(target_feature): https://github.com/rust-lang/rustc_codegen_cranelift/issues/1400
398-
if: startsWith(matrix.target, 'riscv64') && !contains(matrix.flags, 'codegen-backend=cranelift')
399-
# riscv64 +experimental-zacas
399+
if: startsWith(matrix.target, 'riscv') && !contains(matrix.flags, 'codegen-backend=cranelift')
400+
# riscv +experimental-zacas
400401
- run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
401402
env:
402403
RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+experimental-zacas
403404
RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+experimental-zacas
404405
# TODO: cranelift doesn't support cfg(target_feature): https://github.com/rust-lang/rustc_codegen_cranelift/issues/1400
405-
if: startsWith(matrix.target, 'riscv64') && !contains(matrix.flags, 'codegen-backend=cranelift')
406+
if: startsWith(matrix.target, 'riscv') && !contains(matrix.flags, 'codegen-backend=cranelift')
406407
# s390x z196 (arch9)
407408
- run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
408409
env:

‎src/cfgs.rs

Lines changed: 64 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,36 @@ mod atomic_32_macros {
154154
),
155155
target_has_atomic = "64",
156156
not(any(target_pointer_width = "16", target_pointer_width = "32")),
157+
all(
158+
target_arch = "riscv32",
159+
not(any(miri, portable_atomic_sanitize_thread)),
160+
not(portable_atomic_no_asm),
161+
any(
162+
target_feature = "experimental-zacas",
163+
portable_atomic_target_feature = "experimental-zacas",
164+
// TODO(riscv)
165+
// all(
166+
// feature = "fallback",
167+
// not(portable_atomic_no_outline_atomics),
168+
// any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
169+
// any(
170+
// all(
171+
// target_os = "linux",
172+
// any(
173+
// target_env = "gnu",
174+
// all(
175+
// any(target_env = "musl", target_env = "ohos"),
176+
// not(target_feature = "crt-static"),
177+
// ),
178+
// portable_atomic_outline_atomics,
179+
// ),
180+
// ),
181+
// target_os = "android",
182+
// ),
183+
// not(any(miri, portable_atomic_sanitize_thread)),
184+
// ),
185+
),
186+
),
157187
))
158188
)]
159189
#[macro_use]
@@ -201,6 +231,36 @@ mod atomic_64_macros {
201231
),
202232
target_has_atomic = "64",
203233
not(any(target_pointer_width = "16", target_pointer_width = "32")),
234+
all(
235+
target_arch = "riscv32",
236+
not(any(miri, portable_atomic_sanitize_thread)),
237+
not(portable_atomic_no_asm),
238+
any(
239+
target_feature = "experimental-zacas",
240+
portable_atomic_target_feature = "experimental-zacas",
241+
// TODO(riscv)
242+
// all(
243+
// feature = "fallback",
244+
// not(portable_atomic_no_outline_atomics),
245+
// any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
246+
// any(
247+
// all(
248+
// target_os = "linux",
249+
// any(
250+
// target_env = "gnu",
251+
// all(
252+
// any(target_env = "musl", target_env = "ohos"),
253+
// not(target_feature = "crt-static"),
254+
// ),
255+
// portable_atomic_outline_atomics,
256+
// ),
257+
// ),
258+
// target_os = "android",
259+
// ),
260+
// not(any(miri, portable_atomic_sanitize_thread)),
261+
// ),
262+
),
263+
),
204264
)))
205265
)]
206266
#[macro_use]
@@ -247,11 +307,11 @@ mod atomic_64_macros {
247307
any(
248308
target_feature = "experimental-zacas",
249309
portable_atomic_target_feature = "experimental-zacas",
250-
// TODO(riscv64)
310+
// TODO(riscv)
251311
// all(
252312
// feature = "fallback",
253313
// not(portable_atomic_no_outline_atomics),
254-
// any(test, portable_atomic_outline_atomics), // TODO(riscv64): currently disabled by default
314+
// any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
255315
// any(
256316
// all(
257317
// target_os = "linux",
@@ -366,11 +426,11 @@ mod atomic_128_macros {
366426
any(
367427
target_feature = "experimental-zacas",
368428
portable_atomic_target_feature = "experimental-zacas",
369-
// TODO(riscv64)
429+
// TODO(riscv)
370430
// all(
371431
// feature = "fallback",
372432
// not(portable_atomic_no_outline_atomics),
373-
// any(test, portable_atomic_outline_atomics), // TODO(riscv64): currently disabled by default
433+
// any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
374434
// any(
375435
// all(
376436
// target_os = "linux",

‎src/imp/atomic128/riscv64.rs

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@ Generated asm:
1818
- riscv64 (+experimental-zacas) https://godbolt.org/z/5Kc17T1W8
1919
*/
2020

21-
// TODO: 64-bit atomic using amocas.d for riscv32
22-
2321
include!("macros.rs");
2422

2523
// TODO
@@ -31,7 +29,7 @@ include!("macros.rs");
3129
// See detect/auxv.rs for more.
3230
#[cfg(test)] // TODO
3331
#[cfg(not(portable_atomic_no_outline_atomics))]
34-
#[cfg(any(test, portable_atomic_outline_atomics))] // TODO(riscv64): currently disabled by default
32+
#[cfg(any(test, portable_atomic_outline_atomics))] // TODO(riscv): currently disabled by default
3533
#[cfg(any(
3634
test,
3735
not(any(

‎src/imp/atomic64/README.md

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,34 @@
11
# Implementation of 64-bit atomics on 32-bit architectures
22

33
(See the [`atomic128` module](../atomic128) for 128-bit atomics on 64-bit architectures.)
4+
5+
## 64-bit atomics instructions
6+
7+
Here is the table of targets that support 64-bit atomics and the instructions used:
8+
9+
| target_arch | load | store | CAS | RMW | note |
10+
| ----------- | ---- | ----- | --- | --- | ---- |
11+
| x86 | cmpxchg8b or fild or movlps or movq | cmpxchg8b or fistp or movlps | cmpxchg8b | cmpxchg8b | provided by `core::sync::atomic` |
12+
| arm | ldrexd | ldrexd/strexd | ldrexd/strexd | ldrexd/strexd | provided by `core::sync::atomic` for Armv6+, otherwise provided by us for Linux/Android using kuser_cmpxchg64 (see arm_linux.rs for more) |
13+
| riscv32 | amocas.d | amocas.d | amocas.d | amocas.d | Experimental. Requires experimental-zacas target feature. Currently compile-time detection only due to LLVM marking it as experimental. <br> Requires 1.82+ (LLVM 19+) |
14+
15+
If `core::sync::atomic` provides 64-bit atomics, we use them.
16+
On compiler versions or platforms where these are not supported, the fallback implementation is used.
17+
18+
## Run-time CPU feature detection
19+
20+
[detect](../detect) module has run-time CPU feature detection implementations.
21+
22+
Here is the table of targets that support run-time CPU feature detection and the instruction or API used:
23+
24+
| target_arch | target_os/target_env | instruction/API | features | note |
25+
| ----------- | -------------------- | --------------- | -------- | ---- |
26+
| riscv32 | linux | riscv_hwprobe | all | Currently only used in tests due to LLVM marking zacas as experimental |
27+
28+
Run-time detection is enabled by default on most targets and can be disabled with `--cfg portable_atomic_no_outline_atomics`.
29+
30+
On some targets, run-time detection is disabled by default mainly for compatibility with older versions of operating systems or incomplete build environments, and can be enabled by `--cfg portable_atomic_outline_atomics`. (When both cfg are enabled, `*_no_*` cfg is preferred.)
31+
32+
For targets not included in the above table, run-time detection is always disabled and works the same as when `--cfg portable_atomic_no_outline_atomics` is set.
33+
34+
See also [docs on `portable_atomic_no_outline_atomics`](https://github.com/taiki-e/portable-atomic/blob/HEAD/README.md#optional-cfg-no-outline-atomics) in the top-level readme.

0 commit comments

Comments
 (0)