2424#ifndef __BNR_DEFERRED_ATOMIC_SHIMS__
2525#define __BNR_DEFERRED_ATOMIC_SHIMS__
2626
27+ #if !__has_include (< stdatomic .h > ) || !__has_extension (c_atomic )
28+ #error Required compiler features are not available
29+ #endif
30+
2731#include <stdatomic.h>
2832#if defined(__APPLE__ )
2933#include <os/lock.h>
3034#endif
3135#include <pthread.h>
3236#include <dispatch/dispatch.h>
3337
34- #if defined(__has_attribute ) && __has_attribute (always_inline )
3538#define BNR_ATOMIC_INLINE static inline __attribute__((always_inline))
36- #else
37- #define BNR_ATOMIC_INLINE static inline
38- #endif
39+ #define BNR_ATOMIC_OVERLOAD __attribute__((overloadable))
40+ #define BNR_ATOMIC_WARN_UNUSED_RESULT __attribute__((warn_unused_result))
41+ #define BNR_ATOMIC_LIKELY ( expression ) __builtin_expect(!!(expression), 1)
3942
4043#if !defined(SWIFT_ENUM )
4144#define SWIFT_ENUM (_type , _name ) enum _name _name##_t; enum _name
@@ -106,7 +109,8 @@ bool bnr_native_lock_trylock(bnr_native_lock_t address) {
106109 return pthread_mutex_trylock (& address -> impl .legacy ) == 0 ;
107110}
108111
109- BNR_ATOMIC_INLINE void bnr_native_lock_unlock (bnr_native_lock_t address ) {
112+ BNR_ATOMIC_INLINE
113+ void bnr_native_lock_unlock (bnr_native_lock_t address ) {
110114#if defined(__APPLE__ )
111115 if (& os_unfair_lock_unlock != NULL ) {
112116 return os_unfair_lock_unlock (& address -> impl .modern );
@@ -116,80 +120,103 @@ BNR_ATOMIC_INLINE void bnr_native_lock_unlock(bnr_native_lock_t address) {
116120 pthread_mutex_unlock (& address -> impl .legacy );
117121}
118122
119- typedef volatile struct bnr_atomic_ptr_s {
120- _Atomic(const void * _Nullable ) value ;
121- } bnr_atomic_ptr , * _Nonnull bnr_atomic_ptr_t ;
123+ typedef const void * _Nullable volatile * _Nonnull bnr_atomic_ptr_t ;
122124
123- BNR_ATOMIC_INLINE
124- const void * _Nullable bnr_atomic_ptr_load (bnr_atomic_ptr_t target , bnr_atomic_memory_order_t order ) {
125- return atomic_load_explicit ( & target -> value , order );
125+ BNR_ATOMIC_INLINE BNR_ATOMIC_OVERLOAD
126+ void bnr_atomic_init (bnr_atomic_ptr_t target , const void * _Nullable initial ) {
127+ atomic_init (( const void * _Atomic * ) target , initial );
126128}
127129
128- BNR_ATOMIC_INLINE
129- const void * _Nullable bnr_atomic_ptr_exchange (bnr_atomic_ptr_t target , const void * _Nullable desired , bnr_atomic_memory_order_t order ) {
130- return atomic_exchange_explicit ( & target -> value , desired , order );
130+ BNR_ATOMIC_INLINE BNR_ATOMIC_WARN_UNUSED_RESULT BNR_ATOMIC_OVERLOAD
131+ const void * _Nullable bnr_atomic_load (bnr_atomic_ptr_t target , bnr_atomic_memory_order_t order ) {
132+ return atomic_load_explicit (( const void * _Atomic * ) target , order );
131133}
132134
133- BNR_ATOMIC_INLINE
134- bool bnr_atomic_ptr_compare_and_swap ( bnr_atomic_ptr_t target , const void * _Nullable expected , const void * _Nullable desired , bnr_atomic_memory_order_t order ) {
135- return atomic_compare_exchange_strong_explicit ( & target -> value , & expected , desired , order , memory_order_relaxed );
135+ BNR_ATOMIC_INLINE BNR_ATOMIC_WARN_UNUSED_RESULT
136+ const void * _Nullable bnr_atomic_exchange ( bnr_atomic_ptr_t target , const void * _Nullable desired , bnr_atomic_memory_order_t order ) {
137+ return atomic_exchange_explicit (( const void * _Atomic * ) target , desired , order );
136138}
137139
138- typedef volatile struct bnr_atomic_flag_s {
139- _Atomic_Bool ) value ;
140- } bnr_atomic_flag , * _Nonnull bnr_atomic_flag_t ;
140+ BNR_ATOMIC_INLINE BNR_ATOMIC_WARN_UNUSED_RESULT
141+ bool bnr_atomic_compare_and_swap (bnr_atomic_ptr_t target , const void * _Nullable expected , const void * _Nullable desired , bnr_atomic_memory_order_t order ) {
142+ return atomic_compare_exchange_strong_explicit ((const void * _Atomic * )target , & expected , desired , order , memory_order_relaxed );
143+ }
141144
142- BNR_ATOMIC_INLINE
143- bool bnr_atomic_flag_load (bnr_atomic_flag_t target , bnr_atomic_memory_order_t order ) {
144- return atomic_load_explicit (& target -> value , order );
145+ BNR_ATOMIC_INLINE BNR_ATOMIC_WARN_UNUSED_RESULT
146+ const void * _Nonnull bnr_atomic_load_and_wait (bnr_atomic_ptr_t target ) {
147+ const void * result = NULL ;
148+ for (;;) {
149+ if (BNR_ATOMIC_LIKELY (result = bnr_atomic_load (target , bnr_atomic_memory_order_acquire ))) break ;
150+ #if defined(__x86_64__ ) || defined(__i386__ )
151+ __asm__("pause" );
152+ #elif defined(__arm__ ) || defined(__arm64__ )
153+ __asm__("yield" );
154+ #else
155+ __asm__("" );
156+ #endif
157+ }
158+ return result ;
145159}
146160
147- BNR_ATOMIC_INLINE
148- bool bnr_atomic_flag_test_and_set (bnr_atomic_flag_t target , bnr_atomic_memory_order_t order ) {
149- return atomic_exchange_explicit (& target -> value , true, order );
161+ typedef volatile bool * _Nonnull bnr_atomic_flag_t ;
162+
163+ BNR_ATOMIC_INLINE BNR_ATOMIC_OVERLOAD
164+ void bnr_atomic_init (bnr_atomic_flag_t target , bool initial ) {
165+ atomic_init ((atomic_bool * )target , initial );
150166}
151167
152- typedef volatile struct bnr_atomic_bitmask_s {
153- _Atomicuint8_t ) value ;
154- } bnr_atomic_bitmask , * _Nonnull bnr_atomic_bitmask_t ;
168+ BNR_ATOMIC_INLINE BNR_ATOMIC_WARN_UNUSED_RESULT BNR_ATOMIC_OVERLOAD
169+ bool bnr_atomic_load (bnr_atomic_flag_t target , bnr_atomic_memory_order_t order ) {
170+ return atomic_load_explicit ((atomic_bool * )target , order );
171+ }
155172
156- BNR_ATOMIC_INLINE
157- void bnr_atomic_bitmask_init ( bnr_atomic_bitmask_t target , uint8_t mask ) {
158- atomic_init ( & target -> value , mask );
173+ BNR_ATOMIC_INLINE BNR_ATOMIC_OVERLOAD
174+ void bnr_atomic_store ( bnr_atomic_flag_t target , bool desired , bnr_atomic_memory_order_t order ) {
175+ atomic_store_explicit (( atomic_bool * ) target , desired , order );
159176}
160177
161- BNR_ATOMIC_INLINE
162- uint8_t bnr_atomic_bitmask_add (bnr_atomic_bitmask_t target , uint8_t mask , bnr_atomic_memory_order_t order ) {
163- return atomic_fetch_or_explicit (& target -> value , mask , order );
178+ typedef volatile uint8_t * _Nonnull bnr_atomic_bitmask_t ;
179+
180+ BNR_ATOMIC_INLINE BNR_ATOMIC_OVERLOAD
181+ void bnr_atomic_init (bnr_atomic_bitmask_t target , uint8_t mask ) {
182+ atomic_init ((uint8_t _Atomic * )target , mask );
164183}
165184
166- BNR_ATOMIC_INLINE
167- uint8_t bnr_atomic_bitmask_remove (bnr_atomic_bitmask_t target , uint8_t mask , bnr_atomic_memory_order_t order ) {
168- return atomic_fetch_and_explicit ( & target -> value , ~ mask , order );
185+ BNR_ATOMIC_INLINE BNR_ATOMIC_WARN_UNUSED_RESULT BNR_ATOMIC_OVERLOAD
186+ uint8_t bnr_atomic_load (bnr_atomic_bitmask_t target , bnr_atomic_memory_order_t order ) {
187+ return atomic_load_explicit (( uint8_t _Atomic * ) target , order );
169188}
170189
171- BNR_ATOMIC_INLINE
172- bool bnr_atomic_bitmask_test (bnr_atomic_bitmask_t target , uint8_t mask , bnr_atomic_memory_order_t order ) {
173- return ( atomic_load_explicit ( & target -> value , order ) & mask ) != 0 ;
190+ BNR_ATOMIC_INLINE BNR_ATOMIC_OVERLOAD
191+ uint8_t bnr_atomic_fetch_or (bnr_atomic_bitmask_t target , uint8_t mask , bnr_atomic_memory_order_t order ) {
192+ return atomic_fetch_or_explicit (( uint8_t _Atomic * ) target , mask , order ) ;
174193}
175194
176- typedef volatile struct bnr_atomic_counter_s {
177- _Atomiclong ) value ;
178- } bnr_atomic_counter , * _Nonnull bnr_atomic_counter_t ;
195+ BNR_ATOMIC_INLINE BNR_ATOMIC_OVERLOAD
196+ uint8_t bnr_atomic_fetch_and (bnr_atomic_bitmask_t target , uint8_t mask , bnr_atomic_memory_order_t order ) {
197+ return atomic_fetch_and_explicit ((uint8_t _Atomic * )target , mask , order );
198+ }
179199
180- BNR_ATOMIC_INLINE
181- long bnr_atomic_counter_load (bnr_atomic_counter_t target ) {
182- return atomic_load (& target -> value );
200+ typedef volatile long * _Nonnull bnr_atomic_counter_t ;
201+
202+ BNR_ATOMIC_INLINE BNR_ATOMIC_OVERLOAD
203+ void bnr_atomic_init (bnr_atomic_counter_t target ) {
204+ atomic_init ((atomic_long * )target , 0 );
183205}
184206
185- BNR_ATOMIC_INLINE
186- long bnr_atomic_counter_increment (bnr_atomic_counter_t target ) {
187- return atomic_fetch_add ( & target -> value , 1 ) + 1 ;
207+ BNR_ATOMIC_INLINE BNR_ATOMIC_WARN_UNUSED_RESULT BNR_ATOMIC_OVERLOAD
208+ long bnr_atomic_load (bnr_atomic_counter_t target ) {
209+ return atomic_load (( atomic_long * ) target ) ;
188210}
189211
190- BNR_ATOMIC_INLINE
191- long bnr_atomic_counter_decrement (bnr_atomic_counter_t target ) {
192- return atomic_fetch_sub (& target -> value , 1 ) - 1 ;
212+ BNR_ATOMIC_INLINE BNR_ATOMIC_OVERLOAD
213+ long bnr_atomic_fetch_add (bnr_atomic_counter_t target , long offset ) {
214+ return atomic_fetch_add ((atomic_long * )target , offset );
215+ }
216+
217+ BNR_ATOMIC_INLINE BNR_ATOMIC_OVERLOAD
218+ long bnr_atomic_fetch_subtract (bnr_atomic_counter_t target , long offset ) {
219+ return atomic_fetch_sub ((atomic_long * )target , offset );
193220}
194221
195222#undef SWIFT_ENUM
0 commit comments