16 #include <rte_compat.h> 30 static inline void rte_mb(
void);
38 static inline void rte_wmb(
void);
46 static inline void rte_rmb(
void);
119 #define rte_compiler_barrier() do { \ 120 asm volatile ("" : : : "memory"); \ 149 #ifdef RTE_FORCE_INTRINSICS 153 return __sync_bool_compare_and_swap(dst, exp, src);
172 static inline uint16_t
175 #ifdef RTE_FORCE_INTRINSICS 176 static inline uint16_t
179 #if defined(__clang__) 180 return __atomic_exchange_n(dst, val, __ATOMIC_SEQ_CST);
182 return __atomic_exchange_2(dst, val, __ATOMIC_SEQ_CST);
197 #define RTE_ATOMIC16_INIT(val) { (val) } 219 static inline int16_t
250 __sync_fetch_and_add(&v->
cnt, inc);
264 __sync_fetch_and_sub(&v->
cnt, dec);
276 #ifdef RTE_FORCE_INTRINSICS 293 #ifdef RTE_FORCE_INTRINSICS 314 static inline int16_t
317 return __sync_add_and_fetch(&v->
cnt, inc);
334 static inline int16_t
337 return __sync_sub_and_fetch(&v->
cnt, dec);
353 #ifdef RTE_FORCE_INTRINSICS 356 return __sync_add_and_fetch(&v->
cnt, 1) == 0;
373 #ifdef RTE_FORCE_INTRINSICS 376 return __sync_sub_and_fetch(&v->
cnt, 1) == 0;
393 #ifdef RTE_FORCE_INTRINSICS 432 #ifdef RTE_FORCE_INTRINSICS 436 return __sync_bool_compare_and_swap(dst, exp, src);
455 static inline uint32_t
458 #ifdef RTE_FORCE_INTRINSICS 459 static inline uint32_t
462 #if defined(__clang__) 463 return __atomic_exchange_n(dst, val, __ATOMIC_SEQ_CST);
465 return __atomic_exchange_4(dst, val, __ATOMIC_SEQ_CST);
480 #define RTE_ATOMIC32_INIT(val) { (val) } 502 static inline int32_t
533 __sync_fetch_and_add(&v->
cnt, inc);
547 __sync_fetch_and_sub(&v->
cnt, dec);
559 #ifdef RTE_FORCE_INTRINSICS 576 #ifdef RTE_FORCE_INTRINSICS 597 static inline int32_t
600 return __sync_add_and_fetch(&v->
cnt, inc);
617 static inline int32_t
620 return __sync_sub_and_fetch(&v->
cnt, dec);
636 #ifdef RTE_FORCE_INTRINSICS 639 return __sync_add_and_fetch(&v->
cnt, 1) == 0;
656 #ifdef RTE_FORCE_INTRINSICS 659 return __sync_sub_and_fetch(&v->
cnt, 1) == 0;
676 #ifdef RTE_FORCE_INTRINSICS 714 #ifdef RTE_FORCE_INTRINSICS 718 return __sync_bool_compare_and_swap(dst, exp, src);
737 static inline uint64_t
740 #ifdef RTE_FORCE_INTRINSICS 741 static inline uint64_t
744 #if defined(__clang__) 745 return __atomic_exchange_n(dst, val, __ATOMIC_SEQ_CST);
747 return __atomic_exchange_8(dst, val, __ATOMIC_SEQ_CST);
762 #define RTE_ATOMIC64_INIT(val) { (val) } 773 #ifdef RTE_FORCE_INTRINSICS 783 while (success == 0) {
800 static inline int64_t
803 #ifdef RTE_FORCE_INTRINSICS 804 static inline int64_t
813 while (success == 0) {
835 #ifdef RTE_FORCE_INTRINSICS 845 while (success == 0) {
865 #ifdef RTE_FORCE_INTRINSICS 869 __sync_fetch_and_add(&v->
cnt, inc);
884 #ifdef RTE_FORCE_INTRINSICS 888 __sync_fetch_and_sub(&v->
cnt, dec);
901 #ifdef RTE_FORCE_INTRINSICS 918 #ifdef RTE_FORCE_INTRINSICS 939 static inline int64_t
942 #ifdef RTE_FORCE_INTRINSICS 943 static inline int64_t
946 return __sync_add_and_fetch(&v->
cnt, inc);
963 static inline int64_t
966 #ifdef RTE_FORCE_INTRINSICS 967 static inline int64_t
970 return __sync_sub_and_fetch(&v->
cnt, dec);
987 #ifdef RTE_FORCE_INTRINSICS 1007 #ifdef RTE_FORCE_INTRINSICS 1027 #ifdef RTE_FORCE_INTRINSICS 1042 #ifdef RTE_FORCE_INTRINSICS 1060 __extension__ __int128 int128;
1110 const rte_int128_t *src,
static void rte_atomic32_inc(rte_atomic32_t *v)
static int64_t rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
static void rte_smp_rmb(void)
static int rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
static int16_t rte_atomic16_read(const rte_atomic16_t *v)
static void rte_wmb(void)
static void rte_atomic32_add(rte_atomic32_t *v, int32_t inc)
static int rte_atomic16_dec_and_test(rte_atomic16_t *v)
static int64_t rte_atomic64_read(rte_atomic64_t *v)
static int16_t rte_atomic16_sub_return(rte_atomic16_t *v, int16_t dec)
static void rte_atomic32_sub(rte_atomic32_t *v, int32_t dec)
static void rte_atomic32_dec(rte_atomic32_t *v)
static int rte_atomic32_inc_and_test(rte_atomic32_t *v)
static void rte_io_mb(void)
static void rte_atomic64_inc(rte_atomic64_t *v)
static void rte_atomic32_set(rte_atomic32_t *v, int32_t new_value)
static int32_t rte_atomic32_add_return(rte_atomic32_t *v, int32_t inc)
static void rte_atomic16_add(rte_atomic16_t *v, int16_t inc)
static void rte_atomic32_init(rte_atomic32_t *v)
static void rte_atomic16_clear(rte_atomic16_t *v)
static int rte_atomic16_test_and_set(rte_atomic16_t *v)
static int rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
static void rte_atomic16_sub(rte_atomic16_t *v, int16_t dec)
static void rte_atomic16_set(rte_atomic16_t *v, int16_t new_value)
static void rte_io_wmb(void)
static void rte_atomic64_clear(rte_atomic64_t *v)
static int16_t rte_atomic16_add_return(rte_atomic16_t *v, int16_t inc)
static int rte_atomic64_dec_and_test(rte_atomic64_t *v)
static void rte_smp_wmb(void)
static int64_t rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
static void rte_rmb(void)
static void rte_atomic32_clear(rte_atomic32_t *v)
static void rte_atomic16_inc(rte_atomic16_t *v)
static int rte_atomic16_inc_and_test(rte_atomic16_t *v)
static void rte_atomic_thread_fence(int memorder)
static int rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
static int rte_atomic64_inc_and_test(rte_atomic64_t *v)
static int32_t rte_atomic32_sub_return(rte_atomic32_t *v, int32_t dec)
static void rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
static uint16_t rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val)
static int rte_atomic32_test_and_set(rte_atomic32_t *v)
static void rte_smp_mb(void)
static uint32_t rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val)
static void rte_atomic16_dec(rte_atomic16_t *v)
static void rte_atomic64_init(rte_atomic64_t *v)
static void rte_atomic64_dec(rte_atomic64_t *v)
static void rte_atomic16_init(rte_atomic16_t *v)
static int rte_atomic64_test_and_set(rte_atomic64_t *v)
static uint64_t rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val)
static int32_t rte_atomic32_read(const rte_atomic32_t *v)
static __rte_experimental int rte_atomic128_cmp_exchange(rte_int128_t *dst, rte_int128_t *exp, const rte_int128_t *src, unsigned int weak, int success, int failure)
static void rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
static int rte_atomic32_dec_and_test(rte_atomic32_t *v)
static void rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
static void rte_io_rmb(void)