5 #ifndef _RTE_RCU_QSBR_H_
6 #define _RTE_RCU_QSBR_H_
38 extern int rte_rcu_log_type;
40 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
41 #define __RTE_RCU_DP_LOG(level, fmt, args...) \
42 rte_log(RTE_LOG_ ## level, rte_rcu_log_type, \
43 "%s(): " fmt "\n", __func__, ## args)
45 #define __RTE_RCU_DP_LOG(level, fmt, args...)
48 #if defined(RTE_LIBRTE_RCU_DEBUG)
49 #define __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, level, fmt, args...) do {\
50 if (v->qsbr_cnt[thread_id].lock_cnt) \
51 rte_log(RTE_LOG_ ## level, rte_rcu_log_type, \
52 "%s(): " fmt "\n", __func__, ## args); \
55 #define __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, level, fmt, args...)
62 #define __RTE_QSBR_THRID_ARRAY_ELM_SIZE (sizeof(uint64_t) * 8)
63 #define __RTE_QSBR_THRID_ARRAY_SIZE(max_threads) \
64 RTE_ALIGN(RTE_ALIGN_MUL_CEIL(max_threads, \
65 __RTE_QSBR_THRID_ARRAY_ELM_SIZE) >> 3, RTE_CACHE_LINE_SIZE)
66 #define __RTE_QSBR_THRID_ARRAY_ELM(v, i) ((uint64_t *) \
67 ((struct rte_rcu_qsbr_cnt *)(v + 1) + v->max_threads) + i)
68 #define __RTE_QSBR_THRID_INDEX_SHIFT 6
69 #define __RTE_QSBR_THRID_MASK 0x3f
70 #define RTE_QSBR_THRID_INVALID 0xffffffff
73 struct rte_rcu_qsbr_cnt {
84 #define __RTE_QSBR_CNT_THR_OFFLINE 0
85 #define __RTE_QSBR_CNT_INIT 1
86 #define __RTE_QSBR_CNT_MAX ((uint64_t)~0)
104 uint32_t num_threads;
106 uint32_t max_threads;
109 struct rte_rcu_qsbr_cnt qsbr_cnt[0] __rte_cache_aligned;
240 RTE_ASSERT(v != NULL && thread_id < v->max_threads);
242 __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR,
"Lock counter %u\n",
243 v->qsbr_cnt[thread_id].lock_cnt);
250 t = __atomic_load_n(&v->token, __ATOMIC_RELAXED);
255 __atomic_store_n(&v->qsbr_cnt[thread_id].cnt,
256 t, __ATOMIC_RELAXED);
265 #ifdef RTE_ARCH_X86_64
269 __atomic_thread_fence(__ATOMIC_SEQ_CST);
300 RTE_ASSERT(v != NULL && thread_id < v->max_threads);
302 __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR,
"Lock counter %u\n",
303 v->qsbr_cnt[thread_id].lock_cnt);
310 __atomic_store_n(&v->qsbr_cnt[thread_id].cnt,
311 __RTE_QSBR_CNT_THR_OFFLINE, __ATOMIC_RELEASE);
342 RTE_ASSERT(v != NULL && thread_id < v->max_threads);
344 #if defined(RTE_LIBRTE_RCU_DEBUG)
346 __atomic_fetch_add(&v->qsbr_cnt[thread_id].lock_cnt,
347 1, __ATOMIC_ACQUIRE);
379 RTE_ASSERT(v != NULL && thread_id < v->max_threads);
381 #if defined(RTE_LIBRTE_RCU_DEBUG)
383 __atomic_fetch_sub(&v->qsbr_cnt[thread_id].lock_cnt,
384 1, __ATOMIC_RELEASE);
386 __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, WARNING,
387 "Lock counter %u. Nested locks?\n",
388 v->qsbr_cnt[thread_id].lock_cnt);
414 RTE_ASSERT(v != NULL);
421 t = __atomic_add_fetch(&v->token, 1, __ATOMIC_RELEASE);
447 RTE_ASSERT(v != NULL && thread_id < v->max_threads);
449 __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR,
"Lock counter %u\n",
450 v->qsbr_cnt[thread_id].lock_cnt);
457 t = __atomic_load_n(&v->token, __ATOMIC_ACQUIRE);
464 if (t != __atomic_load_n(&v->qsbr_cnt[thread_id].cnt, __ATOMIC_RELAXED))
465 __atomic_store_n(&v->qsbr_cnt[thread_id].cnt,
466 t, __ATOMIC_RELEASE);
468 __RTE_RCU_DP_LOG(DEBUG,
"%s: update: token = %"PRIu64
", Thread ID = %d",
469 __func__, t, thread_id);
476 __rte_rcu_qsbr_check_selective(
struct rte_rcu_qsbr *v, uint64_t t,
bool wait)
481 uint64_t *reg_thread_id;
482 uint64_t acked_token = __RTE_QSBR_CNT_MAX;
484 for (i = 0, reg_thread_id = __RTE_QSBR_THRID_ARRAY_ELM(v, 0);
486 i++, reg_thread_id++) {
490 bmap = __atomic_load_n(reg_thread_id, __ATOMIC_ACQUIRE);
491 id = i << __RTE_QSBR_THRID_INDEX_SHIFT;
494 j = __builtin_ctzl(bmap);
495 __RTE_RCU_DP_LOG(DEBUG,
496 "%s: check: token = %"PRIu64
", wait = %d, Bit Map = 0x%"PRIx64
", Thread ID = %d",
497 __func__, t, wait, bmap,
id + j);
499 &v->qsbr_cnt[
id + j].cnt,
501 __RTE_RCU_DP_LOG(DEBUG,
502 "%s: status: token = %"PRIu64
", wait = %d, Thread QS cnt = %"PRIu64
", Thread ID = %d",
503 __func__, t, wait, c,
id+j);
509 __RTE_QSBR_CNT_THR_OFFLINE && c < t)) {
518 bmap = __atomic_load_n(reg_thread_id,
528 if (c != __RTE_QSBR_CNT_THR_OFFLINE && acked_token > c)
539 if (acked_token != __RTE_QSBR_CNT_MAX)
540 __atomic_store_n(&v->acked_token, acked_token,
550 __rte_rcu_qsbr_check_all(
struct rte_rcu_qsbr *v, uint64_t t,
bool wait)
553 struct rte_rcu_qsbr_cnt *cnt;
555 uint64_t acked_token = __RTE_QSBR_CNT_MAX;
557 for (i = 0, cnt = v->qsbr_cnt; i < v->max_threads; i++, cnt++) {
558 __RTE_RCU_DP_LOG(DEBUG,
559 "%s: check: token = %"PRIu64
", wait = %d, Thread ID = %d",
560 __func__, t, wait, i);
562 c = __atomic_load_n(&cnt->cnt, __ATOMIC_ACQUIRE);
563 __RTE_RCU_DP_LOG(DEBUG,
564 "%s: status: token = %"PRIu64
", wait = %d, Thread QS cnt = %"PRIu64
", Thread ID = %d",
565 __func__, t, wait, c, i);
570 if (
likely(c == __RTE_QSBR_CNT_THR_OFFLINE || c >= t))
583 if (
likely(c != __RTE_QSBR_CNT_THR_OFFLINE && acked_token > c))
591 if (acked_token != __RTE_QSBR_CNT_MAX)
592 __atomic_store_n(&v->acked_token, acked_token,
636 RTE_ASSERT(v != NULL);
639 if (
likely(t <= v->acked_token))
642 if (
likely(v->num_threads == v->max_threads))
643 return __rte_rcu_qsbr_check_all(v, t, wait);
645 return __rte_rcu_qsbr_check_selective(v, t, wait);
static __rte_experimental __rte_always_inline int rte_rcu_qsbr_check(struct rte_rcu_qsbr *v, uint64_t t, bool wait)
__rte_experimental int rte_rcu_qsbr_thread_register(struct rte_rcu_qsbr *v, unsigned int thread_id)
#define __rte_always_inline
static __rte_experimental __rte_always_inline void rte_rcu_qsbr_thread_online(struct rte_rcu_qsbr *v, unsigned int thread_id)
__rte_experimental int rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v)
static __rte_experimental __rte_always_inline void rte_rcu_qsbr_quiescent(struct rte_rcu_qsbr *v, unsigned int thread_id)
static __rte_experimental __rte_always_inline void rte_rcu_qsbr_thread_offline(struct rte_rcu_qsbr *v, unsigned int thread_id)
static void rte_pause(void)
static __rte_experimental __rte_always_inline void rte_rcu_qsbr_unlock(__rte_unused struct rte_rcu_qsbr *v, __rte_unused unsigned int thread_id)
__rte_experimental int rte_rcu_qsbr_thread_unregister(struct rte_rcu_qsbr *v, unsigned int thread_id)
static __rte_experimental __rte_always_inline uint64_t rte_rcu_qsbr_start(struct rte_rcu_qsbr *v)
__rte_experimental void rte_rcu_qsbr_synchronize(struct rte_rcu_qsbr *v, unsigned int thread_id)
static __rte_experimental __rte_always_inline void rte_rcu_qsbr_lock(__rte_unused struct rte_rcu_qsbr *v, __rte_unused unsigned int thread_id)
#define __rte_cache_aligned
static void rte_smp_mb(void)
__rte_experimental size_t rte_rcu_qsbr_get_memsize(uint32_t max_threads)
__rte_experimental int rte_rcu_qsbr_init(struct rte_rcu_qsbr *v, uint32_t max_threads)