10#ifndef _RTE_RING_HTS_ELEM_PVT_H_
11#define _RTE_RING_HTS_ELEM_PVT_H_
13#include <rte_stdatomic.h>
27__rte_ring_hts_update_tail(
struct rte_ring_hts_headtail *ht, uint32_t old_tail,
28 uint32_t num, uint32_t enqueue)
34 tail = old_tail + num;
35 rte_atomic_store_explicit(&ht->ht.pos.tail, tail, rte_memory_order_release);
44__rte_ring_hts_head_wait(
const struct rte_ring_hts_headtail *ht,
45 union __rte_ring_hts_pos *p)
47 while (p->pos.head != p->pos.tail) {
49 p->raw = rte_atomic_load_explicit(&ht->ht.raw, rte_memory_order_acquire);
57__rte_ring_hts_move_prod_head(
struct rte_ring *r,
unsigned int num,
59 uint32_t *free_entries)
62 union __rte_ring_hts_pos np, op;
64 const uint32_t capacity = r->
capacity;
66 op.raw = rte_atomic_load_explicit(&r->hts_prod.ht.raw, rte_memory_order_acquire);
77 __rte_ring_hts_head_wait(&r->hts_prod, &op);
85 *free_entries = capacity + r->cons.tail - op.pos.head;
95 np.pos.tail = op.pos.tail;
96 np.pos.head = op.pos.head + n;
103 }
while (rte_atomic_compare_exchange_strong_explicit(&r->hts_prod.ht.raw,
104 (uint64_t *)(uintptr_t)&op.raw, np.raw,
105 rte_memory_order_acquire, rte_memory_order_acquire) == 0);
107 *old_head = op.pos.head;
115__rte_ring_hts_move_cons_head(
struct rte_ring *r,
unsigned int num,
120 union __rte_ring_hts_pos np, op;
122 op.raw = rte_atomic_load_explicit(&r->hts_cons.ht.raw, rte_memory_order_acquire);
134 __rte_ring_hts_head_wait(&r->hts_cons, &op);
141 *entries = r->prod.tail - op.pos.head;
150 np.pos.tail = op.pos.tail;
151 np.pos.head = op.pos.head + n;
158 }
while (rte_atomic_compare_exchange_strong_explicit(&r->hts_cons.ht.raw,
159 (uint64_t *)(uintptr_t)&op.raw, np.raw,
160 rte_memory_order_acquire, rte_memory_order_acquire) == 0);
162 *old_head = op.pos.head;
189__rte_ring_do_hts_enqueue_elem(
struct rte_ring *r,
const void *obj_table,
191 uint32_t *free_space)
195 n = __rte_ring_hts_move_prod_head(r, n, behavior, &head, &free);
198 __rte_ring_enqueue_elems(r, head, obj_table, esize, n);
199 __rte_ring_hts_update_tail(&r->hts_prod, head, n, 1);
202 if (free_space != NULL)
203 *free_space = free - n;
230__rte_ring_do_hts_dequeue_elem(
struct rte_ring *r,
void *obj_table,
234 uint32_t entries, head;
236 n = __rte_ring_hts_move_cons_head(r, n, behavior, &head, &entries);
239 __rte_ring_dequeue_elems(r, head, obj_table, esize, n);
240 __rte_ring_hts_update_tail(&r->hts_cons, head, n, 0);
243 if (available != NULL)
244 *available = entries - n;
#define __rte_always_inline
static void rte_pause(void)