41 #include <sys/queue.h>
44 #include <rte_config.h>
52 #define RTE_TAILQ_RING_NAME "RTE_RING"
54 enum rte_ring_queue_behavior {
55 RTE_RING_QUEUE_FIXED = 0,
56 RTE_RING_QUEUE_VARIABLE
59 #define RTE_RING_MZ_PREFIX "RG_"
61 #define RTE_RING_NAMESIZE (RTE_MEMZONE_NAMESIZE - \
62 sizeof(RTE_RING_MZ_PREFIX) + 1)
65 struct rte_ring_headtail {
66 volatile uint32_t head;
67 volatile uint32_t tail;
98 struct rte_ring_headtail prod __rte_cache_aligned;
102 struct rte_ring_headtail cons __rte_cache_aligned;
106 #define RING_F_SP_ENQ 0x0001
107 #define RING_F_SC_DEQ 0x0002
116 #define RING_F_EXACT_SZ 0x0004
117 #define RTE_RING_SZ_MASK (0x7fffffffU)
218 int socket_id,
unsigned flags);
240 #define ENQUEUE_PTRS(r, ring_start, prod_head, obj_table, n, obj_type) do { \
242 const uint32_t size = (r)->size; \
243 uint32_t idx = prod_head & (r)->mask; \
244 obj_type *ring = (obj_type *)ring_start; \
245 if (likely(idx + n < size)) { \
246 for (i = 0; i < (n & ((~(unsigned)0x3))); i+=4, idx+=4) { \
247 ring[idx] = obj_table[i]; \
248 ring[idx+1] = obj_table[i+1]; \
249 ring[idx+2] = obj_table[i+2]; \
250 ring[idx+3] = obj_table[i+3]; \
254 ring[idx++] = obj_table[i++]; \
256 ring[idx++] = obj_table[i++]; \
258 ring[idx++] = obj_table[i++]; \
261 for (i = 0; idx < size; i++, idx++)\
262 ring[idx] = obj_table[i]; \
263 for (idx = 0; i < n; i++, idx++) \
264 ring[idx] = obj_table[i]; \
271 #define DEQUEUE_PTRS(r, ring_start, cons_head, obj_table, n, obj_type) do { \
273 uint32_t idx = cons_head & (r)->mask; \
274 const uint32_t size = (r)->size; \
275 obj_type *ring = (obj_type *)ring_start; \
276 if (likely(idx + n < size)) { \
277 for (i = 0; i < (n & (~(unsigned)0x3)); i+=4, idx+=4) {\
278 obj_table[i] = ring[idx]; \
279 obj_table[i+1] = ring[idx+1]; \
280 obj_table[i+2] = ring[idx+2]; \
281 obj_table[i+3] = ring[idx+3]; \
285 obj_table[i++] = ring[idx++]; \
287 obj_table[i++] = ring[idx++]; \
289 obj_table[i++] = ring[idx++]; \
292 for (i = 0; idx < size; i++, idx++) \
293 obj_table[i] = ring[idx]; \
294 for (idx = 0; i < n; i++, idx++) \
295 obj_table[i] = ring[idx]; \
308 #ifdef RTE_USE_C11_MEM_MODEL
309 #include "rte_ring_c11_mem.h"
311 #include "rte_ring_generic.h"
335 __rte_ring_do_enqueue(
struct rte_ring *r,
void *
const *obj_table,
336 unsigned int n,
enum rte_ring_queue_behavior behavior,
337 unsigned int is_sp,
unsigned int *free_space)
339 uint32_t prod_head, prod_next;
340 uint32_t free_entries;
342 n = __rte_ring_move_prod_head(r, is_sp, n, behavior,
343 &prod_head, &prod_next, &free_entries);
347 ENQUEUE_PTRS(r, &r[1], prod_head, obj_table, n,
void *);
349 update_tail(&r->prod, prod_head, prod_next, is_sp, 1);
351 if (free_space != NULL)
352 *free_space = free_entries - n;
377 __rte_ring_do_dequeue(
struct rte_ring *r,
void **obj_table,
378 unsigned int n,
enum rte_ring_queue_behavior behavior,
379 unsigned int is_sc,
unsigned int *available)
381 uint32_t cons_head, cons_next;
384 n = __rte_ring_move_cons_head(r, (
int)is_sc, n, behavior,
385 &cons_head, &cons_next, &entries);
389 DEQUEUE_PTRS(r, &r[1], cons_head, obj_table, n,
void *);
391 update_tail(&r->cons, cons_head, cons_next, is_sc, 0);
394 if (available != NULL)
395 *available = entries - n;
419 unsigned int n,
unsigned int *free_space)
421 return __rte_ring_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_FIXED,
422 __IS_MP, free_space);
442 unsigned int n,
unsigned int *free_space)
444 return __rte_ring_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_FIXED,
445 __IS_SP, free_space);
469 unsigned int n,
unsigned int *free_space)
471 return __rte_ring_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_FIXED,
472 r->prod.single, free_space);
553 unsigned int n,
unsigned int *available)
555 return __rte_ring_do_dequeue(r, obj_table, n, RTE_RING_QUEUE_FIXED,
577 unsigned int n,
unsigned int *available)
579 return __rte_ring_do_dequeue(r, obj_table, n, RTE_RING_QUEUE_FIXED,
604 unsigned int *available)
606 return __rte_ring_do_dequeue(r, obj_table, n, RTE_RING_QUEUE_FIXED,
607 r->cons.single, available);
696 static inline unsigned
699 uint32_t prod_tail = r->prod.tail;
700 uint32_t cons_tail = r->cons.tail;
701 uint32_t count = (prod_tail - cons_tail) & r->
mask;
713 static inline unsigned
759 static inline unsigned int
773 static inline unsigned int
819 unsigned int n,
unsigned int *free_space)
821 return __rte_ring_do_enqueue(r, obj_table, n,
822 RTE_RING_QUEUE_VARIABLE, __IS_MP, free_space);
842 unsigned int n,
unsigned int *free_space)
844 return __rte_ring_do_enqueue(r, obj_table, n,
845 RTE_RING_QUEUE_VARIABLE, __IS_SP, free_space);
869 unsigned int n,
unsigned int *free_space)
871 return __rte_ring_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_VARIABLE,
872 r->prod.single, free_space);
897 unsigned int n,
unsigned int *available)
899 return __rte_ring_do_dequeue(r, obj_table, n,
900 RTE_RING_QUEUE_VARIABLE, __IS_MC, available);
922 unsigned int n,
unsigned int *available)
924 return __rte_ring_do_dequeue(r, obj_table, n,
925 RTE_RING_QUEUE_VARIABLE, __IS_SC, available);
949 unsigned int n,
unsigned int *available)
951 return __rte_ring_do_dequeue(r, obj_table, n,
952 RTE_RING_QUEUE_VARIABLE,
953 r->cons.single, available);
#define __rte_always_inline
const struct rte_memzone * memzone
static __rte_always_inline unsigned int rte_ring_enqueue_bulk(struct rte_ring *r, void *const *obj_table, unsigned int n, unsigned int *free_space)
char pad2 __rte_cache_aligned
char name[RTE_MEMZONE_NAMESIZE] __rte_cache_aligned
static __rte_always_inline int rte_ring_dequeue(struct rte_ring *r, void **obj_p)
static __rte_always_inline unsigned rte_ring_mc_dequeue_burst(struct rte_ring *r, void **obj_table, unsigned int n, unsigned int *available)
static int rte_ring_empty(const struct rte_ring *r)
static __rte_always_inline unsigned rte_ring_mp_enqueue_burst(struct rte_ring *r, void *const *obj_table, unsigned int n, unsigned int *free_space)
static __rte_always_inline int rte_ring_mc_dequeue(struct rte_ring *r, void **obj_p)
void rte_ring_list_dump(FILE *f)
static __rte_always_inline unsigned int rte_ring_mp_enqueue_bulk(struct rte_ring *r, void *const *obj_table, unsigned int n, unsigned int *free_space)
static __rte_always_inline int rte_ring_sp_enqueue(struct rte_ring *r, void *obj)
static __rte_always_inline int rte_ring_mp_enqueue(struct rte_ring *r, void *obj)
static unsigned int rte_ring_get_capacity(const struct rte_ring *r)
static unsigned int rte_ring_get_size(const struct rte_ring *r)
char pad0 __rte_cache_aligned
static __rte_always_inline int rte_ring_sc_dequeue(struct rte_ring *r, void **obj_p)
char pad1 __rte_cache_aligned
void rte_ring_free(struct rte_ring *r)
static __rte_always_inline unsigned rte_ring_sp_enqueue_burst(struct rte_ring *r, void *const *obj_table, unsigned int n, unsigned int *free_space)
static __rte_always_inline unsigned rte_ring_sc_dequeue_burst(struct rte_ring *r, void **obj_table, unsigned int n, unsigned int *available)
static __rte_always_inline unsigned rte_ring_dequeue_burst(struct rte_ring *r, void **obj_table, unsigned int n, unsigned int *available)
static __rte_always_inline unsigned int rte_ring_sp_enqueue_bulk(struct rte_ring *r, void *const *obj_table, unsigned int n, unsigned int *free_space)
void rte_ring_dump(FILE *f, const struct rte_ring *r)
static unsigned rte_ring_count(const struct rte_ring *r)
struct rte_ring * rte_ring_create(const char *name, unsigned count, int socket_id, unsigned flags)
static __rte_always_inline unsigned int rte_ring_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned int n, unsigned int *available)
static __rte_always_inline unsigned int rte_ring_sc_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned int n, unsigned int *available)
struct rte_ring * rte_ring_lookup(const char *name)
static unsigned rte_ring_free_count(const struct rte_ring *r)
static __rte_always_inline int rte_ring_enqueue(struct rte_ring *r, void *obj)
static __rte_always_inline unsigned rte_ring_enqueue_burst(struct rte_ring *r, void *const *obj_table, unsigned int n, unsigned int *free_space)
__rte_experimental void rte_ring_reset(struct rte_ring *r)
static __rte_always_inline unsigned int rte_ring_mc_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned int n, unsigned int *available)
int rte_ring_init(struct rte_ring *r, const char *name, unsigned count, unsigned flags)
static int rte_ring_full(const struct rte_ring *r)
ssize_t rte_ring_get_memsize(unsigned count)
#define RTE_MEMZONE_NAMESIZE