35 #ifndef _RTE_MEMPOOL_H_
36 #define _RTE_MEMPOOL_H_
70 #include <sys/queue.h>
86 #define RTE_MEMPOOL_HEADER_COOKIE1 0xbadbadbadadd2e55ULL
87 #define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL
88 #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL
90 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
94 struct rte_mempool_debug_stats {
97 uint64_t get_success_bulk;
98 uint64_t get_success_objs;
99 uint64_t get_fail_bulk;
100 uint64_t get_fail_objs;
115 void *
objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 3];
130 #define RTE_MEMPOOL_NAMESIZE (RTE_RING_NAMESIZE - \
131 sizeof(RTE_MEMPOOL_MZ_PREFIX) + 1)
132 #define RTE_MEMPOOL_MZ_PREFIX "MP_"
135 #define RTE_MEMPOOL_MZ_FORMAT RTE_MEMPOOL_MZ_PREFIX "%s"
137 #define MEMPOOL_PG_SHIFT_MAX (sizeof(uintptr_t) * CHAR_BIT - 1)
140 #define MEMPOOL_PG_NUM_DEFAULT 1
142 #ifndef RTE_MEMPOOL_ALIGN
143 #define RTE_MEMPOOL_ALIGN RTE_CACHE_LINE_SIZE
146 #define RTE_MEMPOOL_ALIGN_MASK (RTE_MEMPOOL_ALIGN - 1)
161 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
171 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
179 struct rte_mempool_objtlr {
256 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
258 struct rte_mempool_debug_stats stats[RTE_MAX_LCORE];
262 #define MEMPOOL_F_NO_SPREAD 0x0001
263 #define MEMPOOL_F_NO_CACHE_ALIGN 0x0002
264 #define MEMPOOL_F_SP_PUT 0x0004
265 #define MEMPOOL_F_SC_GET 0x0008
266 #define MEMPOOL_F_POOL_CREATED 0x0010
267 #define MEMPOOL_F_NO_PHYS_CONTIG 0x0020
279 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
280 #define __MEMPOOL_STAT_ADD(mp, name, n) do { \
281 unsigned __lcore_id = rte_lcore_id(); \
282 if (__lcore_id < RTE_MAX_LCORE) { \
283 mp->stats[__lcore_id].name##_objs += n; \
284 mp->stats[__lcore_id].name##_bulk += 1; \
288 #define __MEMPOOL_STAT_ADD(mp, name, n) do {} while(0)
299 #define MEMPOOL_HEADER_SIZE(mp, cs) \
300 (sizeof(*(mp)) + (((cs) == 0) ? 0 : \
301 (sizeof(struct rte_mempool_cache) * RTE_MAX_LCORE)))
326 static inline struct rte_mempool_objtlr *__mempool_get_trailer(
void *obj)
346 void rte_mempool_check_cookies(
const struct rte_mempool *mp,
347 void *
const *obj_table_const,
unsigned n,
int free);
349 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
350 #define __mempool_check_cookies(mp, obj_table_const, n, free) \
351 rte_mempool_check_cookies(mp, obj_table_const, n, free)
353 #define __mempool_check_cookies(mp, obj_table_const, n, free) do {} while(0)
356 #define RTE_MEMPOOL_OPS_NAMESIZE 32
368 typedef int (*rte_mempool_alloc_t)(struct rte_mempool *mp);
379 void *
const *obj_table,
unsigned int n);
385 void **obj_table,
unsigned int n);
402 #define RTE_MEMPOOL_MAX_OPS_IDX 16
413 struct rte_mempool_ops_table {
435 rte_mempool_get_ops(
int ops_index)
469 void **obj_table,
unsigned n)
473 ops = rte_mempool_get_ops(mp->
ops_index);
474 return ops->
dequeue(mp, obj_table, n);
491 rte_mempool_ops_enqueue_bulk(
struct rte_mempool *mp,
void *
const *obj_table,
496 ops = rte_mempool_get_ops(mp->
ops_index);
497 return ops->
enqueue(mp, obj_table, n);
509 rte_mempool_ops_get_count(
const struct rte_mempool *mp);
558 #define MEMPOOL_REGISTER_OPS(ops) \
559 void mp_hdlr_init_##ops(void); \
560 void __attribute__((constructor, used)) mp_hdlr_init_##ops(void)\
562 rte_mempool_register_ops(&ops); \
571 void *opaque,
void *obj,
unsigned obj_idx);
742 const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift);
854 const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift,
996 static inline void __attribute__((always_inline))
1000 rte_mempool_ops_enqueue_bulk(mp, cache->objs, cache->len);
1017 if (mp->cache_size == 0)
1020 if (lcore_id >= RTE_MAX_LCORE)
1023 return &mp->local_cache[lcore_id];
1041 static inline void __attribute__((always_inline))
1042 __mempool_generic_put(struct
rte_mempool *mp,
void * const *obj_table,
1048 __MEMPOOL_STAT_ADD(mp, put, n);
1051 if (
unlikely(cache == NULL || n > RTE_MEMPOOL_CACHE_MAX_SIZE))
1054 cache_objs = &cache->objs[cache->len];
1064 rte_memcpy(&cache_objs[0], obj_table,
sizeof(
void *) * n);
1068 if (cache->len >= cache->flushthresh) {
1069 rte_mempool_ops_enqueue_bulk(mp, &cache->objs[cache->size],
1070 cache->len - cache->size);
1071 cache->len = cache->size;
1079 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
1080 if (rte_mempool_ops_enqueue_bulk(mp, obj_table, n) < 0)
1081 rte_panic(
"cannot put objects in mempool\n");
1083 rte_mempool_ops_enqueue_bulk(mp, obj_table, n);
1103 static inline void __attribute__((always_inline))
1108 __mempool_check_cookies(mp, obj_table, n, 0);
1109 __mempool_generic_put(mp, obj_table, n, cache);
1126 static inline void __attribute__((always_inline))
1147 static inline void __attribute__((always_inline))
1170 static inline int __attribute__((always_inline))
1171 __mempool_generic_get(struct
rte_mempool *mp,
void **obj_table,
1175 uint32_t index,
len;
1179 if (
unlikely(cache == NULL || n >= cache->size))
1182 cache_objs = cache->objs;
1185 if (cache->len < n) {
1187 uint32_t req = n + (cache->size - cache->len);
1190 ret = rte_mempool_ops_dequeue_bulk(mp,
1191 &cache->objs[cache->len], req);
1206 for (index = 0, len = cache->len - 1; index < n; ++index, len--, obj_table++)
1207 *obj_table = cache_objs[
len];
1211 __MEMPOOL_STAT_ADD(mp, get_success, n);
1218 ret = rte_mempool_ops_dequeue_bulk(mp, obj_table, n);
1221 __MEMPOOL_STAT_ADD(mp, get_fail, n);
1223 __MEMPOOL_STAT_ADD(mp, get_success, n);
1251 static inline int __attribute__((always_inline))
1256 ret = __mempool_generic_get(mp, obj_table, n, cache);
1258 __mempool_check_cookies(mp, obj_table, n, 1);
1284 static inline int __attribute__((always_inline))
1312 static inline int __attribute__((always_inline))
1526 size_t total_elt_sz,
const phys_addr_t paddr[], uint32_t pg_num,