6 #ifndef _RTE_MEMPOOL_H_
7 #define _RTE_MEMPOOL_H_
41 #include <sys/queue.h>
43 #include <rte_config.h>
58 #define RTE_MEMPOOL_HEADER_COOKIE1 0xbadbadbadadd2e55ULL
59 #define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL
60 #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL
62 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
66 struct rte_mempool_debug_stats {
69 uint64_t get_success_bulk;
70 uint64_t get_success_objs;
71 uint64_t get_fail_bulk;
72 uint64_t get_fail_objs;
74 uint64_t get_success_blks;
76 uint64_t get_fail_blks;
91 void *
objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 3];
106 #define RTE_MEMPOOL_NAMESIZE (RTE_RING_NAMESIZE - \
107 sizeof(RTE_MEMPOOL_MZ_PREFIX) + 1)
108 #define RTE_MEMPOOL_MZ_PREFIX "MP_"
111 #define RTE_MEMPOOL_MZ_FORMAT RTE_MEMPOOL_MZ_PREFIX "%s"
113 #define MEMPOOL_PG_SHIFT_MAX (sizeof(uintptr_t) * CHAR_BIT - 1)
116 #define MEMPOOL_PG_NUM_DEFAULT 1
118 #ifndef RTE_MEMPOOL_ALIGN
119 #define RTE_MEMPOOL_ALIGN RTE_CACHE_LINE_SIZE
122 #define RTE_MEMPOOL_ALIGN_MASK (RTE_MEMPOOL_ALIGN - 1)
141 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
151 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
159 struct rte_mempool_objtlr {
254 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
256 struct rte_mempool_debug_stats stats[RTE_MAX_LCORE];
260 #define MEMPOOL_F_NO_SPREAD 0x0001
261 #define MEMPOOL_F_NO_CACHE_ALIGN 0x0002
262 #define MEMPOOL_F_SP_PUT 0x0004
263 #define MEMPOOL_F_SC_GET 0x0008
264 #define MEMPOOL_F_POOL_CREATED 0x0010
265 #define MEMPOOL_F_NO_IOVA_CONTIG 0x0020
266 #define MEMPOOL_F_NO_PHYS_CONTIG MEMPOOL_F_NO_IOVA_CONTIG
278 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
279 #define __MEMPOOL_STAT_ADD(mp, name, n) do { \
280 unsigned __lcore_id = rte_lcore_id(); \
281 if (__lcore_id < RTE_MAX_LCORE) { \
282 mp->stats[__lcore_id].name##_objs += n; \
283 mp->stats[__lcore_id].name##_bulk += 1; \
286 #define __MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, name, n) do { \
287 unsigned int __lcore_id = rte_lcore_id(); \
288 if (__lcore_id < RTE_MAX_LCORE) { \
289 mp->stats[__lcore_id].name##_blks += n; \
290 mp->stats[__lcore_id].name##_bulk += 1; \
294 #define __MEMPOOL_STAT_ADD(mp, name, n) do {} while(0)
295 #define __MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, name, n) do {} while (0)
306 #define MEMPOOL_HEADER_SIZE(mp, cs) \
307 (sizeof(*(mp)) + (((cs) == 0) ? 0 : \
308 (sizeof(struct rte_mempool_cache) * RTE_MAX_LCORE)))
333 static inline struct rte_mempool_objtlr *__mempool_get_trailer(
void *obj)
353 void rte_mempool_check_cookies(
const struct rte_mempool *mp,
354 void *
const *obj_table_const,
unsigned n,
int free);
356 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
357 #define __mempool_check_cookies(mp, obj_table_const, n, free) \
358 rte_mempool_check_cookies(mp, obj_table_const, n, free)
360 #define __mempool_check_cookies(mp, obj_table_const, n, free) do {} while(0)
382 void *
const *first_obj_table_const,
unsigned int n,
int free);
384 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
385 #define __mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
387 rte_mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
390 #define __mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
395 #define RTE_MEMPOOL_OPS_NAMESIZE 32
407 typedef int (*rte_mempool_alloc_t)(struct rte_mempool *mp);
418 void *
const *obj_table,
unsigned int n);
424 void **obj_table,
unsigned int n);
433 void **first_obj_table,
unsigned int n);
464 uint32_t obj_num, uint32_t pg_shift,
465 size_t *min_chunk_size,
size_t *align);
487 uint32_t obj_num, uint32_t pg_shift,
488 size_t *min_chunk_size,
size_t *align);
534 unsigned int max_objs,
543 unsigned int max_objs,
585 #define RTE_MEMPOOL_MAX_OPS_IDX 16
596 struct rte_mempool_ops_table {
618 rte_mempool_get_ops(
int ops_index)
651 rte_mempool_ops_dequeue_bulk(
struct rte_mempool *mp,
652 void **obj_table,
unsigned n)
656 ops = rte_mempool_get_ops(mp->
ops_index);
657 return ops->
dequeue(mp, obj_table, n);
674 rte_mempool_ops_dequeue_contig_blocks(
struct rte_mempool *mp,
675 void **first_obj_table,
unsigned int n)
679 ops = rte_mempool_get_ops(mp->
ops_index);
698 rte_mempool_ops_enqueue_bulk(
struct rte_mempool *mp,
void *
const *obj_table,
703 ops = rte_mempool_get_ops(mp->
ops_index);
704 return ops->
enqueue(mp, obj_table, n);
716 rte_mempool_ops_get_count(
const struct rte_mempool *mp);
737 ssize_t rte_mempool_ops_calc_mem_size(
const struct rte_mempool *mp,
738 uint32_t obj_num, uint32_t pg_shift,
739 size_t *min_chunk_size,
size_t *align);
764 int rte_mempool_ops_populate(
struct rte_mempool *mp,
unsigned int max_objs,
835 #define MEMPOOL_REGISTER_OPS(ops) \
836 void mp_hdlr_init_##ops(void); \
837 void __attribute__((constructor, used)) mp_hdlr_init_##ops(void)\
839 rte_mempool_register_ops(&ops); \
848 void *opaque,
void *obj,
unsigned obj_idx);
1183 if (lcore_id >= RTE_MAX_LCORE)
1203 if (cache == NULL || cache->
len == 0)
1205 rte_mempool_ops_enqueue_bulk(mp, cache->
objs, cache->
len);
1222 __mempool_generic_put(
struct rte_mempool *mp,
void *
const *obj_table,
1228 __MEMPOOL_STAT_ADD(mp, put, n);
1231 if (
unlikely(cache == NULL || n > RTE_MEMPOOL_CACHE_MAX_SIZE))
1234 cache_objs = &cache->
objs[cache->
len];
1244 rte_memcpy(&cache_objs[0], obj_table,
sizeof(
void *) * n);
1249 rte_mempool_ops_enqueue_bulk(mp, &cache->
objs[cache->
size],
1259 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
1260 if (rte_mempool_ops_enqueue_bulk(mp, obj_table, n) < 0)
1261 rte_panic(
"cannot put objects in mempool\n");
1263 rte_mempool_ops_enqueue_bulk(mp, obj_table, n);
1284 __mempool_check_cookies(mp, obj_table, n, 0);
1285 __mempool_generic_put(mp, obj_table, n, cache);
1344 __mempool_generic_get(
struct rte_mempool *mp,
void **obj_table,
1348 uint32_t index,
len;
1355 cache_objs = cache->
objs;
1358 if (cache->
len < n) {
1360 uint32_t req = n + (cache->
size - cache->
len);
1363 ret = rte_mempool_ops_dequeue_bulk(mp,
1364 &cache->
objs[cache->
len], req);
1379 for (index = 0, len = cache->
len - 1; index < n; ++index, len--, obj_table++)
1380 *obj_table = cache_objs[
len];
1384 __MEMPOOL_STAT_ADD(mp, get_success, n);
1391 ret = rte_mempool_ops_dequeue_bulk(mp, obj_table, n);
1394 __MEMPOOL_STAT_ADD(mp, get_fail, n);
1396 __MEMPOOL_STAT_ADD(mp, get_success, n);
1426 ret = __mempool_generic_get(mp, obj_table, n, cache);
1428 __mempool_check_cookies(mp, obj_table, n, 1);
1515 void **first_obj_table,
unsigned int n)
1519 ret = rte_mempool_ops_dequeue_contig_blocks(mp, first_obj_table, n);
1521 __MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, get_success, n);
1522 __mempool_contig_blocks_check_cookies(mp, first_obj_table, n,
1525 __MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, get_fail, n);