34 #ifndef _RTE_MEMPOOL_H_
35 #define _RTE_MEMPOOL_H_
67 #include <sys/queue.h>
80 #define RTE_MEMPOOL_HEADER_COOKIE1 0xbadbadbadadd2e55ULL
81 #define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL
82 #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL
84 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
88 struct rte_mempool_debug_stats {
91 uint64_t get_success_bulk;
92 uint64_t get_success_objs;
93 uint64_t get_fail_bulk;
94 uint64_t get_fail_objs;
98 #if RTE_MEMPOOL_CACHE_MAX_SIZE > 0
102 struct rte_mempool_cache {
108 void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 3];
123 #define RTE_MEMPOOL_NAMESIZE 32
124 #define RTE_MEMPOOL_MZ_PREFIX "MP_"
127 #define RTE_MEMPOOL_MZ_FORMAT RTE_MEMPOOL_MZ_PREFIX "%s"
129 #ifdef RTE_LIBRTE_XEN_DOM0
132 #define RTE_MEMPOOL_OBJ_NAME "%s_" RTE_MEMPOOL_MZ_PREFIX "elt"
136 #define RTE_MEMPOOL_OBJ_NAME RTE_MEMPOOL_MZ_FORMAT
140 #define MEMPOOL_PG_SHIFT_MAX (sizeof(uintptr_t) * CHAR_BIT - 1)
143 #define MEMPOOL_PG_NUM_DEFAULT 1
145 #ifndef RTE_MEMPOOL_ALIGN
146 #define RTE_MEMPOOL_ALIGN RTE_CACHE_LINE_SIZE
149 #define RTE_MEMPOOL_ALIGN_MASK (RTE_MEMPOOL_ALIGN - 1)
161 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
173 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
197 #if RTE_MEMPOOL_CACHE_MAX_SIZE > 0
199 struct rte_mempool_cache local_cache[RTE_MAX_LCORE];
202 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
204 struct rte_mempool_debug_stats stats[RTE_MAX_LCORE];
222 #define MEMPOOL_F_NO_SPREAD 0x0001
223 #define MEMPOOL_F_NO_CACHE_ALIGN 0x0002
224 #define MEMPOOL_F_SP_PUT 0x0004
225 #define MEMPOOL_F_SC_GET 0x0008
237 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
238 #define __MEMPOOL_STAT_ADD(mp, name, n) do { \
239 unsigned __lcore_id = rte_lcore_id(); \
240 if (__lcore_id < RTE_MAX_LCORE) { \
241 mp->stats[__lcore_id].name##_objs += n; \
242 mp->stats[__lcore_id].name##_bulk += 1; \
246 #define __MEMPOOL_STAT_ADD(mp, name, n) do {} while(0)
257 #define MEMPOOL_HEADER_SIZE(mp, pgn) (sizeof(*(mp)) + \
258 RTE_ALIGN_CEIL(((pgn) - RTE_DIM((mp)->elt_pa)) * \
259 sizeof ((mp)->elt_pa[0]), RTE_CACHE_LINE_SIZE))
264 #define MEMPOOL_IS_CONTIG(mp) \
265 ((mp)->pg_num == MEMPOOL_PG_NUM_DEFAULT && \
266 (mp)->phys_addr == (mp)->elt_pa[0])
310 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
311 #ifndef __INTEL_COMPILER
312 #pragma GCC diagnostic ignored "-Wcast-qual"
314 static inline void __mempool_check_cookies(
const struct rte_mempool *mp,
315 void *
const *obj_table_const,
316 unsigned n,
int free)
327 tmp = (
void *) obj_table_const;
328 obj_table = (
void **) tmp;
334 rte_panic(
"MEMPOOL: object is owned by another "
337 hdr = __mempool_get_header(obj);
338 cookie = hdr->cookie;
344 "obj=%p, mempool=%p, cookie=%" PRIx64
"\n",
345 obj, (
const void *) mp, cookie);
346 rte_panic(
"MEMPOOL: bad header cookie (put)\n");
350 else if (free == 1) {
354 "obj=%p, mempool=%p, cookie=%" PRIx64
"\n",
355 obj, (
const void *) mp, cookie);
356 rte_panic(
"MEMPOOL: bad header cookie (get)\n");
360 else if (free == 2) {
365 "obj=%p, mempool=%p, cookie=%" PRIx64
"\n",
366 obj, (
const void *) mp, cookie);
367 rte_panic(
"MEMPOOL: bad header cookie (audit)\n");
370 tlr = __mempool_get_trailer(obj);
371 cookie = tlr->cookie;
375 "obj=%p, mempool=%p, cookie=%" PRIx64
"\n",
376 obj, (
const void *) mp, cookie);
377 rte_panic(
"MEMPOOL: bad trailer cookie\n");
381 #ifndef __INTEL_COMPILER
382 #pragma GCC diagnostic error "-Wcast-qual"
385 #define __mempool_check_cookies(mp, obj_table_const, n, free) do {} while(0)
432 uint32_t elt_num,
size_t elt_sz,
size_t align,
433 const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift,
540 int socket_id,
unsigned flags);
640 int socket_id,
unsigned flags,
void *vaddr,
729 int socket_id,
unsigned flags);
754 static inline void __attribute__((always_inline))
755 __mempool_put_bulk(struct
rte_mempool *mp,
void * const *obj_table,
756 unsigned n,
int is_mp)
758 #if RTE_MEMPOOL_CACHE_MAX_SIZE > 0
759 struct rte_mempool_cache *cache;
763 uint32_t cache_size = mp->cache_size;
764 uint32_t flushthresh = mp->cache_flushthresh;
768 __MEMPOOL_STAT_ADD(mp, put, n);
770 #if RTE_MEMPOOL_CACHE_MAX_SIZE > 0
772 if (
unlikely(cache_size == 0 || is_mp == 0 ||
773 lcore_id >= RTE_MAX_LCORE))
777 if (
unlikely(n > RTE_MEMPOOL_CACHE_MAX_SIZE))
780 cache = &mp->local_cache[lcore_id];
781 cache_objs = &cache->objs[cache->len];
791 for (index = 0; index < n; ++index, obj_table++)
792 cache_objs[index] = *obj_table;
796 if (cache->len >= flushthresh) {
798 cache->len - cache_size);
799 cache->len = cache_size;
808 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
811 rte_panic(
"cannot put objects in mempool\n");
815 rte_panic(
"cannot put objects in mempool\n");
836 static inline void __attribute__((always_inline))
840 __mempool_check_cookies(mp, obj_table, n, 0);
841 __mempool_put_bulk(mp, obj_table, n, 1);
858 __mempool_check_cookies(mp, obj_table, n, 0);
859 __mempool_put_bulk(mp, obj_table, n, 0);
876 static inline void __attribute__((always_inline))
880 __mempool_check_cookies(mp, obj_table, n, 0);
892 static inline void __attribute__((always_inline))
906 static inline void __attribute__((always_inline))
924 static inline void __attribute__((always_inline))
944 static inline int __attribute__((always_inline))
945 __mempool_get_bulk(struct
rte_mempool *mp,
void **obj_table,
946 unsigned n,
int is_mc)
949 #if RTE_MEMPOOL_CACHE_MAX_SIZE > 0
950 struct rte_mempool_cache *cache;
954 uint32_t cache_size = mp->cache_size;
957 if (
unlikely(cache_size == 0 || is_mc == 0 ||
958 n >= cache_size || lcore_id >= RTE_MAX_LCORE))
961 cache = &mp->local_cache[lcore_id];
962 cache_objs = cache->objs;
965 if (cache->len < n) {
967 uint32_t req = n + (cache_size - cache->len);
985 for (index = 0, len = cache->len - 1; index < n; ++index, len--, obj_table++)
986 *obj_table = cache_objs[len];
990 __MEMPOOL_STAT_ADD(mp, get_success, n);
1004 __MEMPOOL_STAT_ADD(mp, get_fail, n);
1006 __MEMPOOL_STAT_ADD(mp, get_success, n);
1029 static inline int __attribute__((always_inline))
1033 ret = __mempool_get_bulk(mp, obj_table, n, 1);
1035 __mempool_check_cookies(mp, obj_table, n, 1);
1058 static inline int __attribute__((always_inline))
1062 ret = __mempool_get_bulk(mp, obj_table, n, 0);
1064 __mempool_check_cookies(mp, obj_table, n, 1);
1090 static inline int __attribute__((always_inline))
1094 ret = __mempool_get_bulk(mp, obj_table, n,
1097 __mempool_check_cookies(mp, obj_table, n, 1);
1117 static inline int __attribute__((always_inline))
1139 static inline int __attribute__((always_inline))
1165 static inline int __attribute__((always_inline))
1202 static inline unsigned
1262 off = (
const char *)elt - (
const char *)mp->
elt_va_start;