DPDK  21.08.0
rte_mempool.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2014 Intel Corporation.
3  * Copyright(c) 2016 6WIND S.A.
4  */
5 
6 #ifndef _RTE_MEMPOOL_H_
7 #define _RTE_MEMPOOL_H_
8 
36 #include <stdio.h>
37 #include <stdlib.h>
38 #include <stdint.h>
39 #include <errno.h>
40 #include <inttypes.h>
41 #include <sys/queue.h>
42 
43 #include <rte_config.h>
44 #include <rte_spinlock.h>
45 #include <rte_log.h>
46 #include <rte_debug.h>
47 #include <rte_lcore.h>
48 #include <rte_memory.h>
49 #include <rte_branch_prediction.h>
50 #include <rte_ring.h>
51 #include <rte_memcpy.h>
52 #include <rte_common.h>
53 
54 #include "rte_mempool_trace_fp.h"
55 
56 #ifdef __cplusplus
57 extern "C" {
58 #endif
59 
60 #define RTE_MEMPOOL_HEADER_COOKIE1 0xbadbadbadadd2e55ULL
61 #define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL
62 #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL
64 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
65 
71 struct rte_mempool_debug_stats {
72  uint64_t put_bulk;
73  uint64_t put_objs;
74  uint64_t put_common_pool_bulk;
75  uint64_t put_common_pool_objs;
76  uint64_t get_common_pool_bulk;
77  uint64_t get_common_pool_objs;
78  uint64_t get_success_bulk;
79  uint64_t get_success_objs;
80  uint64_t get_fail_bulk;
81  uint64_t get_fail_objs;
82  uint64_t get_success_blks;
83  uint64_t get_fail_blks;
85 #endif
86 
91  uint32_t size;
92  uint32_t flushthresh;
93  uint32_t len;
94  /*
95  * Cache is allocated to this size to allow it to overflow in certain
96  * cases to avoid needless emptying of cache.
97  */
98  void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 3];
100 
105  uint32_t elt_size;
106  uint32_t header_size;
107  uint32_t trailer_size;
108  uint32_t total_size;
110 };
111 
113 #define RTE_MEMPOOL_NAMESIZE (RTE_RING_NAMESIZE - \
114  sizeof(RTE_MEMPOOL_MZ_PREFIX) + 1)
115 #define RTE_MEMPOOL_MZ_PREFIX "MP_"
116 
117 /* "MP_<name>" */
118 #define RTE_MEMPOOL_MZ_FORMAT RTE_MEMPOOL_MZ_PREFIX "%s"
119 
120 #define MEMPOOL_PG_SHIFT_MAX (sizeof(uintptr_t) * CHAR_BIT - 1)
121 
123 #define MEMPOOL_PG_NUM_DEFAULT 1
124 
125 #ifndef RTE_MEMPOOL_ALIGN
126 
129 #define RTE_MEMPOOL_ALIGN RTE_CACHE_LINE_SIZE
130 #endif
131 
132 #define RTE_MEMPOOL_ALIGN_MASK (RTE_MEMPOOL_ALIGN - 1)
133 
144  STAILQ_ENTRY(rte_mempool_objhdr) next;
145  struct rte_mempool *mp;
147 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
148  uint64_t cookie;
149 #endif
150 };
151 
155 STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr);
156 
157 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
158 
165 struct rte_mempool_objtlr {
166  uint64_t cookie;
167 };
168 
169 #endif
170 
174 STAILQ_HEAD(rte_mempool_memhdr_list, rte_mempool_memhdr);
175 
180  void *opaque);
181 
189  STAILQ_ENTRY(rte_mempool_memhdr) next;
190  struct rte_mempool *mp;
191  void *addr;
193  size_t len;
195  void *opaque;
196 };
197 
206  unsigned int contig_block_size;
208 
212 struct rte_mempool {
213  /*
214  * Note: this field kept the RTE_MEMZONE_NAMESIZE size due to ABI
215  * compatibility requirements, it could be changed to
216  * RTE_MEMPOOL_NAMESIZE next time the ABI changes
217  */
218  char name[RTE_MEMZONE_NAMESIZE];
220  union {
221  void *pool_data;
222  uint64_t pool_id;
223  };
224  void *pool_config;
225  const struct rte_memzone *mz;
226  unsigned int flags;
227  int socket_id;
228  uint32_t size;
229  uint32_t cache_size;
232  uint32_t elt_size;
233  uint32_t header_size;
234  uint32_t trailer_size;
236  unsigned private_data_size;
244  int32_t ops_index;
245 
248  uint32_t populated_size;
249  struct rte_mempool_objhdr_list elt_list;
250  uint32_t nb_mem_chunks;
251  struct rte_mempool_memhdr_list mem_list;
253 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
254 
255  struct rte_mempool_debug_stats stats[RTE_MAX_LCORE];
256 #endif
258 
259 #define MEMPOOL_F_NO_SPREAD 0x0001
260 
261 #define MEMPOOL_F_NO_CACHE_ALIGN 0x0002
262 #define MEMPOOL_F_SP_PUT 0x0004
263 #define MEMPOOL_F_SC_GET 0x0008
264 #define MEMPOOL_F_POOL_CREATED 0x0010
265 #define MEMPOOL_F_NO_IOVA_CONTIG 0x0020
277 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
278 #define __MEMPOOL_STAT_ADD(mp, name, n) do { \
279  unsigned __lcore_id = rte_lcore_id(); \
280  if (__lcore_id < RTE_MAX_LCORE) { \
281  mp->stats[__lcore_id].name += n; \
282  } \
283  } while(0)
284 #else
285 #define __MEMPOOL_STAT_ADD(mp, name, n) do {} while(0)
286 #endif
287 
296 #define MEMPOOL_HEADER_SIZE(mp, cs) \
297  (sizeof(*(mp)) + (((cs) == 0) ? 0 : \
298  (sizeof(struct rte_mempool_cache) * RTE_MAX_LCORE)))
299 
300 /* return the header of a mempool object (internal) */
301 static inline struct rte_mempool_objhdr *__mempool_get_header(void *obj)
302 {
303  return (struct rte_mempool_objhdr *)RTE_PTR_SUB(obj,
304  sizeof(struct rte_mempool_objhdr));
305 }
306 
316 static inline struct rte_mempool *rte_mempool_from_obj(void *obj)
317 {
318  struct rte_mempool_objhdr *hdr = __mempool_get_header(obj);
319  return hdr->mp;
320 }
321 
322 /* return the trailer of a mempool object (internal) */
323 static inline struct rte_mempool_objtlr *__mempool_get_trailer(void *obj)
324 {
325  struct rte_mempool *mp = rte_mempool_from_obj(obj);
326  return (struct rte_mempool_objtlr *)RTE_PTR_ADD(obj, mp->elt_size);
327 }
328 
343 void rte_mempool_check_cookies(const struct rte_mempool *mp,
344  void * const *obj_table_const, unsigned n, int free);
345 
346 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
347 #define __mempool_check_cookies(mp, obj_table_const, n, free) \
348  rte_mempool_check_cookies(mp, obj_table_const, n, free)
349 #else
350 #define __mempool_check_cookies(mp, obj_table_const, n, free) do {} while(0)
351 #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
352 
368 void rte_mempool_contig_blocks_check_cookies(const struct rte_mempool *mp,
369  void * const *first_obj_table_const, unsigned int n, int free);
370 
371 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
372 #define __mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
373  free) \
374  rte_mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
375  free)
376 #else
377 #define __mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
378  free) \
379  do {} while (0)
380 #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
381 
382 #define RTE_MEMPOOL_OPS_NAMESIZE 32
394 typedef int (*rte_mempool_alloc_t)(struct rte_mempool *mp);
395 
399 typedef void (*rte_mempool_free_t)(struct rte_mempool *mp);
400 
404 typedef int (*rte_mempool_enqueue_t)(struct rte_mempool *mp,
405  void * const *obj_table, unsigned int n);
406 
410 typedef int (*rte_mempool_dequeue_t)(struct rte_mempool *mp,
411  void **obj_table, unsigned int n);
412 
417  void **first_obj_table, unsigned int n);
418 
422 typedef unsigned (*rte_mempool_get_count)(const struct rte_mempool *mp);
423 
447 typedef ssize_t (*rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp,
448  uint32_t obj_num, uint32_t pg_shift,
449  size_t *min_chunk_size, size_t *align);
450 
486 ssize_t rte_mempool_op_calc_mem_size_helper(const struct rte_mempool *mp,
487  uint32_t obj_num, uint32_t pg_shift, size_t chunk_reserve,
488  size_t *min_chunk_size, size_t *align);
489 
497 ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp,
498  uint32_t obj_num, uint32_t pg_shift,
499  size_t *min_chunk_size, size_t *align);
500 
513 typedef void (rte_mempool_populate_obj_cb_t)(struct rte_mempool *mp,
514  void *opaque, void *vaddr, rte_iova_t iova);
515 
544 typedef int (*rte_mempool_populate_t)(struct rte_mempool *mp,
545  unsigned int max_objs,
546  void *vaddr, rte_iova_t iova, size_t len,
547  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
548 
552 #define RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ 0x0001
553 
586 int rte_mempool_op_populate_helper(struct rte_mempool *mp,
587  unsigned int flags, unsigned int max_objs,
588  void *vaddr, rte_iova_t iova, size_t len,
589  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
590 
598  unsigned int max_objs,
599  void *vaddr, rte_iova_t iova, size_t len,
600  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
601 
605 typedef int (*rte_mempool_get_info_t)(const struct rte_mempool *mp,
606  struct rte_mempool_info *info);
607 
608 
636 
637 #define RTE_MEMPOOL_MAX_OPS_IDX 16
648 struct rte_mempool_ops_table {
650  uint32_t num_ops;
656 
659 
669 static inline struct rte_mempool_ops *
670 rte_mempool_get_ops(int ops_index)
671 {
672  RTE_VERIFY((ops_index >= 0) && (ops_index < RTE_MEMPOOL_MAX_OPS_IDX));
673 
674  return &rte_mempool_ops_table.ops[ops_index];
675 }
676 
686 int
687 rte_mempool_ops_alloc(struct rte_mempool *mp);
688 
702 static inline int
703 rte_mempool_ops_dequeue_bulk(struct rte_mempool *mp,
704  void **obj_table, unsigned n)
705 {
706  struct rte_mempool_ops *ops;
707  int ret;
708 
709  rte_mempool_trace_ops_dequeue_bulk(mp, obj_table, n);
710  ops = rte_mempool_get_ops(mp->ops_index);
711  ret = ops->dequeue(mp, obj_table, n);
712  if (ret == 0) {
713  __MEMPOOL_STAT_ADD(mp, get_common_pool_bulk, 1);
714  __MEMPOOL_STAT_ADD(mp, get_common_pool_objs, n);
715  }
716  return ret;
717 }
718 
732 static inline int
733 rte_mempool_ops_dequeue_contig_blocks(struct rte_mempool *mp,
734  void **first_obj_table, unsigned int n)
735 {
736  struct rte_mempool_ops *ops;
737 
738  ops = rte_mempool_get_ops(mp->ops_index);
739  RTE_ASSERT(ops->dequeue_contig_blocks != NULL);
740  rte_mempool_trace_ops_dequeue_contig_blocks(mp, first_obj_table, n);
741  return ops->dequeue_contig_blocks(mp, first_obj_table, n);
742 }
743 
757 static inline int
758 rte_mempool_ops_enqueue_bulk(struct rte_mempool *mp, void * const *obj_table,
759  unsigned n)
760 {
761  struct rte_mempool_ops *ops;
762 
763  __MEMPOOL_STAT_ADD(mp, put_common_pool_bulk, 1);
764  __MEMPOOL_STAT_ADD(mp, put_common_pool_objs, n);
765  rte_mempool_trace_ops_enqueue_bulk(mp, obj_table, n);
766  ops = rte_mempool_get_ops(mp->ops_index);
767  return ops->enqueue(mp, obj_table, n);
768 }
769 
778 unsigned
779 rte_mempool_ops_get_count(const struct rte_mempool *mp);
780 
800 ssize_t rte_mempool_ops_calc_mem_size(const struct rte_mempool *mp,
801  uint32_t obj_num, uint32_t pg_shift,
802  size_t *min_chunk_size, size_t *align);
803 
827 int rte_mempool_ops_populate(struct rte_mempool *mp, unsigned int max_objs,
828  void *vaddr, rte_iova_t iova, size_t len,
830  void *obj_cb_arg);
831 
844 int rte_mempool_ops_get_info(const struct rte_mempool *mp,
845  struct rte_mempool_info *info);
846 
853 void
854 rte_mempool_ops_free(struct rte_mempool *mp);
855 
873 int
874 rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name,
875  void *pool_config);
876 
887 int rte_mempool_register_ops(const struct rte_mempool_ops *ops);
888 
894 #define MEMPOOL_REGISTER_OPS(ops) \
895  RTE_INIT(mp_hdlr_init_##ops) \
896  { \
897  rte_mempool_register_ops(&ops); \
898  }
899 
905 typedef void (rte_mempool_obj_cb_t)(struct rte_mempool *mp,
906  void *opaque, void *obj, unsigned obj_idx);
907 typedef rte_mempool_obj_cb_t rte_mempool_obj_ctor_t; /* compat */
908 
914 typedef void (rte_mempool_mem_cb_t)(struct rte_mempool *mp,
915  void *opaque, struct rte_mempool_memhdr *memhdr,
916  unsigned mem_idx);
917 
924 typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *);
925 
1005 struct rte_mempool *
1006 rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
1007  unsigned cache_size, unsigned private_data_size,
1008  rte_mempool_ctor_t *mp_init, void *mp_init_arg,
1009  rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
1010  int socket_id, unsigned flags);
1011 
1046 struct rte_mempool *
1047 rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size,
1048  unsigned cache_size, unsigned private_data_size,
1049  int socket_id, unsigned flags);
1060 void
1061 rte_mempool_free(struct rte_mempool *mp);
1062 
1093 int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
1094  rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
1095  void *opaque);
1096 
1123 int
1124 rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
1125  size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb,
1126  void *opaque);
1127 
1142 
1156 int rte_mempool_populate_anon(struct rte_mempool *mp);
1157 
1173 uint32_t rte_mempool_obj_iter(struct rte_mempool *mp,
1174  rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg);
1175 
1191 uint32_t rte_mempool_mem_iter(struct rte_mempool *mp,
1192  rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg);
1193 
1202 void rte_mempool_dump(FILE *f, struct rte_mempool *mp);
1203 
1218 struct rte_mempool_cache *
1219 rte_mempool_cache_create(uint32_t size, int socket_id);
1220 
1227 void
1229 
1241 static __rte_always_inline struct rte_mempool_cache *
1242 rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
1243 {
1244  if (mp->cache_size == 0)
1245  return NULL;
1246 
1247  if (lcore_id >= RTE_MAX_LCORE)
1248  return NULL;
1249 
1250  rte_mempool_trace_default_cache(mp, lcore_id,
1251  &mp->local_cache[lcore_id]);
1252  return &mp->local_cache[lcore_id];
1253 }
1254 
1263 static __rte_always_inline void
1265  struct rte_mempool *mp)
1266 {
1267  if (cache == NULL)
1268  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1269  if (cache == NULL || cache->len == 0)
1270  return;
1271  rte_mempool_trace_cache_flush(cache, mp);
1272  rte_mempool_ops_enqueue_bulk(mp, cache->objs, cache->len);
1273  cache->len = 0;
1274 }
1275 
1288 static __rte_always_inline void
1289 __mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1290  unsigned int n, struct rte_mempool_cache *cache)
1291 {
1292  void **cache_objs;
1293 
1294  /* increment stat now, adding in mempool always success */
1295  __MEMPOOL_STAT_ADD(mp, put_bulk, 1);
1296  __MEMPOOL_STAT_ADD(mp, put_objs, n);
1297 
1298  /* No cache provided or if put would overflow mem allocated for cache */
1299  if (unlikely(cache == NULL || n > RTE_MEMPOOL_CACHE_MAX_SIZE))
1300  goto ring_enqueue;
1301 
1302  cache_objs = &cache->objs[cache->len];
1303 
1304  /*
1305  * The cache follows the following algorithm
1306  * 1. Add the objects to the cache
1307  * 2. Anything greater than the cache min value (if it crosses the
1308  * cache flush threshold) is flushed to the ring.
1309  */
1310 
1311  /* Add elements back into the cache */
1312  rte_memcpy(&cache_objs[0], obj_table, sizeof(void *) * n);
1313 
1314  cache->len += n;
1315 
1316  if (cache->len >= cache->flushthresh) {
1317  rte_mempool_ops_enqueue_bulk(mp, &cache->objs[cache->size],
1318  cache->len - cache->size);
1319  cache->len = cache->size;
1320  }
1321 
1322  return;
1323 
1324 ring_enqueue:
1325 
1326  /* push remaining objects in ring */
1327 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
1328  if (rte_mempool_ops_enqueue_bulk(mp, obj_table, n) < 0)
1329  rte_panic("cannot put objects in mempool\n");
1330 #else
1331  rte_mempool_ops_enqueue_bulk(mp, obj_table, n);
1332 #endif
1333 }
1334 
1335 
1348 static __rte_always_inline void
1349 rte_mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1350  unsigned int n, struct rte_mempool_cache *cache)
1351 {
1352  rte_mempool_trace_generic_put(mp, obj_table, n, cache);
1353  __mempool_check_cookies(mp, obj_table, n, 0);
1354  __mempool_generic_put(mp, obj_table, n, cache);
1355 }
1356 
1371 static __rte_always_inline void
1372 rte_mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,
1373  unsigned int n)
1374 {
1375  struct rte_mempool_cache *cache;
1376  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1377  rte_mempool_trace_put_bulk(mp, obj_table, n, cache);
1378  rte_mempool_generic_put(mp, obj_table, n, cache);
1379 }
1380 
1393 static __rte_always_inline void
1394 rte_mempool_put(struct rte_mempool *mp, void *obj)
1395 {
1396  rte_mempool_put_bulk(mp, &obj, 1);
1397 }
1398 
1413 static __rte_always_inline int
1414 __mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1415  unsigned int n, struct rte_mempool_cache *cache)
1416 {
1417  int ret;
1418  uint32_t index, len;
1419  void **cache_objs;
1420 
1421  /* No cache provided or cannot be satisfied from cache */
1422  if (unlikely(cache == NULL || n >= cache->size))
1423  goto ring_dequeue;
1424 
1425  cache_objs = cache->objs;
1426 
1427  /* Can this be satisfied from the cache? */
1428  if (cache->len < n) {
1429  /* No. Backfill the cache first, and then fill from it */
1430  uint32_t req = n + (cache->size - cache->len);
1431 
1432  /* How many do we require i.e. number to fill the cache + the request */
1433  ret = rte_mempool_ops_dequeue_bulk(mp,
1434  &cache->objs[cache->len], req);
1435  if (unlikely(ret < 0)) {
1436  /*
1437  * In the off chance that we are buffer constrained,
1438  * where we are not able to allocate cache + n, go to
1439  * the ring directly. If that fails, we are truly out of
1440  * buffers.
1441  */
1442  goto ring_dequeue;
1443  }
1444 
1445  cache->len += req;
1446  }
1447 
1448  /* Now fill in the response ... */
1449  for (index = 0, len = cache->len - 1; index < n; ++index, len--, obj_table++)
1450  *obj_table = cache_objs[len];
1451 
1452  cache->len -= n;
1453 
1454  __MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
1455  __MEMPOOL_STAT_ADD(mp, get_success_objs, n);
1456 
1457  return 0;
1458 
1459 ring_dequeue:
1460 
1461  /* get remaining objects from ring */
1462  ret = rte_mempool_ops_dequeue_bulk(mp, obj_table, n);
1463 
1464  if (ret < 0) {
1465  __MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
1466  __MEMPOOL_STAT_ADD(mp, get_fail_objs, n);
1467  } else {
1468  __MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
1469  __MEMPOOL_STAT_ADD(mp, get_success_objs, n);
1470  }
1471 
1472  return ret;
1473 }
1474 
1495 static __rte_always_inline int
1496 rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1497  unsigned int n, struct rte_mempool_cache *cache)
1498 {
1499  int ret;
1500  ret = __mempool_generic_get(mp, obj_table, n, cache);
1501  if (ret == 0)
1502  __mempool_check_cookies(mp, obj_table, n, 1);
1503  rte_mempool_trace_generic_get(mp, obj_table, n, cache);
1504  return ret;
1505 }
1506 
1529 static __rte_always_inline int
1530 rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
1531 {
1532  struct rte_mempool_cache *cache;
1533  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1534  rte_mempool_trace_get_bulk(mp, obj_table, n, cache);
1535  return rte_mempool_generic_get(mp, obj_table, n, cache);
1536 }
1537 
1558 static __rte_always_inline int
1559 rte_mempool_get(struct rte_mempool *mp, void **obj_p)
1560 {
1561  return rte_mempool_get_bulk(mp, obj_p, 1);
1562 }
1563 
1585 static __rte_always_inline int
1587  void **first_obj_table, unsigned int n)
1588 {
1589  int ret;
1590 
1591  ret = rte_mempool_ops_dequeue_contig_blocks(mp, first_obj_table, n);
1592  if (ret == 0) {
1593  __MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
1594  __MEMPOOL_STAT_ADD(mp, get_success_blks, n);
1595  __mempool_contig_blocks_check_cookies(mp, first_obj_table, n,
1596  1);
1597  } else {
1598  __MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
1599  __MEMPOOL_STAT_ADD(mp, get_fail_blks, n);
1600  }
1601 
1602  rte_mempool_trace_get_contig_blocks(mp, first_obj_table, n);
1603  return ret;
1604 }
1605 
1618 unsigned int rte_mempool_avail_count(const struct rte_mempool *mp);
1619 
1632 unsigned int
1633 rte_mempool_in_use_count(const struct rte_mempool *mp);
1634 
1648 static inline int
1650 {
1651  return rte_mempool_avail_count(mp) == mp->size;
1652 }
1653 
1667 static inline int
1669 {
1670  return rte_mempool_avail_count(mp) == 0;
1671 }
1672 
1683 static inline rte_iova_t
1684 rte_mempool_virt2iova(const void *elt)
1685 {
1686  const struct rte_mempool_objhdr *hdr;
1687  hdr = (const struct rte_mempool_objhdr *)RTE_PTR_SUB(elt,
1688  sizeof(*hdr));
1689  return hdr->iova;
1690 }
1691 
1702 void rte_mempool_audit(struct rte_mempool *mp);
1703 
1712 static inline void *rte_mempool_get_priv(struct rte_mempool *mp)
1713 {
1714  return (char *)mp +
1716 }
1717 
1724 void rte_mempool_list_dump(FILE *f);
1725 
1738 struct rte_mempool *rte_mempool_lookup(const char *name);
1739 
1757 uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
1758  struct rte_mempool_objsz *sz);
1759 
1768 void rte_mempool_walk(void (*func)(struct rte_mempool *, void *arg),
1769  void *arg);
1770 
1775 int
1776 rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz);
1777 
1778 #ifdef __cplusplus
1779 }
1780 #endif
1781 
1782 #endif /* _RTE_MEMPOOL_H_ */
uint64_t pool_id
Definition: rte_mempool.h:222
#define __rte_always_inline
Definition: rte_common.h:228
struct rte_mempool * rte_mempool_lookup(const char *name)
struct rte_mempool_cache * rte_mempool_cache_create(uint32_t size, int socket_id)
uint32_t rte_mempool_mem_iter(struct rte_mempool *mp, rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg)
uint32_t header_size
Definition: rte_mempool.h:233
const struct rte_memzone * mz
Definition: rte_mempool.h:225
uint64_t rte_iova_t
Definition: rte_common.h:420
static __rte_always_inline void rte_mempool_put_bulk(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
Definition: rte_mempool.h:1372
struct rte_mempool * mp
Definition: rte_mempool.h:190
void rte_mempool_list_dump(FILE *f)
void() rte_mempool_memchunk_free_cb_t(struct rte_mempool_memhdr *memhdr, void *opaque)
Definition: rte_mempool.h:179
int(* rte_mempool_dequeue_contig_blocks_t)(struct rte_mempool *mp, void **first_obj_table, unsigned int n)
Definition: rte_mempool.h:416
rte_mempool_alloc_t alloc
Definition: rte_mempool.h:612
void() rte_mempool_populate_obj_cb_t(struct rte_mempool *mp, void *opaque, void *vaddr, rte_iova_t iova)
Definition: rte_mempool.h:513
unsigned int flags
Definition: rte_mempool.h:226
static __rte_always_inline void rte_mempool_generic_put(struct rte_mempool *mp, void *const *obj_table, unsigned int n, struct rte_mempool_cache *cache)
Definition: rte_mempool.h:1349
int(* rte_mempool_alloc_t)(struct rte_mempool *mp)
Definition: rte_mempool.h:394
static int rte_mempool_empty(const struct rte_mempool *mp)
Definition: rte_mempool.h:1668
rte_mempool_memchunk_free_cb_t * free_cb
Definition: rte_mempool.h:194
char name[RTE_MEMPOOL_OPS_NAMESIZE]
Definition: rte_mempool.h:611
int(* rte_mempool_dequeue_t)(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:410
#define MEMPOOL_HEADER_SIZE(mp, cs)
Definition: rte_mempool.h:296
rte_mempool_get_info_t get_info
Definition: rte_mempool.h:630
rte_mempool_dequeue_contig_blocks_t dequeue_contig_blocks
Definition: rte_mempool.h:634
uint32_t cache_size
Definition: rte_mempool.h:229
unsigned int rte_mempool_avail_count(const struct rte_mempool *mp)
uint32_t size
Definition: rte_mempool.h:228
#define RTE_PTR_ADD(ptr, x)
Definition: rte_common.h:250
void rte_mempool_cache_free(struct rte_mempool_cache *cache)
uint32_t total_size
Definition: rte_mempool.h:108
uint32_t rte_mempool_obj_iter(struct rte_mempool *mp, rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg)
int rte_mempool_ops_get_info(const struct rte_mempool *mp, struct rte_mempool_info *info)
int rte_mempool_register_ops(const struct rte_mempool_ops *ops)
#define unlikely(x)
static __rte_always_inline int rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table, unsigned int n, struct rte_mempool_cache *cache)
Definition: rte_mempool.h:1496
int rte_mempool_populate_default(struct rte_mempool *mp)
static __rte_always_inline void rte_mempool_cache_flush(struct rte_mempool_cache *cache, struct rte_mempool *mp)
Definition: rte_mempool.h:1264
#define RTE_MEMPOOL_OPS_NAMESIZE
Definition: rte_mempool.h:382
static __rte_always_inline int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
Definition: rte_mempool.h:1559
STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr)
uint32_t nb_mem_chunks
Definition: rte_mempool.h:250
struct rte_mempool * mp
Definition: rte_mempool.h:145
struct rte_mempool_ops ops[RTE_MEMPOOL_MAX_OPS_IDX]
Definition: rte_mempool.h:654
rte_mempool_get_count get_count
Definition: rte_mempool.h:616
ssize_t(* rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp, uint32_t obj_num, uint32_t pg_shift, size_t *min_chunk_size, size_t *align)
Definition: rte_mempool.h:447
static int rte_mempool_full(const struct rte_mempool *mp)
Definition: rte_mempool.h:1649
static unsigned rte_lcore_id(void)
Definition: rte_lcore.h:76
void * pool_config
Definition: rte_mempool.h:224
unsigned int rte_mempool_in_use_count(const struct rte_mempool *mp)
static __rte_always_inline int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:1530
#define rte_panic(...)
Definition: rte_debug.h:43
int rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name, void *pool_config)
int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr, rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
rte_mempool_calc_mem_size_t calc_mem_size
Definition: rte_mempool.h:621
unsigned(* rte_mempool_get_count)(const struct rte_mempool *mp)
Definition: rte_mempool.h:422
uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, struct rte_mempool_objsz *sz)
ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp, uint32_t obj_num, uint32_t pg_shift, size_t *min_chunk_size, size_t *align)
void * objs[RTE_MEMPOOL_CACHE_MAX_SIZE *3]
Definition: rte_mempool.h:98
void * pool_data
Definition: rte_mempool.h:221
unsigned int contig_block_size
Definition: rte_mempool.h:206
int rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
int(* rte_mempool_populate_t)(struct rte_mempool *mp, unsigned int max_objs, void *vaddr, rte_iova_t iova, size_t len, rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
Definition: rte_mempool.h:544
uint32_t elt_size
Definition: rte_mempool.h:232
void rte_mempool_audit(struct rte_mempool *mp)
int(* rte_mempool_get_info_t)(const struct rte_mempool *mp, struct rte_mempool_info *info)
Definition: rte_mempool.h:605
struct rte_mempool * rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, int socket_id, unsigned flags)
unsigned private_data_size
Definition: rte_mempool.h:236
rte_spinlock_t sl
Definition: rte_mempool.h:649
static __rte_always_inline struct rte_mempool_cache * rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
Definition: rte_mempool.h:1242
void rte_mempool_dump(FILE *f, struct rte_mempool *mp)
void() rte_mempool_ctor_t(struct rte_mempool *, void *)
Definition: rte_mempool.h:924
int rte_mempool_populate_anon(struct rte_mempool *mp)
struct rte_mempool_cache * local_cache
Definition: rte_mempool.h:246
#define __rte_cache_aligned
Definition: rte_common.h:402
uint32_t trailer_size
Definition: rte_mempool.h:234
#define RTE_STD_C11
Definition: rte_common.h:42
rte_mempool_dequeue_t dequeue
Definition: rte_mempool.h:615
int(* rte_mempool_enqueue_t)(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
Definition: rte_mempool.h:404
uint32_t flushthresh
Definition: rte_mempool.h:92
void() rte_mempool_obj_cb_t(struct rte_mempool *mp, void *opaque, void *obj, unsigned obj_idx)
Definition: rte_mempool.h:905
void() rte_mempool_mem_cb_t(struct rte_mempool *mp, void *opaque, struct rte_mempool_memhdr *memhdr, unsigned mem_idx)
Definition: rte_mempool.h:914
int rte_mempool_op_populate_default(struct rte_mempool *mp, unsigned int max_objs, void *vaddr, rte_iova_t iova, size_t len, rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
struct rte_mempool * rte_mempool_create(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags)
#define RTE_PTR_SUB(ptr, x)
Definition: rte_common.h:255
static struct rte_mempool * rte_mempool_from_obj(void *obj)
Definition: rte_mempool.h:316
uint32_t header_size
Definition: rte_mempool.h:106
int32_t ops_index
Definition: rte_mempool.h:244
static void * rte_memcpy(void *dst, const void *src, size_t n)
uint32_t populated_size
Definition: rte_mempool.h:248
static rte_iova_t rte_mempool_virt2iova(const void *elt)
Definition: rte_mempool.h:1684
static __rte_always_inline void rte_mempool_put(struct rte_mempool *mp, void *obj)
Definition: rte_mempool.h:1394
rte_mempool_populate_t populate
Definition: rte_mempool.h:626
void rte_mempool_free(struct rte_mempool *mp)
void rte_mempool_walk(void(*func)(struct rte_mempool *, void *arg), void *arg)
static void * rte_mempool_get_priv(struct rte_mempool *mp)
Definition: rte_mempool.h:1712
char name[RTE_MEMZONE_NAMESIZE]
Definition: rte_mempool.h:218
uint32_t trailer_size
Definition: rte_mempool.h:107
#define RTE_MEMPOOL_MAX_OPS_IDX
Definition: rte_mempool.h:637
void(* rte_mempool_free_t)(struct rte_mempool *mp)
Definition: rte_mempool.h:399
rte_mempool_free_t free
Definition: rte_mempool.h:613
static __rte_always_inline int rte_mempool_get_contig_blocks(struct rte_mempool *mp, void **first_obj_table, unsigned int n)
Definition: rte_mempool.h:1586
rte_mempool_enqueue_t enqueue
Definition: rte_mempool.h:614
#define RTE_MEMZONE_NAMESIZE
Definition: rte_memzone.h:51