DPDK  23.07.0
rte_mempool.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2014 Intel Corporation.
3  * Copyright(c) 2016 6WIND S.A.
4  * Copyright(c) 2022 SmartShare Systems
5  */
6 
7 #ifndef _RTE_MEMPOOL_H_
8 #define _RTE_MEMPOOL_H_
9 
37 #include <stdio.h>
38 #include <stdint.h>
39 #include <inttypes.h>
40 
41 #include <rte_compat.h>
42 #include <rte_config.h>
43 #include <rte_spinlock.h>
44 #include <rte_debug.h>
45 #include <rte_lcore.h>
46 #include <rte_branch_prediction.h>
47 #include <rte_ring.h>
48 #include <rte_memcpy.h>
49 #include <rte_common.h>
50 
51 #include "rte_mempool_trace_fp.h"
52 
53 #ifdef __cplusplus
54 extern "C" {
55 #endif
56 
57 #define RTE_MEMPOOL_HEADER_COOKIE1 0xbadbadbadadd2e55ULL
58 #define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL
59 #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL
61 #ifdef RTE_LIBRTE_MEMPOOL_STATS
62 
68 struct rte_mempool_debug_stats {
69  uint64_t put_bulk;
70  uint64_t put_objs;
71  uint64_t put_common_pool_bulk;
72  uint64_t put_common_pool_objs;
73  uint64_t get_common_pool_bulk;
74  uint64_t get_common_pool_objs;
75  uint64_t get_success_bulk;
76  uint64_t get_success_objs;
77  uint64_t get_fail_bulk;
78  uint64_t get_fail_objs;
79  uint64_t get_success_blks;
80  uint64_t get_fail_blks;
82 #endif
83 
88  uint32_t size;
89  uint32_t flushthresh;
90  uint32_t len;
91 #ifdef RTE_LIBRTE_MEMPOOL_STATS
92  uint32_t unused;
93  /*
94  * Alternative location for the most frequently updated mempool statistics (per-lcore),
95  * providing faster update access when using a mempool cache.
96  */
97  struct {
98  uint64_t put_bulk;
99  uint64_t put_objs;
100  uint64_t get_success_bulk;
101  uint64_t get_success_objs;
102  } stats;
103 #endif
104 
110  void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 2] __rte_cache_aligned;
112 
117  uint32_t elt_size;
118  uint32_t header_size;
119  uint32_t trailer_size;
120  uint32_t total_size;
122 };
123 
125 #define RTE_MEMPOOL_NAMESIZE (RTE_RING_NAMESIZE - \
126  sizeof(RTE_MEMPOOL_MZ_PREFIX) + 1)
127 #define RTE_MEMPOOL_MZ_PREFIX "MP_"
128 
129 /* "MP_<name>" */
130 #define RTE_MEMPOOL_MZ_FORMAT RTE_MEMPOOL_MZ_PREFIX "%s"
131 
132 #ifndef RTE_MEMPOOL_ALIGN
133 
136 #define RTE_MEMPOOL_ALIGN RTE_CACHE_LINE_SIZE
137 #endif
138 
139 #define RTE_MEMPOOL_ALIGN_MASK (RTE_MEMPOOL_ALIGN - 1)
140 
151  RTE_STAILQ_ENTRY(rte_mempool_objhdr) next;
152  struct rte_mempool *mp;
154 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
155  uint64_t cookie;
156 #endif
157 };
158 
162 RTE_STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr);
163 
164 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
165 
172 struct rte_mempool_objtlr {
173  uint64_t cookie;
174 };
175 
176 #endif
177 
181 RTE_STAILQ_HEAD(rte_mempool_memhdr_list, rte_mempool_memhdr);
182 
187  void *opaque);
188 
196  RTE_STAILQ_ENTRY(rte_mempool_memhdr) next;
197  struct rte_mempool *mp;
198  void *addr;
200  size_t len;
202  void *opaque;
203 };
204 
213  unsigned int contig_block_size;
215 
219 struct rte_mempool {
220  char name[RTE_MEMPOOL_NAMESIZE];
222  union {
223  void *pool_data;
224  uint64_t pool_id;
225  };
226  void *pool_config;
227  const struct rte_memzone *mz;
228  unsigned int flags;
229  int socket_id;
230  uint32_t size;
231  uint32_t cache_size;
234  uint32_t elt_size;
235  uint32_t header_size;
236  uint32_t trailer_size;
238  unsigned private_data_size;
246  int32_t ops_index;
247 
250  uint32_t populated_size;
251  struct rte_mempool_objhdr_list elt_list;
252  uint32_t nb_mem_chunks;
253  struct rte_mempool_memhdr_list mem_list;
255 #ifdef RTE_LIBRTE_MEMPOOL_STATS
256 
260  struct rte_mempool_debug_stats stats[RTE_MAX_LCORE + 1];
261 #endif
263 
265 #define RTE_MEMPOOL_F_NO_SPREAD 0x0001
266 
270 #define MEMPOOL_F_NO_SPREAD RTE_MEMPOOL_F_NO_SPREAD
271 
272 #define RTE_MEMPOOL_F_NO_CACHE_ALIGN 0x0002
273 
277 #define MEMPOOL_F_NO_CACHE_ALIGN RTE_MEMPOOL_F_NO_CACHE_ALIGN
278 
279 #define RTE_MEMPOOL_F_SP_PUT 0x0004
280 
284 #define MEMPOOL_F_SP_PUT RTE_MEMPOOL_F_SP_PUT
285 
286 #define RTE_MEMPOOL_F_SC_GET 0x0008
287 
291 #define MEMPOOL_F_SC_GET RTE_MEMPOOL_F_SC_GET
292 
293 #define RTE_MEMPOOL_F_POOL_CREATED 0x0010
294 
295 #define RTE_MEMPOOL_F_NO_IOVA_CONTIG 0x0020
296 
300 #define MEMPOOL_F_NO_IOVA_CONTIG RTE_MEMPOOL_F_NO_IOVA_CONTIG
301 
302 #define RTE_MEMPOOL_F_NON_IO 0x0040
303 
307 #define RTE_MEMPOOL_VALID_USER_FLAGS (RTE_MEMPOOL_F_NO_SPREAD \
308  | RTE_MEMPOOL_F_NO_CACHE_ALIGN \
309  | RTE_MEMPOOL_F_SP_PUT \
310  | RTE_MEMPOOL_F_SC_GET \
311  | RTE_MEMPOOL_F_NO_IOVA_CONTIG \
312  )
313 
324 #ifdef RTE_LIBRTE_MEMPOOL_STATS
325 #define RTE_MEMPOOL_STAT_ADD(mp, name, n) do { \
326  unsigned int __lcore_id = rte_lcore_id(); \
327  if (likely(__lcore_id < RTE_MAX_LCORE)) \
328  (mp)->stats[__lcore_id].name += (n); \
329  else \
330  __atomic_fetch_add(&((mp)->stats[RTE_MAX_LCORE].name), \
331  (n), __ATOMIC_RELAXED); \
332  } while (0)
333 #else
334 #define RTE_MEMPOOL_STAT_ADD(mp, name, n) do {} while (0)
335 #endif
336 
347 #ifdef RTE_LIBRTE_MEMPOOL_STATS
348 #define RTE_MEMPOOL_CACHE_STAT_ADD(cache, name, n) ((cache)->stats.name += (n))
349 #else
350 #define RTE_MEMPOOL_CACHE_STAT_ADD(cache, name, n) do {} while (0)
351 #endif
352 
361 #define RTE_MEMPOOL_HEADER_SIZE(mp, cs) \
362  (sizeof(*(mp)) + (((cs) == 0) ? 0 : \
363  (sizeof(struct rte_mempool_cache) * RTE_MAX_LCORE)))
364 
365 /* return the header of a mempool object (internal) */
366 static inline struct rte_mempool_objhdr *
367 rte_mempool_get_header(void *obj)
368 {
369  return (struct rte_mempool_objhdr *)RTE_PTR_SUB(obj,
370  sizeof(struct rte_mempool_objhdr));
371 }
372 
382 static inline struct rte_mempool *rte_mempool_from_obj(void *obj)
383 {
384  struct rte_mempool_objhdr *hdr = rte_mempool_get_header(obj);
385  return hdr->mp;
386 }
387 
388 /* return the trailer of a mempool object (internal) */
389 static inline struct rte_mempool_objtlr *rte_mempool_get_trailer(void *obj)
390 {
391  struct rte_mempool *mp = rte_mempool_from_obj(obj);
392  return (struct rte_mempool_objtlr *)RTE_PTR_ADD(obj, mp->elt_size);
393 }
394 
409 void rte_mempool_check_cookies(const struct rte_mempool *mp,
410  void * const *obj_table_const, unsigned n, int free);
411 
412 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
413 #define RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table_const, n, free) \
414  rte_mempool_check_cookies(mp, obj_table_const, n, free)
415 #else
416 #define RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table_const, n, free) do {} while (0)
417 #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
418 
434 void rte_mempool_contig_blocks_check_cookies(const struct rte_mempool *mp,
435  void * const *first_obj_table_const, unsigned int n, int free);
436 
437 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
438 #define RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table_const, n, \
439  free) \
440  rte_mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
441  free)
442 #else
443 #define RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table_const, n, \
444  free) \
445  do {} while (0)
446 #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
447 
448 #define RTE_MEMPOOL_OPS_NAMESIZE 32
460 typedef int (*rte_mempool_alloc_t)(struct rte_mempool *mp);
461 
465 typedef void (*rte_mempool_free_t)(struct rte_mempool *mp);
466 
470 typedef int (*rte_mempool_enqueue_t)(struct rte_mempool *mp,
471  void * const *obj_table, unsigned int n);
472 
476 typedef int (*rte_mempool_dequeue_t)(struct rte_mempool *mp,
477  void **obj_table, unsigned int n);
478 
483  void **first_obj_table, unsigned int n);
484 
488 typedef unsigned (*rte_mempool_get_count)(const struct rte_mempool *mp);
489 
513 typedef ssize_t (*rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp,
514  uint32_t obj_num, uint32_t pg_shift,
515  size_t *min_chunk_size, size_t *align);
516 
552 ssize_t rte_mempool_op_calc_mem_size_helper(const struct rte_mempool *mp,
553  uint32_t obj_num, uint32_t pg_shift, size_t chunk_reserve,
554  size_t *min_chunk_size, size_t *align);
555 
563 ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp,
564  uint32_t obj_num, uint32_t pg_shift,
565  size_t *min_chunk_size, size_t *align);
566 
579 typedef void (rte_mempool_populate_obj_cb_t)(struct rte_mempool *mp,
580  void *opaque, void *vaddr, rte_iova_t iova);
581 
610 typedef int (*rte_mempool_populate_t)(struct rte_mempool *mp,
611  unsigned int max_objs,
612  void *vaddr, rte_iova_t iova, size_t len,
613  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
614 
618 #define RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ 0x0001
619 
652 int rte_mempool_op_populate_helper(struct rte_mempool *mp,
653  unsigned int flags, unsigned int max_objs,
654  void *vaddr, rte_iova_t iova, size_t len,
655  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
656 
664  unsigned int max_objs,
665  void *vaddr, rte_iova_t iova, size_t len,
666  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
667 
671 typedef int (*rte_mempool_get_info_t)(const struct rte_mempool *mp,
672  struct rte_mempool_info *info);
673 
674 
702 
703 #define RTE_MEMPOOL_MAX_OPS_IDX 16
714 struct rte_mempool_ops_table {
716  uint32_t num_ops;
722 
725 
735 static inline struct rte_mempool_ops *
736 rte_mempool_get_ops(int ops_index)
737 {
738  RTE_VERIFY((ops_index >= 0) && (ops_index < RTE_MEMPOOL_MAX_OPS_IDX));
739 
740  return &rte_mempool_ops_table.ops[ops_index];
741 }
742 
752 int
753 rte_mempool_ops_alloc(struct rte_mempool *mp);
754 
768 static inline int
769 rte_mempool_ops_dequeue_bulk(struct rte_mempool *mp,
770  void **obj_table, unsigned n)
771 {
772  struct rte_mempool_ops *ops;
773  int ret;
774 
775  rte_mempool_trace_ops_dequeue_bulk(mp, obj_table, n);
776  ops = rte_mempool_get_ops(mp->ops_index);
777  ret = ops->dequeue(mp, obj_table, n);
778  if (ret == 0) {
779  RTE_MEMPOOL_STAT_ADD(mp, get_common_pool_bulk, 1);
780  RTE_MEMPOOL_STAT_ADD(mp, get_common_pool_objs, n);
781  }
782  return ret;
783 }
784 
798 static inline int
799 rte_mempool_ops_dequeue_contig_blocks(struct rte_mempool *mp,
800  void **first_obj_table, unsigned int n)
801 {
802  struct rte_mempool_ops *ops;
803 
804  ops = rte_mempool_get_ops(mp->ops_index);
805  RTE_ASSERT(ops->dequeue_contig_blocks != NULL);
806  rte_mempool_trace_ops_dequeue_contig_blocks(mp, first_obj_table, n);
807  return ops->dequeue_contig_blocks(mp, first_obj_table, n);
808 }
809 
823 static inline int
824 rte_mempool_ops_enqueue_bulk(struct rte_mempool *mp, void * const *obj_table,
825  unsigned n)
826 {
827  struct rte_mempool_ops *ops;
828  int ret;
829 
830  RTE_MEMPOOL_STAT_ADD(mp, put_common_pool_bulk, 1);
831  RTE_MEMPOOL_STAT_ADD(mp, put_common_pool_objs, n);
832  rte_mempool_trace_ops_enqueue_bulk(mp, obj_table, n);
833  ops = rte_mempool_get_ops(mp->ops_index);
834  ret = ops->enqueue(mp, obj_table, n);
835 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
836  if (unlikely(ret < 0))
837  RTE_LOG(CRIT, MEMPOOL, "cannot enqueue %u objects to mempool %s\n",
838  n, mp->name);
839 #endif
840  return ret;
841 }
842 
851 unsigned
852 rte_mempool_ops_get_count(const struct rte_mempool *mp);
853 
873 ssize_t rte_mempool_ops_calc_mem_size(const struct rte_mempool *mp,
874  uint32_t obj_num, uint32_t pg_shift,
875  size_t *min_chunk_size, size_t *align);
876 
900 int rte_mempool_ops_populate(struct rte_mempool *mp, unsigned int max_objs,
901  void *vaddr, rte_iova_t iova, size_t len,
903  void *obj_cb_arg);
904 
917 int rte_mempool_ops_get_info(const struct rte_mempool *mp,
918  struct rte_mempool_info *info);
919 
926 void
927 rte_mempool_ops_free(struct rte_mempool *mp);
928 
946 int
947 rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name,
948  void *pool_config);
949 
960 int rte_mempool_register_ops(const struct rte_mempool_ops *ops);
961 
967 #define RTE_MEMPOOL_REGISTER_OPS(ops) \
968  RTE_INIT(mp_hdlr_init_##ops) \
969  { \
970  rte_mempool_register_ops(&ops); \
971  }
972 
978 typedef void (rte_mempool_obj_cb_t)(struct rte_mempool *mp,
979  void *opaque, void *obj, unsigned obj_idx);
980 typedef rte_mempool_obj_cb_t rte_mempool_obj_ctor_t; /* compat */
981 
987 typedef void (rte_mempool_mem_cb_t)(struct rte_mempool *mp,
988  void *opaque, struct rte_mempool_memhdr *memhdr,
989  unsigned mem_idx);
990 
997 typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *);
998 
1077 struct rte_mempool *
1078 rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
1079  unsigned cache_size, unsigned private_data_size,
1080  rte_mempool_ctor_t *mp_init, void *mp_init_arg,
1081  rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
1082  int socket_id, unsigned flags);
1083 
1118 struct rte_mempool *
1119 rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size,
1120  unsigned cache_size, unsigned private_data_size,
1121  int socket_id, unsigned flags);
1133 void
1134 rte_mempool_free(struct rte_mempool *mp);
1135 
1166 int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
1167  rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
1168  void *opaque);
1169 
1196 int
1197 rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
1198  size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb,
1199  void *opaque);
1200 
1215 
1229 int rte_mempool_populate_anon(struct rte_mempool *mp);
1230 
1246 uint32_t rte_mempool_obj_iter(struct rte_mempool *mp,
1247  rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg);
1248 
1264 uint32_t rte_mempool_mem_iter(struct rte_mempool *mp,
1265  rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg);
1266 
1275 void rte_mempool_dump(FILE *f, struct rte_mempool *mp);
1276 
1291 struct rte_mempool_cache *
1292 rte_mempool_cache_create(uint32_t size, int socket_id);
1293 
1300 void
1302 
1314 static __rte_always_inline struct rte_mempool_cache *
1315 rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
1316 {
1317  if (mp->cache_size == 0)
1318  return NULL;
1319 
1320  if (lcore_id >= RTE_MAX_LCORE)
1321  return NULL;
1322 
1323  rte_mempool_trace_default_cache(mp, lcore_id,
1324  &mp->local_cache[lcore_id]);
1325  return &mp->local_cache[lcore_id];
1326 }
1327 
1336 static __rte_always_inline void
1338  struct rte_mempool *mp)
1339 {
1340  if (cache == NULL)
1341  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1342  if (cache == NULL || cache->len == 0)
1343  return;
1344  rte_mempool_trace_cache_flush(cache, mp);
1345  rte_mempool_ops_enqueue_bulk(mp, cache->objs, cache->len);
1346  cache->len = 0;
1347 }
1348 
1361 static __rte_always_inline void
1362 rte_mempool_do_generic_put(struct rte_mempool *mp, void * const *obj_table,
1363  unsigned int n, struct rte_mempool_cache *cache)
1364 {
1365  void **cache_objs;
1366 
1367  /* No cache provided */
1368  if (unlikely(cache == NULL))
1369  goto driver_enqueue;
1370 
1371  /* increment stat now, adding in mempool always success */
1372  RTE_MEMPOOL_CACHE_STAT_ADD(cache, put_bulk, 1);
1373  RTE_MEMPOOL_CACHE_STAT_ADD(cache, put_objs, n);
1374 
1375  /* The request itself is too big for the cache */
1376  if (unlikely(n > cache->flushthresh))
1377  goto driver_enqueue_stats_incremented;
1378 
1379  /*
1380  * The cache follows the following algorithm:
1381  * 1. If the objects cannot be added to the cache without crossing
1382  * the flush threshold, flush the cache to the backend.
1383  * 2. Add the objects to the cache.
1384  */
1385 
1386  if (cache->len + n <= cache->flushthresh) {
1387  cache_objs = &cache->objs[cache->len];
1388  cache->len += n;
1389  } else {
1390  cache_objs = &cache->objs[0];
1391  rte_mempool_ops_enqueue_bulk(mp, cache_objs, cache->len);
1392  cache->len = n;
1393  }
1394 
1395  /* Add the objects to the cache. */
1396  rte_memcpy(cache_objs, obj_table, sizeof(void *) * n);
1397 
1398  return;
1399 
1400 driver_enqueue:
1401 
1402  /* increment stat now, adding in mempool always success */
1403  RTE_MEMPOOL_STAT_ADD(mp, put_bulk, 1);
1404  RTE_MEMPOOL_STAT_ADD(mp, put_objs, n);
1405 
1406 driver_enqueue_stats_incremented:
1407 
1408  /* push objects to the backend */
1409  rte_mempool_ops_enqueue_bulk(mp, obj_table, n);
1410 }
1411 
1412 
1425 static __rte_always_inline void
1426 rte_mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1427  unsigned int n, struct rte_mempool_cache *cache)
1428 {
1429  rte_mempool_trace_generic_put(mp, obj_table, n, cache);
1430  RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table, n, 0);
1431  rte_mempool_do_generic_put(mp, obj_table, n, cache);
1432 }
1433 
1448 static __rte_always_inline void
1449 rte_mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,
1450  unsigned int n)
1451 {
1452  struct rte_mempool_cache *cache;
1453  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1454  rte_mempool_trace_put_bulk(mp, obj_table, n, cache);
1455  rte_mempool_generic_put(mp, obj_table, n, cache);
1456 }
1457 
1470 static __rte_always_inline void
1471 rte_mempool_put(struct rte_mempool *mp, void *obj)
1472 {
1473  rte_mempool_put_bulk(mp, &obj, 1);
1474 }
1475 
1490 static __rte_always_inline int
1491 rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table,
1492  unsigned int n, struct rte_mempool_cache *cache)
1493 {
1494  int ret;
1495  unsigned int remaining;
1496  uint32_t index, len;
1497  void **cache_objs;
1498 
1499  /* No cache provided */
1500  if (unlikely(cache == NULL)) {
1501  remaining = n;
1502  goto driver_dequeue;
1503  }
1504 
1505  /* The cache is a stack, so copy will be in reverse order. */
1506  cache_objs = &cache->objs[cache->len];
1507 
1508  if (__extension__(__builtin_constant_p(n)) && n <= cache->len) {
1509  /*
1510  * The request size is known at build time, and
1511  * the entire request can be satisfied from the cache,
1512  * so let the compiler unroll the fixed length copy loop.
1513  */
1514  cache->len -= n;
1515  for (index = 0; index < n; index++)
1516  *obj_table++ = *--cache_objs;
1517 
1518  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1519  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1520 
1521  return 0;
1522  }
1523 
1524  /*
1525  * Use the cache as much as we have to return hot objects first.
1526  * If the request size 'n' is known at build time, the above comparison
1527  * ensures that n > cache->len here, so omit RTE_MIN().
1528  */
1529  len = __extension__(__builtin_constant_p(n)) ? cache->len :
1530  RTE_MIN(n, cache->len);
1531  cache->len -= len;
1532  remaining = n - len;
1533  for (index = 0; index < len; index++)
1534  *obj_table++ = *--cache_objs;
1535 
1536  /*
1537  * If the request size 'n' is known at build time, the case
1538  * where the entire request can be satisfied from the cache
1539  * has already been handled above, so omit handling it here.
1540  */
1541  if (!__extension__(__builtin_constant_p(n)) && remaining == 0) {
1542  /* The entire request is satisfied from the cache. */
1543 
1544  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1545  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1546 
1547  return 0;
1548  }
1549 
1550  /* if dequeue below would overflow mem allocated for cache */
1551  if (unlikely(remaining > RTE_MEMPOOL_CACHE_MAX_SIZE))
1552  goto driver_dequeue;
1553 
1554  /* Fill the cache from the backend; fetch size + remaining objects. */
1555  ret = rte_mempool_ops_dequeue_bulk(mp, cache->objs,
1556  cache->size + remaining);
1557  if (unlikely(ret < 0)) {
1558  /*
1559  * We are buffer constrained, and not able to allocate
1560  * cache + remaining.
1561  * Do not fill the cache, just satisfy the remaining part of
1562  * the request directly from the backend.
1563  */
1564  goto driver_dequeue;
1565  }
1566 
1567  /* Satisfy the remaining part of the request from the filled cache. */
1568  cache_objs = &cache->objs[cache->size + remaining];
1569  for (index = 0; index < remaining; index++)
1570  *obj_table++ = *--cache_objs;
1571 
1572  cache->len = cache->size;
1573 
1574  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1575  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1576 
1577  return 0;
1578 
1579 driver_dequeue:
1580 
1581  /* Get remaining objects directly from the backend. */
1582  ret = rte_mempool_ops_dequeue_bulk(mp, obj_table, remaining);
1583 
1584  if (ret < 0) {
1585  if (likely(cache != NULL)) {
1586  cache->len = n - remaining;
1587  /*
1588  * No further action is required to roll the first part
1589  * of the request back into the cache, as objects in
1590  * the cache are intact.
1591  */
1592  }
1593 
1594  RTE_MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
1595  RTE_MEMPOOL_STAT_ADD(mp, get_fail_objs, n);
1596  } else {
1597  if (likely(cache != NULL)) {
1598  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1599  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1600  } else {
1601  RTE_MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
1602  RTE_MEMPOOL_STAT_ADD(mp, get_success_objs, n);
1603  }
1604  }
1605 
1606  return ret;
1607 }
1608 
1629 static __rte_always_inline int
1630 rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1631  unsigned int n, struct rte_mempool_cache *cache)
1632 {
1633  int ret;
1634  ret = rte_mempool_do_generic_get(mp, obj_table, n, cache);
1635  if (ret == 0)
1636  RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table, n, 1);
1637  rte_mempool_trace_generic_get(mp, obj_table, n, cache);
1638  return ret;
1639 }
1640 
1663 static __rte_always_inline int
1664 rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
1665 {
1666  struct rte_mempool_cache *cache;
1667  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1668  rte_mempool_trace_get_bulk(mp, obj_table, n, cache);
1669  return rte_mempool_generic_get(mp, obj_table, n, cache);
1670 }
1671 
1692 static __rte_always_inline int
1693 rte_mempool_get(struct rte_mempool *mp, void **obj_p)
1694 {
1695  return rte_mempool_get_bulk(mp, obj_p, 1);
1696 }
1697 
1719 static __rte_always_inline int
1721  void **first_obj_table, unsigned int n)
1722 {
1723  int ret;
1724 
1725  ret = rte_mempool_ops_dequeue_contig_blocks(mp, first_obj_table, n);
1726  if (ret == 0) {
1727  RTE_MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
1728  RTE_MEMPOOL_STAT_ADD(mp, get_success_blks, n);
1729  RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table, n,
1730  1);
1731  } else {
1732  RTE_MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
1733  RTE_MEMPOOL_STAT_ADD(mp, get_fail_blks, n);
1734  }
1735 
1736  rte_mempool_trace_get_contig_blocks(mp, first_obj_table, n);
1737  return ret;
1738 }
1739 
1752 unsigned int rte_mempool_avail_count(const struct rte_mempool *mp);
1753 
1766 unsigned int
1767 rte_mempool_in_use_count(const struct rte_mempool *mp);
1768 
1782 static inline int
1784 {
1785  return rte_mempool_avail_count(mp) == mp->size;
1786 }
1787 
1801 static inline int
1803 {
1804  return rte_mempool_avail_count(mp) == 0;
1805 }
1806 
1817 static inline rte_iova_t
1818 rte_mempool_virt2iova(const void *elt)
1819 {
1820  const struct rte_mempool_objhdr *hdr;
1821  hdr = (const struct rte_mempool_objhdr *)RTE_PTR_SUB(elt,
1822  sizeof(*hdr));
1823  return hdr->iova;
1824 }
1825 
1836 void rte_mempool_audit(struct rte_mempool *mp);
1837 
1846 static inline void *rte_mempool_get_priv(struct rte_mempool *mp)
1847 {
1848  return (char *)mp +
1849  RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size);
1850 }
1851 
1858 void rte_mempool_list_dump(FILE *f);
1859 
1871 struct rte_mempool *rte_mempool_lookup(const char *name);
1872 
1890 uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
1891  struct rte_mempool_objsz *sz);
1892 
1901 void rte_mempool_walk(void (*func)(struct rte_mempool *, void *arg),
1902  void *arg);
1903 
1908 int
1909 rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz);
1910 
1920 };
1921 
1931 typedef void (rte_mempool_event_callback)(
1932  enum rte_mempool_event event,
1933  struct rte_mempool *mp,
1934  void *user_data);
1935 
1952 __rte_internal
1953 int
1954 rte_mempool_event_callback_register(rte_mempool_event_callback *func,
1955  void *user_data);
1956 
1970 __rte_internal
1971 int
1972 rte_mempool_event_callback_unregister(rte_mempool_event_callback *func,
1973  void *user_data);
1974 
1975 #ifdef __cplusplus
1976 }
1977 #endif
1978 
1979 #endif /* _RTE_MEMPOOL_H_ */
uint64_t pool_id
Definition: rte_mempool.h:224
#define __rte_always_inline
Definition: rte_common.h:255
struct rte_mempool * rte_mempool_lookup(const char *name)
struct rte_mempool_cache * rte_mempool_cache_create(uint32_t size, int socket_id)
uint32_t rte_mempool_mem_iter(struct rte_mempool *mp, rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg)
uint32_t header_size
Definition: rte_mempool.h:235
#define likely(x)
const struct rte_memzone * mz
Definition: rte_mempool.h:227
uint64_t rte_iova_t
Definition: rte_common.h:458
static __rte_always_inline void rte_mempool_put_bulk(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
Definition: rte_mempool.h:1449
struct rte_mempool * mp
Definition: rte_mempool.h:197
void rte_mempool_list_dump(FILE *f)
void() rte_mempool_memchunk_free_cb_t(struct rte_mempool_memhdr *memhdr, void *opaque)
Definition: rte_mempool.h:186
int(* rte_mempool_dequeue_contig_blocks_t)(struct rte_mempool *mp, void **first_obj_table, unsigned int n)
Definition: rte_mempool.h:482
rte_mempool_alloc_t alloc
Definition: rte_mempool.h:678
void *objs [RTE_MEMPOOL_CACHE_MAX_SIZE *2] __rte_cache_aligned
Definition: rte_mempool.h:110
void() rte_mempool_populate_obj_cb_t(struct rte_mempool *mp, void *opaque, void *vaddr, rte_iova_t iova)
Definition: rte_mempool.h:579
unsigned int flags
Definition: rte_mempool.h:228
static __rte_always_inline void rte_mempool_generic_put(struct rte_mempool *mp, void *const *obj_table, unsigned int n, struct rte_mempool_cache *cache)
Definition: rte_mempool.h:1426
int(* rte_mempool_alloc_t)(struct rte_mempool *mp)
Definition: rte_mempool.h:460
#define RTE_LOG(l, t,...)
Definition: rte_log.h:335
static int rte_mempool_empty(const struct rte_mempool *mp)
Definition: rte_mempool.h:1802
RTE_STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr)
rte_mempool_memchunk_free_cb_t * free_cb
Definition: rte_mempool.h:201
char name[RTE_MEMPOOL_OPS_NAMESIZE]
Definition: rte_mempool.h:677
int(* rte_mempool_dequeue_t)(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:476
rte_mempool_get_info_t get_info
Definition: rte_mempool.h:696
rte_mempool_dequeue_contig_blocks_t dequeue_contig_blocks
Definition: rte_mempool.h:700
uint32_t cache_size
Definition: rte_mempool.h:231
char name[RTE_MEMPOOL_NAMESIZE]
Definition: rte_mempool.h:220
unsigned int rte_mempool_avail_count(const struct rte_mempool *mp)
uint32_t size
Definition: rte_mempool.h:230
#define RTE_PTR_ADD(ptr, x)
Definition: rte_common.h:290
void rte_mempool_cache_free(struct rte_mempool_cache *cache)
uint32_t total_size
Definition: rte_mempool.h:120
uint32_t rte_mempool_obj_iter(struct rte_mempool *mp, rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg)
int rte_mempool_ops_get_info(const struct rte_mempool *mp, struct rte_mempool_info *info)
int rte_mempool_register_ops(const struct rte_mempool_ops *ops)
#define unlikely(x)
static __rte_always_inline int rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table, unsigned int n, struct rte_mempool_cache *cache)
Definition: rte_mempool.h:1630
int rte_mempool_populate_default(struct rte_mempool *mp)
#define RTE_MIN(a, b)
Definition: rte_common.h:613
static __rte_always_inline void rte_mempool_cache_flush(struct rte_mempool_cache *cache, struct rte_mempool *mp)
Definition: rte_mempool.h:1337
#define RTE_MEMPOOL_OPS_NAMESIZE
Definition: rte_mempool.h:448
static __rte_always_inline int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
Definition: rte_mempool.h:1693
uint32_t nb_mem_chunks
Definition: rte_mempool.h:252
struct rte_mempool * mp
Definition: rte_mempool.h:152
struct rte_mempool_ops ops[RTE_MEMPOOL_MAX_OPS_IDX]
Definition: rte_mempool.h:720
rte_mempool_get_count get_count
Definition: rte_mempool.h:682
ssize_t(* rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp, uint32_t obj_num, uint32_t pg_shift, size_t *min_chunk_size, size_t *align)
Definition: rte_mempool.h:513
static int rte_mempool_full(const struct rte_mempool *mp)
Definition: rte_mempool.h:1783
static unsigned rte_lcore_id(void)
Definition: rte_lcore.h:78
void * pool_config
Definition: rte_mempool.h:226
unsigned int rte_mempool_in_use_count(const struct rte_mempool *mp)
static __rte_always_inline int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:1664
int rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name, void *pool_config)
int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr, rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
rte_mempool_calc_mem_size_t calc_mem_size
Definition: rte_mempool.h:687
unsigned(* rte_mempool_get_count)(const struct rte_mempool *mp)
Definition: rte_mempool.h:488
uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, struct rte_mempool_objsz *sz)
ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp, uint32_t obj_num, uint32_t pg_shift, size_t *min_chunk_size, size_t *align)
void * pool_data
Definition: rte_mempool.h:223
unsigned int contig_block_size
Definition: rte_mempool.h:213
int rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
int(* rte_mempool_populate_t)(struct rte_mempool *mp, unsigned int max_objs, void *vaddr, rte_iova_t iova, size_t len, rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
Definition: rte_mempool.h:610
uint32_t elt_size
Definition: rte_mempool.h:234
void rte_mempool_audit(struct rte_mempool *mp)
int(* rte_mempool_get_info_t)(const struct rte_mempool *mp, struct rte_mempool_info *info)
Definition: rte_mempool.h:671
struct rte_mempool * rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, int socket_id, unsigned flags)
unsigned private_data_size
Definition: rte_mempool.h:238
rte_spinlock_t sl
Definition: rte_mempool.h:715
static __rte_always_inline struct rte_mempool_cache * rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
Definition: rte_mempool.h:1315
void rte_mempool_dump(FILE *f, struct rte_mempool *mp)
void() rte_mempool_ctor_t(struct rte_mempool *, void *)
Definition: rte_mempool.h:997
int rte_mempool_populate_anon(struct rte_mempool *mp)
struct rte_mempool_cache * local_cache
Definition: rte_mempool.h:248
#define __rte_cache_aligned
Definition: rte_common.h:440
uint32_t trailer_size
Definition: rte_mempool.h:236
#define RTE_STD_C11
Definition: rte_common.h:39
rte_mempool_dequeue_t dequeue
Definition: rte_mempool.h:681
int(* rte_mempool_enqueue_t)(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
Definition: rte_mempool.h:470
uint32_t flushthresh
Definition: rte_mempool.h:89
void() rte_mempool_obj_cb_t(struct rte_mempool *mp, void *opaque, void *obj, unsigned obj_idx)
Definition: rte_mempool.h:978
void() rte_mempool_mem_cb_t(struct rte_mempool *mp, void *opaque, struct rte_mempool_memhdr *memhdr, unsigned mem_idx)
Definition: rte_mempool.h:987
int rte_mempool_op_populate_default(struct rte_mempool *mp, unsigned int max_objs, void *vaddr, rte_iova_t iova, size_t len, rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
struct rte_mempool * rte_mempool_create(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags)
#define RTE_PTR_SUB(ptr, x)
Definition: rte_common.h:295
rte_mempool_event
Definition: rte_mempool.h:1915
static struct rte_mempool * rte_mempool_from_obj(void *obj)
Definition: rte_mempool.h:382
uint32_t header_size
Definition: rte_mempool.h:118
int32_t ops_index
Definition: rte_mempool.h:246
static void * rte_memcpy(void *dst, const void *src, size_t n)
uint32_t populated_size
Definition: rte_mempool.h:250
static rte_iova_t rte_mempool_virt2iova(const void *elt)
Definition: rte_mempool.h:1818
static __rte_always_inline void rte_mempool_put(struct rte_mempool *mp, void *obj)
Definition: rte_mempool.h:1471
rte_mempool_populate_t populate
Definition: rte_mempool.h:692
void rte_mempool_free(struct rte_mempool *mp)
void rte_mempool_walk(void(*func)(struct rte_mempool *, void *arg), void *arg)
static void * rte_mempool_get_priv(struct rte_mempool *mp)
Definition: rte_mempool.h:1846
uint32_t trailer_size
Definition: rte_mempool.h:119
#define RTE_MEMPOOL_MAX_OPS_IDX
Definition: rte_mempool.h:703
void(* rte_mempool_free_t)(struct rte_mempool *mp)
Definition: rte_mempool.h:465
rte_mempool_free_t free
Definition: rte_mempool.h:679
static __rte_always_inline int rte_mempool_get_contig_blocks(struct rte_mempool *mp, void **first_obj_table, unsigned int n)
Definition: rte_mempool.h:1720
rte_mempool_enqueue_t enqueue
Definition: rte_mempool.h:680