DPDK  20.11.10
rte_mempool.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2014 Intel Corporation.
3  * Copyright(c) 2016 6WIND S.A.
4  */
5 
6 #ifndef _RTE_MEMPOOL_H_
7 #define _RTE_MEMPOOL_H_
8 
36 #include <stdio.h>
37 #include <stdlib.h>
38 #include <stdint.h>
39 #include <errno.h>
40 #include <inttypes.h>
41 #include <sys/queue.h>
42 
43 #include <rte_config.h>
44 #include <rte_spinlock.h>
45 #include <rte_log.h>
46 #include <rte_debug.h>
47 #include <rte_lcore.h>
48 #include <rte_memory.h>
49 #include <rte_branch_prediction.h>
50 #include <rte_ring.h>
51 #include <rte_memcpy.h>
52 #include <rte_common.h>
53 
54 #include "rte_mempool_trace_fp.h"
55 
56 #ifdef __cplusplus
57 extern "C" {
58 #endif
59 
60 #define RTE_MEMPOOL_HEADER_COOKIE1 0xbadbadbadadd2e55ULL
61 #define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL
62 #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL
64 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
65 
68 struct rte_mempool_debug_stats {
69  uint64_t put_bulk;
70  uint64_t put_objs;
71  uint64_t get_success_bulk;
72  uint64_t get_success_objs;
73  uint64_t get_fail_bulk;
74  uint64_t get_fail_objs;
76  uint64_t get_success_blks;
78  uint64_t get_fail_blks;
80 #endif
81 
86  uint32_t size;
87  uint32_t flushthresh;
88  uint32_t len;
89  /*
90  * Cache is allocated to this size to allow it to overflow in certain
91  * cases to avoid needless emptying of cache.
92  */
93  void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 3];
95 
100  uint32_t elt_size;
101  uint32_t header_size;
102  uint32_t trailer_size;
103  uint32_t total_size;
105 };
106 
108 #define RTE_MEMPOOL_NAMESIZE (RTE_RING_NAMESIZE - \
109  sizeof(RTE_MEMPOOL_MZ_PREFIX) + 1)
110 #define RTE_MEMPOOL_MZ_PREFIX "MP_"
111 
112 /* "MP_<name>" */
113 #define RTE_MEMPOOL_MZ_FORMAT RTE_MEMPOOL_MZ_PREFIX "%s"
114 
115 #define MEMPOOL_PG_SHIFT_MAX \
116  RTE_DEPRECATED(MEMPOOL_PG_SHIFT_MAX) (sizeof(uintptr_t) * CHAR_BIT - 1)
117 
119 #define MEMPOOL_PG_NUM_DEFAULT RTE_DEPRECATED(MEMPOOL_PG_NUM_DEFAULT) 1
120 
121 #ifndef RTE_MEMPOOL_ALIGN
122 
125 #define RTE_MEMPOOL_ALIGN RTE_CACHE_LINE_SIZE
126 #endif
127 
128 #define RTE_MEMPOOL_ALIGN_MASK (RTE_MEMPOOL_ALIGN - 1)
129 
140  STAILQ_ENTRY(rte_mempool_objhdr) next;
141  struct rte_mempool *mp;
143 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
144  uint64_t cookie;
145 #endif
146 };
147 
151 STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr);
152 
153 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
154 
161 struct rte_mempool_objtlr {
162  uint64_t cookie;
163 };
164 
165 #endif
166 
170 STAILQ_HEAD(rte_mempool_memhdr_list, rte_mempool_memhdr);
171 
176  void *opaque);
177 
185  STAILQ_ENTRY(rte_mempool_memhdr) next;
186  struct rte_mempool *mp;
187  void *addr;
189  size_t len;
191  void *opaque;
192 };
193 
202  unsigned int contig_block_size;
204 
208 struct rte_mempool {
209  /*
210  * Note: this field kept the RTE_MEMZONE_NAMESIZE size due to ABI
211  * compatibility requirements, it could be changed to
212  * RTE_MEMPOOL_NAMESIZE next time the ABI changes
213  */
214  char name[RTE_MEMZONE_NAMESIZE];
216  union {
217  void *pool_data;
218  uint64_t pool_id;
219  };
220  void *pool_config;
221  const struct rte_memzone *mz;
222  unsigned int flags;
223  int socket_id;
224  uint32_t size;
225  uint32_t cache_size;
228  uint32_t elt_size;
229  uint32_t header_size;
230  uint32_t trailer_size;
232  unsigned private_data_size;
240  int32_t ops_index;
241 
244  uint32_t populated_size;
245  struct rte_mempool_objhdr_list elt_list;
246  uint32_t nb_mem_chunks;
247  struct rte_mempool_memhdr_list mem_list;
249 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
250 
251  struct rte_mempool_debug_stats stats[RTE_MAX_LCORE];
252 #endif
254 
255 #define MEMPOOL_F_NO_SPREAD 0x0001
256 
257 #define MEMPOOL_F_NO_CACHE_ALIGN 0x0002
258 #define MEMPOOL_F_SP_PUT 0x0004
259 #define MEMPOOL_F_SC_GET 0x0008
260 #define MEMPOOL_F_POOL_CREATED 0x0010
261 #define MEMPOOL_F_NO_IOVA_CONTIG 0x0020
273 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
274 #define __MEMPOOL_STAT_ADD(mp, name, n) do { \
275  unsigned __lcore_id = rte_lcore_id(); \
276  if (__lcore_id < RTE_MAX_LCORE) { \
277  mp->stats[__lcore_id].name##_objs += n; \
278  mp->stats[__lcore_id].name##_bulk += 1; \
279  } \
280  } while(0)
281 #define __MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, name, n) do { \
282  unsigned int __lcore_id = rte_lcore_id(); \
283  if (__lcore_id < RTE_MAX_LCORE) { \
284  mp->stats[__lcore_id].name##_blks += n; \
285  mp->stats[__lcore_id].name##_bulk += 1; \
286  } \
287  } while (0)
288 #else
289 #define __MEMPOOL_STAT_ADD(mp, name, n) do {} while(0)
290 #define __MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, name, n) do {} while (0)
291 #endif
292 
301 #define MEMPOOL_HEADER_SIZE(mp, cs) \
302  (sizeof(*(mp)) + (((cs) == 0) ? 0 : \
303  (sizeof(struct rte_mempool_cache) * RTE_MAX_LCORE)))
304 
305 /* return the header of a mempool object (internal) */
306 static inline struct rte_mempool_objhdr *__mempool_get_header(void *obj)
307 {
308  return (struct rte_mempool_objhdr *)RTE_PTR_SUB(obj,
309  sizeof(struct rte_mempool_objhdr));
310 }
311 
321 static inline struct rte_mempool *rte_mempool_from_obj(void *obj)
322 {
323  struct rte_mempool_objhdr *hdr = __mempool_get_header(obj);
324  return hdr->mp;
325 }
326 
327 /* return the trailer of a mempool object (internal) */
328 static inline struct rte_mempool_objtlr *__mempool_get_trailer(void *obj)
329 {
330  struct rte_mempool *mp = rte_mempool_from_obj(obj);
331  return (struct rte_mempool_objtlr *)RTE_PTR_ADD(obj, mp->elt_size);
332 }
333 
348 void rte_mempool_check_cookies(const struct rte_mempool *mp,
349  void * const *obj_table_const, unsigned n, int free);
350 
351 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
352 #define __mempool_check_cookies(mp, obj_table_const, n, free) \
353  rte_mempool_check_cookies(mp, obj_table_const, n, free)
354 #else
355 #define __mempool_check_cookies(mp, obj_table_const, n, free) do {} while(0)
356 #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
357 
373 void rte_mempool_contig_blocks_check_cookies(const struct rte_mempool *mp,
374  void * const *first_obj_table_const, unsigned int n, int free);
375 
376 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
377 #define __mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
378  free) \
379  rte_mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
380  free)
381 #else
382 #define __mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
383  free) \
384  do {} while (0)
385 #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
386 
387 #define RTE_MEMPOOL_OPS_NAMESIZE 32
399 typedef int (*rte_mempool_alloc_t)(struct rte_mempool *mp);
400 
404 typedef void (*rte_mempool_free_t)(struct rte_mempool *mp);
405 
412 typedef int (*rte_mempool_enqueue_t)(struct rte_mempool *mp,
413  void * const *obj_table, unsigned int n);
414 
421 typedef int (*rte_mempool_dequeue_t)(struct rte_mempool *mp,
422  void **obj_table, unsigned int n);
423 
428  void **first_obj_table, unsigned int n);
429 
433 typedef unsigned (*rte_mempool_get_count)(const struct rte_mempool *mp);
434 
458 typedef ssize_t (*rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp,
459  uint32_t obj_num, uint32_t pg_shift,
460  size_t *min_chunk_size, size_t *align);
461 
497 ssize_t rte_mempool_op_calc_mem_size_helper(const struct rte_mempool *mp,
498  uint32_t obj_num, uint32_t pg_shift, size_t chunk_reserve,
499  size_t *min_chunk_size, size_t *align);
500 
508 ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp,
509  uint32_t obj_num, uint32_t pg_shift,
510  size_t *min_chunk_size, size_t *align);
511 
524 typedef void (rte_mempool_populate_obj_cb_t)(struct rte_mempool *mp,
525  void *opaque, void *vaddr, rte_iova_t iova);
526 
555 typedef int (*rte_mempool_populate_t)(struct rte_mempool *mp,
556  unsigned int max_objs,
557  void *vaddr, rte_iova_t iova, size_t len,
558  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
559 
563 #define RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ 0x0001
564 
597 int rte_mempool_op_populate_helper(struct rte_mempool *mp,
598  unsigned int flags, unsigned int max_objs,
599  void *vaddr, rte_iova_t iova, size_t len,
600  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
601 
609  unsigned int max_objs,
610  void *vaddr, rte_iova_t iova, size_t len,
611  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
612 
616 typedef int (*rte_mempool_get_info_t)(const struct rte_mempool *mp,
617  struct rte_mempool_info *info);
618 
619 
647 
648 #define RTE_MEMPOOL_MAX_OPS_IDX 16
659 struct rte_mempool_ops_table {
661  uint32_t num_ops;
667 
670 
680 static inline struct rte_mempool_ops *
681 rte_mempool_get_ops(int ops_index)
682 {
683  RTE_VERIFY((ops_index >= 0) && (ops_index < RTE_MEMPOOL_MAX_OPS_IDX));
684 
685  return &rte_mempool_ops_table.ops[ops_index];
686 }
687 
697 int
698 rte_mempool_ops_alloc(struct rte_mempool *mp);
699 
713 static inline int
714 rte_mempool_ops_dequeue_bulk(struct rte_mempool *mp,
715  void **obj_table, unsigned n)
716 {
717  struct rte_mempool_ops *ops;
718 
719  rte_mempool_trace_ops_dequeue_bulk(mp, obj_table, n);
720  ops = rte_mempool_get_ops(mp->ops_index);
721  return ops->dequeue(mp, obj_table, n);
722 }
723 
737 static inline int
738 rte_mempool_ops_dequeue_contig_blocks(struct rte_mempool *mp,
739  void **first_obj_table, unsigned int n)
740 {
741  struct rte_mempool_ops *ops;
742 
743  ops = rte_mempool_get_ops(mp->ops_index);
744  RTE_ASSERT(ops->dequeue_contig_blocks != NULL);
745  rte_mempool_trace_ops_dequeue_contig_blocks(mp, first_obj_table, n);
746  return ops->dequeue_contig_blocks(mp, first_obj_table, n);
747 }
748 
762 static inline int
763 rte_mempool_ops_enqueue_bulk(struct rte_mempool *mp, void * const *obj_table,
764  unsigned n)
765 {
766  struct rte_mempool_ops *ops;
767 
768  rte_mempool_trace_ops_enqueue_bulk(mp, obj_table, n);
769  ops = rte_mempool_get_ops(mp->ops_index);
770  return ops->enqueue(mp, obj_table, n);
771 }
772 
781 unsigned
782 rte_mempool_ops_get_count(const struct rte_mempool *mp);
783 
803 ssize_t rte_mempool_ops_calc_mem_size(const struct rte_mempool *mp,
804  uint32_t obj_num, uint32_t pg_shift,
805  size_t *min_chunk_size, size_t *align);
806 
830 int rte_mempool_ops_populate(struct rte_mempool *mp, unsigned int max_objs,
831  void *vaddr, rte_iova_t iova, size_t len,
833  void *obj_cb_arg);
834 
847 int rte_mempool_ops_get_info(const struct rte_mempool *mp,
848  struct rte_mempool_info *info);
849 
856 void
857 rte_mempool_ops_free(struct rte_mempool *mp);
858 
876 int
877 rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name,
878  void *pool_config);
879 
890 int rte_mempool_register_ops(const struct rte_mempool_ops *ops);
891 
897 #define MEMPOOL_REGISTER_OPS(ops) \
898  RTE_INIT(mp_hdlr_init_##ops) \
899  { \
900  rte_mempool_register_ops(&ops); \
901  }
902 
908 typedef void (rte_mempool_obj_cb_t)(struct rte_mempool *mp,
909  void *opaque, void *obj, unsigned obj_idx);
910 typedef rte_mempool_obj_cb_t rte_mempool_obj_ctor_t; /* compat */
911 
917 typedef void (rte_mempool_mem_cb_t)(struct rte_mempool *mp,
918  void *opaque, struct rte_mempool_memhdr *memhdr,
919  unsigned mem_idx);
920 
927 typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *);
928 
1007 struct rte_mempool *
1008 rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
1009  unsigned cache_size, unsigned private_data_size,
1010  rte_mempool_ctor_t *mp_init, void *mp_init_arg,
1011  rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
1012  int socket_id, unsigned flags);
1013 
1048 struct rte_mempool *
1049 rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size,
1050  unsigned cache_size, unsigned private_data_size,
1051  int socket_id, unsigned flags);
1062 void
1063 rte_mempool_free(struct rte_mempool *mp);
1064 
1095 int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
1096  rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
1097  void *opaque);
1098 
1125 int
1126 rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
1127  size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb,
1128  void *opaque);
1129 
1144 
1158 int rte_mempool_populate_anon(struct rte_mempool *mp);
1159 
1175 uint32_t rte_mempool_obj_iter(struct rte_mempool *mp,
1176  rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg);
1177 
1193 uint32_t rte_mempool_mem_iter(struct rte_mempool *mp,
1194  rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg);
1195 
1204 void rte_mempool_dump(FILE *f, struct rte_mempool *mp);
1205 
1220 struct rte_mempool_cache *
1221 rte_mempool_cache_create(uint32_t size, int socket_id);
1222 
1229 void
1231 
1243 static __rte_always_inline struct rte_mempool_cache *
1244 rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
1245 {
1246  if (mp->cache_size == 0)
1247  return NULL;
1248 
1249  if (lcore_id >= RTE_MAX_LCORE)
1250  return NULL;
1251 
1252  rte_mempool_trace_default_cache(mp, lcore_id,
1253  &mp->local_cache[lcore_id]);
1254  return &mp->local_cache[lcore_id];
1255 }
1256 
1265 static __rte_always_inline void
1267  struct rte_mempool *mp)
1268 {
1269  if (cache == NULL)
1270  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1271  if (cache == NULL || cache->len == 0)
1272  return;
1273  rte_mempool_trace_cache_flush(cache, mp);
1274  rte_mempool_ops_enqueue_bulk(mp, cache->objs, cache->len);
1275  cache->len = 0;
1276 }
1277 
1290 static __rte_always_inline void
1291 __mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1292  unsigned int n, struct rte_mempool_cache *cache)
1293 {
1294  void **cache_objs;
1295 
1296  /* increment stat now, adding in mempool always success */
1297  __MEMPOOL_STAT_ADD(mp, put, n);
1298 
1299  /* No cache provided or if put would overflow mem allocated for cache */
1300  if (unlikely(cache == NULL || n > RTE_MEMPOOL_CACHE_MAX_SIZE))
1301  goto ring_enqueue;
1302 
1303  cache_objs = &cache->objs[cache->len];
1304 
1305  /*
1306  * The cache follows the following algorithm
1307  * 1. Add the objects to the cache
1308  * 2. Anything greater than the cache min value (if it crosses the
1309  * cache flush threshold) is flushed to the ring.
1310  */
1311 
1312  /* Add elements back into the cache */
1313  rte_memcpy(&cache_objs[0], obj_table, sizeof(void *) * n);
1314 
1315  cache->len += n;
1316 
1317  if (cache->len >= cache->flushthresh) {
1318  rte_mempool_ops_enqueue_bulk(mp, &cache->objs[cache->size],
1319  cache->len - cache->size);
1320  cache->len = cache->size;
1321  }
1322 
1323  return;
1324 
1325 ring_enqueue:
1326 
1327  /* push remaining objects in ring */
1328 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
1329  if (rte_mempool_ops_enqueue_bulk(mp, obj_table, n) < 0)
1330  rte_panic("cannot put objects in mempool\n");
1331 #else
1332  rte_mempool_ops_enqueue_bulk(mp, obj_table, n);
1333 #endif
1334 }
1335 
1336 
1349 static __rte_always_inline void
1350 rte_mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1351  unsigned int n, struct rte_mempool_cache *cache)
1352 {
1353  rte_mempool_trace_generic_put(mp, obj_table, n, cache);
1354  __mempool_check_cookies(mp, obj_table, n, 0);
1355  __mempool_generic_put(mp, obj_table, n, cache);
1356 }
1357 
1372 static __rte_always_inline void
1373 rte_mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,
1374  unsigned int n)
1375 {
1376  struct rte_mempool_cache *cache;
1377  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1378  rte_mempool_trace_put_bulk(mp, obj_table, n, cache);
1379  rte_mempool_generic_put(mp, obj_table, n, cache);
1380 }
1381 
1394 static __rte_always_inline void
1395 rte_mempool_put(struct rte_mempool *mp, void *obj)
1396 {
1397  rte_mempool_put_bulk(mp, &obj, 1);
1398 }
1399 
1414 static __rte_always_inline int
1415 __mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1416  unsigned int n, struct rte_mempool_cache *cache)
1417 {
1418  int ret;
1419  unsigned int remaining = n;
1420  uint32_t index, len;
1421  void **cache_objs;
1422 
1423  /* No cache provided */
1424  if (unlikely(cache == NULL))
1425  goto driver_dequeue;
1426 
1427  /* Use the cache as much as we have to return hot objects first */
1428  len = RTE_MIN(remaining, cache->len);
1429  cache_objs = &cache->objs[cache->len];
1430  cache->len -= len;
1431  remaining -= len;
1432  for (index = 0; index < len; index++)
1433  *obj_table++ = *--cache_objs;
1434 
1435  if (remaining == 0) {
1436  /* The entire request is satisfied from the cache. */
1437 
1438  __MEMPOOL_STAT_ADD(mp, get_success, n);
1439 
1440  return 0;
1441  }
1442 
1443  /* if dequeue below would overflow mem allocated for cache */
1444  if (unlikely(remaining > RTE_MEMPOOL_CACHE_MAX_SIZE))
1445  goto driver_dequeue;
1446 
1447  /* Fill the cache from the backend; fetch size + remaining objects. */
1448  ret = rte_mempool_ops_dequeue_bulk(mp, cache->objs,
1449  cache->size + remaining);
1450  if (unlikely(ret < 0)) {
1451  /*
1452  * We are buffer constrained, and not able to allocate
1453  * cache + remaining.
1454  * Do not fill the cache, just satisfy the remaining part of
1455  * the request directly from the backend.
1456  */
1457  goto driver_dequeue;
1458  }
1459 
1460  /* Satisfy the remaining part of the request from the filled cache. */
1461  cache_objs = &cache->objs[cache->size + remaining];
1462  for (index = 0; index < remaining; index++)
1463  *obj_table++ = *--cache_objs;
1464 
1465  cache->len = cache->size;
1466 
1467  __MEMPOOL_STAT_ADD(mp, get_success, n);
1468 
1469  return 0;
1470 
1471 driver_dequeue:
1472 
1473  /* Get remaining objects directly from the backend. */
1474  ret = rte_mempool_ops_dequeue_bulk(mp, obj_table, remaining);
1475 
1476  if (ret < 0) {
1477  if (likely(cache != NULL)) {
1478  cache->len = n - remaining;
1479  /*
1480  * No further action is required to roll the first part
1481  * of the request back into the cache, as objects in
1482  * the cache are intact.
1483  */
1484  }
1485 
1486  __MEMPOOL_STAT_ADD(mp, get_fail, n);
1487  } else
1488  __MEMPOOL_STAT_ADD(mp, get_success, n);
1489 
1490  return ret;
1491 }
1492 
1513 static __rte_always_inline int
1514 rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1515  unsigned int n, struct rte_mempool_cache *cache)
1516 {
1517  int ret;
1518  ret = __mempool_generic_get(mp, obj_table, n, cache);
1519  if (ret == 0)
1520  __mempool_check_cookies(mp, obj_table, n, 1);
1521  rte_mempool_trace_generic_get(mp, obj_table, n, cache);
1522  return ret;
1523 }
1524 
1547 static __rte_always_inline int
1548 rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
1549 {
1550  struct rte_mempool_cache *cache;
1551  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1552  rte_mempool_trace_get_bulk(mp, obj_table, n, cache);
1553  return rte_mempool_generic_get(mp, obj_table, n, cache);
1554 }
1555 
1576 static __rte_always_inline int
1577 rte_mempool_get(struct rte_mempool *mp, void **obj_p)
1578 {
1579  return rte_mempool_get_bulk(mp, obj_p, 1);
1580 }
1581 
1603 static __rte_always_inline int
1605  void **first_obj_table, unsigned int n)
1606 {
1607  int ret;
1608 
1609  ret = rte_mempool_ops_dequeue_contig_blocks(mp, first_obj_table, n);
1610  if (ret == 0) {
1611  __MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, get_success, n);
1612  __mempool_contig_blocks_check_cookies(mp, first_obj_table, n,
1613  1);
1614  } else {
1615  __MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, get_fail, n);
1616  }
1617 
1618  rte_mempool_trace_get_contig_blocks(mp, first_obj_table, n);
1619  return ret;
1620 }
1621 
1634 unsigned int rte_mempool_avail_count(const struct rte_mempool *mp);
1635 
1648 unsigned int
1649 rte_mempool_in_use_count(const struct rte_mempool *mp);
1650 
1664 static inline int
1666 {
1667  return rte_mempool_avail_count(mp) == mp->size;
1668 }
1669 
1683 static inline int
1685 {
1686  return rte_mempool_avail_count(mp) == 0;
1687 }
1688 
1699 static inline rte_iova_t
1700 rte_mempool_virt2iova(const void *elt)
1701 {
1702  const struct rte_mempool_objhdr *hdr;
1703  hdr = (const struct rte_mempool_objhdr *)RTE_PTR_SUB(elt,
1704  sizeof(*hdr));
1705  return hdr->iova;
1706 }
1707 
1718 void rte_mempool_audit(struct rte_mempool *mp);
1719 
1728 static inline void *rte_mempool_get_priv(struct rte_mempool *mp)
1729 {
1730  return (char *)mp +
1732 }
1733 
1740 void rte_mempool_list_dump(FILE *f);
1741 
1754 struct rte_mempool *rte_mempool_lookup(const char *name);
1755 
1773 uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
1774  struct rte_mempool_objsz *sz);
1775 
1784 void rte_mempool_walk(void (*func)(struct rte_mempool *, void *arg),
1785  void *arg);
1786 
1791 int
1792 rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz);
1793 
1794 #ifdef __cplusplus
1795 }
1796 #endif
1797 
1798 #endif /* _RTE_MEMPOOL_H_ */
uint64_t pool_id
Definition: rte_mempool.h:218
#define __rte_always_inline
Definition: rte_common.h:231
struct rte_mempool * rte_mempool_lookup(const char *name)
struct rte_mempool_cache * rte_mempool_cache_create(uint32_t size, int socket_id)
uint32_t rte_mempool_mem_iter(struct rte_mempool *mp, rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg)
uint32_t header_size
Definition: rte_mempool.h:229
#define likely(x)
const struct rte_memzone * mz
Definition: rte_mempool.h:221
uint64_t rte_iova_t
Definition: rte_common.h:423
static __rte_always_inline void rte_mempool_put_bulk(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
Definition: rte_mempool.h:1373
struct rte_mempool * mp
Definition: rte_mempool.h:186
void rte_mempool_list_dump(FILE *f)
void() rte_mempool_memchunk_free_cb_t(struct rte_mempool_memhdr *memhdr, void *opaque)
Definition: rte_mempool.h:175
int(* rte_mempool_dequeue_contig_blocks_t)(struct rte_mempool *mp, void **first_obj_table, unsigned int n)
Definition: rte_mempool.h:427
rte_mempool_alloc_t alloc
Definition: rte_mempool.h:623
void() rte_mempool_populate_obj_cb_t(struct rte_mempool *mp, void *opaque, void *vaddr, rte_iova_t iova)
Definition: rte_mempool.h:524
unsigned int flags
Definition: rte_mempool.h:222
static __rte_always_inline void rte_mempool_generic_put(struct rte_mempool *mp, void *const *obj_table, unsigned int n, struct rte_mempool_cache *cache)
Definition: rte_mempool.h:1350
int(* rte_mempool_alloc_t)(struct rte_mempool *mp)
Definition: rte_mempool.h:399
static int rte_mempool_empty(const struct rte_mempool *mp)
Definition: rte_mempool.h:1684
rte_mempool_memchunk_free_cb_t * free_cb
Definition: rte_mempool.h:190
char name[RTE_MEMPOOL_OPS_NAMESIZE]
Definition: rte_mempool.h:622
int(* rte_mempool_dequeue_t)(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:421
#define MEMPOOL_HEADER_SIZE(mp, cs)
Definition: rte_mempool.h:301
rte_mempool_get_info_t get_info
Definition: rte_mempool.h:641
rte_mempool_dequeue_contig_blocks_t dequeue_contig_blocks
Definition: rte_mempool.h:645
uint32_t cache_size
Definition: rte_mempool.h:225
unsigned int rte_mempool_avail_count(const struct rte_mempool *mp)
uint32_t size
Definition: rte_mempool.h:224
#define RTE_PTR_ADD(ptr, x)
Definition: rte_common.h:253
void rte_mempool_cache_free(struct rte_mempool_cache *cache)
uint32_t total_size
Definition: rte_mempool.h:103
uint32_t rte_mempool_obj_iter(struct rte_mempool *mp, rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg)
int rte_mempool_ops_get_info(const struct rte_mempool *mp, struct rte_mempool_info *info)
int rte_mempool_register_ops(const struct rte_mempool_ops *ops)
#define unlikely(x)
static __rte_always_inline int rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table, unsigned int n, struct rte_mempool_cache *cache)
Definition: rte_mempool.h:1514
int rte_mempool_populate_default(struct rte_mempool *mp)
#define RTE_MIN(a, b)
Definition: rte_common.h:578
static __rte_always_inline void rte_mempool_cache_flush(struct rte_mempool_cache *cache, struct rte_mempool *mp)
Definition: rte_mempool.h:1266
#define RTE_MEMPOOL_OPS_NAMESIZE
Definition: rte_mempool.h:387
static __rte_always_inline int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
Definition: rte_mempool.h:1577
STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr)
uint32_t nb_mem_chunks
Definition: rte_mempool.h:246
struct rte_mempool * mp
Definition: rte_mempool.h:141
struct rte_mempool_ops ops[RTE_MEMPOOL_MAX_OPS_IDX]
Definition: rte_mempool.h:665
rte_mempool_get_count get_count
Definition: rte_mempool.h:627
ssize_t(* rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp, uint32_t obj_num, uint32_t pg_shift, size_t *min_chunk_size, size_t *align)
Definition: rte_mempool.h:458
static int rte_mempool_full(const struct rte_mempool *mp)
Definition: rte_mempool.h:1665
static unsigned rte_lcore_id(void)
Definition: rte_lcore.h:75
void * pool_config
Definition: rte_mempool.h:220
unsigned int rte_mempool_in_use_count(const struct rte_mempool *mp)
static __rte_always_inline int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:1548
#define rte_panic(...)
Definition: rte_debug.h:43
int rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name, void *pool_config)
int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr, rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
rte_mempool_calc_mem_size_t calc_mem_size
Definition: rte_mempool.h:632
unsigned(* rte_mempool_get_count)(const struct rte_mempool *mp)
Definition: rte_mempool.h:433
uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, struct rte_mempool_objsz *sz)
ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp, uint32_t obj_num, uint32_t pg_shift, size_t *min_chunk_size, size_t *align)
void * objs[RTE_MEMPOOL_CACHE_MAX_SIZE *3]
Definition: rte_mempool.h:93
void * pool_data
Definition: rte_mempool.h:217
unsigned int contig_block_size
Definition: rte_mempool.h:202
int rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
int(* rte_mempool_populate_t)(struct rte_mempool *mp, unsigned int max_objs, void *vaddr, rte_iova_t iova, size_t len, rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
Definition: rte_mempool.h:555
uint32_t elt_size
Definition: rte_mempool.h:228
void rte_mempool_audit(struct rte_mempool *mp)
int(* rte_mempool_get_info_t)(const struct rte_mempool *mp, struct rte_mempool_info *info)
Definition: rte_mempool.h:616
struct rte_mempool * rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, int socket_id, unsigned flags)
unsigned private_data_size
Definition: rte_mempool.h:232
rte_spinlock_t sl
Definition: rte_mempool.h:660
static __rte_always_inline struct rte_mempool_cache * rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
Definition: rte_mempool.h:1244
void rte_mempool_dump(FILE *f, struct rte_mempool *mp)
void() rte_mempool_ctor_t(struct rte_mempool *, void *)
Definition: rte_mempool.h:927
int rte_mempool_populate_anon(struct rte_mempool *mp)
struct rte_mempool_cache * local_cache
Definition: rte_mempool.h:242
#define __rte_cache_aligned
Definition: rte_common.h:405
uint32_t trailer_size
Definition: rte_mempool.h:230
#define RTE_STD_C11
Definition: rte_common.h:40
rte_mempool_dequeue_t dequeue
Definition: rte_mempool.h:626
int(* rte_mempool_enqueue_t)(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
Definition: rte_mempool.h:412
uint32_t flushthresh
Definition: rte_mempool.h:87
void() rte_mempool_obj_cb_t(struct rte_mempool *mp, void *opaque, void *obj, unsigned obj_idx)
Definition: rte_mempool.h:908
void() rte_mempool_mem_cb_t(struct rte_mempool *mp, void *opaque, struct rte_mempool_memhdr *memhdr, unsigned mem_idx)
Definition: rte_mempool.h:917
int rte_mempool_op_populate_default(struct rte_mempool *mp, unsigned int max_objs, void *vaddr, rte_iova_t iova, size_t len, rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
struct rte_mempool * rte_mempool_create(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags)
#define RTE_PTR_SUB(ptr, x)
Definition: rte_common.h:258
static struct rte_mempool * rte_mempool_from_obj(void *obj)
Definition: rte_mempool.h:321
uint32_t header_size
Definition: rte_mempool.h:101
int32_t ops_index
Definition: rte_mempool.h:240
static void * rte_memcpy(void *dst, const void *src, size_t n)
uint32_t populated_size
Definition: rte_mempool.h:244
static rte_iova_t rte_mempool_virt2iova(const void *elt)
Definition: rte_mempool.h:1700
static __rte_always_inline void rte_mempool_put(struct rte_mempool *mp, void *obj)
Definition: rte_mempool.h:1395
rte_mempool_populate_t populate
Definition: rte_mempool.h:637
void rte_mempool_free(struct rte_mempool *mp)
void rte_mempool_walk(void(*func)(struct rte_mempool *, void *arg), void *arg)
static void * rte_mempool_get_priv(struct rte_mempool *mp)
Definition: rte_mempool.h:1728
char name[RTE_MEMZONE_NAMESIZE]
Definition: rte_mempool.h:214
uint32_t trailer_size
Definition: rte_mempool.h:102
#define RTE_MEMPOOL_MAX_OPS_IDX
Definition: rte_mempool.h:648
void(* rte_mempool_free_t)(struct rte_mempool *mp)
Definition: rte_mempool.h:404
rte_mempool_free_t free
Definition: rte_mempool.h:624
static __rte_always_inline int rte_mempool_get_contig_blocks(struct rte_mempool *mp, void **first_obj_table, unsigned int n)
Definition: rte_mempool.h:1604
rte_mempool_enqueue_t enqueue
Definition: rte_mempool.h:625
#define RTE_MEMZONE_NAMESIZE
Definition: rte_memzone.h:51