DPDK  21.02.0
rte_mempool.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2014 Intel Corporation.
3  * Copyright(c) 2016 6WIND S.A.
4  */
5 
6 #ifndef _RTE_MEMPOOL_H_
7 #define _RTE_MEMPOOL_H_
8 
36 #include <stdio.h>
37 #include <stdlib.h>
38 #include <stdint.h>
39 #include <errno.h>
40 #include <inttypes.h>
41 #include <sys/queue.h>
42 
43 #include <rte_config.h>
44 #include <rte_spinlock.h>
45 #include <rte_log.h>
46 #include <rte_debug.h>
47 #include <rte_lcore.h>
48 #include <rte_memory.h>
49 #include <rte_branch_prediction.h>
50 #include <rte_ring.h>
51 #include <rte_memcpy.h>
52 #include <rte_common.h>
53 
54 #include "rte_mempool_trace_fp.h"
55 
56 #ifdef __cplusplus
57 extern "C" {
58 #endif
59 
60 #define RTE_MEMPOOL_HEADER_COOKIE1 0xbadbadbadadd2e55ULL
61 #define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL
62 #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL
64 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
65 
68 struct rte_mempool_debug_stats {
69  uint64_t put_bulk;
70  uint64_t put_objs;
71  uint64_t get_success_bulk;
72  uint64_t get_success_objs;
73  uint64_t get_fail_bulk;
74  uint64_t get_fail_objs;
76  uint64_t get_success_blks;
78  uint64_t get_fail_blks;
80 #endif
81 
86  uint32_t size;
87  uint32_t flushthresh;
88  uint32_t len;
89  /*
90  * Cache is allocated to this size to allow it to overflow in certain
91  * cases to avoid needless emptying of cache.
92  */
93  void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 3];
95 
100  uint32_t elt_size;
101  uint32_t header_size;
102  uint32_t trailer_size;
103  uint32_t total_size;
105 };
106 
108 #define RTE_MEMPOOL_NAMESIZE (RTE_RING_NAMESIZE - \
109  sizeof(RTE_MEMPOOL_MZ_PREFIX) + 1)
110 #define RTE_MEMPOOL_MZ_PREFIX "MP_"
111 
112 /* "MP_<name>" */
113 #define RTE_MEMPOOL_MZ_FORMAT RTE_MEMPOOL_MZ_PREFIX "%s"
114 
115 #define MEMPOOL_PG_SHIFT_MAX (sizeof(uintptr_t) * CHAR_BIT - 1)
116 
118 #define MEMPOOL_PG_NUM_DEFAULT 1
119 
120 #ifndef RTE_MEMPOOL_ALIGN
121 
124 #define RTE_MEMPOOL_ALIGN RTE_CACHE_LINE_SIZE
125 #endif
126 
127 #define RTE_MEMPOOL_ALIGN_MASK (RTE_MEMPOOL_ALIGN - 1)
128 
140  struct rte_mempool *mp;
142 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
143  uint64_t cookie;
144 #endif
145 };
146 
150 STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr);
151 
152 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
153 
160 struct rte_mempool_objtlr {
161  uint64_t cookie;
162 };
163 
164 #endif
165 
169 STAILQ_HEAD(rte_mempool_memhdr_list, rte_mempool_memhdr);
170 
175  void *opaque);
176 
185  struct rte_mempool *mp;
186  void *addr;
188  size_t len;
190  void *opaque;
191 };
192 
201  unsigned int contig_block_size;
203 
207 struct rte_mempool {
208  /*
209  * Note: this field kept the RTE_MEMZONE_NAMESIZE size due to ABI
210  * compatibility requirements, it could be changed to
211  * RTE_MEMPOOL_NAMESIZE next time the ABI changes
212  */
215  union {
216  void *pool_data;
217  uint64_t pool_id;
218  };
219  void *pool_config;
220  const struct rte_memzone *mz;
221  unsigned int flags;
222  int socket_id;
223  uint32_t size;
224  uint32_t cache_size;
227  uint32_t elt_size;
228  uint32_t header_size;
229  uint32_t trailer_size;
231  unsigned private_data_size;
239  int32_t ops_index;
240 
243  uint32_t populated_size;
244  struct rte_mempool_objhdr_list elt_list;
245  uint32_t nb_mem_chunks;
246  struct rte_mempool_memhdr_list mem_list;
248 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
249 
250  struct rte_mempool_debug_stats stats[RTE_MAX_LCORE];
251 #endif
253 
254 #define MEMPOOL_F_NO_SPREAD 0x0001
255 
256 #define MEMPOOL_F_NO_CACHE_ALIGN 0x0002
257 #define MEMPOOL_F_SP_PUT 0x0004
258 #define MEMPOOL_F_SC_GET 0x0008
259 #define MEMPOOL_F_POOL_CREATED 0x0010
260 #define MEMPOOL_F_NO_IOVA_CONTIG 0x0020
272 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
273 #define __MEMPOOL_STAT_ADD(mp, name, n) do { \
274  unsigned __lcore_id = rte_lcore_id(); \
275  if (__lcore_id < RTE_MAX_LCORE) { \
276  mp->stats[__lcore_id].name##_objs += n; \
277  mp->stats[__lcore_id].name##_bulk += 1; \
278  } \
279  } while(0)
280 #define __MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, name, n) do { \
281  unsigned int __lcore_id = rte_lcore_id(); \
282  if (__lcore_id < RTE_MAX_LCORE) { \
283  mp->stats[__lcore_id].name##_blks += n; \
284  mp->stats[__lcore_id].name##_bulk += 1; \
285  } \
286  } while (0)
287 #else
288 #define __MEMPOOL_STAT_ADD(mp, name, n) do {} while(0)
289 #define __MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, name, n) do {} while (0)
290 #endif
291 
300 #define MEMPOOL_HEADER_SIZE(mp, cs) \
301  (sizeof(*(mp)) + (((cs) == 0) ? 0 : \
302  (sizeof(struct rte_mempool_cache) * RTE_MAX_LCORE)))
303 
304 /* return the header of a mempool object (internal) */
305 static inline struct rte_mempool_objhdr *__mempool_get_header(void *obj)
306 {
307  return (struct rte_mempool_objhdr *)RTE_PTR_SUB(obj,
308  sizeof(struct rte_mempool_objhdr));
309 }
310 
320 static inline struct rte_mempool *rte_mempool_from_obj(void *obj)
321 {
322  struct rte_mempool_objhdr *hdr = __mempool_get_header(obj);
323  return hdr->mp;
324 }
325 
326 /* return the trailer of a mempool object (internal) */
327 static inline struct rte_mempool_objtlr *__mempool_get_trailer(void *obj)
328 {
329  struct rte_mempool *mp = rte_mempool_from_obj(obj);
330  return (struct rte_mempool_objtlr *)RTE_PTR_ADD(obj, mp->elt_size);
331 }
332 
347 void rte_mempool_check_cookies(const struct rte_mempool *mp,
348  void * const *obj_table_const, unsigned n, int free);
349 
350 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
351 #define __mempool_check_cookies(mp, obj_table_const, n, free) \
352  rte_mempool_check_cookies(mp, obj_table_const, n, free)
353 #else
354 #define __mempool_check_cookies(mp, obj_table_const, n, free) do {} while(0)
355 #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
356 
372 void rte_mempool_contig_blocks_check_cookies(const struct rte_mempool *mp,
373  void * const *first_obj_table_const, unsigned int n, int free);
374 
375 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
376 #define __mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
377  free) \
378  rte_mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
379  free)
380 #else
381 #define __mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
382  free) \
383  do {} while (0)
384 #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
385 
386 #define RTE_MEMPOOL_OPS_NAMESIZE 32
398 typedef int (*rte_mempool_alloc_t)(struct rte_mempool *mp);
399 
403 typedef void (*rte_mempool_free_t)(struct rte_mempool *mp);
404 
408 typedef int (*rte_mempool_enqueue_t)(struct rte_mempool *mp,
409  void * const *obj_table, unsigned int n);
410 
414 typedef int (*rte_mempool_dequeue_t)(struct rte_mempool *mp,
415  void **obj_table, unsigned int n);
416 
421  void **first_obj_table, unsigned int n);
422 
426 typedef unsigned (*rte_mempool_get_count)(const struct rte_mempool *mp);
427 
451 typedef ssize_t (*rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp,
452  uint32_t obj_num, uint32_t pg_shift,
453  size_t *min_chunk_size, size_t *align);
454 
490 ssize_t rte_mempool_op_calc_mem_size_helper(const struct rte_mempool *mp,
491  uint32_t obj_num, uint32_t pg_shift, size_t chunk_reserve,
492  size_t *min_chunk_size, size_t *align);
493 
501 ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp,
502  uint32_t obj_num, uint32_t pg_shift,
503  size_t *min_chunk_size, size_t *align);
504 
517 typedef void (rte_mempool_populate_obj_cb_t)(struct rte_mempool *mp,
518  void *opaque, void *vaddr, rte_iova_t iova);
519 
548 typedef int (*rte_mempool_populate_t)(struct rte_mempool *mp,
549  unsigned int max_objs,
550  void *vaddr, rte_iova_t iova, size_t len,
551  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
552 
556 #define RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ 0x0001
557 
590 int rte_mempool_op_populate_helper(struct rte_mempool *mp,
591  unsigned int flags, unsigned int max_objs,
592  void *vaddr, rte_iova_t iova, size_t len,
593  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
594 
602  unsigned int max_objs,
603  void *vaddr, rte_iova_t iova, size_t len,
604  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
605 
609 typedef int (*rte_mempool_get_info_t)(const struct rte_mempool *mp,
610  struct rte_mempool_info *info);
611 
612 
640 
641 #define RTE_MEMPOOL_MAX_OPS_IDX 16
652 struct rte_mempool_ops_table {
654  uint32_t num_ops;
660 
663 
673 static inline struct rte_mempool_ops *
674 rte_mempool_get_ops(int ops_index)
675 {
676  RTE_VERIFY((ops_index >= 0) && (ops_index < RTE_MEMPOOL_MAX_OPS_IDX));
677 
678  return &rte_mempool_ops_table.ops[ops_index];
679 }
680 
690 int
691 rte_mempool_ops_alloc(struct rte_mempool *mp);
692 
706 static inline int
707 rte_mempool_ops_dequeue_bulk(struct rte_mempool *mp,
708  void **obj_table, unsigned n)
709 {
710  struct rte_mempool_ops *ops;
711 
712  rte_mempool_trace_ops_dequeue_bulk(mp, obj_table, n);
713  ops = rte_mempool_get_ops(mp->ops_index);
714  return ops->dequeue(mp, obj_table, n);
715 }
716 
730 static inline int
731 rte_mempool_ops_dequeue_contig_blocks(struct rte_mempool *mp,
732  void **first_obj_table, unsigned int n)
733 {
734  struct rte_mempool_ops *ops;
735 
736  ops = rte_mempool_get_ops(mp->ops_index);
737  RTE_ASSERT(ops->dequeue_contig_blocks != NULL);
738  rte_mempool_trace_ops_dequeue_contig_blocks(mp, first_obj_table, n);
739  return ops->dequeue_contig_blocks(mp, first_obj_table, n);
740 }
741 
755 static inline int
756 rte_mempool_ops_enqueue_bulk(struct rte_mempool *mp, void * const *obj_table,
757  unsigned n)
758 {
759  struct rte_mempool_ops *ops;
760 
761  rte_mempool_trace_ops_enqueue_bulk(mp, obj_table, n);
762  ops = rte_mempool_get_ops(mp->ops_index);
763  return ops->enqueue(mp, obj_table, n);
764 }
765 
774 unsigned
775 rte_mempool_ops_get_count(const struct rte_mempool *mp);
776 
796 ssize_t rte_mempool_ops_calc_mem_size(const struct rte_mempool *mp,
797  uint32_t obj_num, uint32_t pg_shift,
798  size_t *min_chunk_size, size_t *align);
799 
823 int rte_mempool_ops_populate(struct rte_mempool *mp, unsigned int max_objs,
824  void *vaddr, rte_iova_t iova, size_t len,
826  void *obj_cb_arg);
827 
840 int rte_mempool_ops_get_info(const struct rte_mempool *mp,
841  struct rte_mempool_info *info);
842 
849 void
850 rte_mempool_ops_free(struct rte_mempool *mp);
851 
869 int
870 rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name,
871  void *pool_config);
872 
883 int rte_mempool_register_ops(const struct rte_mempool_ops *ops);
884 
890 #define MEMPOOL_REGISTER_OPS(ops) \
891  RTE_INIT(mp_hdlr_init_##ops) \
892  { \
893  rte_mempool_register_ops(&ops); \
894  }
895 
901 typedef void (rte_mempool_obj_cb_t)(struct rte_mempool *mp,
902  void *opaque, void *obj, unsigned obj_idx);
903 typedef rte_mempool_obj_cb_t rte_mempool_obj_ctor_t; /* compat */
904 
910 typedef void (rte_mempool_mem_cb_t)(struct rte_mempool *mp,
911  void *opaque, struct rte_mempool_memhdr *memhdr,
912  unsigned mem_idx);
913 
920 typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *);
921 
1001 struct rte_mempool *
1002 rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
1003  unsigned cache_size, unsigned private_data_size,
1004  rte_mempool_ctor_t *mp_init, void *mp_init_arg,
1005  rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
1006  int socket_id, unsigned flags);
1007 
1042 struct rte_mempool *
1043 rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size,
1044  unsigned cache_size, unsigned private_data_size,
1045  int socket_id, unsigned flags);
1056 void
1057 rte_mempool_free(struct rte_mempool *mp);
1058 
1089 int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
1090  rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
1091  void *opaque);
1092 
1119 int
1120 rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
1121  size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb,
1122  void *opaque);
1123 
1138 
1152 int rte_mempool_populate_anon(struct rte_mempool *mp);
1153 
1169 uint32_t rte_mempool_obj_iter(struct rte_mempool *mp,
1170  rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg);
1171 
1187 uint32_t rte_mempool_mem_iter(struct rte_mempool *mp,
1188  rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg);
1189 
1198 void rte_mempool_dump(FILE *f, struct rte_mempool *mp);
1199 
1214 struct rte_mempool_cache *
1215 rte_mempool_cache_create(uint32_t size, int socket_id);
1216 
1223 void
1225 
1237 static __rte_always_inline struct rte_mempool_cache *
1238 rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
1239 {
1240  if (mp->cache_size == 0)
1241  return NULL;
1242 
1243  if (lcore_id >= RTE_MAX_LCORE)
1244  return NULL;
1245 
1246  rte_mempool_trace_default_cache(mp, lcore_id,
1247  &mp->local_cache[lcore_id]);
1248  return &mp->local_cache[lcore_id];
1249 }
1250 
1259 static __rte_always_inline void
1261  struct rte_mempool *mp)
1262 {
1263  if (cache == NULL)
1264  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1265  if (cache == NULL || cache->len == 0)
1266  return;
1267  rte_mempool_trace_cache_flush(cache, mp);
1268  rte_mempool_ops_enqueue_bulk(mp, cache->objs, cache->len);
1269  cache->len = 0;
1270 }
1271 
1284 static __rte_always_inline void
1285 __mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1286  unsigned int n, struct rte_mempool_cache *cache)
1287 {
1288  void **cache_objs;
1289 
1290  /* increment stat now, adding in mempool always success */
1291  __MEMPOOL_STAT_ADD(mp, put, n);
1292 
1293  /* No cache provided or if put would overflow mem allocated for cache */
1294  if (unlikely(cache == NULL || n > RTE_MEMPOOL_CACHE_MAX_SIZE))
1295  goto ring_enqueue;
1296 
1297  cache_objs = &cache->objs[cache->len];
1298 
1299  /*
1300  * The cache follows the following algorithm
1301  * 1. Add the objects to the cache
1302  * 2. Anything greater than the cache min value (if it crosses the
1303  * cache flush threshold) is flushed to the ring.
1304  */
1305 
1306  /* Add elements back into the cache */
1307  rte_memcpy(&cache_objs[0], obj_table, sizeof(void *) * n);
1308 
1309  cache->len += n;
1310 
1311  if (cache->len >= cache->flushthresh) {
1312  rte_mempool_ops_enqueue_bulk(mp, &cache->objs[cache->size],
1313  cache->len - cache->size);
1314  cache->len = cache->size;
1315  }
1316 
1317  return;
1318 
1319 ring_enqueue:
1320 
1321  /* push remaining objects in ring */
1322 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
1323  if (rte_mempool_ops_enqueue_bulk(mp, obj_table, n) < 0)
1324  rte_panic("cannot put objects in mempool\n");
1325 #else
1326  rte_mempool_ops_enqueue_bulk(mp, obj_table, n);
1327 #endif
1328 }
1329 
1330 
1343 static __rte_always_inline void
1344 rte_mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1345  unsigned int n, struct rte_mempool_cache *cache)
1346 {
1347  rte_mempool_trace_generic_put(mp, obj_table, n, cache);
1348  __mempool_check_cookies(mp, obj_table, n, 0);
1349  __mempool_generic_put(mp, obj_table, n, cache);
1350 }
1351 
1366 static __rte_always_inline void
1367 rte_mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,
1368  unsigned int n)
1369 {
1370  struct rte_mempool_cache *cache;
1371  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1372  rte_mempool_trace_put_bulk(mp, obj_table, n, cache);
1373  rte_mempool_generic_put(mp, obj_table, n, cache);
1374 }
1375 
1388 static __rte_always_inline void
1389 rte_mempool_put(struct rte_mempool *mp, void *obj)
1390 {
1391  rte_mempool_put_bulk(mp, &obj, 1);
1392 }
1393 
1408 static __rte_always_inline int
1409 __mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1410  unsigned int n, struct rte_mempool_cache *cache)
1411 {
1412  int ret;
1413  uint32_t index, len;
1414  void **cache_objs;
1415 
1416  /* No cache provided or cannot be satisfied from cache */
1417  if (unlikely(cache == NULL || n >= cache->size))
1418  goto ring_dequeue;
1419 
1420  cache_objs = cache->objs;
1421 
1422  /* Can this be satisfied from the cache? */
1423  if (cache->len < n) {
1424  /* No. Backfill the cache first, and then fill from it */
1425  uint32_t req = n + (cache->size - cache->len);
1426 
1427  /* How many do we require i.e. number to fill the cache + the request */
1428  ret = rte_mempool_ops_dequeue_bulk(mp,
1429  &cache->objs[cache->len], req);
1430  if (unlikely(ret < 0)) {
1431  /*
1432  * In the off chance that we are buffer constrained,
1433  * where we are not able to allocate cache + n, go to
1434  * the ring directly. If that fails, we are truly out of
1435  * buffers.
1436  */
1437  goto ring_dequeue;
1438  }
1439 
1440  cache->len += req;
1441  }
1442 
1443  /* Now fill in the response ... */
1444  for (index = 0, len = cache->len - 1; index < n; ++index, len--, obj_table++)
1445  *obj_table = cache_objs[len];
1446 
1447  cache->len -= n;
1448 
1449  __MEMPOOL_STAT_ADD(mp, get_success, n);
1450 
1451  return 0;
1452 
1453 ring_dequeue:
1454 
1455  /* get remaining objects from ring */
1456  ret = rte_mempool_ops_dequeue_bulk(mp, obj_table, n);
1457 
1458  if (ret < 0)
1459  __MEMPOOL_STAT_ADD(mp, get_fail, n);
1460  else
1461  __MEMPOOL_STAT_ADD(mp, get_success, n);
1462 
1463  return ret;
1464 }
1465 
1486 static __rte_always_inline int
1487 rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1488  unsigned int n, struct rte_mempool_cache *cache)
1489 {
1490  int ret;
1491  ret = __mempool_generic_get(mp, obj_table, n, cache);
1492  if (ret == 0)
1493  __mempool_check_cookies(mp, obj_table, n, 1);
1494  rte_mempool_trace_generic_get(mp, obj_table, n, cache);
1495  return ret;
1496 }
1497 
1520 static __rte_always_inline int
1521 rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
1522 {
1523  struct rte_mempool_cache *cache;
1524  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1525  rte_mempool_trace_get_bulk(mp, obj_table, n, cache);
1526  return rte_mempool_generic_get(mp, obj_table, n, cache);
1527 }
1528 
1549 static __rte_always_inline int
1550 rte_mempool_get(struct rte_mempool *mp, void **obj_p)
1551 {
1552  return rte_mempool_get_bulk(mp, obj_p, 1);
1553 }
1554 
1576 static __rte_always_inline int
1578  void **first_obj_table, unsigned int n)
1579 {
1580  int ret;
1581 
1582  ret = rte_mempool_ops_dequeue_contig_blocks(mp, first_obj_table, n);
1583  if (ret == 0) {
1584  __MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, get_success, n);
1585  __mempool_contig_blocks_check_cookies(mp, first_obj_table, n,
1586  1);
1587  } else {
1588  __MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, get_fail, n);
1589  }
1590 
1591  rte_mempool_trace_get_contig_blocks(mp, first_obj_table, n);
1592  return ret;
1593 }
1594 
1607 unsigned int rte_mempool_avail_count(const struct rte_mempool *mp);
1608 
1621 unsigned int
1622 rte_mempool_in_use_count(const struct rte_mempool *mp);
1623 
1637 static inline int
1639 {
1640  return rte_mempool_avail_count(mp) == mp->size;
1641 }
1642 
1656 static inline int
1658 {
1659  return rte_mempool_avail_count(mp) == 0;
1660 }
1661 
1672 static inline rte_iova_t
1673 rte_mempool_virt2iova(const void *elt)
1674 {
1675  const struct rte_mempool_objhdr *hdr;
1676  hdr = (const struct rte_mempool_objhdr *)RTE_PTR_SUB(elt,
1677  sizeof(*hdr));
1678  return hdr->iova;
1679 }
1680 
1691 void rte_mempool_audit(struct rte_mempool *mp);
1692 
1701 static inline void *rte_mempool_get_priv(struct rte_mempool *mp)
1702 {
1703  return (char *)mp +
1705 }
1706 
1713 void rte_mempool_list_dump(FILE *f);
1714 
1727 struct rte_mempool *rte_mempool_lookup(const char *name);
1728 
1746 uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
1747  struct rte_mempool_objsz *sz);
1748 
1757 void rte_mempool_walk(void (*func)(struct rte_mempool *, void *arg),
1758  void *arg);
1759 
1764 int
1765 rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz);
1766 
1767 #ifdef __cplusplus
1768 }
1769 #endif
1770 
1771 #endif /* _RTE_MEMPOOL_H_ */
uint64_t pool_id
Definition: rte_mempool.h:217
#define __rte_always_inline
Definition: rte_common.h:226
struct rte_mempool * rte_mempool_lookup(const char *name)
struct rte_mempool_cache * rte_mempool_cache_create(uint32_t size, int socket_id)
uint32_t rte_mempool_mem_iter(struct rte_mempool *mp, rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg)
uint32_t header_size
Definition: rte_mempool.h:228
STAILQ_ENTRY(rte_mempool_objhdr) next
const struct rte_memzone * mz
Definition: rte_mempool.h:220
uint64_t rte_iova_t
Definition: rte_common.h:418
static __rte_always_inline void rte_mempool_put_bulk(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
Definition: rte_mempool.h:1367
struct rte_mempool * mp
Definition: rte_mempool.h:185
void rte_mempool_list_dump(FILE *f)
int(* rte_mempool_dequeue_contig_blocks_t)(struct rte_mempool *mp, void **first_obj_table, unsigned int n)
Definition: rte_mempool.h:420
rte_mempool_alloc_t alloc
Definition: rte_mempool.h:616
void( rte_mempool_memchunk_free_cb_t)(struct rte_mempool_memhdr *memhdr, void *opaque)
Definition: rte_mempool.h:174
unsigned int flags
Definition: rte_mempool.h:221
void( rte_mempool_obj_cb_t)(struct rte_mempool *mp, void *opaque, void *obj, unsigned obj_idx)
Definition: rte_mempool.h:901
static __rte_always_inline void rte_mempool_generic_put(struct rte_mempool *mp, void *const *obj_table, unsigned int n, struct rte_mempool_cache *cache)
Definition: rte_mempool.h:1344
int(* rte_mempool_alloc_t)(struct rte_mempool *mp)
Definition: rte_mempool.h:398
static int rte_mempool_empty(const struct rte_mempool *mp)
Definition: rte_mempool.h:1657
rte_mempool_memchunk_free_cb_t * free_cb
Definition: rte_mempool.h:189
char name[RTE_MEMPOOL_OPS_NAMESIZE]
Definition: rte_mempool.h:615
int(* rte_mempool_dequeue_t)(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:414
#define MEMPOOL_HEADER_SIZE(mp, cs)
Definition: rte_mempool.h:300
rte_mempool_get_info_t get_info
Definition: rte_mempool.h:634
rte_mempool_dequeue_contig_blocks_t dequeue_contig_blocks
Definition: rte_mempool.h:638
uint32_t cache_size
Definition: rte_mempool.h:224
unsigned int rte_mempool_avail_count(const struct rte_mempool *mp)
uint32_t size
Definition: rte_mempool.h:223
struct rte_mempool_objhdr_list elt_list
Definition: rte_mempool.h:244
#define RTE_PTR_ADD(ptr, x)
Definition: rte_common.h:248
void rte_mempool_cache_free(struct rte_mempool_cache *cache)
uint32_t total_size
Definition: rte_mempool.h:103
uint32_t rte_mempool_obj_iter(struct rte_mempool *mp, rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg)
int rte_mempool_ops_get_info(const struct rte_mempool *mp, struct rte_mempool_info *info)
int rte_mempool_register_ops(const struct rte_mempool_ops *ops)
#define unlikely(x)
static __rte_always_inline int rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table, unsigned int n, struct rte_mempool_cache *cache)
Definition: rte_mempool.h:1487
int rte_mempool_populate_default(struct rte_mempool *mp)
static __rte_always_inline void rte_mempool_cache_flush(struct rte_mempool_cache *cache, struct rte_mempool *mp)
Definition: rte_mempool.h:1260
#define RTE_MEMPOOL_OPS_NAMESIZE
Definition: rte_mempool.h:386
static __rte_always_inline int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
Definition: rte_mempool.h:1550
STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr)
uint32_t nb_mem_chunks
Definition: rte_mempool.h:245
struct rte_mempool * mp
Definition: rte_mempool.h:140
struct rte_mempool_ops ops[RTE_MEMPOOL_MAX_OPS_IDX]
Definition: rte_mempool.h:658
rte_mempool_get_count get_count
Definition: rte_mempool.h:620
ssize_t(* rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp, uint32_t obj_num, uint32_t pg_shift, size_t *min_chunk_size, size_t *align)
Definition: rte_mempool.h:451
static int rte_mempool_full(const struct rte_mempool *mp)
Definition: rte_mempool.h:1638
static unsigned rte_lcore_id(void)
Definition: rte_lcore.h:76
void * pool_config
Definition: rte_mempool.h:219
unsigned int rte_mempool_in_use_count(const struct rte_mempool *mp)
void( rte_mempool_mem_cb_t)(struct rte_mempool *mp, void *opaque, struct rte_mempool_memhdr *memhdr, unsigned mem_idx)
Definition: rte_mempool.h:910
static __rte_always_inline int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:1521
#define rte_panic(...)
Definition: rte_debug.h:43
int rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name, void *pool_config)
int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr, rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
rte_mempool_calc_mem_size_t calc_mem_size
Definition: rte_mempool.h:625
unsigned(* rte_mempool_get_count)(const struct rte_mempool *mp)
Definition: rte_mempool.h:426
uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, struct rte_mempool_objsz *sz)
ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp, uint32_t obj_num, uint32_t pg_shift, size_t *min_chunk_size, size_t *align)
void * objs[RTE_MEMPOOL_CACHE_MAX_SIZE *3]
Definition: rte_mempool.h:93
void * pool_data
Definition: rte_mempool.h:216
unsigned int contig_block_size
Definition: rte_mempool.h:201
int rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
int(* rte_mempool_populate_t)(struct rte_mempool *mp, unsigned int max_objs, void *vaddr, rte_iova_t iova, size_t len, rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
Definition: rte_mempool.h:548
uint32_t elt_size
Definition: rte_mempool.h:227
void rte_mempool_audit(struct rte_mempool *mp)
int(* rte_mempool_get_info_t)(const struct rte_mempool *mp, struct rte_mempool_info *info)
Definition: rte_mempool.h:609
struct rte_mempool * rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, int socket_id, unsigned flags)
unsigned private_data_size
Definition: rte_mempool.h:231
void( rte_mempool_ctor_t)(struct rte_mempool *, void *)
Definition: rte_mempool.h:920
rte_spinlock_t sl
Definition: rte_mempool.h:653
static __rte_always_inline struct rte_mempool_cache * rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
Definition: rte_mempool.h:1238
void rte_mempool_dump(FILE *f, struct rte_mempool *mp)
int rte_mempool_populate_anon(struct rte_mempool *mp)
struct rte_mempool_cache * local_cache
Definition: rte_mempool.h:241
#define __rte_cache_aligned
Definition: rte_common.h:400
uint32_t trailer_size
Definition: rte_mempool.h:229
STAILQ_ENTRY(rte_mempool_memhdr) next
#define RTE_STD_C11
Definition: rte_common.h:40
rte_mempool_dequeue_t dequeue
Definition: rte_mempool.h:619
int(* rte_mempool_enqueue_t)(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
Definition: rte_mempool.h:408
uint32_t flushthresh
Definition: rte_mempool.h:87
struct rte_mempool_memhdr_list mem_list
Definition: rte_mempool.h:246
int rte_mempool_op_populate_default(struct rte_mempool *mp, unsigned int max_objs, void *vaddr, rte_iova_t iova, size_t len, rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
struct rte_mempool * rte_mempool_create(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags)
#define RTE_PTR_SUB(ptr, x)
Definition: rte_common.h:253
static struct rte_mempool * rte_mempool_from_obj(void *obj)
Definition: rte_mempool.h:320
uint32_t header_size
Definition: rte_mempool.h:101
int32_t ops_index
Definition: rte_mempool.h:239
static void * rte_memcpy(void *dst, const void *src, size_t n)
uint32_t populated_size
Definition: rte_mempool.h:243
static rte_iova_t rte_mempool_virt2iova(const void *elt)
Definition: rte_mempool.h:1673
static __rte_always_inline void rte_mempool_put(struct rte_mempool *mp, void *obj)
Definition: rte_mempool.h:1389
rte_mempool_populate_t populate
Definition: rte_mempool.h:630
void( rte_mempool_populate_obj_cb_t)(struct rte_mempool *mp, void *opaque, void *vaddr, rte_iova_t iova)
Definition: rte_mempool.h:517
void rte_mempool_free(struct rte_mempool *mp)
void rte_mempool_walk(void(*func)(struct rte_mempool *, void *arg), void *arg)
static void * rte_mempool_get_priv(struct rte_mempool *mp)
Definition: rte_mempool.h:1701
char name[RTE_MEMZONE_NAMESIZE]
Definition: rte_mempool.h:213
uint32_t trailer_size
Definition: rte_mempool.h:102
#define RTE_MEMPOOL_MAX_OPS_IDX
Definition: rte_mempool.h:641
void(* rte_mempool_free_t)(struct rte_mempool *mp)
Definition: rte_mempool.h:403
rte_mempool_free_t free
Definition: rte_mempool.h:617
static __rte_always_inline int rte_mempool_get_contig_blocks(struct rte_mempool *mp, void **first_obj_table, unsigned int n)
Definition: rte_mempool.h:1577
rte_mempool_enqueue_t enqueue
Definition: rte_mempool.h:618
#define RTE_MEMZONE_NAMESIZE
Definition: rte_memzone.h:51