DPDK  24.03.0-rc1
rte_mempool.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2014 Intel Corporation.
3  * Copyright(c) 2016 6WIND S.A.
4  * Copyright(c) 2022 SmartShare Systems
5  */
6 
7 #ifndef _RTE_MEMPOOL_H_
8 #define _RTE_MEMPOOL_H_
9 
37 #include <stdio.h>
38 #include <stdint.h>
39 #include <inttypes.h>
40 
41 #include <rte_compat.h>
42 #include <rte_config.h>
43 #include <rte_spinlock.h>
44 #include <rte_debug.h>
45 #include <rte_lcore.h>
46 #include <rte_log.h>
47 #include <rte_branch_prediction.h>
48 #include <rte_ring.h>
49 #include <rte_memcpy.h>
50 #include <rte_common.h>
51 
52 #include "rte_mempool_trace_fp.h"
53 
54 #ifdef __cplusplus
55 extern "C" {
56 #endif
57 
58 #define RTE_MEMPOOL_HEADER_COOKIE1 0xbadbadbadadd2e55ULL
59 #define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL
60 #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL
62 #ifdef RTE_LIBRTE_MEMPOOL_STATS
63 
69 struct rte_mempool_debug_stats {
70  uint64_t put_bulk;
71  uint64_t put_objs;
72  uint64_t put_common_pool_bulk;
73  uint64_t put_common_pool_objs;
74  uint64_t get_common_pool_bulk;
75  uint64_t get_common_pool_objs;
76  uint64_t get_success_bulk;
77  uint64_t get_success_objs;
78  uint64_t get_fail_bulk;
79  uint64_t get_fail_objs;
80  uint64_t get_success_blks;
81  uint64_t get_fail_blks;
84 #endif
85 
90  uint32_t size;
91  uint32_t flushthresh;
92  uint32_t len;
93 #ifdef RTE_LIBRTE_MEMPOOL_STATS
94  uint32_t unused;
95  /*
96  * Alternative location for the most frequently updated mempool statistics (per-lcore),
97  * providing faster update access when using a mempool cache.
98  */
99  struct {
100  uint64_t put_bulk;
101  uint64_t put_objs;
102  uint64_t get_success_bulk;
103  uint64_t get_success_objs;
104  } stats;
105 #endif
106 
112  void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 2] __rte_cache_aligned;
114 
119  uint32_t elt_size;
120  uint32_t header_size;
121  uint32_t trailer_size;
122  uint32_t total_size;
124 };
125 
127 #define RTE_MEMPOOL_NAMESIZE (RTE_RING_NAMESIZE - \
128  sizeof(RTE_MEMPOOL_MZ_PREFIX) + 1)
129 #define RTE_MEMPOOL_MZ_PREFIX "MP_"
130 
131 /* "MP_<name>" */
132 #define RTE_MEMPOOL_MZ_FORMAT RTE_MEMPOOL_MZ_PREFIX "%s"
133 
134 #ifndef RTE_MEMPOOL_ALIGN
135 
138 #define RTE_MEMPOOL_ALIGN RTE_CACHE_LINE_SIZE
139 #endif
140 
141 #define RTE_MEMPOOL_ALIGN_MASK (RTE_MEMPOOL_ALIGN - 1)
142 
153  RTE_STAILQ_ENTRY(rte_mempool_objhdr) next;
154  struct rte_mempool *mp;
156 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
157  uint64_t cookie;
158 #endif
159 };
160 
164 RTE_STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr);
165 
166 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
167 
174 struct rte_mempool_objtlr {
175  uint64_t cookie;
176 };
177 
178 #endif
179 
183 extern int rte_mempool_logtype;
184 #define RTE_LOGTYPE_MEMPOOL rte_mempool_logtype
185 #define RTE_MEMPOOL_LOG(level, ...) \
186  RTE_LOG_LINE(level, MEMPOOL, "" __VA_ARGS__)
187 
191 RTE_STAILQ_HEAD(rte_mempool_memhdr_list, rte_mempool_memhdr);
192 
197  void *opaque);
198 
206  RTE_STAILQ_ENTRY(rte_mempool_memhdr) next;
207  struct rte_mempool *mp;
208  void *addr;
210  size_t len;
212  void *opaque;
213 };
214 
223  unsigned int contig_block_size;
225 
229 struct rte_mempool {
230  char name[RTE_MEMPOOL_NAMESIZE];
231  union {
232  void *pool_data;
233  uint64_t pool_id;
234  };
235  void *pool_config;
236  const struct rte_memzone *mz;
237  unsigned int flags;
238  int socket_id;
239  uint32_t size;
240  uint32_t cache_size;
243  uint32_t elt_size;
244  uint32_t header_size;
245  uint32_t trailer_size;
247  unsigned private_data_size;
255  int32_t ops_index;
256 
259  uint32_t populated_size;
260  struct rte_mempool_objhdr_list elt_list;
261  uint32_t nb_mem_chunks;
262  struct rte_mempool_memhdr_list mem_list;
264 #ifdef RTE_LIBRTE_MEMPOOL_STATS
265 
269  struct rte_mempool_debug_stats stats[RTE_MAX_LCORE + 1];
270 #endif
272 
274 #define RTE_MEMPOOL_F_NO_SPREAD 0x0001
275 
279 #define MEMPOOL_F_NO_SPREAD RTE_MEMPOOL_F_NO_SPREAD
280 
281 #define RTE_MEMPOOL_F_NO_CACHE_ALIGN 0x0002
282 
286 #define MEMPOOL_F_NO_CACHE_ALIGN RTE_MEMPOOL_F_NO_CACHE_ALIGN
287 
288 #define RTE_MEMPOOL_F_SP_PUT 0x0004
289 
293 #define MEMPOOL_F_SP_PUT RTE_MEMPOOL_F_SP_PUT
294 
295 #define RTE_MEMPOOL_F_SC_GET 0x0008
296 
300 #define MEMPOOL_F_SC_GET RTE_MEMPOOL_F_SC_GET
301 
302 #define RTE_MEMPOOL_F_POOL_CREATED 0x0010
303 
304 #define RTE_MEMPOOL_F_NO_IOVA_CONTIG 0x0020
305 
309 #define MEMPOOL_F_NO_IOVA_CONTIG RTE_MEMPOOL_F_NO_IOVA_CONTIG
310 
311 #define RTE_MEMPOOL_F_NON_IO 0x0040
312 
316 #define RTE_MEMPOOL_VALID_USER_FLAGS (RTE_MEMPOOL_F_NO_SPREAD \
317  | RTE_MEMPOOL_F_NO_CACHE_ALIGN \
318  | RTE_MEMPOOL_F_SP_PUT \
319  | RTE_MEMPOOL_F_SC_GET \
320  | RTE_MEMPOOL_F_NO_IOVA_CONTIG \
321  )
322 
333 #ifdef RTE_LIBRTE_MEMPOOL_STATS
334 #define RTE_MEMPOOL_STAT_ADD(mp, name, n) do { \
335  unsigned int __lcore_id = rte_lcore_id(); \
336  if (likely(__lcore_id < RTE_MAX_LCORE)) \
337  (mp)->stats[__lcore_id].name += (n); \
338  else \
339  rte_atomic_fetch_add_explicit(&((mp)->stats[RTE_MAX_LCORE].name), \
340  (n), rte_memory_order_relaxed); \
341  } while (0)
342 #else
343 #define RTE_MEMPOOL_STAT_ADD(mp, name, n) do {} while (0)
344 #endif
345 
356 #ifdef RTE_LIBRTE_MEMPOOL_STATS
357 #define RTE_MEMPOOL_CACHE_STAT_ADD(cache, name, n) ((cache)->stats.name += (n))
358 #else
359 #define RTE_MEMPOOL_CACHE_STAT_ADD(cache, name, n) do {} while (0)
360 #endif
361 
370 #define RTE_MEMPOOL_HEADER_SIZE(mp, cs) \
371  (sizeof(*(mp)) + (((cs) == 0) ? 0 : \
372  (sizeof(struct rte_mempool_cache) * RTE_MAX_LCORE)))
373 
374 /* return the header of a mempool object (internal) */
375 static inline struct rte_mempool_objhdr *
376 rte_mempool_get_header(void *obj)
377 {
378  return (struct rte_mempool_objhdr *)RTE_PTR_SUB(obj,
379  sizeof(struct rte_mempool_objhdr));
380 }
381 
391 static inline struct rte_mempool *rte_mempool_from_obj(void *obj)
392 {
393  struct rte_mempool_objhdr *hdr = rte_mempool_get_header(obj);
394  return hdr->mp;
395 }
396 
397 /* return the trailer of a mempool object (internal) */
398 static inline struct rte_mempool_objtlr *rte_mempool_get_trailer(void *obj)
399 {
400  struct rte_mempool *mp = rte_mempool_from_obj(obj);
401  return (struct rte_mempool_objtlr *)RTE_PTR_ADD(obj, mp->elt_size);
402 }
403 
418 void rte_mempool_check_cookies(const struct rte_mempool *mp,
419  void * const *obj_table_const, unsigned n, int free);
420 
421 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
422 #define RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table_const, n, free) \
423  rte_mempool_check_cookies(mp, obj_table_const, n, free)
424 #else
425 #define RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table_const, n, free) do {} while (0)
426 #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
427 
443 void rte_mempool_contig_blocks_check_cookies(const struct rte_mempool *mp,
444  void * const *first_obj_table_const, unsigned int n, int free);
445 
446 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
447 #define RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table_const, n, \
448  free) \
449  rte_mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
450  free)
451 #else
452 #define RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table_const, n, \
453  free) \
454  do {} while (0)
455 #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
456 
457 #define RTE_MEMPOOL_OPS_NAMESIZE 32
469 typedef int (*rte_mempool_alloc_t)(struct rte_mempool *mp);
470 
474 typedef void (*rte_mempool_free_t)(struct rte_mempool *mp);
475 
482 typedef int (*rte_mempool_enqueue_t)(struct rte_mempool *mp,
483  void * const *obj_table, unsigned int n);
484 
491 typedef int (*rte_mempool_dequeue_t)(struct rte_mempool *mp,
492  void **obj_table, unsigned int n);
493 
498  void **first_obj_table, unsigned int n);
499 
503 typedef unsigned (*rte_mempool_get_count)(const struct rte_mempool *mp);
504 
528 typedef ssize_t (*rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp,
529  uint32_t obj_num, uint32_t pg_shift,
530  size_t *min_chunk_size, size_t *align);
531 
567 ssize_t rte_mempool_op_calc_mem_size_helper(const struct rte_mempool *mp,
568  uint32_t obj_num, uint32_t pg_shift, size_t chunk_reserve,
569  size_t *min_chunk_size, size_t *align);
570 
578 ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp,
579  uint32_t obj_num, uint32_t pg_shift,
580  size_t *min_chunk_size, size_t *align);
581 
594 typedef void (rte_mempool_populate_obj_cb_t)(struct rte_mempool *mp,
595  void *opaque, void *vaddr, rte_iova_t iova);
596 
625 typedef int (*rte_mempool_populate_t)(struct rte_mempool *mp,
626  unsigned int max_objs,
627  void *vaddr, rte_iova_t iova, size_t len,
628  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
629 
633 #define RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ 0x0001
634 
667 int rte_mempool_op_populate_helper(struct rte_mempool *mp,
668  unsigned int flags, unsigned int max_objs,
669  void *vaddr, rte_iova_t iova, size_t len,
670  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
671 
679  unsigned int max_objs,
680  void *vaddr, rte_iova_t iova, size_t len,
681  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
682 
686 typedef int (*rte_mempool_get_info_t)(const struct rte_mempool *mp,
687  struct rte_mempool_info *info);
688 
689 
717 
718 #define RTE_MEMPOOL_MAX_OPS_IDX 16
729 struct rte_mempool_ops_table {
731  uint32_t num_ops;
737 
740 
750 static inline struct rte_mempool_ops *
751 rte_mempool_get_ops(int ops_index)
752 {
753  RTE_VERIFY((ops_index >= 0) && (ops_index < RTE_MEMPOOL_MAX_OPS_IDX));
754 
755  return &rte_mempool_ops_table.ops[ops_index];
756 }
757 
767 int
768 rte_mempool_ops_alloc(struct rte_mempool *mp);
769 
783 static inline int
784 rte_mempool_ops_dequeue_bulk(struct rte_mempool *mp,
785  void **obj_table, unsigned n)
786 {
787  struct rte_mempool_ops *ops;
788  int ret;
789 
790  rte_mempool_trace_ops_dequeue_bulk(mp, obj_table, n);
791  ops = rte_mempool_get_ops(mp->ops_index);
792  ret = ops->dequeue(mp, obj_table, n);
793  if (ret == 0) {
794  RTE_MEMPOOL_STAT_ADD(mp, get_common_pool_bulk, 1);
795  RTE_MEMPOOL_STAT_ADD(mp, get_common_pool_objs, n);
796  }
797  return ret;
798 }
799 
813 static inline int
814 rte_mempool_ops_dequeue_contig_blocks(struct rte_mempool *mp,
815  void **first_obj_table, unsigned int n)
816 {
817  struct rte_mempool_ops *ops;
818 
819  ops = rte_mempool_get_ops(mp->ops_index);
820  RTE_ASSERT(ops->dequeue_contig_blocks != NULL);
821  rte_mempool_trace_ops_dequeue_contig_blocks(mp, first_obj_table, n);
822  return ops->dequeue_contig_blocks(mp, first_obj_table, n);
823 }
824 
838 static inline int
839 rte_mempool_ops_enqueue_bulk(struct rte_mempool *mp, void * const *obj_table,
840  unsigned n)
841 {
842  struct rte_mempool_ops *ops;
843  int ret;
844 
845  RTE_MEMPOOL_STAT_ADD(mp, put_common_pool_bulk, 1);
846  RTE_MEMPOOL_STAT_ADD(mp, put_common_pool_objs, n);
847  rte_mempool_trace_ops_enqueue_bulk(mp, obj_table, n);
848  ops = rte_mempool_get_ops(mp->ops_index);
849  ret = ops->enqueue(mp, obj_table, n);
850 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
851  if (unlikely(ret < 0))
852  RTE_MEMPOOL_LOG(CRIT, "cannot enqueue %u objects to mempool %s",
853  n, mp->name);
854 #endif
855  return ret;
856 }
857 
866 unsigned
867 rte_mempool_ops_get_count(const struct rte_mempool *mp);
868 
888 ssize_t rte_mempool_ops_calc_mem_size(const struct rte_mempool *mp,
889  uint32_t obj_num, uint32_t pg_shift,
890  size_t *min_chunk_size, size_t *align);
891 
915 int rte_mempool_ops_populate(struct rte_mempool *mp, unsigned int max_objs,
916  void *vaddr, rte_iova_t iova, size_t len,
918  void *obj_cb_arg);
919 
932 int rte_mempool_ops_get_info(const struct rte_mempool *mp,
933  struct rte_mempool_info *info);
934 
941 void
942 rte_mempool_ops_free(struct rte_mempool *mp);
943 
961 int
962 rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name,
963  void *pool_config);
964 
975 int rte_mempool_register_ops(const struct rte_mempool_ops *ops);
976 
982 #define RTE_MEMPOOL_REGISTER_OPS(ops) \
983  RTE_INIT(mp_hdlr_init_##ops) \
984  { \
985  rte_mempool_register_ops(&ops); \
986  }
987 
993 typedef void (rte_mempool_obj_cb_t)(struct rte_mempool *mp,
994  void *opaque, void *obj, unsigned obj_idx);
995 typedef rte_mempool_obj_cb_t rte_mempool_obj_ctor_t; /* compat */
996 
1002 typedef void (rte_mempool_mem_cb_t)(struct rte_mempool *mp,
1003  void *opaque, struct rte_mempool_memhdr *memhdr,
1004  unsigned mem_idx);
1005 
1012 typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *);
1013 
1092 struct rte_mempool *
1093 rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
1094  unsigned cache_size, unsigned private_data_size,
1095  rte_mempool_ctor_t *mp_init, void *mp_init_arg,
1096  rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
1097  int socket_id, unsigned flags);
1098 
1133 struct rte_mempool *
1134 rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size,
1135  unsigned cache_size, unsigned private_data_size,
1136  int socket_id, unsigned flags);
1148 void
1149 rte_mempool_free(struct rte_mempool *mp);
1150 
1181 int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
1182  rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
1183  void *opaque);
1184 
1211 int
1212 rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
1213  size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb,
1214  void *opaque);
1215 
1230 
1244 int rte_mempool_populate_anon(struct rte_mempool *mp);
1245 
1261 uint32_t rte_mempool_obj_iter(struct rte_mempool *mp,
1262  rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg);
1263 
1279 uint32_t rte_mempool_mem_iter(struct rte_mempool *mp,
1280  rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg);
1281 
1290 void rte_mempool_dump(FILE *f, struct rte_mempool *mp);
1291 
1306 struct rte_mempool_cache *
1307 rte_mempool_cache_create(uint32_t size, int socket_id);
1308 
1315 void
1317 
1329 static __rte_always_inline struct rte_mempool_cache *
1330 rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
1331 {
1332  if (mp->cache_size == 0)
1333  return NULL;
1334 
1335  if (lcore_id >= RTE_MAX_LCORE)
1336  return NULL;
1337 
1338  rte_mempool_trace_default_cache(mp, lcore_id,
1339  &mp->local_cache[lcore_id]);
1340  return &mp->local_cache[lcore_id];
1341 }
1342 
1351 static __rte_always_inline void
1353  struct rte_mempool *mp)
1354 {
1355  if (cache == NULL)
1356  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1357  if (cache == NULL || cache->len == 0)
1358  return;
1359  rte_mempool_trace_cache_flush(cache, mp);
1360  rte_mempool_ops_enqueue_bulk(mp, cache->objs, cache->len);
1361  cache->len = 0;
1362 }
1363 
1376 static __rte_always_inline void
1377 rte_mempool_do_generic_put(struct rte_mempool *mp, void * const *obj_table,
1378  unsigned int n, struct rte_mempool_cache *cache)
1379 {
1380  void **cache_objs;
1381 
1382  /* No cache provided */
1383  if (unlikely(cache == NULL))
1384  goto driver_enqueue;
1385 
1386  /* increment stat now, adding in mempool always success */
1387  RTE_MEMPOOL_CACHE_STAT_ADD(cache, put_bulk, 1);
1388  RTE_MEMPOOL_CACHE_STAT_ADD(cache, put_objs, n);
1389 
1390  /* The request itself is too big for the cache */
1391  if (unlikely(n > cache->flushthresh))
1392  goto driver_enqueue_stats_incremented;
1393 
1394  /*
1395  * The cache follows the following algorithm:
1396  * 1. If the objects cannot be added to the cache without crossing
1397  * the flush threshold, flush the cache to the backend.
1398  * 2. Add the objects to the cache.
1399  */
1400 
1401  if (cache->len + n <= cache->flushthresh) {
1402  cache_objs = &cache->objs[cache->len];
1403  cache->len += n;
1404  } else {
1405  cache_objs = &cache->objs[0];
1406  rte_mempool_ops_enqueue_bulk(mp, cache_objs, cache->len);
1407  cache->len = n;
1408  }
1409 
1410  /* Add the objects to the cache. */
1411  rte_memcpy(cache_objs, obj_table, sizeof(void *) * n);
1412 
1413  return;
1414 
1415 driver_enqueue:
1416 
1417  /* increment stat now, adding in mempool always success */
1418  RTE_MEMPOOL_STAT_ADD(mp, put_bulk, 1);
1419  RTE_MEMPOOL_STAT_ADD(mp, put_objs, n);
1420 
1421 driver_enqueue_stats_incremented:
1422 
1423  /* push objects to the backend */
1424  rte_mempool_ops_enqueue_bulk(mp, obj_table, n);
1425 }
1426 
1427 
1440 static __rte_always_inline void
1441 rte_mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1442  unsigned int n, struct rte_mempool_cache *cache)
1443 {
1444  rte_mempool_trace_generic_put(mp, obj_table, n, cache);
1445  RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table, n, 0);
1446  rte_mempool_do_generic_put(mp, obj_table, n, cache);
1447 }
1448 
1463 static __rte_always_inline void
1464 rte_mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,
1465  unsigned int n)
1466 {
1467  struct rte_mempool_cache *cache;
1468  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1469  rte_mempool_trace_put_bulk(mp, obj_table, n, cache);
1470  rte_mempool_generic_put(mp, obj_table, n, cache);
1471 }
1472 
1485 static __rte_always_inline void
1486 rte_mempool_put(struct rte_mempool *mp, void *obj)
1487 {
1488  rte_mempool_put_bulk(mp, &obj, 1);
1489 }
1490 
1505 static __rte_always_inline int
1506 rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table,
1507  unsigned int n, struct rte_mempool_cache *cache)
1508 {
1509  int ret;
1510  unsigned int remaining;
1511  uint32_t index, len;
1512  void **cache_objs;
1513 
1514  /* No cache provided */
1515  if (unlikely(cache == NULL)) {
1516  remaining = n;
1517  goto driver_dequeue;
1518  }
1519 
1520  /* The cache is a stack, so copy will be in reverse order. */
1521  cache_objs = &cache->objs[cache->len];
1522 
1523  if (__extension__(__builtin_constant_p(n)) && n <= cache->len) {
1524  /*
1525  * The request size is known at build time, and
1526  * the entire request can be satisfied from the cache,
1527  * so let the compiler unroll the fixed length copy loop.
1528  */
1529  cache->len -= n;
1530  for (index = 0; index < n; index++)
1531  *obj_table++ = *--cache_objs;
1532 
1533  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1534  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1535 
1536  return 0;
1537  }
1538 
1539  /*
1540  * Use the cache as much as we have to return hot objects first.
1541  * If the request size 'n' is known at build time, the above comparison
1542  * ensures that n > cache->len here, so omit RTE_MIN().
1543  */
1544  len = __extension__(__builtin_constant_p(n)) ? cache->len :
1545  RTE_MIN(n, cache->len);
1546  cache->len -= len;
1547  remaining = n - len;
1548  for (index = 0; index < len; index++)
1549  *obj_table++ = *--cache_objs;
1550 
1551  /*
1552  * If the request size 'n' is known at build time, the case
1553  * where the entire request can be satisfied from the cache
1554  * has already been handled above, so omit handling it here.
1555  */
1556  if (!__extension__(__builtin_constant_p(n)) && remaining == 0) {
1557  /* The entire request is satisfied from the cache. */
1558 
1559  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1560  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1561 
1562  return 0;
1563  }
1564 
1565  /* if dequeue below would overflow mem allocated for cache */
1566  if (unlikely(remaining > RTE_MEMPOOL_CACHE_MAX_SIZE))
1567  goto driver_dequeue;
1568 
1569  /* Fill the cache from the backend; fetch size + remaining objects. */
1570  ret = rte_mempool_ops_dequeue_bulk(mp, cache->objs,
1571  cache->size + remaining);
1572  if (unlikely(ret < 0)) {
1573  /*
1574  * We are buffer constrained, and not able to allocate
1575  * cache + remaining.
1576  * Do not fill the cache, just satisfy the remaining part of
1577  * the request directly from the backend.
1578  */
1579  goto driver_dequeue;
1580  }
1581 
1582  /* Satisfy the remaining part of the request from the filled cache. */
1583  cache_objs = &cache->objs[cache->size + remaining];
1584  for (index = 0; index < remaining; index++)
1585  *obj_table++ = *--cache_objs;
1586 
1587  cache->len = cache->size;
1588 
1589  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1590  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1591 
1592  return 0;
1593 
1594 driver_dequeue:
1595 
1596  /* Get remaining objects directly from the backend. */
1597  ret = rte_mempool_ops_dequeue_bulk(mp, obj_table, remaining);
1598 
1599  if (ret < 0) {
1600  if (likely(cache != NULL)) {
1601  cache->len = n - remaining;
1602  /*
1603  * No further action is required to roll the first part
1604  * of the request back into the cache, as objects in
1605  * the cache are intact.
1606  */
1607  }
1608 
1609  RTE_MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
1610  RTE_MEMPOOL_STAT_ADD(mp, get_fail_objs, n);
1611  } else {
1612  if (likely(cache != NULL)) {
1613  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1614  RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1615  } else {
1616  RTE_MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
1617  RTE_MEMPOOL_STAT_ADD(mp, get_success_objs, n);
1618  }
1619  }
1620 
1621  return ret;
1622 }
1623 
1644 static __rte_always_inline int
1645 rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1646  unsigned int n, struct rte_mempool_cache *cache)
1647 {
1648  int ret;
1649  ret = rte_mempool_do_generic_get(mp, obj_table, n, cache);
1650  if (ret == 0)
1651  RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table, n, 1);
1652  rte_mempool_trace_generic_get(mp, obj_table, n, cache);
1653  return ret;
1654 }
1655 
1678 static __rte_always_inline int
1679 rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
1680 {
1681  struct rte_mempool_cache *cache;
1682  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1683  rte_mempool_trace_get_bulk(mp, obj_table, n, cache);
1684  return rte_mempool_generic_get(mp, obj_table, n, cache);
1685 }
1686 
1707 static __rte_always_inline int
1708 rte_mempool_get(struct rte_mempool *mp, void **obj_p)
1709 {
1710  return rte_mempool_get_bulk(mp, obj_p, 1);
1711 }
1712 
1734 static __rte_always_inline int
1736  void **first_obj_table, unsigned int n)
1737 {
1738  int ret;
1739 
1740  ret = rte_mempool_ops_dequeue_contig_blocks(mp, first_obj_table, n);
1741  if (ret == 0) {
1742  RTE_MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
1743  RTE_MEMPOOL_STAT_ADD(mp, get_success_blks, n);
1744  RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table, n,
1745  1);
1746  } else {
1747  RTE_MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
1748  RTE_MEMPOOL_STAT_ADD(mp, get_fail_blks, n);
1749  }
1750 
1751  rte_mempool_trace_get_contig_blocks(mp, first_obj_table, n);
1752  return ret;
1753 }
1754 
1767 unsigned int rte_mempool_avail_count(const struct rte_mempool *mp);
1768 
1781 unsigned int
1782 rte_mempool_in_use_count(const struct rte_mempool *mp);
1783 
1797 static inline int
1799 {
1800  return rte_mempool_avail_count(mp) == mp->size;
1801 }
1802 
1816 static inline int
1818 {
1819  return rte_mempool_avail_count(mp) == 0;
1820 }
1821 
1832 static inline rte_iova_t
1833 rte_mempool_virt2iova(const void *elt)
1834 {
1835  const struct rte_mempool_objhdr *hdr;
1836  hdr = (const struct rte_mempool_objhdr *)RTE_PTR_SUB(elt,
1837  sizeof(*hdr));
1838  return hdr->iova;
1839 }
1840 
1851 void rte_mempool_audit(struct rte_mempool *mp);
1852 
1861 static inline void *rte_mempool_get_priv(struct rte_mempool *mp)
1862 {
1863  return (char *)mp +
1864  RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size);
1865 }
1866 
1873 void rte_mempool_list_dump(FILE *f);
1874 
1886 struct rte_mempool *rte_mempool_lookup(const char *name);
1887 
1905 uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
1906  struct rte_mempool_objsz *sz);
1907 
1916 void rte_mempool_walk(void (*func)(struct rte_mempool *, void *arg),
1917  void *arg);
1918 
1923 int
1924 rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz);
1925 
1935 };
1936 
1946 typedef void (rte_mempool_event_callback)(
1947  enum rte_mempool_event event,
1948  struct rte_mempool *mp,
1949  void *user_data);
1950 
1967 __rte_internal
1968 int
1969 rte_mempool_event_callback_register(rte_mempool_event_callback *func,
1970  void *user_data);
1971 
1985 __rte_internal
1986 int
1987 rte_mempool_event_callback_unregister(rte_mempool_event_callback *func,
1988  void *user_data);
1989 
1990 #ifdef __cplusplus
1991 }
1992 #endif
1993 
1994 #endif /* _RTE_MEMPOOL_H_ */
uint64_t pool_id
Definition: rte_mempool.h:233
#define __rte_always_inline
Definition: rte_common.h:343
struct rte_mempool * rte_mempool_lookup(const char *name)
struct rte_mempool_cache * rte_mempool_cache_create(uint32_t size, int socket_id)
#define RTE_CACHE_GUARD
Definition: rte_common.h:561
uint32_t rte_mempool_mem_iter(struct rte_mempool *mp, rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg)
uint32_t header_size
Definition: rte_mempool.h:244
#define likely(x)
const struct rte_memzone * mz
Definition: rte_mempool.h:236
uint64_t rte_iova_t
Definition: rte_common.h:576
static __rte_always_inline void rte_mempool_put_bulk(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
Definition: rte_mempool.h:1464
struct rte_mempool * mp
Definition: rte_mempool.h:207
void rte_mempool_list_dump(FILE *f)
void() rte_mempool_memchunk_free_cb_t(struct rte_mempool_memhdr *memhdr, void *opaque)
Definition: rte_mempool.h:196
int(* rte_mempool_dequeue_contig_blocks_t)(struct rte_mempool *mp, void **first_obj_table, unsigned int n)
Definition: rte_mempool.h:497
rte_mempool_alloc_t alloc
Definition: rte_mempool.h:693
void *objs [RTE_MEMPOOL_CACHE_MAX_SIZE *2] __rte_cache_aligned
Definition: rte_mempool.h:112
void() rte_mempool_populate_obj_cb_t(struct rte_mempool *mp, void *opaque, void *vaddr, rte_iova_t iova)
Definition: rte_mempool.h:594
unsigned int flags
Definition: rte_mempool.h:237
static __rte_always_inline void rte_mempool_generic_put(struct rte_mempool *mp, void *const *obj_table, unsigned int n, struct rte_mempool_cache *cache)
Definition: rte_mempool.h:1441
int(* rte_mempool_alloc_t)(struct rte_mempool *mp)
Definition: rte_mempool.h:469
static int rte_mempool_empty(const struct rte_mempool *mp)
Definition: rte_mempool.h:1817
RTE_STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr)
rte_mempool_memchunk_free_cb_t * free_cb
Definition: rte_mempool.h:211
char name[RTE_MEMPOOL_OPS_NAMESIZE]
Definition: rte_mempool.h:692
int(* rte_mempool_dequeue_t)(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:491
rte_mempool_get_info_t get_info
Definition: rte_mempool.h:711
rte_mempool_dequeue_contig_blocks_t dequeue_contig_blocks
Definition: rte_mempool.h:715
uint32_t cache_size
Definition: rte_mempool.h:240
char name[RTE_MEMPOOL_NAMESIZE]
Definition: rte_mempool.h:230
unsigned int rte_mempool_avail_count(const struct rte_mempool *mp)
uint32_t size
Definition: rte_mempool.h:239
#define RTE_PTR_ADD(ptr, x)
Definition: rte_common.h:383
void rte_mempool_cache_free(struct rte_mempool_cache *cache)
uint32_t total_size
Definition: rte_mempool.h:122
uint32_t rte_mempool_obj_iter(struct rte_mempool *mp, rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg)
int rte_mempool_ops_get_info(const struct rte_mempool *mp, struct rte_mempool_info *info)
int rte_mempool_register_ops(const struct rte_mempool_ops *ops)
#define unlikely(x)
static __rte_always_inline int rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table, unsigned int n, struct rte_mempool_cache *cache)
Definition: rte_mempool.h:1645
int rte_mempool_populate_default(struct rte_mempool *mp)
#define RTE_MIN(a, b)
Definition: rte_common.h:601
static __rte_always_inline void rte_mempool_cache_flush(struct rte_mempool_cache *cache, struct rte_mempool *mp)
Definition: rte_mempool.h:1352
#define RTE_MEMPOOL_OPS_NAMESIZE
Definition: rte_mempool.h:457
static __rte_always_inline int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
Definition: rte_mempool.h:1708
uint32_t nb_mem_chunks
Definition: rte_mempool.h:261
struct rte_mempool * mp
Definition: rte_mempool.h:154
struct rte_mempool_ops ops[RTE_MEMPOOL_MAX_OPS_IDX]
Definition: rte_mempool.h:735
rte_mempool_get_count get_count
Definition: rte_mempool.h:697
ssize_t(* rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp, uint32_t obj_num, uint32_t pg_shift, size_t *min_chunk_size, size_t *align)
Definition: rte_mempool.h:528
static int rte_mempool_full(const struct rte_mempool *mp)
Definition: rte_mempool.h:1798
static unsigned rte_lcore_id(void)
Definition: rte_lcore.h:78
void * pool_config
Definition: rte_mempool.h:235
unsigned int rte_mempool_in_use_count(const struct rte_mempool *mp)
static __rte_always_inline int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:1679
int rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name, void *pool_config)
int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr, rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
rte_mempool_calc_mem_size_t calc_mem_size
Definition: rte_mempool.h:702
unsigned(* rte_mempool_get_count)(const struct rte_mempool *mp)
Definition: rte_mempool.h:503
uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, struct rte_mempool_objsz *sz)
ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp, uint32_t obj_num, uint32_t pg_shift, size_t *min_chunk_size, size_t *align)
void * pool_data
Definition: rte_mempool.h:232
unsigned int contig_block_size
Definition: rte_mempool.h:223
int rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
int(* rte_mempool_populate_t)(struct rte_mempool *mp, unsigned int max_objs, void *vaddr, rte_iova_t iova, size_t len, rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
Definition: rte_mempool.h:625
uint32_t elt_size
Definition: rte_mempool.h:243
void rte_mempool_audit(struct rte_mempool *mp)
int(* rte_mempool_get_info_t)(const struct rte_mempool *mp, struct rte_mempool_info *info)
Definition: rte_mempool.h:686
struct rte_mempool * rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, int socket_id, unsigned flags)
unsigned private_data_size
Definition: rte_mempool.h:247
rte_spinlock_t sl
Definition: rte_mempool.h:730
static __rte_always_inline struct rte_mempool_cache * rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
Definition: rte_mempool.h:1330
void rte_mempool_dump(FILE *f, struct rte_mempool *mp)
void() rte_mempool_ctor_t(struct rte_mempool *, void *)
Definition: rte_mempool.h:1012
int rte_mempool_populate_anon(struct rte_mempool *mp)
struct rte_mempool_cache * local_cache
Definition: rte_mempool.h:257
#define __rte_cache_aligned
Definition: rte_common.h:544
uint32_t trailer_size
Definition: rte_mempool.h:245
rte_mempool_dequeue_t dequeue
Definition: rte_mempool.h:696
int(* rte_mempool_enqueue_t)(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
Definition: rte_mempool.h:482
uint32_t flushthresh
Definition: rte_mempool.h:91
void() rte_mempool_obj_cb_t(struct rte_mempool *mp, void *opaque, void *obj, unsigned obj_idx)
Definition: rte_mempool.h:993
void() rte_mempool_mem_cb_t(struct rte_mempool *mp, void *opaque, struct rte_mempool_memhdr *memhdr, unsigned mem_idx)
Definition: rte_mempool.h:1002
int rte_mempool_op_populate_default(struct rte_mempool *mp, unsigned int max_objs, void *vaddr, rte_iova_t iova, size_t len, rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
struct rte_mempool * rte_mempool_create(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags)
#define RTE_PTR_SUB(ptr, x)
Definition: rte_common.h:388
rte_mempool_event
Definition: rte_mempool.h:1930
static struct rte_mempool * rte_mempool_from_obj(void *obj)
Definition: rte_mempool.h:391
uint32_t header_size
Definition: rte_mempool.h:120
int32_t ops_index
Definition: rte_mempool.h:255
static void * rte_memcpy(void *dst, const void *src, size_t n)
uint32_t populated_size
Definition: rte_mempool.h:259
static rte_iova_t rte_mempool_virt2iova(const void *elt)
Definition: rte_mempool.h:1833
static __rte_always_inline void rte_mempool_put(struct rte_mempool *mp, void *obj)
Definition: rte_mempool.h:1486
rte_mempool_populate_t populate
Definition: rte_mempool.h:707
void rte_mempool_free(struct rte_mempool *mp)
void rte_mempool_walk(void(*func)(struct rte_mempool *, void *arg), void *arg)
static void * rte_mempool_get_priv(struct rte_mempool *mp)
Definition: rte_mempool.h:1861
uint32_t trailer_size
Definition: rte_mempool.h:121
#define RTE_MEMPOOL_MAX_OPS_IDX
Definition: rte_mempool.h:718
void(* rte_mempool_free_t)(struct rte_mempool *mp)
Definition: rte_mempool.h:474
rte_mempool_free_t free
Definition: rte_mempool.h:694
static __rte_always_inline int rte_mempool_get_contig_blocks(struct rte_mempool *mp, void **first_obj_table, unsigned int n)
Definition: rte_mempool.h:1735
rte_mempool_enqueue_t enqueue
Definition: rte_mempool.h:695