DPDK  18.02.2
rte_mempool.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2014 Intel Corporation.
3  * Copyright(c) 2016 6WIND S.A.
4  */
5 
6 #ifndef _RTE_MEMPOOL_H_
7 #define _RTE_MEMPOOL_H_
8 
36 #include <stdio.h>
37 #include <stdlib.h>
38 #include <stdint.h>
39 #include <errno.h>
40 #include <inttypes.h>
41 #include <sys/queue.h>
42 
43 #include <rte_config.h>
44 #include <rte_spinlock.h>
45 #include <rte_log.h>
46 #include <rte_debug.h>
47 #include <rte_lcore.h>
48 #include <rte_memory.h>
49 #include <rte_branch_prediction.h>
50 #include <rte_ring.h>
51 #include <rte_memcpy.h>
52 #include <rte_common.h>
53 
54 #ifdef __cplusplus
55 extern "C" {
56 #endif
57 
58 #define RTE_MEMPOOL_HEADER_COOKIE1 0xbadbadbadadd2e55ULL
59 #define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL
60 #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL
62 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
63 
66 struct rte_mempool_debug_stats {
67  uint64_t put_bulk;
68  uint64_t put_objs;
69  uint64_t get_success_bulk;
70  uint64_t get_success_objs;
71  uint64_t get_fail_bulk;
72  uint64_t get_fail_objs;
74 #endif
75 
80  uint32_t size;
81  uint32_t flushthresh;
82  uint32_t len;
83  /*
84  * Cache is allocated to this size to allow it to overflow in certain
85  * cases to avoid needless emptying of cache.
86  */
87  void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 3];
89 
94  uint32_t elt_size;
95  uint32_t header_size;
96  uint32_t trailer_size;
97  uint32_t total_size;
99 };
100 
102 #define RTE_MEMPOOL_NAMESIZE (RTE_RING_NAMESIZE - \
103  sizeof(RTE_MEMPOOL_MZ_PREFIX) + 1)
104 #define RTE_MEMPOOL_MZ_PREFIX "MP_"
105 
106 /* "MP_<name>" */
107 #define RTE_MEMPOOL_MZ_FORMAT RTE_MEMPOOL_MZ_PREFIX "%s"
108 
109 #define MEMPOOL_PG_SHIFT_MAX (sizeof(uintptr_t) * CHAR_BIT - 1)
110 
112 #define MEMPOOL_PG_NUM_DEFAULT 1
113 
114 #ifndef RTE_MEMPOOL_ALIGN
115 #define RTE_MEMPOOL_ALIGN RTE_CACHE_LINE_SIZE
116 #endif
117 
118 #define RTE_MEMPOOL_ALIGN_MASK (RTE_MEMPOOL_ALIGN - 1)
119 
131  struct rte_mempool *mp;
133  union {
136  };
137 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
138  uint64_t cookie;
139 #endif
140 };
141 
145 STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr);
146 
147 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
148 
155 struct rte_mempool_objtlr {
156  uint64_t cookie;
157 };
158 
159 #endif
160 
164 STAILQ_HEAD(rte_mempool_memhdr_list, rte_mempool_memhdr);
165 
170  void *opaque);
171 
180  struct rte_mempool *mp;
181  void *addr;
183  union {
186  };
187  size_t len;
189  void *opaque;
190 };
191 
195 struct rte_mempool {
196  /*
197  * Note: this field kept the RTE_MEMZONE_NAMESIZE size due to ABI
198  * compatibility requirements, it could be changed to
199  * RTE_MEMPOOL_NAMESIZE next time the ABI changes
200  */
203  union {
204  void *pool_data;
205  uint64_t pool_id;
206  };
207  void *pool_config;
208  const struct rte_memzone *mz;
209  unsigned int flags;
210  int socket_id;
211  uint32_t size;
212  uint32_t cache_size;
215  uint32_t elt_size;
216  uint32_t header_size;
217  uint32_t trailer_size;
219  unsigned private_data_size;
227  int32_t ops_index;
228 
231  uint32_t populated_size;
232  struct rte_mempool_objhdr_list elt_list;
233  uint32_t nb_mem_chunks;
234  struct rte_mempool_memhdr_list mem_list;
236 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
237 
238  struct rte_mempool_debug_stats stats[RTE_MAX_LCORE];
239 #endif
241 
242 #define MEMPOOL_F_NO_SPREAD 0x0001
243 #define MEMPOOL_F_NO_CACHE_ALIGN 0x0002
244 #define MEMPOOL_F_SP_PUT 0x0004
245 #define MEMPOOL_F_SC_GET 0x0008
246 #define MEMPOOL_F_POOL_CREATED 0x0010
247 #define MEMPOOL_F_NO_PHYS_CONTIG 0x0020
253 #define MEMPOOL_F_CAPA_PHYS_CONTIG 0x0040
254 
265 #define MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS 0x0080
266 
277 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
278 #define __MEMPOOL_STAT_ADD(mp, name, n) do { \
279  unsigned __lcore_id = rte_lcore_id(); \
280  if (__lcore_id < RTE_MAX_LCORE) { \
281  mp->stats[__lcore_id].name##_objs += n; \
282  mp->stats[__lcore_id].name##_bulk += 1; \
283  } \
284  } while(0)
285 #else
286 #define __MEMPOOL_STAT_ADD(mp, name, n) do {} while(0)
287 #endif
288 
297 #define MEMPOOL_HEADER_SIZE(mp, cs) \
298  (sizeof(*(mp)) + (((cs) == 0) ? 0 : \
299  (sizeof(struct rte_mempool_cache) * RTE_MAX_LCORE)))
300 
301 /* return the header of a mempool object (internal) */
302 static inline struct rte_mempool_objhdr *__mempool_get_header(void *obj)
303 {
304  return (struct rte_mempool_objhdr *)RTE_PTR_SUB(obj,
305  sizeof(struct rte_mempool_objhdr));
306 }
307 
317 static inline struct rte_mempool *rte_mempool_from_obj(void *obj)
318 {
319  struct rte_mempool_objhdr *hdr = __mempool_get_header(obj);
320  return hdr->mp;
321 }
322 
323 /* return the trailer of a mempool object (internal) */
324 static inline struct rte_mempool_objtlr *__mempool_get_trailer(void *obj)
325 {
326  struct rte_mempool *mp = rte_mempool_from_obj(obj);
327  return (struct rte_mempool_objtlr *)RTE_PTR_ADD(obj, mp->elt_size);
328 }
329 
344 void rte_mempool_check_cookies(const struct rte_mempool *mp,
345  void * const *obj_table_const, unsigned n, int free);
346 
347 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
348 #define __mempool_check_cookies(mp, obj_table_const, n, free) \
349  rte_mempool_check_cookies(mp, obj_table_const, n, free)
350 #else
351 #define __mempool_check_cookies(mp, obj_table_const, n, free) do {} while(0)
352 #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
353 
354 #define RTE_MEMPOOL_OPS_NAMESIZE 32
366 typedef int (*rte_mempool_alloc_t)(struct rte_mempool *mp);
367 
371 typedef void (*rte_mempool_free_t)(struct rte_mempool *mp);
372 
376 typedef int (*rte_mempool_enqueue_t)(struct rte_mempool *mp,
377  void * const *obj_table, unsigned int n);
378 
382 typedef int (*rte_mempool_dequeue_t)(struct rte_mempool *mp,
383  void **obj_table, unsigned int n);
384 
388 typedef unsigned (*rte_mempool_get_count)(const struct rte_mempool *mp);
389 
393 typedef int (*rte_mempool_get_capabilities_t)(const struct rte_mempool *mp,
394  unsigned int *flags);
395 
400 (const struct rte_mempool *mp, char *vaddr, rte_iova_t iova, size_t len);
401 
419 
420 #define RTE_MEMPOOL_MAX_OPS_IDX 16
431 struct rte_mempool_ops_table {
433  uint32_t num_ops;
439 
442 
452 static inline struct rte_mempool_ops *
453 rte_mempool_get_ops(int ops_index)
454 {
455  RTE_VERIFY((ops_index >= 0) && (ops_index < RTE_MEMPOOL_MAX_OPS_IDX));
456 
457  return &rte_mempool_ops_table.ops[ops_index];
458 }
459 
469 int
470 rte_mempool_ops_alloc(struct rte_mempool *mp);
471 
485 static inline int
486 rte_mempool_ops_dequeue_bulk(struct rte_mempool *mp,
487  void **obj_table, unsigned n)
488 {
489  struct rte_mempool_ops *ops;
490 
491  ops = rte_mempool_get_ops(mp->ops_index);
492  return ops->dequeue(mp, obj_table, n);
493 }
494 
508 static inline int
509 rte_mempool_ops_enqueue_bulk(struct rte_mempool *mp, void * const *obj_table,
510  unsigned n)
511 {
512  struct rte_mempool_ops *ops;
513 
514  ops = rte_mempool_get_ops(mp->ops_index);
515  return ops->enqueue(mp, obj_table, n);
516 }
517 
526 unsigned
527 rte_mempool_ops_get_count(const struct rte_mempool *mp);
528 
542 int
543 rte_mempool_ops_get_capabilities(const struct rte_mempool *mp,
544  unsigned int *flags);
562 int
563 rte_mempool_ops_register_memory_area(const struct rte_mempool *mp,
564  char *vaddr, rte_iova_t iova, size_t len);
565 
572 void
573 rte_mempool_ops_free(struct rte_mempool *mp);
574 
592 int
593 rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name,
594  void *pool_config);
595 
606 int rte_mempool_register_ops(const struct rte_mempool_ops *ops);
607 
613 #define MEMPOOL_REGISTER_OPS(ops) \
614  void mp_hdlr_init_##ops(void); \
615  void __attribute__((constructor, used)) mp_hdlr_init_##ops(void)\
616  { \
617  rte_mempool_register_ops(&ops); \
618  }
619 
625 typedef void (rte_mempool_obj_cb_t)(struct rte_mempool *mp,
626  void *opaque, void *obj, unsigned obj_idx);
627 typedef rte_mempool_obj_cb_t rte_mempool_obj_ctor_t; /* compat */
628 
634 typedef void (rte_mempool_mem_cb_t)(struct rte_mempool *mp,
635  void *opaque, struct rte_mempool_memhdr *memhdr,
636  unsigned mem_idx);
637 
644 typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *);
645 
725 struct rte_mempool *
726 rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
727  unsigned cache_size, unsigned private_data_size,
728  rte_mempool_ctor_t *mp_init, void *mp_init_arg,
729  rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
730  int socket_id, unsigned flags);
731 
790 struct rte_mempool *
791 rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size,
792  unsigned cache_size, unsigned private_data_size,
793  rte_mempool_ctor_t *mp_init, void *mp_init_arg,
794  rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
795  int socket_id, unsigned flags, void *vaddr,
796  const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift);
797 
832 struct rte_mempool *
833 rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size,
834  unsigned cache_size, unsigned private_data_size,
835  int socket_id, unsigned flags);
846 void
847 rte_mempool_free(struct rte_mempool *mp);
848 
876 int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
877  rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
878  void *opaque);
879 
880 __rte_deprecated
881 int rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr,
882  phys_addr_t paddr, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
883  void *opaque);
884 
911 int rte_mempool_populate_iova_tab(struct rte_mempool *mp, char *vaddr,
912  const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift,
913  rte_mempool_memchunk_free_cb_t *free_cb, void *opaque);
914 
915 __rte_deprecated
916 int rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr,
917  const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift,
918  rte_mempool_memchunk_free_cb_t *free_cb, void *opaque);
919 
944 int
945 rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
946  size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb,
947  void *opaque);
948 
963 
977 int rte_mempool_populate_anon(struct rte_mempool *mp);
978 
994 uint32_t rte_mempool_obj_iter(struct rte_mempool *mp,
995  rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg);
996 
1012 uint32_t rte_mempool_mem_iter(struct rte_mempool *mp,
1013  rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg);
1014 
1023 void rte_mempool_dump(FILE *f, struct rte_mempool *mp);
1024 
1039 struct rte_mempool_cache *
1040 rte_mempool_cache_create(uint32_t size, int socket_id);
1041 
1048 void
1050 
1059 static __rte_always_inline void
1061  struct rte_mempool *mp)
1062 {
1063  rte_mempool_ops_enqueue_bulk(mp, cache->objs, cache->len);
1064  cache->len = 0;
1065 }
1066 
1077 static __rte_always_inline struct rte_mempool_cache *
1078 rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
1079 {
1080  if (mp->cache_size == 0)
1081  return NULL;
1082 
1083  if (lcore_id >= RTE_MAX_LCORE)
1084  return NULL;
1085 
1086  return &mp->local_cache[lcore_id];
1087 }
1088 
1101 static __rte_always_inline void
1102 __mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1103  unsigned int n, struct rte_mempool_cache *cache)
1104 {
1105  void **cache_objs;
1106 
1107  /* increment stat now, adding in mempool always success */
1108  __MEMPOOL_STAT_ADD(mp, put, n);
1109 
1110  /* No cache provided or if put would overflow mem allocated for cache */
1111  if (unlikely(cache == NULL || n > RTE_MEMPOOL_CACHE_MAX_SIZE))
1112  goto ring_enqueue;
1113 
1114  cache_objs = &cache->objs[cache->len];
1115 
1116  /*
1117  * The cache follows the following algorithm
1118  * 1. Add the objects to the cache
1119  * 2. Anything greater than the cache min value (if it crosses the
1120  * cache flush threshold) is flushed to the ring.
1121  */
1122 
1123  /* Add elements back into the cache */
1124  rte_memcpy(&cache_objs[0], obj_table, sizeof(void *) * n);
1125 
1126  cache->len += n;
1127 
1128  if (cache->len >= cache->flushthresh) {
1129  rte_mempool_ops_enqueue_bulk(mp, &cache->objs[cache->size],
1130  cache->len - cache->size);
1131  cache->len = cache->size;
1132  }
1133 
1134  return;
1135 
1136 ring_enqueue:
1137 
1138  /* push remaining objects in ring */
1139 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
1140  if (rte_mempool_ops_enqueue_bulk(mp, obj_table, n) < 0)
1141  rte_panic("cannot put objects in mempool\n");
1142 #else
1143  rte_mempool_ops_enqueue_bulk(mp, obj_table, n);
1144 #endif
1145 }
1146 
1147 
1160 static __rte_always_inline void
1161 rte_mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1162  unsigned int n, struct rte_mempool_cache *cache)
1163 {
1164  __mempool_check_cookies(mp, obj_table, n, 0);
1165  __mempool_generic_put(mp, obj_table, n, cache);
1166 }
1167 
1182 static __rte_always_inline void
1183 rte_mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,
1184  unsigned int n)
1185 {
1186  struct rte_mempool_cache *cache;
1187  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1188  rte_mempool_generic_put(mp, obj_table, n, cache);
1189 }
1190 
1203 static __rte_always_inline void
1204 rte_mempool_put(struct rte_mempool *mp, void *obj)
1205 {
1206  rte_mempool_put_bulk(mp, &obj, 1);
1207 }
1208 
1223 static __rte_always_inline int
1224 __mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1225  unsigned int n, struct rte_mempool_cache *cache)
1226 {
1227  int ret;
1228  uint32_t index, len;
1229  void **cache_objs;
1230 
1231  /* No cache provided or cannot be satisfied from cache */
1232  if (unlikely(cache == NULL || n >= cache->size))
1233  goto ring_dequeue;
1234 
1235  cache_objs = cache->objs;
1236 
1237  /* Can this be satisfied from the cache? */
1238  if (cache->len < n) {
1239  /* No. Backfill the cache first, and then fill from it */
1240  uint32_t req = n + (cache->size - cache->len);
1241 
1242  /* How many do we require i.e. number to fill the cache + the request */
1243  ret = rte_mempool_ops_dequeue_bulk(mp,
1244  &cache->objs[cache->len], req);
1245  if (unlikely(ret < 0)) {
1246  /*
1247  * In the offchance that we are buffer constrained,
1248  * where we are not able to allocate cache + n, go to
1249  * the ring directly. If that fails, we are truly out of
1250  * buffers.
1251  */
1252  goto ring_dequeue;
1253  }
1254 
1255  cache->len += req;
1256  }
1257 
1258  /* Now fill in the response ... */
1259  for (index = 0, len = cache->len - 1; index < n; ++index, len--, obj_table++)
1260  *obj_table = cache_objs[len];
1261 
1262  cache->len -= n;
1263 
1264  __MEMPOOL_STAT_ADD(mp, get_success, n);
1265 
1266  return 0;
1267 
1268 ring_dequeue:
1269 
1270  /* get remaining objects from ring */
1271  ret = rte_mempool_ops_dequeue_bulk(mp, obj_table, n);
1272 
1273  if (ret < 0)
1274  __MEMPOOL_STAT_ADD(mp, get_fail, n);
1275  else
1276  __MEMPOOL_STAT_ADD(mp, get_success, n);
1277 
1278  return ret;
1279 }
1280 
1301 static __rte_always_inline int
1302 rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1303  unsigned int n, struct rte_mempool_cache *cache)
1304 {
1305  int ret;
1306  ret = __mempool_generic_get(mp, obj_table, n, cache);
1307  if (ret == 0)
1308  __mempool_check_cookies(mp, obj_table, n, 1);
1309  return ret;
1310 }
1311 
1334 static __rte_always_inline int
1335 rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
1336 {
1337  struct rte_mempool_cache *cache;
1338  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1339  return rte_mempool_generic_get(mp, obj_table, n, cache);
1340 }
1341 
1362 static __rte_always_inline int
1363 rte_mempool_get(struct rte_mempool *mp, void **obj_p)
1364 {
1365  return rte_mempool_get_bulk(mp, obj_p, 1);
1366 }
1367 
1380 unsigned int rte_mempool_avail_count(const struct rte_mempool *mp);
1381 
1394 unsigned int
1395 rte_mempool_in_use_count(const struct rte_mempool *mp);
1396 
1410 static inline int
1412 {
1413  return !!(rte_mempool_avail_count(mp) == mp->size);
1414 }
1415 
1429 static inline int
1431 {
1432  return !!(rte_mempool_avail_count(mp) == 0);
1433 }
1434 
1445 static inline rte_iova_t
1446 rte_mempool_virt2iova(const void *elt)
1447 {
1448  const struct rte_mempool_objhdr *hdr;
1449  hdr = (const struct rte_mempool_objhdr *)RTE_PTR_SUB(elt,
1450  sizeof(*hdr));
1451  return hdr->iova;
1452 }
1453 
1454 __rte_deprecated
1455 static inline phys_addr_t
1456 rte_mempool_virt2phy(__rte_unused const struct rte_mempool *mp, const void *elt)
1457 {
1458  return rte_mempool_virt2iova(elt);
1459 }
1460 
1471 void rte_mempool_audit(struct rte_mempool *mp);
1472 
1481 static inline void *rte_mempool_get_priv(struct rte_mempool *mp)
1482 {
1483  return (char *)mp +
1485 }
1486 
1493 void rte_mempool_list_dump(FILE *f);
1494 
1507 struct rte_mempool *rte_mempool_lookup(const char *name);
1508 
1526 uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
1527  struct rte_mempool_objsz *sz);
1528 
1552 size_t rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz,
1553  uint32_t pg_shift, unsigned int flags);
1554 
1583 ssize_t rte_mempool_xmem_usage(void *vaddr, uint32_t elt_num,
1584  size_t total_elt_sz, const rte_iova_t iova[], uint32_t pg_num,
1585  uint32_t pg_shift, unsigned int flags);
1586 
1595 void rte_mempool_walk(void (*func)(struct rte_mempool *, void *arg),
1596  void *arg);
1597 
1598 #ifdef __cplusplus
1599 }
1600 #endif
1601 
1602 #endif /* _RTE_MEMPOOL_H_ */