DPDK  24.03.0-rc3
rte_mbuf.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2014 Intel Corporation.
3  * Copyright 2014 6WIND S.A.
4  */
5 
6 #ifndef _RTE_MBUF_H_
7 #define _RTE_MBUF_H_
8 
34 #include <stdint.h>
35 
36 #include <rte_common.h>
37 #include <rte_config.h>
38 #include <rte_mempool.h>
39 #include <rte_prefetch.h>
40 #include <rte_branch_prediction.h>
41 #include <rte_mbuf_ptype.h>
42 #include <rte_mbuf_core.h>
43 
44 #ifdef __cplusplus
45 extern "C" {
46 #endif
47 
56 const char *rte_get_rx_ol_flag_name(uint64_t mask);
57 
70 int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
71 
82 const char *rte_get_tx_ol_flag_name(uint64_t mask);
83 
96 int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
97 
108 static inline void
110 {
111  rte_prefetch0(&m->cacheline0);
112 }
113 
125 static inline void
127 {
128 #if RTE_CACHE_LINE_SIZE == 64
129  rte_prefetch0(&m->cacheline1);
130 #else
131  RTE_SET_USED(m);
132 #endif
133 }
134 
135 
136 static inline uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp);
137 
146 static inline rte_iova_t
147 rte_mbuf_iova_get(const struct rte_mbuf *m)
148 {
149 #if RTE_IOVA_IN_MBUF
150  return m->buf_iova;
151 #else
152  return (rte_iova_t)m->buf_addr;
153 #endif
154 }
155 
164 static inline void
166 {
167 #if RTE_IOVA_IN_MBUF
168  m->buf_iova = iova;
169 #else
170  RTE_SET_USED(m);
171  RTE_SET_USED(iova);
172 #endif
173 }
174 
183 static inline rte_iova_t
184 rte_mbuf_data_iova(const struct rte_mbuf *mb)
185 {
186  return rte_mbuf_iova_get(mb) + mb->data_off;
187 }
188 
201 static inline rte_iova_t
203 {
204  return rte_mbuf_iova_get(mb) + RTE_PKTMBUF_HEADROOM;
205 }
206 
215 static inline struct rte_mbuf *
217 {
218  return (struct rte_mbuf *)RTE_PTR_SUB(mi->buf_addr, sizeof(*mi) + mi->priv_size);
219 }
220 
236 static inline char *
237 rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
238 {
239  return (char *)mb + sizeof(*mb) + rte_pktmbuf_priv_size(mp);
240 }
241 
250 static inline char *
252 {
253  return rte_mbuf_buf_addr(mb, mb->pool) + RTE_PKTMBUF_HEADROOM;
254 }
255 
269 static inline char *
271 {
272  return rte_mbuf_buf_addr(md, md->pool);
273 }
274 
287 static inline void *
289 {
290  return RTE_PTR_ADD(m, sizeof(struct rte_mbuf));
291 }
292 
301  uint16_t mbuf_priv_size;
302  uint32_t flags;
303 };
304 
313 static inline uint32_t
315 {
316  struct rte_pktmbuf_pool_private *mbp_priv;
317 
318  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
319  return mbp_priv->flags;
320 }
321 
328 #define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF (1 << 0)
329 
337 #define RTE_MBUF_HAS_PINNED_EXTBUF(mb) \
338  (rte_pktmbuf_priv_flags(mb->pool) & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF)
339 
340 #ifdef RTE_LIBRTE_MBUF_DEBUG
341 
343 #define __rte_mbuf_sanity_check(m, is_h) rte_mbuf_sanity_check(m, is_h)
344 
345 #else /* RTE_LIBRTE_MBUF_DEBUG */
346 
348 #define __rte_mbuf_sanity_check(m, is_h) do { } while (0)
349 
350 #endif /* RTE_LIBRTE_MBUF_DEBUG */
351 
352 #ifdef RTE_MBUF_REFCNT_ATOMIC
353 
361 static inline uint16_t
362 rte_mbuf_refcnt_read(const struct rte_mbuf *m)
363 {
364  return rte_atomic_load_explicit(&m->refcnt, rte_memory_order_relaxed);
365 }
366 
374 static inline void
375 rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
376 {
377  rte_atomic_store_explicit(&m->refcnt, new_value, rte_memory_order_relaxed);
378 }
379 
380 /* internal */
381 static inline uint16_t
382 __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
383 {
384  return rte_atomic_fetch_add_explicit(&m->refcnt, value,
385  rte_memory_order_acq_rel) + value;
386 }
387 
397 static inline uint16_t
398 rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
399 {
400  /*
401  * The atomic_add is an expensive operation, so we don't want to
402  * call it in the case where we know we are the unique holder of
403  * this mbuf (i.e. ref_cnt == 1). Otherwise, an atomic
404  * operation has to be used because concurrent accesses on the
405  * reference counter can occur.
406  */
407  if (likely(rte_mbuf_refcnt_read(m) == 1)) {
408  ++value;
409  rte_mbuf_refcnt_set(m, (uint16_t)value);
410  return (uint16_t)value;
411  }
412 
413  return __rte_mbuf_refcnt_update(m, value);
414 }
415 
416 #else /* ! RTE_MBUF_REFCNT_ATOMIC */
417 
418 /* internal */
419 static inline uint16_t
420 __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
421 {
422  m->refcnt = (uint16_t)(m->refcnt + value);
423  return m->refcnt;
424 }
425 
429 static inline uint16_t
430 rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
431 {
432  return __rte_mbuf_refcnt_update(m, value);
433 }
434 
438 static inline uint16_t
440 {
441  return m->refcnt;
442 }
443 
447 static inline void
448 rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
449 {
450  m->refcnt = new_value;
451 }
452 
453 #endif /* RTE_MBUF_REFCNT_ATOMIC */
454 
463 static inline uint16_t
465 {
466  return rte_atomic_load_explicit(&shinfo->refcnt, rte_memory_order_relaxed);
467 }
468 
477 static inline void
479  uint16_t new_value)
480 {
481  rte_atomic_store_explicit(&shinfo->refcnt, new_value, rte_memory_order_relaxed);
482 }
483 
495 static inline uint16_t
497  int16_t value)
498 {
499  if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1)) {
500  ++value;
501  rte_mbuf_ext_refcnt_set(shinfo, (uint16_t)value);
502  return (uint16_t)value;
503  }
504 
505  return rte_atomic_fetch_add_explicit(&shinfo->refcnt, value,
506  rte_memory_order_acq_rel) + value;
507 }
508 
510 #define RTE_MBUF_PREFETCH_TO_FREE(m) do { \
511  if ((m) != NULL) \
512  rte_prefetch0(m); \
513 } while (0)
514 
515 
528 void
529 rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header);
530 
550 int rte_mbuf_check(const struct rte_mbuf *m, int is_header,
551  const char **reason);
552 
565 static __rte_always_inline void
567 {
568  RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);
569  RTE_ASSERT(m->next == NULL);
570  RTE_ASSERT(m->nb_segs == 1);
572 }
573 
575 #define MBUF_RAW_ALLOC_CHECK(m) __rte_mbuf_raw_sanity_check(m)
576 
596 static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
597 {
598  struct rte_mbuf *m;
599 
600  if (rte_mempool_get(mp, (void **)&m) < 0)
601  return NULL;
603  return m;
604 }
605 
620 static __rte_always_inline void
622 {
623  RTE_ASSERT(!RTE_MBUF_CLONED(m) &&
626  rte_mempool_put(m->pool, m);
627 }
628 
651 void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg,
652  void *m, unsigned i);
653 
674 void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg);
675 
709 struct rte_mempool *
710 rte_pktmbuf_pool_create(const char *name, unsigned n,
711  unsigned cache_size, uint16_t priv_size, uint16_t data_room_size,
712  int socket_id);
713 
750 struct rte_mempool *
751 rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n,
752  unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size,
753  int socket_id, const char *ops_name);
754 
757  void *buf_ptr;
759  size_t buf_len;
760  uint16_t elt_size;
761 };
762 
803 struct rte_mempool *
804 rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n,
805  unsigned int cache_size, uint16_t priv_size,
806  uint16_t data_room_size, int socket_id,
807  const struct rte_pktmbuf_extmem *ext_mem,
808  unsigned int ext_num);
809 
821 static inline uint16_t
823 {
824  struct rte_pktmbuf_pool_private *mbp_priv;
825 
826  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
827  return mbp_priv->mbuf_data_room_size;
828 }
829 
842 static inline uint16_t
844 {
845  struct rte_pktmbuf_pool_private *mbp_priv;
846 
847  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
848  return mbp_priv->mbuf_priv_size;
849 }
850 
859 static inline void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
860 {
861  m->data_off = (uint16_t)RTE_MIN((uint16_t)RTE_PKTMBUF_HEADROOM,
862  (uint16_t)m->buf_len);
863 }
864 
873 static inline void rte_pktmbuf_reset(struct rte_mbuf *m)
874 {
875  m->next = NULL;
876  m->pkt_len = 0;
877  m->tx_offload = 0;
878  m->vlan_tci = 0;
879  m->vlan_tci_outer = 0;
880  m->nb_segs = 1;
882 
884  m->packet_type = 0;
886 
887  m->data_len = 0;
889 }
890 
904 static inline struct rte_mbuf *rte_pktmbuf_alloc(struct rte_mempool *mp)
905 {
906  struct rte_mbuf *m;
907  if ((m = rte_mbuf_raw_alloc(mp)) != NULL)
909  return m;
910 }
911 
926 static inline int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool,
927  struct rte_mbuf **mbufs, unsigned count)
928 {
929  unsigned idx = 0;
930  int rc;
931 
932  rc = rte_mempool_get_bulk(pool, (void **)mbufs, count);
933  if (unlikely(rc))
934  return rc;
935 
936  /* To understand duff's device on loop unwinding optimization, see
937  * https://en.wikipedia.org/wiki/Duff's_device.
938  * Here while() loop is used rather than do() while{} to avoid extra
939  * check if count is zero.
940  */
941  switch (count % 4) {
942  case 0:
943  while (idx != count) {
944  __rte_mbuf_raw_sanity_check(mbufs[idx]);
945  rte_pktmbuf_reset(mbufs[idx]);
946  idx++;
947  /* fall-through */
948  case 3:
949  __rte_mbuf_raw_sanity_check(mbufs[idx]);
950  rte_pktmbuf_reset(mbufs[idx]);
951  idx++;
952  /* fall-through */
953  case 2:
954  __rte_mbuf_raw_sanity_check(mbufs[idx]);
955  rte_pktmbuf_reset(mbufs[idx]);
956  idx++;
957  /* fall-through */
958  case 1:
959  __rte_mbuf_raw_sanity_check(mbufs[idx]);
960  rte_pktmbuf_reset(mbufs[idx]);
961  idx++;
962  /* fall-through */
963  }
964  }
965  return 0;
966 }
967 
1000 static inline struct rte_mbuf_ext_shared_info *
1001 rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len,
1003 {
1004  struct rte_mbuf_ext_shared_info *shinfo;
1005  void *buf_end = RTE_PTR_ADD(buf_addr, *buf_len);
1006  void *addr;
1007 
1008  addr = RTE_PTR_ALIGN_FLOOR(RTE_PTR_SUB(buf_end, sizeof(*shinfo)),
1009  sizeof(uintptr_t));
1010  if (addr <= buf_addr)
1011  return NULL;
1012 
1013  shinfo = (struct rte_mbuf_ext_shared_info *)addr;
1014  shinfo->free_cb = free_cb;
1015  shinfo->fcb_opaque = fcb_opaque;
1016  rte_mbuf_ext_refcnt_set(shinfo, 1);
1017 
1018  *buf_len = (uint16_t)RTE_PTR_DIFF(shinfo, buf_addr);
1019  return shinfo;
1020 }
1021 
1082 static inline void
1083 rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr,
1084  rte_iova_t buf_iova, uint16_t buf_len,
1085  struct rte_mbuf_ext_shared_info *shinfo)
1086 {
1087  /* mbuf should not be read-only */
1088  RTE_ASSERT(RTE_MBUF_DIRECT(m) && rte_mbuf_refcnt_read(m) == 1);
1089  RTE_ASSERT(shinfo->free_cb != NULL);
1090 
1091  m->buf_addr = buf_addr;
1092  rte_mbuf_iova_set(m, buf_iova);
1093  m->buf_len = buf_len;
1094 
1095  m->data_len = 0;
1096  m->data_off = 0;
1097 
1099  m->shinfo = shinfo;
1100 }
1101 
1109 #define rte_pktmbuf_detach_extbuf(m) rte_pktmbuf_detach(m)
1110 
1119 static inline void
1120 rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
1121 {
1122  memcpy(&mdst->dynfield1, msrc->dynfield1, sizeof(mdst->dynfield1));
1123 }
1124 
1125 /* internal */
1126 static inline void
1127 __rte_pktmbuf_copy_hdr(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
1128 {
1129  mdst->port = msrc->port;
1130  mdst->vlan_tci = msrc->vlan_tci;
1131  mdst->vlan_tci_outer = msrc->vlan_tci_outer;
1132  mdst->tx_offload = msrc->tx_offload;
1133  mdst->hash = msrc->hash;
1134  mdst->packet_type = msrc->packet_type;
1135  rte_mbuf_dynfield_copy(mdst, msrc);
1136 }
1137 
1159 static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
1160 {
1161  RTE_ASSERT(RTE_MBUF_DIRECT(mi) &&
1162  rte_mbuf_refcnt_read(mi) == 1);
1163 
1164  if (RTE_MBUF_HAS_EXTBUF(m)) {
1166  mi->ol_flags = m->ol_flags;
1167  mi->shinfo = m->shinfo;
1168  } else {
1169  /* if m is not direct, get the mbuf that embeds the data */
1171  mi->priv_size = m->priv_size;
1173  }
1174 
1175  __rte_pktmbuf_copy_hdr(mi, m);
1176 
1177  mi->data_off = m->data_off;
1178  mi->data_len = m->data_len;
1180  mi->buf_addr = m->buf_addr;
1181  mi->buf_len = m->buf_len;
1182 
1183  mi->next = NULL;
1184  mi->pkt_len = mi->data_len;
1185  mi->nb_segs = 1;
1186 
1187  __rte_mbuf_sanity_check(mi, 1);
1189 }
1190 
1198 static inline void
1199 __rte_pktmbuf_free_extbuf(struct rte_mbuf *m)
1200 {
1201  RTE_ASSERT(RTE_MBUF_HAS_EXTBUF(m));
1202  RTE_ASSERT(m->shinfo != NULL);
1203 
1204  if (rte_mbuf_ext_refcnt_update(m->shinfo, -1) == 0)
1205  m->shinfo->free_cb(m->buf_addr, m->shinfo->fcb_opaque);
1206 }
1207 
1214 static inline void
1215 __rte_pktmbuf_free_direct(struct rte_mbuf *m)
1216 {
1217  struct rte_mbuf *md;
1218 
1219  RTE_ASSERT(RTE_MBUF_CLONED(m));
1220 
1221  md = rte_mbuf_from_indirect(m);
1222 
1223  if (rte_mbuf_refcnt_update(md, -1) == 0) {
1224  md->next = NULL;
1225  md->nb_segs = 1;
1226  rte_mbuf_refcnt_set(md, 1);
1227  rte_mbuf_raw_free(md);
1228  }
1229 }
1230 
1249 static inline void rte_pktmbuf_detach(struct rte_mbuf *m)
1250 {
1251  struct rte_mempool *mp = m->pool;
1252  uint32_t mbuf_size, buf_len;
1253  uint16_t priv_size;
1254 
1255  if (RTE_MBUF_HAS_EXTBUF(m)) {
1256  /*
1257  * The mbuf has the external attached buffer,
1258  * we should check the type of the memory pool where
1259  * the mbuf was allocated from to detect the pinned
1260  * external buffer.
1261  */
1262  uint32_t flags = rte_pktmbuf_priv_flags(mp);
1263 
1264  if (flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) {
1265  /*
1266  * The pinned external buffer should not be
1267  * detached from its backing mbuf, just exit.
1268  */
1269  return;
1270  }
1271  __rte_pktmbuf_free_extbuf(m);
1272  } else {
1273  __rte_pktmbuf_free_direct(m);
1274  }
1275  priv_size = rte_pktmbuf_priv_size(mp);
1276  mbuf_size = (uint32_t)(sizeof(struct rte_mbuf) + priv_size);
1277  buf_len = rte_pktmbuf_data_room_size(mp);
1278 
1279  m->priv_size = priv_size;
1280  m->buf_addr = (char *)m + mbuf_size;
1281  rte_mbuf_iova_set(m, rte_mempool_virt2iova(m) + mbuf_size);
1282  m->buf_len = (uint16_t)buf_len;
1284  m->data_len = 0;
1285  m->ol_flags = 0;
1286 }
1287 
1301 static inline int __rte_pktmbuf_pinned_extbuf_decref(struct rte_mbuf *m)
1302 {
1303  struct rte_mbuf_ext_shared_info *shinfo;
1304 
1305  /* Clear flags, mbuf is being freed. */
1307  shinfo = m->shinfo;
1308 
1309  /* Optimize for performance - do not dec/reinit */
1310  if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1))
1311  return 0;
1312 
1313  /*
1314  * Direct usage of add primitive to avoid
1315  * duplication of comparing with one.
1316  */
1317  if (likely(rte_atomic_fetch_add_explicit(&shinfo->refcnt, -1,
1318  rte_memory_order_acq_rel) - 1))
1319  return 1;
1320 
1321  /* Reinitialize counter before mbuf freeing. */
1322  rte_mbuf_ext_refcnt_set(shinfo, 1);
1323  return 0;
1324 }
1325 
1340 static __rte_always_inline struct rte_mbuf *
1342 {
1344 
1345  if (likely(rte_mbuf_refcnt_read(m) == 1)) {
1346 
1347  if (!RTE_MBUF_DIRECT(m)) {
1348  rte_pktmbuf_detach(m);
1349  if (RTE_MBUF_HAS_EXTBUF(m) &&
1351  __rte_pktmbuf_pinned_extbuf_decref(m))
1352  return NULL;
1353  }
1354 
1355  if (m->next != NULL)
1356  m->next = NULL;
1357  if (m->nb_segs != 1)
1358  m->nb_segs = 1;
1359 
1360  return m;
1361 
1362  } else if (__rte_mbuf_refcnt_update(m, -1) == 0) {
1363 
1364  if (!RTE_MBUF_DIRECT(m)) {
1365  rte_pktmbuf_detach(m);
1366  if (RTE_MBUF_HAS_EXTBUF(m) &&
1368  __rte_pktmbuf_pinned_extbuf_decref(m))
1369  return NULL;
1370  }
1371 
1372  if (m->next != NULL)
1373  m->next = NULL;
1374  if (m->nb_segs != 1)
1375  m->nb_segs = 1;
1376  rte_mbuf_refcnt_set(m, 1);
1377 
1378  return m;
1379  }
1380  return NULL;
1381 }
1382 
1392 static __rte_always_inline void
1394 {
1395  m = rte_pktmbuf_prefree_seg(m);
1396  if (likely(m != NULL))
1397  rte_mbuf_raw_free(m);
1398 }
1399 
1409 static inline void rte_pktmbuf_free(struct rte_mbuf *m)
1410 {
1411  struct rte_mbuf *m_next;
1412 
1413  if (m != NULL)
1415 
1416  while (m != NULL) {
1417  m_next = m->next;
1419  m = m_next;
1420  }
1421 }
1422 
1435 void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count);
1436 
1454 struct rte_mbuf *
1455 rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp);
1456 
1478 struct rte_mbuf *
1479 rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp,
1480  uint32_t offset, uint32_t length);
1481 
1493 static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
1494 {
1496 
1497  do {
1498  rte_mbuf_refcnt_update(m, v);
1499  } while ((m = m->next) != NULL);
1500 }
1501 
1510 static inline uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
1511 {
1513  return m->data_off;
1514 }
1515 
1524 static inline uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
1525 {
1527  return (uint16_t)(m->buf_len - rte_pktmbuf_headroom(m) -
1528  m->data_len);
1529 }
1530 
1539 static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m)
1540 {
1542  while (m->next != NULL)
1543  m = m->next;
1544  return m;
1545 }
1546 
1555 #define rte_pktmbuf_pkt_len(m) ((m)->pkt_len)
1556 
1565 #define rte_pktmbuf_data_len(m) ((m)->data_len)
1566 
1582 static inline char *rte_pktmbuf_prepend(struct rte_mbuf *m,
1583  uint16_t len)
1584 {
1586 
1587  if (unlikely(len > rte_pktmbuf_headroom(m)))
1588  return NULL;
1589 
1590  /* NB: elaborating the subtraction like this instead of using
1591  * -= allows us to ensure the result type is uint16_t
1592  * avoiding compiler warnings on gcc 8.1 at least */
1593  m->data_off = (uint16_t)(m->data_off - len);
1594  m->data_len = (uint16_t)(m->data_len + len);
1595  m->pkt_len = (m->pkt_len + len);
1596 
1597  return (char *)m->buf_addr + m->data_off;
1598 }
1599 
1615 static inline char *rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
1616 {
1617  void *tail;
1618  struct rte_mbuf *m_last;
1619 
1621 
1622  m_last = rte_pktmbuf_lastseg(m);
1623  if (unlikely(len > rte_pktmbuf_tailroom(m_last)))
1624  return NULL;
1625 
1626  tail = (char *)m_last->buf_addr + m_last->data_off + m_last->data_len;
1627  m_last->data_len = (uint16_t)(m_last->data_len + len);
1628  m->pkt_len = (m->pkt_len + len);
1629  return (char*) tail;
1630 }
1631 
1646 static inline char *rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
1647 {
1649 
1650  if (unlikely(len > m->data_len))
1651  return NULL;
1652 
1653  /* NB: elaborating the addition like this instead of using
1654  * += allows us to ensure the result type is uint16_t
1655  * avoiding compiler warnings on gcc 8.1 at least */
1656  m->data_len = (uint16_t)(m->data_len - len);
1657  m->data_off = (uint16_t)(m->data_off + len);
1658  m->pkt_len = (m->pkt_len - len);
1659  return (char *)m->buf_addr + m->data_off;
1660 }
1661 
1676 static inline int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
1677 {
1678  struct rte_mbuf *m_last;
1679 
1681 
1682  m_last = rte_pktmbuf_lastseg(m);
1683  if (unlikely(len > m_last->data_len))
1684  return -1;
1685 
1686  m_last->data_len = (uint16_t)(m_last->data_len - len);
1687  m->pkt_len = (m->pkt_len - len);
1688  return 0;
1689 }
1690 
1700 static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
1701 {
1703  return m->nb_segs == 1;
1704 }
1705 
1709 const void *__rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off,
1710  uint32_t len, void *buf);
1711 
1732 static inline const void *rte_pktmbuf_read(const struct rte_mbuf *m,
1733  uint32_t off, uint32_t len, void *buf)
1734 {
1735  if (likely(off + len <= rte_pktmbuf_data_len(m)))
1736  return rte_pktmbuf_mtod_offset(m, char *, off);
1737  else
1738  return __rte_pktmbuf_read(m, off, len, buf);
1739 }
1740 
1757 static inline int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
1758 {
1759  struct rte_mbuf *cur_tail;
1760 
1761  /* Check for number-of-segments-overflow */
1762  if (head->nb_segs + tail->nb_segs > RTE_MBUF_MAX_NB_SEGS)
1763  return -EOVERFLOW;
1764 
1765  /* Chain 'tail' onto the old tail */
1766  cur_tail = rte_pktmbuf_lastseg(head);
1767  cur_tail->next = tail;
1768 
1769  /* accumulate number of segments and total length.
1770  * NB: elaborating the addition like this instead of using
1771  * -= allows us to ensure the result type is uint16_t
1772  * avoiding compiler warnings on gcc 8.1 at least */
1773  head->nb_segs = (uint16_t)(head->nb_segs + tail->nb_segs);
1774  head->pkt_len += tail->pkt_len;
1775 
1776  /* pkt_len is only set in the head */
1777  tail->pkt_len = tail->data_len;
1778 
1779  return 0;
1780 }
1781 
1803 static __rte_always_inline uint64_t
1804 rte_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso,
1805  uint64_t ol3, uint64_t ol2, uint64_t unused)
1806 {
1807  return il2 << RTE_MBUF_L2_LEN_OFS |
1808  il3 << RTE_MBUF_L3_LEN_OFS |
1809  il4 << RTE_MBUF_L4_LEN_OFS |
1810  tso << RTE_MBUF_TSO_SEGSZ_OFS |
1811  ol3 << RTE_MBUF_OUTL3_LEN_OFS |
1812  ol2 << RTE_MBUF_OUTL2_LEN_OFS |
1813  unused << RTE_MBUF_TXOFLD_UNUSED_OFS;
1814 }
1815 
1826 static inline int
1828 {
1829  uint64_t ol_flags = m->ol_flags;
1830 
1831  /* Does packet set any of available offloads? */
1832  if (!(ol_flags & RTE_MBUF_F_TX_OFFLOAD_MASK))
1833  return 0;
1834 
1835  /* IP checksum can be counted only for IPv4 packet */
1836  if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) && (ol_flags & RTE_MBUF_F_TX_IPV6))
1837  return -EINVAL;
1838 
1839  /* IP type not set when required */
1840  if (ol_flags & (RTE_MBUF_F_TX_L4_MASK | RTE_MBUF_F_TX_TCP_SEG))
1841  if (!(ol_flags & (RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IPV6)))
1842  return -EINVAL;
1843 
1844  /* Check requirements for TSO packet */
1845  if (ol_flags & RTE_MBUF_F_TX_TCP_SEG)
1846  if ((m->tso_segsz == 0) ||
1847  ((ol_flags & RTE_MBUF_F_TX_IPV4) &&
1848  !(ol_flags & RTE_MBUF_F_TX_IP_CKSUM)))
1849  return -EINVAL;
1850 
1851  /* RTE_MBUF_F_TX_OUTER_IP_CKSUM set for non outer IPv4 packet. */
1852  if ((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) &&
1853  !(ol_flags & RTE_MBUF_F_TX_OUTER_IPV4))
1854  return -EINVAL;
1855 
1856  return 0;
1857 }
1858 
1862 int __rte_pktmbuf_linearize(struct rte_mbuf *mbuf);
1863 
1876 static inline int
1878 {
1879  if (rte_pktmbuf_is_contiguous(mbuf))
1880  return 0;
1881  return __rte_pktmbuf_linearize(mbuf);
1882 }
1883 
1898 void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len);
1899 
1903 static inline uint32_t
1905 {
1906  return m->hash.sched.queue_id;
1907 }
1908 
1912 static inline uint8_t
1914 {
1915  return m->hash.sched.traffic_class;
1916 }
1917 
1921 static inline uint8_t
1923 {
1924  return m->hash.sched.color;
1925 }
1926 
1939 static inline void
1940 rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id,
1941  uint8_t *traffic_class,
1942  uint8_t *color)
1943 {
1944  struct rte_mbuf_sched sched = m->hash.sched;
1945 
1946  *queue_id = sched.queue_id;
1947  *traffic_class = sched.traffic_class;
1948  *color = sched.color;
1949 }
1950 
1954 static inline void
1956 {
1957  m->hash.sched.queue_id = queue_id;
1958 }
1959 
1963 static inline void
1965 {
1966  m->hash.sched.traffic_class = traffic_class;
1967 }
1968 
1972 static inline void
1974 {
1975  m->hash.sched.color = color;
1976 }
1977 
1990 static inline void
1992  uint8_t traffic_class,
1993  uint8_t color)
1994 {
1995  m->hash.sched = (struct rte_mbuf_sched){
1996  .queue_id = queue_id,
1997  .traffic_class = traffic_class,
1998  .color = color,
1999  .reserved = 0,
2000  };
2001 }
2002 
2003 #ifdef __cplusplus
2004 }
2005 #endif
2006 
2007 #endif /* _RTE_MBUF_H_ */
static void rte_pktmbuf_reset(struct rte_mbuf *m)
Definition: rte_mbuf.h:873
struct rte_mbuf_ext_shared_info * shinfo
static rte_iova_t rte_mbuf_data_iova(const struct rte_mbuf *mb)
Definition: rte_mbuf.h:184
struct rte_mbuf * next
uint16_t mbuf_data_room_size
Definition: rte_mbuf.h:300
void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count)
uint16_t vlan_tci_outer
#define __rte_always_inline
Definition: rte_common.h:355
#define RTE_MBUF_F_TX_OFFLOAD_MASK
static struct rte_mbuf * rte_pktmbuf_alloc(struct rte_mempool *mp)
Definition: rte_mbuf.h:904
static void rte_mbuf_sched_traffic_class_set(struct rte_mbuf *m, uint8_t traffic_class)
Definition: rte_mbuf.h:1964
static uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp)
Definition: rte_mbuf.h:843
static int rte_validate_tx_offload(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1827
uint32_t queue_id
#define likely(x)
static void rte_pktmbuf_free(struct rte_mbuf *m)
Definition: rte_mbuf.h:1409
static uint32_t rte_mbuf_sched_queue_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1904
#define RTE_PTR_ALIGN_FLOOR(ptr, align)
Definition: rte_common.h:424
void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg)
uint64_t rte_iova_t
Definition: rte_common.h:584
static __rte_always_inline void rte_pktmbuf_free_seg(struct rte_mbuf *m)
Definition: rte_mbuf.h:1393
void * buf_addr
static struct rte_mbuf * rte_mbuf_from_indirect(struct rte_mbuf *mi)
Definition: rte_mbuf.h:216
uint16_t data_len
rte_mbuf_extbuf_free_callback_t free_cb
static int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
Definition: rte_mbuf.h:1757
static uint8_t rte_mbuf_sched_traffic_class_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1913
#define __rte_unused
Definition: rte_common.h:156
static void rte_mbuf_iova_set(struct rte_mbuf *m, rte_iova_t iova)
Definition: rte_mbuf.h:165
#define RTE_MBUF_F_TX_IPV6
uint64_t tso_segsz
struct rte_mbuf * rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp, uint32_t offset, uint32_t length)
static __rte_always_inline void __rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m)
Definition: rte_mbuf.h:566
void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len)
static uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1510
static int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, struct rte_mbuf **mbufs, unsigned count)
Definition: rte_mbuf.h:926
static void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
Definition: rte_mbuf.h:859
static void * rte_mbuf_to_priv(struct rte_mbuf *m)
Definition: rte_mbuf.h:288
uint32_t cache_size
Definition: rte_mempool.h:241
static void rte_mbuf_prefetch_part2(struct rte_mbuf *m)
Definition: rte_mbuf.h:126
char name[RTE_MEMPOOL_NAMESIZE]
Definition: rte_mempool.h:231
static uint16_t rte_mbuf_ext_refcnt_update(struct rte_mbuf_ext_shared_info *shinfo, int16_t value)
Definition: rte_mbuf.h:496
uint16_t nb_segs
uint16_t port
int rte_mbuf_check(const struct rte_mbuf *m, int is_header, const char **reason)
#define rte_pktmbuf_mtod_offset(m, t, o)
static __rte_always_inline struct rte_mbuf * rte_pktmbuf_prefree_seg(struct rte_mbuf *m)
Definition: rte_mbuf.h:1341
static int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1700
struct rte_mempool * rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n, unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id, const struct rte_pktmbuf_extmem *ext_mem, unsigned int ext_num)
#define RTE_PTR_ADD(ptr, x)
Definition: rte_common.h:395
int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen)
#define RTE_MBUF_MAX_NB_SEGS
static void rte_mbuf_sched_set(struct rte_mbuf *m, uint32_t queue_id, uint8_t traffic_class, uint8_t color)
Definition: rte_mbuf.h:1991
static uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1524
static __rte_always_inline void rte_mbuf_raw_free(struct rte_mbuf *m)
Definition: rte_mbuf.h:621
static void rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id, uint8_t *traffic_class, uint8_t *color)
Definition: rte_mbuf.h:1940
#define unlikely(x)
uint16_t priv_size
static __rte_always_inline uint64_t rte_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso, uint64_t ol3, uint64_t ol2, uint64_t unused)
Definition: rte_mbuf.h:1804
static void rte_mbuf_sched_queue_set(struct rte_mbuf *m, uint32_t queue_id)
Definition: rte_mbuf.h:1955
static char * rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
Definition: rte_mbuf.h:237
void rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header)
static void rte_mbuf_sched_color_set(struct rte_mbuf *m, uint8_t color)
Definition: rte_mbuf.h:1973
void(* rte_mbuf_extbuf_free_callback_t)(void *addr, void *opaque)
#define RTE_MIN(a, b)
Definition: rte_common.h:609
#define __rte_mbuf_sanity_check(m, is_h)
Definition: rte_mbuf.h:348
const char * rte_get_tx_ol_flag_name(uint64_t mask)
static uint16_t rte_mbuf_refcnt_read(const struct rte_mbuf *m)
Definition: rte_mbuf.h:439
static int rte_pktmbuf_linearize(struct rte_mbuf *mbuf)
Definition: rte_mbuf.h:1877
static __rte_always_inline int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
Definition: rte_mempool.h:1709
static uint32_t rte_pktmbuf_priv_flags(struct rte_mempool *mp)
Definition: rte_mbuf.h:314
uint16_t elt_size
Definition: rte_mbuf.h:760
uint16_t refcnt
static char * rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1646
#define RTE_MBUF_DIRECT(mb)
#define RTE_MBUF_CLONED(mb)
static void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
Definition: rte_mbuf.h:1159
static __rte_always_inline int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:1680
#define RTE_MBUF_F_TX_OUTER_IP_CKSUM
uint64_t ol_flags
static void rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
Definition: rte_mbuf.h:1120
static void rte_pktmbuf_detach(struct rte_mbuf *m)
Definition: rte_mbuf.h:1249
static uint16_t rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
Definition: rte_mbuf.h:464
uint32_t pkt_len
uint16_t buf_len
#define rte_pktmbuf_data_len(m)
Definition: rte_mbuf.h:1565
#define RTE_MBUF_F_TX_L4_MASK
static uint16_t rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
Definition: rte_mbuf.h:430
uint32_t packet_type
#define RTE_MBUF_F_TX_IP_CKSUM
static uint16_t rte_pktmbuf_data_room_size(struct rte_mempool *mp)
Definition: rte_mbuf.h:822
#define RTE_MBUF_F_TX_TCP_SEG
#define RTE_MBUF_F_INDIRECT
#define RTE_MBUF_PORT_INVALID
const char * rte_get_rx_ol_flag_name(uint64_t mask)
static struct rte_mbuf_ext_shared_info * rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len, rte_mbuf_extbuf_free_callback_t free_cb, void *fcb_opaque)
Definition: rte_mbuf.h:1001
#define RTE_MBUF_F_TX_OUTER_IPV4
void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg, void *m, unsigned i)
#define RTE_MBUF_HAS_EXTBUF(mb)
struct rte_mempool * pool
static void rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo, uint16_t new_value)
Definition: rte_mbuf.h:478
static char * rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1615
static void rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
Definition: rte_mbuf.h:448
struct rte_mempool * rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n, unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id, const char *ops_name)
static rte_iova_t rte_mbuf_data_iova_default(const struct rte_mbuf *mb)
Definition: rte_mbuf.h:202
static int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1676
static char * rte_mbuf_to_baddr(struct rte_mbuf *md)
Definition: rte_mbuf.h:270
static const void * rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off, uint32_t len, void *buf)
Definition: rte_mbuf.h:1732
static char * rte_pktmbuf_prepend(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1582
static struct rte_mbuf * rte_mbuf_raw_alloc(struct rte_mempool *mp)
Definition: rte_mbuf.h:596
static rte_iova_t rte_mbuf_iova_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:147
static void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
Definition: rte_mbuf.h:1493
#define RTE_PTR_SUB(ptr, x)
Definition: rte_common.h:400
static struct rte_mbuf * rte_pktmbuf_lastseg(struct rte_mbuf *m)
Definition: rte_mbuf.h:1539
static uint8_t rte_mbuf_sched_color_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1922
#define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF
Definition: rte_mbuf.h:328
static char * rte_mbuf_data_addr_default(struct rte_mbuf *mb)
Definition: rte_mbuf.h:251
static rte_iova_t rte_mempool_virt2iova(const void *elt)
Definition: rte_mempool.h:1834
static __rte_always_inline void rte_mempool_put(struct rte_mempool *mp, void *obj)
Definition: rte_mempool.h:1487
rte_iova_t buf_iova
Definition: rte_mbuf.h:758
uint8_t traffic_class
#define RTE_MBUF_F_TX_IPV4
#define RTE_PTR_DIFF(ptr1, ptr2)
Definition: rte_common.h:407
static void rte_mbuf_prefetch_part1(struct rte_mbuf *m)
Definition: rte_mbuf.h:109
#define RTE_MBUF_F_EXTERNAL
static void * rte_mempool_get_priv(struct rte_mempool *mp)
Definition: rte_mempool.h:1862
uint64_t tx_offload
uint16_t vlan_tci
#define RTE_MBUF_HAS_PINNED_EXTBUF(mb)
Definition: rte_mbuf.h:337
struct rte_mbuf * rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp)
#define RTE_SET_USED(x)
Definition: rte_common.h:172
int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen)
static void rte_prefetch0(const volatile void *p)
struct rte_mempool * rte_pktmbuf_pool_create(const char *name, unsigned n, unsigned cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id)
uint32_t dynfield1[9]
static void rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr, rte_iova_t buf_iova, uint16_t buf_len, struct rte_mbuf_ext_shared_info *shinfo)
Definition: rte_mbuf.h:1083