DPDK  22.11.0
rte_mbuf.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2014 Intel Corporation.
3  * Copyright 2014 6WIND S.A.
4  */
5 
6 #ifndef _RTE_MBUF_H_
7 #define _RTE_MBUF_H_
8 
34 #include <stdint.h>
35 #include <rte_compat.h>
36 #include <rte_common.h>
37 #include <rte_config.h>
38 #include <rte_mempool.h>
39 #include <rte_prefetch.h>
40 #include <rte_branch_prediction.h>
41 #include <rte_mbuf_ptype.h>
42 #include <rte_mbuf_core.h>
43 
44 #ifdef __cplusplus
45 extern "C" {
46 #endif
47 
56 const char *rte_get_rx_ol_flag_name(uint64_t mask);
57 
70 int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
71 
82 const char *rte_get_tx_ol_flag_name(uint64_t mask);
83 
96 int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
97 
108 static inline void
110 {
111  rte_prefetch0(&m->cacheline0);
112 }
113 
125 static inline void
127 {
128 #if RTE_CACHE_LINE_SIZE == 64
129  rte_prefetch0(&m->cacheline1);
130 #else
131  RTE_SET_USED(m);
132 #endif
133 }
134 
135 
136 static inline uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp);
137 
146 static inline rte_iova_t
147 rte_mbuf_iova_get(const struct rte_mbuf *m)
148 {
149 #if RTE_IOVA_AS_PA
150  return m->buf_iova;
151 #else
152  return (rte_iova_t)m->buf_addr;
153 #endif
154 }
155 
164 static inline void
166 {
167 #if RTE_IOVA_AS_PA
168  m->buf_iova = iova;
169 #else
170  RTE_SET_USED(m);
171  RTE_SET_USED(iova);
172 #endif
173 }
174 
183 static inline rte_iova_t
184 rte_mbuf_data_iova(const struct rte_mbuf *mb)
185 {
186  return rte_mbuf_iova_get(mb) + mb->data_off;
187 }
188 
201 static inline rte_iova_t
203 {
204  return rte_mbuf_iova_get(mb) + RTE_PKTMBUF_HEADROOM;
205 }
206 
215 static inline struct rte_mbuf *
217 {
218  return (struct rte_mbuf *)RTE_PTR_SUB(mi->buf_addr, sizeof(*mi) + mi->priv_size);
219 }
220 
236 static inline char *
237 rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
238 {
239  return (char *)mb + sizeof(*mb) + rte_pktmbuf_priv_size(mp);
240 }
241 
250 static inline char *
252 {
253  return rte_mbuf_buf_addr(mb, mb->pool) + RTE_PKTMBUF_HEADROOM;
254 }
255 
269 static inline char *
271 {
272  return rte_mbuf_buf_addr(md, md->pool);
273 }
274 
287 static inline void *
289 {
290  return RTE_PTR_ADD(m, sizeof(struct rte_mbuf));
291 }
292 
301  uint16_t mbuf_priv_size;
302  uint32_t flags;
303 };
304 
313 static inline uint32_t
315 {
316  struct rte_pktmbuf_pool_private *mbp_priv;
317 
318  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
319  return mbp_priv->flags;
320 }
321 
328 #define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF (1 << 0)
329 
337 #define RTE_MBUF_HAS_PINNED_EXTBUF(mb) \
338  (rte_pktmbuf_priv_flags(mb->pool) & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF)
339 
340 #ifdef RTE_LIBRTE_MBUF_DEBUG
341 
343 #define __rte_mbuf_sanity_check(m, is_h) rte_mbuf_sanity_check(m, is_h)
344 
345 #else /* RTE_LIBRTE_MBUF_DEBUG */
346 
348 #define __rte_mbuf_sanity_check(m, is_h) do { } while (0)
349 
350 #endif /* RTE_LIBRTE_MBUF_DEBUG */
351 
352 #ifdef RTE_MBUF_REFCNT_ATOMIC
353 
361 static inline uint16_t
362 rte_mbuf_refcnt_read(const struct rte_mbuf *m)
363 {
364  return __atomic_load_n(&m->refcnt, __ATOMIC_RELAXED);
365 }
366 
374 static inline void
375 rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
376 {
377  __atomic_store_n(&m->refcnt, new_value, __ATOMIC_RELAXED);
378 }
379 
380 /* internal */
381 static inline uint16_t
382 __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
383 {
384  return __atomic_add_fetch(&m->refcnt, (uint16_t)value,
385  __ATOMIC_ACQ_REL);
386 }
387 
397 static inline uint16_t
398 rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
399 {
400  /*
401  * The atomic_add is an expensive operation, so we don't want to
402  * call it in the case where we know we are the unique holder of
403  * this mbuf (i.e. ref_cnt == 1). Otherwise, an atomic
404  * operation has to be used because concurrent accesses on the
405  * reference counter can occur.
406  */
407  if (likely(rte_mbuf_refcnt_read(m) == 1)) {
408  ++value;
409  rte_mbuf_refcnt_set(m, (uint16_t)value);
410  return (uint16_t)value;
411  }
412 
413  return __rte_mbuf_refcnt_update(m, value);
414 }
415 
416 #else /* ! RTE_MBUF_REFCNT_ATOMIC */
417 
418 /* internal */
419 static inline uint16_t
420 __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
421 {
422  m->refcnt = (uint16_t)(m->refcnt + value);
423  return m->refcnt;
424 }
425 
429 static inline uint16_t
430 rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
431 {
432  return __rte_mbuf_refcnt_update(m, value);
433 }
434 
438 static inline uint16_t
440 {
441  return m->refcnt;
442 }
443 
447 static inline void
448 rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
449 {
450  m->refcnt = new_value;
451 }
452 
453 #endif /* RTE_MBUF_REFCNT_ATOMIC */
454 
463 static inline uint16_t
465 {
466  return __atomic_load_n(&shinfo->refcnt, __ATOMIC_RELAXED);
467 }
468 
477 static inline void
479  uint16_t new_value)
480 {
481  __atomic_store_n(&shinfo->refcnt, new_value, __ATOMIC_RELAXED);
482 }
483 
495 static inline uint16_t
497  int16_t value)
498 {
499  if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1)) {
500  ++value;
501  rte_mbuf_ext_refcnt_set(shinfo, (uint16_t)value);
502  return (uint16_t)value;
503  }
504 
505  return __atomic_add_fetch(&shinfo->refcnt, (uint16_t)value,
506  __ATOMIC_ACQ_REL);
507 }
508 
510 #define RTE_MBUF_PREFETCH_TO_FREE(m) do { \
511  if ((m) != NULL) \
512  rte_prefetch0(m); \
513 } while (0)
514 
515 
528 void
529 rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header);
530 
550 int rte_mbuf_check(const struct rte_mbuf *m, int is_header,
551  const char **reason);
552 
565 static __rte_always_inline void
567 {
568  RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);
569  RTE_ASSERT(m->next == NULL);
570  RTE_ASSERT(m->nb_segs == 1);
572 }
573 
575 #define MBUF_RAW_ALLOC_CHECK(m) __rte_mbuf_raw_sanity_check(m)
576 
596 static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
597 {
598  struct rte_mbuf *m;
599 
600  if (rte_mempool_get(mp, (void **)&m) < 0)
601  return NULL;
603  return m;
604 }
605 
620 static __rte_always_inline void
622 {
623  RTE_ASSERT(!RTE_MBUF_CLONED(m) &&
626  rte_mempool_put(m->pool, m);
627 }
628 
651 void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg,
652  void *m, unsigned i);
653 
674 void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg);
675 
709 struct rte_mempool *
710 rte_pktmbuf_pool_create(const char *name, unsigned n,
711  unsigned cache_size, uint16_t priv_size, uint16_t data_room_size,
712  int socket_id);
713 
750 struct rte_mempool *
751 rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n,
752  unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size,
753  int socket_id, const char *ops_name);
754 
757  void *buf_ptr;
759  size_t buf_len;
760  uint16_t elt_size;
761 };
762 
803 __rte_experimental
804 struct rte_mempool *
805 rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n,
806  unsigned int cache_size, uint16_t priv_size,
807  uint16_t data_room_size, int socket_id,
808  const struct rte_pktmbuf_extmem *ext_mem,
809  unsigned int ext_num);
810 
822 static inline uint16_t
824 {
825  struct rte_pktmbuf_pool_private *mbp_priv;
826 
827  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
828  return mbp_priv->mbuf_data_room_size;
829 }
830 
843 static inline uint16_t
845 {
846  struct rte_pktmbuf_pool_private *mbp_priv;
847 
848  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
849  return mbp_priv->mbuf_priv_size;
850 }
851 
860 static inline void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
861 {
862  m->data_off = (uint16_t)RTE_MIN((uint16_t)RTE_PKTMBUF_HEADROOM,
863  (uint16_t)m->buf_len);
864 }
865 
874 static inline void rte_pktmbuf_reset(struct rte_mbuf *m)
875 {
876  m->next = NULL;
877  m->pkt_len = 0;
878  m->tx_offload = 0;
879  m->vlan_tci = 0;
880  m->vlan_tci_outer = 0;
881  m->nb_segs = 1;
883 
885  m->packet_type = 0;
887 
888  m->data_len = 0;
890 }
891 
905 static inline struct rte_mbuf *rte_pktmbuf_alloc(struct rte_mempool *mp)
906 {
907  struct rte_mbuf *m;
908  if ((m = rte_mbuf_raw_alloc(mp)) != NULL)
910  return m;
911 }
912 
927 static inline int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool,
928  struct rte_mbuf **mbufs, unsigned count)
929 {
930  unsigned idx = 0;
931  int rc;
932 
933  rc = rte_mempool_get_bulk(pool, (void **)mbufs, count);
934  if (unlikely(rc))
935  return rc;
936 
937  /* To understand duff's device on loop unwinding optimization, see
938  * https://en.wikipedia.org/wiki/Duff's_device.
939  * Here while() loop is used rather than do() while{} to avoid extra
940  * check if count is zero.
941  */
942  switch (count % 4) {
943  case 0:
944  while (idx != count) {
945  __rte_mbuf_raw_sanity_check(mbufs[idx]);
946  rte_pktmbuf_reset(mbufs[idx]);
947  idx++;
948  /* fall-through */
949  case 3:
950  __rte_mbuf_raw_sanity_check(mbufs[idx]);
951  rte_pktmbuf_reset(mbufs[idx]);
952  idx++;
953  /* fall-through */
954  case 2:
955  __rte_mbuf_raw_sanity_check(mbufs[idx]);
956  rte_pktmbuf_reset(mbufs[idx]);
957  idx++;
958  /* fall-through */
959  case 1:
960  __rte_mbuf_raw_sanity_check(mbufs[idx]);
961  rte_pktmbuf_reset(mbufs[idx]);
962  idx++;
963  /* fall-through */
964  }
965  }
966  return 0;
967 }
968 
1001 static inline struct rte_mbuf_ext_shared_info *
1002 rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len,
1004 {
1005  struct rte_mbuf_ext_shared_info *shinfo;
1006  void *buf_end = RTE_PTR_ADD(buf_addr, *buf_len);
1007  void *addr;
1008 
1009  addr = RTE_PTR_ALIGN_FLOOR(RTE_PTR_SUB(buf_end, sizeof(*shinfo)),
1010  sizeof(uintptr_t));
1011  if (addr <= buf_addr)
1012  return NULL;
1013 
1014  shinfo = (struct rte_mbuf_ext_shared_info *)addr;
1015  shinfo->free_cb = free_cb;
1016  shinfo->fcb_opaque = fcb_opaque;
1017  rte_mbuf_ext_refcnt_set(shinfo, 1);
1018 
1019  *buf_len = (uint16_t)RTE_PTR_DIFF(shinfo, buf_addr);
1020  return shinfo;
1021 }
1022 
1083 static inline void
1084 rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr,
1085  rte_iova_t buf_iova, uint16_t buf_len,
1086  struct rte_mbuf_ext_shared_info *shinfo)
1087 {
1088  /* mbuf should not be read-only */
1089  RTE_ASSERT(RTE_MBUF_DIRECT(m) && rte_mbuf_refcnt_read(m) == 1);
1090  RTE_ASSERT(shinfo->free_cb != NULL);
1091 
1092  m->buf_addr = buf_addr;
1093  rte_mbuf_iova_set(m, buf_iova);
1094  m->buf_len = buf_len;
1095 
1096  m->data_len = 0;
1097  m->data_off = 0;
1098 
1100  m->shinfo = shinfo;
1101 }
1102 
1110 #define rte_pktmbuf_detach_extbuf(m) rte_pktmbuf_detach(m)
1111 
1120 static inline void
1121 rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
1122 {
1123  memcpy(&mdst->dynfield1, msrc->dynfield1, sizeof(mdst->dynfield1));
1124 }
1125 
1126 /* internal */
1127 static inline void
1128 __rte_pktmbuf_copy_hdr(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
1129 {
1130  mdst->port = msrc->port;
1131  mdst->vlan_tci = msrc->vlan_tci;
1132  mdst->vlan_tci_outer = msrc->vlan_tci_outer;
1133  mdst->tx_offload = msrc->tx_offload;
1134  mdst->hash = msrc->hash;
1135  mdst->packet_type = msrc->packet_type;
1136  rte_mbuf_dynfield_copy(mdst, msrc);
1137 }
1138 
1160 static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
1161 {
1162  RTE_ASSERT(RTE_MBUF_DIRECT(mi) &&
1163  rte_mbuf_refcnt_read(mi) == 1);
1164 
1165  if (RTE_MBUF_HAS_EXTBUF(m)) {
1167  mi->ol_flags = m->ol_flags;
1168  mi->shinfo = m->shinfo;
1169  } else {
1170  /* if m is not direct, get the mbuf that embeds the data */
1172  mi->priv_size = m->priv_size;
1174  }
1175 
1176  __rte_pktmbuf_copy_hdr(mi, m);
1177 
1178  mi->data_off = m->data_off;
1179  mi->data_len = m->data_len;
1181  mi->buf_addr = m->buf_addr;
1182  mi->buf_len = m->buf_len;
1183 
1184  mi->next = NULL;
1185  mi->pkt_len = mi->data_len;
1186  mi->nb_segs = 1;
1187 
1188  __rte_mbuf_sanity_check(mi, 1);
1190 }
1191 
1199 static inline void
1200 __rte_pktmbuf_free_extbuf(struct rte_mbuf *m)
1201 {
1202  RTE_ASSERT(RTE_MBUF_HAS_EXTBUF(m));
1203  RTE_ASSERT(m->shinfo != NULL);
1204 
1205  if (rte_mbuf_ext_refcnt_update(m->shinfo, -1) == 0)
1206  m->shinfo->free_cb(m->buf_addr, m->shinfo->fcb_opaque);
1207 }
1208 
1215 static inline void
1216 __rte_pktmbuf_free_direct(struct rte_mbuf *m)
1217 {
1218  struct rte_mbuf *md;
1219 
1220  RTE_ASSERT(RTE_MBUF_CLONED(m));
1221 
1222  md = rte_mbuf_from_indirect(m);
1223 
1224  if (rte_mbuf_refcnt_update(md, -1) == 0) {
1225  md->next = NULL;
1226  md->nb_segs = 1;
1227  rte_mbuf_refcnt_set(md, 1);
1228  rte_mbuf_raw_free(md);
1229  }
1230 }
1231 
1250 static inline void rte_pktmbuf_detach(struct rte_mbuf *m)
1251 {
1252  struct rte_mempool *mp = m->pool;
1253  uint32_t mbuf_size, buf_len;
1254  uint16_t priv_size;
1255 
1256  if (RTE_MBUF_HAS_EXTBUF(m)) {
1257  /*
1258  * The mbuf has the external attached buffer,
1259  * we should check the type of the memory pool where
1260  * the mbuf was allocated from to detect the pinned
1261  * external buffer.
1262  */
1263  uint32_t flags = rte_pktmbuf_priv_flags(mp);
1264 
1265  if (flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) {
1266  /*
1267  * The pinned external buffer should not be
1268  * detached from its backing mbuf, just exit.
1269  */
1270  return;
1271  }
1272  __rte_pktmbuf_free_extbuf(m);
1273  } else {
1274  __rte_pktmbuf_free_direct(m);
1275  }
1276  priv_size = rte_pktmbuf_priv_size(mp);
1277  mbuf_size = (uint32_t)(sizeof(struct rte_mbuf) + priv_size);
1278  buf_len = rte_pktmbuf_data_room_size(mp);
1279 
1280  m->priv_size = priv_size;
1281  m->buf_addr = (char *)m + mbuf_size;
1282  rte_mbuf_iova_set(m, rte_mempool_virt2iova(m) + mbuf_size);
1283  m->buf_len = (uint16_t)buf_len;
1285  m->data_len = 0;
1286  m->ol_flags = 0;
1287 }
1288 
1302 static inline int __rte_pktmbuf_pinned_extbuf_decref(struct rte_mbuf *m)
1303 {
1304  struct rte_mbuf_ext_shared_info *shinfo;
1305 
1306  /* Clear flags, mbuf is being freed. */
1308  shinfo = m->shinfo;
1309 
1310  /* Optimize for performance - do not dec/reinit */
1311  if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1))
1312  return 0;
1313 
1314  /*
1315  * Direct usage of add primitive to avoid
1316  * duplication of comparing with one.
1317  */
1318  if (likely(__atomic_add_fetch(&shinfo->refcnt, (uint16_t)-1,
1319  __ATOMIC_ACQ_REL)))
1320  return 1;
1321 
1322  /* Reinitialize counter before mbuf freeing. */
1323  rte_mbuf_ext_refcnt_set(shinfo, 1);
1324  return 0;
1325 }
1326 
1341 static __rte_always_inline struct rte_mbuf *
1343 {
1345 
1346  if (likely(rte_mbuf_refcnt_read(m) == 1)) {
1347 
1348  if (!RTE_MBUF_DIRECT(m)) {
1349  rte_pktmbuf_detach(m);
1350  if (RTE_MBUF_HAS_EXTBUF(m) &&
1352  __rte_pktmbuf_pinned_extbuf_decref(m))
1353  return NULL;
1354  }
1355 
1356  if (m->next != NULL)
1357  m->next = NULL;
1358  if (m->nb_segs != 1)
1359  m->nb_segs = 1;
1360 
1361  return m;
1362 
1363  } else if (__rte_mbuf_refcnt_update(m, -1) == 0) {
1364 
1365  if (!RTE_MBUF_DIRECT(m)) {
1366  rte_pktmbuf_detach(m);
1367  if (RTE_MBUF_HAS_EXTBUF(m) &&
1369  __rte_pktmbuf_pinned_extbuf_decref(m))
1370  return NULL;
1371  }
1372 
1373  if (m->next != NULL)
1374  m->next = NULL;
1375  if (m->nb_segs != 1)
1376  m->nb_segs = 1;
1377  rte_mbuf_refcnt_set(m, 1);
1378 
1379  return m;
1380  }
1381  return NULL;
1382 }
1383 
1393 static __rte_always_inline void
1395 {
1396  m = rte_pktmbuf_prefree_seg(m);
1397  if (likely(m != NULL))
1398  rte_mbuf_raw_free(m);
1399 }
1400 
1410 static inline void rte_pktmbuf_free(struct rte_mbuf *m)
1411 {
1412  struct rte_mbuf *m_next;
1413 
1414  if (m != NULL)
1416 
1417  while (m != NULL) {
1418  m_next = m->next;
1420  m = m_next;
1421  }
1422 }
1423 
1436 void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count);
1437 
1455 struct rte_mbuf *
1456 rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp);
1457 
1479 struct rte_mbuf *
1480 rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp,
1481  uint32_t offset, uint32_t length);
1482 
1494 static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
1495 {
1497 
1498  do {
1499  rte_mbuf_refcnt_update(m, v);
1500  } while ((m = m->next) != NULL);
1501 }
1502 
1511 static inline uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
1512 {
1514  return m->data_off;
1515 }
1516 
1525 static inline uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
1526 {
1528  return (uint16_t)(m->buf_len - rte_pktmbuf_headroom(m) -
1529  m->data_len);
1530 }
1531 
1540 static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m)
1541 {
1543  while (m->next != NULL)
1544  m = m->next;
1545  return m;
1546 }
1547 
1556 #define rte_pktmbuf_pkt_len(m) ((m)->pkt_len)
1557 
1566 #define rte_pktmbuf_data_len(m) ((m)->data_len)
1567 
1583 static inline char *rte_pktmbuf_prepend(struct rte_mbuf *m,
1584  uint16_t len)
1585 {
1587 
1588  if (unlikely(len > rte_pktmbuf_headroom(m)))
1589  return NULL;
1590 
1591  /* NB: elaborating the subtraction like this instead of using
1592  * -= allows us to ensure the result type is uint16_t
1593  * avoiding compiler warnings on gcc 8.1 at least */
1594  m->data_off = (uint16_t)(m->data_off - len);
1595  m->data_len = (uint16_t)(m->data_len + len);
1596  m->pkt_len = (m->pkt_len + len);
1597 
1598  return (char *)m->buf_addr + m->data_off;
1599 }
1600 
1616 static inline char *rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
1617 {
1618  void *tail;
1619  struct rte_mbuf *m_last;
1620 
1622 
1623  m_last = rte_pktmbuf_lastseg(m);
1624  if (unlikely(len > rte_pktmbuf_tailroom(m_last)))
1625  return NULL;
1626 
1627  tail = (char *)m_last->buf_addr + m_last->data_off + m_last->data_len;
1628  m_last->data_len = (uint16_t)(m_last->data_len + len);
1629  m->pkt_len = (m->pkt_len + len);
1630  return (char*) tail;
1631 }
1632 
1647 static inline char *rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
1648 {
1650 
1651  if (unlikely(len > m->data_len))
1652  return NULL;
1653 
1654  /* NB: elaborating the addition like this instead of using
1655  * += allows us to ensure the result type is uint16_t
1656  * avoiding compiler warnings on gcc 8.1 at least */
1657  m->data_len = (uint16_t)(m->data_len - len);
1658  m->data_off = (uint16_t)(m->data_off + len);
1659  m->pkt_len = (m->pkt_len - len);
1660  return (char *)m->buf_addr + m->data_off;
1661 }
1662 
1677 static inline int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
1678 {
1679  struct rte_mbuf *m_last;
1680 
1682 
1683  m_last = rte_pktmbuf_lastseg(m);
1684  if (unlikely(len > m_last->data_len))
1685  return -1;
1686 
1687  m_last->data_len = (uint16_t)(m_last->data_len - len);
1688  m->pkt_len = (m->pkt_len - len);
1689  return 0;
1690 }
1691 
1701 static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
1702 {
1704  return m->nb_segs == 1;
1705 }
1706 
1710 const void *__rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off,
1711  uint32_t len, void *buf);
1712 
1733 static inline const void *rte_pktmbuf_read(const struct rte_mbuf *m,
1734  uint32_t off, uint32_t len, void *buf)
1735 {
1736  if (likely(off + len <= rte_pktmbuf_data_len(m)))
1737  return rte_pktmbuf_mtod_offset(m, char *, off);
1738  else
1739  return __rte_pktmbuf_read(m, off, len, buf);
1740 }
1741 
1758 static inline int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
1759 {
1760  struct rte_mbuf *cur_tail;
1761 
1762  /* Check for number-of-segments-overflow */
1763  if (head->nb_segs + tail->nb_segs > RTE_MBUF_MAX_NB_SEGS)
1764  return -EOVERFLOW;
1765 
1766  /* Chain 'tail' onto the old tail */
1767  cur_tail = rte_pktmbuf_lastseg(head);
1768  cur_tail->next = tail;
1769 
1770  /* accumulate number of segments and total length.
1771  * NB: elaborating the addition like this instead of using
1772  * -= allows us to ensure the result type is uint16_t
1773  * avoiding compiler warnings on gcc 8.1 at least */
1774  head->nb_segs = (uint16_t)(head->nb_segs + tail->nb_segs);
1775  head->pkt_len += tail->pkt_len;
1776 
1777  /* pkt_len is only set in the head */
1778  tail->pkt_len = tail->data_len;
1779 
1780  return 0;
1781 }
1782 
1804 static __rte_always_inline uint64_t
1805 rte_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso,
1806  uint64_t ol3, uint64_t ol2, uint64_t unused)
1807 {
1808  return il2 << RTE_MBUF_L2_LEN_OFS |
1809  il3 << RTE_MBUF_L3_LEN_OFS |
1810  il4 << RTE_MBUF_L4_LEN_OFS |
1811  tso << RTE_MBUF_TSO_SEGSZ_OFS |
1812  ol3 << RTE_MBUF_OUTL3_LEN_OFS |
1813  ol2 << RTE_MBUF_OUTL2_LEN_OFS |
1814  unused << RTE_MBUF_TXOFLD_UNUSED_OFS;
1815 }
1816 
1827 static inline int
1829 {
1830  uint64_t ol_flags = m->ol_flags;
1831 
1832  /* Does packet set any of available offloads? */
1833  if (!(ol_flags & RTE_MBUF_F_TX_OFFLOAD_MASK))
1834  return 0;
1835 
1836  /* IP checksum can be counted only for IPv4 packet */
1837  if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) && (ol_flags & RTE_MBUF_F_TX_IPV6))
1838  return -EINVAL;
1839 
1840  /* IP type not set when required */
1841  if (ol_flags & (RTE_MBUF_F_TX_L4_MASK | RTE_MBUF_F_TX_TCP_SEG))
1842  if (!(ol_flags & (RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IPV6)))
1843  return -EINVAL;
1844 
1845  /* Check requirements for TSO packet */
1846  if (ol_flags & RTE_MBUF_F_TX_TCP_SEG)
1847  if ((m->tso_segsz == 0) ||
1848  ((ol_flags & RTE_MBUF_F_TX_IPV4) &&
1849  !(ol_flags & RTE_MBUF_F_TX_IP_CKSUM)))
1850  return -EINVAL;
1851 
1852  /* RTE_MBUF_F_TX_OUTER_IP_CKSUM set for non outer IPv4 packet. */
1853  if ((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) &&
1854  !(ol_flags & RTE_MBUF_F_TX_OUTER_IPV4))
1855  return -EINVAL;
1856 
1857  return 0;
1858 }
1859 
1863 int __rte_pktmbuf_linearize(struct rte_mbuf *mbuf);
1864 
1877 static inline int
1879 {
1880  if (rte_pktmbuf_is_contiguous(mbuf))
1881  return 0;
1882  return __rte_pktmbuf_linearize(mbuf);
1883 }
1884 
1899 void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len);
1900 
1904 static inline uint32_t
1906 {
1907  return m->hash.sched.queue_id;
1908 }
1909 
1913 static inline uint8_t
1915 {
1916  return m->hash.sched.traffic_class;
1917 }
1918 
1922 static inline uint8_t
1924 {
1925  return m->hash.sched.color;
1926 }
1927 
1940 static inline void
1941 rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id,
1942  uint8_t *traffic_class,
1943  uint8_t *color)
1944 {
1945  struct rte_mbuf_sched sched = m->hash.sched;
1946 
1947  *queue_id = sched.queue_id;
1948  *traffic_class = sched.traffic_class;
1949  *color = sched.color;
1950 }
1951 
1955 static inline void
1957 {
1958  m->hash.sched.queue_id = queue_id;
1959 }
1960 
1964 static inline void
1966 {
1967  m->hash.sched.traffic_class = traffic_class;
1968 }
1969 
1973 static inline void
1975 {
1976  m->hash.sched.color = color;
1977 }
1978 
1991 static inline void
1993  uint8_t traffic_class,
1994  uint8_t color)
1995 {
1996  m->hash.sched = (struct rte_mbuf_sched){
1997  .queue_id = queue_id,
1998  .traffic_class = traffic_class,
1999  .color = color,
2000  .reserved = 0,
2001  };
2002 }
2003 
2004 #ifdef __cplusplus
2005 }
2006 #endif
2007 
2008 #endif /* _RTE_MBUF_H_ */
static void rte_pktmbuf_reset(struct rte_mbuf *m)
Definition: rte_mbuf.h:874
struct rte_mbuf_ext_shared_info * shinfo
static rte_iova_t rte_mbuf_data_iova(const struct rte_mbuf *mb)
Definition: rte_mbuf.h:184
struct rte_mbuf * next
uint16_t mbuf_data_room_size
Definition: rte_mbuf.h:300
void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count)
uint16_t vlan_tci_outer
#define __rte_always_inline
Definition: rte_common.h:255
#define RTE_MBUF_F_TX_OFFLOAD_MASK
static struct rte_mbuf * rte_pktmbuf_alloc(struct rte_mempool *mp)
Definition: rte_mbuf.h:905
static void rte_mbuf_sched_traffic_class_set(struct rte_mbuf *m, uint8_t traffic_class)
Definition: rte_mbuf.h:1965
static uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp)
Definition: rte_mbuf.h:844
static int rte_validate_tx_offload(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1828
uint32_t queue_id
#define likely(x)
static void rte_pktmbuf_free(struct rte_mbuf *m)
Definition: rte_mbuf.h:1410
static uint32_t rte_mbuf_sched_queue_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1905
#define RTE_PTR_ALIGN_FLOOR(ptr, align)
Definition: rte_common.h:319
void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg)
uint64_t rte_iova_t
Definition: rte_common.h:458
static __rte_always_inline void rte_pktmbuf_free_seg(struct rte_mbuf *m)
Definition: rte_mbuf.h:1394
void * buf_addr
static struct rte_mbuf * rte_mbuf_from_indirect(struct rte_mbuf *mi)
Definition: rte_mbuf.h:216
uint16_t data_len
rte_mbuf_extbuf_free_callback_t free_cb
static int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
Definition: rte_mbuf.h:1758
static uint8_t rte_mbuf_sched_traffic_class_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1914
#define __rte_unused
Definition: rte_common.h:120
static void rte_mbuf_iova_set(struct rte_mbuf *m, rte_iova_t iova)
Definition: rte_mbuf.h:165
#define RTE_MBUF_F_TX_IPV6
uint64_t tso_segsz
struct rte_mbuf * rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp, uint32_t offset, uint32_t length)
static __rte_always_inline void __rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m)
Definition: rte_mbuf.h:566
void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len)
static uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1511
static int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, struct rte_mbuf **mbufs, unsigned count)
Definition: rte_mbuf.h:927
static void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
Definition: rte_mbuf.h:860
static void * rte_mbuf_to_priv(struct rte_mbuf *m)
Definition: rte_mbuf.h:288
uint32_t cache_size
Definition: rte_mempool.h:231
static void rte_mbuf_prefetch_part2(struct rte_mbuf *m)
Definition: rte_mbuf.h:126
char name[RTE_MEMPOOL_NAMESIZE]
Definition: rte_mempool.h:220
static uint16_t rte_mbuf_ext_refcnt_update(struct rte_mbuf_ext_shared_info *shinfo, int16_t value)
Definition: rte_mbuf.h:496
uint16_t nb_segs
uint16_t port
int rte_mbuf_check(const struct rte_mbuf *m, int is_header, const char **reason)
#define rte_pktmbuf_mtod_offset(m, t, o)
static __rte_always_inline struct rte_mbuf * rte_pktmbuf_prefree_seg(struct rte_mbuf *m)
Definition: rte_mbuf.h:1342
static int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1701
#define RTE_PTR_ADD(ptr, x)
Definition: rte_common.h:290
int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen)
#define RTE_MBUF_MAX_NB_SEGS
static void rte_mbuf_sched_set(struct rte_mbuf *m, uint32_t queue_id, uint8_t traffic_class, uint8_t color)
Definition: rte_mbuf.h:1992
static uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1525
static __rte_always_inline void rte_mbuf_raw_free(struct rte_mbuf *m)
Definition: rte_mbuf.h:621
static void rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id, uint8_t *traffic_class, uint8_t *color)
Definition: rte_mbuf.h:1941
#define unlikely(x)
uint16_t priv_size
static __rte_always_inline uint64_t rte_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso, uint64_t ol3, uint64_t ol2, uint64_t unused)
Definition: rte_mbuf.h:1805
static void rte_mbuf_sched_queue_set(struct rte_mbuf *m, uint32_t queue_id)
Definition: rte_mbuf.h:1956
static char * rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
Definition: rte_mbuf.h:237
void rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header)
static void rte_mbuf_sched_color_set(struct rte_mbuf *m, uint8_t color)
Definition: rte_mbuf.h:1974
void(* rte_mbuf_extbuf_free_callback_t)(void *addr, void *opaque)
#define RTE_MIN(a, b)
Definition: rte_common.h:613
#define __rte_mbuf_sanity_check(m, is_h)
Definition: rte_mbuf.h:348
const char * rte_get_tx_ol_flag_name(uint64_t mask)
static uint16_t rte_mbuf_refcnt_read(const struct rte_mbuf *m)
Definition: rte_mbuf.h:439
static int rte_pktmbuf_linearize(struct rte_mbuf *mbuf)
Definition: rte_mbuf.h:1878
static __rte_always_inline int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
Definition: rte_mempool.h:1663
static uint32_t rte_pktmbuf_priv_flags(struct rte_mempool *mp)
Definition: rte_mbuf.h:314
uint16_t elt_size
Definition: rte_mbuf.h:760
uint16_t refcnt
static char * rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1647
#define RTE_MBUF_DIRECT(mb)
#define RTE_MBUF_CLONED(mb)
static void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
Definition: rte_mbuf.h:1160
static __rte_always_inline int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:1634
#define RTE_MBUF_F_TX_OUTER_IP_CKSUM
uint64_t ol_flags
static void rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
Definition: rte_mbuf.h:1121
static void rte_pktmbuf_detach(struct rte_mbuf *m)
Definition: rte_mbuf.h:1250
static uint16_t rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
Definition: rte_mbuf.h:464
uint32_t pkt_len
uint16_t buf_len
#define rte_pktmbuf_data_len(m)
Definition: rte_mbuf.h:1566
#define RTE_MBUF_F_TX_L4_MASK
static uint16_t rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
Definition: rte_mbuf.h:430
uint32_t packet_type
#define RTE_MBUF_F_TX_IP_CKSUM
static uint16_t rte_pktmbuf_data_room_size(struct rte_mempool *mp)
Definition: rte_mbuf.h:823
#define RTE_MBUF_F_TX_TCP_SEG
#define RTE_MBUF_F_INDIRECT
#define RTE_MBUF_PORT_INVALID
const char * rte_get_rx_ol_flag_name(uint64_t mask)
static struct rte_mbuf_ext_shared_info * rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len, rte_mbuf_extbuf_free_callback_t free_cb, void *fcb_opaque)
Definition: rte_mbuf.h:1002
#define RTE_MBUF_F_TX_OUTER_IPV4
void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg, void *m, unsigned i)
#define RTE_MBUF_HAS_EXTBUF(mb)
struct rte_mempool * pool
static void rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo, uint16_t new_value)
Definition: rte_mbuf.h:478
static char * rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1616
static void rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
Definition: rte_mbuf.h:448
struct rte_mempool * rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n, unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id, const char *ops_name)
static rte_iova_t rte_mbuf_data_iova_default(const struct rte_mbuf *mb)
Definition: rte_mbuf.h:202
static int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1677
static char * rte_mbuf_to_baddr(struct rte_mbuf *md)
Definition: rte_mbuf.h:270
static const void * rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off, uint32_t len, void *buf)
Definition: rte_mbuf.h:1733
static char * rte_pktmbuf_prepend(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1583
static struct rte_mbuf * rte_mbuf_raw_alloc(struct rte_mempool *mp)
Definition: rte_mbuf.h:596
static rte_iova_t rte_mbuf_iova_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:147
static void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
Definition: rte_mbuf.h:1494
#define RTE_PTR_SUB(ptr, x)
Definition: rte_common.h:295
static struct rte_mbuf * rte_pktmbuf_lastseg(struct rte_mbuf *m)
Definition: rte_mbuf.h:1540
__rte_experimental struct rte_mempool * rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n, unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id, const struct rte_pktmbuf_extmem *ext_mem, unsigned int ext_num)
static uint8_t rte_mbuf_sched_color_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1923
#define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF
Definition: rte_mbuf.h:328
static char * rte_mbuf_data_addr_default(struct rte_mbuf *mb)
Definition: rte_mbuf.h:251
static rte_iova_t rte_mempool_virt2iova(const void *elt)
Definition: rte_mempool.h:1788
static __rte_always_inline void rte_mempool_put(struct rte_mempool *mp, void *obj)
Definition: rte_mempool.h:1471
rte_iova_t buf_iova
Definition: rte_mbuf.h:758
uint8_t traffic_class
#define RTE_MBUF_F_TX_IPV4
#define RTE_PTR_DIFF(ptr1, ptr2)
Definition: rte_common.h:302
static void rte_mbuf_prefetch_part1(struct rte_mbuf *m)
Definition: rte_mbuf.h:109
#define RTE_MBUF_F_EXTERNAL
static void * rte_mempool_get_priv(struct rte_mempool *mp)
Definition: rte_mempool.h:1816
uint64_t tx_offload
uint16_t vlan_tci
#define RTE_MBUF_HAS_PINNED_EXTBUF(mb)
Definition: rte_mbuf.h:337
struct rte_mbuf * rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp)
#define RTE_SET_USED(x)
Definition: rte_common.h:135
int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen)
static void rte_prefetch0(const volatile void *p)
struct rte_mempool * rte_pktmbuf_pool_create(const char *name, unsigned n, unsigned cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id)
uint32_t dynfield1[9]
static void rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr, rte_iova_t buf_iova, uint16_t buf_len, struct rte_mbuf_ext_shared_info *shinfo)
Definition: rte_mbuf.h:1084