DPDK  22.11.0-rc0
rte_mbuf.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2014 Intel Corporation.
3  * Copyright 2014 6WIND S.A.
4  */
5 
6 #ifndef _RTE_MBUF_H_
7 #define _RTE_MBUF_H_
8 
34 #include <stdint.h>
35 #include <rte_compat.h>
36 #include <rte_common.h>
37 #include <rte_config.h>
38 #include <rte_mempool.h>
39 #include <rte_prefetch.h>
40 #include <rte_branch_prediction.h>
41 #include <rte_mbuf_ptype.h>
42 #include <rte_mbuf_core.h>
43 
44 #ifdef __cplusplus
45 extern "C" {
46 #endif
47 
56 const char *rte_get_rx_ol_flag_name(uint64_t mask);
57 
70 int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
71 
82 const char *rte_get_tx_ol_flag_name(uint64_t mask);
83 
96 int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
97 
108 static inline void
110 {
111  rte_prefetch0(&m->cacheline0);
112 }
113 
125 static inline void
127 {
128 #if RTE_CACHE_LINE_SIZE == 64
129  rte_prefetch0(&m->cacheline1);
130 #else
131  RTE_SET_USED(m);
132 #endif
133 }
134 
135 
136 static inline uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp);
137 
146 static inline rte_iova_t
147 rte_mbuf_data_iova(const struct rte_mbuf *mb)
148 {
149  return mb->buf_iova + mb->data_off;
150 }
151 
164 static inline rte_iova_t
166 {
167  return mb->buf_iova + RTE_PKTMBUF_HEADROOM;
168 }
169 
178 static inline struct rte_mbuf *
180 {
181  return (struct rte_mbuf *)RTE_PTR_SUB(mi->buf_addr, sizeof(*mi) + mi->priv_size);
182 }
183 
199 static inline char *
200 rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
201 {
202  return (char *)mb + sizeof(*mb) + rte_pktmbuf_priv_size(mp);
203 }
204 
213 static inline char *
215 {
216  return rte_mbuf_buf_addr(mb, mb->pool) + RTE_PKTMBUF_HEADROOM;
217 }
218 
232 static inline char *
234 {
235  return rte_mbuf_buf_addr(md, md->pool);
236 }
237 
250 static inline void *
252 {
253  return RTE_PTR_ADD(m, sizeof(struct rte_mbuf));
254 }
255 
264  uint16_t mbuf_priv_size;
265  uint32_t flags;
266 };
267 
276 static inline uint32_t
278 {
279  struct rte_pktmbuf_pool_private *mbp_priv;
280 
281  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
282  return mbp_priv->flags;
283 }
284 
291 #define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF (1 << 0)
292 
300 #define RTE_MBUF_HAS_PINNED_EXTBUF(mb) \
301  (rte_pktmbuf_priv_flags(mb->pool) & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF)
302 
303 #ifdef RTE_LIBRTE_MBUF_DEBUG
304 
306 #define __rte_mbuf_sanity_check(m, is_h) rte_mbuf_sanity_check(m, is_h)
307 
308 #else /* RTE_LIBRTE_MBUF_DEBUG */
309 
311 #define __rte_mbuf_sanity_check(m, is_h) do { } while (0)
312 
313 #endif /* RTE_LIBRTE_MBUF_DEBUG */
314 
315 #ifdef RTE_MBUF_REFCNT_ATOMIC
316 
324 static inline uint16_t
325 rte_mbuf_refcnt_read(const struct rte_mbuf *m)
326 {
327  return __atomic_load_n(&m->refcnt, __ATOMIC_RELAXED);
328 }
329 
337 static inline void
338 rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
339 {
340  __atomic_store_n(&m->refcnt, new_value, __ATOMIC_RELAXED);
341 }
342 
343 /* internal */
344 static inline uint16_t
345 __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
346 {
347  return __atomic_add_fetch(&m->refcnt, (uint16_t)value,
348  __ATOMIC_ACQ_REL);
349 }
350 
360 static inline uint16_t
361 rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
362 {
363  /*
364  * The atomic_add is an expensive operation, so we don't want to
365  * call it in the case where we know we are the unique holder of
366  * this mbuf (i.e. ref_cnt == 1). Otherwise, an atomic
367  * operation has to be used because concurrent accesses on the
368  * reference counter can occur.
369  */
370  if (likely(rte_mbuf_refcnt_read(m) == 1)) {
371  ++value;
372  rte_mbuf_refcnt_set(m, (uint16_t)value);
373  return (uint16_t)value;
374  }
375 
376  return __rte_mbuf_refcnt_update(m, value);
377 }
378 
379 #else /* ! RTE_MBUF_REFCNT_ATOMIC */
380 
381 /* internal */
382 static inline uint16_t
383 __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
384 {
385  m->refcnt = (uint16_t)(m->refcnt + value);
386  return m->refcnt;
387 }
388 
392 static inline uint16_t
393 rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
394 {
395  return __rte_mbuf_refcnt_update(m, value);
396 }
397 
401 static inline uint16_t
403 {
404  return m->refcnt;
405 }
406 
410 static inline void
411 rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
412 {
413  m->refcnt = new_value;
414 }
415 
416 #endif /* RTE_MBUF_REFCNT_ATOMIC */
417 
426 static inline uint16_t
428 {
429  return __atomic_load_n(&shinfo->refcnt, __ATOMIC_RELAXED);
430 }
431 
440 static inline void
442  uint16_t new_value)
443 {
444  __atomic_store_n(&shinfo->refcnt, new_value, __ATOMIC_RELAXED);
445 }
446 
458 static inline uint16_t
460  int16_t value)
461 {
462  if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1)) {
463  ++value;
464  rte_mbuf_ext_refcnt_set(shinfo, (uint16_t)value);
465  return (uint16_t)value;
466  }
467 
468  return __atomic_add_fetch(&shinfo->refcnt, (uint16_t)value,
469  __ATOMIC_ACQ_REL);
470 }
471 
473 #define RTE_MBUF_PREFETCH_TO_FREE(m) do { \
474  if ((m) != NULL) \
475  rte_prefetch0(m); \
476 } while (0)
477 
478 
491 void
492 rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header);
493 
513 int rte_mbuf_check(const struct rte_mbuf *m, int is_header,
514  const char **reason);
515 
528 static __rte_always_inline void
530 {
531  RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);
532  RTE_ASSERT(m->next == NULL);
533  RTE_ASSERT(m->nb_segs == 1);
535 }
536 
538 #define MBUF_RAW_ALLOC_CHECK(m) __rte_mbuf_raw_sanity_check(m)
539 
559 static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
560 {
561  struct rte_mbuf *m;
562 
563  if (rte_mempool_get(mp, (void **)&m) < 0)
564  return NULL;
566  return m;
567 }
568 
583 static __rte_always_inline void
585 {
586  RTE_ASSERT(!RTE_MBUF_CLONED(m) &&
589  rte_mempool_put(m->pool, m);
590 }
591 
614 void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg,
615  void *m, unsigned i);
616 
637 void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg);
638 
673 struct rte_mempool *
674 rte_pktmbuf_pool_create(const char *name, unsigned n,
675  unsigned cache_size, uint16_t priv_size, uint16_t data_room_size,
676  int socket_id);
677 
715 struct rte_mempool *
716 rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n,
717  unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size,
718  int socket_id, const char *ops_name);
719 
722  void *buf_ptr;
724  size_t buf_len;
725  uint16_t elt_size;
726 };
727 
769 __rte_experimental
770 struct rte_mempool *
771 rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n,
772  unsigned int cache_size, uint16_t priv_size,
773  uint16_t data_room_size, int socket_id,
774  const struct rte_pktmbuf_extmem *ext_mem,
775  unsigned int ext_num);
776 
788 static inline uint16_t
790 {
791  struct rte_pktmbuf_pool_private *mbp_priv;
792 
793  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
794  return mbp_priv->mbuf_data_room_size;
795 }
796 
809 static inline uint16_t
811 {
812  struct rte_pktmbuf_pool_private *mbp_priv;
813 
814  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
815  return mbp_priv->mbuf_priv_size;
816 }
817 
826 static inline void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
827 {
828  m->data_off = (uint16_t)RTE_MIN((uint16_t)RTE_PKTMBUF_HEADROOM,
829  (uint16_t)m->buf_len);
830 }
831 
840 static inline void rte_pktmbuf_reset(struct rte_mbuf *m)
841 {
842  m->next = NULL;
843  m->pkt_len = 0;
844  m->tx_offload = 0;
845  m->vlan_tci = 0;
846  m->vlan_tci_outer = 0;
847  m->nb_segs = 1;
849 
851  m->packet_type = 0;
853 
854  m->data_len = 0;
856 }
857 
871 static inline struct rte_mbuf *rte_pktmbuf_alloc(struct rte_mempool *mp)
872 {
873  struct rte_mbuf *m;
874  if ((m = rte_mbuf_raw_alloc(mp)) != NULL)
876  return m;
877 }
878 
893 static inline int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool,
894  struct rte_mbuf **mbufs, unsigned count)
895 {
896  unsigned idx = 0;
897  int rc;
898 
899  rc = rte_mempool_get_bulk(pool, (void **)mbufs, count);
900  if (unlikely(rc))
901  return rc;
902 
903  /* To understand duff's device on loop unwinding optimization, see
904  * https://en.wikipedia.org/wiki/Duff's_device.
905  * Here while() loop is used rather than do() while{} to avoid extra
906  * check if count is zero.
907  */
908  switch (count % 4) {
909  case 0:
910  while (idx != count) {
911  __rte_mbuf_raw_sanity_check(mbufs[idx]);
912  rte_pktmbuf_reset(mbufs[idx]);
913  idx++;
914  /* fall-through */
915  case 3:
916  __rte_mbuf_raw_sanity_check(mbufs[idx]);
917  rte_pktmbuf_reset(mbufs[idx]);
918  idx++;
919  /* fall-through */
920  case 2:
921  __rte_mbuf_raw_sanity_check(mbufs[idx]);
922  rte_pktmbuf_reset(mbufs[idx]);
923  idx++;
924  /* fall-through */
925  case 1:
926  __rte_mbuf_raw_sanity_check(mbufs[idx]);
927  rte_pktmbuf_reset(mbufs[idx]);
928  idx++;
929  /* fall-through */
930  }
931  }
932  return 0;
933 }
934 
967 static inline struct rte_mbuf_ext_shared_info *
968 rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len,
970 {
971  struct rte_mbuf_ext_shared_info *shinfo;
972  void *buf_end = RTE_PTR_ADD(buf_addr, *buf_len);
973  void *addr;
974 
975  addr = RTE_PTR_ALIGN_FLOOR(RTE_PTR_SUB(buf_end, sizeof(*shinfo)),
976  sizeof(uintptr_t));
977  if (addr <= buf_addr)
978  return NULL;
979 
980  shinfo = (struct rte_mbuf_ext_shared_info *)addr;
981  shinfo->free_cb = free_cb;
982  shinfo->fcb_opaque = fcb_opaque;
983  rte_mbuf_ext_refcnt_set(shinfo, 1);
984 
985  *buf_len = (uint16_t)RTE_PTR_DIFF(shinfo, buf_addr);
986  return shinfo;
987 }
988 
1049 static inline void
1050 rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr,
1051  rte_iova_t buf_iova, uint16_t buf_len,
1052  struct rte_mbuf_ext_shared_info *shinfo)
1053 {
1054  /* mbuf should not be read-only */
1055  RTE_ASSERT(RTE_MBUF_DIRECT(m) && rte_mbuf_refcnt_read(m) == 1);
1056  RTE_ASSERT(shinfo->free_cb != NULL);
1057 
1058  m->buf_addr = buf_addr;
1059  m->buf_iova = buf_iova;
1060  m->buf_len = buf_len;
1061 
1062  m->data_len = 0;
1063  m->data_off = 0;
1064 
1066  m->shinfo = shinfo;
1067 }
1068 
1076 #define rte_pktmbuf_detach_extbuf(m) rte_pktmbuf_detach(m)
1077 
1086 static inline void
1087 rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
1088 {
1089  memcpy(&mdst->dynfield1, msrc->dynfield1, sizeof(mdst->dynfield1));
1090 }
1091 
1092 /* internal */
1093 static inline void
1094 __rte_pktmbuf_copy_hdr(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
1095 {
1096  mdst->port = msrc->port;
1097  mdst->vlan_tci = msrc->vlan_tci;
1098  mdst->vlan_tci_outer = msrc->vlan_tci_outer;
1099  mdst->tx_offload = msrc->tx_offload;
1100  mdst->hash = msrc->hash;
1101  mdst->packet_type = msrc->packet_type;
1102  rte_mbuf_dynfield_copy(mdst, msrc);
1103 }
1104 
1126 static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
1127 {
1128  RTE_ASSERT(RTE_MBUF_DIRECT(mi) &&
1129  rte_mbuf_refcnt_read(mi) == 1);
1130 
1131  if (RTE_MBUF_HAS_EXTBUF(m)) {
1133  mi->ol_flags = m->ol_flags;
1134  mi->shinfo = m->shinfo;
1135  } else {
1136  /* if m is not direct, get the mbuf that embeds the data */
1138  mi->priv_size = m->priv_size;
1140  }
1141 
1142  __rte_pktmbuf_copy_hdr(mi, m);
1143 
1144  mi->data_off = m->data_off;
1145  mi->data_len = m->data_len;
1146  mi->buf_iova = m->buf_iova;
1147  mi->buf_addr = m->buf_addr;
1148  mi->buf_len = m->buf_len;
1149 
1150  mi->next = NULL;
1151  mi->pkt_len = mi->data_len;
1152  mi->nb_segs = 1;
1153 
1154  __rte_mbuf_sanity_check(mi, 1);
1156 }
1157 
1165 static inline void
1166 __rte_pktmbuf_free_extbuf(struct rte_mbuf *m)
1167 {
1168  RTE_ASSERT(RTE_MBUF_HAS_EXTBUF(m));
1169  RTE_ASSERT(m->shinfo != NULL);
1170 
1171  if (rte_mbuf_ext_refcnt_update(m->shinfo, -1) == 0)
1172  m->shinfo->free_cb(m->buf_addr, m->shinfo->fcb_opaque);
1173 }
1174 
1181 static inline void
1182 __rte_pktmbuf_free_direct(struct rte_mbuf *m)
1183 {
1184  struct rte_mbuf *md;
1185 
1186  RTE_ASSERT(RTE_MBUF_CLONED(m));
1187 
1188  md = rte_mbuf_from_indirect(m);
1189 
1190  if (rte_mbuf_refcnt_update(md, -1) == 0) {
1191  md->next = NULL;
1192  md->nb_segs = 1;
1193  rte_mbuf_refcnt_set(md, 1);
1194  rte_mbuf_raw_free(md);
1195  }
1196 }
1197 
1216 static inline void rte_pktmbuf_detach(struct rte_mbuf *m)
1217 {
1218  struct rte_mempool *mp = m->pool;
1219  uint32_t mbuf_size, buf_len;
1220  uint16_t priv_size;
1221 
1222  if (RTE_MBUF_HAS_EXTBUF(m)) {
1223  /*
1224  * The mbuf has the external attached buffer,
1225  * we should check the type of the memory pool where
1226  * the mbuf was allocated from to detect the pinned
1227  * external buffer.
1228  */
1229  uint32_t flags = rte_pktmbuf_priv_flags(mp);
1230 
1231  if (flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) {
1232  /*
1233  * The pinned external buffer should not be
1234  * detached from its backing mbuf, just exit.
1235  */
1236  return;
1237  }
1238  __rte_pktmbuf_free_extbuf(m);
1239  } else {
1240  __rte_pktmbuf_free_direct(m);
1241  }
1242  priv_size = rte_pktmbuf_priv_size(mp);
1243  mbuf_size = (uint32_t)(sizeof(struct rte_mbuf) + priv_size);
1244  buf_len = rte_pktmbuf_data_room_size(mp);
1245 
1246  m->priv_size = priv_size;
1247  m->buf_addr = (char *)m + mbuf_size;
1248  m->buf_iova = rte_mempool_virt2iova(m) + mbuf_size;
1249  m->buf_len = (uint16_t)buf_len;
1251  m->data_len = 0;
1252  m->ol_flags = 0;
1253 }
1254 
1268 static inline int __rte_pktmbuf_pinned_extbuf_decref(struct rte_mbuf *m)
1269 {
1270  struct rte_mbuf_ext_shared_info *shinfo;
1271 
1272  /* Clear flags, mbuf is being freed. */
1274  shinfo = m->shinfo;
1275 
1276  /* Optimize for performance - do not dec/reinit */
1277  if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1))
1278  return 0;
1279 
1280  /*
1281  * Direct usage of add primitive to avoid
1282  * duplication of comparing with one.
1283  */
1284  if (likely(__atomic_add_fetch(&shinfo->refcnt, (uint16_t)-1,
1285  __ATOMIC_ACQ_REL)))
1286  return 1;
1287 
1288  /* Reinitialize counter before mbuf freeing. */
1289  rte_mbuf_ext_refcnt_set(shinfo, 1);
1290  return 0;
1291 }
1292 
1307 static __rte_always_inline struct rte_mbuf *
1309 {
1311 
1312  if (likely(rte_mbuf_refcnt_read(m) == 1)) {
1313 
1314  if (!RTE_MBUF_DIRECT(m)) {
1315  rte_pktmbuf_detach(m);
1316  if (RTE_MBUF_HAS_EXTBUF(m) &&
1318  __rte_pktmbuf_pinned_extbuf_decref(m))
1319  return NULL;
1320  }
1321 
1322  if (m->next != NULL)
1323  m->next = NULL;
1324  if (m->nb_segs != 1)
1325  m->nb_segs = 1;
1326 
1327  return m;
1328 
1329  } else if (__rte_mbuf_refcnt_update(m, -1) == 0) {
1330 
1331  if (!RTE_MBUF_DIRECT(m)) {
1332  rte_pktmbuf_detach(m);
1333  if (RTE_MBUF_HAS_EXTBUF(m) &&
1335  __rte_pktmbuf_pinned_extbuf_decref(m))
1336  return NULL;
1337  }
1338 
1339  if (m->next != NULL)
1340  m->next = NULL;
1341  if (m->nb_segs != 1)
1342  m->nb_segs = 1;
1343  rte_mbuf_refcnt_set(m, 1);
1344 
1345  return m;
1346  }
1347  return NULL;
1348 }
1349 
1359 static __rte_always_inline void
1361 {
1362  m = rte_pktmbuf_prefree_seg(m);
1363  if (likely(m != NULL))
1364  rte_mbuf_raw_free(m);
1365 }
1366 
1376 static inline void rte_pktmbuf_free(struct rte_mbuf *m)
1377 {
1378  struct rte_mbuf *m_next;
1379 
1380  if (m != NULL)
1382 
1383  while (m != NULL) {
1384  m_next = m->next;
1386  m = m_next;
1387  }
1388 }
1389 
1402 void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count);
1403 
1421 struct rte_mbuf *
1422 rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp);
1423 
1445 struct rte_mbuf *
1446 rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp,
1447  uint32_t offset, uint32_t length);
1448 
1460 static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
1461 {
1463 
1464  do {
1465  rte_mbuf_refcnt_update(m, v);
1466  } while ((m = m->next) != NULL);
1467 }
1468 
1477 static inline uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
1478 {
1480  return m->data_off;
1481 }
1482 
1491 static inline uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
1492 {
1494  return (uint16_t)(m->buf_len - rte_pktmbuf_headroom(m) -
1495  m->data_len);
1496 }
1497 
1506 static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m)
1507 {
1509  while (m->next != NULL)
1510  m = m->next;
1511  return m;
1512 }
1513 
1522 #define rte_pktmbuf_pkt_len(m) ((m)->pkt_len)
1523 
1532 #define rte_pktmbuf_data_len(m) ((m)->data_len)
1533 
1549 static inline char *rte_pktmbuf_prepend(struct rte_mbuf *m,
1550  uint16_t len)
1551 {
1553 
1554  if (unlikely(len > rte_pktmbuf_headroom(m)))
1555  return NULL;
1556 
1557  /* NB: elaborating the subtraction like this instead of using
1558  * -= allows us to ensure the result type is uint16_t
1559  * avoiding compiler warnings on gcc 8.1 at least */
1560  m->data_off = (uint16_t)(m->data_off - len);
1561  m->data_len = (uint16_t)(m->data_len + len);
1562  m->pkt_len = (m->pkt_len + len);
1563 
1564  return (char *)m->buf_addr + m->data_off;
1565 }
1566 
1582 static inline char *rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
1583 {
1584  void *tail;
1585  struct rte_mbuf *m_last;
1586 
1588 
1589  m_last = rte_pktmbuf_lastseg(m);
1590  if (unlikely(len > rte_pktmbuf_tailroom(m_last)))
1591  return NULL;
1592 
1593  tail = (char *)m_last->buf_addr + m_last->data_off + m_last->data_len;
1594  m_last->data_len = (uint16_t)(m_last->data_len + len);
1595  m->pkt_len = (m->pkt_len + len);
1596  return (char*) tail;
1597 }
1598 
1613 static inline char *rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
1614 {
1616 
1617  if (unlikely(len > m->data_len))
1618  return NULL;
1619 
1620  /* NB: elaborating the addition like this instead of using
1621  * += allows us to ensure the result type is uint16_t
1622  * avoiding compiler warnings on gcc 8.1 at least */
1623  m->data_len = (uint16_t)(m->data_len - len);
1624  m->data_off = (uint16_t)(m->data_off + len);
1625  m->pkt_len = (m->pkt_len - len);
1626  return (char *)m->buf_addr + m->data_off;
1627 }
1628 
1643 static inline int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
1644 {
1645  struct rte_mbuf *m_last;
1646 
1648 
1649  m_last = rte_pktmbuf_lastseg(m);
1650  if (unlikely(len > m_last->data_len))
1651  return -1;
1652 
1653  m_last->data_len = (uint16_t)(m_last->data_len - len);
1654  m->pkt_len = (m->pkt_len - len);
1655  return 0;
1656 }
1657 
1667 static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
1668 {
1670  return m->nb_segs == 1;
1671 }
1672 
1676 const void *__rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off,
1677  uint32_t len, void *buf);
1678 
1699 static inline const void *rte_pktmbuf_read(const struct rte_mbuf *m,
1700  uint32_t off, uint32_t len, void *buf)
1701 {
1702  if (likely(off + len <= rte_pktmbuf_data_len(m)))
1703  return rte_pktmbuf_mtod_offset(m, char *, off);
1704  else
1705  return __rte_pktmbuf_read(m, off, len, buf);
1706 }
1707 
1724 static inline int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
1725 {
1726  struct rte_mbuf *cur_tail;
1727 
1728  /* Check for number-of-segments-overflow */
1729  if (head->nb_segs + tail->nb_segs > RTE_MBUF_MAX_NB_SEGS)
1730  return -EOVERFLOW;
1731 
1732  /* Chain 'tail' onto the old tail */
1733  cur_tail = rte_pktmbuf_lastseg(head);
1734  cur_tail->next = tail;
1735 
1736  /* accumulate number of segments and total length.
1737  * NB: elaborating the addition like this instead of using
1738  * -= allows us to ensure the result type is uint16_t
1739  * avoiding compiler warnings on gcc 8.1 at least */
1740  head->nb_segs = (uint16_t)(head->nb_segs + tail->nb_segs);
1741  head->pkt_len += tail->pkt_len;
1742 
1743  /* pkt_len is only set in the head */
1744  tail->pkt_len = tail->data_len;
1745 
1746  return 0;
1747 }
1748 
1770 static __rte_always_inline uint64_t
1771 rte_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso,
1772  uint64_t ol3, uint64_t ol2, uint64_t unused)
1773 {
1774  return il2 << RTE_MBUF_L2_LEN_OFS |
1775  il3 << RTE_MBUF_L3_LEN_OFS |
1776  il4 << RTE_MBUF_L4_LEN_OFS |
1777  tso << RTE_MBUF_TSO_SEGSZ_OFS |
1778  ol3 << RTE_MBUF_OUTL3_LEN_OFS |
1779  ol2 << RTE_MBUF_OUTL2_LEN_OFS |
1780  unused << RTE_MBUF_TXOFLD_UNUSED_OFS;
1781 }
1782 
1793 static inline int
1795 {
1796  uint64_t ol_flags = m->ol_flags;
1797 
1798  /* Does packet set any of available offloads? */
1799  if (!(ol_flags & RTE_MBUF_F_TX_OFFLOAD_MASK))
1800  return 0;
1801 
1802  /* IP checksum can be counted only for IPv4 packet */
1803  if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) && (ol_flags & RTE_MBUF_F_TX_IPV6))
1804  return -EINVAL;
1805 
1806  /* IP type not set when required */
1807  if (ol_flags & (RTE_MBUF_F_TX_L4_MASK | RTE_MBUF_F_TX_TCP_SEG))
1808  if (!(ol_flags & (RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IPV6)))
1809  return -EINVAL;
1810 
1811  /* Check requirements for TSO packet */
1812  if (ol_flags & RTE_MBUF_F_TX_TCP_SEG)
1813  if ((m->tso_segsz == 0) ||
1814  ((ol_flags & RTE_MBUF_F_TX_IPV4) &&
1815  !(ol_flags & RTE_MBUF_F_TX_IP_CKSUM)))
1816  return -EINVAL;
1817 
1818  /* RTE_MBUF_F_TX_OUTER_IP_CKSUM set for non outer IPv4 packet. */
1819  if ((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) &&
1820  !(ol_flags & RTE_MBUF_F_TX_OUTER_IPV4))
1821  return -EINVAL;
1822 
1823  return 0;
1824 }
1825 
1829 int __rte_pktmbuf_linearize(struct rte_mbuf *mbuf);
1830 
1843 static inline int
1845 {
1846  if (rte_pktmbuf_is_contiguous(mbuf))
1847  return 0;
1848  return __rte_pktmbuf_linearize(mbuf);
1849 }
1850 
1865 void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len);
1866 
1870 static inline uint32_t
1872 {
1873  return m->hash.sched.queue_id;
1874 }
1875 
1879 static inline uint8_t
1881 {
1882  return m->hash.sched.traffic_class;
1883 }
1884 
1888 static inline uint8_t
1890 {
1891  return m->hash.sched.color;
1892 }
1893 
1906 static inline void
1907 rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id,
1908  uint8_t *traffic_class,
1909  uint8_t *color)
1910 {
1911  struct rte_mbuf_sched sched = m->hash.sched;
1912 
1913  *queue_id = sched.queue_id;
1914  *traffic_class = sched.traffic_class;
1915  *color = sched.color;
1916 }
1917 
1921 static inline void
1923 {
1924  m->hash.sched.queue_id = queue_id;
1925 }
1926 
1930 static inline void
1932 {
1933  m->hash.sched.traffic_class = traffic_class;
1934 }
1935 
1939 static inline void
1941 {
1942  m->hash.sched.color = color;
1943 }
1944 
1957 static inline void
1959  uint8_t traffic_class,
1960  uint8_t color)
1961 {
1962  m->hash.sched = (struct rte_mbuf_sched){
1963  .queue_id = queue_id,
1964  .traffic_class = traffic_class,
1965  .color = color,
1966  .reserved = 0,
1967  };
1968 }
1969 
1970 #ifdef __cplusplus
1971 }
1972 #endif
1973 
1974 #endif /* _RTE_MBUF_H_ */
static void rte_pktmbuf_reset(struct rte_mbuf *m)
Definition: rte_mbuf.h:840
struct rte_mbuf_ext_shared_info * shinfo
static rte_iova_t rte_mbuf_data_iova(const struct rte_mbuf *mb)
Definition: rte_mbuf.h:147
struct rte_mbuf * next
uint16_t mbuf_data_room_size
Definition: rte_mbuf.h:263
void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count)
uint16_t vlan_tci_outer
#define __rte_always_inline
Definition: rte_common.h:255
#define RTE_MBUF_F_TX_OFFLOAD_MASK
static struct rte_mbuf * rte_pktmbuf_alloc(struct rte_mempool *mp)
Definition: rte_mbuf.h:871
static void rte_mbuf_sched_traffic_class_set(struct rte_mbuf *m, uint8_t traffic_class)
Definition: rte_mbuf.h:1931
static uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp)
Definition: rte_mbuf.h:810
static int rte_validate_tx_offload(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1794
uint32_t queue_id
#define likely(x)
static void rte_pktmbuf_free(struct rte_mbuf *m)
Definition: rte_mbuf.h:1376
static uint32_t rte_mbuf_sched_queue_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1871
#define RTE_PTR_ALIGN_FLOOR(ptr, align)
Definition: rte_common.h:319
void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg)
uint64_t rte_iova_t
Definition: rte_common.h:458
static __rte_always_inline void rte_pktmbuf_free_seg(struct rte_mbuf *m)
Definition: rte_mbuf.h:1360
void * buf_addr
static struct rte_mbuf * rte_mbuf_from_indirect(struct rte_mbuf *mi)
Definition: rte_mbuf.h:179
uint16_t data_len
rte_mbuf_extbuf_free_callback_t free_cb
static int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
Definition: rte_mbuf.h:1724
static uint8_t rte_mbuf_sched_traffic_class_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1880
#define __rte_unused
Definition: rte_common.h:120
#define RTE_MBUF_F_TX_IPV6
uint64_t tso_segsz
struct rte_mbuf * rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp, uint32_t offset, uint32_t length)
static __rte_always_inline void __rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m)
Definition: rte_mbuf.h:529
void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len)
static uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1477
static int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, struct rte_mbuf **mbufs, unsigned count)
Definition: rte_mbuf.h:893
static void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
Definition: rte_mbuf.h:826
static void * rte_mbuf_to_priv(struct rte_mbuf *m)
Definition: rte_mbuf.h:251
uint32_t cache_size
Definition: rte_mempool.h:214
static void rte_mbuf_prefetch_part2(struct rte_mbuf *m)
Definition: rte_mbuf.h:126
char name[RTE_MEMPOOL_NAMESIZE]
Definition: rte_mempool.h:203
static uint16_t rte_mbuf_ext_refcnt_update(struct rte_mbuf_ext_shared_info *shinfo, int16_t value)
Definition: rte_mbuf.h:459
uint16_t nb_segs
uint16_t port
int rte_mbuf_check(const struct rte_mbuf *m, int is_header, const char **reason)
#define rte_pktmbuf_mtod_offset(m, t, o)
static __rte_always_inline struct rte_mbuf * rte_pktmbuf_prefree_seg(struct rte_mbuf *m)
Definition: rte_mbuf.h:1308
static int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1667
#define RTE_PTR_ADD(ptr, x)
Definition: rte_common.h:290
int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen)
#define RTE_MBUF_MAX_NB_SEGS
static void rte_mbuf_sched_set(struct rte_mbuf *m, uint32_t queue_id, uint8_t traffic_class, uint8_t color)
Definition: rte_mbuf.h:1958
static uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1491
static __rte_always_inline void rte_mbuf_raw_free(struct rte_mbuf *m)
Definition: rte_mbuf.h:584
static void rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id, uint8_t *traffic_class, uint8_t *color)
Definition: rte_mbuf.h:1907
#define unlikely(x)
uint16_t priv_size
static __rte_always_inline uint64_t rte_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso, uint64_t ol3, uint64_t ol2, uint64_t unused)
Definition: rte_mbuf.h:1771
static void rte_mbuf_sched_queue_set(struct rte_mbuf *m, uint32_t queue_id)
Definition: rte_mbuf.h:1922
static char * rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
Definition: rte_mbuf.h:200
void rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header)
static void rte_mbuf_sched_color_set(struct rte_mbuf *m, uint8_t color)
Definition: rte_mbuf.h:1940
void(* rte_mbuf_extbuf_free_callback_t)(void *addr, void *opaque)
#define RTE_MIN(a, b)
Definition: rte_common.h:613
#define __rte_mbuf_sanity_check(m, is_h)
Definition: rte_mbuf.h:311
const char * rte_get_tx_ol_flag_name(uint64_t mask)
static uint16_t rte_mbuf_refcnt_read(const struct rte_mbuf *m)
Definition: rte_mbuf.h:402
static int rte_pktmbuf_linearize(struct rte_mbuf *mbuf)
Definition: rte_mbuf.h:1844
static __rte_always_inline int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
Definition: rte_mempool.h:1587
static uint32_t rte_pktmbuf_priv_flags(struct rte_mempool *mp)
Definition: rte_mbuf.h:277
uint16_t elt_size
Definition: rte_mbuf.h:725
uint16_t refcnt
static char * rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1613
#define RTE_MBUF_DIRECT(mb)
#define RTE_MBUF_CLONED(mb)
static void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
Definition: rte_mbuf.h:1126
static __rte_always_inline int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:1558
#define RTE_MBUF_F_TX_OUTER_IP_CKSUM
uint64_t ol_flags
static void rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
Definition: rte_mbuf.h:1087
static void rte_pktmbuf_detach(struct rte_mbuf *m)
Definition: rte_mbuf.h:1216
static uint16_t rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
Definition: rte_mbuf.h:427
uint32_t pkt_len
uint16_t buf_len
#define rte_pktmbuf_data_len(m)
Definition: rte_mbuf.h:1532
#define RTE_MBUF_F_TX_L4_MASK
static uint16_t rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
Definition: rte_mbuf.h:393
uint32_t packet_type
#define RTE_MBUF_F_TX_IP_CKSUM
static uint16_t rte_pktmbuf_data_room_size(struct rte_mempool *mp)
Definition: rte_mbuf.h:789
#define RTE_MBUF_F_TX_TCP_SEG
#define RTE_MBUF_F_INDIRECT
#define RTE_MBUF_PORT_INVALID
const char * rte_get_rx_ol_flag_name(uint64_t mask)
static struct rte_mbuf_ext_shared_info * rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len, rte_mbuf_extbuf_free_callback_t free_cb, void *fcb_opaque)
Definition: rte_mbuf.h:968
#define RTE_MBUF_F_TX_OUTER_IPV4
void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg, void *m, unsigned i)
#define RTE_MBUF_HAS_EXTBUF(mb)
struct rte_mempool * pool
static void rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo, uint16_t new_value)
Definition: rte_mbuf.h:441
static char * rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1582
static void rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
Definition: rte_mbuf.h:411
struct rte_mempool * rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n, unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id, const char *ops_name)
static rte_iova_t rte_mbuf_data_iova_default(const struct rte_mbuf *mb)
Definition: rte_mbuf.h:165
static int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1643
static char * rte_mbuf_to_baddr(struct rte_mbuf *md)
Definition: rte_mbuf.h:233
static const void * rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off, uint32_t len, void *buf)
Definition: rte_mbuf.h:1699
static char * rte_pktmbuf_prepend(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1549
static struct rte_mbuf * rte_mbuf_raw_alloc(struct rte_mempool *mp)
Definition: rte_mbuf.h:559
static void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
Definition: rte_mbuf.h:1460
#define RTE_PTR_SUB(ptr, x)
Definition: rte_common.h:295
static struct rte_mbuf * rte_pktmbuf_lastseg(struct rte_mbuf *m)
Definition: rte_mbuf.h:1506
__rte_experimental struct rte_mempool * rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n, unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id, const struct rte_pktmbuf_extmem *ext_mem, unsigned int ext_num)
static uint8_t rte_mbuf_sched_color_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1889
#define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF
Definition: rte_mbuf.h:291
static char * rte_mbuf_data_addr_default(struct rte_mbuf *mb)
Definition: rte_mbuf.h:214
static rte_iova_t rte_mempool_virt2iova(const void *elt)
Definition: rte_mempool.h:1712
static __rte_always_inline void rte_mempool_put(struct rte_mempool *mp, void *obj)
Definition: rte_mempool.h:1422
rte_iova_t buf_iova
Definition: rte_mbuf.h:723
uint8_t traffic_class
#define RTE_MBUF_F_TX_IPV4
#define RTE_PTR_DIFF(ptr1, ptr2)
Definition: rte_common.h:302
static void rte_mbuf_prefetch_part1(struct rte_mbuf *m)
Definition: rte_mbuf.h:109
#define RTE_MBUF_F_EXTERNAL
static void * rte_mempool_get_priv(struct rte_mempool *mp)
Definition: rte_mempool.h:1740
uint64_t tx_offload
uint16_t vlan_tci
#define RTE_MBUF_HAS_PINNED_EXTBUF(mb)
Definition: rte_mbuf.h:300
struct rte_mbuf * rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp)
#define RTE_SET_USED(x)
Definition: rte_common.h:135
int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen)
static void rte_prefetch0(const volatile void *p)
struct rte_mempool * rte_pktmbuf_pool_create(const char *name, unsigned n, unsigned cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id)
uint32_t dynfield1[9]
static void rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr, rte_iova_t buf_iova, uint16_t buf_len, struct rte_mbuf_ext_shared_info *shinfo)
Definition: rte_mbuf.h:1050