DPDK  21.11.0-rc0
rte_mbuf.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2014 Intel Corporation.
3  * Copyright 2014 6WIND S.A.
4  */
5 
6 #ifndef _RTE_MBUF_H_
7 #define _RTE_MBUF_H_
8 
34 #include <stdint.h>
35 #include <rte_compat.h>
36 #include <rte_common.h>
37 #include <rte_config.h>
38 #include <rte_mempool.h>
39 #include <rte_memory.h>
40 #include <rte_prefetch.h>
41 #include <rte_branch_prediction.h>
42 #include <rte_byteorder.h>
43 #include <rte_mbuf_ptype.h>
44 #include <rte_mbuf_core.h>
45 
46 #ifdef __cplusplus
47 extern "C" {
48 #endif
49 
58 const char *rte_get_rx_ol_flag_name(uint64_t mask);
59 
72 int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
73 
84 const char *rte_get_tx_ol_flag_name(uint64_t mask);
85 
98 int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
99 
110 static inline void
112 {
113  rte_prefetch0(&m->cacheline0);
114 }
115 
127 static inline void
129 {
130 #if RTE_CACHE_LINE_SIZE == 64
131  rte_prefetch0(&m->cacheline1);
132 #else
133  RTE_SET_USED(m);
134 #endif
135 }
136 
137 
138 static inline uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp);
139 
148 static inline rte_iova_t
149 rte_mbuf_data_iova(const struct rte_mbuf *mb)
150 {
151  return mb->buf_iova + mb->data_off;
152 }
153 
166 static inline rte_iova_t
168 {
169  return mb->buf_iova + RTE_PKTMBUF_HEADROOM;
170 }
171 
180 static inline struct rte_mbuf *
182 {
183  return (struct rte_mbuf *)RTE_PTR_SUB(mi->buf_addr, sizeof(*mi) + mi->priv_size);
184 }
185 
206 __rte_experimental
207 static inline char *
208 rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
209 {
210  return (char *)mb + sizeof(*mb) + rte_pktmbuf_priv_size(mp);
211 }
212 
224 __rte_experimental
225 static inline char *
227 {
228  /* gcc complains about calling this experimental function even
229  * when not using it. Hide it with ALLOW_EXPERIMENTAL_API.
230  */
231 #ifdef ALLOW_EXPERIMENTAL_API
232  return rte_mbuf_buf_addr(mb, mb->pool) + RTE_PKTMBUF_HEADROOM;
233 #else
234  return NULL;
235 #endif
236 }
237 
251 static inline char *
253 {
254 #ifdef ALLOW_EXPERIMENTAL_API
255  return rte_mbuf_buf_addr(md, md->pool);
256 #else
257  char *buffer_addr;
258  buffer_addr = (char *)md + sizeof(*md) + rte_pktmbuf_priv_size(md->pool);
259  return buffer_addr;
260 #endif
261 }
262 
275 __rte_experimental
276 static inline void *
278 {
279  return RTE_PTR_ADD(m, sizeof(struct rte_mbuf));
280 }
281 
290  uint16_t mbuf_priv_size;
291  uint32_t flags;
292 };
293 
302 static inline uint32_t
304 {
305  struct rte_pktmbuf_pool_private *mbp_priv;
306 
307  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
308  return mbp_priv->flags;
309 }
310 
317 #define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF (1 << 0)
318 
326 #define RTE_MBUF_HAS_PINNED_EXTBUF(mb) \
327  (rte_pktmbuf_priv_flags(mb->pool) & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF)
328 
329 #ifdef RTE_LIBRTE_MBUF_DEBUG
330 
332 #define __rte_mbuf_sanity_check(m, is_h) rte_mbuf_sanity_check(m, is_h)
333 
334 #else /* RTE_LIBRTE_MBUF_DEBUG */
335 
337 #define __rte_mbuf_sanity_check(m, is_h) do { } while (0)
338 
339 #endif /* RTE_LIBRTE_MBUF_DEBUG */
340 
341 #ifdef RTE_MBUF_REFCNT_ATOMIC
342 
350 static inline uint16_t
351 rte_mbuf_refcnt_read(const struct rte_mbuf *m)
352 {
353  return __atomic_load_n(&m->refcnt, __ATOMIC_RELAXED);
354 }
355 
363 static inline void
364 rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
365 {
366  __atomic_store_n(&m->refcnt, new_value, __ATOMIC_RELAXED);
367 }
368 
369 /* internal */
370 static inline uint16_t
371 __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
372 {
373  return __atomic_add_fetch(&m->refcnt, (uint16_t)value,
374  __ATOMIC_ACQ_REL);
375 }
376 
386 static inline uint16_t
387 rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
388 {
389  /*
390  * The atomic_add is an expensive operation, so we don't want to
391  * call it in the case where we know we are the unique holder of
392  * this mbuf (i.e. ref_cnt == 1). Otherwise, an atomic
393  * operation has to be used because concurrent accesses on the
394  * reference counter can occur.
395  */
396  if (likely(rte_mbuf_refcnt_read(m) == 1)) {
397  ++value;
398  rte_mbuf_refcnt_set(m, (uint16_t)value);
399  return (uint16_t)value;
400  }
401 
402  return __rte_mbuf_refcnt_update(m, value);
403 }
404 
405 #else /* ! RTE_MBUF_REFCNT_ATOMIC */
406 
407 /* internal */
408 static inline uint16_t
409 __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
410 {
411  m->refcnt = (uint16_t)(m->refcnt + value);
412  return m->refcnt;
413 }
414 
418 static inline uint16_t
419 rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
420 {
421  return __rte_mbuf_refcnt_update(m, value);
422 }
423 
427 static inline uint16_t
429 {
430  return m->refcnt;
431 }
432 
436 static inline void
437 rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
438 {
439  m->refcnt = new_value;
440 }
441 
442 #endif /* RTE_MBUF_REFCNT_ATOMIC */
443 
452 static inline uint16_t
454 {
455  return __atomic_load_n(&shinfo->refcnt, __ATOMIC_RELAXED);
456 }
457 
466 static inline void
468  uint16_t new_value)
469 {
470  __atomic_store_n(&shinfo->refcnt, new_value, __ATOMIC_RELAXED);
471 }
472 
484 static inline uint16_t
486  int16_t value)
487 {
488  if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1)) {
489  ++value;
490  rte_mbuf_ext_refcnt_set(shinfo, (uint16_t)value);
491  return (uint16_t)value;
492  }
493 
494  return __atomic_add_fetch(&shinfo->refcnt, (uint16_t)value,
495  __ATOMIC_ACQ_REL);
496 }
497 
499 #define RTE_MBUF_PREFETCH_TO_FREE(m) do { \
500  if ((m) != NULL) \
501  rte_prefetch0(m); \
502 } while (0)
503 
504 
517 void
518 rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header);
519 
539 __rte_experimental
540 int rte_mbuf_check(const struct rte_mbuf *m, int is_header,
541  const char **reason);
542 
555 static __rte_always_inline void
557 {
558  RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);
559  RTE_ASSERT(m->next == NULL);
560  RTE_ASSERT(m->nb_segs == 1);
562 }
563 
565 #define MBUF_RAW_ALLOC_CHECK(m) __rte_mbuf_raw_sanity_check(m)
566 
586 static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
587 {
588  struct rte_mbuf *m;
589 
590  if (rte_mempool_get(mp, (void **)&m) < 0)
591  return NULL;
593  return m;
594 }
595 
610 static __rte_always_inline void
612 {
613  RTE_ASSERT(!RTE_MBUF_CLONED(m) &&
616  rte_mempool_put(m->pool, m);
617 }
618 
641 void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg,
642  void *m, unsigned i);
643 
664 void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg);
665 
700 struct rte_mempool *
701 rte_pktmbuf_pool_create(const char *name, unsigned n,
702  unsigned cache_size, uint16_t priv_size, uint16_t data_room_size,
703  int socket_id);
704 
742 struct rte_mempool *
743 rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n,
744  unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size,
745  int socket_id, const char *ops_name);
746 
749  void *buf_ptr;
751  size_t buf_len;
752  uint16_t elt_size;
753 };
754 
796 __rte_experimental
797 struct rte_mempool *
798 rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n,
799  unsigned int cache_size, uint16_t priv_size,
800  uint16_t data_room_size, int socket_id,
801  const struct rte_pktmbuf_extmem *ext_mem,
802  unsigned int ext_num);
803 
815 static inline uint16_t
817 {
818  struct rte_pktmbuf_pool_private *mbp_priv;
819 
820  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
821  return mbp_priv->mbuf_data_room_size;
822 }
823 
836 static inline uint16_t
838 {
839  struct rte_pktmbuf_pool_private *mbp_priv;
840 
841  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
842  return mbp_priv->mbuf_priv_size;
843 }
844 
853 static inline void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
854 {
855  m->data_off = (uint16_t)RTE_MIN((uint16_t)RTE_PKTMBUF_HEADROOM,
856  (uint16_t)m->buf_len);
857 }
858 
867 static inline void rte_pktmbuf_reset(struct rte_mbuf *m)
868 {
869  m->next = NULL;
870  m->pkt_len = 0;
871  m->tx_offload = 0;
872  m->vlan_tci = 0;
873  m->vlan_tci_outer = 0;
874  m->nb_segs = 1;
876 
878  m->packet_type = 0;
880 
881  m->data_len = 0;
883 }
884 
898 static inline struct rte_mbuf *rte_pktmbuf_alloc(struct rte_mempool *mp)
899 {
900  struct rte_mbuf *m;
901  if ((m = rte_mbuf_raw_alloc(mp)) != NULL)
903  return m;
904 }
905 
920 static inline int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool,
921  struct rte_mbuf **mbufs, unsigned count)
922 {
923  unsigned idx = 0;
924  int rc;
925 
926  rc = rte_mempool_get_bulk(pool, (void **)mbufs, count);
927  if (unlikely(rc))
928  return rc;
929 
930  /* To understand duff's device on loop unwinding optimization, see
931  * https://en.wikipedia.org/wiki/Duff's_device.
932  * Here while() loop is used rather than do() while{} to avoid extra
933  * check if count is zero.
934  */
935  switch (count % 4) {
936  case 0:
937  while (idx != count) {
938  __rte_mbuf_raw_sanity_check(mbufs[idx]);
939  rte_pktmbuf_reset(mbufs[idx]);
940  idx++;
941  /* fall-through */
942  case 3:
943  __rte_mbuf_raw_sanity_check(mbufs[idx]);
944  rte_pktmbuf_reset(mbufs[idx]);
945  idx++;
946  /* fall-through */
947  case 2:
948  __rte_mbuf_raw_sanity_check(mbufs[idx]);
949  rte_pktmbuf_reset(mbufs[idx]);
950  idx++;
951  /* fall-through */
952  case 1:
953  __rte_mbuf_raw_sanity_check(mbufs[idx]);
954  rte_pktmbuf_reset(mbufs[idx]);
955  idx++;
956  /* fall-through */
957  }
958  }
959  return 0;
960 }
961 
994 static inline struct rte_mbuf_ext_shared_info *
995 rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len,
997 {
998  struct rte_mbuf_ext_shared_info *shinfo;
999  void *buf_end = RTE_PTR_ADD(buf_addr, *buf_len);
1000  void *addr;
1001 
1002  addr = RTE_PTR_ALIGN_FLOOR(RTE_PTR_SUB(buf_end, sizeof(*shinfo)),
1003  sizeof(uintptr_t));
1004  if (addr <= buf_addr)
1005  return NULL;
1006 
1007  shinfo = (struct rte_mbuf_ext_shared_info *)addr;
1008  shinfo->free_cb = free_cb;
1009  shinfo->fcb_opaque = fcb_opaque;
1010  rte_mbuf_ext_refcnt_set(shinfo, 1);
1011 
1012  *buf_len = (uint16_t)RTE_PTR_DIFF(shinfo, buf_addr);
1013  return shinfo;
1014 }
1015 
1076 static inline void
1077 rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr,
1078  rte_iova_t buf_iova, uint16_t buf_len,
1079  struct rte_mbuf_ext_shared_info *shinfo)
1080 {
1081  /* mbuf should not be read-only */
1082  RTE_ASSERT(RTE_MBUF_DIRECT(m) && rte_mbuf_refcnt_read(m) == 1);
1083  RTE_ASSERT(shinfo->free_cb != NULL);
1084 
1085  m->buf_addr = buf_addr;
1086  m->buf_iova = buf_iova;
1087  m->buf_len = buf_len;
1088 
1089  m->data_len = 0;
1090  m->data_off = 0;
1091 
1093  m->shinfo = shinfo;
1094 }
1095 
1103 #define rte_pktmbuf_detach_extbuf(m) rte_pktmbuf_detach(m)
1104 
1113 static inline void
1114 rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
1115 {
1116  memcpy(&mdst->dynfield1, msrc->dynfield1, sizeof(mdst->dynfield1));
1117 }
1118 
1119 /* internal */
1120 static inline void
1121 __rte_pktmbuf_copy_hdr(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
1122 {
1123  mdst->port = msrc->port;
1124  mdst->vlan_tci = msrc->vlan_tci;
1125  mdst->vlan_tci_outer = msrc->vlan_tci_outer;
1126  mdst->tx_offload = msrc->tx_offload;
1127  mdst->hash = msrc->hash;
1128  mdst->packet_type = msrc->packet_type;
1129  rte_mbuf_dynfield_copy(mdst, msrc);
1130 }
1131 
1153 static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
1154 {
1155  RTE_ASSERT(RTE_MBUF_DIRECT(mi) &&
1156  rte_mbuf_refcnt_read(mi) == 1);
1157 
1158  if (RTE_MBUF_HAS_EXTBUF(m)) {
1160  mi->ol_flags = m->ol_flags;
1161  mi->shinfo = m->shinfo;
1162  } else {
1163  /* if m is not direct, get the mbuf that embeds the data */
1165  mi->priv_size = m->priv_size;
1166  mi->ol_flags = m->ol_flags | IND_ATTACHED_MBUF;
1167  }
1168 
1169  __rte_pktmbuf_copy_hdr(mi, m);
1170 
1171  mi->data_off = m->data_off;
1172  mi->data_len = m->data_len;
1173  mi->buf_iova = m->buf_iova;
1174  mi->buf_addr = m->buf_addr;
1175  mi->buf_len = m->buf_len;
1176 
1177  mi->next = NULL;
1178  mi->pkt_len = mi->data_len;
1179  mi->nb_segs = 1;
1180 
1181  __rte_mbuf_sanity_check(mi, 1);
1183 }
1184 
1192 static inline void
1193 __rte_pktmbuf_free_extbuf(struct rte_mbuf *m)
1194 {
1195  RTE_ASSERT(RTE_MBUF_HAS_EXTBUF(m));
1196  RTE_ASSERT(m->shinfo != NULL);
1197 
1198  if (rte_mbuf_ext_refcnt_update(m->shinfo, -1) == 0)
1199  m->shinfo->free_cb(m->buf_addr, m->shinfo->fcb_opaque);
1200 }
1201 
1208 static inline void
1209 __rte_pktmbuf_free_direct(struct rte_mbuf *m)
1210 {
1211  struct rte_mbuf *md;
1212 
1213  RTE_ASSERT(RTE_MBUF_CLONED(m));
1214 
1215  md = rte_mbuf_from_indirect(m);
1216 
1217  if (rte_mbuf_refcnt_update(md, -1) == 0) {
1218  md->next = NULL;
1219  md->nb_segs = 1;
1220  rte_mbuf_refcnt_set(md, 1);
1221  rte_mbuf_raw_free(md);
1222  }
1223 }
1224 
1243 static inline void rte_pktmbuf_detach(struct rte_mbuf *m)
1244 {
1245  struct rte_mempool *mp = m->pool;
1246  uint32_t mbuf_size, buf_len;
1247  uint16_t priv_size;
1248 
1249  if (RTE_MBUF_HAS_EXTBUF(m)) {
1250  /*
1251  * The mbuf has the external attached buffer,
1252  * we should check the type of the memory pool where
1253  * the mbuf was allocated from to detect the pinned
1254  * external buffer.
1255  */
1256  uint32_t flags = rte_pktmbuf_priv_flags(mp);
1257 
1258  if (flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) {
1259  /*
1260  * The pinned external buffer should not be
1261  * detached from its backing mbuf, just exit.
1262  */
1263  return;
1264  }
1265  __rte_pktmbuf_free_extbuf(m);
1266  } else {
1267  __rte_pktmbuf_free_direct(m);
1268  }
1269  priv_size = rte_pktmbuf_priv_size(mp);
1270  mbuf_size = (uint32_t)(sizeof(struct rte_mbuf) + priv_size);
1271  buf_len = rte_pktmbuf_data_room_size(mp);
1272 
1273  m->priv_size = priv_size;
1274  m->buf_addr = (char *)m + mbuf_size;
1275  m->buf_iova = rte_mempool_virt2iova(m) + mbuf_size;
1276  m->buf_len = (uint16_t)buf_len;
1278  m->data_len = 0;
1279  m->ol_flags = 0;
1280 }
1281 
1295 static inline int __rte_pktmbuf_pinned_extbuf_decref(struct rte_mbuf *m)
1296 {
1297  struct rte_mbuf_ext_shared_info *shinfo;
1298 
1299  /* Clear flags, mbuf is being freed. */
1301  shinfo = m->shinfo;
1302 
1303  /* Optimize for performance - do not dec/reinit */
1304  if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1))
1305  return 0;
1306 
1307  /*
1308  * Direct usage of add primitive to avoid
1309  * duplication of comparing with one.
1310  */
1311  if (likely(__atomic_add_fetch(&shinfo->refcnt, (uint16_t)-1,
1312  __ATOMIC_ACQ_REL)))
1313  return 1;
1314 
1315  /* Reinitialize counter before mbuf freeing. */
1316  rte_mbuf_ext_refcnt_set(shinfo, 1);
1317  return 0;
1318 }
1319 
1334 static __rte_always_inline struct rte_mbuf *
1336 {
1338 
1339  if (likely(rte_mbuf_refcnt_read(m) == 1)) {
1340 
1341  if (!RTE_MBUF_DIRECT(m)) {
1342  rte_pktmbuf_detach(m);
1343  if (RTE_MBUF_HAS_EXTBUF(m) &&
1345  __rte_pktmbuf_pinned_extbuf_decref(m))
1346  return NULL;
1347  }
1348 
1349  if (m->next != NULL) {
1350  m->next = NULL;
1351  m->nb_segs = 1;
1352  }
1353 
1354  return m;
1355 
1356  } else if (__rte_mbuf_refcnt_update(m, -1) == 0) {
1357 
1358  if (!RTE_MBUF_DIRECT(m)) {
1359  rte_pktmbuf_detach(m);
1360  if (RTE_MBUF_HAS_EXTBUF(m) &&
1362  __rte_pktmbuf_pinned_extbuf_decref(m))
1363  return NULL;
1364  }
1365 
1366  if (m->next != NULL) {
1367  m->next = NULL;
1368  m->nb_segs = 1;
1369  }
1370  rte_mbuf_refcnt_set(m, 1);
1371 
1372  return m;
1373  }
1374  return NULL;
1375 }
1376 
1386 static __rte_always_inline void
1388 {
1389  m = rte_pktmbuf_prefree_seg(m);
1390  if (likely(m != NULL))
1391  rte_mbuf_raw_free(m);
1392 }
1393 
1403 static inline void rte_pktmbuf_free(struct rte_mbuf *m)
1404 {
1405  struct rte_mbuf *m_next;
1406 
1407  if (m != NULL)
1409 
1410  while (m != NULL) {
1411  m_next = m->next;
1413  m = m_next;
1414  }
1415 }
1416 
1429 __rte_experimental
1430 void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count);
1431 
1449 struct rte_mbuf *
1450 rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp);
1451 
1473 __rte_experimental
1474 struct rte_mbuf *
1475 rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp,
1476  uint32_t offset, uint32_t length);
1477 
1489 static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
1490 {
1492 
1493  do {
1494  rte_mbuf_refcnt_update(m, v);
1495  } while ((m = m->next) != NULL);
1496 }
1497 
1506 static inline uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
1507 {
1509  return m->data_off;
1510 }
1511 
1520 static inline uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
1521 {
1523  return (uint16_t)(m->buf_len - rte_pktmbuf_headroom(m) -
1524  m->data_len);
1525 }
1526 
1535 static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m)
1536 {
1538  while (m->next != NULL)
1539  m = m->next;
1540  return m;
1541 }
1542 
1551 #define rte_pktmbuf_pkt_len(m) ((m)->pkt_len)
1552 
1561 #define rte_pktmbuf_data_len(m) ((m)->data_len)
1562 
1578 static inline char *rte_pktmbuf_prepend(struct rte_mbuf *m,
1579  uint16_t len)
1580 {
1582 
1583  if (unlikely(len > rte_pktmbuf_headroom(m)))
1584  return NULL;
1585 
1586  /* NB: elaborating the subtraction like this instead of using
1587  * -= allows us to ensure the result type is uint16_t
1588  * avoiding compiler warnings on gcc 8.1 at least */
1589  m->data_off = (uint16_t)(m->data_off - len);
1590  m->data_len = (uint16_t)(m->data_len + len);
1591  m->pkt_len = (m->pkt_len + len);
1592 
1593  return (char *)m->buf_addr + m->data_off;
1594 }
1595 
1611 static inline char *rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
1612 {
1613  void *tail;
1614  struct rte_mbuf *m_last;
1615 
1617 
1618  m_last = rte_pktmbuf_lastseg(m);
1619  if (unlikely(len > rte_pktmbuf_tailroom(m_last)))
1620  return NULL;
1621 
1622  tail = (char *)m_last->buf_addr + m_last->data_off + m_last->data_len;
1623  m_last->data_len = (uint16_t)(m_last->data_len + len);
1624  m->pkt_len = (m->pkt_len + len);
1625  return (char*) tail;
1626 }
1627 
1642 static inline char *rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
1643 {
1645 
1646  if (unlikely(len > m->data_len))
1647  return NULL;
1648 
1649  /* NB: elaborating the addition like this instead of using
1650  * += allows us to ensure the result type is uint16_t
1651  * avoiding compiler warnings on gcc 8.1 at least */
1652  m->data_len = (uint16_t)(m->data_len - len);
1653  m->data_off = (uint16_t)(m->data_off + len);
1654  m->pkt_len = (m->pkt_len - len);
1655  return (char *)m->buf_addr + m->data_off;
1656 }
1657 
1672 static inline int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
1673 {
1674  struct rte_mbuf *m_last;
1675 
1677 
1678  m_last = rte_pktmbuf_lastseg(m);
1679  if (unlikely(len > m_last->data_len))
1680  return -1;
1681 
1682  m_last->data_len = (uint16_t)(m_last->data_len - len);
1683  m->pkt_len = (m->pkt_len - len);
1684  return 0;
1685 }
1686 
1696 static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
1697 {
1699  return m->nb_segs == 1;
1700 }
1701 
1705 const void *__rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off,
1706  uint32_t len, void *buf);
1707 
1728 static inline const void *rte_pktmbuf_read(const struct rte_mbuf *m,
1729  uint32_t off, uint32_t len, void *buf)
1730 {
1731  if (likely(off + len <= rte_pktmbuf_data_len(m)))
1732  return rte_pktmbuf_mtod_offset(m, char *, off);
1733  else
1734  return __rte_pktmbuf_read(m, off, len, buf);
1735 }
1736 
1753 static inline int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
1754 {
1755  struct rte_mbuf *cur_tail;
1756 
1757  /* Check for number-of-segments-overflow */
1758  if (head->nb_segs + tail->nb_segs > RTE_MBUF_MAX_NB_SEGS)
1759  return -EOVERFLOW;
1760 
1761  /* Chain 'tail' onto the old tail */
1762  cur_tail = rte_pktmbuf_lastseg(head);
1763  cur_tail->next = tail;
1764 
1765  /* accumulate number of segments and total length.
1766  * NB: elaborating the addition like this instead of using
1767  * -= allows us to ensure the result type is uint16_t
1768  * avoiding compiler warnings on gcc 8.1 at least */
1769  head->nb_segs = (uint16_t)(head->nb_segs + tail->nb_segs);
1770  head->pkt_len += tail->pkt_len;
1771 
1772  /* pkt_len is only set in the head */
1773  tail->pkt_len = tail->data_len;
1774 
1775  return 0;
1776 }
1777 
1778 /*
1779  * @warning
1780  * @b EXPERIMENTAL: This API may change without prior notice.
1781  *
1782  * For given input values generate raw tx_offload value.
1783  * Note that it is caller responsibility to make sure that input parameters
1784  * don't exceed maximum bit-field values.
1785  * @param il2
1786  * l2_len value.
1787  * @param il3
1788  * l3_len value.
1789  * @param il4
1790  * l4_len value.
1791  * @param tso
1792  * tso_segsz value.
1793  * @param ol3
1794  * outer_l3_len value.
1795  * @param ol2
1796  * outer_l2_len value.
1797  * @param unused
1798  * unused value.
1799  * @return
1800  * raw tx_offload value.
1801  */
1802 static __rte_always_inline uint64_t
1803 rte_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso,
1804  uint64_t ol3, uint64_t ol2, uint64_t unused)
1805 {
1806  return il2 << RTE_MBUF_L2_LEN_OFS |
1807  il3 << RTE_MBUF_L3_LEN_OFS |
1808  il4 << RTE_MBUF_L4_LEN_OFS |
1809  tso << RTE_MBUF_TSO_SEGSZ_OFS |
1810  ol3 << RTE_MBUF_OUTL3_LEN_OFS |
1811  ol2 << RTE_MBUF_OUTL2_LEN_OFS |
1812  unused << RTE_MBUF_TXOFLD_UNUSED_OFS;
1813 }
1814 
1825 static inline int
1827 {
1828  uint64_t ol_flags = m->ol_flags;
1829 
1830  /* Does packet set any of available offloads? */
1831  if (!(ol_flags & PKT_TX_OFFLOAD_MASK))
1832  return 0;
1833 
1834  /* IP checksum can be counted only for IPv4 packet */
1835  if ((ol_flags & PKT_TX_IP_CKSUM) && (ol_flags & PKT_TX_IPV6))
1836  return -EINVAL;
1837 
1838  /* IP type not set when required */
1839  if (ol_flags & (PKT_TX_L4_MASK | PKT_TX_TCP_SEG))
1840  if (!(ol_flags & (PKT_TX_IPV4 | PKT_TX_IPV6)))
1841  return -EINVAL;
1842 
1843  /* Check requirements for TSO packet */
1844  if (ol_flags & PKT_TX_TCP_SEG)
1845  if ((m->tso_segsz == 0) ||
1846  ((ol_flags & PKT_TX_IPV4) &&
1847  !(ol_flags & PKT_TX_IP_CKSUM)))
1848  return -EINVAL;
1849 
1850  /* PKT_TX_OUTER_IP_CKSUM set for non outer IPv4 packet. */
1851  if ((ol_flags & PKT_TX_OUTER_IP_CKSUM) &&
1852  !(ol_flags & PKT_TX_OUTER_IPV4))
1853  return -EINVAL;
1854 
1855  return 0;
1856 }
1857 
1861 int __rte_pktmbuf_linearize(struct rte_mbuf *mbuf);
1862 
1875 static inline int
1877 {
1878  if (rte_pktmbuf_is_contiguous(mbuf))
1879  return 0;
1880  return __rte_pktmbuf_linearize(mbuf);
1881 }
1882 
1897 void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len);
1898 
1902 static inline uint32_t
1904 {
1905  return m->hash.sched.queue_id;
1906 }
1907 
1911 static inline uint8_t
1913 {
1914  return m->hash.sched.traffic_class;
1915 }
1916 
1920 static inline uint8_t
1922 {
1923  return m->hash.sched.color;
1924 }
1925 
1938 static inline void
1939 rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id,
1940  uint8_t *traffic_class,
1941  uint8_t *color)
1942 {
1943  struct rte_mbuf_sched sched = m->hash.sched;
1944 
1945  *queue_id = sched.queue_id;
1946  *traffic_class = sched.traffic_class;
1947  *color = sched.color;
1948 }
1949 
1953 static inline void
1955 {
1956  m->hash.sched.queue_id = queue_id;
1957 }
1958 
1962 static inline void
1964 {
1965  m->hash.sched.traffic_class = traffic_class;
1966 }
1967 
1971 static inline void
1973 {
1974  m->hash.sched.color = color;
1975 }
1976 
1989 static inline void
1991  uint8_t traffic_class,
1992  uint8_t color)
1993 {
1994  m->hash.sched = (struct rte_mbuf_sched){
1995  .queue_id = queue_id,
1996  .traffic_class = traffic_class,
1997  .color = color,
1998  .reserved = 0,
1999  };
2000 }
2001 
2002 #ifdef __cplusplus
2003 }
2004 #endif
2005 
2006 #endif /* _RTE_MBUF_H_ */
static void rte_pktmbuf_reset(struct rte_mbuf *m)
Definition: rte_mbuf.h:867
struct rte_mbuf_ext_shared_info * shinfo
static rte_iova_t rte_mbuf_data_iova(const struct rte_mbuf *mb)
Definition: rte_mbuf.h:149
struct rte_mbuf * next
uint16_t mbuf_data_room_size
Definition: rte_mbuf.h:289
uint16_t vlan_tci_outer
#define __rte_always_inline
Definition: rte_common.h:228
static struct rte_mbuf * rte_pktmbuf_alloc(struct rte_mempool *mp)
Definition: rte_mbuf.h:898
static void rte_mbuf_sched_traffic_class_set(struct rte_mbuf *m, uint8_t traffic_class)
Definition: rte_mbuf.h:1963
static uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp)
Definition: rte_mbuf.h:837
static int rte_validate_tx_offload(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1826
__rte_experimental struct rte_mbuf * rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp, uint32_t offset, uint32_t length)
uint32_t queue_id
#define likely(x)
static void rte_pktmbuf_free(struct rte_mbuf *m)
Definition: rte_mbuf.h:1403
#define PKT_TX_OUTER_IP_CKSUM
static uint32_t rte_mbuf_sched_queue_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1903
#define RTE_PTR_ALIGN_FLOOR(ptr, align)
Definition: rte_common.h:279
void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg)
uint64_t rte_iova_t
Definition: rte_common.h:420
static __rte_always_inline void rte_pktmbuf_free_seg(struct rte_mbuf *m)
Definition: rte_mbuf.h:1387
void * buf_addr
static struct rte_mbuf * rte_mbuf_from_indirect(struct rte_mbuf *mi)
Definition: rte_mbuf.h:181
uint16_t data_len
rte_mbuf_extbuf_free_callback_t free_cb
static int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
Definition: rte_mbuf.h:1753
__rte_experimental void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count)
static uint8_t rte_mbuf_sched_traffic_class_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1912
#define PKT_TX_IPV4
static __rte_experimental char * rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
Definition: rte_mbuf.h:208
#define __rte_unused
Definition: rte_common.h:118
__rte_experimental int rte_mbuf_check(const struct rte_mbuf *m, int is_header, const char **reason)
uint64_t tso_segsz
static __rte_always_inline void __rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m)
Definition: rte_mbuf.h:556
void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len)
static uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1506
static int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, struct rte_mbuf **mbufs, unsigned count)
Definition: rte_mbuf.h:920
static void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
Definition: rte_mbuf.h:853
uint32_t cache_size
Definition: rte_mempool.h:229
static void rte_mbuf_prefetch_part2(struct rte_mbuf *m)
Definition: rte_mbuf.h:128
#define PKT_TX_OUTER_IPV4
static uint16_t rte_mbuf_ext_refcnt_update(struct rte_mbuf_ext_shared_info *shinfo, int16_t value)
Definition: rte_mbuf.h:485
uint16_t nb_segs
#define IND_ATTACHED_MBUF
uint16_t port
#define rte_pktmbuf_mtod_offset(m, t, o)
static __rte_always_inline struct rte_mbuf * rte_pktmbuf_prefree_seg(struct rte_mbuf *m)
Definition: rte_mbuf.h:1335
static int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1696
#define RTE_PTR_ADD(ptr, x)
Definition: rte_common.h:250
int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen)
#define RTE_MBUF_MAX_NB_SEGS
static void rte_mbuf_sched_set(struct rte_mbuf *m, uint32_t queue_id, uint8_t traffic_class, uint8_t color)
Definition: rte_mbuf.h:1990
static uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1520
static __rte_always_inline void rte_mbuf_raw_free(struct rte_mbuf *m)
Definition: rte_mbuf.h:611
static void rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id, uint8_t *traffic_class, uint8_t *color)
Definition: rte_mbuf.h:1939
#define unlikely(x)
uint16_t priv_size
static void rte_mbuf_sched_queue_set(struct rte_mbuf *m, uint32_t queue_id)
Definition: rte_mbuf.h:1954
void rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header)
static void rte_mbuf_sched_color_set(struct rte_mbuf *m, uint8_t color)
Definition: rte_mbuf.h:1972
void(* rte_mbuf_extbuf_free_callback_t)(void *addr, void *opaque)
#define RTE_MIN(a, b)
Definition: rte_common.h:575
#define __rte_mbuf_sanity_check(m, is_h)
Definition: rte_mbuf.h:337
const char * rte_get_tx_ol_flag_name(uint64_t mask)
static uint16_t rte_mbuf_refcnt_read(const struct rte_mbuf *m)
Definition: rte_mbuf.h:428
#define PKT_TX_TCP_SEG
static int rte_pktmbuf_linearize(struct rte_mbuf *mbuf)
Definition: rte_mbuf.h:1876
static __rte_always_inline int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
Definition: rte_mempool.h:1559
static uint32_t rte_pktmbuf_priv_flags(struct rte_mempool *mp)
Definition: rte_mbuf.h:303
uint16_t elt_size
Definition: rte_mbuf.h:752
uint16_t refcnt
static char * rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1642
static __rte_experimental void * rte_mbuf_to_priv(struct rte_mbuf *m)
Definition: rte_mbuf.h:277
#define RTE_MBUF_DIRECT(mb)
#define RTE_MBUF_CLONED(mb)
static void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
Definition: rte_mbuf.h:1153
static __rte_always_inline int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:1530
#define PKT_TX_IP_CKSUM
uint64_t ol_flags
static void rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
Definition: rte_mbuf.h:1114
static void rte_pktmbuf_detach(struct rte_mbuf *m)
Definition: rte_mbuf.h:1243
static uint16_t rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
Definition: rte_mbuf.h:453
uint32_t pkt_len
uint16_t buf_len
#define rte_pktmbuf_data_len(m)
Definition: rte_mbuf.h:1561
static uint16_t rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
Definition: rte_mbuf.h:419
uint32_t packet_type
static uint16_t rte_pktmbuf_data_room_size(struct rte_mempool *mp)
Definition: rte_mbuf.h:816
#define RTE_MBUF_PORT_INVALID
const char * rte_get_rx_ol_flag_name(uint64_t mask)
static struct rte_mbuf_ext_shared_info * rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len, rte_mbuf_extbuf_free_callback_t free_cb, void *fcb_opaque)
Definition: rte_mbuf.h:995
#define PKT_TX_IPV6
#define PKT_TX_L4_MASK
void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg, void *m, unsigned i)
#define RTE_MBUF_HAS_EXTBUF(mb)
struct rte_mempool * pool
static void rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo, uint16_t new_value)
Definition: rte_mbuf.h:467
static char * rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1611
static void rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
Definition: rte_mbuf.h:437
struct rte_mempool * rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n, unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id, const char *ops_name)
static rte_iova_t rte_mbuf_data_iova_default(const struct rte_mbuf *mb)
Definition: rte_mbuf.h:167
static int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1672
static char * rte_mbuf_to_baddr(struct rte_mbuf *md)
Definition: rte_mbuf.h:252
static const void * rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off, uint32_t len, void *buf)
Definition: rte_mbuf.h:1728
static char * rte_pktmbuf_prepend(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1578
static struct rte_mbuf * rte_mbuf_raw_alloc(struct rte_mempool *mp)
Definition: rte_mbuf.h:586
static void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
Definition: rte_mbuf.h:1489
#define RTE_PTR_SUB(ptr, x)
Definition: rte_common.h:255
static struct rte_mbuf * rte_pktmbuf_lastseg(struct rte_mbuf *m)
Definition: rte_mbuf.h:1535
static __rte_experimental char * rte_mbuf_data_addr_default(__rte_unused struct rte_mbuf *mb)
Definition: rte_mbuf.h:226
__rte_experimental struct rte_mempool * rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n, unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id, const struct rte_pktmbuf_extmem *ext_mem, unsigned int ext_num)
static uint8_t rte_mbuf_sched_color_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1921
#define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF
Definition: rte_mbuf.h:317
static rte_iova_t rte_mempool_virt2iova(const void *elt)
Definition: rte_mempool.h:1684
static __rte_always_inline void rte_mempool_put(struct rte_mempool *mp, void *obj)
Definition: rte_mempool.h:1394
rte_iova_t buf_iova
Definition: rte_mbuf.h:750
uint8_t traffic_class
#define RTE_PTR_DIFF(ptr1, ptr2)
Definition: rte_common.h:262
static void rte_mbuf_prefetch_part1(struct rte_mbuf *m)
Definition: rte_mbuf.h:111
#define PKT_TX_OFFLOAD_MASK
static void * rte_mempool_get_priv(struct rte_mempool *mp)
Definition: rte_mempool.h:1712
uint64_t tx_offload
char name[RTE_MEMZONE_NAMESIZE]
Definition: rte_mempool.h:218
uint16_t vlan_tci
#define RTE_MBUF_HAS_PINNED_EXTBUF(mb)
Definition: rte_mbuf.h:326
struct rte_mbuf * rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp)
#define RTE_SET_USED(x)
Definition: rte_common.h:133
int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen)
static void rte_prefetch0(const volatile void *p)
struct rte_mempool * rte_pktmbuf_pool_create(const char *name, unsigned n, unsigned cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id)
uint32_t dynfield1[9]
#define EXT_ATTACHED_MBUF
static void rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr, rte_iova_t buf_iova, uint16_t buf_len, struct rte_mbuf_ext_shared_info *shinfo)
Definition: rte_mbuf.h:1077