DPDK  21.02.0
rte_mbuf.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2014 Intel Corporation.
3  * Copyright 2014 6WIND S.A.
4  */
5 
6 #ifndef _RTE_MBUF_H_
7 #define _RTE_MBUF_H_
8 
34 #include <stdint.h>
35 #include <rte_compat.h>
36 #include <rte_common.h>
37 #include <rte_config.h>
38 #include <rte_mempool.h>
39 #include <rte_memory.h>
40 #include <rte_prefetch.h>
41 #include <rte_branch_prediction.h>
42 #include <rte_byteorder.h>
43 #include <rte_mbuf_ptype.h>
44 #include <rte_mbuf_core.h>
45 
46 #ifdef __cplusplus
47 extern "C" {
48 #endif
49 
58 const char *rte_get_rx_ol_flag_name(uint64_t mask);
59 
72 int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
73 
84 const char *rte_get_tx_ol_flag_name(uint64_t mask);
85 
98 int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
99 
110 static inline void
112 {
113  rte_prefetch0(&m->cacheline0);
114 }
115 
127 static inline void
129 {
130 #if RTE_CACHE_LINE_SIZE == 64
131  rte_prefetch0(&m->cacheline1);
132 #else
133  RTE_SET_USED(m);
134 #endif
135 }
136 
137 
138 static inline uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp);
139 
148 static inline rte_iova_t
149 rte_mbuf_data_iova(const struct rte_mbuf *mb)
150 {
151  return mb->buf_iova + mb->data_off;
152 }
153 
166 static inline rte_iova_t
168 {
169  return mb->buf_iova + RTE_PKTMBUF_HEADROOM;
170 }
171 
180 static inline struct rte_mbuf *
182 {
183  return (struct rte_mbuf *)RTE_PTR_SUB(mi->buf_addr, sizeof(*mi) + mi->priv_size);
184 }
185 
206 __rte_experimental
207 static inline char *
208 rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
209 {
210  return (char *)mb + sizeof(*mb) + rte_pktmbuf_priv_size(mp);
211 }
212 
224 __rte_experimental
225 static inline char *
227 {
228  /* gcc complains about calling this experimental function even
229  * when not using it. Hide it with ALLOW_EXPERIMENTAL_API.
230  */
231 #ifdef ALLOW_EXPERIMENTAL_API
232  return rte_mbuf_buf_addr(mb, mb->pool) + RTE_PKTMBUF_HEADROOM;
233 #else
234  return NULL;
235 #endif
236 }
237 
251 static inline char *
253 {
254 #ifdef ALLOW_EXPERIMENTAL_API
255  return rte_mbuf_buf_addr(md, md->pool);
256 #else
257  char *buffer_addr;
258  buffer_addr = (char *)md + sizeof(*md) + rte_pktmbuf_priv_size(md->pool);
259  return buffer_addr;
260 #endif
261 }
262 
275 __rte_experimental
276 static inline void *
278 {
279  return RTE_PTR_ADD(m, sizeof(struct rte_mbuf));
280 }
281 
290  uint16_t mbuf_priv_size;
291  uint32_t flags;
292 };
293 
302 static inline uint32_t
304 {
305  struct rte_pktmbuf_pool_private *mbp_priv;
306 
307  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
308  return mbp_priv->flags;
309 }
310 
317 #define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF (1 << 0)
318 
326 #define RTE_MBUF_HAS_PINNED_EXTBUF(mb) \
327  (rte_pktmbuf_priv_flags(mb->pool) & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF)
328 
329 #ifdef RTE_LIBRTE_MBUF_DEBUG
330 
332 #define __rte_mbuf_sanity_check(m, is_h) rte_mbuf_sanity_check(m, is_h)
333 
334 #else /* RTE_LIBRTE_MBUF_DEBUG */
335 
337 #define __rte_mbuf_sanity_check(m, is_h) do { } while (0)
338 
339 #endif /* RTE_LIBRTE_MBUF_DEBUG */
340 
341 #ifdef RTE_MBUF_REFCNT_ATOMIC
342 
350 static inline uint16_t
351 rte_mbuf_refcnt_read(const struct rte_mbuf *m)
352 {
353  return __atomic_load_n(&m->refcnt, __ATOMIC_RELAXED);
354 }
355 
363 static inline void
364 rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
365 {
366  __atomic_store_n(&m->refcnt, new_value, __ATOMIC_RELAXED);
367 }
368 
369 /* internal */
370 static inline uint16_t
371 __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
372 {
373  return __atomic_add_fetch(&m->refcnt, (uint16_t)value,
374  __ATOMIC_ACQ_REL);
375 }
376 
386 static inline uint16_t
387 rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
388 {
389  /*
390  * The atomic_add is an expensive operation, so we don't want to
391  * call it in the case where we know we are the unique holder of
392  * this mbuf (i.e. ref_cnt == 1). Otherwise, an atomic
393  * operation has to be used because concurrent accesses on the
394  * reference counter can occur.
395  */
396  if (likely(rte_mbuf_refcnt_read(m) == 1)) {
397  ++value;
398  rte_mbuf_refcnt_set(m, (uint16_t)value);
399  return (uint16_t)value;
400  }
401 
402  return __rte_mbuf_refcnt_update(m, value);
403 }
404 
405 #else /* ! RTE_MBUF_REFCNT_ATOMIC */
406 
407 /* internal */
408 static inline uint16_t
409 __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
410 {
411  m->refcnt = (uint16_t)(m->refcnt + value);
412  return m->refcnt;
413 }
414 
418 static inline uint16_t
419 rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
420 {
421  return __rte_mbuf_refcnt_update(m, value);
422 }
423 
427 static inline uint16_t
429 {
430  return m->refcnt;
431 }
432 
436 static inline void
437 rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
438 {
439  m->refcnt = new_value;
440 }
441 
442 #endif /* RTE_MBUF_REFCNT_ATOMIC */
443 
452 static inline uint16_t
454 {
455  return __atomic_load_n(&shinfo->refcnt, __ATOMIC_RELAXED);
456 }
457 
466 static inline void
468  uint16_t new_value)
469 {
470  __atomic_store_n(&shinfo->refcnt, new_value, __ATOMIC_RELAXED);
471 }
472 
484 static inline uint16_t
486  int16_t value)
487 {
488  if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1)) {
489  ++value;
490  rte_mbuf_ext_refcnt_set(shinfo, (uint16_t)value);
491  return (uint16_t)value;
492  }
493 
494  return __atomic_add_fetch(&shinfo->refcnt, (uint16_t)value,
495  __ATOMIC_ACQ_REL);
496 }
497 
499 #define RTE_MBUF_PREFETCH_TO_FREE(m) do { \
500  if ((m) != NULL) \
501  rte_prefetch0(m); \
502 } while (0)
503 
504 
517 void
518 rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header);
519 
539 __rte_experimental
540 int rte_mbuf_check(const struct rte_mbuf *m, int is_header,
541  const char **reason);
542 
555 static __rte_always_inline void
557 {
558  RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);
559  RTE_ASSERT(m->next == NULL);
560  RTE_ASSERT(m->nb_segs == 1);
562 }
563 
565 #define MBUF_RAW_ALLOC_CHECK(m) __rte_mbuf_raw_sanity_check(m)
566 
586 static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
587 {
588  struct rte_mbuf *m;
589 
590  if (rte_mempool_get(mp, (void **)&m) < 0)
591  return NULL;
593  return m;
594 }
595 
610 static __rte_always_inline void
612 {
613  RTE_ASSERT(!RTE_MBUF_CLONED(m) &&
616  rte_mempool_put(m->pool, m);
617 }
618 
638 void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg,
639  void *m, unsigned i);
640 
658 void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg);
659 
694 struct rte_mempool *
695 rte_pktmbuf_pool_create(const char *name, unsigned n,
696  unsigned cache_size, uint16_t priv_size, uint16_t data_room_size,
697  int socket_id);
698 
736 struct rte_mempool *
737 rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n,
738  unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size,
739  int socket_id, const char *ops_name);
740 
743  void *buf_ptr;
745  size_t buf_len;
746  uint16_t elt_size;
747 };
748 
790 __rte_experimental
791 struct rte_mempool *
792 rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n,
793  unsigned int cache_size, uint16_t priv_size,
794  uint16_t data_room_size, int socket_id,
795  const struct rte_pktmbuf_extmem *ext_mem,
796  unsigned int ext_num);
797 
809 static inline uint16_t
811 {
812  struct rte_pktmbuf_pool_private *mbp_priv;
813 
814  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
815  return mbp_priv->mbuf_data_room_size;
816 }
817 
830 static inline uint16_t
832 {
833  struct rte_pktmbuf_pool_private *mbp_priv;
834 
835  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
836  return mbp_priv->mbuf_priv_size;
837 }
838 
847 static inline void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
848 {
849  m->data_off = (uint16_t)RTE_MIN((uint16_t)RTE_PKTMBUF_HEADROOM,
850  (uint16_t)m->buf_len);
851 }
852 
861 static inline void rte_pktmbuf_reset(struct rte_mbuf *m)
862 {
863  m->next = NULL;
864  m->pkt_len = 0;
865  m->tx_offload = 0;
866  m->vlan_tci = 0;
867  m->vlan_tci_outer = 0;
868  m->nb_segs = 1;
870 
872  m->packet_type = 0;
874 
875  m->data_len = 0;
877 }
878 
892 static inline struct rte_mbuf *rte_pktmbuf_alloc(struct rte_mempool *mp)
893 {
894  struct rte_mbuf *m;
895  if ((m = rte_mbuf_raw_alloc(mp)) != NULL)
897  return m;
898 }
899 
914 static inline int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool,
915  struct rte_mbuf **mbufs, unsigned count)
916 {
917  unsigned idx = 0;
918  int rc;
919 
920  rc = rte_mempool_get_bulk(pool, (void **)mbufs, count);
921  if (unlikely(rc))
922  return rc;
923 
924  /* To understand duff's device on loop unwinding optimization, see
925  * https://en.wikipedia.org/wiki/Duff's_device.
926  * Here while() loop is used rather than do() while{} to avoid extra
927  * check if count is zero.
928  */
929  switch (count % 4) {
930  case 0:
931  while (idx != count) {
932  __rte_mbuf_raw_sanity_check(mbufs[idx]);
933  rte_pktmbuf_reset(mbufs[idx]);
934  idx++;
935  /* fall-through */
936  case 3:
937  __rte_mbuf_raw_sanity_check(mbufs[idx]);
938  rte_pktmbuf_reset(mbufs[idx]);
939  idx++;
940  /* fall-through */
941  case 2:
942  __rte_mbuf_raw_sanity_check(mbufs[idx]);
943  rte_pktmbuf_reset(mbufs[idx]);
944  idx++;
945  /* fall-through */
946  case 1:
947  __rte_mbuf_raw_sanity_check(mbufs[idx]);
948  rte_pktmbuf_reset(mbufs[idx]);
949  idx++;
950  /* fall-through */
951  }
952  }
953  return 0;
954 }
955 
988 static inline struct rte_mbuf_ext_shared_info *
989 rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len,
991 {
992  struct rte_mbuf_ext_shared_info *shinfo;
993  void *buf_end = RTE_PTR_ADD(buf_addr, *buf_len);
994  void *addr;
995 
996  addr = RTE_PTR_ALIGN_FLOOR(RTE_PTR_SUB(buf_end, sizeof(*shinfo)),
997  sizeof(uintptr_t));
998  if (addr <= buf_addr)
999  return NULL;
1000 
1001  shinfo = (struct rte_mbuf_ext_shared_info *)addr;
1002  shinfo->free_cb = free_cb;
1003  shinfo->fcb_opaque = fcb_opaque;
1004  rte_mbuf_ext_refcnt_set(shinfo, 1);
1005 
1006  *buf_len = (uint16_t)RTE_PTR_DIFF(shinfo, buf_addr);
1007  return shinfo;
1008 }
1009 
1070 static inline void
1071 rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr,
1072  rte_iova_t buf_iova, uint16_t buf_len,
1073  struct rte_mbuf_ext_shared_info *shinfo)
1074 {
1075  /* mbuf should not be read-only */
1076  RTE_ASSERT(RTE_MBUF_DIRECT(m) && rte_mbuf_refcnt_read(m) == 1);
1077  RTE_ASSERT(shinfo->free_cb != NULL);
1078 
1079  m->buf_addr = buf_addr;
1080  m->buf_iova = buf_iova;
1081  m->buf_len = buf_len;
1082 
1083  m->data_len = 0;
1084  m->data_off = 0;
1085 
1087  m->shinfo = shinfo;
1088 }
1089 
1097 #define rte_pktmbuf_detach_extbuf(m) rte_pktmbuf_detach(m)
1098 
1107 static inline void
1108 rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
1109 {
1110  memcpy(&mdst->dynfield1, msrc->dynfield1, sizeof(mdst->dynfield1));
1111 }
1112 
1113 /* internal */
1114 static inline void
1115 __rte_pktmbuf_copy_hdr(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
1116 {
1117  mdst->port = msrc->port;
1118  mdst->vlan_tci = msrc->vlan_tci;
1119  mdst->vlan_tci_outer = msrc->vlan_tci_outer;
1120  mdst->tx_offload = msrc->tx_offload;
1121  mdst->hash = msrc->hash;
1122  mdst->packet_type = msrc->packet_type;
1123  rte_mbuf_dynfield_copy(mdst, msrc);
1124 }
1125 
1147 static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
1148 {
1149  RTE_ASSERT(RTE_MBUF_DIRECT(mi) &&
1150  rte_mbuf_refcnt_read(mi) == 1);
1151 
1152  if (RTE_MBUF_HAS_EXTBUF(m)) {
1154  mi->ol_flags = m->ol_flags;
1155  mi->shinfo = m->shinfo;
1156  } else {
1157  /* if m is not direct, get the mbuf that embeds the data */
1159  mi->priv_size = m->priv_size;
1160  mi->ol_flags = m->ol_flags | IND_ATTACHED_MBUF;
1161  }
1162 
1163  __rte_pktmbuf_copy_hdr(mi, m);
1164 
1165  mi->data_off = m->data_off;
1166  mi->data_len = m->data_len;
1167  mi->buf_iova = m->buf_iova;
1168  mi->buf_addr = m->buf_addr;
1169  mi->buf_len = m->buf_len;
1170 
1171  mi->next = NULL;
1172  mi->pkt_len = mi->data_len;
1173  mi->nb_segs = 1;
1174 
1175  __rte_mbuf_sanity_check(mi, 1);
1177 }
1178 
1186 static inline void
1187 __rte_pktmbuf_free_extbuf(struct rte_mbuf *m)
1188 {
1189  RTE_ASSERT(RTE_MBUF_HAS_EXTBUF(m));
1190  RTE_ASSERT(m->shinfo != NULL);
1191 
1192  if (rte_mbuf_ext_refcnt_update(m->shinfo, -1) == 0)
1193  m->shinfo->free_cb(m->buf_addr, m->shinfo->fcb_opaque);
1194 }
1195 
1202 static inline void
1203 __rte_pktmbuf_free_direct(struct rte_mbuf *m)
1204 {
1205  struct rte_mbuf *md;
1206 
1207  RTE_ASSERT(RTE_MBUF_CLONED(m));
1208 
1209  md = rte_mbuf_from_indirect(m);
1210 
1211  if (rte_mbuf_refcnt_update(md, -1) == 0) {
1212  md->next = NULL;
1213  md->nb_segs = 1;
1214  rte_mbuf_refcnt_set(md, 1);
1215  rte_mbuf_raw_free(md);
1216  }
1217 }
1218 
1237 static inline void rte_pktmbuf_detach(struct rte_mbuf *m)
1238 {
1239  struct rte_mempool *mp = m->pool;
1240  uint32_t mbuf_size, buf_len;
1241  uint16_t priv_size;
1242 
1243  if (RTE_MBUF_HAS_EXTBUF(m)) {
1244  /*
1245  * The mbuf has the external attached buffer,
1246  * we should check the type of the memory pool where
1247  * the mbuf was allocated from to detect the pinned
1248  * external buffer.
1249  */
1250  uint32_t flags = rte_pktmbuf_priv_flags(mp);
1251 
1252  if (flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) {
1253  /*
1254  * The pinned external buffer should not be
1255  * detached from its backing mbuf, just exit.
1256  */
1257  return;
1258  }
1259  __rte_pktmbuf_free_extbuf(m);
1260  } else {
1261  __rte_pktmbuf_free_direct(m);
1262  }
1263  priv_size = rte_pktmbuf_priv_size(mp);
1264  mbuf_size = (uint32_t)(sizeof(struct rte_mbuf) + priv_size);
1265  buf_len = rte_pktmbuf_data_room_size(mp);
1266 
1267  m->priv_size = priv_size;
1268  m->buf_addr = (char *)m + mbuf_size;
1269  m->buf_iova = rte_mempool_virt2iova(m) + mbuf_size;
1270  m->buf_len = (uint16_t)buf_len;
1272  m->data_len = 0;
1273  m->ol_flags = 0;
1274 }
1275 
1289 static inline int __rte_pktmbuf_pinned_extbuf_decref(struct rte_mbuf *m)
1290 {
1291  struct rte_mbuf_ext_shared_info *shinfo;
1292 
1293  /* Clear flags, mbuf is being freed. */
1295  shinfo = m->shinfo;
1296 
1297  /* Optimize for performance - do not dec/reinit */
1298  if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1))
1299  return 0;
1300 
1301  /*
1302  * Direct usage of add primitive to avoid
1303  * duplication of comparing with one.
1304  */
1305  if (likely(__atomic_add_fetch(&shinfo->refcnt, (uint16_t)-1,
1306  __ATOMIC_ACQ_REL)))
1307  return 1;
1308 
1309  /* Reinitialize counter before mbuf freeing. */
1310  rte_mbuf_ext_refcnt_set(shinfo, 1);
1311  return 0;
1312 }
1313 
1328 static __rte_always_inline struct rte_mbuf *
1330 {
1332 
1333  if (likely(rte_mbuf_refcnt_read(m) == 1)) {
1334 
1335  if (!RTE_MBUF_DIRECT(m)) {
1336  rte_pktmbuf_detach(m);
1337  if (RTE_MBUF_HAS_EXTBUF(m) &&
1339  __rte_pktmbuf_pinned_extbuf_decref(m))
1340  return NULL;
1341  }
1342 
1343  if (m->next != NULL) {
1344  m->next = NULL;
1345  m->nb_segs = 1;
1346  }
1347 
1348  return m;
1349 
1350  } else if (__rte_mbuf_refcnt_update(m, -1) == 0) {
1351 
1352  if (!RTE_MBUF_DIRECT(m)) {
1353  rte_pktmbuf_detach(m);
1354  if (RTE_MBUF_HAS_EXTBUF(m) &&
1356  __rte_pktmbuf_pinned_extbuf_decref(m))
1357  return NULL;
1358  }
1359 
1360  if (m->next != NULL) {
1361  m->next = NULL;
1362  m->nb_segs = 1;
1363  }
1364  rte_mbuf_refcnt_set(m, 1);
1365 
1366  return m;
1367  }
1368  return NULL;
1369 }
1370 
1380 static __rte_always_inline void
1382 {
1383  m = rte_pktmbuf_prefree_seg(m);
1384  if (likely(m != NULL))
1385  rte_mbuf_raw_free(m);
1386 }
1387 
1397 static inline void rte_pktmbuf_free(struct rte_mbuf *m)
1398 {
1399  struct rte_mbuf *m_next;
1400 
1401  if (m != NULL)
1403 
1404  while (m != NULL) {
1405  m_next = m->next;
1407  m = m_next;
1408  }
1409 }
1410 
1423 __rte_experimental
1424 void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count);
1425 
1443 struct rte_mbuf *
1444 rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp);
1445 
1467 __rte_experimental
1468 struct rte_mbuf *
1469 rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp,
1470  uint32_t offset, uint32_t length);
1471 
1483 static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
1484 {
1486 
1487  do {
1488  rte_mbuf_refcnt_update(m, v);
1489  } while ((m = m->next) != NULL);
1490 }
1491 
1500 static inline uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
1501 {
1503  return m->data_off;
1504 }
1505 
1514 static inline uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
1515 {
1517  return (uint16_t)(m->buf_len - rte_pktmbuf_headroom(m) -
1518  m->data_len);
1519 }
1520 
1529 static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m)
1530 {
1532  while (m->next != NULL)
1533  m = m->next;
1534  return m;
1535 }
1536 
1545 #define rte_pktmbuf_pkt_len(m) ((m)->pkt_len)
1546 
1555 #define rte_pktmbuf_data_len(m) ((m)->data_len)
1556 
1572 static inline char *rte_pktmbuf_prepend(struct rte_mbuf *m,
1573  uint16_t len)
1574 {
1576 
1577  if (unlikely(len > rte_pktmbuf_headroom(m)))
1578  return NULL;
1579 
1580  /* NB: elaborating the subtraction like this instead of using
1581  * -= allows us to ensure the result type is uint16_t
1582  * avoiding compiler warnings on gcc 8.1 at least */
1583  m->data_off = (uint16_t)(m->data_off - len);
1584  m->data_len = (uint16_t)(m->data_len + len);
1585  m->pkt_len = (m->pkt_len + len);
1586 
1587  return (char *)m->buf_addr + m->data_off;
1588 }
1589 
1605 static inline char *rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
1606 {
1607  void *tail;
1608  struct rte_mbuf *m_last;
1609 
1611 
1612  m_last = rte_pktmbuf_lastseg(m);
1613  if (unlikely(len > rte_pktmbuf_tailroom(m_last)))
1614  return NULL;
1615 
1616  tail = (char *)m_last->buf_addr + m_last->data_off + m_last->data_len;
1617  m_last->data_len = (uint16_t)(m_last->data_len + len);
1618  m->pkt_len = (m->pkt_len + len);
1619  return (char*) tail;
1620 }
1621 
1636 static inline char *rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
1637 {
1639 
1640  if (unlikely(len > m->data_len))
1641  return NULL;
1642 
1643  /* NB: elaborating the addition like this instead of using
1644  * += allows us to ensure the result type is uint16_t
1645  * avoiding compiler warnings on gcc 8.1 at least */
1646  m->data_len = (uint16_t)(m->data_len - len);
1647  m->data_off = (uint16_t)(m->data_off + len);
1648  m->pkt_len = (m->pkt_len - len);
1649  return (char *)m->buf_addr + m->data_off;
1650 }
1651 
1666 static inline int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
1667 {
1668  struct rte_mbuf *m_last;
1669 
1671 
1672  m_last = rte_pktmbuf_lastseg(m);
1673  if (unlikely(len > m_last->data_len))
1674  return -1;
1675 
1676  m_last->data_len = (uint16_t)(m_last->data_len - len);
1677  m->pkt_len = (m->pkt_len - len);
1678  return 0;
1679 }
1680 
1690 static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
1691 {
1693  return m->nb_segs == 1;
1694 }
1695 
1699 const void *__rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off,
1700  uint32_t len, void *buf);
1701 
1722 static inline const void *rte_pktmbuf_read(const struct rte_mbuf *m,
1723  uint32_t off, uint32_t len, void *buf)
1724 {
1725  if (likely(off + len <= rte_pktmbuf_data_len(m)))
1726  return rte_pktmbuf_mtod_offset(m, char *, off);
1727  else
1728  return __rte_pktmbuf_read(m, off, len, buf);
1729 }
1730 
1747 static inline int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
1748 {
1749  struct rte_mbuf *cur_tail;
1750 
1751  /* Check for number-of-segments-overflow */
1752  if (head->nb_segs + tail->nb_segs > RTE_MBUF_MAX_NB_SEGS)
1753  return -EOVERFLOW;
1754 
1755  /* Chain 'tail' onto the old tail */
1756  cur_tail = rte_pktmbuf_lastseg(head);
1757  cur_tail->next = tail;
1758 
1759  /* accumulate number of segments and total length.
1760  * NB: elaborating the addition like this instead of using
1761  * -= allows us to ensure the result type is uint16_t
1762  * avoiding compiler warnings on gcc 8.1 at least */
1763  head->nb_segs = (uint16_t)(head->nb_segs + tail->nb_segs);
1764  head->pkt_len += tail->pkt_len;
1765 
1766  /* pkt_len is only set in the head */
1767  tail->pkt_len = tail->data_len;
1768 
1769  return 0;
1770 }
1771 
1772 /*
1773  * @warning
1774  * @b EXPERIMENTAL: This API may change without prior notice.
1775  *
1776  * For given input values generate raw tx_offload value.
1777  * Note that it is caller responsibility to make sure that input parameters
1778  * don't exceed maximum bit-field values.
1779  * @param il2
1780  * l2_len value.
1781  * @param il3
1782  * l3_len value.
1783  * @param il4
1784  * l4_len value.
1785  * @param tso
1786  * tso_segsz value.
1787  * @param ol3
1788  * outer_l3_len value.
1789  * @param ol2
1790  * outer_l2_len value.
1791  * @param unused
1792  * unused value.
1793  * @return
1794  * raw tx_offload value.
1795  */
1796 static __rte_always_inline uint64_t
1797 rte_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso,
1798  uint64_t ol3, uint64_t ol2, uint64_t unused)
1799 {
1800  return il2 << RTE_MBUF_L2_LEN_OFS |
1801  il3 << RTE_MBUF_L3_LEN_OFS |
1802  il4 << RTE_MBUF_L4_LEN_OFS |
1803  tso << RTE_MBUF_TSO_SEGSZ_OFS |
1804  ol3 << RTE_MBUF_OUTL3_LEN_OFS |
1805  ol2 << RTE_MBUF_OUTL2_LEN_OFS |
1806  unused << RTE_MBUF_TXOFLD_UNUSED_OFS;
1807 }
1808 
1819 static inline int
1821 {
1822  uint64_t ol_flags = m->ol_flags;
1823 
1824  /* Does packet set any of available offloads? */
1825  if (!(ol_flags & PKT_TX_OFFLOAD_MASK))
1826  return 0;
1827 
1828  /* IP checksum can be counted only for IPv4 packet */
1829  if ((ol_flags & PKT_TX_IP_CKSUM) && (ol_flags & PKT_TX_IPV6))
1830  return -EINVAL;
1831 
1832  /* IP type not set when required */
1833  if (ol_flags & (PKT_TX_L4_MASK | PKT_TX_TCP_SEG))
1834  if (!(ol_flags & (PKT_TX_IPV4 | PKT_TX_IPV6)))
1835  return -EINVAL;
1836 
1837  /* Check requirements for TSO packet */
1838  if (ol_flags & PKT_TX_TCP_SEG)
1839  if ((m->tso_segsz == 0) ||
1840  ((ol_flags & PKT_TX_IPV4) &&
1841  !(ol_flags & PKT_TX_IP_CKSUM)))
1842  return -EINVAL;
1843 
1844  /* PKT_TX_OUTER_IP_CKSUM set for non outer IPv4 packet. */
1845  if ((ol_flags & PKT_TX_OUTER_IP_CKSUM) &&
1846  !(ol_flags & PKT_TX_OUTER_IPV4))
1847  return -EINVAL;
1848 
1849  return 0;
1850 }
1851 
1855 int __rte_pktmbuf_linearize(struct rte_mbuf *mbuf);
1856 
1869 static inline int
1871 {
1872  if (rte_pktmbuf_is_contiguous(mbuf))
1873  return 0;
1874  return __rte_pktmbuf_linearize(mbuf);
1875 }
1876 
1891 void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len);
1892 
1896 static inline uint32_t
1898 {
1899  return m->hash.sched.queue_id;
1900 }
1901 
1905 static inline uint8_t
1907 {
1908  return m->hash.sched.traffic_class;
1909 }
1910 
1914 static inline uint8_t
1916 {
1917  return m->hash.sched.color;
1918 }
1919 
1932 static inline void
1933 rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id,
1934  uint8_t *traffic_class,
1935  uint8_t *color)
1936 {
1937  struct rte_mbuf_sched sched = m->hash.sched;
1938 
1939  *queue_id = sched.queue_id;
1940  *traffic_class = sched.traffic_class;
1941  *color = sched.color;
1942 }
1943 
1947 static inline void
1949 {
1950  m->hash.sched.queue_id = queue_id;
1951 }
1952 
1956 static inline void
1958 {
1959  m->hash.sched.traffic_class = traffic_class;
1960 }
1961 
1965 static inline void
1967 {
1968  m->hash.sched.color = color;
1969 }
1970 
1983 static inline void
1985  uint8_t traffic_class,
1986  uint8_t color)
1987 {
1988  m->hash.sched = (struct rte_mbuf_sched){
1989  .queue_id = queue_id,
1990  .traffic_class = traffic_class,
1991  .color = color,
1992  .reserved = 0,
1993  };
1994 }
1995 
1996 #ifdef __cplusplus
1997 }
1998 #endif
1999 
2000 #endif /* _RTE_MBUF_H_ */
static void rte_pktmbuf_reset(struct rte_mbuf *m)
Definition: rte_mbuf.h:861
struct rte_mbuf_ext_shared_info * shinfo
static rte_iova_t rte_mbuf_data_iova(const struct rte_mbuf *mb)
Definition: rte_mbuf.h:149
struct rte_mbuf * next
uint16_t mbuf_data_room_size
Definition: rte_mbuf.h:289
uint16_t vlan_tci_outer
#define __rte_always_inline
Definition: rte_common.h:226
static struct rte_mbuf * rte_pktmbuf_alloc(struct rte_mempool *mp)
Definition: rte_mbuf.h:892
static void rte_mbuf_sched_traffic_class_set(struct rte_mbuf *m, uint8_t traffic_class)
Definition: rte_mbuf.h:1957
static uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp)
Definition: rte_mbuf.h:831
static int rte_validate_tx_offload(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1820
__rte_experimental struct rte_mbuf * rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp, uint32_t offset, uint32_t length)
uint32_t queue_id
#define likely(x)
static void rte_pktmbuf_free(struct rte_mbuf *m)
Definition: rte_mbuf.h:1397
#define PKT_TX_OUTER_IP_CKSUM
static uint32_t rte_mbuf_sched_queue_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1897
#define RTE_PTR_ALIGN_FLOOR(ptr, align)
Definition: rte_common.h:277
void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg)
uint64_t rte_iova_t
Definition: rte_common.h:418
static __rte_always_inline void rte_pktmbuf_free_seg(struct rte_mbuf *m)
Definition: rte_mbuf.h:1381
void * buf_addr
static struct rte_mbuf * rte_mbuf_from_indirect(struct rte_mbuf *mi)
Definition: rte_mbuf.h:181
uint16_t data_len
rte_mbuf_extbuf_free_callback_t free_cb
static int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
Definition: rte_mbuf.h:1747
__rte_experimental void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count)
static uint8_t rte_mbuf_sched_traffic_class_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1906
#define PKT_TX_IPV4
static __rte_experimental char * rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
Definition: rte_mbuf.h:208
unsigned int flags
Definition: rte_mempool.h:221
#define __rte_unused
Definition: rte_common.h:116
__rte_experimental int rte_mbuf_check(const struct rte_mbuf *m, int is_header, const char **reason)
uint64_t tso_segsz
static __rte_always_inline void __rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m)
Definition: rte_mbuf.h:556
void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len)
static uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1500
static int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, struct rte_mbuf **mbufs, unsigned count)
Definition: rte_mbuf.h:914
static void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
Definition: rte_mbuf.h:847
uint32_t cache_size
Definition: rte_mempool.h:224
static void rte_mbuf_prefetch_part2(struct rte_mbuf *m)
Definition: rte_mbuf.h:128
#define PKT_TX_OUTER_IPV4
static uint16_t rte_mbuf_ext_refcnt_update(struct rte_mbuf_ext_shared_info *shinfo, int16_t value)
Definition: rte_mbuf.h:485
uint16_t nb_segs
#define IND_ATTACHED_MBUF
uint16_t port
#define rte_pktmbuf_mtod_offset(m, t, o)
static __rte_always_inline struct rte_mbuf * rte_pktmbuf_prefree_seg(struct rte_mbuf *m)
Definition: rte_mbuf.h:1329
static int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1690
#define RTE_PTR_ADD(ptr, x)
Definition: rte_common.h:248
int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen)
#define RTE_MBUF_MAX_NB_SEGS
static void rte_mbuf_sched_set(struct rte_mbuf *m, uint32_t queue_id, uint8_t traffic_class, uint8_t color)
Definition: rte_mbuf.h:1984
static uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1514
static __rte_always_inline void rte_mbuf_raw_free(struct rte_mbuf *m)
Definition: rte_mbuf.h:611
static void rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id, uint8_t *traffic_class, uint8_t *color)
Definition: rte_mbuf.h:1933
#define unlikely(x)
uint16_t priv_size
static void rte_mbuf_sched_queue_set(struct rte_mbuf *m, uint32_t queue_id)
Definition: rte_mbuf.h:1948
void rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header)
static void rte_mbuf_sched_color_set(struct rte_mbuf *m, uint8_t color)
Definition: rte_mbuf.h:1966
void(* rte_mbuf_extbuf_free_callback_t)(void *addr, void *opaque)
#define RTE_MIN(a, b)
Definition: rte_common.h:573
#define __rte_mbuf_sanity_check(m, is_h)
Definition: rte_mbuf.h:337
const char * rte_get_tx_ol_flag_name(uint64_t mask)
static uint16_t rte_mbuf_refcnt_read(const struct rte_mbuf *m)
Definition: rte_mbuf.h:428
#define PKT_TX_TCP_SEG
static int rte_pktmbuf_linearize(struct rte_mbuf *mbuf)
Definition: rte_mbuf.h:1870
static __rte_always_inline int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
Definition: rte_mempool.h:1550
static uint32_t rte_pktmbuf_priv_flags(struct rte_mempool *mp)
Definition: rte_mbuf.h:303
uint16_t elt_size
Definition: rte_mbuf.h:746
uint16_t refcnt
static char * rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1636
static __rte_experimental void * rte_mbuf_to_priv(struct rte_mbuf *m)
Definition: rte_mbuf.h:277
#define RTE_MBUF_DIRECT(mb)
#define RTE_MBUF_CLONED(mb)
static void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
Definition: rte_mbuf.h:1147
static __rte_always_inline int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:1521
#define PKT_TX_IP_CKSUM
uint64_t ol_flags
static void rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
Definition: rte_mbuf.h:1108
static void rte_pktmbuf_detach(struct rte_mbuf *m)
Definition: rte_mbuf.h:1237
static uint16_t rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
Definition: rte_mbuf.h:453
uint32_t pkt_len
uint16_t buf_len
#define rte_pktmbuf_data_len(m)
Definition: rte_mbuf.h:1555
static uint16_t rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
Definition: rte_mbuf.h:419
uint32_t packet_type
static uint16_t rte_pktmbuf_data_room_size(struct rte_mempool *mp)
Definition: rte_mbuf.h:810
#define RTE_MBUF_PORT_INVALID
const char * rte_get_rx_ol_flag_name(uint64_t mask)
static struct rte_mbuf_ext_shared_info * rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len, rte_mbuf_extbuf_free_callback_t free_cb, void *fcb_opaque)
Definition: rte_mbuf.h:989
#define PKT_TX_IPV6
#define PKT_TX_L4_MASK
void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg, void *m, unsigned i)
#define RTE_MBUF_HAS_EXTBUF(mb)
struct rte_mempool * pool
static void rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo, uint16_t new_value)
Definition: rte_mbuf.h:467
static char * rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1605
static void rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
Definition: rte_mbuf.h:437
struct rte_mempool * rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n, unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id, const char *ops_name)
static rte_iova_t rte_mbuf_data_iova_default(const struct rte_mbuf *mb)
Definition: rte_mbuf.h:167
static int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1666
static char * rte_mbuf_to_baddr(struct rte_mbuf *md)
Definition: rte_mbuf.h:252
static const void * rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off, uint32_t len, void *buf)
Definition: rte_mbuf.h:1722
static char * rte_pktmbuf_prepend(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1572
static struct rte_mbuf * rte_mbuf_raw_alloc(struct rte_mempool *mp)
Definition: rte_mbuf.h:586
static void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
Definition: rte_mbuf.h:1483
#define RTE_PTR_SUB(ptr, x)
Definition: rte_common.h:253
static struct rte_mbuf * rte_pktmbuf_lastseg(struct rte_mbuf *m)
Definition: rte_mbuf.h:1529
static __rte_experimental char * rte_mbuf_data_addr_default(__rte_unused struct rte_mbuf *mb)
Definition: rte_mbuf.h:226
__rte_experimental struct rte_mempool * rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n, unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id, const struct rte_pktmbuf_extmem *ext_mem, unsigned int ext_num)
static uint8_t rte_mbuf_sched_color_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1915
#define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF
Definition: rte_mbuf.h:317
static rte_iova_t rte_mempool_virt2iova(const void *elt)
Definition: rte_mempool.h:1673
static __rte_always_inline void rte_mempool_put(struct rte_mempool *mp, void *obj)
Definition: rte_mempool.h:1389
rte_iova_t buf_iova
Definition: rte_mbuf.h:744
uint8_t traffic_class
#define RTE_PTR_DIFF(ptr1, ptr2)
Definition: rte_common.h:260
static void rte_mbuf_prefetch_part1(struct rte_mbuf *m)
Definition: rte_mbuf.h:111
#define PKT_TX_OFFLOAD_MASK
static void * rte_mempool_get_priv(struct rte_mempool *mp)
Definition: rte_mempool.h:1701
uint64_t tx_offload
char name[RTE_MEMZONE_NAMESIZE]
Definition: rte_mempool.h:213
uint16_t vlan_tci
#define RTE_MBUF_HAS_PINNED_EXTBUF(mb)
Definition: rte_mbuf.h:326
struct rte_mbuf * rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp)
#define RTE_SET_USED(x)
Definition: rte_common.h:131
int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen)
static void rte_prefetch0(const volatile void *p)
struct rte_mempool * rte_pktmbuf_pool_create(const char *name, unsigned n, unsigned cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id)
uint32_t dynfield1[9]
#define EXT_ATTACHED_MBUF
static void rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr, rte_iova_t buf_iova, uint16_t buf_len, struct rte_mbuf_ext_shared_info *shinfo)
Definition: rte_mbuf.h:1071