DPDK  20.05.0
rte_mbuf.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2014 Intel Corporation.
3  * Copyright 2014 6WIND S.A.
4  */
5 
6 #ifndef _RTE_MBUF_H_
7 #define _RTE_MBUF_H_
8 
34 #include <stdint.h>
35 #include <rte_compat.h>
36 #include <rte_common.h>
37 #include <rte_config.h>
38 #include <rte_mempool.h>
39 #include <rte_memory.h>
40 #include <rte_atomic.h>
41 #include <rte_prefetch.h>
42 #include <rte_branch_prediction.h>
43 #include <rte_byteorder.h>
44 #include <rte_mbuf_ptype.h>
45 #include <rte_mbuf_core.h>
46 
47 #ifdef __cplusplus
48 extern "C" {
49 #endif
50 
59 const char *rte_get_rx_ol_flag_name(uint64_t mask);
60 
73 int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
74 
85 const char *rte_get_tx_ol_flag_name(uint64_t mask);
86 
99 int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
100 
111 static inline void
113 {
114  rte_prefetch0(&m->cacheline0);
115 }
116 
128 static inline void
130 {
131 #if RTE_CACHE_LINE_SIZE == 64
132  rte_prefetch0(&m->cacheline1);
133 #else
134  RTE_SET_USED(m);
135 #endif
136 }
137 
138 
139 static inline uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp);
140 
149 static inline rte_iova_t
150 rte_mbuf_data_iova(const struct rte_mbuf *mb)
151 {
152  return mb->buf_iova + mb->data_off;
153 }
154 
155 __rte_deprecated
156 static inline phys_addr_t
157 rte_mbuf_data_dma_addr(const struct rte_mbuf *mb)
158 {
159  return rte_mbuf_data_iova(mb);
160 }
161 
174 static inline rte_iova_t
176 {
177  return mb->buf_iova + RTE_PKTMBUF_HEADROOM;
178 }
179 
180 __rte_deprecated
181 static inline phys_addr_t
182 rte_mbuf_data_dma_addr_default(const struct rte_mbuf *mb)
183 {
184  return rte_mbuf_data_iova_default(mb);
185 }
186 
195 static inline struct rte_mbuf *
197 {
198  return (struct rte_mbuf *)RTE_PTR_SUB(mi->buf_addr, sizeof(*mi) + mi->priv_size);
199 }
200 
221 __rte_experimental
222 static inline char *
223 rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
224 {
225  return (char *)mb + sizeof(*mb) + rte_pktmbuf_priv_size(mp);
226 }
227 
239 __rte_experimental
240 static inline char *
242 {
243  /* gcc complains about calling this experimental function even
244  * when not using it. Hide it with ALLOW_EXPERIMENTAL_API.
245  */
246 #ifdef ALLOW_EXPERIMENTAL_API
247  return rte_mbuf_buf_addr(mb, mb->pool) + RTE_PKTMBUF_HEADROOM;
248 #else
249  return NULL;
250 #endif
251 }
252 
266 static inline char *
268 {
269 #ifdef ALLOW_EXPERIMENTAL_API
270  return rte_mbuf_buf_addr(md, md->pool);
271 #else
272  char *buffer_addr;
273  buffer_addr = (char *)md + sizeof(*md) + rte_pktmbuf_priv_size(md->pool);
274  return buffer_addr;
275 #endif
276 }
277 
290 __rte_experimental
291 static inline void *
293 {
294  return RTE_PTR_ADD(m, sizeof(struct rte_mbuf));
295 }
296 
305  uint16_t mbuf_priv_size;
306  uint32_t flags;
307 };
308 
317 static inline uint32_t
319 {
320  struct rte_pktmbuf_pool_private *mbp_priv;
321 
322  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
323  return mbp_priv->flags;
324 }
325 
332 #define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF (1 << 0)
333 
341 #define RTE_MBUF_HAS_PINNED_EXTBUF(mb) \
342  (rte_pktmbuf_priv_flags(mb->pool) & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF)
343 
344 #ifdef RTE_LIBRTE_MBUF_DEBUG
345 
347 #define __rte_mbuf_sanity_check(m, is_h) rte_mbuf_sanity_check(m, is_h)
348 
349 #else /* RTE_LIBRTE_MBUF_DEBUG */
350 
352 #define __rte_mbuf_sanity_check(m, is_h) do { } while (0)
353 
354 #endif /* RTE_LIBRTE_MBUF_DEBUG */
355 
356 #ifdef RTE_MBUF_REFCNT_ATOMIC
357 
365 static inline uint16_t
366 rte_mbuf_refcnt_read(const struct rte_mbuf *m)
367 {
368  return (uint16_t)(rte_atomic16_read(&m->refcnt_atomic));
369 }
370 
378 static inline void
379 rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
380 {
381  rte_atomic16_set(&m->refcnt_atomic, (int16_t)new_value);
382 }
383 
384 /* internal */
385 static inline uint16_t
386 __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
387 {
388  return (uint16_t)(rte_atomic16_add_return(&m->refcnt_atomic, value));
389 }
390 
400 static inline uint16_t
401 rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
402 {
403  /*
404  * The atomic_add is an expensive operation, so we don't want to
405  * call it in the case where we know we are the unique holder of
406  * this mbuf (i.e. ref_cnt == 1). Otherwise, an atomic
407  * operation has to be used because concurrent accesses on the
408  * reference counter can occur.
409  */
410  if (likely(rte_mbuf_refcnt_read(m) == 1)) {
411  ++value;
412  rte_mbuf_refcnt_set(m, (uint16_t)value);
413  return (uint16_t)value;
414  }
415 
416  return __rte_mbuf_refcnt_update(m, value);
417 }
418 
419 #else /* ! RTE_MBUF_REFCNT_ATOMIC */
420 
421 /* internal */
422 static inline uint16_t
423 __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
424 {
425  m->refcnt = (uint16_t)(m->refcnt + value);
426  return m->refcnt;
427 }
428 
432 static inline uint16_t
433 rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
434 {
435  return __rte_mbuf_refcnt_update(m, value);
436 }
437 
441 static inline uint16_t
443 {
444  return m->refcnt;
445 }
446 
450 static inline void
451 rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
452 {
453  m->refcnt = new_value;
454 }
455 
456 #endif /* RTE_MBUF_REFCNT_ATOMIC */
457 
466 static inline uint16_t
468 {
469  return (uint16_t)(rte_atomic16_read(&shinfo->refcnt_atomic));
470 }
471 
480 static inline void
482  uint16_t new_value)
483 {
484  rte_atomic16_set(&shinfo->refcnt_atomic, (int16_t)new_value);
485 }
486 
498 static inline uint16_t
500  int16_t value)
501 {
502  if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1)) {
503  ++value;
504  rte_mbuf_ext_refcnt_set(shinfo, (uint16_t)value);
505  return (uint16_t)value;
506  }
507 
508  return (uint16_t)rte_atomic16_add_return(&shinfo->refcnt_atomic, value);
509 }
510 
512 #define RTE_MBUF_PREFETCH_TO_FREE(m) do { \
513  if ((m) != NULL) \
514  rte_prefetch0(m); \
515 } while (0)
516 
517 
530 void
531 rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header);
532 
552 __rte_experimental
553 int rte_mbuf_check(const struct rte_mbuf *m, int is_header,
554  const char **reason);
555 
556 #define MBUF_RAW_ALLOC_CHECK(m) do { \
557  RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1); \
558  RTE_ASSERT((m)->next == NULL); \
559  RTE_ASSERT((m)->nb_segs == 1); \
560  __rte_mbuf_sanity_check(m, 0); \
561 } while (0)
562 
582 static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
583 {
584  struct rte_mbuf *m;
585 
586  if (rte_mempool_get(mp, (void **)&m) < 0)
587  return NULL;
588  MBUF_RAW_ALLOC_CHECK(m);
589  return m;
590 }
591 
606 static __rte_always_inline void
608 {
609  RTE_ASSERT(!RTE_MBUF_CLONED(m) &&
611  RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);
612  RTE_ASSERT(m->next == NULL);
613  RTE_ASSERT(m->nb_segs == 1);
615  rte_mempool_put(m->pool, m);
616 }
617 
637 void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg,
638  void *m, unsigned i);
639 
657 void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg);
658 
693 struct rte_mempool *
694 rte_pktmbuf_pool_create(const char *name, unsigned n,
695  unsigned cache_size, uint16_t priv_size, uint16_t data_room_size,
696  int socket_id);
697 
735 struct rte_mempool *
736 rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n,
737  unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size,
738  int socket_id, const char *ops_name);
739 
742  void *buf_ptr;
744  size_t buf_len;
745  uint16_t elt_size;
746 };
747 
789 __rte_experimental
790 struct rte_mempool *
791 rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n,
792  unsigned int cache_size, uint16_t priv_size,
793  uint16_t data_room_size, int socket_id,
794  const struct rte_pktmbuf_extmem *ext_mem,
795  unsigned int ext_num);
796 
808 static inline uint16_t
810 {
811  struct rte_pktmbuf_pool_private *mbp_priv;
812 
813  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
814  return mbp_priv->mbuf_data_room_size;
815 }
816 
829 static inline uint16_t
831 {
832  struct rte_pktmbuf_pool_private *mbp_priv;
833 
834  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
835  return mbp_priv->mbuf_priv_size;
836 }
837 
846 static inline void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
847 {
848  m->data_off = (uint16_t)RTE_MIN((uint16_t)RTE_PKTMBUF_HEADROOM,
849  (uint16_t)m->buf_len);
850 }
851 
860 #define MBUF_INVALID_PORT UINT16_MAX
861 
862 static inline void rte_pktmbuf_reset(struct rte_mbuf *m)
863 {
864  m->next = NULL;
865  m->pkt_len = 0;
866  m->tx_offload = 0;
867  m->vlan_tci = 0;
868  m->vlan_tci_outer = 0;
869  m->nb_segs = 1;
870  m->port = MBUF_INVALID_PORT;
871 
873  m->packet_type = 0;
875 
876  m->data_len = 0;
878 }
879 
893 static inline struct rte_mbuf *rte_pktmbuf_alloc(struct rte_mempool *mp)
894 {
895  struct rte_mbuf *m;
896  if ((m = rte_mbuf_raw_alloc(mp)) != NULL)
897  rte_pktmbuf_reset(m);
898  return m;
899 }
900 
915 static inline int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool,
916  struct rte_mbuf **mbufs, unsigned count)
917 {
918  unsigned idx = 0;
919  int rc;
920 
921  rc = rte_mempool_get_bulk(pool, (void **)mbufs, count);
922  if (unlikely(rc))
923  return rc;
924 
925  /* To understand duff's device on loop unwinding optimization, see
926  * https://en.wikipedia.org/wiki/Duff's_device.
927  * Here while() loop is used rather than do() while{} to avoid extra
928  * check if count is zero.
929  */
930  switch (count % 4) {
931  case 0:
932  while (idx != count) {
933  MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
934  rte_pktmbuf_reset(mbufs[idx]);
935  idx++;
936  /* fall-through */
937  case 3:
938  MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
939  rte_pktmbuf_reset(mbufs[idx]);
940  idx++;
941  /* fall-through */
942  case 2:
943  MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
944  rte_pktmbuf_reset(mbufs[idx]);
945  idx++;
946  /* fall-through */
947  case 1:
948  MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
949  rte_pktmbuf_reset(mbufs[idx]);
950  idx++;
951  /* fall-through */
952  }
953  }
954  return 0;
955 }
956 
989 static inline struct rte_mbuf_ext_shared_info *
990 rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len,
992 {
993  struct rte_mbuf_ext_shared_info *shinfo;
994  void *buf_end = RTE_PTR_ADD(buf_addr, *buf_len);
995  void *addr;
996 
997  addr = RTE_PTR_ALIGN_FLOOR(RTE_PTR_SUB(buf_end, sizeof(*shinfo)),
998  sizeof(uintptr_t));
999  if (addr <= buf_addr)
1000  return NULL;
1001 
1002  shinfo = (struct rte_mbuf_ext_shared_info *)addr;
1003  shinfo->free_cb = free_cb;
1004  shinfo->fcb_opaque = fcb_opaque;
1005  rte_mbuf_ext_refcnt_set(shinfo, 1);
1006 
1007  *buf_len = (uint16_t)RTE_PTR_DIFF(shinfo, buf_addr);
1008  return shinfo;
1009 }
1010 
1071 static inline void
1072 rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr,
1073  rte_iova_t buf_iova, uint16_t buf_len,
1074  struct rte_mbuf_ext_shared_info *shinfo)
1075 {
1076  /* mbuf should not be read-only */
1077  RTE_ASSERT(RTE_MBUF_DIRECT(m) && rte_mbuf_refcnt_read(m) == 1);
1078  RTE_ASSERT(shinfo->free_cb != NULL);
1079 
1080  m->buf_addr = buf_addr;
1081  m->buf_iova = buf_iova;
1082  m->buf_len = buf_len;
1083 
1084  m->data_len = 0;
1085  m->data_off = 0;
1086 
1088  m->shinfo = shinfo;
1089 }
1090 
1098 #define rte_pktmbuf_detach_extbuf(m) rte_pktmbuf_detach(m)
1099 
1108 static inline void
1109 rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
1110 {
1111  memcpy(&mdst->dynfield1, msrc->dynfield1, sizeof(mdst->dynfield1));
1112 }
1113 
1114 /* internal */
1115 static inline void
1116 __rte_pktmbuf_copy_hdr(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
1117 {
1118  mdst->port = msrc->port;
1119  mdst->vlan_tci = msrc->vlan_tci;
1120  mdst->vlan_tci_outer = msrc->vlan_tci_outer;
1121  mdst->tx_offload = msrc->tx_offload;
1122  mdst->hash = msrc->hash;
1123  mdst->packet_type = msrc->packet_type;
1124  mdst->timestamp = msrc->timestamp;
1125  rte_mbuf_dynfield_copy(mdst, msrc);
1126 }
1127 
1149 static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
1150 {
1151  RTE_ASSERT(RTE_MBUF_DIRECT(mi) &&
1152  rte_mbuf_refcnt_read(mi) == 1);
1153 
1154  if (RTE_MBUF_HAS_EXTBUF(m)) {
1156  mi->ol_flags = m->ol_flags;
1157  mi->shinfo = m->shinfo;
1158  } else {
1159  /* if m is not direct, get the mbuf that embeds the data */
1161  mi->priv_size = m->priv_size;
1162  mi->ol_flags = m->ol_flags | IND_ATTACHED_MBUF;
1163  }
1164 
1165  __rte_pktmbuf_copy_hdr(mi, m);
1166 
1167  mi->data_off = m->data_off;
1168  mi->data_len = m->data_len;
1169  mi->buf_iova = m->buf_iova;
1170  mi->buf_addr = m->buf_addr;
1171  mi->buf_len = m->buf_len;
1172 
1173  mi->next = NULL;
1174  mi->pkt_len = mi->data_len;
1175  mi->nb_segs = 1;
1176 
1177  __rte_mbuf_sanity_check(mi, 1);
1179 }
1180 
1188 static inline void
1189 __rte_pktmbuf_free_extbuf(struct rte_mbuf *m)
1190 {
1191  RTE_ASSERT(RTE_MBUF_HAS_EXTBUF(m));
1192  RTE_ASSERT(m->shinfo != NULL);
1193 
1194  if (rte_mbuf_ext_refcnt_update(m->shinfo, -1) == 0)
1195  m->shinfo->free_cb(m->buf_addr, m->shinfo->fcb_opaque);
1196 }
1197 
1204 static inline void
1205 __rte_pktmbuf_free_direct(struct rte_mbuf *m)
1206 {
1207  struct rte_mbuf *md;
1208 
1209  RTE_ASSERT(RTE_MBUF_CLONED(m));
1210 
1211  md = rte_mbuf_from_indirect(m);
1212 
1213  if (rte_mbuf_refcnt_update(md, -1) == 0) {
1214  md->next = NULL;
1215  md->nb_segs = 1;
1216  rte_mbuf_refcnt_set(md, 1);
1217  rte_mbuf_raw_free(md);
1218  }
1219 }
1220 
1239 static inline void rte_pktmbuf_detach(struct rte_mbuf *m)
1240 {
1241  struct rte_mempool *mp = m->pool;
1242  uint32_t mbuf_size, buf_len;
1243  uint16_t priv_size;
1244 
1245  if (RTE_MBUF_HAS_EXTBUF(m)) {
1246  /*
1247  * The mbuf has the external attached buffer,
1248  * we should check the type of the memory pool where
1249  * the mbuf was allocated from to detect the pinned
1250  * external buffer.
1251  */
1252  uint32_t flags = rte_pktmbuf_priv_flags(mp);
1253 
1254  if (flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) {
1255  /*
1256  * The pinned external buffer should not be
1257  * detached from its backing mbuf, just exit.
1258  */
1259  return;
1260  }
1261  __rte_pktmbuf_free_extbuf(m);
1262  } else {
1263  __rte_pktmbuf_free_direct(m);
1264  }
1265  priv_size = rte_pktmbuf_priv_size(mp);
1266  mbuf_size = (uint32_t)(sizeof(struct rte_mbuf) + priv_size);
1267  buf_len = rte_pktmbuf_data_room_size(mp);
1268 
1269  m->priv_size = priv_size;
1270  m->buf_addr = (char *)m + mbuf_size;
1271  m->buf_iova = rte_mempool_virt2iova(m) + mbuf_size;
1272  m->buf_len = (uint16_t)buf_len;
1274  m->data_len = 0;
1275  m->ol_flags = 0;
1276 }
1277 
1291 static inline int __rte_pktmbuf_pinned_extbuf_decref(struct rte_mbuf *m)
1292 {
1293  struct rte_mbuf_ext_shared_info *shinfo;
1294 
1295  /* Clear flags, mbuf is being freed. */
1297  shinfo = m->shinfo;
1298 
1299  /* Optimize for performance - do not dec/reinit */
1300  if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1))
1301  return 0;
1302 
1303  /*
1304  * Direct usage of add primitive to avoid
1305  * duplication of comparing with one.
1306  */
1308  (&shinfo->refcnt_atomic, -1)))
1309  return 1;
1310 
1311  /* Reinitialize counter before mbuf freeing. */
1312  rte_mbuf_ext_refcnt_set(shinfo, 1);
1313  return 0;
1314 }
1315 
1330 static __rte_always_inline struct rte_mbuf *
1332 {
1334 
1335  if (likely(rte_mbuf_refcnt_read(m) == 1)) {
1336 
1337  if (!RTE_MBUF_DIRECT(m)) {
1338  rte_pktmbuf_detach(m);
1339  if (RTE_MBUF_HAS_EXTBUF(m) &&
1341  __rte_pktmbuf_pinned_extbuf_decref(m))
1342  return NULL;
1343  }
1344 
1345  if (m->next != NULL) {
1346  m->next = NULL;
1347  m->nb_segs = 1;
1348  }
1349 
1350  return m;
1351 
1352  } else if (__rte_mbuf_refcnt_update(m, -1) == 0) {
1353 
1354  if (!RTE_MBUF_DIRECT(m)) {
1355  rte_pktmbuf_detach(m);
1356  if (RTE_MBUF_HAS_EXTBUF(m) &&
1358  __rte_pktmbuf_pinned_extbuf_decref(m))
1359  return NULL;
1360  }
1361 
1362  if (m->next != NULL) {
1363  m->next = NULL;
1364  m->nb_segs = 1;
1365  }
1366  rte_mbuf_refcnt_set(m, 1);
1367 
1368  return m;
1369  }
1370  return NULL;
1371 }
1372 
1382 static __rte_always_inline void
1384 {
1385  m = rte_pktmbuf_prefree_seg(m);
1386  if (likely(m != NULL))
1387  rte_mbuf_raw_free(m);
1388 }
1389 
1399 static inline void rte_pktmbuf_free(struct rte_mbuf *m)
1400 {
1401  struct rte_mbuf *m_next;
1402 
1403  if (m != NULL)
1405 
1406  while (m != NULL) {
1407  m_next = m->next;
1409  m = m_next;
1410  }
1411 }
1412 
1425 __rte_experimental
1426 void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count);
1427 
1445 struct rte_mbuf *
1446 rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp);
1447 
1469 __rte_experimental
1470 struct rte_mbuf *
1471 rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp,
1472  uint32_t offset, uint32_t length);
1473 
1485 static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
1486 {
1488 
1489  do {
1490  rte_mbuf_refcnt_update(m, v);
1491  } while ((m = m->next) != NULL);
1492 }
1493 
1502 static inline uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
1503 {
1505  return m->data_off;
1506 }
1507 
1516 static inline uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
1517 {
1519  return (uint16_t)(m->buf_len - rte_pktmbuf_headroom(m) -
1520  m->data_len);
1521 }
1522 
1531 static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m)
1532 {
1534  while (m->next != NULL)
1535  m = m->next;
1536  return m;
1537 }
1538 
1539 /* deprecated */
1540 #define rte_pktmbuf_mtophys_offset(m, o) \
1541  rte_pktmbuf_iova_offset(m, o)
1542 
1543 /* deprecated */
1544 #define rte_pktmbuf_mtophys(m) rte_pktmbuf_iova(m)
1545 
1554 #define rte_pktmbuf_pkt_len(m) ((m)->pkt_len)
1555 
1564 #define rte_pktmbuf_data_len(m) ((m)->data_len)
1565 
1581 static inline char *rte_pktmbuf_prepend(struct rte_mbuf *m,
1582  uint16_t len)
1583 {
1585 
1586  if (unlikely(len > rte_pktmbuf_headroom(m)))
1587  return NULL;
1588 
1589  /* NB: elaborating the subtraction like this instead of using
1590  * -= allows us to ensure the result type is uint16_t
1591  * avoiding compiler warnings on gcc 8.1 at least */
1592  m->data_off = (uint16_t)(m->data_off - len);
1593  m->data_len = (uint16_t)(m->data_len + len);
1594  m->pkt_len = (m->pkt_len + len);
1595 
1596  return (char *)m->buf_addr + m->data_off;
1597 }
1598 
1614 static inline char *rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
1615 {
1616  void *tail;
1617  struct rte_mbuf *m_last;
1618 
1620 
1621  m_last = rte_pktmbuf_lastseg(m);
1622  if (unlikely(len > rte_pktmbuf_tailroom(m_last)))
1623  return NULL;
1624 
1625  tail = (char *)m_last->buf_addr + m_last->data_off + m_last->data_len;
1626  m_last->data_len = (uint16_t)(m_last->data_len + len);
1627  m->pkt_len = (m->pkt_len + len);
1628  return (char*) tail;
1629 }
1630 
1645 static inline char *rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
1646 {
1648 
1649  if (unlikely(len > m->data_len))
1650  return NULL;
1651 
1652  /* NB: elaborating the addition like this instead of using
1653  * += allows us to ensure the result type is uint16_t
1654  * avoiding compiler warnings on gcc 8.1 at least */
1655  m->data_len = (uint16_t)(m->data_len - len);
1656  m->data_off = (uint16_t)(m->data_off + len);
1657  m->pkt_len = (m->pkt_len - len);
1658  return (char *)m->buf_addr + m->data_off;
1659 }
1660 
1675 static inline int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
1676 {
1677  struct rte_mbuf *m_last;
1678 
1680 
1681  m_last = rte_pktmbuf_lastseg(m);
1682  if (unlikely(len > m_last->data_len))
1683  return -1;
1684 
1685  m_last->data_len = (uint16_t)(m_last->data_len - len);
1686  m->pkt_len = (m->pkt_len - len);
1687  return 0;
1688 }
1689 
1699 static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
1700 {
1702  return m->nb_segs == 1;
1703 }
1704 
1708 const void *__rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off,
1709  uint32_t len, void *buf);
1710 
1731 static inline const void *rte_pktmbuf_read(const struct rte_mbuf *m,
1732  uint32_t off, uint32_t len, void *buf)
1733 {
1734  if (likely(off + len <= rte_pktmbuf_data_len(m)))
1735  return rte_pktmbuf_mtod_offset(m, char *, off);
1736  else
1737  return __rte_pktmbuf_read(m, off, len, buf);
1738 }
1739 
1756 static inline int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
1757 {
1758  struct rte_mbuf *cur_tail;
1759 
1760  /* Check for number-of-segments-overflow */
1761  if (head->nb_segs + tail->nb_segs > RTE_MBUF_MAX_NB_SEGS)
1762  return -EOVERFLOW;
1763 
1764  /* Chain 'tail' onto the old tail */
1765  cur_tail = rte_pktmbuf_lastseg(head);
1766  cur_tail->next = tail;
1767 
1768  /* accumulate number of segments and total length.
1769  * NB: elaborating the addition like this instead of using
1770  * -= allows us to ensure the result type is uint16_t
1771  * avoiding compiler warnings on gcc 8.1 at least */
1772  head->nb_segs = (uint16_t)(head->nb_segs + tail->nb_segs);
1773  head->pkt_len += tail->pkt_len;
1774 
1775  /* pkt_len is only set in the head */
1776  tail->pkt_len = tail->data_len;
1777 
1778  return 0;
1779 }
1780 
1781 /*
1782  * @warning
1783  * @b EXPERIMENTAL: This API may change without prior notice.
1784  *
1785  * For given input values generate raw tx_offload value.
1786  * Note that it is caller responsibility to make sure that input parameters
1787  * don't exceed maximum bit-field values.
1788  * @param il2
1789  * l2_len value.
1790  * @param il3
1791  * l3_len value.
1792  * @param il4
1793  * l4_len value.
1794  * @param tso
1795  * tso_segsz value.
1796  * @param ol3
1797  * outer_l3_len value.
1798  * @param ol2
1799  * outer_l2_len value.
1800  * @param unused
1801  * unused value.
1802  * @return
1803  * raw tx_offload value.
1804  */
1805 static __rte_always_inline uint64_t
1806 rte_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso,
1807  uint64_t ol3, uint64_t ol2, uint64_t unused)
1808 {
1809  return il2 << RTE_MBUF_L2_LEN_OFS |
1810  il3 << RTE_MBUF_L3_LEN_OFS |
1811  il4 << RTE_MBUF_L4_LEN_OFS |
1812  tso << RTE_MBUF_TSO_SEGSZ_OFS |
1813  ol3 << RTE_MBUF_OUTL3_LEN_OFS |
1814  ol2 << RTE_MBUF_OUTL2_LEN_OFS |
1815  unused << RTE_MBUF_TXOFLD_UNUSED_OFS;
1816 }
1817 
1828 static inline int
1830 {
1831  uint64_t ol_flags = m->ol_flags;
1832 
1833  /* Does packet set any of available offloads? */
1834  if (!(ol_flags & PKT_TX_OFFLOAD_MASK))
1835  return 0;
1836 
1837  /* IP checksum can be counted only for IPv4 packet */
1838  if ((ol_flags & PKT_TX_IP_CKSUM) && (ol_flags & PKT_TX_IPV6))
1839  return -EINVAL;
1840 
1841  /* IP type not set when required */
1842  if (ol_flags & (PKT_TX_L4_MASK | PKT_TX_TCP_SEG))
1843  if (!(ol_flags & (PKT_TX_IPV4 | PKT_TX_IPV6)))
1844  return -EINVAL;
1845 
1846  /* Check requirements for TSO packet */
1847  if (ol_flags & PKT_TX_TCP_SEG)
1848  if ((m->tso_segsz == 0) ||
1849  ((ol_flags & PKT_TX_IPV4) &&
1850  !(ol_flags & PKT_TX_IP_CKSUM)))
1851  return -EINVAL;
1852 
1853  /* PKT_TX_OUTER_IP_CKSUM set for non outer IPv4 packet. */
1854  if ((ol_flags & PKT_TX_OUTER_IP_CKSUM) &&
1855  !(ol_flags & PKT_TX_OUTER_IPV4))
1856  return -EINVAL;
1857 
1858  return 0;
1859 }
1860 
1864 int __rte_pktmbuf_linearize(struct rte_mbuf *mbuf);
1865 
1878 static inline int
1880 {
1881  if (rte_pktmbuf_is_contiguous(mbuf))
1882  return 0;
1883  return __rte_pktmbuf_linearize(mbuf);
1884 }
1885 
1900 void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len);
1901 
1905 static inline uint32_t
1907 {
1908  return m->hash.sched.queue_id;
1909 }
1910 
1914 static inline uint8_t
1916 {
1917  return m->hash.sched.traffic_class;
1918 }
1919 
1923 static inline uint8_t
1925 {
1926  return m->hash.sched.color;
1927 }
1928 
1941 static inline void
1942 rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id,
1943  uint8_t *traffic_class,
1944  uint8_t *color)
1945 {
1946  struct rte_mbuf_sched sched = m->hash.sched;
1947 
1948  *queue_id = sched.queue_id;
1949  *traffic_class = sched.traffic_class;
1950  *color = sched.color;
1951 }
1952 
1956 static inline void
1958 {
1959  m->hash.sched.queue_id = queue_id;
1960 }
1961 
1965 static inline void
1967 {
1968  m->hash.sched.traffic_class = traffic_class;
1969 }
1970 
1974 static inline void
1976 {
1977  m->hash.sched.color = color;
1978 }
1979 
1992 static inline void
1994  uint8_t traffic_class,
1995  uint8_t color)
1996 {
1997  m->hash.sched = (struct rte_mbuf_sched){
1998  .queue_id = queue_id,
1999  .traffic_class = traffic_class,
2000  .color = color,
2001  .reserved = 0,
2002  };
2003 }
2004 
2005 #ifdef __cplusplus
2006 }
2007 #endif
2008 
2009 #endif /* _RTE_MBUF_H_ */
struct rte_mbuf_ext_shared_info * shinfo
static rte_iova_t rte_mbuf_data_iova(const struct rte_mbuf *mb)
Definition: rte_mbuf.h:150
struct rte_mbuf * next
uint16_t mbuf_data_room_size
Definition: rte_mbuf.h:304
uint64_t timestamp
uint16_t vlan_tci_outer
#define __rte_always_inline
Definition: rte_common.h:193
static int16_t rte_atomic16_read(const rte_atomic16_t *v)
Definition: rte_atomic.h:253
static struct rte_mbuf * rte_pktmbuf_alloc(struct rte_mempool *mp)
Definition: rte_mbuf.h:893
static void rte_mbuf_sched_traffic_class_set(struct rte_mbuf *m, uint8_t traffic_class)
Definition: rte_mbuf.h:1966
static uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp)
Definition: rte_mbuf.h:830
static int rte_validate_tx_offload(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1829
__rte_experimental struct rte_mbuf * rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp, uint32_t offset, uint32_t length)
uint32_t queue_id
#define likely(x)
static void rte_pktmbuf_free(struct rte_mbuf *m)
Definition: rte_mbuf.h:1399
#define PKT_TX_OUTER_IP_CKSUM
static uint32_t rte_mbuf_sched_queue_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1906
#define RTE_PTR_ALIGN_FLOOR(ptr, align)
Definition: rte_common.h:244
void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg)
uint64_t rte_iova_t
Definition: rte_common.h:385
static __rte_always_inline void rte_pktmbuf_free_seg(struct rte_mbuf *m)
Definition: rte_mbuf.h:1383
void * buf_addr
static struct rte_mbuf * rte_mbuf_from_indirect(struct rte_mbuf *mi)
Definition: rte_mbuf.h:196
uint16_t data_len
rte_mbuf_extbuf_free_callback_t free_cb
static int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
Definition: rte_mbuf.h:1756
__rte_experimental void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count)
static uint8_t rte_mbuf_sched_traffic_class_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1915
#define PKT_TX_IPV4
static __rte_experimental char * rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
Definition: rte_mbuf.h:223
unsigned int flags
Definition: rte_mempool.h:232
#define __rte_unused
Definition: rte_common.h:104
__rte_experimental int rte_mbuf_check(const struct rte_mbuf *m, int is_header, const char **reason)
uint64_t tso_segsz
void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len)
static uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1502
static int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, struct rte_mbuf **mbufs, unsigned count)
Definition: rte_mbuf.h:915
static void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
Definition: rte_mbuf.h:846
uint32_t cache_size
Definition: rte_mempool.h:235
static void rte_mbuf_prefetch_part2(struct rte_mbuf *m)
Definition: rte_mbuf.h:129
#define PKT_TX_OUTER_IPV4
static uint16_t rte_mbuf_ext_refcnt_update(struct rte_mbuf_ext_shared_info *shinfo, int16_t value)
Definition: rte_mbuf.h:499
uint16_t nb_segs
#define IND_ATTACHED_MBUF
uint16_t port
#define rte_pktmbuf_mtod_offset(m, t, o)
static __rte_always_inline struct rte_mbuf * rte_pktmbuf_prefree_seg(struct rte_mbuf *m)
Definition: rte_mbuf.h:1331
static int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1699
#define RTE_PTR_ADD(ptr, x)
Definition: rte_common.h:215
int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen)
static void rte_mbuf_sched_set(struct rte_mbuf *m, uint32_t queue_id, uint8_t traffic_class, uint8_t color)
Definition: rte_mbuf.h:1993
static uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1516
static __rte_always_inline void rte_mbuf_raw_free(struct rte_mbuf *m)
Definition: rte_mbuf.h:607
static void rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id, uint8_t *traffic_class, uint8_t *color)
Definition: rte_mbuf.h:1942
#define unlikely(x)
uint16_t priv_size
static void rte_mbuf_sched_queue_set(struct rte_mbuf *m, uint32_t queue_id)
Definition: rte_mbuf.h:1957
void rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header)
static void rte_mbuf_sched_color_set(struct rte_mbuf *m, uint8_t color)
Definition: rte_mbuf.h:1975
void(* rte_mbuf_extbuf_free_callback_t)(void *addr, void *opaque)
#define RTE_MIN(a, b)
Definition: rte_common.h:540
#define __rte_mbuf_sanity_check(m, is_h)
Definition: rte_mbuf.h:352
const char * rte_get_tx_ol_flag_name(uint64_t mask)
static uint16_t rte_mbuf_refcnt_read(const struct rte_mbuf *m)
Definition: rte_mbuf.h:442
#define PKT_TX_TCP_SEG
static int rte_pktmbuf_linearize(struct rte_mbuf *mbuf)
Definition: rte_mbuf.h:1879
static __rte_always_inline int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
Definition: rte_mempool.h:1582
uint64_t phys_addr_t
Definition: rte_common.h:375
static void rte_atomic16_set(rte_atomic16_t *v, int16_t new_value)
Definition: rte_atomic.h:267
static uint32_t rte_pktmbuf_priv_flags(struct rte_mempool *mp)
Definition: rte_mbuf.h:318
uint16_t elt_size
Definition: rte_mbuf.h:745
uint16_t refcnt
static char * rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1645
static __rte_experimental void * rte_mbuf_to_priv(struct rte_mbuf *m)
Definition: rte_mbuf.h:292
#define RTE_MBUF_DIRECT(mb)
#define RTE_MBUF_CLONED(mb)
static void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
Definition: rte_mbuf.h:1149
static __rte_always_inline int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:1553
#define PKT_TX_IP_CKSUM
static int16_t rte_atomic16_add_return(rte_atomic16_t *v, int16_t inc)
Definition: rte_atomic.h:348
uint64_t ol_flags
static void rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
Definition: rte_mbuf.h:1109
static void rte_pktmbuf_detach(struct rte_mbuf *m)
Definition: rte_mbuf.h:1239
static uint16_t rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
Definition: rte_mbuf.h:467
uint32_t pkt_len
uint64_t dynfield1[2]
uint16_t buf_len
#define rte_pktmbuf_data_len(m)
Definition: rte_mbuf.h:1564
static uint16_t rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
Definition: rte_mbuf.h:433
uint32_t packet_type
#define MBUF_INVALID_PORT
Definition: rte_mbuf.h:860
static uint16_t rte_pktmbuf_data_room_size(struct rte_mempool *mp)
Definition: rte_mbuf.h:809
const char * rte_get_rx_ol_flag_name(uint64_t mask)
static struct rte_mbuf_ext_shared_info * rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len, rte_mbuf_extbuf_free_callback_t free_cb, void *fcb_opaque)
Definition: rte_mbuf.h:990
#define PKT_TX_IPV6
#define PKT_TX_L4_MASK
void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg, void *m, unsigned i)
#define RTE_MBUF_HAS_EXTBUF(mb)
struct rte_mempool * pool
static void rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo, uint16_t new_value)
Definition: rte_mbuf.h:481
static char * rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1614
static void rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
Definition: rte_mbuf.h:451
struct rte_mempool * rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n, unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id, const char *ops_name)
static rte_iova_t rte_mbuf_data_iova_default(const struct rte_mbuf *mb)
Definition: rte_mbuf.h:175
static int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1675
static char * rte_mbuf_to_baddr(struct rte_mbuf *md)
Definition: rte_mbuf.h:267
static const void * rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off, uint32_t len, void *buf)
Definition: rte_mbuf.h:1731
static char * rte_pktmbuf_prepend(struct rte_mbuf *m, uint16_t len)
Definition: rte_mbuf.h:1581
static struct rte_mbuf * rte_mbuf_raw_alloc(struct rte_mempool *mp)
Definition: rte_mbuf.h:582
static void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
Definition: rte_mbuf.h:1485
#define RTE_PTR_SUB(ptr, x)
Definition: rte_common.h:220
static struct rte_mbuf * rte_pktmbuf_lastseg(struct rte_mbuf *m)
Definition: rte_mbuf.h:1531
static __rte_experimental char * rte_mbuf_data_addr_default(__rte_unused struct rte_mbuf *mb)
Definition: rte_mbuf.h:241
__rte_experimental struct rte_mempool * rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n, unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id, const struct rte_pktmbuf_extmem *ext_mem, unsigned int ext_num)
static uint8_t rte_mbuf_sched_color_get(const struct rte_mbuf *m)
Definition: rte_mbuf.h:1924
#define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF
Definition: rte_mbuf.h:332
static rte_iova_t rte_mempool_virt2iova(const void *elt)
Definition: rte_mempool.h:1709
static __rte_always_inline void rte_mempool_put(struct rte_mempool *mp, void *obj)
Definition: rte_mempool.h:1421
rte_iova_t buf_iova
Definition: rte_mbuf.h:743
uint8_t traffic_class
#define RTE_PTR_DIFF(ptr1, ptr2)
Definition: rte_common.h:227
static void rte_mbuf_prefetch_part1(struct rte_mbuf *m)
Definition: rte_mbuf.h:112
rte_atomic16_t refcnt_atomic
#define PKT_TX_OFFLOAD_MASK
static void * rte_mempool_get_priv(struct rte_mempool *mp)
Definition: rte_mempool.h:1737
uint64_t tx_offload
char name[RTE_MEMZONE_NAMESIZE]
Definition: rte_mempool.h:224
uint16_t vlan_tci
#define RTE_MBUF_HAS_PINNED_EXTBUF(mb)
Definition: rte_mbuf.h:341
rte_atomic16_t refcnt_atomic
struct rte_mbuf * rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp)
#define RTE_SET_USED(x)
Definition: rte_common.h:110
int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen)
static void rte_prefetch0(const volatile void *p)
struct rte_mempool * rte_pktmbuf_pool_create(const char *name, unsigned n, unsigned cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id)
#define EXT_ATTACHED_MBUF
static void rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr, rte_iova_t buf_iova, uint16_t buf_len, struct rte_mbuf_ext_shared_info *shinfo)
Definition: rte_mbuf.h:1072