69 #pragma GCC poison RTE_MBUF_SCATTER_GATHER
70 #pragma GCC poison RTE_MBUF_REFCNT
86 #define PKT_RX_VLAN_PKT (1ULL << 0)
87 #define PKT_RX_RSS_HASH (1ULL << 1)
88 #define PKT_RX_FDIR (1ULL << 2)
89 #define PKT_RX_L4_CKSUM_BAD (1ULL << 3)
90 #define PKT_RX_IP_CKSUM_BAD (1ULL << 4)
91 #define PKT_RX_EIP_CKSUM_BAD (1ULL << 5)
92 #define PKT_RX_OVERSIZE (0ULL << 0)
93 #define PKT_RX_HBUF_OVERFLOW (0ULL << 0)
94 #define PKT_RX_RECIP_ERR (0ULL << 0)
95 #define PKT_RX_MAC_ERR (0ULL << 0)
96 #define PKT_RX_IEEE1588_PTP (1ULL << 9)
97 #define PKT_RX_IEEE1588_TMST (1ULL << 10)
98 #define PKT_RX_FDIR_ID (1ULL << 13)
99 #define PKT_RX_FDIR_FLX (1ULL << 14)
100 #define PKT_RX_QINQ_PKT (1ULL << 15)
108 #define PKT_TX_QINQ_PKT (1ULL << 49)
123 #define PKT_TX_TCP_SEG (1ULL << 50)
125 #define PKT_TX_IEEE1588_TMST (1ULL << 51)
138 #define PKT_TX_L4_NO_CKSUM (0ULL << 52)
139 #define PKT_TX_TCP_CKSUM (1ULL << 52)
140 #define PKT_TX_SCTP_CKSUM (2ULL << 52)
141 #define PKT_TX_UDP_CKSUM (3ULL << 52)
142 #define PKT_TX_L4_MASK (3ULL << 52)
151 #define PKT_TX_IP_CKSUM (1ULL << 54)
159 #define PKT_TX_IPV4 (1ULL << 55)
167 #define PKT_TX_IPV6 (1ULL << 56)
169 #define PKT_TX_VLAN_PKT (1ULL << 57)
179 #define PKT_TX_OUTER_IP_CKSUM (1ULL << 58)
186 #define PKT_TX_OUTER_IPV4 (1ULL << 59)
193 #define PKT_TX_OUTER_IPV6 (1ULL << 60)
195 #define __RESERVED (1ULL << 61)
197 #define IND_ATTACHED_MBUF (1ULL << 62)
200 #define CTRL_MBUF_FLAG (1ULL << 63)
250 #define RTE_PTYPE_UNKNOWN 0x00000000
258 #define RTE_PTYPE_L2_ETHER 0x00000001
265 #define RTE_PTYPE_L2_ETHER_TIMESYNC 0x00000002
272 #define RTE_PTYPE_L2_ETHER_ARP 0x00000003
279 #define RTE_PTYPE_L2_ETHER_LLDP 0x00000004
284 #define RTE_PTYPE_L2_MASK 0x0000000f
294 #define RTE_PTYPE_L3_IPV4 0x00000010
304 #define RTE_PTYPE_L3_IPV4_EXT 0x00000030
314 #define RTE_PTYPE_L3_IPV6 0x00000040
324 #define RTE_PTYPE_L3_IPV4_EXT_UNKNOWN 0x00000090
335 #define RTE_PTYPE_L3_IPV6_EXT 0x000000c0
346 #define RTE_PTYPE_L3_IPV6_EXT_UNKNOWN 0x000000e0
351 #define RTE_PTYPE_L3_MASK 0x000000f0
363 #define RTE_PTYPE_L4_TCP 0x00000100
375 #define RTE_PTYPE_L4_UDP 0x00000200
392 #define RTE_PTYPE_L4_FRAG 0x00000300
404 #define RTE_PTYPE_L4_SCTP 0x00000400
416 #define RTE_PTYPE_L4_ICMP 0x00000500
432 #define RTE_PTYPE_L4_NONFRAG 0x00000600
437 #define RTE_PTYPE_L4_MASK 0x00000f00
448 #define RTE_PTYPE_TUNNEL_IP 0x00001000
459 #define RTE_PTYPE_TUNNEL_GRE 0x00002000
472 #define RTE_PTYPE_TUNNEL_VXLAN 0x00003000
486 #define RTE_PTYPE_TUNNEL_NVGRE 0x00004000
499 #define RTE_PTYPE_TUNNEL_GENEVE 0x00005000
506 #define RTE_PTYPE_TUNNEL_GRENAT 0x00006000
510 #define RTE_PTYPE_TUNNEL_MASK 0x0000f000
518 #define RTE_PTYPE_INNER_L2_ETHER 0x00010000
525 #define RTE_PTYPE_INNER_L2_ETHER_VLAN 0x00020000
529 #define RTE_PTYPE_INNER_L2_MASK 0x000f0000
538 #define RTE_PTYPE_INNER_L3_IPV4 0x00100000
547 #define RTE_PTYPE_INNER_L3_IPV4_EXT 0x00200000
556 #define RTE_PTYPE_INNER_L3_IPV6 0x00300000
565 #define RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN 0x00400000
575 #define RTE_PTYPE_INNER_L3_IPV6_EXT 0x00500000
586 #define RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN 0x00600000
590 #define RTE_PTYPE_INNER_L3_MASK 0x00f00000
602 #define RTE_PTYPE_INNER_L4_TCP 0x01000000
614 #define RTE_PTYPE_INNER_L4_UDP 0x02000000
626 #define RTE_PTYPE_INNER_L4_FRAG 0x03000000
638 #define RTE_PTYPE_INNER_L4_SCTP 0x04000000
650 #define RTE_PTYPE_INNER_L4_ICMP 0x05000000
663 #define RTE_PTYPE_INNER_L4_NONFRAG 0x06000000
667 #define RTE_PTYPE_INNER_L4_MASK 0x0f000000
674 #define RTE_ETH_IS_IPV4_HDR(ptype) ((ptype) & RTE_PTYPE_L3_IPV4)
681 #define RTE_ETH_IS_IPV6_HDR(ptype) ((ptype) & RTE_PTYPE_L3_IPV6)
684 #define RTE_ETH_IS_TUNNEL_PKT(ptype) ((ptype) & (RTE_PTYPE_TUNNEL_MASK | \
685 RTE_PTYPE_INNER_L2_MASK | \
686 RTE_PTYPE_INNER_L3_MASK | \
687 RTE_PTYPE_INNER_L4_MASK))
690 #define RTE_MBUF_PRIV_ALIGN 8
720 #define RTE_MBUF_DEFAULT_DATAROOM 2048
721 #define RTE_MBUF_DEFAULT_BUF_SIZE \
722 (RTE_MBUF_DEFAULT_DATAROOM + RTE_PKTMBUF_HEADROOM)
764 MARKER rx_descriptor_fields1;
814 MARKER cacheline1 __rte_cache_min_aligned;
916 #define RTE_MBUF_INDIRECT(mb) ((mb)->ol_flags & IND_ATTACHED_MBUF)
921 #define RTE_MBUF_DIRECT(mb) (!RTE_MBUF_INDIRECT(mb))
934 #ifdef RTE_LIBRTE_MBUF_DEBUG
937 #define __rte_mbuf_sanity_check(m, is_h) rte_mbuf_sanity_check(m, is_h)
940 #define __rte_mbuf_sanity_check_raw(m, is_h) do { \
942 rte_mbuf_sanity_check(m, is_h); \
946 #define RTE_MBUF_ASSERT(exp) \
948 rte_panic("line%d\tassert \"" #exp "\" failed\n", __LINE__); \
954 #define __rte_mbuf_sanity_check(m, is_h) do { } while (0)
957 #define __rte_mbuf_sanity_check_raw(m, is_h) do { } while (0)
960 #define RTE_MBUF_ASSERT(exp) do { } while (0)
964 #ifdef RTE_MBUF_REFCNT_ATOMIC
973 static inline uint16_t
1001 static inline uint16_t
1024 static inline uint16_t
1034 static inline uint16_t
1052 #define RTE_MBUF_PREFETCH_TO_FREE(m) do { \
1104 static inline void __attribute__((always_inline))
1105 __rte_mbuf_raw_free(struct
rte_mbuf *m)
1133 void *m,
unsigned i);
1147 #define rte_ctrlmbuf_alloc(mp) rte_pktmbuf_alloc(mp)
1155 #define rte_ctrlmbuf_free(m) rte_pktmbuf_free(m)
1165 #define rte_ctrlmbuf_data(m) ((char *)((m)->buf_addr) + (m)->data_off)
1175 #define rte_ctrlmbuf_len(m) rte_pktmbuf_data_len(m)
1214 void *m,
unsigned i);
1272 unsigned cache_size, uint16_t priv_size, uint16_t data_room_size,
1286 static inline uint16_t
1307 static inline uint16_t
1336 m->data_off = (RTE_PKTMBUF_HEADROOM <= m->
buf_len) ?
1337 RTE_PKTMBUF_HEADROOM : m->
buf_len;
1359 if ((m = __rte_mbuf_raw_alloc(mp)) != NULL)
1378 struct rte_mbuf **mbufs,
unsigned count)
1392 switch (count % 4) {
1394 while (idx != count) {
1454 mi->data_off = m->data_off;
1485 uint32_t mbuf_size, buf_len, priv_size;
1488 mbuf_size =
sizeof(
struct rte_mbuf) + priv_size;
1492 m->
buf_addr = (
char *)m + mbuf_size;
1494 m->
buf_len = (uint16_t)buf_len;
1495 m->data_off =
RTE_MIN(RTE_PKTMBUF_HEADROOM, (uint16_t)m->
buf_len);
1500 static inline struct rte_mbuf* __attribute__((always_inline))
1501 __rte_pktmbuf_prefree_seg(struct
rte_mbuf *m)
1515 __rte_mbuf_raw_free(md);
1531 static inline void __attribute__((always_inline))
1534 if (
likely(NULL != (m = __rte_pktmbuf_prefree_seg(m)))) {
1536 __rte_mbuf_raw_free(m);
1599 }
while ((md = md->
next) != NULL &&
1633 }
while ((m = m->
next) != NULL);
1678 while (m2->
next != NULL)
1697 #define rte_pktmbuf_mtod_offset(m, t, o) \
1698 ((t)((char *)(m)->buf_addr + (m)->data_off + (o)))
1712 #define rte_pktmbuf_mtod(m, t) rte_pktmbuf_mtod_offset(m, t, 0)
1723 #define rte_pktmbuf_mtophys_offset(m, o) \
1724 (phys_addr_t)((m)->buf_physaddr + (m)->data_off + (o))
1733 #define rte_pktmbuf_mtophys(m) rte_pktmbuf_mtophys_offset(m, 0)
1743 #define rte_pktmbuf_pkt_len(m) ((m)->pkt_len)
1753 #define rte_pktmbuf_data_len(m) ((m)->data_len)
1782 return (
char *)m->
buf_addr + m->data_off;
1814 return (
char*) tail;
1841 return (
char *)m->
buf_addr + m->data_off;
1914 cur_tail->
next = tail;