DPDK  16.04.0
rte_mbuf.h
Go to the documentation of this file.
1 /*-
2  * BSD LICENSE
3  *
4  * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
5  * Copyright 2014 6WIND S.A.
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * * Redistributions of source code must retain the above copyright
13  * notice, this list of conditions and the following disclaimer.
14  * * Redistributions in binary form must reproduce the above copyright
15  * notice, this list of conditions and the following disclaimer in
16  * the documentation and/or other materials provided with the
17  * distribution.
18  * * Neither the name of Intel Corporation nor the names of its
19  * contributors may be used to endorse or promote products derived
20  * from this software without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33  */
34 
35 #ifndef _RTE_MBUF_H_
36 #define _RTE_MBUF_H_
37 
56 #include <stdint.h>
57 #include <rte_common.h>
58 #include <rte_mempool.h>
59 #include <rte_memory.h>
60 #include <rte_atomic.h>
61 #include <rte_prefetch.h>
62 #include <rte_branch_prediction.h>
63 
64 #ifdef __cplusplus
65 extern "C" {
66 #endif
67 
68 /* deprecated options */
69 #pragma GCC poison RTE_MBUF_SCATTER_GATHER
70 #pragma GCC poison RTE_MBUF_REFCNT
71 
72 /*
73  * Packet Offload Features Flags. It also carry packet type information.
74  * Critical resources. Both rx/tx shared these bits. Be cautious on any change
75  *
76  * - RX flags start at bit position zero, and get added to the left of previous
77  * flags.
78  * - The most-significant 3 bits are reserved for generic mbuf flags
79  * - TX flags therefore start at bit position 60 (i.e. 63-3), and new flags get
80  * added to the right of the previously defined flags i.e. they should count
81  * downwards, not upwards.
82  *
83  * Keep these flags synchronized with rte_get_rx_ol_flag_name() and
84  * rte_get_tx_ol_flag_name().
85  */
86 #define PKT_RX_VLAN_PKT (1ULL << 0)
87 #define PKT_RX_RSS_HASH (1ULL << 1)
88 #define PKT_RX_FDIR (1ULL << 2)
89 #define PKT_RX_L4_CKSUM_BAD (1ULL << 3)
90 #define PKT_RX_IP_CKSUM_BAD (1ULL << 4)
91 #define PKT_RX_EIP_CKSUM_BAD (1ULL << 5)
92 #define PKT_RX_OVERSIZE (0ULL << 0)
93 #define PKT_RX_HBUF_OVERFLOW (0ULL << 0)
94 #define PKT_RX_RECIP_ERR (0ULL << 0)
95 #define PKT_RX_MAC_ERR (0ULL << 0)
96 #define PKT_RX_IEEE1588_PTP (1ULL << 9)
97 #define PKT_RX_IEEE1588_TMST (1ULL << 10)
98 #define PKT_RX_FDIR_ID (1ULL << 13)
99 #define PKT_RX_FDIR_FLX (1ULL << 14)
100 #define PKT_RX_QINQ_PKT (1ULL << 15)
101 /* add new RX flags here */
102 
103 /* add new TX flags here */
104 
108 #define PKT_TX_QINQ_PKT (1ULL << 49)
123 #define PKT_TX_TCP_SEG (1ULL << 50)
124 
125 #define PKT_TX_IEEE1588_TMST (1ULL << 51)
138 #define PKT_TX_L4_NO_CKSUM (0ULL << 52)
139 #define PKT_TX_TCP_CKSUM (1ULL << 52)
140 #define PKT_TX_SCTP_CKSUM (2ULL << 52)
141 #define PKT_TX_UDP_CKSUM (3ULL << 52)
142 #define PKT_TX_L4_MASK (3ULL << 52)
151 #define PKT_TX_IP_CKSUM (1ULL << 54)
152 
159 #define PKT_TX_IPV4 (1ULL << 55)
160 
167 #define PKT_TX_IPV6 (1ULL << 56)
168 
169 #define PKT_TX_VLAN_PKT (1ULL << 57)
179 #define PKT_TX_OUTER_IP_CKSUM (1ULL << 58)
180 
186 #define PKT_TX_OUTER_IPV4 (1ULL << 59)
187 
193 #define PKT_TX_OUTER_IPV6 (1ULL << 60)
194 
195 #define __RESERVED (1ULL << 61)
197 #define IND_ATTACHED_MBUF (1ULL << 62)
199 /* Use final bit of flags to indicate a control mbuf */
200 #define CTRL_MBUF_FLAG (1ULL << 63)
202 /*
203  * 32 bits are divided into several fields to mark packet types. Note that
204  * each field is indexical.
205  * - Bit 3:0 is for L2 types.
206  * - Bit 7:4 is for L3 or outer L3 (for tunneling case) types.
207  * - Bit 11:8 is for L4 or outer L4 (for tunneling case) types.
208  * - Bit 15:12 is for tunnel types.
209  * - Bit 19:16 is for inner L2 types.
210  * - Bit 23:20 is for inner L3 types.
211  * - Bit 27:24 is for inner L4 types.
212  * - Bit 31:28 is reserved.
213  *
214  * To be compatible with Vector PMD, RTE_PTYPE_L3_IPV4, RTE_PTYPE_L3_IPV4_EXT,
215  * RTE_PTYPE_L3_IPV6, RTE_PTYPE_L3_IPV6_EXT, RTE_PTYPE_L4_TCP, RTE_PTYPE_L4_UDP
216  * and RTE_PTYPE_L4_SCTP should be kept as below in a contiguous 7 bits.
217  *
218  * Note that L3 types values are selected for checking IPV4/IPV6 header from
219  * performance point of view. Reading annotations of RTE_ETH_IS_IPV4_HDR and
220  * RTE_ETH_IS_IPV6_HDR is needed for any future changes of L3 type values.
221  *
222  * Note that the packet types of the same packet recognized by different
223  * hardware may be different, as different hardware may have different
224  * capability of packet type recognition.
225  *
226  * examples:
227  * <'ether type'=0x0800
228  * | 'version'=4, 'protocol'=0x29
229  * | 'version'=6, 'next header'=0x3A
230  * | 'ICMPv6 header'>
231  * will be recognized on i40e hardware as packet type combination of,
232  * RTE_PTYPE_L2_ETHER |
233  * RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
234  * RTE_PTYPE_TUNNEL_IP |
235  * RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
236  * RTE_PTYPE_INNER_L4_ICMP.
237  *
238  * <'ether type'=0x86DD
239  * | 'version'=6, 'next header'=0x2F
240  * | 'GRE header'
241  * | 'version'=6, 'next header'=0x11
242  * | 'UDP header'>
243  * will be recognized on i40e hardware as packet type combination of,
244  * RTE_PTYPE_L2_ETHER |
245  * RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
246  * RTE_PTYPE_TUNNEL_GRENAT |
247  * RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
248  * RTE_PTYPE_INNER_L4_UDP.
249  */
250 #define RTE_PTYPE_UNKNOWN 0x00000000
251 
258 #define RTE_PTYPE_L2_ETHER 0x00000001
259 
265 #define RTE_PTYPE_L2_ETHER_TIMESYNC 0x00000002
266 
272 #define RTE_PTYPE_L2_ETHER_ARP 0x00000003
273 
279 #define RTE_PTYPE_L2_ETHER_LLDP 0x00000004
280 
284 #define RTE_PTYPE_L2_MASK 0x0000000f
285 
294 #define RTE_PTYPE_L3_IPV4 0x00000010
295 
304 #define RTE_PTYPE_L3_IPV4_EXT 0x00000030
305 
314 #define RTE_PTYPE_L3_IPV6 0x00000040
315 
324 #define RTE_PTYPE_L3_IPV4_EXT_UNKNOWN 0x00000090
325 
335 #define RTE_PTYPE_L3_IPV6_EXT 0x000000c0
336 
346 #define RTE_PTYPE_L3_IPV6_EXT_UNKNOWN 0x000000e0
347 
351 #define RTE_PTYPE_L3_MASK 0x000000f0
352 
363 #define RTE_PTYPE_L4_TCP 0x00000100
364 
375 #define RTE_PTYPE_L4_UDP 0x00000200
376 
392 #define RTE_PTYPE_L4_FRAG 0x00000300
393 
404 #define RTE_PTYPE_L4_SCTP 0x00000400
405 
416 #define RTE_PTYPE_L4_ICMP 0x00000500
417 
432 #define RTE_PTYPE_L4_NONFRAG 0x00000600
433 
437 #define RTE_PTYPE_L4_MASK 0x00000f00
438 
448 #define RTE_PTYPE_TUNNEL_IP 0x00001000
449 
459 #define RTE_PTYPE_TUNNEL_GRE 0x00002000
460 
472 #define RTE_PTYPE_TUNNEL_VXLAN 0x00003000
473 
486 #define RTE_PTYPE_TUNNEL_NVGRE 0x00004000
487 
499 #define RTE_PTYPE_TUNNEL_GENEVE 0x00005000
500 
506 #define RTE_PTYPE_TUNNEL_GRENAT 0x00006000
507 
510 #define RTE_PTYPE_TUNNEL_MASK 0x0000f000
511 
518 #define RTE_PTYPE_INNER_L2_ETHER 0x00010000
519 
525 #define RTE_PTYPE_INNER_L2_ETHER_VLAN 0x00020000
526 
529 #define RTE_PTYPE_INNER_L2_MASK 0x000f0000
530 
538 #define RTE_PTYPE_INNER_L3_IPV4 0x00100000
539 
547 #define RTE_PTYPE_INNER_L3_IPV4_EXT 0x00200000
548 
556 #define RTE_PTYPE_INNER_L3_IPV6 0x00300000
557 
565 #define RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN 0x00400000
566 
575 #define RTE_PTYPE_INNER_L3_IPV6_EXT 0x00500000
576 
586 #define RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN 0x00600000
587 
590 #define RTE_PTYPE_INNER_L3_MASK 0x00f00000
591 
602 #define RTE_PTYPE_INNER_L4_TCP 0x01000000
603 
614 #define RTE_PTYPE_INNER_L4_UDP 0x02000000
615 
626 #define RTE_PTYPE_INNER_L4_FRAG 0x03000000
627 
638 #define RTE_PTYPE_INNER_L4_SCTP 0x04000000
639 
650 #define RTE_PTYPE_INNER_L4_ICMP 0x05000000
651 
663 #define RTE_PTYPE_INNER_L4_NONFRAG 0x06000000
664 
667 #define RTE_PTYPE_INNER_L4_MASK 0x0f000000
668 
674 #define RTE_ETH_IS_IPV4_HDR(ptype) ((ptype) & RTE_PTYPE_L3_IPV4)
675 
681 #define RTE_ETH_IS_IPV6_HDR(ptype) ((ptype) & RTE_PTYPE_L3_IPV6)
682 
683 /* Check if it is a tunneling packet */
684 #define RTE_ETH_IS_TUNNEL_PKT(ptype) ((ptype) & (RTE_PTYPE_TUNNEL_MASK | \
685  RTE_PTYPE_INNER_L2_MASK | \
686  RTE_PTYPE_INNER_L3_MASK | \
687  RTE_PTYPE_INNER_L4_MASK))
688 
690 #define RTE_MBUF_PRIV_ALIGN 8
691 
700 const char *rte_get_rx_ol_flag_name(uint64_t mask);
701 
712 const char *rte_get_tx_ol_flag_name(uint64_t mask);
713 
720 #define RTE_MBUF_DEFAULT_DATAROOM 2048
721 #define RTE_MBUF_DEFAULT_BUF_SIZE \
722  (RTE_MBUF_DEFAULT_DATAROOM + RTE_PKTMBUF_HEADROOM)
723 
724 /* define a set of marker types that can be used to refer to set points in the
725  * mbuf */
726 typedef void *MARKER[0];
727 typedef uint8_t MARKER8[0];
728 typedef uint64_t MARKER64[0];
734 struct rte_mbuf {
735  MARKER cacheline0;
736 
737  void *buf_addr;
740  uint16_t buf_len;
742  /* next 6 bytes are initialised on RX descriptor rearm */
743  MARKER8 rearm_data;
744  uint16_t data_off;
745 
754  union {
756  uint16_t refcnt;
757  };
758  uint8_t nb_segs;
759  uint8_t port;
761  uint64_t ol_flags;
763  /* remaining bytes are set on RX when pulling packet from descriptor */
764  MARKER rx_descriptor_fields1;
765 
766  /*
767  * The packet type, which is the combination of outer/inner L2, L3, L4
768  * and tunnel types.
769  */
770  union {
771  uint32_t packet_type;
772  struct {
773  uint32_t l2_type:4;
774  uint32_t l3_type:4;
775  uint32_t l4_type:4;
776  uint32_t tun_type:4;
777  uint32_t inner_l2_type:4;
778  uint32_t inner_l3_type:4;
779  uint32_t inner_l4_type:4;
780  };
781  };
782 
783  uint32_t pkt_len;
784  uint16_t data_len;
785  uint16_t vlan_tci;
787  union {
788  uint32_t rss;
789  struct {
790  union {
791  struct {
792  uint16_t hash;
793  uint16_t id;
794  };
795  uint32_t lo;
797  };
798  uint32_t hi;
801  } fdir;
802  struct {
803  uint32_t lo;
804  uint32_t hi;
805  } sched;
806  uint32_t usr;
807  } hash;
809  uint32_t seqn;
811  uint16_t vlan_tci_outer;
813  /* second cache line - fields only used in slow path or on TX */
814  MARKER cacheline1 __rte_cache_min_aligned;
815 
816  union {
817  void *userdata;
818  uint64_t udata64;
819  };
820 
821  struct rte_mempool *pool;
822  struct rte_mbuf *next;
824  /* fields to support TX offloads */
825  union {
826  uint64_t tx_offload;
827  struct {
828  uint64_t l2_len:7;
829  uint64_t l3_len:9;
830  uint64_t l4_len:8;
831  uint64_t tso_segsz:16;
833  /* fields for TX offloading of tunnels */
834  uint64_t outer_l3_len:9;
835  uint64_t outer_l2_len:7;
837  /* uint64_t unused:8; */
838  };
839  };
840 
843  uint16_t priv_size;
844 
846  uint16_t timesync;
848 
849 static inline uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp);
850 
859 static inline phys_addr_t
861 {
862  return mb->buf_physaddr + mb->data_off;
863 }
864 
877 static inline phys_addr_t
879 {
880  return mb->buf_physaddr + RTE_PKTMBUF_HEADROOM;
881 }
882 
891 static inline struct rte_mbuf *
893 {
894  return (struct rte_mbuf *)RTE_PTR_SUB(mi->buf_addr, sizeof(*mi) + mi->priv_size);
895 }
896 
905 static inline char *
907 {
908  char *buffer_addr;
909  buffer_addr = (char *)md + sizeof(*md) + rte_pktmbuf_priv_size(md->pool);
910  return buffer_addr;
911 }
912 
916 #define RTE_MBUF_INDIRECT(mb) ((mb)->ol_flags & IND_ATTACHED_MBUF)
917 
921 #define RTE_MBUF_DIRECT(mb) (!RTE_MBUF_INDIRECT(mb))
922 
931  uint16_t mbuf_priv_size;
932 };
933 
934 #ifdef RTE_LIBRTE_MBUF_DEBUG
935 
937 #define __rte_mbuf_sanity_check(m, is_h) rte_mbuf_sanity_check(m, is_h)
938 
940 #define __rte_mbuf_sanity_check_raw(m, is_h) do { \
941  if ((m) != NULL) \
942  rte_mbuf_sanity_check(m, is_h); \
943 } while (0)
944 
946 #define RTE_MBUF_ASSERT(exp) \
947 if (!(exp)) { \
948  rte_panic("line%d\tassert \"" #exp "\" failed\n", __LINE__); \
949 }
950 
951 #else /* RTE_LIBRTE_MBUF_DEBUG */
952 
954 #define __rte_mbuf_sanity_check(m, is_h) do { } while (0)
955 
957 #define __rte_mbuf_sanity_check_raw(m, is_h) do { } while (0)
958 
960 #define RTE_MBUF_ASSERT(exp) do { } while (0)
961 
962 #endif /* RTE_LIBRTE_MBUF_DEBUG */
963 
964 #ifdef RTE_MBUF_REFCNT_ATOMIC
965 
973 static inline uint16_t
974 rte_mbuf_refcnt_read(const struct rte_mbuf *m)
975 {
976  return (uint16_t)(rte_atomic16_read(&m->refcnt_atomic));
977 }
978 
986 static inline void
987 rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
988 {
989  rte_atomic16_set(&m->refcnt_atomic, new_value);
990 }
991 
1001 static inline uint16_t
1002 rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
1003 {
1004  /*
1005  * The atomic_add is an expensive operation, so we don't want to
1006  * call it in the case where we know we are the uniq holder of
1007  * this mbuf (i.e. ref_cnt == 1). Otherwise, an atomic
1008  * operation has to be used because concurrent accesses on the
1009  * reference counter can occur.
1010  */
1011  if (likely(rte_mbuf_refcnt_read(m) == 1)) {
1012  rte_mbuf_refcnt_set(m, 1 + value);
1013  return 1 + value;
1014  }
1015 
1016  return (uint16_t)(rte_atomic16_add_return(&m->refcnt_atomic, value));
1017 }
1018 
1019 #else /* ! RTE_MBUF_REFCNT_ATOMIC */
1020 
1024 static inline uint16_t
1025 rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
1026 {
1027  m->refcnt = (uint16_t)(m->refcnt + value);
1028  return m->refcnt;
1029 }
1030 
1034 static inline uint16_t
1036 {
1037  return m->refcnt;
1038 }
1039 
1043 static inline void
1044 rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
1045 {
1046  m->refcnt = new_value;
1047 }
1048 
1049 #endif /* RTE_MBUF_REFCNT_ATOMIC */
1050 
1052 #define RTE_MBUF_PREFETCH_TO_FREE(m) do { \
1053  if ((m) != NULL) \
1054  rte_prefetch0(m); \
1055 } while (0)
1056 
1057 
1070 void
1071 rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header);
1072 
1084 static inline struct rte_mbuf *__rte_mbuf_raw_alloc(struct rte_mempool *mp)
1085 {
1086  struct rte_mbuf *m;
1087  void *mb = NULL;
1088  if (rte_mempool_get(mp, &mb) < 0)
1089  return NULL;
1090  m = (struct rte_mbuf *)mb;
1092  rte_mbuf_refcnt_set(m, 1);
1093  return m;
1094 }
1095 
1104 static inline void __attribute__((always_inline))
1105 __rte_mbuf_raw_free(struct rte_mbuf *m)
1106 {
1108  rte_mempool_put(m->pool, m);
1109 }
1110 
1111 /* Operations on ctrl mbuf */
1112 
1132 void rte_ctrlmbuf_init(struct rte_mempool *mp, void *opaque_arg,
1133  void *m, unsigned i);
1134 
1147 #define rte_ctrlmbuf_alloc(mp) rte_pktmbuf_alloc(mp)
1148 
1155 #define rte_ctrlmbuf_free(m) rte_pktmbuf_free(m)
1156 
1165 #define rte_ctrlmbuf_data(m) ((char *)((m)->buf_addr) + (m)->data_off)
1166 
1175 #define rte_ctrlmbuf_len(m) rte_pktmbuf_data_len(m)
1176 
1186 static inline int
1188 {
1189  return !!(m->ol_flags & CTRL_MBUF_FLAG);
1190 }
1191 
1192 /* Operations on pkt mbuf */
1193 
1213 void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg,
1214  void *m, unsigned i);
1215 
1216 
1233 void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg);
1234 
1270 struct rte_mempool *
1271 rte_pktmbuf_pool_create(const char *name, unsigned n,
1272  unsigned cache_size, uint16_t priv_size, uint16_t data_room_size,
1273  int socket_id);
1274 
1286 static inline uint16_t
1288 {
1289  struct rte_pktmbuf_pool_private *mbp_priv;
1290 
1291  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
1292  return mbp_priv->mbuf_data_room_size;
1293 }
1294 
1307 static inline uint16_t
1309 {
1310  struct rte_pktmbuf_pool_private *mbp_priv;
1311 
1312  mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
1313  return mbp_priv->mbuf_priv_size;
1314 }
1315 
1324 static inline void rte_pktmbuf_reset(struct rte_mbuf *m)
1325 {
1326  m->next = NULL;
1327  m->pkt_len = 0;
1328  m->tx_offload = 0;
1329  m->vlan_tci = 0;
1330  m->vlan_tci_outer = 0;
1331  m->nb_segs = 1;
1332  m->port = 0xff;
1333 
1334  m->ol_flags = 0;
1335  m->packet_type = 0;
1336  m->data_off = (RTE_PKTMBUF_HEADROOM <= m->buf_len) ?
1337  RTE_PKTMBUF_HEADROOM : m->buf_len;
1338 
1339  m->data_len = 0;
1341 }
1342 
1356 static inline struct rte_mbuf *rte_pktmbuf_alloc(struct rte_mempool *mp)
1357 {
1358  struct rte_mbuf *m;
1359  if ((m = __rte_mbuf_raw_alloc(mp)) != NULL)
1360  rte_pktmbuf_reset(m);
1361  return m;
1362 }
1363 
1377 static inline int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool,
1378  struct rte_mbuf **mbufs, unsigned count)
1379 {
1380  unsigned idx = 0;
1381  int rc;
1382 
1383  rc = rte_mempool_get_bulk(pool, (void **)mbufs, count);
1384  if (unlikely(rc))
1385  return rc;
1386 
1387  /* To understand duff's device on loop unwinding optimization, see
1388  * https://en.wikipedia.org/wiki/Duff's_device.
1389  * Here while() loop is used rather than do() while{} to avoid extra
1390  * check if count is zero.
1391  */
1392  switch (count % 4) {
1393  case 0:
1394  while (idx != count) {
1395  RTE_MBUF_ASSERT(rte_mbuf_refcnt_read(mbufs[idx]) == 0);
1396  rte_mbuf_refcnt_set(mbufs[idx], 1);
1397  rte_pktmbuf_reset(mbufs[idx]);
1398  idx++;
1399  case 3:
1400  RTE_MBUF_ASSERT(rte_mbuf_refcnt_read(mbufs[idx]) == 0);
1401  rte_mbuf_refcnt_set(mbufs[idx], 1);
1402  rte_pktmbuf_reset(mbufs[idx]);
1403  idx++;
1404  case 2:
1405  RTE_MBUF_ASSERT(rte_mbuf_refcnt_read(mbufs[idx]) == 0);
1406  rte_mbuf_refcnt_set(mbufs[idx], 1);
1407  rte_pktmbuf_reset(mbufs[idx]);
1408  idx++;
1409  case 1:
1410  RTE_MBUF_ASSERT(rte_mbuf_refcnt_read(mbufs[idx]) == 0);
1411  rte_mbuf_refcnt_set(mbufs[idx], 1);
1412  rte_pktmbuf_reset(mbufs[idx]);
1413  idx++;
1414  }
1415  }
1416  return 0;
1417 }
1418 
1434 static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
1435 {
1436  struct rte_mbuf *md;
1437 
1439  rte_mbuf_refcnt_read(mi) == 1);
1440 
1441  /* if m is not direct, get the mbuf that embeds the data */
1442  if (RTE_MBUF_DIRECT(m))
1443  md = m;
1444  else
1445  md = rte_mbuf_from_indirect(m);
1446 
1447  rte_mbuf_refcnt_update(md, 1);
1448  mi->priv_size = m->priv_size;
1449  mi->buf_physaddr = m->buf_physaddr;
1450  mi->buf_addr = m->buf_addr;
1451  mi->buf_len = m->buf_len;
1452 
1453  mi->next = m->next;
1454  mi->data_off = m->data_off;
1455  mi->data_len = m->data_len;
1456  mi->port = m->port;
1457  mi->vlan_tci = m->vlan_tci;
1458  mi->vlan_tci_outer = m->vlan_tci_outer;
1459  mi->tx_offload = m->tx_offload;
1460  mi->hash = m->hash;
1461 
1462  mi->next = NULL;
1463  mi->pkt_len = mi->data_len;
1464  mi->nb_segs = 1;
1465  mi->ol_flags = m->ol_flags | IND_ATTACHED_MBUF;
1466  mi->packet_type = m->packet_type;
1467 
1468  __rte_mbuf_sanity_check(mi, 1);
1470 }
1471 
1482 static inline void rte_pktmbuf_detach(struct rte_mbuf *m)
1483 {
1484  struct rte_mempool *mp = m->pool;
1485  uint32_t mbuf_size, buf_len, priv_size;
1486 
1487  priv_size = rte_pktmbuf_priv_size(mp);
1488  mbuf_size = sizeof(struct rte_mbuf) + priv_size;
1489  buf_len = rte_pktmbuf_data_room_size(mp);
1490 
1491  m->priv_size = priv_size;
1492  m->buf_addr = (char *)m + mbuf_size;
1493  m->buf_physaddr = rte_mempool_virt2phy(mp, m) + mbuf_size;
1494  m->buf_len = (uint16_t)buf_len;
1495  m->data_off = RTE_MIN(RTE_PKTMBUF_HEADROOM, (uint16_t)m->buf_len);
1496  m->data_len = 0;
1497  m->ol_flags = 0;
1498 }
1499 
1500 static inline struct rte_mbuf* __attribute__((always_inline))
1501 __rte_pktmbuf_prefree_seg(struct rte_mbuf *m)
1502 {
1504 
1505  if (likely(rte_mbuf_refcnt_update(m, -1) == 0)) {
1506 
1507  /* if this is an indirect mbuf, then
1508  * - detach mbuf
1509  * - free attached mbuf segment
1510  */
1511  if (RTE_MBUF_INDIRECT(m)) {
1512  struct rte_mbuf *md = rte_mbuf_from_indirect(m);
1513  rte_pktmbuf_detach(m);
1514  if (rte_mbuf_refcnt_update(md, -1) == 0)
1515  __rte_mbuf_raw_free(md);
1516  }
1517  return m;
1518  }
1519  return NULL;
1520 }
1521 
1531 static inline void __attribute__((always_inline))
1533 {
1534  if (likely(NULL != (m = __rte_pktmbuf_prefree_seg(m)))) {
1535  m->next = NULL;
1536  __rte_mbuf_raw_free(m);
1537  }
1538 }
1539 
1549 static inline void rte_pktmbuf_free(struct rte_mbuf *m)
1550 {
1551  struct rte_mbuf *m_next;
1552 
1554 
1555  while (m != NULL) {
1556  m_next = m->next;
1558  m = m_next;
1559  }
1560 }
1561 
1579 static inline struct rte_mbuf *rte_pktmbuf_clone(struct rte_mbuf *md,
1580  struct rte_mempool *mp)
1581 {
1582  struct rte_mbuf *mc, *mi, **prev;
1583  uint32_t pktlen;
1584  uint8_t nseg;
1585 
1586  if (unlikely ((mc = rte_pktmbuf_alloc(mp)) == NULL))
1587  return NULL;
1588 
1589  mi = mc;
1590  prev = &mi->next;
1591  pktlen = md->pkt_len;
1592  nseg = 0;
1593 
1594  do {
1595  nseg++;
1596  rte_pktmbuf_attach(mi, md);
1597  *prev = mi;
1598  prev = &mi->next;
1599  } while ((md = md->next) != NULL &&
1600  (mi = rte_pktmbuf_alloc(mp)) != NULL);
1601 
1602  *prev = NULL;
1603  mc->nb_segs = nseg;
1604  mc->pkt_len = pktlen;
1605 
1606  /* Allocation of new indirect segment failed */
1607  if (unlikely (mi == NULL)) {
1608  rte_pktmbuf_free(mc);
1609  return NULL;
1610  }
1611 
1612  __rte_mbuf_sanity_check(mc, 1);
1613  return mc;
1614 }
1615 
1627 static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
1628 {
1630 
1631  do {
1632  rte_mbuf_refcnt_update(m, v);
1633  } while ((m = m->next) != NULL);
1634 }
1635 
1644 static inline uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
1645 {
1647  return m->data_off;
1648 }
1649 
1658 static inline uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
1659 {
1661  return (uint16_t)(m->buf_len - rte_pktmbuf_headroom(m) -
1662  m->data_len);
1663 }
1664 
1673 static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m)
1674 {
1675  struct rte_mbuf *m2 = (struct rte_mbuf *)m;
1676 
1678  while (m2->next != NULL)
1679  m2 = m2->next;
1680  return m2;
1681 }
1682 
1697 #define rte_pktmbuf_mtod_offset(m, t, o) \
1698  ((t)((char *)(m)->buf_addr + (m)->data_off + (o)))
1699 
1712 #define rte_pktmbuf_mtod(m, t) rte_pktmbuf_mtod_offset(m, t, 0)
1713 
1723 #define rte_pktmbuf_mtophys_offset(m, o) \
1724  (phys_addr_t)((m)->buf_physaddr + (m)->data_off + (o))
1725 
1733 #define rte_pktmbuf_mtophys(m) rte_pktmbuf_mtophys_offset(m, 0)
1734 
1743 #define rte_pktmbuf_pkt_len(m) ((m)->pkt_len)
1744 
1753 #define rte_pktmbuf_data_len(m) ((m)->data_len)
1754 
1770 static inline char *rte_pktmbuf_prepend(struct rte_mbuf *m,
1771  uint16_t len)
1772 {
1774 
1775  if (unlikely(len > rte_pktmbuf_headroom(m)))
1776  return NULL;
1777 
1778  m->data_off -= len;
1779  m->data_len = (uint16_t)(m->data_len + len);
1780  m->pkt_len = (m->pkt_len + len);
1781 
1782  return (char *)m->buf_addr + m->data_off;
1783 }
1784 
1800 static inline char *rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
1801 {
1802  void *tail;
1803  struct rte_mbuf *m_last;
1804 
1806 
1807  m_last = rte_pktmbuf_lastseg(m);
1808  if (unlikely(len > rte_pktmbuf_tailroom(m_last)))
1809  return NULL;
1810 
1811  tail = (char *)m_last->buf_addr + m_last->data_off + m_last->data_len;
1812  m_last->data_len = (uint16_t)(m_last->data_len + len);
1813  m->pkt_len = (m->pkt_len + len);
1814  return (char*) tail;
1815 }
1816 
1831 static inline char *rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
1832 {
1834 
1835  if (unlikely(len > m->data_len))
1836  return NULL;
1837 
1838  m->data_len = (uint16_t)(m->data_len - len);
1839  m->data_off += len;
1840  m->pkt_len = (m->pkt_len - len);
1841  return (char *)m->buf_addr + m->data_off;
1842 }
1843 
1858 static inline int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
1859 {
1860  struct rte_mbuf *m_last;
1861 
1863 
1864  m_last = rte_pktmbuf_lastseg(m);
1865  if (unlikely(len > m_last->data_len))
1866  return -1;
1867 
1868  m_last->data_len = (uint16_t)(m_last->data_len - len);
1869  m->pkt_len = (m->pkt_len - len);
1870  return 0;
1871 }
1872 
1882 static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
1883 {
1885  return !!(m->nb_segs == 1);
1886 }
1887 
1904 static inline int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
1905 {
1906  struct rte_mbuf *cur_tail;
1907 
1908  /* Check for number-of-segments-overflow */
1909  if (head->nb_segs + tail->nb_segs >= 1 << (sizeof(head->nb_segs) * 8))
1910  return -EOVERFLOW;
1911 
1912  /* Chain 'tail' onto the old tail */
1913  cur_tail = rte_pktmbuf_lastseg(head);
1914  cur_tail->next = tail;
1915 
1916  /* accumulate number of segments and total length. */
1917  head->nb_segs = (uint8_t)(head->nb_segs + tail->nb_segs);
1918  head->pkt_len += tail->pkt_len;
1919 
1920  /* pkt_len is only set in the head */
1921  tail->pkt_len = tail->data_len;
1922 
1923  return 0;
1924 }
1925 
1940 void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len);
1941 
1942 #ifdef __cplusplus
1943 }
1944 #endif
1945 
1946 #endif /* _RTE_MBUF_H_ */