rte_mbuf.h

Go to the documentation of this file.

/*-
 *   BSD LICENSE
 *
 *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
 *   Copyright 2014 6WIND S.A.
 *   All rights reserved.
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef _RTE_MBUF_H_
#define _RTE_MBUF_H_

#include <stdint.h>
#include <rte_common.h>
#include <rte_mempool.h>
#include <rte_memory.h>
#include <rte_atomic.h>
#include <rte_prefetch.h>
#include <rte_branch_prediction.h>

#ifdef __cplusplus
extern "C" {
#endif

/* deprecated options */
#pragma GCC poison RTE_MBUF_SCATTER_GATHER
#pragma GCC poison RTE_MBUF_REFCNT

/*
 * Packet Offload Features Flags. It also carry packet type information.
 * Critical resources. Both rx/tx shared these bits. Be cautious on any change
 *
 * - RX flags start at bit position zero, and get added to the left of previous
 *   flags.
 * - The most-significant 3 bits are reserved for generic mbuf flags
 * - TX flags therefore start at bit position 60 (i.e. 63-3), and new flags get
 *   added to the right of the previously defined flags i.e. they should count
 *   downwards, not upwards.
 *
 * Keep these flags synchronized with rte_get_rx_ol_flag_name() and
 * rte_get_tx_ol_flag_name().
 */
#define PKT_RX_VLAN_PKT      (1ULL << 0)  
#define PKT_RX_RSS_HASH      (1ULL << 1)  
#define PKT_RX_FDIR          (1ULL << 2)  
#define PKT_RX_L4_CKSUM_BAD  (1ULL << 3)  
#define PKT_RX_IP_CKSUM_BAD  (1ULL << 4)  
#define PKT_RX_EIP_CKSUM_BAD (0ULL << 0)  
#define PKT_RX_OVERSIZE      (0ULL << 0)  
#define PKT_RX_HBUF_OVERFLOW (0ULL << 0)  
#define PKT_RX_RECIP_ERR     (0ULL << 0)  
#define PKT_RX_MAC_ERR       (0ULL << 0)  
#ifndef RTE_NEXT_ABI
#define PKT_RX_IPV4_HDR      (1ULL << 5)  
#define PKT_RX_IPV4_HDR_EXT  (1ULL << 6)  
#define PKT_RX_IPV6_HDR      (1ULL << 7)  
#define PKT_RX_IPV6_HDR_EXT  (1ULL << 8)  
#endif /* RTE_NEXT_ABI */
#define PKT_RX_IEEE1588_PTP  (1ULL << 9)  
#define PKT_RX_IEEE1588_TMST (1ULL << 10) 
#ifndef RTE_NEXT_ABI
#define PKT_RX_TUNNEL_IPV4_HDR (1ULL << 11) 
#define PKT_RX_TUNNEL_IPV6_HDR (1ULL << 12) 
#endif /* RTE_NEXT_ABI */
#define PKT_RX_FDIR_ID       (1ULL << 13) 
#define PKT_RX_FDIR_FLX      (1ULL << 14) 
#define PKT_RX_QINQ_PKT      (1ULL << 15)  
/* add new RX flags here */

/* add new TX flags here */

#define PKT_TX_QINQ_PKT    (1ULL << 49)   
#define PKT_TX_TCP_SEG       (1ULL << 50)

#define PKT_TX_IEEE1588_TMST (1ULL << 51) 
#define PKT_TX_L4_NO_CKSUM   (0ULL << 52) 
#define PKT_TX_TCP_CKSUM     (1ULL << 52) 
#define PKT_TX_SCTP_CKSUM    (2ULL << 52) 
#define PKT_TX_UDP_CKSUM     (3ULL << 52) 
#define PKT_TX_L4_MASK       (3ULL << 52) 
#define PKT_TX_IP_CKSUM      (1ULL << 54)

#define PKT_TX_IPV4          (1ULL << 55)

#define PKT_TX_IPV6          (1ULL << 56)

#define PKT_TX_VLAN_PKT      (1ULL << 57) 
#define PKT_TX_OUTER_IP_CKSUM   (1ULL << 58)

#define PKT_TX_OUTER_IPV4   (1ULL << 59)

#define PKT_TX_OUTER_IPV6    (1ULL << 60)

#define __RESERVED           (1ULL << 61) 
#define IND_ATTACHED_MBUF    (1ULL << 62) 
/* Use final bit of flags to indicate a control mbuf */
#define CTRL_MBUF_FLAG       (1ULL << 63) 
#ifdef RTE_NEXT_ABI
/*
 * 32 bits are divided into several fields to mark packet types. Note that
 * each field is indexical.
 * - Bit 3:0 is for L2 types.
 * - Bit 7:4 is for L3 or outer L3 (for tunneling case) types.
 * - Bit 11:8 is for L4 or outer L4 (for tunneling case) types.
 * - Bit 15:12 is for tunnel types.
 * - Bit 19:16 is for inner L2 types.
 * - Bit 23:20 is for inner L3 types.
 * - Bit 27:24 is for inner L4 types.
 * - Bit 31:28 is reserved.
 *
 * To be compatible with Vector PMD, RTE_PTYPE_L3_IPV4, RTE_PTYPE_L3_IPV4_EXT,
 * RTE_PTYPE_L3_IPV6, RTE_PTYPE_L3_IPV6_EXT, RTE_PTYPE_L4_TCP, RTE_PTYPE_L4_UDP
 * and RTE_PTYPE_L4_SCTP should be kept as below in a contiguous 7 bits.
 *
 * Note that L3 types values are selected for checking IPV4/IPV6 header from
 * performance point of view. Reading annotations of RTE_ETH_IS_IPV4_HDR and
 * RTE_ETH_IS_IPV6_HDR is needed for any future changes of L3 type values.
 *
 * Note that the packet types of the same packet recognized by different
 * hardware may be different, as different hardware may have different
 * capability of packet type recognition.
 *
 * examples:
 * <'ether type'=0x0800
 * | 'version'=4, 'protocol'=0x29
 * | 'version'=6, 'next header'=0x3A
 * | 'ICMPv6 header'>
 * will be recognized on i40e hardware as packet type combination of,
 * RTE_PTYPE_L2_ETHER |
 * RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 * RTE_PTYPE_TUNNEL_IP |
 * RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
 * RTE_PTYPE_INNER_L4_ICMP.
 *
 * <'ether type'=0x86DD
 * | 'version'=6, 'next header'=0x2F
 * | 'GRE header'
 * | 'version'=6, 'next header'=0x11
 * | 'UDP header'>
 * will be recognized on i40e hardware as packet type combination of,
 * RTE_PTYPE_L2_ETHER |
 * RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 * RTE_PTYPE_TUNNEL_GRENAT |
 * RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
 * RTE_PTYPE_INNER_L4_UDP.
 */
#define RTE_PTYPE_UNKNOWN                   0x00000000

#define RTE_PTYPE_L2_ETHER                  0x00000001

#define RTE_PTYPE_L2_ETHER_TIMESYNC         0x00000002

#define RTE_PTYPE_L2_ETHER_ARP              0x00000003

#define RTE_PTYPE_L2_ETHER_LLDP             0x00000004

#define RTE_PTYPE_L2_MASK                   0x0000000f

#define RTE_PTYPE_L3_IPV4                   0x00000010

#define RTE_PTYPE_L3_IPV4_EXT               0x00000030

#define RTE_PTYPE_L3_IPV6                   0x00000040

#define RTE_PTYPE_L3_IPV4_EXT_UNKNOWN       0x00000090

#define RTE_PTYPE_L3_IPV6_EXT               0x000000c0

#define RTE_PTYPE_L3_IPV6_EXT_UNKNOWN       0x000000e0

#define RTE_PTYPE_L3_MASK                   0x000000f0

#define RTE_PTYPE_L4_TCP                    0x00000100

#define RTE_PTYPE_L4_UDP                    0x00000200

#define RTE_PTYPE_L4_FRAG                   0x00000300

#define RTE_PTYPE_L4_SCTP                   0x00000400

#define RTE_PTYPE_L4_ICMP                   0x00000500

#define RTE_PTYPE_L4_NONFRAG                0x00000600

#define RTE_PTYPE_L4_MASK                   0x00000f00

#define RTE_PTYPE_TUNNEL_IP                 0x00001000

#define RTE_PTYPE_TUNNEL_GRE                0x00002000

#define RTE_PTYPE_TUNNEL_VXLAN              0x00003000

#define RTE_PTYPE_TUNNEL_NVGRE              0x00004000

#define RTE_PTYPE_TUNNEL_GENEVE             0x00005000

#define RTE_PTYPE_TUNNEL_GRENAT             0x00006000

#define RTE_PTYPE_TUNNEL_MASK               0x0000f000

#define RTE_PTYPE_INNER_L2_ETHER            0x00010000

#define RTE_PTYPE_INNER_L2_ETHER_VLAN       0x00020000

#define RTE_PTYPE_INNER_L2_MASK             0x000f0000

#define RTE_PTYPE_INNER_L3_IPV4             0x00100000

#define RTE_PTYPE_INNER_L3_IPV4_EXT         0x00200000

#define RTE_PTYPE_INNER_L3_IPV6             0x00300000

#define RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN 0x00400000

#define RTE_PTYPE_INNER_L3_IPV6_EXT         0x00500000

#define RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN 0x00600000

#define RTE_PTYPE_INNER_L3_MASK             0x00f00000

#define RTE_PTYPE_INNER_L4_TCP              0x01000000

#define RTE_PTYPE_INNER_L4_UDP              0x02000000

#define RTE_PTYPE_INNER_L4_FRAG             0x03000000

#define RTE_PTYPE_INNER_L4_SCTP             0x04000000

#define RTE_PTYPE_INNER_L4_ICMP             0x05000000

#define RTE_PTYPE_INNER_L4_NONFRAG          0x06000000

#define RTE_PTYPE_INNER_L4_MASK             0x0f000000

#define  RTE_ETH_IS_IPV4_HDR(ptype) ((ptype) & RTE_PTYPE_L3_IPV4)

#define  RTE_ETH_IS_IPV6_HDR(ptype) ((ptype) & RTE_PTYPE_L3_IPV6)

/* Check if it is a tunneling packet */
#define RTE_ETH_IS_TUNNEL_PKT(ptype) ((ptype) & (RTE_PTYPE_TUNNEL_MASK | \
                                                 RTE_PTYPE_INNER_L2_MASK | \
                                                 RTE_PTYPE_INNER_L3_MASK | \
                                                 RTE_PTYPE_INNER_L4_MASK))
#endif /* RTE_NEXT_ABI */

#define RTE_MBUF_PRIV_ALIGN 8

const char *rte_get_rx_ol_flag_name(uint64_t mask);

const char *rte_get_tx_ol_flag_name(uint64_t mask);

#define RTE_MBUF_DEFAULT_DATAROOM       2048
#define RTE_MBUF_DEFAULT_BUF_SIZE       \
        (RTE_MBUF_DEFAULT_DATAROOM + RTE_PKTMBUF_HEADROOM)

/* define a set of marker types that can be used to refer to set points in the
 * mbuf */
typedef void    *MARKER[0];   
typedef uint8_t  MARKER8[0];  
typedef uint64_t MARKER64[0]; 
struct rte_mbuf {
        MARKER cacheline0;

        void *buf_addr;           
        phys_addr_t buf_physaddr; 
        uint16_t buf_len;         
        /* next 6 bytes are initialised on RX descriptor rearm */
        MARKER8 rearm_data;
        uint16_t data_off;

        union {
                rte_atomic16_t refcnt_atomic; 
                uint16_t refcnt;              
        };
        uint8_t nb_segs;          
        uint8_t port;             
        uint64_t ol_flags;        
        /* remaining bytes are set on RX when pulling packet from descriptor */
        MARKER rx_descriptor_fields1;

#ifdef RTE_NEXT_ABI
        /*
         * The packet type, which is the combination of outer/inner L2, L3, L4
         * and tunnel types.
         */
        union {
                uint32_t packet_type; 
                struct {
                        uint32_t l2_type:4; 
                        uint32_t l3_type:4; 
                        uint32_t l4_type:4; 
                        uint32_t tun_type:4; 
                        uint32_t inner_l2_type:4; 
                        uint32_t inner_l3_type:4; 
                        uint32_t inner_l4_type:4; 
                };
        };

        uint32_t pkt_len;         
        uint16_t data_len;        
        uint16_t vlan_tci;        
#else /* RTE_NEXT_ABI */

        uint16_t packet_type;

        uint16_t data_len;        
        uint32_t pkt_len;         
        uint16_t vlan_tci;        
        uint16_t vlan_tci_outer;  
#endif /* RTE_NEXT_ABI */
        union {
                uint32_t rss;     
                struct {
                        union {
                                struct {
                                        uint16_t hash;
                                        uint16_t id;
                                };
                                uint32_t lo;
                        };
                        uint32_t hi;
                } fdir;           
                uint32_t sched;   
                uint32_t usr;     
        } hash;                   
        uint32_t seqn; 
#ifdef RTE_NEXT_ABI
        uint16_t vlan_tci_outer;  
#endif /* RTE_NEXT_ABI */

        /* second cache line - fields only used in slow path or on TX */
        MARKER cacheline1 __rte_cache_aligned;

        union {
                void *userdata;   
                uint64_t udata64; 
        };

        struct rte_mempool *pool; 
        struct rte_mbuf *next;    
        /* fields to support TX offloads */
        union {
                uint64_t tx_offload;       
                struct {
                        uint64_t l2_len:7; 
                        uint64_t l3_len:9; 
                        uint64_t l4_len:8; 
                        uint64_t tso_segsz:16; 
                        /* fields for TX offloading of tunnels */
                        uint64_t outer_l3_len:9; 
                        uint64_t outer_l2_len:7; 
                        /* uint64_t unused:8; */
                };
        };

        uint16_t priv_size;

        uint16_t timesync;
} __rte_cache_aligned;

static inline uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp);

static inline struct rte_mbuf *
rte_mbuf_from_indirect(struct rte_mbuf *mi)
{
        return (struct rte_mbuf *)RTE_PTR_SUB(mi->buf_addr, sizeof(*mi) + mi->priv_size);
}

static inline char *
rte_mbuf_to_baddr(struct rte_mbuf *md)
{
        char *buffer_addr;
        buffer_addr = (char *)md + sizeof(*md) + rte_pktmbuf_priv_size(md->pool);
        return buffer_addr;
}

#define RTE_MBUF_INDIRECT(mb)   ((mb)->ol_flags & IND_ATTACHED_MBUF)

#define RTE_MBUF_DIRECT(mb)     (!RTE_MBUF_INDIRECT(mb))

struct rte_pktmbuf_pool_private {
        uint16_t mbuf_data_room_size; 
        uint16_t mbuf_priv_size;      
};

#ifdef RTE_LIBRTE_MBUF_DEBUG

#define __rte_mbuf_sanity_check(m, is_h) rte_mbuf_sanity_check(m, is_h)

#define __rte_mbuf_sanity_check_raw(m, is_h)    do {       \
        if ((m) != NULL)                                   \
                rte_mbuf_sanity_check(m, is_h);          \
} while (0)

#define RTE_MBUF_ASSERT(exp)                                         \
if (!(exp)) {                                                        \
        rte_panic("line%d\tassert \"" #exp "\" failed\n", __LINE__); \
}

#else /*  RTE_LIBRTE_MBUF_DEBUG */

#define __rte_mbuf_sanity_check(m, is_h) do { } while (0)

#define __rte_mbuf_sanity_check_raw(m, is_h) do { } while (0)

#define RTE_MBUF_ASSERT(exp)                do { } while (0)

#endif /*  RTE_LIBRTE_MBUF_DEBUG */

#ifdef RTE_MBUF_REFCNT_ATOMIC

static inline uint16_t
rte_mbuf_refcnt_read(const struct rte_mbuf *m)
{
        return (uint16_t)(rte_atomic16_read(&m->refcnt_atomic));
}

static inline void
rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
{
        rte_atomic16_set(&m->refcnt_atomic, new_value);
}

static inline uint16_t
rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
{
        /*
         * The atomic_add is an expensive operation, so we don't want to
         * call it in the case where we know we are the uniq holder of
         * this mbuf (i.e. ref_cnt == 1). Otherwise, an atomic
         * operation has to be used because concurrent accesses on the
         * reference counter can occur.
         */
        if (likely(rte_mbuf_refcnt_read(m) == 1)) {
                rte_mbuf_refcnt_set(m, 1 + value);
                return 1 + value;
        }

        return (uint16_t)(rte_atomic16_add_return(&m->refcnt_atomic, value));
}

#else /* ! RTE_MBUF_REFCNT_ATOMIC */

static inline uint16_t
rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
{
        m->refcnt = (uint16_t)(m->refcnt + value);
        return m->refcnt;
}

static inline uint16_t
rte_mbuf_refcnt_read(const struct rte_mbuf *m)
{
        return m->refcnt;
}

static inline void
rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
{
        m->refcnt = new_value;
}

#endif /* RTE_MBUF_REFCNT_ATOMIC */

#define RTE_MBUF_PREFETCH_TO_FREE(m) do {       \
        if ((m) != NULL)                        \
                rte_prefetch0(m);               \
} while (0)


void
rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header);

static inline struct rte_mbuf *__rte_mbuf_raw_alloc(struct rte_mempool *mp)
{
        struct rte_mbuf *m;
        void *mb = NULL;
        if (rte_mempool_get(mp, &mb) < 0)
                return NULL;
        m = (struct rte_mbuf *)mb;
        RTE_MBUF_ASSERT(rte_mbuf_refcnt_read(m) == 0);
        rte_mbuf_refcnt_set(m, 1);
        return m;
}

static inline void __attribute__((always_inline))
__rte_mbuf_raw_free(struct rte_mbuf *m)
{
        RTE_MBUF_ASSERT(rte_mbuf_refcnt_read(m) == 0);
        rte_mempool_put(m->pool, m);
}

/* Operations on ctrl mbuf */

void rte_ctrlmbuf_init(struct rte_mempool *mp, void *opaque_arg,
                void *m, unsigned i);

#define rte_ctrlmbuf_alloc(mp) rte_pktmbuf_alloc(mp)

#define rte_ctrlmbuf_free(m) rte_pktmbuf_free(m)

#define rte_ctrlmbuf_data(m) ((char *)((m)->buf_addr) + (m)->data_off)

#define rte_ctrlmbuf_len(m) rte_pktmbuf_data_len(m)

static inline int
rte_is_ctrlmbuf(struct rte_mbuf *m)
{
        return !!(m->ol_flags & CTRL_MBUF_FLAG);
}

/* Operations on pkt mbuf */

void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg,
                      void *m, unsigned i);


void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg);

struct rte_mempool *
rte_pktmbuf_pool_create(const char *name, unsigned n,
        unsigned cache_size, uint16_t priv_size, uint16_t data_room_size,
        int socket_id);

static inline uint16_t
rte_pktmbuf_data_room_size(struct rte_mempool *mp)
{
        struct rte_pktmbuf_pool_private *mbp_priv;

        mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
        return mbp_priv->mbuf_data_room_size;
}

static inline uint16_t
rte_pktmbuf_priv_size(struct rte_mempool *mp)
{
        struct rte_pktmbuf_pool_private *mbp_priv;

        mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
        return mbp_priv->mbuf_priv_size;
}

static inline void rte_pktmbuf_reset(struct rte_mbuf *m)
{
        m->next = NULL;
        m->pkt_len = 0;
        m->tx_offload = 0;
        m->vlan_tci = 0;
        m->vlan_tci_outer = 0;
        m->nb_segs = 1;
        m->port = 0xff;

        m->ol_flags = 0;
        m->packet_type = 0;
        m->data_off = (RTE_PKTMBUF_HEADROOM <= m->buf_len) ?
                        RTE_PKTMBUF_HEADROOM : m->buf_len;

        m->data_len = 0;
        __rte_mbuf_sanity_check(m, 1);
}

static inline struct rte_mbuf *rte_pktmbuf_alloc(struct rte_mempool *mp)
{
        struct rte_mbuf *m;
        if ((m = __rte_mbuf_raw_alloc(mp)) != NULL)
                rte_pktmbuf_reset(m);
        return m;
}

static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
{
        struct rte_mbuf *md;

        RTE_MBUF_ASSERT(RTE_MBUF_DIRECT(mi) &&
            rte_mbuf_refcnt_read(mi) == 1);

        /* if m is not direct, get the mbuf that embeds the data */
        if (RTE_MBUF_DIRECT(m))
                md = m;
        else
                md = rte_mbuf_from_indirect(m);

        rte_mbuf_refcnt_update(md, 1);
        mi->priv_size = m->priv_size;
        mi->buf_physaddr = m->buf_physaddr;
        mi->buf_addr = m->buf_addr;
        mi->buf_len = m->buf_len;

        mi->next = m->next;
        mi->data_off = m->data_off;
        mi->data_len = m->data_len;
        mi->port = m->port;
        mi->vlan_tci = m->vlan_tci;
        mi->vlan_tci_outer = m->vlan_tci_outer;
        mi->tx_offload = m->tx_offload;
        mi->hash = m->hash;

        mi->next = NULL;
        mi->pkt_len = mi->data_len;
        mi->nb_segs = 1;
        mi->ol_flags = m->ol_flags | IND_ATTACHED_MBUF;
        mi->packet_type = m->packet_type;

        __rte_mbuf_sanity_check(mi, 1);
        __rte_mbuf_sanity_check(m, 0);
}

static inline void rte_pktmbuf_detach(struct rte_mbuf *m)
{
        struct rte_mempool *mp = m->pool;
        uint32_t mbuf_size, buf_len, priv_size;

        priv_size = rte_pktmbuf_priv_size(mp);
        mbuf_size = sizeof(struct rte_mbuf) + priv_size;
        buf_len = rte_pktmbuf_data_room_size(mp);

        m->priv_size = priv_size;
        m->buf_addr = (char *)m + mbuf_size;
        m->buf_physaddr = rte_mempool_virt2phy(mp, m) + mbuf_size;
        m->buf_len = (uint16_t)buf_len;
        m->data_off = RTE_MIN(RTE_PKTMBUF_HEADROOM, (uint16_t)m->buf_len);
        m->data_len = 0;
        m->ol_flags = 0;
}

static inline struct rte_mbuf* __attribute__((always_inline))
__rte_pktmbuf_prefree_seg(struct rte_mbuf *m)
{
        __rte_mbuf_sanity_check(m, 0);

        if (likely(rte_mbuf_refcnt_update(m, -1) == 0)) {

                /* if this is an indirect mbuf, then
                 *  - detach mbuf
                 *  - free attached mbuf segment
                 */
                if (RTE_MBUF_INDIRECT(m)) {
                        struct rte_mbuf *md = rte_mbuf_from_indirect(m);
                        rte_pktmbuf_detach(m);
                        if (rte_mbuf_refcnt_update(md, -1) == 0)
                                __rte_mbuf_raw_free(md);
                }
                return m;
        }
        return NULL;
}

static inline void __attribute__((always_inline))
rte_pktmbuf_free_seg(struct rte_mbuf *m)
{
        if (likely(NULL != (m = __rte_pktmbuf_prefree_seg(m)))) {
                m->next = NULL;
                __rte_mbuf_raw_free(m);
        }
}

static inline void rte_pktmbuf_free(struct rte_mbuf *m)
{
        struct rte_mbuf *m_next;

        __rte_mbuf_sanity_check(m, 1);

        while (m != NULL) {
                m_next = m->next;
                rte_pktmbuf_free_seg(m);
                m = m_next;
        }
}

static inline struct rte_mbuf *rte_pktmbuf_clone(struct rte_mbuf *md,
                struct rte_mempool *mp)
{
        struct rte_mbuf *mc, *mi, **prev;
        uint32_t pktlen;
        uint8_t nseg;

        if (unlikely ((mc = rte_pktmbuf_alloc(mp)) == NULL))
                return NULL;

        mi = mc;
        prev = &mi->next;
        pktlen = md->pkt_len;
        nseg = 0;

        do {
                nseg++;
                rte_pktmbuf_attach(mi, md);
                *prev = mi;
                prev = &mi->next;
        } while ((md = md->next) != NULL &&
            (mi = rte_pktmbuf_alloc(mp)) != NULL);

        *prev = NULL;
        mc->nb_segs = nseg;
        mc->pkt_len = pktlen;

        /* Allocation of new indirect segment failed */
        if (unlikely (mi == NULL)) {
                rte_pktmbuf_free(mc);
                return NULL;
        }

        __rte_mbuf_sanity_check(mc, 1);
        return mc;
}

static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
{
        __rte_mbuf_sanity_check(m, 1);

        do {
                rte_mbuf_refcnt_update(m, v);
        } while ((m = m->next) != NULL);
}

static inline uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
{
        __rte_mbuf_sanity_check(m, 1);
        return m->data_off;
}

static inline uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
{
        __rte_mbuf_sanity_check(m, 1);
        return (uint16_t)(m->buf_len - rte_pktmbuf_headroom(m) -
                          m->data_len);
}

static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m)
{
        struct rte_mbuf *m2 = (struct rte_mbuf *)m;

        __rte_mbuf_sanity_check(m, 1);
        while (m2->next != NULL)
                m2 = m2->next;
        return m2;
}

#define rte_pktmbuf_mtod_offset(m, t, o)        \
        ((t)((char *)(m)->buf_addr + (m)->data_off + (o)))

#define rte_pktmbuf_mtod(m, t) rte_pktmbuf_mtod_offset(m, t, 0)

#define rte_pktmbuf_pkt_len(m) ((m)->pkt_len)

#define rte_pktmbuf_data_len(m) ((m)->data_len)

static inline char *rte_pktmbuf_prepend(struct rte_mbuf *m,
                                        uint16_t len)
{
        __rte_mbuf_sanity_check(m, 1);

        if (unlikely(len > rte_pktmbuf_headroom(m)))
                return NULL;

        m->data_off -= len;
        m->data_len = (uint16_t)(m->data_len + len);
        m->pkt_len  = (m->pkt_len + len);

        return (char *)m->buf_addr + m->data_off;
}

static inline char *rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
{
        void *tail;
        struct rte_mbuf *m_last;

        __rte_mbuf_sanity_check(m, 1);

        m_last = rte_pktmbuf_lastseg(m);
        if (unlikely(len > rte_pktmbuf_tailroom(m_last)))
                return NULL;

        tail = (char *)m_last->buf_addr + m_last->data_off + m_last->data_len;
        m_last->data_len = (uint16_t)(m_last->data_len + len);
        m->pkt_len  = (m->pkt_len + len);
        return (char*) tail;
}

static inline char *rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
{
        __rte_mbuf_sanity_check(m, 1);

        if (unlikely(len > m->data_len))
                return NULL;

        m->data_len = (uint16_t)(m->data_len - len);
        m->data_off += len;
        m->pkt_len  = (m->pkt_len - len);
        return (char *)m->buf_addr + m->data_off;
}

static inline int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
{
        struct rte_mbuf *m_last;

        __rte_mbuf_sanity_check(m, 1);

        m_last = rte_pktmbuf_lastseg(m);
        if (unlikely(len > m_last->data_len))
                return -1;

        m_last->data_len = (uint16_t)(m_last->data_len - len);
        m->pkt_len  = (m->pkt_len - len);
        return 0;
}

static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
{
        __rte_mbuf_sanity_check(m, 1);
        return !!(m->nb_segs == 1);
}

void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len);

#ifdef __cplusplus
}
#endif

#endif /* _RTE_MBUF_H_ */