rte_mbuf.h

Go to the documentation of this file.

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2010-2014 Intel Corporation.
 * Copyright 2014 6WIND S.A.
 */

#ifndef _RTE_MBUF_H_
#define _RTE_MBUF_H_

#include <stdint.h>
#include <rte_compat.h>
#include <rte_common.h>
#include <rte_config.h>
#include <rte_mempool.h>
#include <rte_memory.h>
#include <rte_atomic.h>
#include <rte_prefetch.h>
#include <rte_branch_prediction.h>
#include <rte_byteorder.h>
#include <rte_mbuf_ptype.h>

#ifdef __cplusplus
extern "C" {
#endif

/*
 * Packet Offload Features Flags. It also carry packet type information.
 * Critical resources. Both rx/tx shared these bits. Be cautious on any change
 *
 * - RX flags start at bit position zero, and get added to the left of previous
 *   flags.
 * - The most-significant 3 bits are reserved for generic mbuf flags
 * - TX flags therefore start at bit position 60 (i.e. 63-3), and new flags get
 *   added to the right of the previously defined flags i.e. they should count
 *   downwards, not upwards.
 *
 * Keep these flags synchronized with rte_get_rx_ol_flag_name() and
 * rte_get_tx_ol_flag_name().
 */

#define PKT_RX_VLAN          (1ULL << 0)

#define PKT_RX_RSS_HASH      (1ULL << 1)  
#define PKT_RX_FDIR          (1ULL << 2)  
#define PKT_RX_L4_CKSUM_BAD  (1ULL << 3)

#define PKT_RX_IP_CKSUM_BAD  (1ULL << 4)

#define PKT_RX_EIP_CKSUM_BAD (1ULL << 5)  
#define PKT_RX_VLAN_STRIPPED (1ULL << 6)

#define PKT_RX_IP_CKSUM_MASK ((1ULL << 4) | (1ULL << 7))

#define PKT_RX_IP_CKSUM_UNKNOWN 0
#define PKT_RX_IP_CKSUM_BAD     (1ULL << 4)
#define PKT_RX_IP_CKSUM_GOOD    (1ULL << 7)
#define PKT_RX_IP_CKSUM_NONE    ((1ULL << 4) | (1ULL << 7))

#define PKT_RX_L4_CKSUM_MASK ((1ULL << 3) | (1ULL << 8))

#define PKT_RX_L4_CKSUM_UNKNOWN 0
#define PKT_RX_L4_CKSUM_BAD     (1ULL << 3)
#define PKT_RX_L4_CKSUM_GOOD    (1ULL << 8)
#define PKT_RX_L4_CKSUM_NONE    ((1ULL << 3) | (1ULL << 8))

#define PKT_RX_IEEE1588_PTP  (1ULL << 9)  
#define PKT_RX_IEEE1588_TMST (1ULL << 10) 
#define PKT_RX_FDIR_ID       (1ULL << 13) 
#define PKT_RX_FDIR_FLX      (1ULL << 14) 
#define PKT_RX_QINQ_STRIPPED (1ULL << 15)

#define PKT_RX_LRO           (1ULL << 16)

#define PKT_RX_TIMESTAMP     (1ULL << 17)

#define PKT_RX_SEC_OFFLOAD      (1ULL << 18)

#define PKT_RX_SEC_OFFLOAD_FAILED   (1ULL << 19)

#define PKT_RX_QINQ          (1ULL << 20)

#define PKT_RX_OUTER_L4_CKSUM_MASK  ((1ULL << 21) | (1ULL << 22))

#define PKT_RX_OUTER_L4_CKSUM_UNKNOWN   0
#define PKT_RX_OUTER_L4_CKSUM_BAD   (1ULL << 21)
#define PKT_RX_OUTER_L4_CKSUM_GOOD  (1ULL << 22)
#define PKT_RX_OUTER_L4_CKSUM_INVALID   ((1ULL << 21) | (1ULL << 22))

/* add new RX flags here */

/* add new TX flags here */

#define PKT_TX_METADATA (1ULL << 40)

#define PKT_TX_OUTER_UDP_CKSUM     (1ULL << 41)

#define PKT_TX_UDP_SEG  (1ULL << 42)

#define PKT_TX_SEC_OFFLOAD      (1ULL << 43)

#define PKT_TX_MACSEC        (1ULL << 44)

#define PKT_TX_TUNNEL_VXLAN   (0x1ULL << 45)
#define PKT_TX_TUNNEL_GRE     (0x2ULL << 45)
#define PKT_TX_TUNNEL_IPIP    (0x3ULL << 45)
#define PKT_TX_TUNNEL_GENEVE  (0x4ULL << 45)

#define PKT_TX_TUNNEL_MPLSINUDP (0x5ULL << 45)
#define PKT_TX_TUNNEL_VXLAN_GPE (0x6ULL << 45)

#define PKT_TX_TUNNEL_IP (0xDULL << 45)

#define PKT_TX_TUNNEL_UDP (0xEULL << 45)
/* add new TX TUNNEL type here */
#define PKT_TX_TUNNEL_MASK    (0xFULL << 45)

#define PKT_TX_QINQ        (1ULL << 49)
/* this old name is deprecated */
#define PKT_TX_QINQ_PKT    PKT_TX_QINQ

#define PKT_TX_TCP_SEG       (1ULL << 50)

#define PKT_TX_IEEE1588_TMST (1ULL << 51) 
#define PKT_TX_L4_NO_CKSUM   (0ULL << 52) 
#define PKT_TX_TCP_CKSUM     (1ULL << 52) 
#define PKT_TX_SCTP_CKSUM    (2ULL << 52) 
#define PKT_TX_UDP_CKSUM     (3ULL << 52) 
#define PKT_TX_L4_MASK       (3ULL << 52) 
#define PKT_TX_IP_CKSUM      (1ULL << 54)

#define PKT_TX_IPV4          (1ULL << 55)

#define PKT_TX_IPV6          (1ULL << 56)

#define PKT_TX_VLAN          (1ULL << 57)
/* this old name is deprecated */
#define PKT_TX_VLAN_PKT      PKT_TX_VLAN

#define PKT_TX_OUTER_IP_CKSUM   (1ULL << 58)

#define PKT_TX_OUTER_IPV4   (1ULL << 59)

#define PKT_TX_OUTER_IPV6    (1ULL << 60)

#define PKT_TX_OFFLOAD_MASK (    \
        PKT_TX_OUTER_IPV6 |  \
        PKT_TX_OUTER_IPV4 |  \
        PKT_TX_OUTER_IP_CKSUM |  \
        PKT_TX_VLAN_PKT |        \
        PKT_TX_IPV6 |        \
        PKT_TX_IPV4 |        \
        PKT_TX_IP_CKSUM |        \
        PKT_TX_L4_MASK |         \
        PKT_TX_IEEE1588_TMST |   \
        PKT_TX_TCP_SEG |         \
        PKT_TX_QINQ_PKT |        \
        PKT_TX_TUNNEL_MASK |     \
        PKT_TX_MACSEC |      \
        PKT_TX_SEC_OFFLOAD |     \
        PKT_TX_UDP_SEG |     \
        PKT_TX_OUTER_UDP_CKSUM | \
        PKT_TX_METADATA)

#define EXT_ATTACHED_MBUF    (1ULL << 61)

#define IND_ATTACHED_MBUF    (1ULL << 62) 
#define RTE_MBUF_PRIV_ALIGN 8

const char *rte_get_rx_ol_flag_name(uint64_t mask);

int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);

const char *rte_get_tx_ol_flag_name(uint64_t mask);

int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);

#define RTE_MBUF_DEFAULT_DATAROOM   2048
#define RTE_MBUF_DEFAULT_BUF_SIZE   \
    (RTE_MBUF_DEFAULT_DATAROOM + RTE_PKTMBUF_HEADROOM)

/* define a set of marker types that can be used to refer to set points in the
 * mbuf */
__extension__
typedef void    *MARKER[0];   
__extension__
typedef uint8_t  MARKER8[0];  
__extension__
typedef uint64_t MARKER64[0]; 
struct rte_mbuf_sched {
    uint32_t queue_id;   
    uint8_t traffic_class;
    uint8_t color;
    uint16_t reserved;   
}; 
enum {
    RTE_MBUF_L2_LEN_BITS = 7,
    RTE_MBUF_L3_LEN_BITS = 9,
    RTE_MBUF_L4_LEN_BITS = 8,
    RTE_MBUF_TSO_SEGSZ_BITS = 16,
    RTE_MBUF_OUTL3_LEN_BITS = 9,
    RTE_MBUF_OUTL2_LEN_BITS = 7,
    RTE_MBUF_TXOFLD_UNUSED_BITS = sizeof(uint64_t) * CHAR_BIT -
        RTE_MBUF_L2_LEN_BITS -
        RTE_MBUF_L3_LEN_BITS -
        RTE_MBUF_L4_LEN_BITS -
        RTE_MBUF_TSO_SEGSZ_BITS -
        RTE_MBUF_OUTL3_LEN_BITS -
        RTE_MBUF_OUTL2_LEN_BITS,
#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
    RTE_MBUF_L2_LEN_OFS =
        sizeof(uint64_t) * CHAR_BIT - RTE_MBUF_L2_LEN_BITS,
    RTE_MBUF_L3_LEN_OFS = RTE_MBUF_L2_LEN_OFS - RTE_MBUF_L3_LEN_BITS,
    RTE_MBUF_L4_LEN_OFS = RTE_MBUF_L3_LEN_OFS - RTE_MBUF_L4_LEN_BITS,
    RTE_MBUF_TSO_SEGSZ_OFS = RTE_MBUF_L4_LEN_OFS - RTE_MBUF_TSO_SEGSZ_BITS,
    RTE_MBUF_OUTL3_LEN_OFS =
        RTE_MBUF_TSO_SEGSZ_OFS - RTE_MBUF_OUTL3_LEN_BITS,
    RTE_MBUF_OUTL2_LEN_OFS =
        RTE_MBUF_OUTL3_LEN_OFS - RTE_MBUF_OUTL2_LEN_BITS,
    RTE_MBUF_TXOFLD_UNUSED_OFS =
        RTE_MBUF_OUTL2_LEN_OFS - RTE_MBUF_TXOFLD_UNUSED_BITS,
#else
    RTE_MBUF_L2_LEN_OFS = 0,
    RTE_MBUF_L3_LEN_OFS = RTE_MBUF_L2_LEN_OFS + RTE_MBUF_L2_LEN_BITS,
    RTE_MBUF_L4_LEN_OFS = RTE_MBUF_L3_LEN_OFS + RTE_MBUF_L3_LEN_BITS,
    RTE_MBUF_TSO_SEGSZ_OFS = RTE_MBUF_L4_LEN_OFS + RTE_MBUF_L4_LEN_BITS,
    RTE_MBUF_OUTL3_LEN_OFS =
        RTE_MBUF_TSO_SEGSZ_OFS + RTE_MBUF_TSO_SEGSZ_BITS,
    RTE_MBUF_OUTL2_LEN_OFS =
        RTE_MBUF_OUTL3_LEN_OFS + RTE_MBUF_OUTL3_LEN_BITS,
    RTE_MBUF_TXOFLD_UNUSED_OFS =
        RTE_MBUF_OUTL2_LEN_OFS + RTE_MBUF_OUTL2_LEN_BITS,
#endif
};

struct rte_mbuf {
    MARKER cacheline0;

    void *buf_addr;           
    RTE_STD_C11
    union {
        rte_iova_t buf_iova;
        rte_iova_t buf_physaddr; 
    } __rte_aligned(sizeof(rte_iova_t));

    /* next 8 bytes are initialised on RX descriptor rearm */
    MARKER64 rearm_data;
    uint16_t data_off;

    RTE_STD_C11
    union {
        rte_atomic16_t refcnt_atomic; 
        uint16_t refcnt;              
    };
    uint16_t nb_segs;         
    uint16_t port;

    uint64_t ol_flags;        
    /* remaining bytes are set on RX when pulling packet from descriptor */
    MARKER rx_descriptor_fields1;

    /*
     * The packet type, which is the combination of outer/inner L2, L3, L4
     * and tunnel types. The packet_type is about data really present in the
     * mbuf. Example: if vlan stripping is enabled, a received vlan packet
     * would have RTE_PTYPE_L2_ETHER and not RTE_PTYPE_L2_VLAN because the
     * vlan is stripped from the data.
     */
    RTE_STD_C11
    union {
        uint32_t packet_type; 
        struct {
            uint32_t l2_type:4; 
            uint32_t l3_type:4; 
            uint32_t l4_type:4; 
            uint32_t tun_type:4; 
            RTE_STD_C11
            union {
                uint8_t inner_esp_next_proto;
                __extension__
                struct {
                    uint8_t inner_l2_type:4;
                    uint8_t inner_l3_type:4;
                };
            };
            uint32_t inner_l4_type:4; 
        };
    };

    uint32_t pkt_len;         
    uint16_t data_len;        
    uint16_t vlan_tci;

    RTE_STD_C11
    union {
        union {
            uint32_t rss;     
            struct {
                union {
                    struct {
                        uint16_t hash;
                        uint16_t id;
                    };
                    uint32_t lo;
                };
                uint32_t hi;
            } fdir; 
            struct rte_mbuf_sched sched;
            struct {
                uint32_t reserved1;
                uint16_t reserved2;
                uint16_t txq;
            } txadapter; 
            uint32_t usr;
        } hash;                   
        struct {
            uint32_t tx_metadata;
            uint32_t reserved;
        };
    };

    uint16_t vlan_tci_outer;

    uint16_t buf_len;         
    uint64_t timestamp;

    /* second cache line - fields only used in slow path or on TX */
    MARKER cacheline1 __rte_cache_min_aligned;

    RTE_STD_C11
    union {
        void *userdata;   
        uint64_t udata64; 
    };

    struct rte_mempool *pool; 
    struct rte_mbuf *next;    
    /* fields to support TX offloads */
    RTE_STD_C11
    union {
        uint64_t tx_offload;       
        __extension__
        struct {
            uint64_t l2_len:RTE_MBUF_L2_LEN_BITS;
            uint64_t l3_len:RTE_MBUF_L3_LEN_BITS;
            uint64_t l4_len:RTE_MBUF_L4_LEN_BITS;
            uint64_t tso_segsz:RTE_MBUF_TSO_SEGSZ_BITS;
            /*
             * Fields for Tx offloading of tunnels.
             * These are undefined for packets which don't request
             * any tunnel offloads (outer IP or UDP checksum,
             * tunnel TSO).
             *
             * PMDs should not use these fields unconditionally
             * when calculating offsets.
             *
             * Applications are expected to set appropriate tunnel
             * offload flags when they fill in these fields.
             */
            uint64_t outer_l3_len:RTE_MBUF_OUTL3_LEN_BITS;
            uint64_t outer_l2_len:RTE_MBUF_OUTL2_LEN_BITS;
            /* uint64_t unused:RTE_MBUF_TXOFLD_UNUSED_BITS; */
        };
    };

    uint16_t priv_size;

    uint16_t timesync;

    uint32_t seqn;

    struct rte_mbuf_ext_shared_info *shinfo;

} __rte_cache_aligned;

typedef void (*rte_mbuf_extbuf_free_callback_t)(void *addr, void *opaque);

struct rte_mbuf_ext_shared_info {
    rte_mbuf_extbuf_free_callback_t free_cb; 
    void *fcb_opaque;                        
    rte_atomic16_t refcnt_atomic;        
};

#define RTE_MBUF_MAX_NB_SEGS    UINT16_MAX

static inline void
rte_mbuf_prefetch_part1(struct rte_mbuf *m)
{
    rte_prefetch0(&m->cacheline0);
}

static inline void
rte_mbuf_prefetch_part2(struct rte_mbuf *m)
{
#if RTE_CACHE_LINE_SIZE == 64
    rte_prefetch0(&m->cacheline1);
#else
    RTE_SET_USED(m);
#endif
}


static inline uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp);

static inline rte_iova_t
rte_mbuf_data_iova(const struct rte_mbuf *mb)
{
    return mb->buf_iova + mb->data_off;
}

__rte_deprecated
static inline phys_addr_t
rte_mbuf_data_dma_addr(const struct rte_mbuf *mb)
{
    return rte_mbuf_data_iova(mb);
}

static inline rte_iova_t
rte_mbuf_data_iova_default(const struct rte_mbuf *mb)
{
    return mb->buf_iova + RTE_PKTMBUF_HEADROOM;
}

__rte_deprecated
static inline phys_addr_t
rte_mbuf_data_dma_addr_default(const struct rte_mbuf *mb)
{
    return rte_mbuf_data_iova_default(mb);
}

static inline struct rte_mbuf *
rte_mbuf_from_indirect(struct rte_mbuf *mi)
{
    return (struct rte_mbuf *)RTE_PTR_SUB(mi->buf_addr, sizeof(*mi) + mi->priv_size);
}

__rte_experimental
static inline char *
rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
{
    return (char *)mb + sizeof(*mb) + rte_pktmbuf_priv_size(mp);
}

__rte_experimental
static inline char *
rte_mbuf_data_addr_default(__rte_unused struct rte_mbuf *mb)
{
    /* gcc complains about calling this experimental function even
     * when not using it. Hide it with ALLOW_EXPERIMENTAL_API.
     */
#ifdef ALLOW_EXPERIMENTAL_API
    return rte_mbuf_buf_addr(mb, mb->pool) + RTE_PKTMBUF_HEADROOM;
#else
    return NULL;
#endif
}

static inline char *
rte_mbuf_to_baddr(struct rte_mbuf *md)
{
#ifdef ALLOW_EXPERIMENTAL_API
    return rte_mbuf_buf_addr(md, md->pool);
#else
    char *buffer_addr;
    buffer_addr = (char *)md + sizeof(*md) + rte_pktmbuf_priv_size(md->pool);
    return buffer_addr;
#endif
}

__rte_experimental
static inline void *
rte_mbuf_to_priv(struct rte_mbuf *m)
{
    return RTE_PTR_ADD(m, sizeof(struct rte_mbuf));
}

#define RTE_MBUF_CLONED(mb)     ((mb)->ol_flags & IND_ATTACHED_MBUF)

#define RTE_MBUF_HAS_EXTBUF(mb) ((mb)->ol_flags & EXT_ATTACHED_MBUF)

#define RTE_MBUF_DIRECT(mb) \
    (!((mb)->ol_flags & (IND_ATTACHED_MBUF | EXT_ATTACHED_MBUF)))

struct rte_pktmbuf_pool_private {
    uint16_t mbuf_data_room_size; 
    uint16_t mbuf_priv_size;      
};

#ifdef RTE_LIBRTE_MBUF_DEBUG

#define __rte_mbuf_sanity_check(m, is_h) rte_mbuf_sanity_check(m, is_h)

#else /*  RTE_LIBRTE_MBUF_DEBUG */

#define __rte_mbuf_sanity_check(m, is_h) do { } while (0)

#endif /*  RTE_LIBRTE_MBUF_DEBUG */

#ifdef RTE_MBUF_REFCNT_ATOMIC

static inline uint16_t
rte_mbuf_refcnt_read(const struct rte_mbuf *m)
{
    return (uint16_t)(rte_atomic16_read(&m->refcnt_atomic));
}

static inline void
rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
{
    rte_atomic16_set(&m->refcnt_atomic, (int16_t)new_value);
}

/* internal */
static inline uint16_t
__rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
{
    return (uint16_t)(rte_atomic16_add_return(&m->refcnt_atomic, value));
}

static inline uint16_t
rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
{
    /*
     * The atomic_add is an expensive operation, so we don't want to
     * call it in the case where we know we are the unique holder of
     * this mbuf (i.e. ref_cnt == 1). Otherwise, an atomic
     * operation has to be used because concurrent accesses on the
     * reference counter can occur.
     */
    if (likely(rte_mbuf_refcnt_read(m) == 1)) {
        ++value;
        rte_mbuf_refcnt_set(m, (uint16_t)value);
        return (uint16_t)value;
    }

    return __rte_mbuf_refcnt_update(m, value);
}

#else /* ! RTE_MBUF_REFCNT_ATOMIC */

/* internal */
static inline uint16_t
__rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
{
    m->refcnt = (uint16_t)(m->refcnt + value);
    return m->refcnt;
}

static inline uint16_t
rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
{
    return __rte_mbuf_refcnt_update(m, value);
}

static inline uint16_t
rte_mbuf_refcnt_read(const struct rte_mbuf *m)
{
    return m->refcnt;
}

static inline void
rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
{
    m->refcnt = new_value;
}

#endif /* RTE_MBUF_REFCNT_ATOMIC */

static inline uint16_t
rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
{
    return (uint16_t)(rte_atomic16_read(&shinfo->refcnt_atomic));
}

static inline void
rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo,
    uint16_t new_value)
{
    rte_atomic16_set(&shinfo->refcnt_atomic, (int16_t)new_value);
}

static inline uint16_t
rte_mbuf_ext_refcnt_update(struct rte_mbuf_ext_shared_info *shinfo,
    int16_t value)
{
    if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1)) {
        ++value;
        rte_mbuf_ext_refcnt_set(shinfo, (uint16_t)value);
        return (uint16_t)value;
    }

    return (uint16_t)rte_atomic16_add_return(&shinfo->refcnt_atomic, value);
}

#define RTE_MBUF_PREFETCH_TO_FREE(m) do {       \
    if ((m) != NULL)                        \
        rte_prefetch0(m);               \
} while (0)


void
rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header);

__rte_experimental
int rte_mbuf_check(const struct rte_mbuf *m, int is_header,
           const char **reason);

#define MBUF_RAW_ALLOC_CHECK(m) do {                \
    RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);       \
    RTE_ASSERT((m)->next == NULL);              \
    RTE_ASSERT((m)->nb_segs == 1);              \
    __rte_mbuf_sanity_check(m, 0);              \
} while (0)

static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
{
    struct rte_mbuf *m;

    if (rte_mempool_get(mp, (void **)&m) < 0)
        return NULL;
    MBUF_RAW_ALLOC_CHECK(m);
    return m;
}

static __rte_always_inline void
rte_mbuf_raw_free(struct rte_mbuf *m)
{
    RTE_ASSERT(RTE_MBUF_DIRECT(m));
    RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);
    RTE_ASSERT(m->next == NULL);
    RTE_ASSERT(m->nb_segs == 1);
    __rte_mbuf_sanity_check(m, 0);
    rte_mempool_put(m->pool, m);
}

void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg,
              void *m, unsigned i);


void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg);

struct rte_mempool *
rte_pktmbuf_pool_create(const char *name, unsigned n,
    unsigned cache_size, uint16_t priv_size, uint16_t data_room_size,
    int socket_id);

struct rte_mempool *
rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n,
    unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size,
    int socket_id, const char *ops_name);

static inline uint16_t
rte_pktmbuf_data_room_size(struct rte_mempool *mp)
{
    struct rte_pktmbuf_pool_private *mbp_priv;

    mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
    return mbp_priv->mbuf_data_room_size;
}

static inline uint16_t
rte_pktmbuf_priv_size(struct rte_mempool *mp)
{
    struct rte_pktmbuf_pool_private *mbp_priv;

    mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
    return mbp_priv->mbuf_priv_size;
}

static inline void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
{
    m->data_off = (uint16_t)RTE_MIN((uint16_t)RTE_PKTMBUF_HEADROOM,
                    (uint16_t)m->buf_len);
}

#define MBUF_INVALID_PORT UINT16_MAX

static inline void rte_pktmbuf_reset(struct rte_mbuf *m)
{
    m->next = NULL;
    m->pkt_len = 0;
    m->tx_offload = 0;
    m->vlan_tci = 0;
    m->vlan_tci_outer = 0;
    m->nb_segs = 1;
    m->port = MBUF_INVALID_PORT;

    m->ol_flags = 0;
    m->packet_type = 0;
    rte_pktmbuf_reset_headroom(m);

    m->data_len = 0;
    __rte_mbuf_sanity_check(m, 1);
}

static inline struct rte_mbuf *rte_pktmbuf_alloc(struct rte_mempool *mp)
{
    struct rte_mbuf *m;
    if ((m = rte_mbuf_raw_alloc(mp)) != NULL)
        rte_pktmbuf_reset(m);
    return m;
}

static inline int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool,
     struct rte_mbuf **mbufs, unsigned count)
{
    unsigned idx = 0;
    int rc;

    rc = rte_mempool_get_bulk(pool, (void **)mbufs, count);
    if (unlikely(rc))
        return rc;

    /* To understand duff's device on loop unwinding optimization, see
     * https://en.wikipedia.org/wiki/Duff's_device.
     * Here while() loop is used rather than do() while{} to avoid extra
     * check if count is zero.
     */
    switch (count % 4) {
    case 0:
        while (idx != count) {
            MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
            rte_pktmbuf_reset(mbufs[idx]);
            idx++;
            /* fall-through */
    case 3:
            MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
            rte_pktmbuf_reset(mbufs[idx]);
            idx++;
            /* fall-through */
    case 2:
            MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
            rte_pktmbuf_reset(mbufs[idx]);
            idx++;
            /* fall-through */
    case 1:
            MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
            rte_pktmbuf_reset(mbufs[idx]);
            idx++;
            /* fall-through */
        }
    }
    return 0;
}

static inline struct rte_mbuf_ext_shared_info *
rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len,
    rte_mbuf_extbuf_free_callback_t free_cb, void *fcb_opaque)
{
    struct rte_mbuf_ext_shared_info *shinfo;
    void *buf_end = RTE_PTR_ADD(buf_addr, *buf_len);
    void *addr;

    addr = RTE_PTR_ALIGN_FLOOR(RTE_PTR_SUB(buf_end, sizeof(*shinfo)),
                   sizeof(uintptr_t));
    if (addr <= buf_addr)
        return NULL;

    shinfo = (struct rte_mbuf_ext_shared_info *)addr;
    shinfo->free_cb = free_cb;
    shinfo->fcb_opaque = fcb_opaque;
    rte_mbuf_ext_refcnt_set(shinfo, 1);

    *buf_len = (uint16_t)RTE_PTR_DIFF(shinfo, buf_addr);
    return shinfo;
}

static inline void
rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr,
    rte_iova_t buf_iova, uint16_t buf_len,
    struct rte_mbuf_ext_shared_info *shinfo)
{
    /* mbuf should not be read-only */
    RTE_ASSERT(RTE_MBUF_DIRECT(m) && rte_mbuf_refcnt_read(m) == 1);
    RTE_ASSERT(shinfo->free_cb != NULL);

    m->buf_addr = buf_addr;
    m->buf_iova = buf_iova;
    m->buf_len = buf_len;

    m->data_len = 0;
    m->data_off = 0;

    m->ol_flags |= EXT_ATTACHED_MBUF;
    m->shinfo = shinfo;
}

#define rte_pktmbuf_detach_extbuf(m) rte_pktmbuf_detach(m)

static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
{
    RTE_ASSERT(RTE_MBUF_DIRECT(mi) &&
        rte_mbuf_refcnt_read(mi) == 1);

    if (RTE_MBUF_HAS_EXTBUF(m)) {
        rte_mbuf_ext_refcnt_update(m->shinfo, 1);
        mi->ol_flags = m->ol_flags;
        mi->shinfo = m->shinfo;
    } else {
        /* if m is not direct, get the mbuf that embeds the data */
        rte_mbuf_refcnt_update(rte_mbuf_from_indirect(m), 1);
        mi->priv_size = m->priv_size;
        mi->ol_flags = m->ol_flags | IND_ATTACHED_MBUF;
    }

    mi->buf_iova = m->buf_iova;
    mi->buf_addr = m->buf_addr;
    mi->buf_len = m->buf_len;

    mi->data_off = m->data_off;
    mi->data_len = m->data_len;
    mi->port = m->port;
    mi->vlan_tci = m->vlan_tci;
    mi->vlan_tci_outer = m->vlan_tci_outer;
    mi->tx_offload = m->tx_offload;
    mi->hash = m->hash;

    mi->next = NULL;
    mi->pkt_len = mi->data_len;
    mi->nb_segs = 1;
    mi->packet_type = m->packet_type;
    mi->timestamp = m->timestamp;

    __rte_mbuf_sanity_check(mi, 1);
    __rte_mbuf_sanity_check(m, 0);
}

static inline void
__rte_pktmbuf_free_extbuf(struct rte_mbuf *m)
{
    RTE_ASSERT(RTE_MBUF_HAS_EXTBUF(m));
    RTE_ASSERT(m->shinfo != NULL);

    if (rte_mbuf_ext_refcnt_update(m->shinfo, -1) == 0)
        m->shinfo->free_cb(m->buf_addr, m->shinfo->fcb_opaque);
}

static inline void
__rte_pktmbuf_free_direct(struct rte_mbuf *m)
{
    struct rte_mbuf *md;

    RTE_ASSERT(RTE_MBUF_CLONED(m));

    md = rte_mbuf_from_indirect(m);

    if (rte_mbuf_refcnt_update(md, -1) == 0) {
        md->next = NULL;
        md->nb_segs = 1;
        rte_mbuf_refcnt_set(md, 1);
        rte_mbuf_raw_free(md);
    }
}

static inline void rte_pktmbuf_detach(struct rte_mbuf *m)
{
    struct rte_mempool *mp = m->pool;
    uint32_t mbuf_size, buf_len;
    uint16_t priv_size;

    if (RTE_MBUF_HAS_EXTBUF(m))
        __rte_pktmbuf_free_extbuf(m);
    else
        __rte_pktmbuf_free_direct(m);

    priv_size = rte_pktmbuf_priv_size(mp);
    mbuf_size = (uint32_t)(sizeof(struct rte_mbuf) + priv_size);
    buf_len = rte_pktmbuf_data_room_size(mp);

    m->priv_size = priv_size;
    m->buf_addr = (char *)m + mbuf_size;
    m->buf_iova = rte_mempool_virt2iova(m) + mbuf_size;
    m->buf_len = (uint16_t)buf_len;
    rte_pktmbuf_reset_headroom(m);
    m->data_len = 0;
    m->ol_flags = 0;
}

static __rte_always_inline struct rte_mbuf *
rte_pktmbuf_prefree_seg(struct rte_mbuf *m)
{
    __rte_mbuf_sanity_check(m, 0);

    if (likely(rte_mbuf_refcnt_read(m) == 1)) {

        if (!RTE_MBUF_DIRECT(m))
            rte_pktmbuf_detach(m);

        if (m->next != NULL) {
            m->next = NULL;
            m->nb_segs = 1;
        }

        return m;

    } else if (__rte_mbuf_refcnt_update(m, -1) == 0) {

        if (!RTE_MBUF_DIRECT(m))
            rte_pktmbuf_detach(m);

        if (m->next != NULL) {
            m->next = NULL;
            m->nb_segs = 1;
        }
        rte_mbuf_refcnt_set(m, 1);

        return m;
    }
    return NULL;
}

static __rte_always_inline void
rte_pktmbuf_free_seg(struct rte_mbuf *m)
{
    m = rte_pktmbuf_prefree_seg(m);
    if (likely(m != NULL))
        rte_mbuf_raw_free(m);
}

static inline void rte_pktmbuf_free(struct rte_mbuf *m)
{
    struct rte_mbuf *m_next;

    if (m != NULL)
        __rte_mbuf_sanity_check(m, 1);

    while (m != NULL) {
        m_next = m->next;
        rte_pktmbuf_free_seg(m);
        m = m_next;
    }
}

static inline struct rte_mbuf *rte_pktmbuf_clone(struct rte_mbuf *md,
        struct rte_mempool *mp)
{
    struct rte_mbuf *mc, *mi, **prev;
    uint32_t pktlen;
    uint16_t nseg;

    if (unlikely ((mc = rte_pktmbuf_alloc(mp)) == NULL))
        return NULL;

    mi = mc;
    prev = &mi->next;
    pktlen = md->pkt_len;
    nseg = 0;

    do {
        nseg++;
        rte_pktmbuf_attach(mi, md);
        *prev = mi;
        prev = &mi->next;
    } while ((md = md->next) != NULL &&
        (mi = rte_pktmbuf_alloc(mp)) != NULL);

    *prev = NULL;
    mc->nb_segs = nseg;
    mc->pkt_len = pktlen;

    /* Allocation of new indirect segment failed */
    if (unlikely (mi == NULL)) {
        rte_pktmbuf_free(mc);
        return NULL;
    }

    __rte_mbuf_sanity_check(mc, 1);
    return mc;
}

static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
{
    __rte_mbuf_sanity_check(m, 1);

    do {
        rte_mbuf_refcnt_update(m, v);
    } while ((m = m->next) != NULL);
}

static inline uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
{
    __rte_mbuf_sanity_check(m, 0);
    return m->data_off;
}

static inline uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
{
    __rte_mbuf_sanity_check(m, 0);
    return (uint16_t)(m->buf_len - rte_pktmbuf_headroom(m) -
              m->data_len);
}

static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m)
{
    __rte_mbuf_sanity_check(m, 1);
    while (m->next != NULL)
        m = m->next;
    return m;
}

#define rte_pktmbuf_mtod_offset(m, t, o)    \
    ((t)((char *)(m)->buf_addr + (m)->data_off + (o)))

#define rte_pktmbuf_mtod(m, t) rte_pktmbuf_mtod_offset(m, t, 0)

#define rte_pktmbuf_iova_offset(m, o) \
    (rte_iova_t)((m)->buf_iova + (m)->data_off + (o))

/* deprecated */
#define rte_pktmbuf_mtophys_offset(m, o) \
    rte_pktmbuf_iova_offset(m, o)

#define rte_pktmbuf_iova(m) rte_pktmbuf_iova_offset(m, 0)

/* deprecated */
#define rte_pktmbuf_mtophys(m) rte_pktmbuf_iova(m)

#define rte_pktmbuf_pkt_len(m) ((m)->pkt_len)

#define rte_pktmbuf_data_len(m) ((m)->data_len)

static inline char *rte_pktmbuf_prepend(struct rte_mbuf *m,
                    uint16_t len)
{
    __rte_mbuf_sanity_check(m, 1);

    if (unlikely(len > rte_pktmbuf_headroom(m)))
        return NULL;

    /* NB: elaborating the subtraction like this instead of using
     *     -= allows us to ensure the result type is uint16_t
     *     avoiding compiler warnings on gcc 8.1 at least */
    m->data_off = (uint16_t)(m->data_off - len);
    m->data_len = (uint16_t)(m->data_len + len);
    m->pkt_len  = (m->pkt_len + len);

    return (char *)m->buf_addr + m->data_off;
}

static inline char *rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
{
    void *tail;
    struct rte_mbuf *m_last;

    __rte_mbuf_sanity_check(m, 1);

    m_last = rte_pktmbuf_lastseg(m);
    if (unlikely(len > rte_pktmbuf_tailroom(m_last)))
        return NULL;

    tail = (char *)m_last->buf_addr + m_last->data_off + m_last->data_len;
    m_last->data_len = (uint16_t)(m_last->data_len + len);
    m->pkt_len  = (m->pkt_len + len);
    return (char*) tail;
}

static inline char *rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
{
    __rte_mbuf_sanity_check(m, 1);

    if (unlikely(len > m->data_len))
        return NULL;

    /* NB: elaborating the addition like this instead of using
     *     += allows us to ensure the result type is uint16_t
     *     avoiding compiler warnings on gcc 8.1 at least */
    m->data_len = (uint16_t)(m->data_len - len);
    m->data_off = (uint16_t)(m->data_off + len);
    m->pkt_len  = (m->pkt_len - len);
    return (char *)m->buf_addr + m->data_off;
}

static inline int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
{
    struct rte_mbuf *m_last;

    __rte_mbuf_sanity_check(m, 1);

    m_last = rte_pktmbuf_lastseg(m);
    if (unlikely(len > m_last->data_len))
        return -1;

    m_last->data_len = (uint16_t)(m_last->data_len - len);
    m->pkt_len  = (m->pkt_len - len);
    return 0;
}

static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
{
    __rte_mbuf_sanity_check(m, 1);
    return !!(m->nb_segs == 1);
}

const void *__rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off,
    uint32_t len, void *buf);

static inline const void *rte_pktmbuf_read(const struct rte_mbuf *m,
    uint32_t off, uint32_t len, void *buf)
{
    if (likely(off + len <= rte_pktmbuf_data_len(m)))
        return rte_pktmbuf_mtod_offset(m, char *, off);
    else
        return __rte_pktmbuf_read(m, off, len, buf);
}

static inline int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
{
    struct rte_mbuf *cur_tail;

    /* Check for number-of-segments-overflow */
    if (head->nb_segs + tail->nb_segs > RTE_MBUF_MAX_NB_SEGS)
        return -EOVERFLOW;

    /* Chain 'tail' onto the old tail */
    cur_tail = rte_pktmbuf_lastseg(head);
    cur_tail->next = tail;

    /* accumulate number of segments and total length.
     * NB: elaborating the addition like this instead of using
     *     -= allows us to ensure the result type is uint16_t
     *     avoiding compiler warnings on gcc 8.1 at least */
    head->nb_segs = (uint16_t)(head->nb_segs + tail->nb_segs);
    head->pkt_len += tail->pkt_len;

    /* pkt_len is only set in the head */
    tail->pkt_len = tail->data_len;

    return 0;
}

/*
 * @warning
 * @b EXPERIMENTAL: This API may change without prior notice.
 *
 * For given input values generate raw tx_offload value.
 * Note that it is caller responsibility to make sure that input parameters
 * don't exceed maximum bit-field values.
 * @param il2
 *   l2_len value.
 * @param il3
 *   l3_len value.
 * @param il4
 *   l4_len value.
 * @param tso
 *   tso_segsz value.
 * @param ol3
 *   outer_l3_len value.
 * @param ol2
 *   outer_l2_len value.
 * @param unused
 *   unused value.
 * @return
 *   raw tx_offload value.
 */
static __rte_always_inline uint64_t
rte_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso,
    uint64_t ol3, uint64_t ol2, uint64_t unused)
{
    return il2 << RTE_MBUF_L2_LEN_OFS |
        il3 << RTE_MBUF_L3_LEN_OFS |
        il4 << RTE_MBUF_L4_LEN_OFS |
        tso << RTE_MBUF_TSO_SEGSZ_OFS |
        ol3 << RTE_MBUF_OUTL3_LEN_OFS |
        ol2 << RTE_MBUF_OUTL2_LEN_OFS |
        unused << RTE_MBUF_TXOFLD_UNUSED_OFS;
}

static inline int
rte_validate_tx_offload(const struct rte_mbuf *m)
{
    uint64_t ol_flags = m->ol_flags;

    /* Does packet set any of available offloads? */
    if (!(ol_flags & PKT_TX_OFFLOAD_MASK))
        return 0;

    /* IP checksum can be counted only for IPv4 packet */
    if ((ol_flags & PKT_TX_IP_CKSUM) && (ol_flags & PKT_TX_IPV6))
        return -EINVAL;

    /* IP type not set when required */
    if (ol_flags & (PKT_TX_L4_MASK | PKT_TX_TCP_SEG))
        if (!(ol_flags & (PKT_TX_IPV4 | PKT_TX_IPV6)))
            return -EINVAL;

    /* Check requirements for TSO packet */
    if (ol_flags & PKT_TX_TCP_SEG)
        if ((m->tso_segsz == 0) ||
                ((ol_flags & PKT_TX_IPV4) &&
                !(ol_flags & PKT_TX_IP_CKSUM)))
            return -EINVAL;

    /* PKT_TX_OUTER_IP_CKSUM set for non outer IPv4 packet. */
    if ((ol_flags & PKT_TX_OUTER_IP_CKSUM) &&
            !(ol_flags & PKT_TX_OUTER_IPV4))
        return -EINVAL;

    return 0;
}

static inline int
rte_pktmbuf_linearize(struct rte_mbuf *mbuf)
{
    size_t seg_len, copy_len;
    struct rte_mbuf *m;
    struct rte_mbuf *m_next;
    char *buffer;

    if (rte_pktmbuf_is_contiguous(mbuf))
        return 0;

    /* Extend first segment to the total packet length */
    copy_len = rte_pktmbuf_pkt_len(mbuf) - rte_pktmbuf_data_len(mbuf);

    if (unlikely(copy_len > rte_pktmbuf_tailroom(mbuf)))
        return -1;

    buffer = rte_pktmbuf_mtod_offset(mbuf, char *, mbuf->data_len);
    mbuf->data_len = (uint16_t)(mbuf->pkt_len);

    /* Append data from next segments to the first one */
    m = mbuf->next;
    while (m != NULL) {
        m_next = m->next;

        seg_len = rte_pktmbuf_data_len(m);
        rte_memcpy(buffer, rte_pktmbuf_mtod(m, char *), seg_len);
        buffer += seg_len;

        rte_pktmbuf_free_seg(m);
        m = m_next;
    }

    mbuf->next = NULL;
    mbuf->nb_segs = 1;

    return 0;
}

void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len);

static inline uint32_t
rte_mbuf_sched_queue_get(const struct rte_mbuf *m)
{
    return m->hash.sched.queue_id;
}

static inline uint8_t
rte_mbuf_sched_traffic_class_get(const struct rte_mbuf *m)
{
    return m->hash.sched.traffic_class;
}

static inline uint8_t
rte_mbuf_sched_color_get(const struct rte_mbuf *m)
{
    return m->hash.sched.color;
}

static inline void
rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id,
            uint8_t *traffic_class,
            uint8_t *color)
{
    struct rte_mbuf_sched sched = m->hash.sched;

    *queue_id = sched.queue_id;
    *traffic_class = sched.traffic_class;
    *color = sched.color;
}

static inline void
rte_mbuf_sched_queue_set(struct rte_mbuf *m, uint32_t queue_id)
{
    m->hash.sched.queue_id = queue_id;
}

static inline void
rte_mbuf_sched_traffic_class_set(struct rte_mbuf *m, uint8_t traffic_class)
{
    m->hash.sched.traffic_class = traffic_class;
}

static inline void
rte_mbuf_sched_color_set(struct rte_mbuf *m, uint8_t color)
{
    m->hash.sched.color = color;
}

static inline void
rte_mbuf_sched_set(struct rte_mbuf *m, uint32_t queue_id,
            uint8_t traffic_class,
            uint8_t color)
{
    m->hash.sched = (struct rte_mbuf_sched){
                .queue_id = queue_id,
                .traffic_class = traffic_class,
                .color = color,
                .reserved = 0,
            };
}

#ifdef __cplusplus
}
#endif

#endif /* _RTE_MBUF_H_ */