rte_mbuf.h

Go to the documentation of this file.

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2010-2014 Intel Corporation.
 * Copyright 2014 6WIND S.A.
 */

#ifndef _RTE_MBUF_H_
#define _RTE_MBUF_H_

#include <stdint.h>
#include <rte_compat.h>
#include <rte_common.h>
#include <rte_config.h>
#include <rte_mempool.h>
#include <rte_memory.h>
#include <rte_atomic.h>
#include <rte_prefetch.h>
#include <rte_branch_prediction.h>
#include <rte_byteorder.h>
#include <rte_mbuf_ptype.h>

#ifdef __cplusplus
extern "C" {
#endif

/*
 * Packet Offload Features Flags. It also carry packet type information.
 * Critical resources. Both rx/tx shared these bits. Be cautious on any change
 *
 * - RX flags start at bit position zero, and get added to the left of previous
 *   flags.
 * - The most-significant 3 bits are reserved for generic mbuf flags
 * - TX flags therefore start at bit position 60 (i.e. 63-3), and new flags get
 *   added to the right of the previously defined flags i.e. they should count
 *   downwards, not upwards.
 *
 * Keep these flags synchronized with rte_get_rx_ol_flag_name() and
 * rte_get_tx_ol_flag_name().
 */

#define PKT_RX_VLAN          (1ULL << 0)

#define PKT_RX_RSS_HASH      (1ULL << 1)  
#define PKT_RX_FDIR          (1ULL << 2)  
#define PKT_RX_L4_CKSUM_BAD  (1ULL << 3)

#define PKT_RX_IP_CKSUM_BAD  (1ULL << 4)

#define PKT_RX_EIP_CKSUM_BAD (1ULL << 5)  
#define PKT_RX_VLAN_STRIPPED (1ULL << 6)

#define PKT_RX_IP_CKSUM_MASK ((1ULL << 4) | (1ULL << 7))

#define PKT_RX_IP_CKSUM_UNKNOWN 0
#define PKT_RX_IP_CKSUM_BAD     (1ULL << 4)
#define PKT_RX_IP_CKSUM_GOOD    (1ULL << 7)
#define PKT_RX_IP_CKSUM_NONE    ((1ULL << 4) | (1ULL << 7))

#define PKT_RX_L4_CKSUM_MASK ((1ULL << 3) | (1ULL << 8))

#define PKT_RX_L4_CKSUM_UNKNOWN 0
#define PKT_RX_L4_CKSUM_BAD     (1ULL << 3)
#define PKT_RX_L4_CKSUM_GOOD    (1ULL << 8)
#define PKT_RX_L4_CKSUM_NONE    ((1ULL << 3) | (1ULL << 8))

#define PKT_RX_IEEE1588_PTP  (1ULL << 9)  
#define PKT_RX_IEEE1588_TMST (1ULL << 10) 
#define PKT_RX_FDIR_ID       (1ULL << 13) 
#define PKT_RX_FDIR_FLX      (1ULL << 14) 
#define PKT_RX_QINQ_STRIPPED (1ULL << 15)

#define PKT_RX_LRO           (1ULL << 16)

#define PKT_RX_TIMESTAMP     (1ULL << 17)

#define PKT_RX_SEC_OFFLOAD              (1ULL << 18)

#define PKT_RX_SEC_OFFLOAD_FAILED       (1ULL << 19)

#define PKT_RX_QINQ          (1ULL << 20)

#define PKT_RX_OUTER_L4_CKSUM_MASK      ((1ULL << 21) | (1ULL << 22))

#define PKT_RX_OUTER_L4_CKSUM_UNKNOWN   0
#define PKT_RX_OUTER_L4_CKSUM_BAD       (1ULL << 21)
#define PKT_RX_OUTER_L4_CKSUM_GOOD      (1ULL << 22)
#define PKT_RX_OUTER_L4_CKSUM_INVALID   ((1ULL << 21) | (1ULL << 22))

/* add new RX flags here */

/* add new TX flags here */

#define PKT_TX_METADATA (1ULL << 40)

#define PKT_TX_OUTER_UDP_CKSUM     (1ULL << 41)

#define PKT_TX_UDP_SEG  (1ULL << 42)

#define PKT_TX_SEC_OFFLOAD              (1ULL << 43)

#define PKT_TX_MACSEC        (1ULL << 44)

#define PKT_TX_TUNNEL_VXLAN   (0x1ULL << 45)
#define PKT_TX_TUNNEL_GRE     (0x2ULL << 45)
#define PKT_TX_TUNNEL_IPIP    (0x3ULL << 45)
#define PKT_TX_TUNNEL_GENEVE  (0x4ULL << 45)

#define PKT_TX_TUNNEL_MPLSINUDP (0x5ULL << 45)
#define PKT_TX_TUNNEL_VXLAN_GPE (0x6ULL << 45)

#define PKT_TX_TUNNEL_IP (0xDULL << 45)

#define PKT_TX_TUNNEL_UDP (0xEULL << 45)
/* add new TX TUNNEL type here */
#define PKT_TX_TUNNEL_MASK    (0xFULL << 45)

#define PKT_TX_QINQ        (1ULL << 49)
/* this old name is deprecated */
#define PKT_TX_QINQ_PKT    PKT_TX_QINQ

#define PKT_TX_TCP_SEG       (1ULL << 50)

#define PKT_TX_IEEE1588_TMST (1ULL << 51) 
#define PKT_TX_L4_NO_CKSUM   (0ULL << 52) 
#define PKT_TX_TCP_CKSUM     (1ULL << 52) 
#define PKT_TX_SCTP_CKSUM    (2ULL << 52) 
#define PKT_TX_UDP_CKSUM     (3ULL << 52) 
#define PKT_TX_L4_MASK       (3ULL << 52) 
#define PKT_TX_IP_CKSUM      (1ULL << 54)

#define PKT_TX_IPV4          (1ULL << 55)

#define PKT_TX_IPV6          (1ULL << 56)

#define PKT_TX_VLAN          (1ULL << 57)
/* this old name is deprecated */
#define PKT_TX_VLAN_PKT      PKT_TX_VLAN

#define PKT_TX_OUTER_IP_CKSUM   (1ULL << 58)

#define PKT_TX_OUTER_IPV4   (1ULL << 59)

#define PKT_TX_OUTER_IPV6    (1ULL << 60)

#define PKT_TX_OFFLOAD_MASK (    \
                PKT_TX_OUTER_IPV6 |      \
                PKT_TX_OUTER_IPV4 |      \
                PKT_TX_OUTER_IP_CKSUM |  \
                PKT_TX_VLAN_PKT |        \
                PKT_TX_IPV6 |            \
                PKT_TX_IPV4 |            \
                PKT_TX_IP_CKSUM |        \
                PKT_TX_L4_MASK |         \
                PKT_TX_IEEE1588_TMST |   \
                PKT_TX_TCP_SEG |         \
                PKT_TX_QINQ_PKT |        \
                PKT_TX_TUNNEL_MASK |     \
                PKT_TX_MACSEC |          \
                PKT_TX_SEC_OFFLOAD |     \
                PKT_TX_UDP_SEG |         \
                PKT_TX_OUTER_UDP_CKSUM | \
                PKT_TX_METADATA)

#define EXT_ATTACHED_MBUF    (1ULL << 61)

#define IND_ATTACHED_MBUF    (1ULL << 62) 
#define RTE_MBUF_PRIV_ALIGN 8

const char *rte_get_rx_ol_flag_name(uint64_t mask);

int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);

const char *rte_get_tx_ol_flag_name(uint64_t mask);

int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);

#define RTE_MBUF_DEFAULT_DATAROOM       2048
#define RTE_MBUF_DEFAULT_BUF_SIZE       \
        (RTE_MBUF_DEFAULT_DATAROOM + RTE_PKTMBUF_HEADROOM)

/* define a set of marker types that can be used to refer to set points in the
 * mbuf */
__extension__
typedef void    *MARKER[0];   
__extension__
typedef uint8_t  MARKER8[0];  
__extension__
typedef uint64_t MARKER64[0]; 
struct rte_mbuf_sched {
        uint32_t queue_id;   
        uint8_t traffic_class;
        uint8_t color;
        uint16_t reserved;   
}; 
enum {
        RTE_MBUF_L2_LEN_BITS = 7,
        RTE_MBUF_L3_LEN_BITS = 9,
        RTE_MBUF_L4_LEN_BITS = 8,
        RTE_MBUF_TSO_SEGSZ_BITS = 16,
        RTE_MBUF_OUTL3_LEN_BITS = 9,
        RTE_MBUF_OUTL2_LEN_BITS = 7,
        RTE_MBUF_TXOFLD_UNUSED_BITS = sizeof(uint64_t) * CHAR_BIT -
                RTE_MBUF_L2_LEN_BITS -
                RTE_MBUF_L3_LEN_BITS -
                RTE_MBUF_L4_LEN_BITS -
                RTE_MBUF_TSO_SEGSZ_BITS -
                RTE_MBUF_OUTL3_LEN_BITS -
                RTE_MBUF_OUTL2_LEN_BITS,
#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
        RTE_MBUF_L2_LEN_OFS =
                sizeof(uint64_t) * CHAR_BIT - RTE_MBUF_L2_LEN_BITS,
        RTE_MBUF_L3_LEN_OFS = RTE_MBUF_L2_LEN_OFS - RTE_MBUF_L3_LEN_BITS,
        RTE_MBUF_L4_LEN_OFS = RTE_MBUF_L3_LEN_OFS - RTE_MBUF_L4_LEN_BITS,
        RTE_MBUF_TSO_SEGSZ_OFS = RTE_MBUF_L4_LEN_OFS - RTE_MBUF_TSO_SEGSZ_BITS,
        RTE_MBUF_OUTL3_LEN_OFS =
                RTE_MBUF_TSO_SEGSZ_OFS - RTE_MBUF_OUTL3_LEN_BITS,
        RTE_MBUF_OUTL2_LEN_OFS =
                RTE_MBUF_OUTL3_LEN_OFS - RTE_MBUF_OUTL2_LEN_BITS,
        RTE_MBUF_TXOFLD_UNUSED_OFS =
                RTE_MBUF_OUTL2_LEN_OFS - RTE_MBUF_TXOFLD_UNUSED_BITS,
#else
        RTE_MBUF_L2_LEN_OFS = 0,
        RTE_MBUF_L3_LEN_OFS = RTE_MBUF_L2_LEN_OFS + RTE_MBUF_L2_LEN_BITS,
        RTE_MBUF_L4_LEN_OFS = RTE_MBUF_L3_LEN_OFS + RTE_MBUF_L3_LEN_BITS,
        RTE_MBUF_TSO_SEGSZ_OFS = RTE_MBUF_L4_LEN_OFS + RTE_MBUF_L4_LEN_BITS,
        RTE_MBUF_OUTL3_LEN_OFS =
                RTE_MBUF_TSO_SEGSZ_OFS + RTE_MBUF_TSO_SEGSZ_BITS,
        RTE_MBUF_OUTL2_LEN_OFS =
                RTE_MBUF_OUTL3_LEN_OFS + RTE_MBUF_OUTL3_LEN_BITS,
        RTE_MBUF_TXOFLD_UNUSED_OFS =
                RTE_MBUF_OUTL2_LEN_OFS + RTE_MBUF_OUTL2_LEN_BITS,
#endif
};

struct rte_mbuf {
        MARKER cacheline0;

        void *buf_addr;           
        RTE_STD_C11
        union {
                rte_iova_t buf_iova;
                rte_iova_t buf_physaddr; 
        } __rte_aligned(sizeof(rte_iova_t));

        /* next 8 bytes are initialised on RX descriptor rearm */
        MARKER64 rearm_data;
        uint16_t data_off;

        RTE_STD_C11
        union {
                rte_atomic16_t refcnt_atomic; 
                uint16_t refcnt;              
        };
        uint16_t nb_segs;         
        uint16_t port;

        uint64_t ol_flags;        
        /* remaining bytes are set on RX when pulling packet from descriptor */
        MARKER rx_descriptor_fields1;

        /*
         * The packet type, which is the combination of outer/inner L2, L3, L4
         * and tunnel types. The packet_type is about data really present in the
         * mbuf. Example: if vlan stripping is enabled, a received vlan packet
         * would have RTE_PTYPE_L2_ETHER and not RTE_PTYPE_L2_VLAN because the
         * vlan is stripped from the data.
         */
        RTE_STD_C11
        union {
                uint32_t packet_type; 
                struct {
                        uint32_t l2_type:4; 
                        uint32_t l3_type:4; 
                        uint32_t l4_type:4; 
                        uint32_t tun_type:4; 
                        RTE_STD_C11
                        union {
                                uint8_t inner_esp_next_proto;
                                __extension__
                                struct {
                                        uint8_t inner_l2_type:4;
                                        uint8_t inner_l3_type:4;
                                };
                        };
                        uint32_t inner_l4_type:4; 
                };
        };

        uint32_t pkt_len;         
        uint16_t data_len;        
        uint16_t vlan_tci;

        RTE_STD_C11
        union {
                union {
                        uint32_t rss;     
                        struct {
                                union {
                                        struct {
                                                uint16_t hash;
                                                uint16_t id;
                                        };
                                        uint32_t lo;
                                };
                                uint32_t hi;
                        } fdir; 
                        struct rte_mbuf_sched sched;
                        struct {
                                uint32_t reserved1;
                                uint16_t reserved2;
                                uint16_t txq;
                        } txadapter; 
                        uint32_t usr;
                } hash;                   
                struct {
                        uint32_t tx_metadata;
                        uint32_t reserved;
                };
        };

        uint16_t vlan_tci_outer;

        uint16_t buf_len;         
        uint64_t timestamp;

        /* second cache line - fields only used in slow path or on TX */
        MARKER cacheline1 __rte_cache_min_aligned;

        RTE_STD_C11
        union {
                void *userdata;   
                uint64_t udata64; 
        };

        struct rte_mempool *pool; 
        struct rte_mbuf *next;    
        /* fields to support TX offloads */
        RTE_STD_C11
        union {
                uint64_t tx_offload;       
                __extension__
                struct {
                        uint64_t l2_len:RTE_MBUF_L2_LEN_BITS;
                        uint64_t l3_len:RTE_MBUF_L3_LEN_BITS;
                        uint64_t l4_len:RTE_MBUF_L4_LEN_BITS;
                        uint64_t tso_segsz:RTE_MBUF_TSO_SEGSZ_BITS;
                        /* fields for TX offloading of tunnels */
                        uint64_t outer_l3_len:RTE_MBUF_OUTL3_LEN_BITS;
                        uint64_t outer_l2_len:RTE_MBUF_OUTL2_LEN_BITS;
                        /* uint64_t unused:RTE_MBUF_TXOFLD_UNUSED_BITS; */
                };
        };

        uint16_t priv_size;

        uint16_t timesync;

        uint32_t seqn;

        struct rte_mbuf_ext_shared_info *shinfo;

} __rte_cache_aligned;

typedef void (*rte_mbuf_extbuf_free_callback_t)(void *addr, void *opaque);

struct rte_mbuf_ext_shared_info {
        rte_mbuf_extbuf_free_callback_t free_cb; 
        void *fcb_opaque;                        
        rte_atomic16_t refcnt_atomic;        
};

#define RTE_MBUF_MAX_NB_SEGS    UINT16_MAX

static inline void
rte_mbuf_prefetch_part1(struct rte_mbuf *m)
{
        rte_prefetch0(&m->cacheline0);
}

static inline void
rte_mbuf_prefetch_part2(struct rte_mbuf *m)
{
#if RTE_CACHE_LINE_SIZE == 64
        rte_prefetch0(&m->cacheline1);
#else
        RTE_SET_USED(m);
#endif
}


static inline uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp);

static inline rte_iova_t
rte_mbuf_data_iova(const struct rte_mbuf *mb)
{
        return mb->buf_iova + mb->data_off;
}

__rte_deprecated
static inline phys_addr_t
rte_mbuf_data_dma_addr(const struct rte_mbuf *mb)
{
        return rte_mbuf_data_iova(mb);
}

static inline rte_iova_t
rte_mbuf_data_iova_default(const struct rte_mbuf *mb)
{
        return mb->buf_iova + RTE_PKTMBUF_HEADROOM;
}

__rte_deprecated
static inline phys_addr_t
rte_mbuf_data_dma_addr_default(const struct rte_mbuf *mb)
{
        return rte_mbuf_data_iova_default(mb);
}

static inline struct rte_mbuf *
rte_mbuf_from_indirect(struct rte_mbuf *mi)
{
        return (struct rte_mbuf *)RTE_PTR_SUB(mi->buf_addr, sizeof(*mi) + mi->priv_size);
}

static inline char * __rte_experimental
rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
{
        return (char *)mb + sizeof(*mb) + rte_pktmbuf_priv_size(mp);
}

static inline char * __rte_experimental
rte_mbuf_data_addr_default(struct rte_mbuf *mb)
{
        return rte_mbuf_buf_addr(mb, mb->pool) + RTE_PKTMBUF_HEADROOM;
}

static inline char *
rte_mbuf_to_baddr(struct rte_mbuf *md)
{
#ifdef ALLOW_EXPERIMENTAL_API
        return rte_mbuf_buf_addr(md, md->pool);
#else
        char *buffer_addr;
        buffer_addr = (char *)md + sizeof(*md) + rte_pktmbuf_priv_size(md->pool);
        return buffer_addr;
#endif
}

static inline void * __rte_experimental
rte_mbuf_to_priv(struct rte_mbuf *m)
{
        return RTE_PTR_ADD(m, sizeof(struct rte_mbuf));
}

#define RTE_MBUF_CLONED(mb)     ((mb)->ol_flags & IND_ATTACHED_MBUF)

#define RTE_MBUF_HAS_EXTBUF(mb) ((mb)->ol_flags & EXT_ATTACHED_MBUF)

#define RTE_MBUF_DIRECT(mb) \
        (!((mb)->ol_flags & (IND_ATTACHED_MBUF | EXT_ATTACHED_MBUF)))

struct rte_pktmbuf_pool_private {
        uint16_t mbuf_data_room_size; 
        uint16_t mbuf_priv_size;      
};

#ifdef RTE_LIBRTE_MBUF_DEBUG

#define __rte_mbuf_sanity_check(m, is_h) rte_mbuf_sanity_check(m, is_h)

#else /*  RTE_LIBRTE_MBUF_DEBUG */

#define __rte_mbuf_sanity_check(m, is_h) do { } while (0)

#endif /*  RTE_LIBRTE_MBUF_DEBUG */

#ifdef RTE_MBUF_REFCNT_ATOMIC

static inline uint16_t
rte_mbuf_refcnt_read(const struct rte_mbuf *m)
{
        return (uint16_t)(rte_atomic16_read(&m->refcnt_atomic));
}

static inline void
rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
{
        rte_atomic16_set(&m->refcnt_atomic, (int16_t)new_value);
}

/* internal */
static inline uint16_t
__rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
{
        return (uint16_t)(rte_atomic16_add_return(&m->refcnt_atomic, value));
}

static inline uint16_t
rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
{
        /*
         * The atomic_add is an expensive operation, so we don't want to
         * call it in the case where we know we are the unique holder of
         * this mbuf (i.e. ref_cnt == 1). Otherwise, an atomic
         * operation has to be used because concurrent accesses on the
         * reference counter can occur.
         */
        if (likely(rte_mbuf_refcnt_read(m) == 1)) {
                ++value;
                rte_mbuf_refcnt_set(m, (uint16_t)value);
                return (uint16_t)value;
        }

        return __rte_mbuf_refcnt_update(m, value);
}

#else /* ! RTE_MBUF_REFCNT_ATOMIC */

/* internal */
static inline uint16_t
__rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
{
        m->refcnt = (uint16_t)(m->refcnt + value);
        return m->refcnt;
}

static inline uint16_t
rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
{
        return __rte_mbuf_refcnt_update(m, value);
}

static inline uint16_t
rte_mbuf_refcnt_read(const struct rte_mbuf *m)
{
        return m->refcnt;
}

static inline void
rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
{
        m->refcnt = new_value;
}

#endif /* RTE_MBUF_REFCNT_ATOMIC */

static inline uint16_t
rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
{
        return (uint16_t)(rte_atomic16_read(&shinfo->refcnt_atomic));
}

static inline void
rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo,
        uint16_t new_value)
{
        rte_atomic16_set(&shinfo->refcnt_atomic, (int16_t)new_value);
}

static inline uint16_t
rte_mbuf_ext_refcnt_update(struct rte_mbuf_ext_shared_info *shinfo,
        int16_t value)
{
        if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1)) {
                ++value;
                rte_mbuf_ext_refcnt_set(shinfo, (uint16_t)value);
                return (uint16_t)value;
        }

        return (uint16_t)rte_atomic16_add_return(&shinfo->refcnt_atomic, value);
}

#define RTE_MBUF_PREFETCH_TO_FREE(m) do {       \
        if ((m) != NULL)                        \
                rte_prefetch0(m);               \
} while (0)


void
rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header);

__rte_experimental
int rte_mbuf_check(const struct rte_mbuf *m, int is_header,
                   const char **reason);

#define MBUF_RAW_ALLOC_CHECK(m) do {                            \
        RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);               \
        RTE_ASSERT((m)->next == NULL);                          \
        RTE_ASSERT((m)->nb_segs == 1);                          \
        __rte_mbuf_sanity_check(m, 0);                          \
} while (0)

static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
{
        struct rte_mbuf *m;

        if (rte_mempool_get(mp, (void **)&m) < 0)
                return NULL;
        MBUF_RAW_ALLOC_CHECK(m);
        return m;
}

static __rte_always_inline void
rte_mbuf_raw_free(struct rte_mbuf *m)
{
        RTE_ASSERT(RTE_MBUF_DIRECT(m));
        RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);
        RTE_ASSERT(m->next == NULL);
        RTE_ASSERT(m->nb_segs == 1);
        __rte_mbuf_sanity_check(m, 0);
        rte_mempool_put(m->pool, m);
}

void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg,
                      void *m, unsigned i);


void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg);

struct rte_mempool *
rte_pktmbuf_pool_create(const char *name, unsigned n,
        unsigned cache_size, uint16_t priv_size, uint16_t data_room_size,
        int socket_id);

struct rte_mempool *
rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n,
        unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size,
        int socket_id, const char *ops_name);

static inline uint16_t
rte_pktmbuf_data_room_size(struct rte_mempool *mp)
{
        struct rte_pktmbuf_pool_private *mbp_priv;

        mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
        return mbp_priv->mbuf_data_room_size;
}

static inline uint16_t
rte_pktmbuf_priv_size(struct rte_mempool *mp)
{
        struct rte_pktmbuf_pool_private *mbp_priv;

        mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
        return mbp_priv->mbuf_priv_size;
}

static inline void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
{
        m->data_off = (uint16_t)RTE_MIN((uint16_t)RTE_PKTMBUF_HEADROOM,
                                        (uint16_t)m->buf_len);
}

#define MBUF_INVALID_PORT UINT16_MAX

static inline void rte_pktmbuf_reset(struct rte_mbuf *m)
{
        m->next = NULL;
        m->pkt_len = 0;
        m->tx_offload = 0;
        m->vlan_tci = 0;
        m->vlan_tci_outer = 0;
        m->nb_segs = 1;
        m->port = MBUF_INVALID_PORT;

        m->ol_flags = 0;
        m->packet_type = 0;
        rte_pktmbuf_reset_headroom(m);

        m->data_len = 0;
        __rte_mbuf_sanity_check(m, 1);
}

static inline struct rte_mbuf *rte_pktmbuf_alloc(struct rte_mempool *mp)
{
        struct rte_mbuf *m;
        if ((m = rte_mbuf_raw_alloc(mp)) != NULL)
                rte_pktmbuf_reset(m);
        return m;
}

static inline int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool,
         struct rte_mbuf **mbufs, unsigned count)
{
        unsigned idx = 0;
        int rc;

        rc = rte_mempool_get_bulk(pool, (void **)mbufs, count);
        if (unlikely(rc))
                return rc;

        /* To understand duff's device on loop unwinding optimization, see
         * https://en.wikipedia.org/wiki/Duff's_device.
         * Here while() loop is used rather than do() while{} to avoid extra
         * check if count is zero.
         */
        switch (count % 4) {
        case 0:
                while (idx != count) {
                        MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
                        rte_pktmbuf_reset(mbufs[idx]);
                        idx++;
                        /* fall-through */
        case 3:
                        MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
                        rte_pktmbuf_reset(mbufs[idx]);
                        idx++;
                        /* fall-through */
        case 2:
                        MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
                        rte_pktmbuf_reset(mbufs[idx]);
                        idx++;
                        /* fall-through */
        case 1:
                        MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
                        rte_pktmbuf_reset(mbufs[idx]);
                        idx++;
                        /* fall-through */
                }
        }
        return 0;
}

static inline struct rte_mbuf_ext_shared_info *
rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len,
        rte_mbuf_extbuf_free_callback_t free_cb, void *fcb_opaque)
{
        struct rte_mbuf_ext_shared_info *shinfo;
        void *buf_end = RTE_PTR_ADD(buf_addr, *buf_len);
        void *addr;

        addr = RTE_PTR_ALIGN_FLOOR(RTE_PTR_SUB(buf_end, sizeof(*shinfo)),
                                   sizeof(uintptr_t));
        if (addr <= buf_addr)
                return NULL;

        shinfo = (struct rte_mbuf_ext_shared_info *)addr;
        shinfo->free_cb = free_cb;
        shinfo->fcb_opaque = fcb_opaque;
        rte_mbuf_ext_refcnt_set(shinfo, 1);

        *buf_len = (uint16_t)RTE_PTR_DIFF(shinfo, buf_addr);
        return shinfo;
}

static inline void
rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr,
        rte_iova_t buf_iova, uint16_t buf_len,
        struct rte_mbuf_ext_shared_info *shinfo)
{
        /* mbuf should not be read-only */
        RTE_ASSERT(RTE_MBUF_DIRECT(m) && rte_mbuf_refcnt_read(m) == 1);
        RTE_ASSERT(shinfo->free_cb != NULL);

        m->buf_addr = buf_addr;
        m->buf_iova = buf_iova;
        m->buf_len = buf_len;

        m->data_len = 0;
        m->data_off = 0;

        m->ol_flags |= EXT_ATTACHED_MBUF;
        m->shinfo = shinfo;
}

#define rte_pktmbuf_detach_extbuf(m) rte_pktmbuf_detach(m)

static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
{
        RTE_ASSERT(RTE_MBUF_DIRECT(mi) &&
            rte_mbuf_refcnt_read(mi) == 1);

        if (RTE_MBUF_HAS_EXTBUF(m)) {
                rte_mbuf_ext_refcnt_update(m->shinfo, 1);
                mi->ol_flags = m->ol_flags;
                mi->shinfo = m->shinfo;
        } else {
                /* if m is not direct, get the mbuf that embeds the data */
                rte_mbuf_refcnt_update(rte_mbuf_from_indirect(m), 1);
                mi->priv_size = m->priv_size;
                mi->ol_flags = m->ol_flags | IND_ATTACHED_MBUF;
        }

        mi->buf_iova = m->buf_iova;
        mi->buf_addr = m->buf_addr;
        mi->buf_len = m->buf_len;

        mi->data_off = m->data_off;
        mi->data_len = m->data_len;
        mi->port = m->port;
        mi->vlan_tci = m->vlan_tci;
        mi->vlan_tci_outer = m->vlan_tci_outer;
        mi->tx_offload = m->tx_offload;
        mi->hash = m->hash;

        mi->next = NULL;
        mi->pkt_len = mi->data_len;
        mi->nb_segs = 1;
        mi->packet_type = m->packet_type;
        mi->timestamp = m->timestamp;

        __rte_mbuf_sanity_check(mi, 1);
        __rte_mbuf_sanity_check(m, 0);
}

static inline void
__rte_pktmbuf_free_extbuf(struct rte_mbuf *m)
{
        RTE_ASSERT(RTE_MBUF_HAS_EXTBUF(m));
        RTE_ASSERT(m->shinfo != NULL);

        if (rte_mbuf_ext_refcnt_update(m->shinfo, -1) == 0)
                m->shinfo->free_cb(m->buf_addr, m->shinfo->fcb_opaque);
}

static inline void
__rte_pktmbuf_free_direct(struct rte_mbuf *m)
{
        struct rte_mbuf *md;

        RTE_ASSERT(RTE_MBUF_CLONED(m));

        md = rte_mbuf_from_indirect(m);

        if (rte_mbuf_refcnt_update(md, -1) == 0) {
                md->next = NULL;
                md->nb_segs = 1;
                rte_mbuf_refcnt_set(md, 1);
                rte_mbuf_raw_free(md);
        }
}

static inline void rte_pktmbuf_detach(struct rte_mbuf *m)
{
        struct rte_mempool *mp = m->pool;
        uint32_t mbuf_size, buf_len;
        uint16_t priv_size;

        if (RTE_MBUF_HAS_EXTBUF(m))
                __rte_pktmbuf_free_extbuf(m);
        else
                __rte_pktmbuf_free_direct(m);

        priv_size = rte_pktmbuf_priv_size(mp);
        mbuf_size = (uint32_t)(sizeof(struct rte_mbuf) + priv_size);
        buf_len = rte_pktmbuf_data_room_size(mp);

        m->priv_size = priv_size;
        m->buf_addr = (char *)m + mbuf_size;
        m->buf_iova = rte_mempool_virt2iova(m) + mbuf_size;
        m->buf_len = (uint16_t)buf_len;
        rte_pktmbuf_reset_headroom(m);
        m->data_len = 0;
        m->ol_flags = 0;
}

static __rte_always_inline struct rte_mbuf *
rte_pktmbuf_prefree_seg(struct rte_mbuf *m)
{
        __rte_mbuf_sanity_check(m, 0);

        if (likely(rte_mbuf_refcnt_read(m) == 1)) {

                if (!RTE_MBUF_DIRECT(m))
                        rte_pktmbuf_detach(m);

                if (m->next != NULL) {
                        m->next = NULL;
                        m->nb_segs = 1;
                }

                return m;

        } else if (__rte_mbuf_refcnt_update(m, -1) == 0) {

                if (!RTE_MBUF_DIRECT(m))
                        rte_pktmbuf_detach(m);

                if (m->next != NULL) {
                        m->next = NULL;
                        m->nb_segs = 1;
                }
                rte_mbuf_refcnt_set(m, 1);

                return m;
        }
        return NULL;
}

static __rte_always_inline void
rte_pktmbuf_free_seg(struct rte_mbuf *m)
{
        m = rte_pktmbuf_prefree_seg(m);
        if (likely(m != NULL))
                rte_mbuf_raw_free(m);
}

static inline void rte_pktmbuf_free(struct rte_mbuf *m)
{
        struct rte_mbuf *m_next;

        if (m != NULL)
                __rte_mbuf_sanity_check(m, 1);

        while (m != NULL) {
                m_next = m->next;
                rte_pktmbuf_free_seg(m);
                m = m_next;
        }
}

static inline struct rte_mbuf *rte_pktmbuf_clone(struct rte_mbuf *md,
                struct rte_mempool *mp)
{
        struct rte_mbuf *mc, *mi, **prev;
        uint32_t pktlen;
        uint16_t nseg;

        if (unlikely ((mc = rte_pktmbuf_alloc(mp)) == NULL))
                return NULL;

        mi = mc;
        prev = &mi->next;
        pktlen = md->pkt_len;
        nseg = 0;

        do {
                nseg++;
                rte_pktmbuf_attach(mi, md);
                *prev = mi;
                prev = &mi->next;
        } while ((md = md->next) != NULL &&
            (mi = rte_pktmbuf_alloc(mp)) != NULL);

        *prev = NULL;
        mc->nb_segs = nseg;
        mc->pkt_len = pktlen;

        /* Allocation of new indirect segment failed */
        if (unlikely (mi == NULL)) {
                rte_pktmbuf_free(mc);
                return NULL;
        }

        __rte_mbuf_sanity_check(mc, 1);
        return mc;
}

static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
{
        __rte_mbuf_sanity_check(m, 1);

        do {
                rte_mbuf_refcnt_update(m, v);
        } while ((m = m->next) != NULL);
}

static inline uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
{
        __rte_mbuf_sanity_check(m, 0);
        return m->data_off;
}

static inline uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
{
        __rte_mbuf_sanity_check(m, 0);
        return (uint16_t)(m->buf_len - rte_pktmbuf_headroom(m) -
                          m->data_len);
}

static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m)
{
        __rte_mbuf_sanity_check(m, 1);
        while (m->next != NULL)
                m = m->next;
        return m;
}

#define rte_pktmbuf_mtod_offset(m, t, o)        \
        ((t)((char *)(m)->buf_addr + (m)->data_off + (o)))

#define rte_pktmbuf_mtod(m, t) rte_pktmbuf_mtod_offset(m, t, 0)

#define rte_pktmbuf_iova_offset(m, o) \
        (rte_iova_t)((m)->buf_iova + (m)->data_off + (o))

/* deprecated */
#define rte_pktmbuf_mtophys_offset(m, o) \
        rte_pktmbuf_iova_offset(m, o)

#define rte_pktmbuf_iova(m) rte_pktmbuf_iova_offset(m, 0)

/* deprecated */
#define rte_pktmbuf_mtophys(m) rte_pktmbuf_iova(m)

#define rte_pktmbuf_pkt_len(m) ((m)->pkt_len)

#define rte_pktmbuf_data_len(m) ((m)->data_len)

static inline char *rte_pktmbuf_prepend(struct rte_mbuf *m,
                                        uint16_t len)
{
        __rte_mbuf_sanity_check(m, 1);

        if (unlikely(len > rte_pktmbuf_headroom(m)))
                return NULL;

        /* NB: elaborating the subtraction like this instead of using
         *     -= allows us to ensure the result type is uint16_t
         *     avoiding compiler warnings on gcc 8.1 at least */
        m->data_off = (uint16_t)(m->data_off - len);
        m->data_len = (uint16_t)(m->data_len + len);
        m->pkt_len  = (m->pkt_len + len);

        return (char *)m->buf_addr + m->data_off;
}

static inline char *rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
{
        void *tail;
        struct rte_mbuf *m_last;

        __rte_mbuf_sanity_check(m, 1);

        m_last = rte_pktmbuf_lastseg(m);
        if (unlikely(len > rte_pktmbuf_tailroom(m_last)))
                return NULL;

        tail = (char *)m_last->buf_addr + m_last->data_off + m_last->data_len;
        m_last->data_len = (uint16_t)(m_last->data_len + len);
        m->pkt_len  = (m->pkt_len + len);
        return (char*) tail;
}

static inline char *rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
{
        __rte_mbuf_sanity_check(m, 1);

        if (unlikely(len > m->data_len))
                return NULL;

        /* NB: elaborating the addition like this instead of using
         *     += allows us to ensure the result type is uint16_t
         *     avoiding compiler warnings on gcc 8.1 at least */
        m->data_len = (uint16_t)(m->data_len - len);
        m->data_off = (uint16_t)(m->data_off + len);
        m->pkt_len  = (m->pkt_len - len);
        return (char *)m->buf_addr + m->data_off;
}

static inline int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
{
        struct rte_mbuf *m_last;

        __rte_mbuf_sanity_check(m, 1);

        m_last = rte_pktmbuf_lastseg(m);
        if (unlikely(len > m_last->data_len))
                return -1;

        m_last->data_len = (uint16_t)(m_last->data_len - len);
        m->pkt_len  = (m->pkt_len - len);
        return 0;
}

static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
{
        __rte_mbuf_sanity_check(m, 1);
        return !!(m->nb_segs == 1);
}

const void *__rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off,
        uint32_t len, void *buf);

static inline const void *rte_pktmbuf_read(const struct rte_mbuf *m,
        uint32_t off, uint32_t len, void *buf)
{
        if (likely(off + len <= rte_pktmbuf_data_len(m)))
                return rte_pktmbuf_mtod_offset(m, char *, off);
        else
                return __rte_pktmbuf_read(m, off, len, buf);
}

static inline int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
{
        struct rte_mbuf *cur_tail;

        /* Check for number-of-segments-overflow */
        if (head->nb_segs + tail->nb_segs > RTE_MBUF_MAX_NB_SEGS)
                return -EOVERFLOW;

        /* Chain 'tail' onto the old tail */
        cur_tail = rte_pktmbuf_lastseg(head);
        cur_tail->next = tail;

        /* accumulate number of segments and total length.
         * NB: elaborating the addition like this instead of using
         *     -= allows us to ensure the result type is uint16_t
         *     avoiding compiler warnings on gcc 8.1 at least */
        head->nb_segs = (uint16_t)(head->nb_segs + tail->nb_segs);
        head->pkt_len += tail->pkt_len;

        /* pkt_len is only set in the head */
        tail->pkt_len = tail->data_len;

        return 0;
}

/*
 * @warning
 * @b EXPERIMENTAL: This API may change without prior notice.
 *
 * For given input values generate raw tx_offload value.
 * Note that it is caller responsibility to make sure that input parameters
 * don't exceed maximum bit-field values.
 * @param il2
 *   l2_len value.
 * @param il3
 *   l3_len value.
 * @param il4
 *   l4_len value.
 * @param tso
 *   tso_segsz value.
 * @param ol3
 *   outer_l3_len value.
 * @param ol2
 *   outer_l2_len value.
 * @param unused
 *   unused value.
 * @return
 *   raw tx_offload value.
 */
static __rte_always_inline uint64_t
rte_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso,
        uint64_t ol3, uint64_t ol2, uint64_t unused)
{
        return il2 << RTE_MBUF_L2_LEN_OFS |
                il3 << RTE_MBUF_L3_LEN_OFS |
                il4 << RTE_MBUF_L4_LEN_OFS |
                tso << RTE_MBUF_TSO_SEGSZ_OFS |
                ol3 << RTE_MBUF_OUTL3_LEN_OFS |
                ol2 << RTE_MBUF_OUTL2_LEN_OFS |
                unused << RTE_MBUF_TXOFLD_UNUSED_OFS;
}

static inline int
rte_validate_tx_offload(const struct rte_mbuf *m)
{
        uint64_t ol_flags = m->ol_flags;

        /* Does packet set any of available offloads? */
        if (!(ol_flags & PKT_TX_OFFLOAD_MASK))
                return 0;

        /* IP checksum can be counted only for IPv4 packet */
        if ((ol_flags & PKT_TX_IP_CKSUM) && (ol_flags & PKT_TX_IPV6))
                return -EINVAL;

        /* IP type not set when required */
        if (ol_flags & (PKT_TX_L4_MASK | PKT_TX_TCP_SEG))
                if (!(ol_flags & (PKT_TX_IPV4 | PKT_TX_IPV6)))
                        return -EINVAL;

        /* Check requirements for TSO packet */
        if (ol_flags & PKT_TX_TCP_SEG)
                if ((m->tso_segsz == 0) ||
                                ((ol_flags & PKT_TX_IPV4) &&
                                !(ol_flags & PKT_TX_IP_CKSUM)))
                        return -EINVAL;

        /* PKT_TX_OUTER_IP_CKSUM set for non outer IPv4 packet. */
        if ((ol_flags & PKT_TX_OUTER_IP_CKSUM) &&
                        !(ol_flags & PKT_TX_OUTER_IPV4))
                return -EINVAL;

        return 0;
}

static inline int
rte_pktmbuf_linearize(struct rte_mbuf *mbuf)
{
        size_t seg_len, copy_len;
        struct rte_mbuf *m;
        struct rte_mbuf *m_next;
        char *buffer;

        if (rte_pktmbuf_is_contiguous(mbuf))
                return 0;

        /* Extend first segment to the total packet length */
        copy_len = rte_pktmbuf_pkt_len(mbuf) - rte_pktmbuf_data_len(mbuf);

        if (unlikely(copy_len > rte_pktmbuf_tailroom(mbuf)))
                return -1;

        buffer = rte_pktmbuf_mtod_offset(mbuf, char *, mbuf->data_len);
        mbuf->data_len = (uint16_t)(mbuf->pkt_len);

        /* Append data from next segments to the first one */
        m = mbuf->next;
        while (m != NULL) {
                m_next = m->next;

                seg_len = rte_pktmbuf_data_len(m);
                rte_memcpy(buffer, rte_pktmbuf_mtod(m, char *), seg_len);
                buffer += seg_len;

                rte_pktmbuf_free_seg(m);
                m = m_next;
        }

        mbuf->next = NULL;
        mbuf->nb_segs = 1;

        return 0;
}

void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len);

static inline uint32_t
rte_mbuf_sched_queue_get(const struct rte_mbuf *m)
{
        return m->hash.sched.queue_id;
}

static inline uint8_t
rte_mbuf_sched_traffic_class_get(const struct rte_mbuf *m)
{
        return m->hash.sched.traffic_class;
}

static inline uint8_t
rte_mbuf_sched_color_get(const struct rte_mbuf *m)
{
        return m->hash.sched.color;
}

static inline void
rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id,
                        uint8_t *traffic_class,
                        uint8_t *color)
{
        struct rte_mbuf_sched sched = m->hash.sched;

        *queue_id = sched.queue_id;
        *traffic_class = sched.traffic_class;
        *color = sched.color;
}

static inline void
rte_mbuf_sched_queue_set(struct rte_mbuf *m, uint32_t queue_id)
{
        m->hash.sched.queue_id = queue_id;
}

static inline void
rte_mbuf_sched_traffic_class_set(struct rte_mbuf *m, uint8_t traffic_class)
{
        m->hash.sched.traffic_class = traffic_class;
}

static inline void
rte_mbuf_sched_color_set(struct rte_mbuf *m, uint8_t color)
{
        m->hash.sched.color = color;
}

static inline void
rte_mbuf_sched_set(struct rte_mbuf *m, uint32_t queue_id,
                        uint8_t traffic_class,
                        uint8_t color)
{
        m->hash.sched = (struct rte_mbuf_sched){
                                .queue_id = queue_id,
                                .traffic_class = traffic_class,
                                .color = color,
                                .reserved = 0,
                        };
}

#ifdef __cplusplus
}
#endif

#endif /* _RTE_MBUF_H_ */