rte_mbuf.h

Go to the documentation of this file.

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2010-2014 Intel Corporation.
 * Copyright 2014 6WIND S.A.
 */

#ifndef _RTE_MBUF_H_
#define _RTE_MBUF_H_

#include <stdint.h>

#include <rte_common.h>
#include <rte_config.h>
#include <rte_mempool.h>
#include <rte_prefetch.h>
#include <rte_branch_prediction.h>
#include <rte_mbuf_ptype.h>
#include <rte_mbuf_core.h>

#ifdef __cplusplus
extern "C" {
#endif

const char *rte_get_rx_ol_flag_name(uint64_t mask);

int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);

const char *rte_get_tx_ol_flag_name(uint64_t mask);

int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);

static inline void
rte_mbuf_prefetch_part1(struct rte_mbuf *m)
{
    rte_prefetch0(m);
}

static inline void
rte_mbuf_prefetch_part2(struct rte_mbuf *m)
{
#if RTE_CACHE_LINE_SIZE == 64
    rte_prefetch0(RTE_PTR_ADD(m, RTE_CACHE_LINE_MIN_SIZE));
#else
    RTE_SET_USED(m);
#endif
}


static inline uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp);

static inline rte_iova_t
rte_mbuf_iova_get(const struct rte_mbuf *m)
{
#if RTE_IOVA_IN_MBUF
    return m->buf_iova;
#else
    return (rte_iova_t)m->buf_addr;
#endif
}

static inline void
rte_mbuf_iova_set(struct rte_mbuf *m, rte_iova_t iova)
{
#if RTE_IOVA_IN_MBUF
    m->buf_iova = iova;
#else
    RTE_SET_USED(m);
    RTE_SET_USED(iova);
#endif
}

static inline rte_iova_t
rte_mbuf_data_iova(const struct rte_mbuf *mb)
{
    return rte_mbuf_iova_get(mb) + mb->data_off;
}

static inline rte_iova_t
rte_mbuf_data_iova_default(const struct rte_mbuf *mb)
{
    return rte_mbuf_iova_get(mb) + RTE_PKTMBUF_HEADROOM;
}

static inline struct rte_mbuf *
rte_mbuf_from_indirect(struct rte_mbuf *mi)
{
    return (struct rte_mbuf *)RTE_PTR_SUB(mi->buf_addr, sizeof(*mi) + mi->priv_size);
}

static inline char *
rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
{
    return (char *)mb + sizeof(*mb) + rte_pktmbuf_priv_size(mp);
}

static inline char *
rte_mbuf_data_addr_default(struct rte_mbuf *mb)
{
    return rte_mbuf_buf_addr(mb, mb->pool) + RTE_PKTMBUF_HEADROOM;
}

static inline char *
rte_mbuf_to_baddr(struct rte_mbuf *md)
{
    return rte_mbuf_buf_addr(md, md->pool);
}

static inline void *
rte_mbuf_to_priv(struct rte_mbuf *m)
{
    return RTE_PTR_ADD(m, sizeof(struct rte_mbuf));
}

struct rte_pktmbuf_pool_private {
    uint16_t mbuf_data_room_size; 
    uint16_t mbuf_priv_size;      
    uint32_t flags; 
};

static inline uint32_t
rte_pktmbuf_priv_flags(struct rte_mempool *mp)
{
    struct rte_pktmbuf_pool_private *mbp_priv;

    mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
    return mbp_priv->flags;
}

#define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF (1 << 0)

#define RTE_MBUF_HAS_PINNED_EXTBUF(mb) \
    (rte_pktmbuf_priv_flags(mb->pool) & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF)

#ifdef RTE_LIBRTE_MBUF_DEBUG

#define __rte_mbuf_sanity_check(m, is_h) rte_mbuf_sanity_check(m, is_h)

#else /*  RTE_LIBRTE_MBUF_DEBUG */

#define __rte_mbuf_sanity_check(m, is_h) do { } while (0)

#endif /*  RTE_LIBRTE_MBUF_DEBUG */

#ifdef RTE_MBUF_REFCNT_ATOMIC

static inline uint16_t
rte_mbuf_refcnt_read(const struct rte_mbuf *m)
{
    return rte_atomic_load_explicit(&m->refcnt, rte_memory_order_relaxed);
}

static inline void
rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
{
    rte_atomic_store_explicit(&m->refcnt, new_value, rte_memory_order_relaxed);
}

/* internal */
static inline uint16_t
__rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
{
    return rte_atomic_fetch_add_explicit(&m->refcnt, value,
                 rte_memory_order_acq_rel) + value;
}

static inline uint16_t
rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
{
    /*
     * The atomic_add is an expensive operation, so we don't want to
     * call it in the case where we know we are the unique holder of
     * this mbuf (i.e. ref_cnt == 1). Otherwise, an atomic
     * operation has to be used because concurrent accesses on the
     * reference counter can occur.
     */
    if (likely(rte_mbuf_refcnt_read(m) == 1)) {
        ++value;
        rte_mbuf_refcnt_set(m, (uint16_t)value);
        return (uint16_t)value;
    }

    return __rte_mbuf_refcnt_update(m, value);
}

#else /* ! RTE_MBUF_REFCNT_ATOMIC */

/* internal */
static inline uint16_t
__rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
{
    m->refcnt = (uint16_t)(m->refcnt + value);
    return m->refcnt;
}

static inline uint16_t
rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
{
    return __rte_mbuf_refcnt_update(m, value);
}

static inline uint16_t
rte_mbuf_refcnt_read(const struct rte_mbuf *m)
{
    return m->refcnt;
}

static inline void
rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
{
    m->refcnt = new_value;
}

#endif /* RTE_MBUF_REFCNT_ATOMIC */

static inline uint16_t
rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
{
    return rte_atomic_load_explicit(&shinfo->refcnt, rte_memory_order_relaxed);
}

static inline void
rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo,
    uint16_t new_value)
{
    rte_atomic_store_explicit(&shinfo->refcnt, new_value, rte_memory_order_relaxed);
}

static inline uint16_t
rte_mbuf_ext_refcnt_update(struct rte_mbuf_ext_shared_info *shinfo,
    int16_t value)
{
    if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1)) {
        ++value;
        rte_mbuf_ext_refcnt_set(shinfo, (uint16_t)value);
        return (uint16_t)value;
    }

    return rte_atomic_fetch_add_explicit(&shinfo->refcnt, value,
                 rte_memory_order_acq_rel) + value;
}

#define RTE_MBUF_PREFETCH_TO_FREE(m) do {       \
    if ((m) != NULL)                        \
        rte_prefetch0(m);               \
} while (0)


void
rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header);

int rte_mbuf_check(const struct rte_mbuf *m, int is_header,
           const char **reason);

static __rte_always_inline void
__rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m)
{
    RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);
    RTE_ASSERT(m->next == NULL);
    RTE_ASSERT(m->nb_segs == 1);
    __rte_mbuf_sanity_check(m, 0);
}

#define MBUF_RAW_ALLOC_CHECK(m) __rte_mbuf_raw_sanity_check(m)

static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
{
    struct rte_mbuf *m;

    if (rte_mempool_get(mp, (void **)&m) < 0)
        return NULL;
    __rte_mbuf_raw_sanity_check(m);
    return m;
}

static __rte_always_inline void
rte_mbuf_raw_free(struct rte_mbuf *m)
{
    RTE_ASSERT(!RTE_MBUF_CLONED(m) &&
          (!RTE_MBUF_HAS_EXTBUF(m) || RTE_MBUF_HAS_PINNED_EXTBUF(m)));
    __rte_mbuf_raw_sanity_check(m);
    rte_mempool_put(m->pool, m);
}

void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg,
              void *m, unsigned i);

void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg);

struct rte_mempool *
rte_pktmbuf_pool_create(const char *name, unsigned n,
    unsigned cache_size, uint16_t priv_size, uint16_t data_room_size,
    int socket_id);

struct rte_mempool *
rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n,
    unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size,
    int socket_id, const char *ops_name);

struct rte_pktmbuf_extmem {
    void *buf_ptr;      
    rte_iova_t buf_iova;    
    size_t buf_len;     
    uint16_t elt_size;  
};

struct rte_mempool *
rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n,
    unsigned int cache_size, uint16_t priv_size,
    uint16_t data_room_size, int socket_id,
    const struct rte_pktmbuf_extmem *ext_mem,
    unsigned int ext_num);

static inline uint16_t
rte_pktmbuf_data_room_size(struct rte_mempool *mp)
{
    struct rte_pktmbuf_pool_private *mbp_priv;

    mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
    return mbp_priv->mbuf_data_room_size;
}

static inline uint16_t
rte_pktmbuf_priv_size(struct rte_mempool *mp)
{
    struct rte_pktmbuf_pool_private *mbp_priv;

    mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
    return mbp_priv->mbuf_priv_size;
}

static inline void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
{
    m->data_off = (uint16_t)RTE_MIN((uint16_t)RTE_PKTMBUF_HEADROOM,
                    (uint16_t)m->buf_len);
}

static inline void rte_pktmbuf_reset(struct rte_mbuf *m)
{
    m->next = NULL;
    m->pkt_len = 0;
    m->tx_offload = 0;
    m->vlan_tci = 0;
    m->vlan_tci_outer = 0;
    m->nb_segs = 1;
    m->port = RTE_MBUF_PORT_INVALID;

    m->ol_flags &= RTE_MBUF_F_EXTERNAL;
    m->packet_type = 0;
    rte_pktmbuf_reset_headroom(m);

    m->data_len = 0;
    __rte_mbuf_sanity_check(m, 1);
}

static inline struct rte_mbuf *rte_pktmbuf_alloc(struct rte_mempool *mp)
{
    struct rte_mbuf *m;
    if ((m = rte_mbuf_raw_alloc(mp)) != NULL)
        rte_pktmbuf_reset(m);
    return m;
}

static inline int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool,
     struct rte_mbuf **mbufs, unsigned count)
{
    unsigned idx = 0;
    int rc;

    rc = rte_mempool_get_bulk(pool, (void **)mbufs, count);
    if (unlikely(rc))
        return rc;

    /* To understand duff's device on loop unwinding optimization, see
     * https://en.wikipedia.org/wiki/Duff's_device.
     * Here while() loop is used rather than do() while{} to avoid extra
     * check if count is zero.
     */
    switch (count % 4) {
    case 0:
        while (idx != count) {
            __rte_mbuf_raw_sanity_check(mbufs[idx]);
            rte_pktmbuf_reset(mbufs[idx]);
            idx++;
            /* fall-through */
    case 3:
            __rte_mbuf_raw_sanity_check(mbufs[idx]);
            rte_pktmbuf_reset(mbufs[idx]);
            idx++;
            /* fall-through */
    case 2:
            __rte_mbuf_raw_sanity_check(mbufs[idx]);
            rte_pktmbuf_reset(mbufs[idx]);
            idx++;
            /* fall-through */
    case 1:
            __rte_mbuf_raw_sanity_check(mbufs[idx]);
            rte_pktmbuf_reset(mbufs[idx]);
            idx++;
            /* fall-through */
        }
    }
    return 0;
}

static inline struct rte_mbuf_ext_shared_info *
rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len,
    rte_mbuf_extbuf_free_callback_t free_cb, void *fcb_opaque)
{
    struct rte_mbuf_ext_shared_info *shinfo;
    void *buf_end = RTE_PTR_ADD(buf_addr, *buf_len);
    void *addr;

    addr = RTE_PTR_ALIGN_FLOOR(RTE_PTR_SUB(buf_end, sizeof(*shinfo)),
                   sizeof(uintptr_t));
    if (addr <= buf_addr)
        return NULL;

    shinfo = (struct rte_mbuf_ext_shared_info *)addr;
    shinfo->free_cb = free_cb;
    shinfo->fcb_opaque = fcb_opaque;
    rte_mbuf_ext_refcnt_set(shinfo, 1);

    *buf_len = (uint16_t)RTE_PTR_DIFF(shinfo, buf_addr);
    return shinfo;
}

static inline void
rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr,
    rte_iova_t buf_iova, uint16_t buf_len,
    struct rte_mbuf_ext_shared_info *shinfo)
{
    /* mbuf should not be read-only */
    RTE_ASSERT(RTE_MBUF_DIRECT(m) && rte_mbuf_refcnt_read(m) == 1);
    RTE_ASSERT(shinfo->free_cb != NULL);

    m->buf_addr = buf_addr;
    rte_mbuf_iova_set(m, buf_iova);
    m->buf_len = buf_len;

    m->data_len = 0;
    m->data_off = 0;

    m->ol_flags |= RTE_MBUF_F_EXTERNAL;
    m->shinfo = shinfo;
}

#define rte_pktmbuf_detach_extbuf(m) rte_pktmbuf_detach(m)

static inline void
rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
{
#if !RTE_IOVA_IN_MBUF
    mdst->dynfield2 = msrc->dynfield2;
#endif
    memcpy(&mdst->dynfield1, msrc->dynfield1, sizeof(mdst->dynfield1));
}

/* internal */
static inline void
__rte_pktmbuf_copy_hdr(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
{
    mdst->port = msrc->port;
    mdst->vlan_tci = msrc->vlan_tci;
    mdst->vlan_tci_outer = msrc->vlan_tci_outer;
    mdst->tx_offload = msrc->tx_offload;
    mdst->hash = msrc->hash;
    mdst->packet_type = msrc->packet_type;
    rte_mbuf_dynfield_copy(mdst, msrc);
}

static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
{
    RTE_ASSERT(RTE_MBUF_DIRECT(mi) &&
        rte_mbuf_refcnt_read(mi) == 1);

    if (RTE_MBUF_HAS_EXTBUF(m)) {
        rte_mbuf_ext_refcnt_update(m->shinfo, 1);
        mi->ol_flags = m->ol_flags;
        mi->shinfo = m->shinfo;
    } else {
        /* if m is not direct, get the mbuf that embeds the data */
        rte_mbuf_refcnt_update(rte_mbuf_from_indirect(m), 1);
        mi->priv_size = m->priv_size;
        mi->ol_flags = m->ol_flags | RTE_MBUF_F_INDIRECT;
    }

    __rte_pktmbuf_copy_hdr(mi, m);

    mi->data_off = m->data_off;
    mi->data_len = m->data_len;
    rte_mbuf_iova_set(mi, rte_mbuf_iova_get(m));
    mi->buf_addr = m->buf_addr;
    mi->buf_len = m->buf_len;

    mi->next = NULL;
    mi->pkt_len = mi->data_len;
    mi->nb_segs = 1;

    __rte_mbuf_sanity_check(mi, 1);
    __rte_mbuf_sanity_check(m, 0);
}

static inline void
__rte_pktmbuf_free_extbuf(struct rte_mbuf *m)
{
    RTE_ASSERT(RTE_MBUF_HAS_EXTBUF(m));
    RTE_ASSERT(m->shinfo != NULL);

    if (rte_mbuf_ext_refcnt_update(m->shinfo, -1) == 0)
        m->shinfo->free_cb(m->buf_addr, m->shinfo->fcb_opaque);
}

static inline void
__rte_pktmbuf_free_direct(struct rte_mbuf *m)
{
    struct rte_mbuf *md;

    RTE_ASSERT(RTE_MBUF_CLONED(m));

    md = rte_mbuf_from_indirect(m);

    if (rte_mbuf_refcnt_update(md, -1) == 0) {
        md->next = NULL;
        md->nb_segs = 1;
        rte_mbuf_refcnt_set(md, 1);
        rte_mbuf_raw_free(md);
    }
}

static inline void rte_pktmbuf_detach(struct rte_mbuf *m)
{
    struct rte_mempool *mp = m->pool;
    uint32_t mbuf_size, buf_len;
    uint16_t priv_size;

    if (RTE_MBUF_HAS_EXTBUF(m)) {
        /*
         * The mbuf has the external attached buffer,
         * we should check the type of the memory pool where
         * the mbuf was allocated from to detect the pinned
         * external buffer.
         */
        uint32_t flags = rte_pktmbuf_priv_flags(mp);

        if (flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) {
            /*
             * The pinned external buffer should not be
             * detached from its backing mbuf, just exit.
             */
            return;
        }
        __rte_pktmbuf_free_extbuf(m);
    } else {
        __rte_pktmbuf_free_direct(m);
    }
    priv_size = rte_pktmbuf_priv_size(mp);
    mbuf_size = (uint32_t)(sizeof(struct rte_mbuf) + priv_size);
    buf_len = rte_pktmbuf_data_room_size(mp);

    m->priv_size = priv_size;
    m->buf_addr = (char *)m + mbuf_size;
    rte_mbuf_iova_set(m, rte_mempool_virt2iova(m) + mbuf_size);
    m->buf_len = (uint16_t)buf_len;
    rte_pktmbuf_reset_headroom(m);
    m->data_len = 0;
    m->ol_flags = 0;
}

static inline int __rte_pktmbuf_pinned_extbuf_decref(struct rte_mbuf *m)
{
    struct rte_mbuf_ext_shared_info *shinfo;

    /* Clear flags, mbuf is being freed. */
    m->ol_flags = RTE_MBUF_F_EXTERNAL;
    shinfo = m->shinfo;

    /* Optimize for performance - do not dec/reinit */
    if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1))
        return 0;

    /*
     * Direct usage of add primitive to avoid
     * duplication of comparing with one.
     */
    if (likely(rte_atomic_fetch_add_explicit(&shinfo->refcnt, -1,
                     rte_memory_order_acq_rel) - 1))
        return 1;

    /* Reinitialize counter before mbuf freeing. */
    rte_mbuf_ext_refcnt_set(shinfo, 1);
    return 0;
}

static __rte_always_inline struct rte_mbuf *
rte_pktmbuf_prefree_seg(struct rte_mbuf *m)
{
    __rte_mbuf_sanity_check(m, 0);

    if (likely(rte_mbuf_refcnt_read(m) == 1)) {

        if (!RTE_MBUF_DIRECT(m)) {
            rte_pktmbuf_detach(m);
            if (RTE_MBUF_HAS_EXTBUF(m) &&
                RTE_MBUF_HAS_PINNED_EXTBUF(m) &&
                __rte_pktmbuf_pinned_extbuf_decref(m))
                return NULL;
        }

        if (m->next != NULL)
            m->next = NULL;
        if (m->nb_segs != 1)
            m->nb_segs = 1;

        return m;

    } else if (__rte_mbuf_refcnt_update(m, -1) == 0) {

        if (!RTE_MBUF_DIRECT(m)) {
            rte_pktmbuf_detach(m);
            if (RTE_MBUF_HAS_EXTBUF(m) &&
                RTE_MBUF_HAS_PINNED_EXTBUF(m) &&
                __rte_pktmbuf_pinned_extbuf_decref(m))
                return NULL;
        }

        if (m->next != NULL)
            m->next = NULL;
        if (m->nb_segs != 1)
            m->nb_segs = 1;
        rte_mbuf_refcnt_set(m, 1);

        return m;
    }
    return NULL;
}

static __rte_always_inline void
rte_pktmbuf_free_seg(struct rte_mbuf *m)
{
    m = rte_pktmbuf_prefree_seg(m);
    if (likely(m != NULL))
        rte_mbuf_raw_free(m);
}

static inline void rte_pktmbuf_free(struct rte_mbuf *m)
{
    struct rte_mbuf *m_next;

    if (m != NULL)
        __rte_mbuf_sanity_check(m, 1);

    while (m != NULL) {
        m_next = m->next;
        rte_pktmbuf_free_seg(m);
        m = m_next;
    }
}

void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count);

struct rte_mbuf *
rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp);

struct rte_mbuf *
rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp,
         uint32_t offset, uint32_t length);

static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
{
    __rte_mbuf_sanity_check(m, 1);

    do {
        rte_mbuf_refcnt_update(m, v);
    } while ((m = m->next) != NULL);
}

static inline uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
{
    __rte_mbuf_sanity_check(m, 0);
    return m->data_off;
}

static inline uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
{
    __rte_mbuf_sanity_check(m, 0);
    return (uint16_t)(m->buf_len - rte_pktmbuf_headroom(m) -
              m->data_len);
}

static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m)
{
    __rte_mbuf_sanity_check(m, 1);
    while (m->next != NULL)
        m = m->next;
    return m;
}

#define rte_pktmbuf_pkt_len(m) ((m)->pkt_len)

#define rte_pktmbuf_data_len(m) ((m)->data_len)

static inline char *rte_pktmbuf_prepend(struct rte_mbuf *m,
                    uint16_t len)
{
    __rte_mbuf_sanity_check(m, 1);

    if (unlikely(len > rte_pktmbuf_headroom(m)))
        return NULL;

    /* NB: elaborating the subtraction like this instead of using
     *     -= allows us to ensure the result type is uint16_t
     *     avoiding compiler warnings on gcc 8.1 at least */
    m->data_off = (uint16_t)(m->data_off - len);
    m->data_len = (uint16_t)(m->data_len + len);
    m->pkt_len  = (m->pkt_len + len);

    return (char *)m->buf_addr + m->data_off;
}

static inline char *rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
{
    void *tail;
    struct rte_mbuf *m_last;

    __rte_mbuf_sanity_check(m, 1);

    m_last = rte_pktmbuf_lastseg(m);
    if (unlikely(len > rte_pktmbuf_tailroom(m_last)))
        return NULL;

    tail = (char *)m_last->buf_addr + m_last->data_off + m_last->data_len;
    m_last->data_len = (uint16_t)(m_last->data_len + len);
    m->pkt_len  = (m->pkt_len + len);
    return (char*) tail;
}

static inline char *rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
{
    __rte_mbuf_sanity_check(m, 1);

    if (unlikely(len > m->data_len))
        return NULL;

    /* NB: elaborating the addition like this instead of using
     *     += allows us to ensure the result type is uint16_t
     *     avoiding compiler warnings on gcc 8.1 at least */
    m->data_len = (uint16_t)(m->data_len - len);
    m->data_off = (uint16_t)(m->data_off + len);
    m->pkt_len  = (m->pkt_len - len);
    return (char *)m->buf_addr + m->data_off;
}

static inline int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
{
    struct rte_mbuf *m_last;

    __rte_mbuf_sanity_check(m, 1);

    m_last = rte_pktmbuf_lastseg(m);
    if (unlikely(len > m_last->data_len))
        return -1;

    m_last->data_len = (uint16_t)(m_last->data_len - len);
    m->pkt_len  = (m->pkt_len - len);
    return 0;
}

static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
{
    __rte_mbuf_sanity_check(m, 1);
    return m->nb_segs == 1;
}

const void *__rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off,
    uint32_t len, void *buf);

static inline const void *rte_pktmbuf_read(const struct rte_mbuf *m,
    uint32_t off, uint32_t len, void *buf)
{
    if (likely(off + len <= rte_pktmbuf_data_len(m)))
        return rte_pktmbuf_mtod_offset(m, char *, off);
    else
        return __rte_pktmbuf_read(m, off, len, buf);
}

static inline int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
{
    struct rte_mbuf *cur_tail;

    /* Check for number-of-segments-overflow */
    if (head->nb_segs + tail->nb_segs > RTE_MBUF_MAX_NB_SEGS)
        return -EOVERFLOW;

    /* Chain 'tail' onto the old tail */
    cur_tail = rte_pktmbuf_lastseg(head);
    cur_tail->next = tail;

    /* accumulate number of segments and total length.
     * NB: elaborating the addition like this instead of using
     *     -= allows us to ensure the result type is uint16_t
     *     avoiding compiler warnings on gcc 8.1 at least */
    head->nb_segs = (uint16_t)(head->nb_segs + tail->nb_segs);
    head->pkt_len += tail->pkt_len;

    /* pkt_len is only set in the head */
    tail->pkt_len = tail->data_len;

    return 0;
}

static __rte_always_inline uint64_t
rte_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso,
    uint64_t ol3, uint64_t ol2, uint64_t unused)
{
    return il2 << RTE_MBUF_L2_LEN_OFS |
        il3 << RTE_MBUF_L3_LEN_OFS |
        il4 << RTE_MBUF_L4_LEN_OFS |
        tso << RTE_MBUF_TSO_SEGSZ_OFS |
        ol3 << RTE_MBUF_OUTL3_LEN_OFS |
        ol2 << RTE_MBUF_OUTL2_LEN_OFS |
        unused << RTE_MBUF_TXOFLD_UNUSED_OFS;
}

static inline int
rte_validate_tx_offload(const struct rte_mbuf *m)
{
    uint64_t ol_flags = m->ol_flags;

    /* Does packet set any of available offloads? */
    if (!(ol_flags & RTE_MBUF_F_TX_OFFLOAD_MASK))
        return 0;

    /* IP checksum can be counted only for IPv4 packet */
    if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) && (ol_flags & RTE_MBUF_F_TX_IPV6))
        return -EINVAL;

    /* IP type not set when required */
    if (ol_flags & (RTE_MBUF_F_TX_L4_MASK | RTE_MBUF_F_TX_TCP_SEG))
        if (!(ol_flags & (RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IPV6)))
            return -EINVAL;

    /* Check requirements for TSO packet */
    if (ol_flags & RTE_MBUF_F_TX_TCP_SEG)
        if ((m->tso_segsz == 0) ||
                ((ol_flags & RTE_MBUF_F_TX_IPV4) &&
                 !(ol_flags & RTE_MBUF_F_TX_IP_CKSUM)))
            return -EINVAL;

    /* RTE_MBUF_F_TX_OUTER_IP_CKSUM set for non outer IPv4 packet. */
    if ((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) &&
            !(ol_flags & RTE_MBUF_F_TX_OUTER_IPV4))
        return -EINVAL;

    return 0;
}

int __rte_pktmbuf_linearize(struct rte_mbuf *mbuf);

static inline int
rte_pktmbuf_linearize(struct rte_mbuf *mbuf)
{
    if (rte_pktmbuf_is_contiguous(mbuf))
        return 0;
    return __rte_pktmbuf_linearize(mbuf);
}

void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len);

static inline uint32_t
rte_mbuf_sched_queue_get(const struct rte_mbuf *m)
{
    return m->hash.sched.queue_id;
}

static inline uint8_t
rte_mbuf_sched_traffic_class_get(const struct rte_mbuf *m)
{
    return m->hash.sched.traffic_class;
}

static inline uint8_t
rte_mbuf_sched_color_get(const struct rte_mbuf *m)
{
    return m->hash.sched.color;
}

static inline void
rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id,
            uint8_t *traffic_class,
            uint8_t *color)
{
    struct rte_mbuf_sched sched = m->hash.sched;

    *queue_id = sched.queue_id;
    *traffic_class = sched.traffic_class;
    *color = sched.color;
}

static inline void
rte_mbuf_sched_queue_set(struct rte_mbuf *m, uint32_t queue_id)
{
    m->hash.sched.queue_id = queue_id;
}

static inline void
rte_mbuf_sched_traffic_class_set(struct rte_mbuf *m, uint8_t traffic_class)
{
    m->hash.sched.traffic_class = traffic_class;
}

static inline void
rte_mbuf_sched_color_set(struct rte_mbuf *m, uint8_t color)
{
    m->hash.sched.color = color;
}

static inline void
rte_mbuf_sched_set(struct rte_mbuf *m, uint32_t queue_id,
            uint8_t traffic_class,
            uint8_t color)
{
    m->hash.sched = (struct rte_mbuf_sched){
                .queue_id = queue_id,
                .traffic_class = traffic_class,
                .color = color,
                .reserved = 0,
            };
}

#ifdef __cplusplus
}
#endif

#endif /* _RTE_MBUF_H_ */