rte_mbuf_core.h

Go to the documentation of this file.

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2010-2014 Intel Corporation.
 * Copyright 2014 6WIND S.A.
 */

#ifndef _RTE_MBUF_CORE_H_
#define _RTE_MBUF_CORE_H_

#include <stdint.h>

#include <rte_compat.h>
#include <rte_byteorder.h>
#include <generic/rte_atomic.h>

#ifdef __cplusplus
extern "C" {
#endif

/*
 * Packet Offload Features Flags. It also carry packet type information.
 * Critical resources. Both rx/tx shared these bits. Be cautious on any change
 *
 * - RX flags start at bit position zero, and get added to the left of previous
 *   flags.
 * - The most-significant 3 bits are reserved for generic mbuf flags
 * - TX flags therefore start at bit position 60 (i.e. 63-3), and new flags get
 *   added to the right of the previously defined flags i.e. they should count
 *   downwards, not upwards.
 *
 * Keep these flags synchronized with rte_get_rx_ol_flag_name() and
 * rte_get_tx_ol_flag_name().
 */

#define PKT_RX_VLAN          (1ULL << 0)

#define PKT_RX_RSS_HASH      (1ULL << 1)

#define PKT_RX_FDIR          (1ULL << 2)

#define PKT_RX_L4_CKSUM_BAD  (1ULL << 3)

#define PKT_RX_IP_CKSUM_BAD  (1ULL << 4)

#define PKT_RX_EIP_CKSUM_BAD (1ULL << 5)

#define PKT_RX_VLAN_STRIPPED (1ULL << 6)

#define PKT_RX_IP_CKSUM_MASK ((1ULL << 4) | (1ULL << 7))

#define PKT_RX_IP_CKSUM_UNKNOWN 0
#define PKT_RX_IP_CKSUM_BAD     (1ULL << 4)
#define PKT_RX_IP_CKSUM_GOOD    (1ULL << 7)
#define PKT_RX_IP_CKSUM_NONE    ((1ULL << 4) | (1ULL << 7))

#define PKT_RX_L4_CKSUM_MASK ((1ULL << 3) | (1ULL << 8))

#define PKT_RX_L4_CKSUM_UNKNOWN 0
#define PKT_RX_L4_CKSUM_BAD     (1ULL << 3)
#define PKT_RX_L4_CKSUM_GOOD    (1ULL << 8)
#define PKT_RX_L4_CKSUM_NONE    ((1ULL << 3) | (1ULL << 8))

#define PKT_RX_IEEE1588_PTP  (1ULL << 9)

#define PKT_RX_IEEE1588_TMST (1ULL << 10)

#define PKT_RX_FDIR_ID       (1ULL << 13)

#define PKT_RX_FDIR_FLX      (1ULL << 14)

#define PKT_RX_QINQ_STRIPPED (1ULL << 15)

#define PKT_RX_LRO           (1ULL << 16)

#define PKT_RX_TIMESTAMP     (1ULL << 17)

#define PKT_RX_SEC_OFFLOAD  (1ULL << 18)

#define PKT_RX_SEC_OFFLOAD_FAILED   (1ULL << 19)

#define PKT_RX_QINQ          (1ULL << 20)

#define PKT_RX_OUTER_L4_CKSUM_MASK  ((1ULL << 21) | (1ULL << 22))

#define PKT_RX_OUTER_L4_CKSUM_UNKNOWN   0
#define PKT_RX_OUTER_L4_CKSUM_BAD   (1ULL << 21)
#define PKT_RX_OUTER_L4_CKSUM_GOOD  (1ULL << 22)
#define PKT_RX_OUTER_L4_CKSUM_INVALID   ((1ULL << 21) | (1ULL << 22))

/* add new RX flags here, don't forget to update PKT_FIRST_FREE */

#define PKT_FIRST_FREE (1ULL << 23)
#define PKT_LAST_FREE (1ULL << 40)

/* add new TX flags here, don't forget to update PKT_LAST_FREE  */

#define PKT_TX_OUTER_UDP_CKSUM     (1ULL << 41)

#define PKT_TX_UDP_SEG  (1ULL << 42)

#define PKT_TX_SEC_OFFLOAD  (1ULL << 43)

#define PKT_TX_MACSEC        (1ULL << 44)

#define PKT_TX_TUNNEL_VXLAN   (0x1ULL << 45)
#define PKT_TX_TUNNEL_GRE     (0x2ULL << 45)
#define PKT_TX_TUNNEL_IPIP    (0x3ULL << 45)
#define PKT_TX_TUNNEL_GENEVE  (0x4ULL << 45)

#define PKT_TX_TUNNEL_MPLSINUDP (0x5ULL << 45)
#define PKT_TX_TUNNEL_VXLAN_GPE (0x6ULL << 45)
#define PKT_TX_TUNNEL_GTP       (0x7ULL << 45)

#define PKT_TX_TUNNEL_IP (0xDULL << 45)

#define PKT_TX_TUNNEL_UDP (0xEULL << 45)
/* add new TX TUNNEL type here */
#define PKT_TX_TUNNEL_MASK    (0xFULL << 45)

#define PKT_TX_QINQ        (1ULL << 49)
/* this old name is deprecated */
#define PKT_TX_QINQ_PKT    PKT_TX_QINQ

#define PKT_TX_TCP_SEG       (1ULL << 50)

#define PKT_TX_IEEE1588_TMST (1ULL << 51)

#define PKT_TX_L4_NO_CKSUM   (0ULL << 52) 
#define PKT_TX_TCP_CKSUM     (1ULL << 52)

#define PKT_TX_SCTP_CKSUM    (2ULL << 52)

#define PKT_TX_UDP_CKSUM     (3ULL << 52)

#define PKT_TX_L4_MASK       (3ULL << 52)

#define PKT_TX_IP_CKSUM      (1ULL << 54)

#define PKT_TX_IPV4          (1ULL << 55)

#define PKT_TX_IPV6          (1ULL << 56)

#define PKT_TX_VLAN          (1ULL << 57)
/* this old name is deprecated */
#define PKT_TX_VLAN_PKT      PKT_TX_VLAN

#define PKT_TX_OUTER_IP_CKSUM   (1ULL << 58)

#define PKT_TX_OUTER_IPV4   (1ULL << 59)

#define PKT_TX_OUTER_IPV6    (1ULL << 60)

#define PKT_TX_OFFLOAD_MASK (    \
        PKT_TX_OUTER_IPV6 |  \
        PKT_TX_OUTER_IPV4 |  \
        PKT_TX_OUTER_IP_CKSUM |  \
        PKT_TX_VLAN_PKT |        \
        PKT_TX_IPV6 |        \
        PKT_TX_IPV4 |        \
        PKT_TX_IP_CKSUM |        \
        PKT_TX_L4_MASK |         \
        PKT_TX_IEEE1588_TMST |   \
        PKT_TX_TCP_SEG |         \
        PKT_TX_QINQ_PKT |        \
        PKT_TX_TUNNEL_MASK |     \
        PKT_TX_MACSEC |      \
        PKT_TX_SEC_OFFLOAD |     \
        PKT_TX_UDP_SEG |     \
        PKT_TX_OUTER_UDP_CKSUM)

#define EXT_ATTACHED_MBUF    (1ULL << 61)

#define IND_ATTACHED_MBUF    (1ULL << 62) 
#define RTE_MBUF_PRIV_ALIGN 8

#define RTE_MBUF_DEFAULT_DATAROOM   2048
#define RTE_MBUF_DEFAULT_BUF_SIZE   \
    (RTE_MBUF_DEFAULT_DATAROOM + RTE_PKTMBUF_HEADROOM)

/*
 * define a set of marker types that can be used to refer to set points in the
 * mbuf.
 */
__extension__
typedef void    *MARKER[0];   
__extension__
typedef uint8_t  MARKER8[0];  
__extension__
typedef uint64_t MARKER64[0];

struct rte_mbuf_sched {
    uint32_t queue_id;   
    uint8_t traffic_class;
    uint8_t color;
    uint16_t reserved;   
}; 
enum {
    RTE_MBUF_L2_LEN_BITS = 7,
    RTE_MBUF_L3_LEN_BITS = 9,
    RTE_MBUF_L4_LEN_BITS = 8,
    RTE_MBUF_TSO_SEGSZ_BITS = 16,
    RTE_MBUF_OUTL3_LEN_BITS = 9,
    RTE_MBUF_OUTL2_LEN_BITS = 7,
    RTE_MBUF_TXOFLD_UNUSED_BITS = sizeof(uint64_t) * CHAR_BIT -
        RTE_MBUF_L2_LEN_BITS -
        RTE_MBUF_L3_LEN_BITS -
        RTE_MBUF_L4_LEN_BITS -
        RTE_MBUF_TSO_SEGSZ_BITS -
        RTE_MBUF_OUTL3_LEN_BITS -
        RTE_MBUF_OUTL2_LEN_BITS,
#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
    RTE_MBUF_L2_LEN_OFS =
        sizeof(uint64_t) * CHAR_BIT - RTE_MBUF_L2_LEN_BITS,
    RTE_MBUF_L3_LEN_OFS = RTE_MBUF_L2_LEN_OFS - RTE_MBUF_L3_LEN_BITS,
    RTE_MBUF_L4_LEN_OFS = RTE_MBUF_L3_LEN_OFS - RTE_MBUF_L4_LEN_BITS,
    RTE_MBUF_TSO_SEGSZ_OFS = RTE_MBUF_L4_LEN_OFS - RTE_MBUF_TSO_SEGSZ_BITS,
    RTE_MBUF_OUTL3_LEN_OFS =
        RTE_MBUF_TSO_SEGSZ_OFS - RTE_MBUF_OUTL3_LEN_BITS,
    RTE_MBUF_OUTL2_LEN_OFS =
        RTE_MBUF_OUTL3_LEN_OFS - RTE_MBUF_OUTL2_LEN_BITS,
    RTE_MBUF_TXOFLD_UNUSED_OFS =
        RTE_MBUF_OUTL2_LEN_OFS - RTE_MBUF_TXOFLD_UNUSED_BITS,
#else
    RTE_MBUF_L2_LEN_OFS = 0,
    RTE_MBUF_L3_LEN_OFS = RTE_MBUF_L2_LEN_OFS + RTE_MBUF_L2_LEN_BITS,
    RTE_MBUF_L4_LEN_OFS = RTE_MBUF_L3_LEN_OFS + RTE_MBUF_L3_LEN_BITS,
    RTE_MBUF_TSO_SEGSZ_OFS = RTE_MBUF_L4_LEN_OFS + RTE_MBUF_L4_LEN_BITS,
    RTE_MBUF_OUTL3_LEN_OFS =
        RTE_MBUF_TSO_SEGSZ_OFS + RTE_MBUF_TSO_SEGSZ_BITS,
    RTE_MBUF_OUTL2_LEN_OFS =
        RTE_MBUF_OUTL3_LEN_OFS + RTE_MBUF_OUTL3_LEN_BITS,
    RTE_MBUF_TXOFLD_UNUSED_OFS =
        RTE_MBUF_OUTL2_LEN_OFS + RTE_MBUF_OUTL2_LEN_BITS,
#endif
};

struct rte_mbuf {
    MARKER cacheline0;

    void *buf_addr;           
    RTE_STD_C11
    union {
        rte_iova_t buf_iova;
        rte_iova_t buf_physaddr; 
    } __rte_aligned(sizeof(rte_iova_t));

    /* next 8 bytes are initialised on RX descriptor rearm */
    MARKER64 rearm_data;
    uint16_t data_off;

    RTE_STD_C11
    union {
        rte_atomic16_t refcnt_atomic; 
        uint16_t refcnt;
    };
    uint16_t nb_segs;         
    uint16_t port;

    uint64_t ol_flags;        
    /* remaining bytes are set on RX when pulling packet from descriptor */
    MARKER rx_descriptor_fields1;

    /*
     * The packet type, which is the combination of outer/inner L2, L3, L4
     * and tunnel types. The packet_type is about data really present in the
     * mbuf. Example: if vlan stripping is enabled, a received vlan packet
     * would have RTE_PTYPE_L2_ETHER and not RTE_PTYPE_L2_VLAN because the
     * vlan is stripped from the data.
     */
    RTE_STD_C11
    union {
        uint32_t packet_type; 
        struct {
            uint32_t l2_type:4; 
            uint32_t l3_type:4; 
            uint32_t l4_type:4; 
            uint32_t tun_type:4; 
            RTE_STD_C11
            union {
                uint8_t inner_esp_next_proto;
                __extension__
                struct {
                    uint8_t inner_l2_type:4;
                    uint8_t inner_l3_type:4;
                };
            };
            uint32_t inner_l4_type:4; 
        };
    };

    uint32_t pkt_len;         
    uint16_t data_len;        
    uint16_t vlan_tci;

    RTE_STD_C11
    union {
        union {
            uint32_t rss;     
            struct {
                union {
                    struct {
                        uint16_t hash;
                        uint16_t id;
                    };
                    uint32_t lo;
                };
                uint32_t hi;
            } fdir; 
            struct rte_mbuf_sched sched;
            struct {
                uint32_t reserved1;
                uint16_t reserved2;
                uint16_t txq;
            } txadapter; 
            uint32_t usr;
        } hash;                   
    };

    uint16_t vlan_tci_outer;

    uint16_t buf_len;         
    uint64_t timestamp;

    /* second cache line - fields only used in slow path or on TX */
    MARKER cacheline1 __rte_cache_min_aligned;

    RTE_STD_C11
    union {
        void *userdata;   
        uint64_t udata64; 
    };

    struct rte_mempool *pool; 
    struct rte_mbuf *next;    
    /* fields to support TX offloads */
    RTE_STD_C11
    union {
        uint64_t tx_offload;       
        __extension__
        struct {
            uint64_t l2_len:RTE_MBUF_L2_LEN_BITS;
            uint64_t l3_len:RTE_MBUF_L3_LEN_BITS;
            uint64_t l4_len:RTE_MBUF_L4_LEN_BITS;
            uint64_t tso_segsz:RTE_MBUF_TSO_SEGSZ_BITS;
            /*
             * Fields for Tx offloading of tunnels.
             * These are undefined for packets which don't request
             * any tunnel offloads (outer IP or UDP checksum,
             * tunnel TSO).
             *
             * PMDs should not use these fields unconditionally
             * when calculating offsets.
             *
             * Applications are expected to set appropriate tunnel
             * offload flags when they fill in these fields.
             */
            uint64_t outer_l3_len:RTE_MBUF_OUTL3_LEN_BITS;
            uint64_t outer_l2_len:RTE_MBUF_OUTL2_LEN_BITS;
            /* uint64_t unused:RTE_MBUF_TXOFLD_UNUSED_BITS; */
        };
    };

    uint16_t priv_size;

    uint16_t timesync;

    uint32_t seqn;

    struct rte_mbuf_ext_shared_info *shinfo;

    uint64_t dynfield1[2]; 
} __rte_cache_aligned;

typedef void (*rte_mbuf_extbuf_free_callback_t)(void *addr, void *opaque);

struct rte_mbuf_ext_shared_info {
    rte_mbuf_extbuf_free_callback_t free_cb; 
    void *fcb_opaque;                        
    rte_atomic16_t refcnt_atomic;        
};

#define RTE_MBUF_MAX_NB_SEGS    UINT16_MAX

#define RTE_MBUF_CLONED(mb)     ((mb)->ol_flags & IND_ATTACHED_MBUF)

#define RTE_MBUF_HAS_EXTBUF(mb) ((mb)->ol_flags & EXT_ATTACHED_MBUF)

#define RTE_MBUF_DIRECT(mb) \
    (!((mb)->ol_flags & (IND_ATTACHED_MBUF | EXT_ATTACHED_MBUF)))

#define MBUF_INVALID_PORT UINT16_MAX

#define rte_pktmbuf_mtod_offset(m, t, o)    \
    ((t)(void *)((char *)(m)->buf_addr + (m)->data_off + (o)))

#define rte_pktmbuf_mtod(m, t) rte_pktmbuf_mtod_offset(m, t, 0)

#define rte_pktmbuf_iova_offset(m, o) \
    (rte_iova_t)((m)->buf_iova + (m)->data_off + (o))

#define rte_pktmbuf_iova(m) rte_pktmbuf_iova_offset(m, 0)

#ifdef __cplusplus
}
#endif

#endif /* _RTE_MBUF_CORE_H_ */