rte_ethdev.h

Go to the documentation of this file.

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2010-2017 Intel Corporation
 */

#ifndef _RTE_ETHDEV_H_
#define _RTE_ETHDEV_H_

#ifdef __cplusplus
extern "C" {
#endif

#include <stdint.h>

/* Use this macro to check if LRO API is supported */
#define RTE_ETHDEV_HAS_LRO_SUPPORT

#include <rte_compat.h>
#include <rte_log.h>
#include <rte_interrupts.h>
#include <rte_dev.h>
#include <rte_devargs.h>
#include <rte_errno.h>
#include <rte_common.h>
#include <rte_config.h>

#include "rte_ether.h"
#include "rte_eth_ctrl.h"
#include "rte_dev_info.h"

struct rte_mbuf;

struct rte_eth_stats {
        uint64_t ipackets;  
        uint64_t opackets;  
        uint64_t ibytes;    
        uint64_t obytes;    
        uint64_t imissed;
        uint64_t ierrors;   
        uint64_t oerrors;   
        uint64_t rx_nombuf; 
        uint64_t q_ipackets[RTE_ETHDEV_QUEUE_STAT_CNTRS];
        uint64_t q_opackets[RTE_ETHDEV_QUEUE_STAT_CNTRS];
        uint64_t q_ibytes[RTE_ETHDEV_QUEUE_STAT_CNTRS];
        uint64_t q_obytes[RTE_ETHDEV_QUEUE_STAT_CNTRS];
        uint64_t q_errors[RTE_ETHDEV_QUEUE_STAT_CNTRS];
};

#define ETH_LINK_SPEED_AUTONEG  (0 <<  0)  
#define ETH_LINK_SPEED_FIXED    (1 <<  0)  
#define ETH_LINK_SPEED_10M_HD   (1 <<  1)  
#define ETH_LINK_SPEED_10M      (1 <<  2)  
#define ETH_LINK_SPEED_100M_HD  (1 <<  3)  
#define ETH_LINK_SPEED_100M     (1 <<  4)  
#define ETH_LINK_SPEED_1G       (1 <<  5)  
#define ETH_LINK_SPEED_2_5G     (1 <<  6)  
#define ETH_LINK_SPEED_5G       (1 <<  7)  
#define ETH_LINK_SPEED_10G      (1 <<  8)  
#define ETH_LINK_SPEED_20G      (1 <<  9)  
#define ETH_LINK_SPEED_25G      (1 << 10)  
#define ETH_LINK_SPEED_40G      (1 << 11)  
#define ETH_LINK_SPEED_50G      (1 << 12)  
#define ETH_LINK_SPEED_56G      (1 << 13)  
#define ETH_LINK_SPEED_100G     (1 << 14)  
#define ETH_SPEED_NUM_NONE         0 
#define ETH_SPEED_NUM_10M         10 
#define ETH_SPEED_NUM_100M       100 
#define ETH_SPEED_NUM_1G        1000 
#define ETH_SPEED_NUM_2_5G      2500 
#define ETH_SPEED_NUM_5G        5000 
#define ETH_SPEED_NUM_10G      10000 
#define ETH_SPEED_NUM_20G      20000 
#define ETH_SPEED_NUM_25G      25000 
#define ETH_SPEED_NUM_40G      40000 
#define ETH_SPEED_NUM_50G      50000 
#define ETH_SPEED_NUM_56G      56000 
#define ETH_SPEED_NUM_100G    100000 
__extension__
struct rte_eth_link {
        uint32_t link_speed;        
        uint16_t link_duplex  : 1;  
        uint16_t link_autoneg : 1;  
        uint16_t link_status  : 1;  
} __attribute__((aligned(8)));      
/* Utility constants */
#define ETH_LINK_HALF_DUPLEX 0 
#define ETH_LINK_FULL_DUPLEX 1 
#define ETH_LINK_DOWN        0 
#define ETH_LINK_UP          1 
#define ETH_LINK_FIXED       0 
#define ETH_LINK_AUTONEG     1 
struct rte_eth_thresh {
        uint8_t pthresh; 
        uint8_t hthresh; 
        uint8_t wthresh; 
};

#define ETH_MQ_RX_RSS_FLAG  0x1
#define ETH_MQ_RX_DCB_FLAG  0x2
#define ETH_MQ_RX_VMDQ_FLAG 0x4

enum rte_eth_rx_mq_mode {
        ETH_MQ_RX_NONE = 0,

        ETH_MQ_RX_RSS = ETH_MQ_RX_RSS_FLAG,
        ETH_MQ_RX_DCB = ETH_MQ_RX_DCB_FLAG,
        ETH_MQ_RX_DCB_RSS = ETH_MQ_RX_RSS_FLAG | ETH_MQ_RX_DCB_FLAG,

        ETH_MQ_RX_VMDQ_ONLY = ETH_MQ_RX_VMDQ_FLAG,
        ETH_MQ_RX_VMDQ_RSS = ETH_MQ_RX_RSS_FLAG | ETH_MQ_RX_VMDQ_FLAG,
        ETH_MQ_RX_VMDQ_DCB = ETH_MQ_RX_VMDQ_FLAG | ETH_MQ_RX_DCB_FLAG,
        ETH_MQ_RX_VMDQ_DCB_RSS = ETH_MQ_RX_RSS_FLAG | ETH_MQ_RX_DCB_FLAG |
                                 ETH_MQ_RX_VMDQ_FLAG,
};

#define ETH_RSS                       ETH_MQ_RX_RSS
#define VMDQ_DCB                      ETH_MQ_RX_VMDQ_DCB
#define ETH_DCB_RX                    ETH_MQ_RX_DCB

enum rte_eth_tx_mq_mode {
        ETH_MQ_TX_NONE    = 0,  
        ETH_MQ_TX_DCB,          
        ETH_MQ_TX_VMDQ_DCB,     
        ETH_MQ_TX_VMDQ_ONLY,    
};

#define ETH_DCB_NONE                ETH_MQ_TX_NONE
#define ETH_VMDQ_DCB_TX             ETH_MQ_TX_VMDQ_DCB
#define ETH_DCB_TX                  ETH_MQ_TX_DCB

struct rte_eth_rxmode {
        enum rte_eth_rx_mq_mode mq_mode;
        uint32_t max_rx_pkt_len;  
        uint16_t split_hdr_size;  
        uint64_t offloads;
        __extension__
        uint16_t header_split : 1, 
                hw_ip_checksum   : 1, 
                hw_vlan_filter   : 1, 
                hw_vlan_strip    : 1, 
                hw_vlan_extend   : 1, 
                jumbo_frame      : 1, 
                hw_strip_crc     : 1, 
                enable_scatter   : 1, 
                enable_lro       : 1, 
                hw_timestamp     : 1, 
                security         : 1, 
                ignore_offload_bitfield : 1;
};

enum rte_vlan_type {
        ETH_VLAN_TYPE_UNKNOWN = 0,
        ETH_VLAN_TYPE_INNER, 
        ETH_VLAN_TYPE_OUTER, 
        ETH_VLAN_TYPE_MAX,
};

struct rte_vlan_filter_conf {
        uint64_t ids[64];
};

struct rte_eth_rss_conf {
        uint8_t *rss_key;    
        uint8_t rss_key_len; 
        uint64_t rss_hf;     
};

/*
 * The RSS offload types are defined based on flow types which are defined
 * in rte_eth_ctrl.h. Different NIC hardwares may support different RSS offload
 * types. The supported flow types or RSS offload types can be queried by
 * rte_eth_dev_info_get().
 */
#define ETH_RSS_IPV4               (1ULL << RTE_ETH_FLOW_IPV4)
#define ETH_RSS_FRAG_IPV4          (1ULL << RTE_ETH_FLOW_FRAG_IPV4)
#define ETH_RSS_NONFRAG_IPV4_TCP   (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_TCP)
#define ETH_RSS_NONFRAG_IPV4_UDP   (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_UDP)
#define ETH_RSS_NONFRAG_IPV4_SCTP  (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_SCTP)
#define ETH_RSS_NONFRAG_IPV4_OTHER (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_OTHER)
#define ETH_RSS_IPV6               (1ULL << RTE_ETH_FLOW_IPV6)
#define ETH_RSS_FRAG_IPV6          (1ULL << RTE_ETH_FLOW_FRAG_IPV6)
#define ETH_RSS_NONFRAG_IPV6_TCP   (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_TCP)
#define ETH_RSS_NONFRAG_IPV6_UDP   (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_UDP)
#define ETH_RSS_NONFRAG_IPV6_SCTP  (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_SCTP)
#define ETH_RSS_NONFRAG_IPV6_OTHER (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_OTHER)
#define ETH_RSS_L2_PAYLOAD         (1ULL << RTE_ETH_FLOW_L2_PAYLOAD)
#define ETH_RSS_IPV6_EX            (1ULL << RTE_ETH_FLOW_IPV6_EX)
#define ETH_RSS_IPV6_TCP_EX        (1ULL << RTE_ETH_FLOW_IPV6_TCP_EX)
#define ETH_RSS_IPV6_UDP_EX        (1ULL << RTE_ETH_FLOW_IPV6_UDP_EX)
#define ETH_RSS_PORT               (1ULL << RTE_ETH_FLOW_PORT)
#define ETH_RSS_VXLAN              (1ULL << RTE_ETH_FLOW_VXLAN)
#define ETH_RSS_GENEVE             (1ULL << RTE_ETH_FLOW_GENEVE)
#define ETH_RSS_NVGRE              (1ULL << RTE_ETH_FLOW_NVGRE)

#define ETH_RSS_IP ( \
        ETH_RSS_IPV4 | \
        ETH_RSS_FRAG_IPV4 | \
        ETH_RSS_NONFRAG_IPV4_OTHER | \
        ETH_RSS_IPV6 | \
        ETH_RSS_FRAG_IPV6 | \
        ETH_RSS_NONFRAG_IPV6_OTHER | \
        ETH_RSS_IPV6_EX)

#define ETH_RSS_UDP ( \
        ETH_RSS_NONFRAG_IPV4_UDP | \
        ETH_RSS_NONFRAG_IPV6_UDP | \
        ETH_RSS_IPV6_UDP_EX)

#define ETH_RSS_TCP ( \
        ETH_RSS_NONFRAG_IPV4_TCP | \
        ETH_RSS_NONFRAG_IPV6_TCP | \
        ETH_RSS_IPV6_TCP_EX)

#define ETH_RSS_SCTP ( \
        ETH_RSS_NONFRAG_IPV4_SCTP | \
        ETH_RSS_NONFRAG_IPV6_SCTP)

#define ETH_RSS_TUNNEL ( \
        ETH_RSS_VXLAN  | \
        ETH_RSS_GENEVE | \
        ETH_RSS_NVGRE)

#define ETH_RSS_PROTO_MASK ( \
        ETH_RSS_IPV4 | \
        ETH_RSS_FRAG_IPV4 | \
        ETH_RSS_NONFRAG_IPV4_TCP | \
        ETH_RSS_NONFRAG_IPV4_UDP | \
        ETH_RSS_NONFRAG_IPV4_SCTP | \
        ETH_RSS_NONFRAG_IPV4_OTHER | \
        ETH_RSS_IPV6 | \
        ETH_RSS_FRAG_IPV6 | \
        ETH_RSS_NONFRAG_IPV6_TCP | \
        ETH_RSS_NONFRAG_IPV6_UDP | \
        ETH_RSS_NONFRAG_IPV6_SCTP | \
        ETH_RSS_NONFRAG_IPV6_OTHER | \
        ETH_RSS_L2_PAYLOAD | \
        ETH_RSS_IPV6_EX | \
        ETH_RSS_IPV6_TCP_EX | \
        ETH_RSS_IPV6_UDP_EX | \
        ETH_RSS_PORT  | \
        ETH_RSS_VXLAN | \
        ETH_RSS_GENEVE | \
        ETH_RSS_NVGRE)

/*
 * Definitions used for redirection table entry size.
 * Some RSS RETA sizes may not be supported by some drivers, check the
 * documentation or the description of relevant functions for more details.
 */
#define ETH_RSS_RETA_SIZE_64  64
#define ETH_RSS_RETA_SIZE_128 128
#define ETH_RSS_RETA_SIZE_256 256
#define ETH_RSS_RETA_SIZE_512 512
#define RTE_RETA_GROUP_SIZE   64

/* Definitions used for VMDQ and DCB functionality */
#define ETH_VMDQ_MAX_VLAN_FILTERS   64 
#define ETH_DCB_NUM_USER_PRIORITIES 8  
#define ETH_VMDQ_DCB_NUM_QUEUES     128 
#define ETH_DCB_NUM_QUEUES          128 
/* DCB capability defines */
#define ETH_DCB_PG_SUPPORT      0x00000001 
#define ETH_DCB_PFC_SUPPORT     0x00000002 
/* Definitions used for VLAN Offload functionality */
#define ETH_VLAN_STRIP_OFFLOAD   0x0001 
#define ETH_VLAN_FILTER_OFFLOAD  0x0002 
#define ETH_VLAN_EXTEND_OFFLOAD  0x0004 
/* Definitions used for mask VLAN setting */
#define ETH_VLAN_STRIP_MASK   0x0001 
#define ETH_VLAN_FILTER_MASK  0x0002 
#define ETH_VLAN_EXTEND_MASK  0x0004 
#define ETH_VLAN_ID_MAX       0x0FFF 
/* Definitions used for receive MAC address   */
#define ETH_NUM_RECEIVE_MAC_ADDR  128 
/* Definitions used for unicast hash  */
#define ETH_VMDQ_NUM_UC_HASH_ARRAY  128 
/* Definitions used for VMDQ pool rx mode setting */
#define ETH_VMDQ_ACCEPT_UNTAG   0x0001 
#define ETH_VMDQ_ACCEPT_HASH_MC 0x0002 
#define ETH_VMDQ_ACCEPT_HASH_UC 0x0004 
#define ETH_VMDQ_ACCEPT_BROADCAST   0x0008 
#define ETH_VMDQ_ACCEPT_MULTICAST   0x0010 
#define ETH_MIRROR_MAX_VLANS       64

#define ETH_MIRROR_VIRTUAL_POOL_UP     0x01  
#define ETH_MIRROR_UPLINK_PORT         0x02  
#define ETH_MIRROR_DOWNLINK_PORT       0x04  
#define ETH_MIRROR_VLAN                0x08  
#define ETH_MIRROR_VIRTUAL_POOL_DOWN   0x10  
struct rte_eth_vlan_mirror {
        uint64_t vlan_mask; 
        uint16_t vlan_id[ETH_MIRROR_MAX_VLANS];
};

struct rte_eth_mirror_conf {
        uint8_t rule_type; 
        uint8_t dst_pool;  
        uint64_t pool_mask; 
        struct rte_eth_vlan_mirror vlan;
};

struct rte_eth_rss_reta_entry64 {
        uint64_t mask;
        uint16_t reta[RTE_RETA_GROUP_SIZE];
};

enum rte_eth_nb_tcs {
        ETH_4_TCS = 4, 
        ETH_8_TCS = 8  
};

enum rte_eth_nb_pools {
        ETH_8_POOLS = 8,    
        ETH_16_POOLS = 16,  
        ETH_32_POOLS = 32,  
        ETH_64_POOLS = 64   
};

/* This structure may be extended in future. */
struct rte_eth_dcb_rx_conf {
        enum rte_eth_nb_tcs nb_tcs; 
        uint8_t dcb_tc[ETH_DCB_NUM_USER_PRIORITIES];
};

struct rte_eth_vmdq_dcb_tx_conf {
        enum rte_eth_nb_pools nb_queue_pools; 
        uint8_t dcb_tc[ETH_DCB_NUM_USER_PRIORITIES];
};

struct rte_eth_dcb_tx_conf {
        enum rte_eth_nb_tcs nb_tcs; 
        uint8_t dcb_tc[ETH_DCB_NUM_USER_PRIORITIES];
};

struct rte_eth_vmdq_tx_conf {
        enum rte_eth_nb_pools nb_queue_pools; 
};

struct rte_eth_vmdq_dcb_conf {
        enum rte_eth_nb_pools nb_queue_pools; 
        uint8_t enable_default_pool; 
        uint8_t default_pool; 
        uint8_t nb_pool_maps; 
        struct {
                uint16_t vlan_id; 
                uint64_t pools;   
        } pool_map[ETH_VMDQ_MAX_VLAN_FILTERS]; 
        uint8_t dcb_tc[ETH_DCB_NUM_USER_PRIORITIES];
};

struct rte_eth_vmdq_rx_conf {
        enum rte_eth_nb_pools nb_queue_pools; 
        uint8_t enable_default_pool; 
        uint8_t default_pool; 
        uint8_t enable_loop_back; 
        uint8_t nb_pool_maps; 
        uint32_t rx_mode; 
        struct {
                uint16_t vlan_id; 
                uint64_t pools;   
        } pool_map[ETH_VMDQ_MAX_VLAN_FILTERS]; 
};

struct rte_eth_txmode {
        enum rte_eth_tx_mq_mode mq_mode; 
        uint64_t offloads;

        /* For i40e specifically */
        uint16_t pvid;
        __extension__
        uint8_t hw_vlan_reject_tagged : 1,
                hw_vlan_reject_untagged : 1,
                hw_vlan_insert_pvid : 1;
};

struct rte_eth_rxconf {
        struct rte_eth_thresh rx_thresh; 
        uint16_t rx_free_thresh; 
        uint8_t rx_drop_en; 
        uint8_t rx_deferred_start; 
        uint64_t offloads;
};

#define ETH_TXQ_FLAGS_NOMULTSEGS 0x0001 
#define ETH_TXQ_FLAGS_NOREFCOUNT 0x0002 
#define ETH_TXQ_FLAGS_NOMULTMEMP 0x0004 
#define ETH_TXQ_FLAGS_NOVLANOFFL 0x0100 
#define ETH_TXQ_FLAGS_NOXSUMSCTP 0x0200 
#define ETH_TXQ_FLAGS_NOXSUMUDP  0x0400 
#define ETH_TXQ_FLAGS_NOXSUMTCP  0x0800 
#define ETH_TXQ_FLAGS_NOOFFLOADS \
                (ETH_TXQ_FLAGS_NOVLANOFFL | ETH_TXQ_FLAGS_NOXSUMSCTP | \
                 ETH_TXQ_FLAGS_NOXSUMUDP  | ETH_TXQ_FLAGS_NOXSUMTCP)
#define ETH_TXQ_FLAGS_NOXSUMS \
                (ETH_TXQ_FLAGS_NOXSUMSCTP | ETH_TXQ_FLAGS_NOXSUMUDP | \
                 ETH_TXQ_FLAGS_NOXSUMTCP)

#define ETH_TXQ_FLAGS_IGNORE    0x8000

struct rte_eth_txconf {
        struct rte_eth_thresh tx_thresh; 
        uint16_t tx_rs_thresh; 
        uint16_t tx_free_thresh; 
        uint32_t txq_flags; 
        uint8_t tx_deferred_start; 
        uint64_t offloads;
};

struct rte_eth_desc_lim {
        uint16_t nb_max;   
        uint16_t nb_min;   
        uint16_t nb_align; 
        uint16_t nb_seg_max;

        uint16_t nb_mtu_seg_max;
};

enum rte_eth_fc_mode {
        RTE_FC_NONE = 0, 
        RTE_FC_RX_PAUSE, 
        RTE_FC_TX_PAUSE, 
        RTE_FC_FULL      
};

struct rte_eth_fc_conf {
        uint32_t high_water;  
        uint32_t low_water;   
        uint16_t pause_time;  
        uint16_t send_xon;    
        enum rte_eth_fc_mode mode;  
        uint8_t mac_ctrl_frame_fwd; 
        uint8_t autoneg;      
};

struct rte_eth_pfc_conf {
        struct rte_eth_fc_conf fc; 
        uint8_t priority;          
};

enum rte_fdir_pballoc_type {
        RTE_FDIR_PBALLOC_64K = 0,  
        RTE_FDIR_PBALLOC_128K,     
        RTE_FDIR_PBALLOC_256K,     
};

enum rte_fdir_status_mode {
        RTE_FDIR_NO_REPORT_STATUS = 0, 
        RTE_FDIR_REPORT_STATUS, 
        RTE_FDIR_REPORT_STATUS_ALWAYS, 
};

struct rte_fdir_conf {
        enum rte_fdir_mode mode; 
        enum rte_fdir_pballoc_type pballoc; 
        enum rte_fdir_status_mode status;  
        uint8_t drop_queue;
        struct rte_eth_fdir_masks mask;
        struct rte_eth_fdir_flex_conf flex_conf;
};

struct rte_eth_udp_tunnel {
        uint16_t udp_port; 
        uint8_t prot_type; 
};

struct rte_intr_conf {
        uint32_t lsc:1;
        uint32_t rxq:1;
        uint32_t rmv:1;
};

struct rte_eth_conf {
        uint32_t link_speeds; 
        struct rte_eth_rxmode rxmode; 
        struct rte_eth_txmode txmode; 
        uint32_t lpbk_mode; 
        struct {
                struct rte_eth_rss_conf rss_conf; 
                struct rte_eth_vmdq_dcb_conf vmdq_dcb_conf;
                struct rte_eth_dcb_rx_conf dcb_rx_conf;
                struct rte_eth_vmdq_rx_conf vmdq_rx_conf;
        } rx_adv_conf; 
        union {
                struct rte_eth_vmdq_dcb_tx_conf vmdq_dcb_tx_conf;
                struct rte_eth_dcb_tx_conf dcb_tx_conf;
                struct rte_eth_vmdq_tx_conf vmdq_tx_conf;
        } tx_adv_conf; 
        uint32_t dcb_capability_en;
        struct rte_fdir_conf fdir_conf; 
        struct rte_intr_conf intr_conf; 
};

#define DEV_RX_OFFLOAD_VLAN_STRIP  0x00000001
#define DEV_RX_OFFLOAD_IPV4_CKSUM  0x00000002
#define DEV_RX_OFFLOAD_UDP_CKSUM   0x00000004
#define DEV_RX_OFFLOAD_TCP_CKSUM   0x00000008
#define DEV_RX_OFFLOAD_TCP_LRO     0x00000010
#define DEV_RX_OFFLOAD_QINQ_STRIP  0x00000020
#define DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM 0x00000040
#define DEV_RX_OFFLOAD_MACSEC_STRIP     0x00000080
#define DEV_RX_OFFLOAD_HEADER_SPLIT     0x00000100
#define DEV_RX_OFFLOAD_VLAN_FILTER      0x00000200
#define DEV_RX_OFFLOAD_VLAN_EXTEND      0x00000400
#define DEV_RX_OFFLOAD_JUMBO_FRAME      0x00000800
#define DEV_RX_OFFLOAD_CRC_STRIP        0x00001000
#define DEV_RX_OFFLOAD_SCATTER          0x00002000
#define DEV_RX_OFFLOAD_TIMESTAMP        0x00004000
#define DEV_RX_OFFLOAD_SECURITY         0x00008000
#define DEV_RX_OFFLOAD_CHECKSUM (DEV_RX_OFFLOAD_IPV4_CKSUM | \
                                 DEV_RX_OFFLOAD_UDP_CKSUM | \
                                 DEV_RX_OFFLOAD_TCP_CKSUM)
#define DEV_RX_OFFLOAD_VLAN (DEV_RX_OFFLOAD_VLAN_STRIP | \
                             DEV_RX_OFFLOAD_VLAN_FILTER | \
                             DEV_RX_OFFLOAD_VLAN_EXTEND)

/*
 * If new Rx offload capabilities are defined, they also must be
 * mentioned in rte_rx_offload_names in rte_ethdev.c file.
 */

#define DEV_TX_OFFLOAD_VLAN_INSERT 0x00000001
#define DEV_TX_OFFLOAD_IPV4_CKSUM  0x00000002
#define DEV_TX_OFFLOAD_UDP_CKSUM   0x00000004
#define DEV_TX_OFFLOAD_TCP_CKSUM   0x00000008
#define DEV_TX_OFFLOAD_SCTP_CKSUM  0x00000010
#define DEV_TX_OFFLOAD_TCP_TSO     0x00000020
#define DEV_TX_OFFLOAD_UDP_TSO     0x00000040
#define DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM 0x00000080 
#define DEV_TX_OFFLOAD_QINQ_INSERT 0x00000100
#define DEV_TX_OFFLOAD_VXLAN_TNL_TSO    0x00000200    
#define DEV_TX_OFFLOAD_GRE_TNL_TSO      0x00000400    
#define DEV_TX_OFFLOAD_IPIP_TNL_TSO     0x00000800    
#define DEV_TX_OFFLOAD_GENEVE_TNL_TSO   0x00001000    
#define DEV_TX_OFFLOAD_MACSEC_INSERT    0x00002000
#define DEV_TX_OFFLOAD_MT_LOCKFREE      0x00004000

#define DEV_TX_OFFLOAD_MULTI_SEGS       0x00008000

#define DEV_TX_OFFLOAD_MBUF_FAST_FREE   0x00010000

#define DEV_TX_OFFLOAD_SECURITY         0x00020000

#define DEV_TX_OFFLOAD_UDP_TNL_TSO      0x00040000

#define DEV_TX_OFFLOAD_IP_TNL_TSO       0x00080000

#define RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP 0x00000001

#define RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP 0x00000002

/*
 * If new Tx offload capabilities are defined, they also must be
 * mentioned in rte_tx_offload_names in rte_ethdev.c file.
 */

struct rte_pci_device;

/*
 * Fallback default preferred Rx/Tx port parameters.
 * These are used if an application requests default parameters
 * but the PMD does not provide preferred values.
 */
#define RTE_ETH_DEV_FALLBACK_RX_RINGSIZE 512
#define RTE_ETH_DEV_FALLBACK_TX_RINGSIZE 512
#define RTE_ETH_DEV_FALLBACK_RX_NBQUEUES 1
#define RTE_ETH_DEV_FALLBACK_TX_NBQUEUES 1

struct rte_eth_dev_portconf {
        uint16_t burst_size; 
        uint16_t ring_size; 
        uint16_t nb_queues; 
};

#define RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID    (0)

struct rte_eth_switch_info {
        const char *name;       
        uint16_t domain_id;     
        uint16_t port_id;
};

struct rte_eth_dev_info {
        struct rte_device *device; 
        const char *driver_name; 
        unsigned int if_index; 
        const uint32_t *dev_flags; 
        uint32_t min_rx_bufsize; 
        uint32_t max_rx_pktlen; 
        uint16_t max_rx_queues; 
        uint16_t max_tx_queues; 
        uint32_t max_mac_addrs; 
        uint32_t max_hash_mac_addrs;
        uint16_t max_vfs; 
        uint16_t max_vmdq_pools; 
        uint64_t rx_offload_capa;
        uint64_t tx_offload_capa;
        uint64_t rx_queue_offload_capa;
        uint64_t tx_queue_offload_capa;
        uint16_t reta_size;
        uint8_t hash_key_size; 
        uint64_t flow_type_rss_offloads;
        struct rte_eth_rxconf default_rxconf; 
        struct rte_eth_txconf default_txconf; 
        uint16_t vmdq_queue_base; 
        uint16_t vmdq_queue_num;  
        uint16_t vmdq_pool_base;  
        struct rte_eth_desc_lim rx_desc_lim;  
        struct rte_eth_desc_lim tx_desc_lim;  
        uint32_t speed_capa;  
        uint16_t nb_rx_queues; 
        uint16_t nb_tx_queues; 
        struct rte_eth_dev_portconf default_rxportconf;
        struct rte_eth_dev_portconf default_txportconf;
        uint64_t dev_capa;
        struct rte_eth_switch_info switch_info;
};

struct rte_eth_rxq_info {
        struct rte_mempool *mp;     
        struct rte_eth_rxconf conf; 
        uint8_t scattered_rx;       
        uint16_t nb_desc;           
} __rte_cache_min_aligned;

struct rte_eth_txq_info {
        struct rte_eth_txconf conf; 
        uint16_t nb_desc;           
} __rte_cache_min_aligned;

#define RTE_ETH_XSTATS_NAME_SIZE 64

struct rte_eth_xstat {
        uint64_t id;        
        uint64_t value;     
};

struct rte_eth_xstat_name {
        char name[RTE_ETH_XSTATS_NAME_SIZE]; 
};

#define ETH_DCB_NUM_TCS    8
#define ETH_MAX_VMDQ_POOL  64

struct rte_eth_dcb_tc_queue_mapping {
        struct {
                uint8_t base;
                uint8_t nb_queue;
        } tc_rxq[ETH_MAX_VMDQ_POOL][ETH_DCB_NUM_TCS];
        struct {
                uint8_t base;
                uint8_t nb_queue;
        } tc_txq[ETH_MAX_VMDQ_POOL][ETH_DCB_NUM_TCS];
};

struct rte_eth_dcb_info {
        uint8_t nb_tcs;        
        uint8_t prio_tc[ETH_DCB_NUM_USER_PRIORITIES]; 
        uint8_t tc_bws[ETH_DCB_NUM_TCS]; 
        struct rte_eth_dcb_tc_queue_mapping tc_queue;
};

#define RTE_ETH_QUEUE_STATE_STOPPED 0
#define RTE_ETH_QUEUE_STATE_STARTED 1

#define RTE_ETH_ALL RTE_MAX_ETHPORTS

/* Macros to check for valid port */
#define RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, retval) do { \
        if (!rte_eth_dev_is_valid_port(port_id)) { \
                RTE_PMD_DEBUG_TRACE("Invalid port_id=%d\n", port_id); \
                return retval; \
        } \
} while (0)

#define RTE_ETH_VALID_PORTID_OR_RET(port_id) do { \
        if (!rte_eth_dev_is_valid_port(port_id)) { \
                RTE_PMD_DEBUG_TRACE("Invalid port_id=%d\n", port_id); \
                return; \
        } \
} while (0)

#define ETH_L2_TUNNEL_ENABLE_MASK       0x00000001

#define ETH_L2_TUNNEL_INSERTION_MASK    0x00000002

#define ETH_L2_TUNNEL_STRIPPING_MASK    0x00000004

#define ETH_L2_TUNNEL_FORWARDING_MASK   0x00000008

typedef uint16_t (*rte_rx_callback_fn)(uint16_t port_id, uint16_t queue,
        struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t max_pkts,
        void *user_param);

typedef uint16_t (*rte_tx_callback_fn)(uint16_t port_id, uint16_t queue,
        struct rte_mbuf *pkts[], uint16_t nb_pkts, void *user_param);

enum rte_eth_dev_state {
        RTE_ETH_DEV_UNUSED = 0,
        RTE_ETH_DEV_ATTACHED,
        RTE_ETH_DEV_DEFERRED,
        RTE_ETH_DEV_REMOVED,
};

struct rte_eth_dev_sriov {
        uint8_t active;               
        uint8_t nb_q_per_pool;        
        uint16_t def_vmdq_idx;        
        uint16_t def_pool_q_idx;      
};
#define RTE_ETH_DEV_SRIOV(dev)         ((dev)->data->sriov)

#define RTE_ETH_NAME_MAX_LEN RTE_DEV_NAME_MAX_LEN

#define RTE_ETH_DEV_NO_OWNER 0

#define RTE_ETH_MAX_OWNER_NAME_LEN 64

struct rte_eth_dev_owner {
        uint64_t id; 
        char name[RTE_ETH_MAX_OWNER_NAME_LEN]; 
};

#define RTE_ETH_DEV_INTR_LSC     0x0002

#define RTE_ETH_DEV_BONDED_SLAVE 0x0004

#define RTE_ETH_DEV_INTR_RMV     0x0008

#define RTE_ETH_DEV_REPRESENTOR  0x0010

uint64_t rte_eth_find_next_owned_by(uint16_t port_id,
                const uint64_t owner_id);

#define RTE_ETH_FOREACH_DEV_OWNED_BY(p, o) \
        for (p = rte_eth_find_next_owned_by(0, o); \
             (unsigned int)p < (unsigned int)RTE_MAX_ETHPORTS; \
             p = rte_eth_find_next_owned_by(p + 1, o))

uint16_t rte_eth_find_next(uint16_t port_id);

#define RTE_ETH_FOREACH_DEV(p) \
        RTE_ETH_FOREACH_DEV_OWNED_BY(p, RTE_ETH_DEV_NO_OWNER)


int __rte_experimental rte_eth_dev_owner_new(uint64_t *owner_id);

int __rte_experimental rte_eth_dev_owner_set(const uint16_t port_id,
                const struct rte_eth_dev_owner *owner);

int __rte_experimental rte_eth_dev_owner_unset(const uint16_t port_id,
                const uint64_t owner_id);

void __rte_experimental rte_eth_dev_owner_delete(const uint64_t owner_id);

int __rte_experimental rte_eth_dev_owner_get(const uint16_t port_id,
                struct rte_eth_dev_owner *owner);

__rte_deprecated
uint16_t rte_eth_dev_count(void);

uint16_t rte_eth_dev_count_avail(void);

uint16_t __rte_experimental rte_eth_dev_count_total(void);

int rte_eth_dev_attach(const char *devargs, uint16_t *port_id);

int rte_eth_dev_detach(uint16_t port_id, char *devname);

uint32_t rte_eth_speed_bitflag(uint32_t speed, int duplex);

const char * __rte_experimental rte_eth_dev_rx_offload_name(uint64_t offload);

const char * __rte_experimental rte_eth_dev_tx_offload_name(uint64_t offload);

int rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_queue,
                uint16_t nb_tx_queue, const struct rte_eth_conf *eth_conf);

int __rte_experimental
rte_eth_dev_is_removed(uint16_t port_id);

int rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
                uint16_t nb_rx_desc, unsigned int socket_id,
                const struct rte_eth_rxconf *rx_conf,
                struct rte_mempool *mb_pool);

int rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
                uint16_t nb_tx_desc, unsigned int socket_id,
                const struct rte_eth_txconf *tx_conf);

int rte_eth_dev_socket_id(uint16_t port_id);

int rte_eth_dev_is_valid_port(uint16_t port_id);

int rte_eth_dev_rx_queue_start(uint16_t port_id, uint16_t rx_queue_id);

int rte_eth_dev_rx_queue_stop(uint16_t port_id, uint16_t rx_queue_id);

int rte_eth_dev_tx_queue_start(uint16_t port_id, uint16_t tx_queue_id);

int rte_eth_dev_tx_queue_stop(uint16_t port_id, uint16_t tx_queue_id);

int rte_eth_dev_start(uint16_t port_id);

void rte_eth_dev_stop(uint16_t port_id);

int rte_eth_dev_set_link_up(uint16_t port_id);

int rte_eth_dev_set_link_down(uint16_t port_id);

void rte_eth_dev_close(uint16_t port_id);

int rte_eth_dev_reset(uint16_t port_id);

void rte_eth_promiscuous_enable(uint16_t port_id);

void rte_eth_promiscuous_disable(uint16_t port_id);

int rte_eth_promiscuous_get(uint16_t port_id);

void rte_eth_allmulticast_enable(uint16_t port_id);

void rte_eth_allmulticast_disable(uint16_t port_id);

int rte_eth_allmulticast_get(uint16_t port_id);

void rte_eth_link_get(uint16_t port_id, struct rte_eth_link *link);

void rte_eth_link_get_nowait(uint16_t port_id, struct rte_eth_link *link);

int rte_eth_stats_get(uint16_t port_id, struct rte_eth_stats *stats);

int rte_eth_stats_reset(uint16_t port_id);

int rte_eth_xstats_get_names(uint16_t port_id,
                struct rte_eth_xstat_name *xstats_names,
                unsigned int size);

int rte_eth_xstats_get(uint16_t port_id, struct rte_eth_xstat *xstats,
                unsigned int n);

int
rte_eth_xstats_get_names_by_id(uint16_t port_id,
        struct rte_eth_xstat_name *xstats_names, unsigned int size,
        uint64_t *ids);

int rte_eth_xstats_get_by_id(uint16_t port_id, const uint64_t *ids,
                             uint64_t *values, unsigned int size);

int rte_eth_xstats_get_id_by_name(uint16_t port_id, const char *xstat_name,
                uint64_t *id);

void rte_eth_xstats_reset(uint16_t port_id);

int rte_eth_dev_set_tx_queue_stats_mapping(uint16_t port_id,
                uint16_t tx_queue_id, uint8_t stat_idx);

int rte_eth_dev_set_rx_queue_stats_mapping(uint16_t port_id,
                                           uint16_t rx_queue_id,
                                           uint8_t stat_idx);

void rte_eth_macaddr_get(uint16_t port_id, struct ether_addr *mac_addr);

void rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info);

int rte_eth_dev_fw_version_get(uint16_t port_id,
                               char *fw_version, size_t fw_size);

int rte_eth_dev_get_supported_ptypes(uint16_t port_id, uint32_t ptype_mask,
                                     uint32_t *ptypes, int num);

int rte_eth_dev_get_mtu(uint16_t port_id, uint16_t *mtu);

int rte_eth_dev_set_mtu(uint16_t port_id, uint16_t mtu);

int rte_eth_dev_vlan_filter(uint16_t port_id, uint16_t vlan_id, int on);

int rte_eth_dev_set_vlan_strip_on_queue(uint16_t port_id, uint16_t rx_queue_id,
                int on);

int rte_eth_dev_set_vlan_ether_type(uint16_t port_id,
                                    enum rte_vlan_type vlan_type,
                                    uint16_t tag_type);

int rte_eth_dev_set_vlan_offload(uint16_t port_id, int offload_mask);

int rte_eth_dev_get_vlan_offload(uint16_t port_id);

int rte_eth_dev_set_vlan_pvid(uint16_t port_id, uint16_t pvid, int on);

typedef void (*buffer_tx_error_fn)(struct rte_mbuf **unsent, uint16_t count,
                void *userdata);

struct rte_eth_dev_tx_buffer {
        buffer_tx_error_fn error_callback;
        void *error_userdata;
        uint16_t size;           
        uint16_t length;         
        struct rte_mbuf *pkts[];
};

#define RTE_ETH_TX_BUFFER_SIZE(sz) \
        (sizeof(struct rte_eth_dev_tx_buffer) + (sz) * sizeof(struct rte_mbuf *))

int
rte_eth_tx_buffer_init(struct rte_eth_dev_tx_buffer *buffer, uint16_t size);

int
rte_eth_tx_buffer_set_err_callback(struct rte_eth_dev_tx_buffer *buffer,
                buffer_tx_error_fn callback, void *userdata);

void
rte_eth_tx_buffer_drop_callback(struct rte_mbuf **pkts, uint16_t unsent,
                void *userdata);

void
rte_eth_tx_buffer_count_callback(struct rte_mbuf **pkts, uint16_t unsent,
                void *userdata);

int
rte_eth_tx_done_cleanup(uint16_t port_id, uint16_t queue_id, uint32_t free_cnt);

enum rte_eth_event_ipsec_subtype {
        RTE_ETH_EVENT_IPSEC_UNKNOWN = 0,
        RTE_ETH_EVENT_IPSEC_ESN_OVERFLOW,
        RTE_ETH_EVENT_IPSEC_SA_TIME_EXPIRY,
        RTE_ETH_EVENT_IPSEC_SA_BYTE_EXPIRY,
        RTE_ETH_EVENT_IPSEC_MAX
};

struct rte_eth_event_ipsec_desc {
        enum rte_eth_event_ipsec_subtype subtype;
        uint64_t metadata;
};

enum rte_eth_event_type {
        RTE_ETH_EVENT_UNKNOWN,  
        RTE_ETH_EVENT_INTR_LSC, 
        RTE_ETH_EVENT_QUEUE_STATE,
        RTE_ETH_EVENT_INTR_RESET,
        RTE_ETH_EVENT_VF_MBOX,  
        RTE_ETH_EVENT_MACSEC,   
        RTE_ETH_EVENT_INTR_RMV, 
        RTE_ETH_EVENT_NEW,      
        RTE_ETH_EVENT_DESTROY,  
        RTE_ETH_EVENT_IPSEC,    
        RTE_ETH_EVENT_MAX       
};

typedef int (*rte_eth_dev_cb_fn)(uint16_t port_id,
                enum rte_eth_event_type event, void *cb_arg, void *ret_param);
int rte_eth_dev_callback_register(uint16_t port_id,
                        enum rte_eth_event_type event,
                rte_eth_dev_cb_fn cb_fn, void *cb_arg);

int rte_eth_dev_callback_unregister(uint16_t port_id,
                        enum rte_eth_event_type event,
                rte_eth_dev_cb_fn cb_fn, void *cb_arg);

int rte_eth_dev_rx_intr_enable(uint16_t port_id, uint16_t queue_id);

int rte_eth_dev_rx_intr_disable(uint16_t port_id, uint16_t queue_id);

int rte_eth_dev_rx_intr_ctl(uint16_t port_id, int epfd, int op, void *data);

int rte_eth_dev_rx_intr_ctl_q(uint16_t port_id, uint16_t queue_id,
                              int epfd, int op, void *data);

int  rte_eth_led_on(uint16_t port_id);

int  rte_eth_led_off(uint16_t port_id);

int rte_eth_dev_flow_ctrl_get(uint16_t port_id,
                              struct rte_eth_fc_conf *fc_conf);

int rte_eth_dev_flow_ctrl_set(uint16_t port_id,
                              struct rte_eth_fc_conf *fc_conf);

int rte_eth_dev_priority_flow_ctrl_set(uint16_t port_id,
                                struct rte_eth_pfc_conf *pfc_conf);

int rte_eth_dev_mac_addr_add(uint16_t port_id, struct ether_addr *mac_addr,
                                uint32_t pool);

int rte_eth_dev_mac_addr_remove(uint16_t port_id, struct ether_addr *mac_addr);

int rte_eth_dev_default_mac_addr_set(uint16_t port_id,
                struct ether_addr *mac_addr);

int rte_eth_dev_rss_reta_update(uint16_t port_id,
                                struct rte_eth_rss_reta_entry64 *reta_conf,
                                uint16_t reta_size);

int rte_eth_dev_rss_reta_query(uint16_t port_id,
                               struct rte_eth_rss_reta_entry64 *reta_conf,
                               uint16_t reta_size);

int rte_eth_dev_uc_hash_table_set(uint16_t port_id, struct ether_addr *addr,
                                  uint8_t on);

int rte_eth_dev_uc_all_hash_table_set(uint16_t port_id, uint8_t on);

int rte_eth_mirror_rule_set(uint16_t port_id,
                        struct rte_eth_mirror_conf *mirror_conf,
                        uint8_t rule_id,
                        uint8_t on);

int rte_eth_mirror_rule_reset(uint16_t port_id,
                                         uint8_t rule_id);

int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx,
                        uint16_t tx_rate);

int rte_eth_dev_rss_hash_update(uint16_t port_id,
                                struct rte_eth_rss_conf *rss_conf);

int
rte_eth_dev_rss_hash_conf_get(uint16_t port_id,
                              struct rte_eth_rss_conf *rss_conf);

int
rte_eth_dev_udp_tunnel_port_add(uint16_t port_id,
                                struct rte_eth_udp_tunnel *tunnel_udp);

int
rte_eth_dev_udp_tunnel_port_delete(uint16_t port_id,
                                   struct rte_eth_udp_tunnel *tunnel_udp);

int rte_eth_dev_filter_supported(uint16_t port_id,
                enum rte_filter_type filter_type);

int rte_eth_dev_filter_ctrl(uint16_t port_id, enum rte_filter_type filter_type,
                        enum rte_filter_op filter_op, void *arg);

int rte_eth_dev_get_dcb_info(uint16_t port_id,
                             struct rte_eth_dcb_info *dcb_info);

struct rte_eth_rxtx_callback;

const struct rte_eth_rxtx_callback *
rte_eth_add_rx_callback(uint16_t port_id, uint16_t queue_id,
                rte_rx_callback_fn fn, void *user_param);

const struct rte_eth_rxtx_callback *
rte_eth_add_first_rx_callback(uint16_t port_id, uint16_t queue_id,
                rte_rx_callback_fn fn, void *user_param);

const struct rte_eth_rxtx_callback *
rte_eth_add_tx_callback(uint16_t port_id, uint16_t queue_id,
                rte_tx_callback_fn fn, void *user_param);

int rte_eth_remove_rx_callback(uint16_t port_id, uint16_t queue_id,
                const struct rte_eth_rxtx_callback *user_cb);

int rte_eth_remove_tx_callback(uint16_t port_id, uint16_t queue_id,
                const struct rte_eth_rxtx_callback *user_cb);

int rte_eth_rx_queue_info_get(uint16_t port_id, uint16_t queue_id,
        struct rte_eth_rxq_info *qinfo);

int rte_eth_tx_queue_info_get(uint16_t port_id, uint16_t queue_id,
        struct rte_eth_txq_info *qinfo);

int rte_eth_dev_get_reg_info(uint16_t port_id, struct rte_dev_reg_info *info);

int rte_eth_dev_get_eeprom_length(uint16_t port_id);

int rte_eth_dev_get_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info);

int rte_eth_dev_set_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info);

int __rte_experimental
rte_eth_dev_get_module_info(uint16_t port_id,
                            struct rte_eth_dev_module_info *modinfo);

int __rte_experimental
rte_eth_dev_get_module_eeprom(uint16_t port_id,
                              struct rte_dev_eeprom_info *info);

int rte_eth_dev_set_mc_addr_list(uint16_t port_id,
                                 struct ether_addr *mc_addr_set,
                                 uint32_t nb_mc_addr);

int rte_eth_timesync_enable(uint16_t port_id);

int rte_eth_timesync_disable(uint16_t port_id);

int rte_eth_timesync_read_rx_timestamp(uint16_t port_id,
                struct timespec *timestamp, uint32_t flags);

int rte_eth_timesync_read_tx_timestamp(uint16_t port_id,
                struct timespec *timestamp);

int rte_eth_timesync_adjust_time(uint16_t port_id, int64_t delta);

int rte_eth_timesync_read_time(uint16_t port_id, struct timespec *time);

int rte_eth_timesync_write_time(uint16_t port_id, const struct timespec *time);

int
rte_eth_dev_l2_tunnel_eth_type_conf(uint16_t port_id,
                                    struct rte_eth_l2_tunnel_conf *l2_tunnel);

int
rte_eth_dev_l2_tunnel_offload_set(uint16_t port_id,
                                  struct rte_eth_l2_tunnel_conf *l2_tunnel,
                                  uint32_t mask,
                                  uint8_t en);

int
rte_eth_dev_get_port_by_name(const char *name, uint16_t *port_id);

int
rte_eth_dev_get_name_by_port(uint16_t port_id, char *name);

int rte_eth_dev_adjust_nb_rx_tx_desc(uint16_t port_id,
                                     uint16_t *nb_rx_desc,
                                     uint16_t *nb_tx_desc);

int
rte_eth_dev_pool_ops_supported(uint16_t port_id, const char *pool);

void *
rte_eth_dev_get_sec_ctx(uint16_t port_id);


#include <rte_ethdev_core.h>

static inline uint16_t
rte_eth_rx_burst(uint16_t port_id, uint16_t queue_id,
                 struct rte_mbuf **rx_pkts, const uint16_t nb_pkts)
{
        struct rte_eth_dev *dev = &rte_eth_devices[port_id];
        uint16_t nb_rx;

#ifdef RTE_LIBRTE_ETHDEV_DEBUG
        RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0);
        RTE_FUNC_PTR_OR_ERR_RET(*dev->rx_pkt_burst, 0);

        if (queue_id >= dev->data->nb_rx_queues) {
                RTE_PMD_DEBUG_TRACE("Invalid RX queue_id=%d\n", queue_id);
                return 0;
        }
#endif
        nb_rx = (*dev->rx_pkt_burst)(dev->data->rx_queues[queue_id],
                                     rx_pkts, nb_pkts);

#ifdef RTE_ETHDEV_RXTX_CALLBACKS
        if (unlikely(dev->post_rx_burst_cbs[queue_id] != NULL)) {
                struct rte_eth_rxtx_callback *cb =
                                dev->post_rx_burst_cbs[queue_id];

                do {
                        nb_rx = cb->fn.rx(port_id, queue_id, rx_pkts, nb_rx,
                                                nb_pkts, cb->param);
                        cb = cb->next;
                } while (cb != NULL);
        }
#endif

        return nb_rx;
}

static inline int
rte_eth_rx_queue_count(uint16_t port_id, uint16_t queue_id)
{
        struct rte_eth_dev *dev;

        RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_count, -ENOTSUP);
        if (queue_id >= dev->data->nb_rx_queues)
                return -EINVAL;

        return (int)(*dev->dev_ops->rx_queue_count)(dev, queue_id);
}

static inline int
rte_eth_rx_descriptor_done(uint16_t port_id, uint16_t queue_id, uint16_t offset)
{
        struct rte_eth_dev *dev = &rte_eth_devices[port_id];
        RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_descriptor_done, -ENOTSUP);
        return (*dev->dev_ops->rx_descriptor_done)( \
                dev->data->rx_queues[queue_id], offset);
}

#define RTE_ETH_RX_DESC_AVAIL    0 
#define RTE_ETH_RX_DESC_DONE     1 
#define RTE_ETH_RX_DESC_UNAVAIL  2 
static inline int
rte_eth_rx_descriptor_status(uint16_t port_id, uint16_t queue_id,
        uint16_t offset)
{
        struct rte_eth_dev *dev;
        void *rxq;

#ifdef RTE_LIBRTE_ETHDEV_DEBUG
        RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
#endif
        dev = &rte_eth_devices[port_id];
#ifdef RTE_LIBRTE_ETHDEV_DEBUG
        if (queue_id >= dev->data->nb_rx_queues)
                return -ENODEV;
#endif
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_descriptor_status, -ENOTSUP);
        rxq = dev->data->rx_queues[queue_id];

        return (*dev->dev_ops->rx_descriptor_status)(rxq, offset);
}

#define RTE_ETH_TX_DESC_FULL    0 
#define RTE_ETH_TX_DESC_DONE    1 
#define RTE_ETH_TX_DESC_UNAVAIL 2 
static inline int rte_eth_tx_descriptor_status(uint16_t port_id,
        uint16_t queue_id, uint16_t offset)
{
        struct rte_eth_dev *dev;
        void *txq;

#ifdef RTE_LIBRTE_ETHDEV_DEBUG
        RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
#endif
        dev = &rte_eth_devices[port_id];
#ifdef RTE_LIBRTE_ETHDEV_DEBUG
        if (queue_id >= dev->data->nb_tx_queues)
                return -ENODEV;
#endif
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_descriptor_status, -ENOTSUP);
        txq = dev->data->tx_queues[queue_id];

        return (*dev->dev_ops->tx_descriptor_status)(txq, offset);
}

static inline uint16_t
rte_eth_tx_burst(uint16_t port_id, uint16_t queue_id,
                 struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
        struct rte_eth_dev *dev = &rte_eth_devices[port_id];

#ifdef RTE_LIBRTE_ETHDEV_DEBUG
        RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0);
        RTE_FUNC_PTR_OR_ERR_RET(*dev->tx_pkt_burst, 0);

        if (queue_id >= dev->data->nb_tx_queues) {
                RTE_PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", queue_id);
                return 0;
        }
#endif

#ifdef RTE_ETHDEV_RXTX_CALLBACKS
        struct rte_eth_rxtx_callback *cb = dev->pre_tx_burst_cbs[queue_id];

        if (unlikely(cb != NULL)) {
                do {
                        nb_pkts = cb->fn.tx(port_id, queue_id, tx_pkts, nb_pkts,
                                        cb->param);
                        cb = cb->next;
                } while (cb != NULL);
        }
#endif

        return (*dev->tx_pkt_burst)(dev->data->tx_queues[queue_id], tx_pkts, nb_pkts);
}

#ifndef RTE_ETHDEV_TX_PREPARE_NOOP

static inline uint16_t
rte_eth_tx_prepare(uint16_t port_id, uint16_t queue_id,
                struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
        struct rte_eth_dev *dev;

#ifdef RTE_LIBRTE_ETHDEV_DEBUG
        if (!rte_eth_dev_is_valid_port(port_id)) {
                RTE_PMD_DEBUG_TRACE("Invalid TX port_id=%d\n", port_id);
                rte_errno = -EINVAL;
                return 0;
        }
#endif

        dev = &rte_eth_devices[port_id];

#ifdef RTE_LIBRTE_ETHDEV_DEBUG
        if (queue_id >= dev->data->nb_tx_queues) {
                RTE_PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", queue_id);
                rte_errno = -EINVAL;
                return 0;
        }
#endif

        if (!dev->tx_pkt_prepare)
                return nb_pkts;

        return (*dev->tx_pkt_prepare)(dev->data->tx_queues[queue_id],
                        tx_pkts, nb_pkts);
}

#else

/*
 * Native NOOP operation for compilation targets which doesn't require any
 * preparations steps, and functional NOOP may introduce unnecessary performance
 * drop.
 *
 * Generally this is not a good idea to turn it on globally and didn't should
 * be used if behavior of tx_preparation can change.
 */

static inline uint16_t
rte_eth_tx_prepare(__rte_unused uint16_t port_id,
                __rte_unused uint16_t queue_id,
                __rte_unused struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
        return nb_pkts;
}

#endif

static inline uint16_t
rte_eth_tx_buffer_flush(uint16_t port_id, uint16_t queue_id,
                struct rte_eth_dev_tx_buffer *buffer)
{
        uint16_t sent;
        uint16_t to_send = buffer->length;

        if (to_send == 0)
                return 0;

        sent = rte_eth_tx_burst(port_id, queue_id, buffer->pkts, to_send);

        buffer->length = 0;

        /* All packets sent, or to be dealt with by callback below */
        if (unlikely(sent != to_send))
                buffer->error_callback(&buffer->pkts[sent],
                                       (uint16_t)(to_send - sent),
                                       buffer->error_userdata);

        return sent;
}

static __rte_always_inline uint16_t
rte_eth_tx_buffer(uint16_t port_id, uint16_t queue_id,
                struct rte_eth_dev_tx_buffer *buffer, struct rte_mbuf *tx_pkt)
{
        buffer->pkts[buffer->length++] = tx_pkt;
        if (buffer->length < buffer->size)
                return 0;

        return rte_eth_tx_buffer_flush(port_id, queue_id, buffer);
}

#ifdef __cplusplus
}
#endif

#endif /* _RTE_ETHDEV_H_ */