rte_eventdev.h

Go to the documentation of this file.

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2016 Cavium, Inc.
 * Copyright(c) 2016-2018 Intel Corporation.
 * Copyright 2016 NXP
 * All rights reserved.
 */

#ifndef _RTE_EVENTDEV_H_
#define _RTE_EVENTDEV_H_

#ifdef __cplusplus
extern "C" {
#endif

#include <rte_compat.h>
#include <rte_common.h>
#include <rte_errno.h>
#include <rte_mbuf_pool_ops.h>
#include <rte_mempool.h>

#include "rte_eventdev_trace_fp.h"

struct rte_mbuf; /* we just use mbuf pointers; no need to include rte_mbuf.h */
struct rte_event;

/* Event device capability bitmap flags */
#define RTE_EVENT_DEV_CAP_QUEUE_QOS           (1ULL << 0)

#define RTE_EVENT_DEV_CAP_EVENT_QOS           (1ULL << 1)

#define RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED   (1ULL << 2)

#define RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES     (1ULL << 3)

#define RTE_EVENT_DEV_CAP_BURST_MODE          (1ULL << 4)

#define RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE    (1ULL << 5)

#define RTE_EVENT_DEV_CAP_NONSEQ_MODE         (1ULL << 6)

#define RTE_EVENT_DEV_CAP_RUNTIME_PORT_LINK   (1ULL << 7)

#define RTE_EVENT_DEV_CAP_MULTIPLE_QUEUE_PORT (1ULL << 8)

#define RTE_EVENT_DEV_CAP_CARRY_FLOW_ID (1ULL << 9)

#define RTE_EVENT_DEV_CAP_MAINTENANCE_FREE (1ULL << 10)

#define RTE_EVENT_DEV_CAP_RUNTIME_QUEUE_ATTR (1ULL << 11)

#define RTE_EVENT_DEV_CAP_PROFILE_LINK (1ULL << 12)

#define RTE_EVENT_DEV_CAP_ATOMIC  (1ULL << 13)

#define RTE_EVENT_DEV_CAP_ORDERED  (1ULL << 14)

#define RTE_EVENT_DEV_CAP_PARALLEL  (1ULL << 15)

/* Event device priority levels */
#define RTE_EVENT_DEV_PRIORITY_HIGHEST   0

#define RTE_EVENT_DEV_PRIORITY_NORMAL    128

#define RTE_EVENT_DEV_PRIORITY_LOWEST    255

/* Event queue scheduling weights */
#define RTE_EVENT_QUEUE_WEIGHT_HIGHEST 255

#define RTE_EVENT_QUEUE_WEIGHT_LOWEST 0

/* Event queue scheduling affinity */
#define RTE_EVENT_QUEUE_AFFINITY_HIGHEST 255

#define RTE_EVENT_QUEUE_AFFINITY_LOWEST 0

uint8_t
rte_event_dev_count(void);

int
rte_event_dev_get_dev_id(const char *name);

int
rte_event_dev_socket_id(uint8_t dev_id);

struct rte_event_dev_info {
    const char *driver_name;    
    struct rte_device *dev; 
    uint32_t min_dequeue_timeout_ns;
    uint32_t max_dequeue_timeout_ns;
    uint32_t dequeue_timeout_ns;
    uint8_t max_event_queues;
    uint32_t max_event_queue_flows;
    uint8_t max_event_queue_priority_levels;
    uint8_t max_event_priority_levels;
    uint8_t max_event_ports;
    uint8_t max_event_port_dequeue_depth;
    uint32_t max_event_port_enqueue_depth;
    uint8_t max_event_port_links;
    int32_t max_num_events;
    uint32_t event_dev_cap;
    uint8_t max_single_link_event_port_queue_pairs;
    uint8_t max_profiles_per_port;
};

int
rte_event_dev_info_get(uint8_t dev_id, struct rte_event_dev_info *dev_info);

#define RTE_EVENT_DEV_ATTR_PORT_COUNT 0

#define RTE_EVENT_DEV_ATTR_QUEUE_COUNT 1

#define RTE_EVENT_DEV_ATTR_STARTED 2

int
rte_event_dev_attr_get(uint8_t dev_id, uint32_t attr_id,
               uint32_t *attr_value);


/* Event device configuration bitmap flags */
#define RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT (1ULL << 0)

struct rte_event_dev_config {
    uint32_t dequeue_timeout_ns;
    int32_t nb_events_limit;
    uint8_t nb_event_queues;
    uint8_t nb_event_ports;
    uint32_t nb_event_queue_flows;
    uint32_t nb_event_port_dequeue_depth;
    uint32_t nb_event_port_enqueue_depth;
    uint32_t event_dev_cfg;
    uint8_t nb_single_link_event_port_queues;
};

int
rte_event_dev_configure(uint8_t dev_id,
            const struct rte_event_dev_config *dev_conf);

/* Event queue specific APIs */

/* Event queue configuration bitmap flags */
#define RTE_EVENT_QUEUE_CFG_ALL_TYPES          (1ULL << 0)

#define RTE_EVENT_QUEUE_CFG_SINGLE_LINK        (1ULL << 1)

struct rte_event_queue_conf {
    uint32_t nb_atomic_flows;
    uint32_t nb_atomic_order_sequences;
    uint32_t event_queue_cfg;
    uint8_t schedule_type;
    uint8_t priority;
    uint8_t weight;
    uint8_t affinity;
};

int
rte_event_queue_default_conf_get(uint8_t dev_id, uint8_t queue_id,
                 struct rte_event_queue_conf *queue_conf);

int
rte_event_queue_setup(uint8_t dev_id, uint8_t queue_id,
              const struct rte_event_queue_conf *queue_conf);

#define RTE_EVENT_QUEUE_ATTR_PRIORITY 0

#define RTE_EVENT_QUEUE_ATTR_NB_ATOMIC_FLOWS 1

#define RTE_EVENT_QUEUE_ATTR_NB_ATOMIC_ORDER_SEQUENCES 2

#define RTE_EVENT_QUEUE_ATTR_EVENT_QUEUE_CFG 3

#define RTE_EVENT_QUEUE_ATTR_SCHEDULE_TYPE 4

#define RTE_EVENT_QUEUE_ATTR_WEIGHT 5

#define RTE_EVENT_QUEUE_ATTR_AFFINITY 6

int
rte_event_queue_attr_get(uint8_t dev_id, uint8_t queue_id, uint32_t attr_id,
            uint32_t *attr_value);

int
rte_event_queue_attr_set(uint8_t dev_id, uint8_t queue_id, uint32_t attr_id,
             uint64_t attr_value);

/* Event port specific APIs */

/* Event port configuration bitmap flags */
#define RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL    (1ULL << 0)

#define RTE_EVENT_PORT_CFG_SINGLE_LINK         (1ULL << 1)

#define RTE_EVENT_PORT_CFG_HINT_PRODUCER       (1ULL << 2)

#define RTE_EVENT_PORT_CFG_HINT_CONSUMER       (1ULL << 3)

#define RTE_EVENT_PORT_CFG_HINT_WORKER         (1ULL << 4)

struct rte_event_port_conf {
    int32_t new_event_threshold;
    uint16_t dequeue_depth;
    uint16_t enqueue_depth;
    uint32_t event_port_cfg; 
};

int
rte_event_port_default_conf_get(uint8_t dev_id, uint8_t port_id,
                struct rte_event_port_conf *port_conf);

int
rte_event_port_setup(uint8_t dev_id, uint8_t port_id,
             const struct rte_event_port_conf *port_conf);

typedef void (*rte_eventdev_port_flush_t)(uint8_t dev_id,
                      struct rte_event event, void *arg);
void
rte_event_port_quiesce(uint8_t dev_id, uint8_t port_id,
               rte_eventdev_port_flush_t release_cb, void *args);

#define RTE_EVENT_PORT_ATTR_ENQ_DEPTH 0

#define RTE_EVENT_PORT_ATTR_DEQ_DEPTH 1

#define RTE_EVENT_PORT_ATTR_NEW_EVENT_THRESHOLD 2

#define RTE_EVENT_PORT_ATTR_IMPLICIT_RELEASE_DISABLE 3

int
rte_event_port_attr_get(uint8_t dev_id, uint8_t port_id, uint32_t attr_id,
            uint32_t *attr_value);

int
rte_event_dev_start(uint8_t dev_id);

void
rte_event_dev_stop(uint8_t dev_id);

typedef void (*rte_eventdev_stop_flush_t)(uint8_t dev_id,
                      struct rte_event event, void *arg);
int rte_event_dev_stop_flush_callback_register(uint8_t dev_id,
                           rte_eventdev_stop_flush_t callback, void *userdata);

int
rte_event_dev_close(uint8_t dev_id);

struct __rte_aligned(16) rte_event_vector {
    uint16_t nb_elem;
    uint16_t elem_offset : 12;
    uint16_t rsvd : 3;
    uint16_t attr_valid : 1;
    union {
        /* Used by Rx/Tx adapter.
         * Indicates that all the elements in this vector belong to the
         * same port and queue pair when originating from Rx adapter,
         * valid only when event type is ETHDEV_VECTOR or
         * ETH_RX_ADAPTER_VECTOR.
         * Can also be used to indicate the Tx adapter the destination
         * port and queue of the mbufs in the vector
         */
        struct {
            uint16_t port;   
            uint16_t queue;  
        };
    };
    uint64_t impl_opaque;

/* empty structures do not have zero size in C++ leading to compilation errors
 * with clang about structure having different sizes in C and C++.
 * Since these are all zero-sized arrays, we can omit the "union" wrapper for
 * C++ builds, removing the warning.
 */
#ifndef __cplusplus

    union __rte_aligned(16) {
#endif
        struct rte_mbuf *mbufs[0];
        void *ptrs[0];
        uint64_t u64s[0];
#ifndef __cplusplus
    };
#endif

};

/* Scheduler type definitions */
#define RTE_SCHED_TYPE_ORDERED          0

#define RTE_SCHED_TYPE_ATOMIC           1

#define RTE_SCHED_TYPE_PARALLEL         2

/* Event types to classify the event source */
#define RTE_EVENT_TYPE_ETHDEV           0x0

#define RTE_EVENT_TYPE_CRYPTODEV        0x1

#define RTE_EVENT_TYPE_TIMER        0x2

#define RTE_EVENT_TYPE_CPU              0x3

#define RTE_EVENT_TYPE_ETH_RX_ADAPTER   0x4

#define RTE_EVENT_TYPE_DMADEV           0x5

#define RTE_EVENT_TYPE_VECTOR           0x8

#define RTE_EVENT_TYPE_ETHDEV_VECTOR                                           \
    (RTE_EVENT_TYPE_VECTOR | RTE_EVENT_TYPE_ETHDEV)

#define RTE_EVENT_TYPE_CPU_VECTOR (RTE_EVENT_TYPE_VECTOR | RTE_EVENT_TYPE_CPU)

#define RTE_EVENT_TYPE_ETH_RX_ADAPTER_VECTOR                                   \
    (RTE_EVENT_TYPE_VECTOR | RTE_EVENT_TYPE_ETH_RX_ADAPTER)

#define RTE_EVENT_TYPE_CRYPTODEV_VECTOR                                        \
    (RTE_EVENT_TYPE_VECTOR | RTE_EVENT_TYPE_CRYPTODEV)

#define RTE_EVENT_TYPE_MAX              0x10

/* Event enqueue operations */
#define RTE_EVENT_OP_NEW                0

#define RTE_EVENT_OP_FORWARD            1

#define RTE_EVENT_OP_RELEASE            2

struct rte_event {
    /* WORD0 */
    union {
        uint64_t event;
        struct {
            uint32_t flow_id:20;
            uint32_t sub_event_type:8;
            uint32_t event_type:4;
            uint8_t op:2;
            uint8_t rsvd:4;
            uint8_t sched_type:2;
            uint8_t queue_id;
            uint8_t priority;
            uint8_t impl_opaque;
        };
    };
    /* WORD1 */
    union {
        uint64_t u64;
        void *event_ptr;
        struct rte_mbuf *mbuf;
        struct rte_event_vector *vec;
    };
};

/* Ethdev Rx adapter capability bitmap flags */
#define RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT  0x1

#define RTE_EVENT_ETH_RX_ADAPTER_CAP_MULTI_EVENTQ   0x2

#define RTE_EVENT_ETH_RX_ADAPTER_CAP_OVERRIDE_FLOW_ID   0x4

#define RTE_EVENT_ETH_RX_ADAPTER_CAP_EVENT_VECTOR   0x8

int
rte_event_eth_rx_adapter_caps_get(uint8_t dev_id, uint16_t eth_port_id,
                uint32_t *caps);

#define RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT (1ULL << 0)

#define RTE_EVENT_TIMER_ADAPTER_CAP_PERIODIC      (1ULL << 1)

int
rte_event_timer_adapter_caps_get(uint8_t dev_id, uint32_t *caps);

/* Crypto adapter capability bitmap flag */
#define RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW   0x1

#define RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD   0x2

#define RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND  0x4

#define RTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA   0x8

#define RTE_EVENT_CRYPTO_ADAPTER_CAP_EVENT_VECTOR   0x10

int
rte_event_crypto_adapter_caps_get(uint8_t dev_id, uint8_t cdev_id,
                  uint32_t *caps);

/* DMA adapter capability bitmap flag */
#define RTE_EVENT_DMA_ADAPTER_CAP_INTERNAL_PORT_OP_NEW 0x1

#define RTE_EVENT_DMA_ADAPTER_CAP_INTERNAL_PORT_OP_FWD 0x2

#define RTE_EVENT_DMA_ADAPTER_CAP_INTERNAL_PORT_VCHAN_EV_BIND 0x4

__rte_experimental
int
rte_event_dma_adapter_caps_get(uint8_t dev_id, uint8_t dmadev_id, uint32_t *caps);

/* Ethdev Tx adapter capability bitmap flags */
#define RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT  0x1

#define RTE_EVENT_ETH_TX_ADAPTER_CAP_EVENT_VECTOR   0x2

int
rte_event_eth_tx_adapter_caps_get(uint8_t dev_id, uint16_t eth_port_id,
                uint32_t *caps);

int
rte_event_dequeue_timeout_ticks(uint8_t dev_id, uint64_t ns,
                    uint64_t *timeout_ticks);

int
rte_event_port_link(uint8_t dev_id, uint8_t port_id,
            const uint8_t queues[], const uint8_t priorities[],
            uint16_t nb_links);

int
rte_event_port_unlink(uint8_t dev_id, uint8_t port_id,
              uint8_t queues[], uint16_t nb_unlinks);

__rte_experimental
int
rte_event_port_profile_links_set(uint8_t dev_id, uint8_t port_id, const uint8_t queues[],
                 const uint8_t priorities[], uint16_t nb_links, uint8_t profile_id);

__rte_experimental
int
rte_event_port_profile_unlink(uint8_t dev_id, uint8_t port_id, uint8_t queues[],
                  uint16_t nb_unlinks, uint8_t profile_id);

int
rte_event_port_unlinks_in_progress(uint8_t dev_id, uint8_t port_id);

int
rte_event_port_links_get(uint8_t dev_id, uint8_t port_id,
             uint8_t queues[], uint8_t priorities[]);

__rte_experimental
int
rte_event_port_profile_links_get(uint8_t dev_id, uint8_t port_id, uint8_t queues[],
                 uint8_t priorities[], uint8_t profile_id);

int
rte_event_dev_service_id_get(uint8_t dev_id, uint32_t *service_id);

int
rte_event_dev_dump(uint8_t dev_id, FILE *f);

#define RTE_EVENT_DEV_XSTATS_NAME_SIZE 64

enum rte_event_dev_xstats_mode {
    RTE_EVENT_DEV_XSTATS_DEVICE,
    RTE_EVENT_DEV_XSTATS_PORT,
    RTE_EVENT_DEV_XSTATS_QUEUE,
};

struct rte_event_dev_xstats_name {
    char name[RTE_EVENT_DEV_XSTATS_NAME_SIZE];
};

int
rte_event_dev_xstats_names_get(uint8_t dev_id,
                   enum rte_event_dev_xstats_mode mode,
                   uint8_t queue_port_id,
                   struct rte_event_dev_xstats_name *xstats_names,
                   uint64_t *ids,
                   unsigned int size);

int
rte_event_dev_xstats_get(uint8_t dev_id,
             enum rte_event_dev_xstats_mode mode,
             uint8_t queue_port_id,
             const uint64_t ids[],
             uint64_t values[], unsigned int n);

uint64_t
rte_event_dev_xstats_by_name_get(uint8_t dev_id, const char *name,
                 uint64_t *id);

int
rte_event_dev_xstats_reset(uint8_t dev_id,
               enum rte_event_dev_xstats_mode mode,
               int16_t queue_port_id,
               const uint64_t ids[],
               uint32_t nb_ids);

int rte_event_dev_selftest(uint8_t dev_id);

struct rte_mempool *
rte_event_vector_pool_create(const char *name, unsigned int n,
                 unsigned int cache_size, uint16_t nb_elem,
                 int socket_id);

#include <rte_eventdev_core.h>

static __rte_always_inline uint16_t
__rte_event_enqueue_burst(uint8_t dev_id, uint8_t port_id,
              const struct rte_event ev[], uint16_t nb_events,
              const event_enqueue_burst_t fn)
{
    const struct rte_event_fp_ops *fp_ops;
    void *port;

    fp_ops = &rte_event_fp_ops[dev_id];
    port = fp_ops->data[port_id];
#ifdef RTE_LIBRTE_EVENTDEV_DEBUG
    if (dev_id >= RTE_EVENT_MAX_DEVS ||
        port_id >= RTE_EVENT_MAX_PORTS_PER_DEV) {
        rte_errno = EINVAL;
        return 0;
    }

    if (port == NULL) {
        rte_errno = EINVAL;
        return 0;
    }
#endif
    rte_eventdev_trace_enq_burst(dev_id, port_id, ev, nb_events, (void *)fn);
    /*
     * Allow zero cost non burst mode routine invocation if application
     * requests nb_events as const one
     */
    if (nb_events == 1)
        return (fp_ops->enqueue)(port, ev);
    else
        return fn(port, ev, nb_events);
}

static inline uint16_t
rte_event_enqueue_burst(uint8_t dev_id, uint8_t port_id,
            const struct rte_event ev[], uint16_t nb_events)
{
    const struct rte_event_fp_ops *fp_ops;

    fp_ops = &rte_event_fp_ops[dev_id];
    return __rte_event_enqueue_burst(dev_id, port_id, ev, nb_events,
                     fp_ops->enqueue_burst);
}

static inline uint16_t
rte_event_enqueue_new_burst(uint8_t dev_id, uint8_t port_id,
                const struct rte_event ev[], uint16_t nb_events)
{
    const struct rte_event_fp_ops *fp_ops;

    fp_ops = &rte_event_fp_ops[dev_id];
    return __rte_event_enqueue_burst(dev_id, port_id, ev, nb_events,
                     fp_ops->enqueue_new_burst);
}

static inline uint16_t
rte_event_enqueue_forward_burst(uint8_t dev_id, uint8_t port_id,
                const struct rte_event ev[], uint16_t nb_events)
{
    const struct rte_event_fp_ops *fp_ops;

    fp_ops = &rte_event_fp_ops[dev_id];
    return __rte_event_enqueue_burst(dev_id, port_id, ev, nb_events,
                     fp_ops->enqueue_forward_burst);
}

static inline uint16_t
rte_event_dequeue_burst(uint8_t dev_id, uint8_t port_id, struct rte_event ev[],
            uint16_t nb_events, uint64_t timeout_ticks)
{
    const struct rte_event_fp_ops *fp_ops;
    void *port;

    fp_ops = &rte_event_fp_ops[dev_id];
    port = fp_ops->data[port_id];
#ifdef RTE_LIBRTE_EVENTDEV_DEBUG
    if (dev_id >= RTE_EVENT_MAX_DEVS ||
        port_id >= RTE_EVENT_MAX_PORTS_PER_DEV) {
        rte_errno = EINVAL;
        return 0;
    }

    if (port == NULL) {
        rte_errno = EINVAL;
        return 0;
    }
#endif
    rte_eventdev_trace_deq_burst(dev_id, port_id, ev, nb_events);
    /*
     * Allow zero cost non burst mode routine invocation if application
     * requests nb_events as const one
     */
    if (nb_events == 1)
        return (fp_ops->dequeue)(port, ev, timeout_ticks);
    else
        return (fp_ops->dequeue_burst)(port, ev, nb_events,
                           timeout_ticks);
}

#define RTE_EVENT_DEV_MAINT_OP_FLUSH          (1 << 0)

static inline int
rte_event_maintain(uint8_t dev_id, uint8_t port_id, int op)
{
    const struct rte_event_fp_ops *fp_ops;
    void *port;

    fp_ops = &rte_event_fp_ops[dev_id];
    port = fp_ops->data[port_id];
#ifdef RTE_LIBRTE_EVENTDEV_DEBUG
    if (dev_id >= RTE_EVENT_MAX_DEVS ||
        port_id >= RTE_EVENT_MAX_PORTS_PER_DEV)
        return -EINVAL;

    if (port == NULL)
        return -EINVAL;

    if (op & (~RTE_EVENT_DEV_MAINT_OP_FLUSH))
        return -EINVAL;
#endif
    rte_eventdev_trace_maintain(dev_id, port_id, op);

    if (fp_ops->maintain != NULL)
        fp_ops->maintain(port, op);

    return 0;
}

static inline uint8_t
rte_event_port_profile_switch(uint8_t dev_id, uint8_t port_id, uint8_t profile_id)
{
    const struct rte_event_fp_ops *fp_ops;
    void *port;

    fp_ops = &rte_event_fp_ops[dev_id];
    port = fp_ops->data[port_id];

#ifdef RTE_LIBRTE_EVENTDEV_DEBUG
    if (dev_id >= RTE_EVENT_MAX_DEVS ||
        port_id >= RTE_EVENT_MAX_PORTS_PER_DEV)
        return -EINVAL;

    if (port == NULL)
        return -EINVAL;

    if (profile_id >= RTE_EVENT_MAX_PROFILES_PER_PORT)
        return -EINVAL;
#endif
    rte_eventdev_trace_port_profile_switch(dev_id, port_id, profile_id);

    return fp_ops->profile_switch(port, profile_id);
}

#ifdef __cplusplus
}
#endif

#endif /* _RTE_EVENTDEV_H_ */