rte_cryptodev.h

Go to the documentation of this file.

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2015-2020 Intel Corporation.
 */

#ifndef _RTE_CRYPTODEV_H_
#define _RTE_CRYPTODEV_H_

#ifdef __cplusplus
extern "C" {
#endif

#include <rte_compat.h>
#include "rte_kvargs.h"
#include "rte_crypto.h"
#include <rte_common.h>
#include <rte_rcu_qsbr.h>

#include "rte_cryptodev_trace_fp.h"

extern const char **rte_cyptodev_names;

/* Logging Macros */

#define CDEV_LOG_ERR(...) \
    RTE_LOG(ERR, CRYPTODEV, \
        RTE_FMT("%s() line %u: " RTE_FMT_HEAD(__VA_ARGS__,) "\n", \
            __func__, __LINE__, RTE_FMT_TAIL(__VA_ARGS__,)))

#define CDEV_LOG_INFO(...) \
    RTE_LOG(INFO, CRYPTODEV, \
        RTE_FMT(RTE_FMT_HEAD(__VA_ARGS__,) "\n", \
            RTE_FMT_TAIL(__VA_ARGS__,)))

#define CDEV_LOG_DEBUG(...) \
    RTE_LOG(DEBUG, CRYPTODEV, \
        RTE_FMT("%s() line %u: " RTE_FMT_HEAD(__VA_ARGS__,) "\n", \
            __func__, __LINE__, RTE_FMT_TAIL(__VA_ARGS__,)))

#define CDEV_PMD_TRACE(...) \
    RTE_LOG(DEBUG, CRYPTODEV, \
        RTE_FMT("[%s] %s: " RTE_FMT_HEAD(__VA_ARGS__,) "\n", \
            dev, __func__, RTE_FMT_TAIL(__VA_ARGS__,)))

#define rte_crypto_op_ctod_offset(c, t, o)  \
    ((t)((char *)(c) + (o)))

#define rte_crypto_op_ctophys_offset(c, o)  \
    (rte_iova_t)((c)->phys_addr + (o))

struct rte_crypto_param_range {
    uint16_t min;   
    uint16_t max;   
    uint16_t increment;
};

#define RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES             RTE_BIT32(0)
#define RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES            RTE_BIT32(1)
#define RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_1_MEGABYTES           RTE_BIT32(2)

struct rte_cryptodev_symmetric_capability {
    enum rte_crypto_sym_xform_type xform_type;
    RTE_STD_C11
    union {
        struct {
            enum rte_crypto_auth_algorithm algo;
            uint16_t block_size;
            struct rte_crypto_param_range key_size;
            struct rte_crypto_param_range digest_size;
            struct rte_crypto_param_range aad_size;
            struct rte_crypto_param_range iv_size;
        } auth;
        struct {
            enum rte_crypto_cipher_algorithm algo;
            uint16_t block_size;
            struct rte_crypto_param_range key_size;
            struct rte_crypto_param_range iv_size;
            uint32_t dataunit_set;
        } cipher;
        struct {
            enum rte_crypto_aead_algorithm algo;
            uint16_t block_size;
            struct rte_crypto_param_range key_size;
            struct rte_crypto_param_range digest_size;
            struct rte_crypto_param_range aad_size;
            struct rte_crypto_param_range iv_size;
        } aead;
    };
};

struct rte_cryptodev_asymmetric_xform_capability {
    enum rte_crypto_asym_xform_type xform_type;
    uint32_t op_types;
    __extension__
    union {
        struct rte_crypto_param_range modlen;
    };
};

struct rte_cryptodev_asymmetric_capability {
    struct rte_cryptodev_asymmetric_xform_capability xform_capa;
};


struct rte_cryptodev_capabilities {
    enum rte_crypto_op_type op;
    RTE_STD_C11
    union {
        struct rte_cryptodev_symmetric_capability sym;
        struct rte_cryptodev_asymmetric_capability asym;
    };
};

struct rte_cryptodev_sym_capability_idx {
    enum rte_crypto_sym_xform_type type;
    union {
        enum rte_crypto_cipher_algorithm cipher;
        enum rte_crypto_auth_algorithm auth;
        enum rte_crypto_aead_algorithm aead;
    } algo;
};

struct rte_cryptodev_asym_capability_idx {
    enum rte_crypto_asym_xform_type type;
};

const struct rte_cryptodev_symmetric_capability *
rte_cryptodev_sym_capability_get(uint8_t dev_id,
        const struct rte_cryptodev_sym_capability_idx *idx);

__rte_experimental
const struct rte_cryptodev_asymmetric_xform_capability *
rte_cryptodev_asym_capability_get(uint8_t dev_id,
        const struct rte_cryptodev_asym_capability_idx *idx);

int
rte_cryptodev_sym_capability_check_cipher(
        const struct rte_cryptodev_symmetric_capability *capability,
        uint16_t key_size, uint16_t iv_size);

int
rte_cryptodev_sym_capability_check_auth(
        const struct rte_cryptodev_symmetric_capability *capability,
        uint16_t key_size, uint16_t digest_size, uint16_t iv_size);

int
rte_cryptodev_sym_capability_check_aead(
        const struct rte_cryptodev_symmetric_capability *capability,
        uint16_t key_size, uint16_t digest_size, uint16_t aad_size,
        uint16_t iv_size);

__rte_experimental
int
rte_cryptodev_asym_xform_capability_check_optype(
    const struct rte_cryptodev_asymmetric_xform_capability *capability,
        enum rte_crypto_asym_op_type op_type);

__rte_experimental
int
rte_cryptodev_asym_xform_capability_check_modlen(
    const struct rte_cryptodev_asymmetric_xform_capability *capability,
        uint16_t modlen);

int
rte_cryptodev_get_cipher_algo_enum(enum rte_crypto_cipher_algorithm *algo_enum,
        const char *algo_string);

int
rte_cryptodev_get_auth_algo_enum(enum rte_crypto_auth_algorithm *algo_enum,
        const char *algo_string);

int
rte_cryptodev_get_aead_algo_enum(enum rte_crypto_aead_algorithm *algo_enum,
        const char *algo_string);

__rte_experimental
int
rte_cryptodev_asym_get_xform_enum(enum rte_crypto_asym_xform_type *xform_enum,
        const char *xform_string);

__rte_experimental
const char *
rte_cryptodev_get_cipher_algo_string(enum rte_crypto_cipher_algorithm algo_enum);

__rte_experimental
const char *
rte_cryptodev_get_auth_algo_string(enum rte_crypto_auth_algorithm algo_enum);

__rte_experimental
const char *
rte_cryptodev_get_aead_algo_string(enum rte_crypto_aead_algorithm algo_enum);

__rte_experimental
const char *
rte_cryptodev_asym_get_xform_string(enum rte_crypto_asym_xform_type xform_enum);


#define RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() \
    { RTE_CRYPTO_OP_TYPE_UNDEFINED }


#define RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO       (1ULL << 0)

#define RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO      (1ULL << 1)

#define RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING     (1ULL << 2)

#define RTE_CRYPTODEV_FF_CPU_SSE            (1ULL << 3)

#define RTE_CRYPTODEV_FF_CPU_AVX            (1ULL << 4)

#define RTE_CRYPTODEV_FF_CPU_AVX2           (1ULL << 5)

#define RTE_CRYPTODEV_FF_CPU_AESNI          (1ULL << 6)

#define RTE_CRYPTODEV_FF_HW_ACCELERATED         (1ULL << 7)

#define RTE_CRYPTODEV_FF_CPU_AVX512         (1ULL << 8)

#define RTE_CRYPTODEV_FF_IN_PLACE_SGL           (1ULL << 9)

#define RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT     (1ULL << 10)

#define RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT      (1ULL << 11)

#define RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT      (1ULL << 12)

#define RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT       (1ULL << 13)

#define RTE_CRYPTODEV_FF_CPU_NEON           (1ULL << 14)

#define RTE_CRYPTODEV_FF_CPU_ARM_CE         (1ULL << 15)

#define RTE_CRYPTODEV_FF_SECURITY           (1ULL << 16)

#define RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP        (1ULL << 17)

#define RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT     (1ULL << 18)

#define RTE_CRYPTODEV_FF_DIGEST_ENCRYPTED       (1ULL << 19)

#define RTE_CRYPTODEV_FF_ASYM_SESSIONLESS       (1ULL << 20)

#define RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO         (1ULL << 21)

#define RTE_CRYPTODEV_FF_SYM_SESSIONLESS        (1ULL << 22)

#define RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA      (1ULL << 23)

#define RTE_CRYPTODEV_FF_SYM_RAW_DP         (1ULL << 24)

#define RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS (1ULL << 25)

#define RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY     (1ULL << 26)

#define RTE_CRYPTODEV_FF_SECURITY_INNER_CSUM        (1ULL << 27)

extern const char *
rte_cryptodev_get_feature_name(uint64_t flag);

/* Structure rte_cryptodev_info 8< */
struct rte_cryptodev_info {
    const char *driver_name;    
    uint8_t driver_id;      
    struct rte_device *device;  
    uint64_t feature_flags;
    const struct rte_cryptodev_capabilities *capabilities;
    unsigned max_nb_queue_pairs;
    uint16_t min_mbuf_headroom_req;
    uint16_t min_mbuf_tailroom_req;
    struct {
        unsigned max_nb_sessions;
    } sym;
};
/* >8 End of structure rte_cryptodev_info. */

#define RTE_CRYPTODEV_DETACHED  (0)
#define RTE_CRYPTODEV_ATTACHED  (1)

enum rte_cryptodev_event_type {
    RTE_CRYPTODEV_EVENT_UNKNOWN,    
    RTE_CRYPTODEV_EVENT_ERROR,  
    RTE_CRYPTODEV_EVENT_MAX     
};

/* Structure rte_cryptodev_qp_conf 8<*/
struct rte_cryptodev_qp_conf {
    uint32_t nb_descriptors; 
    struct rte_mempool *mp_session;
};
/* >8 End of structure rte_cryptodev_qp_conf. */

typedef uint16_t (*rte_cryptodev_callback_fn)(uint16_t dev_id, uint16_t qp_id,
        struct rte_crypto_op **ops, uint16_t nb_ops, void *user_param);

typedef void (*rte_cryptodev_cb_fn)(uint8_t dev_id,
        enum rte_cryptodev_event_type event, void *cb_arg);


struct rte_cryptodev_stats {
    uint64_t enqueued_count;
    uint64_t dequeued_count;
    uint64_t enqueue_err_count;
    uint64_t dequeue_err_count;
};

#define RTE_CRYPTODEV_NAME_MAX_LEN  (64)

extern int
rte_cryptodev_get_dev_id(const char *name);

extern const char *
rte_cryptodev_name_get(uint8_t dev_id);

extern uint8_t
rte_cryptodev_count(void);

extern uint8_t
rte_cryptodev_device_count_by_driver(uint8_t driver_id);

uint8_t
rte_cryptodev_devices_get(const char *driver_name, uint8_t *devices,
        uint8_t nb_devices);
/*
 * Return the NUMA socket to which a device is connected
 *
 * @param dev_id
 *   The identifier of the device
 * @return
 *   The NUMA socket id to which the device is connected or
 *   a default of zero if the socket could not be determined.
 *   -1 if returned is the dev_id value is out of range.
 */
extern int
rte_cryptodev_socket_id(uint8_t dev_id);

/* Structure rte_cryptodev_config 8< */
struct rte_cryptodev_config {
    int socket_id;          
    uint16_t nb_queue_pairs;
    uint64_t ff_disable;
};
/* >8 End of structure rte_cryptodev_config. */

extern int
rte_cryptodev_configure(uint8_t dev_id, struct rte_cryptodev_config *config);

extern int
rte_cryptodev_start(uint8_t dev_id);

extern void
rte_cryptodev_stop(uint8_t dev_id);

extern int
rte_cryptodev_close(uint8_t dev_id);

extern int
rte_cryptodev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
        const struct rte_cryptodev_qp_conf *qp_conf, int socket_id);

__rte_experimental
int
rte_cryptodev_get_qp_status(uint8_t dev_id, uint16_t queue_pair_id);

extern uint16_t
rte_cryptodev_queue_pair_count(uint8_t dev_id);


extern int
rte_cryptodev_stats_get(uint8_t dev_id, struct rte_cryptodev_stats *stats);

extern void
rte_cryptodev_stats_reset(uint8_t dev_id);

extern void
rte_cryptodev_info_get(uint8_t dev_id, struct rte_cryptodev_info *dev_info);


extern int
rte_cryptodev_callback_register(uint8_t dev_id,
        enum rte_cryptodev_event_type event,
        rte_cryptodev_cb_fn cb_fn, void *cb_arg);

extern int
rte_cryptodev_callback_unregister(uint8_t dev_id,
        enum rte_cryptodev_event_type event,
        rte_cryptodev_cb_fn cb_fn, void *cb_arg);

__rte_experimental
int
rte_cryptodev_queue_pair_event_error_query(uint8_t dev_id, uint16_t qp_id);

struct rte_cryptodev_callback;

RTE_TAILQ_HEAD(rte_cryptodev_cb_list, rte_cryptodev_callback);

struct rte_cryptodev_cb {
    struct rte_cryptodev_cb *next;
    rte_cryptodev_callback_fn fn;
    void *arg;
};

struct rte_cryptodev_cb_rcu {
    struct rte_cryptodev_cb *next;
    struct rte_rcu_qsbr *qsbr;
};

void *
rte_cryptodev_get_sec_ctx(uint8_t dev_id);

__rte_experimental
struct rte_mempool *
rte_cryptodev_sym_session_pool_create(const char *name, uint32_t nb_elts,
    uint32_t elt_size, uint32_t cache_size, uint16_t priv_size,
    int socket_id);


__rte_experimental
struct rte_mempool *
rte_cryptodev_asym_session_pool_create(const char *name, uint32_t nb_elts,
    uint32_t cache_size, uint16_t user_data_size, int socket_id);

void *
rte_cryptodev_sym_session_create(uint8_t dev_id,
        struct rte_crypto_sym_xform *xforms,
        struct rte_mempool *mp);
__rte_experimental
int
rte_cryptodev_asym_session_create(uint8_t dev_id,
        struct rte_crypto_asym_xform *xforms, struct rte_mempool *mp,
        void **session);

int
rte_cryptodev_sym_session_free(uint8_t dev_id,
    void *sess);

__rte_experimental
int
rte_cryptodev_asym_session_free(uint8_t dev_id, void *sess);

__rte_experimental
unsigned int
rte_cryptodev_asym_get_header_session_size(void);

unsigned int
rte_cryptodev_sym_get_private_session_size(uint8_t dev_id);

__rte_experimental
unsigned int
rte_cryptodev_asym_get_private_session_size(uint8_t dev_id);

unsigned int
rte_cryptodev_is_valid_dev(uint8_t dev_id);

int rte_cryptodev_driver_id_get(const char *name);

const char *rte_cryptodev_driver_name_get(uint8_t driver_id);

__rte_experimental
int
rte_cryptodev_sym_session_set_user_data(void *sess,
                    void *data,
                    uint16_t size);

#define CRYPTO_SESS_OPAQUE_DATA_OFF 0

static inline uint64_t
rte_cryptodev_sym_session_opaque_data_get(void *sess)
{
    return *((uint64_t *)sess + CRYPTO_SESS_OPAQUE_DATA_OFF);
}

static inline void
rte_cryptodev_sym_session_opaque_data_set(void *sess, uint64_t opaque)
{
    uint64_t *data;
    data = (((uint64_t *)sess) + CRYPTO_SESS_OPAQUE_DATA_OFF);
    *data = opaque;
}

__rte_experimental
void *
rte_cryptodev_sym_session_get_user_data(void *sess);

__rte_experimental
int
rte_cryptodev_asym_session_set_user_data(void *sess, void *data, uint16_t size);

__rte_experimental
void *
rte_cryptodev_asym_session_get_user_data(void *sess);

__rte_experimental
uint32_t
rte_cryptodev_sym_cpu_crypto_process(uint8_t dev_id,
    void *sess, union rte_crypto_sym_ofs ofs,
    struct rte_crypto_sym_vec *vec);

__rte_experimental
int
rte_cryptodev_get_raw_dp_ctx_size(uint8_t dev_id);

__rte_experimental
int
rte_cryptodev_session_event_mdata_set(uint8_t dev_id, void *sess,
    enum rte_crypto_op_type op_type,
    enum rte_crypto_op_sess_type sess_type,
    void *ev_mdata, uint16_t size);

union rte_cryptodev_session_ctx {void *crypto_sess;
    struct rte_crypto_sym_xform *xform;
    struct rte_security_session *sec_sess;
};

typedef uint32_t (*cryptodev_sym_raw_enqueue_burst_t)(
    void *qp, uint8_t *drv_ctx, struct rte_crypto_sym_vec *vec,
    union rte_crypto_sym_ofs ofs, void *user_data[], int *enqueue_status);

typedef int (*cryptodev_sym_raw_enqueue_t)(
    void *qp, uint8_t *drv_ctx, struct rte_crypto_vec *data_vec,
    uint16_t n_data_vecs, union rte_crypto_sym_ofs ofs,
    struct rte_crypto_va_iova_ptr *iv,
    struct rte_crypto_va_iova_ptr *digest,
    struct rte_crypto_va_iova_ptr *aad_or_auth_iv,
    void *user_data);

typedef int (*cryptodev_sym_raw_operation_done_t)(void *qp, uint8_t *drv_ctx,
    uint32_t n);

typedef uint32_t (*rte_cryptodev_raw_get_dequeue_count_t)(void *user_data);

typedef void (*rte_cryptodev_raw_post_dequeue_t)(void *user_data,
    uint32_t index, uint8_t is_op_success);

typedef uint32_t (*cryptodev_sym_raw_dequeue_burst_t)(void *qp,
    uint8_t *drv_ctx,
    rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
    uint32_t max_nb_to_dequeue,
    rte_cryptodev_raw_post_dequeue_t post_dequeue,
    void **out_user_data, uint8_t is_user_data_array,
    uint32_t *n_success, int *dequeue_status);

typedef void * (*cryptodev_sym_raw_dequeue_t)(
        void *qp, uint8_t *drv_ctx, int *dequeue_status,
        enum rte_crypto_op_status *op_status);

struct rte_crypto_raw_dp_ctx {
    void *qp_data;

    cryptodev_sym_raw_enqueue_t enqueue;
    cryptodev_sym_raw_enqueue_burst_t enqueue_burst;
    cryptodev_sym_raw_operation_done_t enqueue_done;
    cryptodev_sym_raw_dequeue_t dequeue;
    cryptodev_sym_raw_dequeue_burst_t dequeue_burst;
    cryptodev_sym_raw_operation_done_t dequeue_done;

    /* Driver specific context data */
    __extension__ uint8_t drv_ctx_data[];
};

__rte_experimental
int
rte_cryptodev_configure_raw_dp_ctx(uint8_t dev_id, uint16_t qp_id,
    struct rte_crypto_raw_dp_ctx *ctx,
    enum rte_crypto_op_sess_type sess_type,
    union rte_cryptodev_session_ctx session_ctx,
    uint8_t is_update);

__rte_experimental
uint32_t
rte_cryptodev_raw_enqueue_burst(struct rte_crypto_raw_dp_ctx *ctx,
    struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
    void **user_data, int *enqueue_status);

__rte_experimental
static __rte_always_inline int
rte_cryptodev_raw_enqueue(struct rte_crypto_raw_dp_ctx *ctx,
    struct rte_crypto_vec *data_vec, uint16_t n_data_vecs,
    union rte_crypto_sym_ofs ofs,
    struct rte_crypto_va_iova_ptr *iv,
    struct rte_crypto_va_iova_ptr *digest,
    struct rte_crypto_va_iova_ptr *aad_or_auth_iv,
    void *user_data)
{
    return (*ctx->enqueue)(ctx->qp_data, ctx->drv_ctx_data, data_vec,
        n_data_vecs, ofs, iv, digest, aad_or_auth_iv, user_data);
}

__rte_experimental
int
rte_cryptodev_raw_enqueue_done(struct rte_crypto_raw_dp_ctx *ctx,
        uint32_t n);

__rte_experimental
uint32_t
rte_cryptodev_raw_dequeue_burst(struct rte_crypto_raw_dp_ctx *ctx,
    rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
    uint32_t max_nb_to_dequeue,
    rte_cryptodev_raw_post_dequeue_t post_dequeue,
    void **out_user_data, uint8_t is_user_data_array,
    uint32_t *n_success, int *dequeue_status);

__rte_experimental
static __rte_always_inline void *
rte_cryptodev_raw_dequeue(struct rte_crypto_raw_dp_ctx *ctx,
        int *dequeue_status, enum rte_crypto_op_status *op_status)
{
    return (*ctx->dequeue)(ctx->qp_data, ctx->drv_ctx_data, dequeue_status,
            op_status);
}

__rte_experimental
int
rte_cryptodev_raw_dequeue_done(struct rte_crypto_raw_dp_ctx *ctx,
        uint32_t n);

__rte_experimental
struct rte_cryptodev_cb *
rte_cryptodev_add_enq_callback(uint8_t dev_id,
                   uint16_t qp_id,
                   rte_cryptodev_callback_fn cb_fn,
                   void *cb_arg);

__rte_experimental
int rte_cryptodev_remove_enq_callback(uint8_t dev_id,
                      uint16_t qp_id,
                      struct rte_cryptodev_cb *cb);

__rte_experimental
struct rte_cryptodev_cb *
rte_cryptodev_add_deq_callback(uint8_t dev_id,
                   uint16_t qp_id,
                   rte_cryptodev_callback_fn cb_fn,
                   void *cb_arg);

__rte_experimental
int rte_cryptodev_remove_deq_callback(uint8_t dev_id,
                      uint16_t qp_id,
                      struct rte_cryptodev_cb *cb);

#include <rte_cryptodev_core.h>
static inline uint16_t
rte_cryptodev_dequeue_burst(uint8_t dev_id, uint16_t qp_id,
        struct rte_crypto_op **ops, uint16_t nb_ops)
{
    const struct rte_crypto_fp_ops *fp_ops;
    void *qp;

    rte_cryptodev_trace_dequeue_burst(dev_id, qp_id, (void **)ops, nb_ops);

    fp_ops = &rte_crypto_fp_ops[dev_id];
    qp = fp_ops->qp.data[qp_id];

    nb_ops = fp_ops->dequeue_burst(qp, ops, nb_ops);

#ifdef RTE_CRYPTO_CALLBACKS
    if (unlikely(fp_ops->qp.deq_cb != NULL)) {
        struct rte_cryptodev_cb_rcu *list;
        struct rte_cryptodev_cb *cb;

        /* __ATOMIC_RELEASE memory order was used when the
         * call back was inserted into the list.
         * Since there is a clear dependency between loading
         * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
         * not required.
         */
        list = &fp_ops->qp.deq_cb[qp_id];
        rte_rcu_qsbr_thread_online(list->qsbr, 0);
        cb = __atomic_load_n(&list->next, __ATOMIC_RELAXED);

        while (cb != NULL) {
            nb_ops = cb->fn(dev_id, qp_id, ops, nb_ops,
                    cb->arg);
            cb = cb->next;
        };

        rte_rcu_qsbr_thread_offline(list->qsbr, 0);
    }
#endif
    return nb_ops;
}

static inline uint16_t
rte_cryptodev_enqueue_burst(uint8_t dev_id, uint16_t qp_id,
        struct rte_crypto_op **ops, uint16_t nb_ops)
{
    const struct rte_crypto_fp_ops *fp_ops;
    void *qp;

    fp_ops = &rte_crypto_fp_ops[dev_id];
    qp = fp_ops->qp.data[qp_id];
#ifdef RTE_CRYPTO_CALLBACKS
    if (unlikely(fp_ops->qp.enq_cb != NULL)) {
        struct rte_cryptodev_cb_rcu *list;
        struct rte_cryptodev_cb *cb;

        /* __ATOMIC_RELEASE memory order was used when the
         * call back was inserted into the list.
         * Since there is a clear dependency between loading
         * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
         * not required.
         */
        list = &fp_ops->qp.enq_cb[qp_id];
        rte_rcu_qsbr_thread_online(list->qsbr, 0);
        cb = __atomic_load_n(&list->next, __ATOMIC_RELAXED);

        while (cb != NULL) {
            nb_ops = cb->fn(dev_id, qp_id, ops, nb_ops,
                    cb->arg);
            cb = cb->next;
        };

        rte_rcu_qsbr_thread_offline(list->qsbr, 0);
    }
#endif

    rte_cryptodev_trace_enqueue_burst(dev_id, qp_id, (void **)ops, nb_ops);
    return fp_ops->enqueue_burst(qp, ops, nb_ops);
}



#ifdef __cplusplus
}
#endif

#endif /* _RTE_CRYPTODEV_H_ */