rte_dmadev.h

Go to the documentation of this file.

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2021 HiSilicon Limited
 * Copyright(c) 2021 Intel Corporation
 * Copyright(c) 2021 Marvell International Ltd
 * Copyright(c) 2021 SmartShare Systems
 */

#ifndef RTE_DMADEV_H
#define RTE_DMADEV_H

#include <stdint.h>

#include <rte_bitops.h>
#include <rte_common.h>
#include <rte_compat.h>

#ifdef __cplusplus
extern "C" {
#endif

#define RTE_DMADEV_DEFAULT_MAX 64

__rte_experimental
int rte_dma_dev_max(size_t dev_max);

__rte_experimental
int rte_dma_get_dev_id_by_name(const char *name);

__rte_experimental
bool rte_dma_is_valid(int16_t dev_id);

__rte_experimental
uint16_t rte_dma_count_avail(void);

__rte_experimental
int16_t rte_dma_next_dev(int16_t start_dev_id);

#define RTE_DMA_FOREACH_DEV(p) \
    for (p = rte_dma_next_dev(0); \
         p != -1; \
         p = rte_dma_next_dev(p + 1))


#define RTE_DMA_CAPA_MEM_TO_MEM     RTE_BIT64(0)

#define RTE_DMA_CAPA_MEM_TO_DEV     RTE_BIT64(1)

#define RTE_DMA_CAPA_DEV_TO_MEM     RTE_BIT64(2)

#define RTE_DMA_CAPA_DEV_TO_DEV     RTE_BIT64(3)

#define RTE_DMA_CAPA_SVA                RTE_BIT64(4)

#define RTE_DMA_CAPA_SILENT             RTE_BIT64(5)

#define RTE_DMA_CAPA_HANDLES_ERRORS RTE_BIT64(6)

#define RTE_DMA_CAPA_OPS_COPY           RTE_BIT64(32)

#define RTE_DMA_CAPA_OPS_COPY_SG    RTE_BIT64(33)

#define RTE_DMA_CAPA_OPS_FILL       RTE_BIT64(34)

struct rte_dma_info {
    const char *dev_name; 
    uint64_t dev_capa;
    uint16_t max_vchans;
    uint16_t max_desc;
    uint16_t min_desc;
    uint16_t max_sges;
    int16_t numa_node;
    uint16_t nb_vchans;
};

__rte_experimental
int rte_dma_info_get(int16_t dev_id, struct rte_dma_info *dev_info);

struct rte_dma_conf {
    uint16_t nb_vchans;
    bool enable_silent;
};

__rte_experimental
int rte_dma_configure(int16_t dev_id, const struct rte_dma_conf *dev_conf);

__rte_experimental
int rte_dma_start(int16_t dev_id);

__rte_experimental
int rte_dma_stop(int16_t dev_id);

__rte_experimental
int rte_dma_close(int16_t dev_id);

enum rte_dma_direction {
    RTE_DMA_DIR_MEM_TO_MEM,
    RTE_DMA_DIR_MEM_TO_DEV,
    RTE_DMA_DIR_DEV_TO_MEM,
    RTE_DMA_DIR_DEV_TO_DEV,
};

enum rte_dma_port_type {
    RTE_DMA_PORT_NONE,
    RTE_DMA_PORT_PCIE, 
};

struct rte_dma_port_param {
    enum rte_dma_port_type port_type;
    RTE_STD_C11
    union {
        __extension__
        struct {
            uint64_t coreid : 4; 
            uint64_t pfid : 8; 
            uint64_t vfen : 1; 
            uint64_t vfid : 16; 
            uint64_t pasid : 20;
            uint64_t attr : 3;
            uint64_t ph : 2;
            uint64_t st : 16;
        } pcie;
    };
    uint64_t reserved[2]; 
};

struct rte_dma_vchan_conf {
    enum rte_dma_direction direction;
    uint16_t nb_desc;
    struct rte_dma_port_param src_port;
    struct rte_dma_port_param dst_port;
};

__rte_experimental
int rte_dma_vchan_setup(int16_t dev_id, uint16_t vchan,
            const struct rte_dma_vchan_conf *conf);

struct rte_dma_stats {
    uint64_t submitted;
    uint64_t completed;
    uint64_t errors;
};

#define RTE_DMA_ALL_VCHAN   0xFFFFu

__rte_experimental
int rte_dma_stats_get(int16_t dev_id, uint16_t vchan,
              struct rte_dma_stats *stats);

__rte_experimental
int rte_dma_stats_reset(int16_t dev_id, uint16_t vchan);

enum rte_dma_vchan_status {
    RTE_DMA_VCHAN_IDLE,          
    RTE_DMA_VCHAN_ACTIVE,        
    RTE_DMA_VCHAN_HALTED_ERROR,  
};

__rte_experimental
int
rte_dma_vchan_status(int16_t dev_id, uint16_t vchan, enum rte_dma_vchan_status *status);

__rte_experimental
int rte_dma_dump(int16_t dev_id, FILE *f);

enum rte_dma_status_code {
    RTE_DMA_STATUS_SUCCESSFUL,
    RTE_DMA_STATUS_USER_ABORT,
    RTE_DMA_STATUS_NOT_ATTEMPTED,
    RTE_DMA_STATUS_INVALID_SRC_ADDR,
    RTE_DMA_STATUS_INVALID_DST_ADDR,
    RTE_DMA_STATUS_INVALID_ADDR,
    RTE_DMA_STATUS_INVALID_LENGTH,
    RTE_DMA_STATUS_INVALID_OPCODE,
    RTE_DMA_STATUS_BUS_READ_ERROR,
    RTE_DMA_STATUS_BUS_WRITE_ERROR,
    RTE_DMA_STATUS_BUS_ERROR,
    RTE_DMA_STATUS_DATA_POISION,
    RTE_DMA_STATUS_DESCRIPTOR_READ_ERROR,
    RTE_DMA_STATUS_DEV_LINK_ERROR,
    RTE_DMA_STATUS_PAGE_FAULT,
    RTE_DMA_STATUS_ERROR_UNKNOWN = 0x100,
};

struct rte_dma_sge {
    rte_iova_t addr; 
    uint32_t length; 
};

#include "rte_dmadev_core.h"

#define RTE_DMA_OP_FLAG_FENCE   RTE_BIT64(0)

#define RTE_DMA_OP_FLAG_SUBMIT  RTE_BIT64(1)

#define RTE_DMA_OP_FLAG_LLC     RTE_BIT64(2)

__rte_experimental
static inline int
rte_dma_copy(int16_t dev_id, uint16_t vchan, rte_iova_t src, rte_iova_t dst,
         uint32_t length, uint64_t flags)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id) || length == 0)
        return -EINVAL;
    RTE_FUNC_PTR_OR_ERR_RET(*obj->copy, -ENOTSUP);
#endif

    return (*obj->copy)(obj->dev_private, vchan, src, dst, length, flags);
}

__rte_experimental
static inline int
rte_dma_copy_sg(int16_t dev_id, uint16_t vchan, struct rte_dma_sge *src,
        struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst,
        uint64_t flags)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id) || src == NULL || dst == NULL ||
        nb_src == 0 || nb_dst == 0)
        return -EINVAL;
    RTE_FUNC_PTR_OR_ERR_RET(*obj->copy_sg, -ENOTSUP);
#endif

    return (*obj->copy_sg)(obj->dev_private, vchan, src, dst, nb_src,
                   nb_dst, flags);
}

__rte_experimental
static inline int
rte_dma_fill(int16_t dev_id, uint16_t vchan, uint64_t pattern,
         rte_iova_t dst, uint32_t length, uint64_t flags)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id) || length == 0)
        return -EINVAL;
    RTE_FUNC_PTR_OR_ERR_RET(*obj->fill, -ENOTSUP);
#endif

    return (*obj->fill)(obj->dev_private, vchan, pattern, dst, length,
                flags);
}

__rte_experimental
static inline int
rte_dma_submit(int16_t dev_id, uint16_t vchan)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id))
        return -EINVAL;
    RTE_FUNC_PTR_OR_ERR_RET(*obj->submit, -ENOTSUP);
#endif

    return (*obj->submit)(obj->dev_private, vchan);
}

__rte_experimental
static inline uint16_t
rte_dma_completed(int16_t dev_id, uint16_t vchan, const uint16_t nb_cpls,
          uint16_t *last_idx, bool *has_error)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
    uint16_t idx;
    bool err;

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id) || nb_cpls == 0)
        return 0;
    RTE_FUNC_PTR_OR_ERR_RET(*obj->completed, 0);
#endif

    /* Ensure the pointer values are non-null to simplify drivers.
     * In most cases these should be compile time evaluated, since this is
     * an inline function.
     * - If NULL is explicitly passed as parameter, then compiler knows the
     *   value is NULL
     * - If address of local variable is passed as parameter, then compiler
     *   can know it's non-NULL.
     */
    if (last_idx == NULL)
        last_idx = &idx;
    if (has_error == NULL)
        has_error = &err;

    *has_error = false;
    return (*obj->completed)(obj->dev_private, vchan, nb_cpls, last_idx,
                 has_error);
}

__rte_experimental
static inline uint16_t
rte_dma_completed_status(int16_t dev_id, uint16_t vchan,
             const uint16_t nb_cpls, uint16_t *last_idx,
             enum rte_dma_status_code *status)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
    uint16_t idx;

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id) || nb_cpls == 0 || status == NULL)
        return 0;
    RTE_FUNC_PTR_OR_ERR_RET(*obj->completed_status, 0);
#endif

    if (last_idx == NULL)
        last_idx = &idx;

    return (*obj->completed_status)(obj->dev_private, vchan, nb_cpls,
                    last_idx, status);
}

__rte_experimental
static inline uint16_t
rte_dma_burst_capacity(int16_t dev_id, uint16_t vchan)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id))
        return 0;
    RTE_FUNC_PTR_OR_ERR_RET(*obj->burst_capacity, 0);
#endif
    return (*obj->burst_capacity)(obj->dev_private, vchan);
}

#ifdef __cplusplus
}
#endif

#endif /* RTE_DMADEV_H */