rte_dmadev.h

Go to the documentation of this file.

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2021 HiSilicon Limited
 * Copyright(c) 2021 Intel Corporation
 * Copyright(c) 2021 Marvell International Ltd
 * Copyright(c) 2021 SmartShare Systems
 */

#ifndef RTE_DMADEV_H
#define RTE_DMADEV_H

#include <stdint.h>

#include <rte_bitops.h>
#include <rte_common.h>

#ifdef __cplusplus
extern "C" {
#endif

#define RTE_DMADEV_DEFAULT_MAX 64

int rte_dma_dev_max(size_t dev_max);

int rte_dma_get_dev_id_by_name(const char *name);

bool rte_dma_is_valid(int16_t dev_id);

uint16_t rte_dma_count_avail(void);

int16_t rte_dma_next_dev(int16_t start_dev_id);

#define RTE_DMA_FOREACH_DEV(p) \
    for (p = rte_dma_next_dev(0); \
         p != -1; \
         p = rte_dma_next_dev(p + 1))


#define RTE_DMA_CAPA_MEM_TO_MEM     RTE_BIT64(0)

#define RTE_DMA_CAPA_MEM_TO_DEV     RTE_BIT64(1)

#define RTE_DMA_CAPA_DEV_TO_MEM     RTE_BIT64(2)

#define RTE_DMA_CAPA_DEV_TO_DEV     RTE_BIT64(3)

#define RTE_DMA_CAPA_SVA                RTE_BIT64(4)

#define RTE_DMA_CAPA_SILENT             RTE_BIT64(5)

#define RTE_DMA_CAPA_HANDLES_ERRORS RTE_BIT64(6)

#define RTE_DMA_CAPA_M2D_AUTO_FREE      RTE_BIT64(7)

#define RTE_DMA_CAPA_PRI_POLICY_SP  RTE_BIT64(8)

#define RTE_DMA_CAPA_OPS_COPY           RTE_BIT64(32)

#define RTE_DMA_CAPA_OPS_COPY_SG    RTE_BIT64(33)

#define RTE_DMA_CAPA_OPS_FILL       RTE_BIT64(34)

struct rte_dma_info {
    const char *dev_name; 
    uint64_t dev_capa;
    uint16_t max_vchans;
    uint16_t max_desc;
    uint16_t min_desc;
    uint16_t max_sges;
    int16_t numa_node;
    uint16_t nb_vchans;
    uint16_t nb_priorities;
};

int rte_dma_info_get(int16_t dev_id, struct rte_dma_info *dev_info);

struct rte_dma_conf {
    uint16_t nb_vchans;
    bool enable_silent;
    /* The priority of the DMA device.
     * This value should be lower than the field 'nb_priorities' of struct
     * rte_dma_info which get from rte_dma_info_get(). If the DMA device
     * does not support priority scheduling, this value should be zero.
     *
     * Lowest value indicates higher priority and vice-versa.
     */
    uint16_t priority;
};

int rte_dma_configure(int16_t dev_id, const struct rte_dma_conf *dev_conf);

int rte_dma_start(int16_t dev_id);

int rte_dma_stop(int16_t dev_id);

int rte_dma_close(int16_t dev_id);

enum rte_dma_direction {
    RTE_DMA_DIR_MEM_TO_MEM,
    RTE_DMA_DIR_MEM_TO_DEV,
    RTE_DMA_DIR_DEV_TO_MEM,
    RTE_DMA_DIR_DEV_TO_DEV,
};

enum rte_dma_port_type {
    RTE_DMA_PORT_NONE,
    RTE_DMA_PORT_PCIE, 
};

struct rte_dma_port_param {
    enum rte_dma_port_type port_type;
    union {
        __extension__
        struct {
            uint64_t coreid : 4; 
            uint64_t pfid : 8; 
            uint64_t vfen : 1; 
            uint64_t vfid : 16; 
            uint64_t pasid : 20;
            uint64_t attr : 3;
            uint64_t ph : 2;
            uint64_t st : 16;
        } pcie;
    };
    uint64_t reserved[2]; 
};

struct rte_dma_auto_free_param {
    union {
        struct {
            struct rte_mempool *pool;
        } m2d;
    };
    uint64_t reserved[2];
};

struct rte_dma_vchan_conf {
    enum rte_dma_direction direction;
    uint16_t nb_desc;
    struct rte_dma_port_param src_port;
    struct rte_dma_port_param dst_port;
    struct rte_dma_auto_free_param auto_free;
};

int rte_dma_vchan_setup(int16_t dev_id, uint16_t vchan,
            const struct rte_dma_vchan_conf *conf);

struct rte_dma_stats {
    uint64_t submitted;
    uint64_t completed;
    uint64_t errors;
};

#define RTE_DMA_ALL_VCHAN   0xFFFFu

int rte_dma_stats_get(int16_t dev_id, uint16_t vchan,
              struct rte_dma_stats *stats);

int rte_dma_stats_reset(int16_t dev_id, uint16_t vchan);

enum rte_dma_vchan_status {
    RTE_DMA_VCHAN_IDLE,          
    RTE_DMA_VCHAN_ACTIVE,        
    RTE_DMA_VCHAN_HALTED_ERROR,  
};

int
rte_dma_vchan_status(int16_t dev_id, uint16_t vchan, enum rte_dma_vchan_status *status);

int rte_dma_dump(int16_t dev_id, FILE *f);

enum rte_dma_status_code {
    RTE_DMA_STATUS_SUCCESSFUL,
    RTE_DMA_STATUS_USER_ABORT,
    RTE_DMA_STATUS_NOT_ATTEMPTED,
    RTE_DMA_STATUS_INVALID_SRC_ADDR,
    RTE_DMA_STATUS_INVALID_DST_ADDR,
    RTE_DMA_STATUS_INVALID_ADDR,
    RTE_DMA_STATUS_INVALID_LENGTH,
    RTE_DMA_STATUS_INVALID_OPCODE,
    RTE_DMA_STATUS_BUS_READ_ERROR,
    RTE_DMA_STATUS_BUS_WRITE_ERROR,
    RTE_DMA_STATUS_BUS_ERROR,
    RTE_DMA_STATUS_DATA_POISION,
    RTE_DMA_STATUS_DESCRIPTOR_READ_ERROR,
    RTE_DMA_STATUS_DEV_LINK_ERROR,
    RTE_DMA_STATUS_PAGE_FAULT,
    RTE_DMA_STATUS_ERROR_UNKNOWN = 0x100,
};

struct rte_dma_sge {
    rte_iova_t addr; 
    uint32_t length; 
};

#ifdef __cplusplus
}
#endif

#include "rte_dmadev_core.h"
#include "rte_dmadev_trace_fp.h"

#ifdef __cplusplus
extern "C" {
#endif

#define RTE_DMA_OP_FLAG_FENCE   RTE_BIT64(0)

#define RTE_DMA_OP_FLAG_SUBMIT  RTE_BIT64(1)

#define RTE_DMA_OP_FLAG_LLC     RTE_BIT64(2)

#define RTE_DMA_OP_FLAG_AUTO_FREE   RTE_BIT64(3)

static inline int
rte_dma_copy(int16_t dev_id, uint16_t vchan, rte_iova_t src, rte_iova_t dst,
         uint32_t length, uint64_t flags)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
    int ret;

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id) || length == 0)
        return -EINVAL;
    if (*obj->copy == NULL)
        return -ENOTSUP;
#endif

    ret = (*obj->copy)(obj->dev_private, vchan, src, dst, length, flags);
    rte_dma_trace_copy(dev_id, vchan, src, dst, length, flags, ret);

    return ret;
}

static inline int
rte_dma_copy_sg(int16_t dev_id, uint16_t vchan, struct rte_dma_sge *src,
        struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst,
        uint64_t flags)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
    int ret;

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id) || src == NULL || dst == NULL ||
        nb_src == 0 || nb_dst == 0)
        return -EINVAL;
    if (*obj->copy_sg == NULL)
        return -ENOTSUP;
#endif

    ret = (*obj->copy_sg)(obj->dev_private, vchan, src, dst, nb_src,
                  nb_dst, flags);
    rte_dma_trace_copy_sg(dev_id, vchan, src, dst, nb_src, nb_dst, flags,
                  ret);

    return ret;
}

static inline int
rte_dma_fill(int16_t dev_id, uint16_t vchan, uint64_t pattern,
         rte_iova_t dst, uint32_t length, uint64_t flags)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
    int ret;

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id) || length == 0)
        return -EINVAL;
    if (*obj->fill == NULL)
        return -ENOTSUP;
#endif

    ret = (*obj->fill)(obj->dev_private, vchan, pattern, dst, length,
               flags);
    rte_dma_trace_fill(dev_id, vchan, pattern, dst, length, flags, ret);

    return ret;
}

static inline int
rte_dma_submit(int16_t dev_id, uint16_t vchan)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
    int ret;

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id))
        return -EINVAL;
    if (*obj->submit == NULL)
        return -ENOTSUP;
#endif

    ret = (*obj->submit)(obj->dev_private, vchan);
    rte_dma_trace_submit(dev_id, vchan, ret);

    return ret;
}

static inline uint16_t
rte_dma_completed(int16_t dev_id, uint16_t vchan, const uint16_t nb_cpls,
          uint16_t *last_idx, bool *has_error)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
    uint16_t idx, ret;
    bool err;

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id) || nb_cpls == 0)
        return 0;
    if (*obj->completed == NULL)
        return 0;
#endif

    /* Ensure the pointer values are non-null to simplify drivers.
     * In most cases these should be compile time evaluated, since this is
     * an inline function.
     * - If NULL is explicitly passed as parameter, then compiler knows the
     *   value is NULL
     * - If address of local variable is passed as parameter, then compiler
     *   can know it's non-NULL.
     */
    if (last_idx == NULL)
        last_idx = &idx;
    if (has_error == NULL)
        has_error = &err;

    *has_error = false;
    ret = (*obj->completed)(obj->dev_private, vchan, nb_cpls, last_idx,
                has_error);
    rte_dma_trace_completed(dev_id, vchan, nb_cpls, last_idx, has_error,
                ret);

    return ret;
}

static inline uint16_t
rte_dma_completed_status(int16_t dev_id, uint16_t vchan,
             const uint16_t nb_cpls, uint16_t *last_idx,
             enum rte_dma_status_code *status)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
    uint16_t idx, ret;

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id) || nb_cpls == 0 || status == NULL)
        return 0;
    if (*obj->completed_status == NULL)
        return 0;
#endif

    if (last_idx == NULL)
        last_idx = &idx;

    ret = (*obj->completed_status)(obj->dev_private, vchan, nb_cpls,
                       last_idx, status);
    rte_dma_trace_completed_status(dev_id, vchan, nb_cpls, last_idx, status,
                       ret);

    return ret;
}

static inline uint16_t
rte_dma_burst_capacity(int16_t dev_id, uint16_t vchan)
{
    struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
    uint16_t ret;

#ifdef RTE_DMADEV_DEBUG
    if (!rte_dma_is_valid(dev_id))
        return 0;
    if (*obj->burst_capacity == NULL)
        return 0;
#endif
    ret = (*obj->burst_capacity)(obj->dev_private, vchan);
    rte_dma_trace_burst_capacity(dev_id, vchan, ret);

    return ret;
}

#ifdef __cplusplus
}
#endif

#endif /* RTE_DMADEV_H */