rte_graph_worker_common.h

Go to the documentation of this file.

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(C) 2020 Marvell International Ltd.
 */
 
#ifndef _RTE_GRAPH_WORKER_COMMON_H_
#define _RTE_GRAPH_WORKER_COMMON_H_
 
#include <assert.h>
#include <stdalign.h>
#include <stddef.h>
 
#include <rte_common.h>
#include <rte_cycles.h>
#include <rte_prefetch.h>
#include <rte_memcpy.h>
#include <rte_memory.h>
 
#include "rte_graph.h"
 
#ifdef __cplusplus
extern "C" {
#endif
 
/* When adding a new graph model entry, update rte_graph_model_is_valid() implementation. */
#define RTE_GRAPH_MODEL_RTC 0 
#define RTE_GRAPH_MODEL_MCORE_DISPATCH 1
#define RTE_GRAPH_MODEL_DEFAULT RTE_GRAPH_MODEL_RTC 
SLIST_HEAD(rte_graph_rq_head, rte_graph);
 
struct __rte_cache_aligned rte_graph {
    /* Fast path area. */
    rte_node_t nb_nodes;         
    rte_graph_off_t nodes_start; 
    rte_graph_off_t *sched_table; 
    uint64_t *pending;       
    uint64_t *src_pending;       
    uint16_t nb_sched_words;     
    uint8_t model;           
    /* 26 bytes padding */
    union {
        /* Fast schedule area for mcore dispatch model */
        struct {
            alignas(RTE_CACHE_LINE_SIZE) struct rte_graph_rq_head *rq;
                /* The run-queue */
            struct rte_graph_rq_head rq_head; /* The head for run-queue list */
 
            unsigned int lcore_id;  
            struct rte_ring *wq;    
            struct rte_mempool *mp; 
            packets_enqueued_cb notify_cb; 
            uint64_t cb_priv;       
        } dispatch; 
    };
    SLIST_ENTRY(rte_graph) next;   /* The next for rte_graph list */
    /* End of Fast path area.*/
    rte_graph_t id; 
    int socket; 
    char name[RTE_GRAPH_NAMESIZE];  
    bool pcap_enable;           
    uint64_t nb_pkt_captured;
    uint64_t nb_pkt_to_capture;
    char pcap_filename[RTE_GRAPH_PCAP_FILE_SZ];  
    uint64_t fence;         
};
 
struct __rte_cache_aligned rte_node {
    /* Slow path area  */
    uint64_t fence;     
    rte_graph_off_t next;   
    rte_node_t id;      
    rte_node_t parent_id;   
    rte_edge_t nb_edges;    
    uint16_t sched_idx; 
    uint32_t realloc_count; 
    char parent[RTE_NODE_NAMESIZE]; 
    char name[RTE_NODE_NAMESIZE];   
    rte_node_process_t original_process;
 
    union {
        alignas(RTE_CACHE_LINE_MIN_SIZE) struct {
            unsigned int lcore_id;  
            uint64_t total_sched_objs; 
            uint64_t total_sched_fail; 
            struct rte_graph *graph;  
        } dispatch;
    };
 
    alignas(RTE_CACHE_LINE_MIN_SIZE)
    rte_graph_off_t xstat_off; 
#ifdef RTE_GRAPH_PROFILE
    alignas(RTE_CACHE_LINE_MIN_SIZE)
    struct {
        uint64_t calls;     
        uint64_t cycles;    
    } usage_stats_01[2];
    struct {
        uint64_t calls;     
        uint64_t cycles;    
        uint64_t objs;      
    } usage_stats[4];
#endif
    __extension__ struct __rte_cache_aligned {
#define RTE_NODE_CTX_SZ 16
        union {
            uint8_t ctx[RTE_NODE_CTX_SZ];
            __extension__ struct {
                void *ctx_ptr;
                void *ctx_ptr2;
            };
        }; 
        uint16_t size;      
        uint16_t idx;       
        rte_graph_off_t off;    
        uint64_t total_cycles;  
        uint64_t total_calls;   
        uint64_t total_objs;    
        union {
            void **objs;       
            uint64_t objs_u64;
        };
        union {
            rte_node_process_t process; 
            uint64_t process_u64;
        };
        alignas(RTE_CACHE_LINE_MIN_SIZE) struct rte_node *nodes[]; 
    };
};
 
#ifndef RTE_GRAPH_PROFILE
static_assert(offsetof(struct rte_node, nodes) - offsetof(struct rte_node, ctx)
    == RTE_CACHE_LINE_MIN_SIZE, "rte_node fast path area must fit in 64 bytes");
#endif
 
void __rte_node_stream_alloc(struct rte_graph *graph, struct rte_node *node);
 
void __rte_node_stream_alloc_size(struct rte_graph *graph,
                  struct rte_node *node, uint16_t req_size);
 
/* Fast path helper functions */
 
static __rte_always_inline void
__rte_node_process(struct rte_graph *graph, struct rte_node *node)
{
    uint64_t cycles;
    uint16_t rc;
    void **objs;
 
    RTE_ASSERT(node->fence == RTE_GRAPH_FENCE);
    objs = node->objs;
    rte_prefetch0(objs);
 
    if (rte_graph_has_stats_feature()) {
        cycles = rte_rdtsc();
        rc = node->process(graph, node, objs, node->idx);
        cycles = rte_rdtsc() - cycles;
        node->total_cycles += cycles;
        node->total_calls++;
        node->total_objs += rc;
#ifdef RTE_GRAPH_PROFILE
        static_assert(RTE_GRAPH_PROFILE_BURST_SMALL >= 2,
            "BURST_SMALL must be >= 2");
        static_assert(RTE_GRAPH_PROFILE_BURST_SMALL < RTE_GRAPH_PROFILE_BURST_MEDIUM,
            "BURST_SMALL must be < BURST_MEDIUM");
        static_assert(RTE_GRAPH_PROFILE_BURST_MEDIUM < RTE_GRAPH_PROFILE_BURST_LARGE,
            "BURST_MEDIUM must be < BURST_LARGE");
        if (rc < 2) {
            node->usage_stats_01[rc].calls++;
            node->usage_stats_01[rc].cycles += cycles;
        } else {
            unsigned int index;
 
            if (rc >= RTE_GRAPH_PROFILE_BURST_LARGE)
                index = 3;
            else if (rc >= RTE_GRAPH_PROFILE_BURST_MEDIUM)
                index = 2;
            else if (rc >= RTE_GRAPH_PROFILE_BURST_SMALL)
                index = 1;
            else
                index = 0;
 
            node->usage_stats[index].calls++;
            node->usage_stats[index].cycles += cycles;
            node->usage_stats[index].objs += rc;
        }
#endif
    } else {
        node->process(graph, node, objs, node->idx);
    }
    node->idx = 0;
}
 
static inline struct rte_node *
__rte_graph_pending_node(struct rte_graph *graph, uint16_t word, uint16_t bit)
{
    const uint16_t index = (word * sizeof(*graph->pending) * CHAR_BIT) + bit;
    const rte_graph_off_t node_offset = graph->sched_table[index];
    return (struct rte_node *)RTE_PTR_ADD(graph, node_offset);
}
 
static inline void
__rte_node_pending_set(uint64_t *bitmap, struct rte_node *node)
{
    const uint16_t word = node->sched_idx / (sizeof(*bitmap) * CHAR_BIT);
    const uint16_t bit = node->sched_idx % (sizeof(*bitmap) * CHAR_BIT);
    bitmap[word] |= 1ULL << bit;
}
 
static __rte_always_inline void
__rte_node_enqueue_prologue(struct rte_graph *graph, struct rte_node *node,
                const uint16_t idx, const uint16_t space)
{
 
    if (idx == 0)
        __rte_node_pending_set(graph->pending, node);
 
    if (unlikely(node->size < (idx + space)))
        __rte_node_stream_alloc_size(graph, node, node->size + space);
}
 
static __rte_always_inline struct rte_node *
__rte_node_next_node_get(struct rte_node *node, rte_edge_t next)
{
    RTE_ASSERT(next < node->nb_edges);
    RTE_ASSERT(node->fence == RTE_GRAPH_FENCE);
    node = node->nodes[next];
    RTE_ASSERT(node->fence == RTE_GRAPH_FENCE);
 
    return node;
}
 
static inline void
rte_node_enqueue(struct rte_graph *graph, struct rte_node *node,
         rte_edge_t next, void **objs, uint16_t nb_objs)
{
    node = __rte_node_next_node_get(node, next);
    const uint16_t idx = node->idx;
 
    __rte_node_enqueue_prologue(graph, node, idx, nb_objs);
 
    rte_memcpy(&node->objs[idx], objs, nb_objs * sizeof(void *));
    node->idx = idx + nb_objs;
}
 
static inline void
rte_node_enqueue_x1(struct rte_graph *graph, struct rte_node *node,
            rte_edge_t next, void *obj)
{
    node = __rte_node_next_node_get(node, next);
    uint16_t idx = node->idx;
 
    __rte_node_enqueue_prologue(graph, node, idx, 1);
 
    node->objs[idx++] = obj;
    node->idx = idx;
}
 
static inline void
rte_node_enqueue_x2(struct rte_graph *graph, struct rte_node *node,
            rte_edge_t next, void *obj0, void *obj1)
{
    node = __rte_node_next_node_get(node, next);
    uint16_t idx = node->idx;
 
    __rte_node_enqueue_prologue(graph, node, idx, 2);
 
    node->objs[idx++] = obj0;
    node->objs[idx++] = obj1;
    node->idx = idx;
}
 
static inline void
rte_node_enqueue_x4(struct rte_graph *graph, struct rte_node *node,
            rte_edge_t next, void *obj0, void *obj1, void *obj2,
            void *obj3)
{
    node = __rte_node_next_node_get(node, next);
    uint16_t idx = node->idx;
 
    __rte_node_enqueue_prologue(graph, node, idx, 4);
 
    node->objs[idx++] = obj0;
    node->objs[idx++] = obj1;
    node->objs[idx++] = obj2;
    node->objs[idx++] = obj3;
    node->idx = idx;
}
 
static inline void
rte_node_enqueue_next(struct rte_graph *graph, struct rte_node *node,
              rte_edge_t *nexts, void **objs, uint16_t nb_objs)
{
    uint16_t i;
 
    for (i = 0; i < nb_objs; i++)
        rte_node_enqueue_x1(graph, node, nexts[i], objs[i]);
}
 
static inline void **
rte_node_next_stream_get(struct rte_graph *graph, struct rte_node *node,
             rte_edge_t next, uint16_t nb_objs)
{
    node = __rte_node_next_node_get(node, next);
    const uint16_t idx = node->idx;
    uint16_t free_space = node->size - idx;
 
    if (unlikely(free_space < nb_objs))
        __rte_node_stream_alloc_size(graph, node, node->size + nb_objs);
 
    return &node->objs[idx];
}
 
static inline void
rte_node_next_stream_put(struct rte_graph *graph, struct rte_node *node,
             rte_edge_t next, uint16_t idx)
{
    if (unlikely(!idx))
        return;
 
    node = __rte_node_next_node_get(node, next);
    if (node->idx == 0)
        __rte_node_pending_set(graph->pending, node);
 
    node->idx += idx;
}
 
static inline void
rte_node_next_stream_move(struct rte_graph *graph, struct rte_node *src,
              rte_edge_t next)
{
    struct rte_node *dst = __rte_node_next_node_get(src, next);
 
    /* Let swap the pointers if dst don't have valid objs */
    if (likely(dst->idx == 0)) {
        void **dobjs = dst->objs;
        uint16_t dsz = dst->size;
        dst->objs = src->objs;
        dst->size = src->size;
        src->objs = dobjs;
        src->size = dsz;
        dst->idx = src->idx;
        __rte_node_pending_set(graph->pending, dst);
    } else { /* Move the objects from src node to dst node */
        rte_node_enqueue(graph, src, next, src->objs, src->idx);
    }
}
 
bool
rte_graph_model_is_valid(uint8_t model);
 
int rte_graph_worker_model_set(uint8_t model);
 
uint8_t rte_graph_worker_model_get(struct rte_graph *graph);
 
static __rte_always_inline
uint8_t rte_graph_worker_model_no_check_get(struct rte_graph *graph)
{
    return graph->model;
}
 
__rte_experimental
static inline void
rte_node_xstat_increment(struct rte_node *node, uint16_t xstat_id, uint64_t value)
{
    if (rte_graph_has_stats_feature()) {
        uint64_t *xstat = (uint64_t *)RTE_PTR_ADD(node, node->xstat_off);
        xstat[xstat_id] += value;
    }
}
 
#ifdef __cplusplus
}
#endif
 
#endif /* _RTE_GRAPH_WORKER_COIMMON_H_ */