rte_mempool.h

Go to the documentation of this file.

/*-
 *   BSD LICENSE
 *
 *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
 *   All rights reserved.
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef _RTE_MEMPOOL_H_
#define _RTE_MEMPOOL_H_

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <errno.h>
#include <inttypes.h>
#include <sys/queue.h>

#include <rte_log.h>
#include <rte_debug.h>
#include <rte_lcore.h>
#include <rte_memory.h>
#include <rte_branch_prediction.h>
#include <rte_ring.h>

#ifdef __cplusplus
extern "C" {
#endif

#define RTE_MEMPOOL_HEADER_COOKIE1  0xbadbadbadadd2e55ULL 
#define RTE_MEMPOOL_HEADER_COOKIE2  0xf2eef2eedadd2e55ULL 
#define RTE_MEMPOOL_TRAILER_COOKIE  0xadd2e55badbadbadULL 
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG

struct rte_mempool_debug_stats {
        uint64_t put_bulk;         
        uint64_t put_objs;         
        uint64_t get_success_bulk; 
        uint64_t get_success_objs; 
        uint64_t get_fail_bulk;    
        uint64_t get_fail_objs;    
} __rte_cache_aligned;
#endif

#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0

struct rte_mempool_cache {
        unsigned len; 
        /*
         * Cache is allocated to this size to allow it to overflow in certain
         * cases to avoid needless emptying of cache.
         */
        void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 3]; 
} __rte_cache_aligned;
#endif /* RTE_MEMPOOL_CACHE_MAX_SIZE > 0 */

struct rte_mempool_objsz {
        uint32_t elt_size;     
        uint32_t header_size;  
        uint32_t trailer_size; 
        uint32_t total_size;
};

#define RTE_MEMPOOL_NAMESIZE 32 
#define RTE_MEMPOOL_MZ_PREFIX "MP_"

/* "MP_<name>" */
#define RTE_MEMPOOL_MZ_FORMAT   RTE_MEMPOOL_MZ_PREFIX "%s"

#ifdef RTE_LIBRTE_XEN_DOM0

/* "<name>_MP_elt" */
#define RTE_MEMPOOL_OBJ_NAME    "%s_" RTE_MEMPOOL_MZ_PREFIX "elt"

#else

#define RTE_MEMPOOL_OBJ_NAME    RTE_MEMPOOL_MZ_FORMAT

#endif /* RTE_LIBRTE_XEN_DOM0 */

#define MEMPOOL_PG_SHIFT_MAX    (sizeof(uintptr_t) * CHAR_BIT - 1)

#define MEMPOOL_PG_NUM_DEFAULT  1

#ifndef RTE_MEMPOOL_ALIGN
#define RTE_MEMPOOL_ALIGN       RTE_CACHE_LINE_SIZE
#endif

#define RTE_MEMPOOL_ALIGN_MASK  (RTE_MEMPOOL_ALIGN - 1)

struct rte_mempool_objhdr {
        struct rte_mempool *mp;          
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
        uint64_t cookie;                 
#endif
};

struct rte_mempool_objtlr {
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
        uint64_t cookie;                 
#endif
};

struct rte_mempool {
        char name[RTE_MEMPOOL_NAMESIZE]; 
        struct rte_ring *ring;           
        phys_addr_t phys_addr;           
        int flags;                       
        uint32_t size;                   
        uint32_t cache_size;             
        uint32_t cache_flushthresh;
        uint32_t elt_size;               
        uint32_t header_size;            
        uint32_t trailer_size;           
        unsigned private_data_size;      
#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0

        struct rte_mempool_cache local_cache[RTE_MAX_LCORE];
#endif

#ifdef RTE_LIBRTE_MEMPOOL_DEBUG

        struct rte_mempool_debug_stats stats[RTE_MAX_LCORE];
#endif

        /* Address translation support, starts from next cache line. */

        uint32_t    pg_num __rte_cache_aligned;
        uint32_t    pg_shift;     
        uintptr_t   pg_mask;      
        uintptr_t   elt_va_start;
        uintptr_t   elt_va_end;
        phys_addr_t elt_pa[MEMPOOL_PG_NUM_DEFAULT];
}  __rte_cache_aligned;

#define MEMPOOL_F_NO_SPREAD      0x0001 
#define MEMPOOL_F_NO_CACHE_ALIGN 0x0002 
#define MEMPOOL_F_SP_PUT         0x0004 
#define MEMPOOL_F_SC_GET         0x0008 
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
#define __MEMPOOL_STAT_ADD(mp, name, n) do {                    \
                unsigned __lcore_id = rte_lcore_id();           \
                if (__lcore_id < RTE_MAX_LCORE) {               \
                        mp->stats[__lcore_id].name##_objs += n; \
                        mp->stats[__lcore_id].name##_bulk += 1; \
                }                                               \
        } while(0)
#else
#define __MEMPOOL_STAT_ADD(mp, name, n) do {} while(0)
#endif

#define MEMPOOL_HEADER_SIZE(mp, pgn)    (sizeof(*(mp)) + \
        RTE_ALIGN_CEIL(((pgn) - RTE_DIM((mp)->elt_pa)) * \
        sizeof ((mp)->elt_pa[0]), RTE_CACHE_LINE_SIZE))

#define MEMPOOL_IS_CONTIG(mp)                      \
        ((mp)->pg_num == MEMPOOL_PG_NUM_DEFAULT && \
        (mp)->phys_addr == (mp)->elt_pa[0])

/* return the header of a mempool object (internal) */
static inline struct rte_mempool_objhdr *__mempool_get_header(void *obj)
{
        return (struct rte_mempool_objhdr *)RTE_PTR_SUB(obj, sizeof(struct rte_mempool_objhdr));
}

static inline struct rte_mempool *rte_mempool_from_obj(void *obj)
{
        struct rte_mempool_objhdr *hdr = __mempool_get_header(obj);
        return hdr->mp;
}

/* return the trailer of a mempool object (internal) */
static inline struct rte_mempool_objtlr *__mempool_get_trailer(void *obj)
{
        struct rte_mempool *mp = rte_mempool_from_obj(obj);
        return (struct rte_mempool_objtlr *)RTE_PTR_ADD(obj, mp->elt_size);
}

#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
#ifndef __INTEL_COMPILER
#pragma GCC diagnostic ignored "-Wcast-qual"
#endif
static inline void __mempool_check_cookies(const struct rte_mempool *mp,
                                           void * const *obj_table_const,
                                           unsigned n, int free)
{
        struct rte_mempool_objhdr *hdr;
        struct rte_mempool_objtlr *tlr;
        uint64_t cookie;
        void *tmp;
        void *obj;
        void **obj_table;

        /* Force to drop the "const" attribute. This is done only when
         * DEBUG is enabled */
        tmp = (void *) obj_table_const;
        obj_table = (void **) tmp;

        while (n--) {
                obj = obj_table[n];

                if (rte_mempool_from_obj(obj) != mp)
                        rte_panic("MEMPOOL: object is owned by another "
                                  "mempool\n");

                hdr = __mempool_get_header(obj);
                cookie = hdr->cookie;

                if (free == 0) {
                        if (cookie != RTE_MEMPOOL_HEADER_COOKIE1) {
                                rte_log_set_history(0);
                                RTE_LOG(CRIT, MEMPOOL,
                                        "obj=%p, mempool=%p, cookie=%" PRIx64 "\n",
                                        obj, (const void *) mp, cookie);
                                rte_panic("MEMPOOL: bad header cookie (put)\n");
                        }
                        hdr->cookie = RTE_MEMPOOL_HEADER_COOKIE2;
                }
                else if (free == 1) {
                        if (cookie != RTE_MEMPOOL_HEADER_COOKIE2) {
                                rte_log_set_history(0);
                                RTE_LOG(CRIT, MEMPOOL,
                                        "obj=%p, mempool=%p, cookie=%" PRIx64 "\n",
                                        obj, (const void *) mp, cookie);
                                rte_panic("MEMPOOL: bad header cookie (get)\n");
                        }
                        hdr->cookie = RTE_MEMPOOL_HEADER_COOKIE1;
                }
                else if (free == 2) {
                        if (cookie != RTE_MEMPOOL_HEADER_COOKIE1 &&
                            cookie != RTE_MEMPOOL_HEADER_COOKIE2) {
                                rte_log_set_history(0);
                                RTE_LOG(CRIT, MEMPOOL,
                                        "obj=%p, mempool=%p, cookie=%" PRIx64 "\n",
                                        obj, (const void *) mp, cookie);
                                rte_panic("MEMPOOL: bad header cookie (audit)\n");
                        }
                }
                tlr = __mempool_get_trailer(obj);
                cookie = tlr->cookie;
                if (cookie != RTE_MEMPOOL_TRAILER_COOKIE) {
                        rte_log_set_history(0);
                        RTE_LOG(CRIT, MEMPOOL,
                                "obj=%p, mempool=%p, cookie=%" PRIx64 "\n",
                                obj, (const void *) mp, cookie);
                        rte_panic("MEMPOOL: bad trailer cookie\n");
                }
        }
}
#ifndef __INTEL_COMPILER
#pragma GCC diagnostic error "-Wcast-qual"
#endif
#else
#define __mempool_check_cookies(mp, obj_table_const, n, free) do {} while(0)
#endif /* RTE_LIBRTE_MEMPOOL_DEBUG */

typedef void (*rte_mempool_obj_iter_t)(void * /*obj_iter_arg*/,
        void * /*obj_start*/,
        void * /*obj_end*/,
        uint32_t /*obj_index */);

uint32_t rte_mempool_obj_iter(void *vaddr,
        uint32_t elt_num, size_t elt_sz, size_t align,
        const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift,
        rte_mempool_obj_iter_t obj_iter, void *obj_iter_arg);

typedef void (rte_mempool_obj_ctor_t)(struct rte_mempool *, void *,
                                      void *, unsigned);

typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *);

struct rte_mempool *
rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
                   unsigned cache_size, unsigned private_data_size,
                   rte_mempool_ctor_t *mp_init, void *mp_init_arg,
                   rte_mempool_obj_ctor_t *obj_init, void *obj_init_arg,
                   int socket_id, unsigned flags);

struct rte_mempool *
rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size,
                unsigned cache_size, unsigned private_data_size,
                rte_mempool_ctor_t *mp_init, void *mp_init_arg,
                rte_mempool_obj_ctor_t *obj_init, void *obj_init_arg,
                int socket_id, unsigned flags, void *vaddr,
                const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift);

struct rte_mempool *
rte_dom0_mempool_create(const char *name, unsigned n, unsigned elt_size,
                unsigned cache_size, unsigned private_data_size,
                rte_mempool_ctor_t *mp_init, void *mp_init_arg,
                rte_mempool_obj_ctor_t *obj_init, void *obj_init_arg,
                int socket_id, unsigned flags);


void rte_mempool_dump(FILE *f, const struct rte_mempool *mp);

static inline void __attribute__((always_inline))
__mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,
                    unsigned n, int is_mp)
{
#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0
        struct rte_mempool_cache *cache;
        uint32_t index;
        void **cache_objs;
        unsigned lcore_id = rte_lcore_id();
        uint32_t cache_size = mp->cache_size;
        uint32_t flushthresh = mp->cache_flushthresh;
#endif /* RTE_MEMPOOL_CACHE_MAX_SIZE > 0 */

        /* increment stat now, adding in mempool always success */
        __MEMPOOL_STAT_ADD(mp, put, n);

#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0
        /* cache is not enabled or single producer or non-EAL thread */
        if (unlikely(cache_size == 0 || is_mp == 0 ||
                     lcore_id >= RTE_MAX_LCORE))
                goto ring_enqueue;

        /* Go straight to ring if put would overflow mem allocated for cache */
        if (unlikely(n > RTE_MEMPOOL_CACHE_MAX_SIZE))
                goto ring_enqueue;

        cache = &mp->local_cache[lcore_id];
        cache_objs = &cache->objs[cache->len];

        /*
         * The cache follows the following algorithm
         *   1. Add the objects to the cache
         *   2. Anything greater than the cache min value (if it crosses the
         *   cache flush threshold) is flushed to the ring.
         */

        /* Add elements back into the cache */
        for (index = 0; index < n; ++index, obj_table++)
                cache_objs[index] = *obj_table;

        cache->len += n;

        if (cache->len >= flushthresh) {
                rte_ring_mp_enqueue_bulk(mp->ring, &cache->objs[cache_size],
                                cache->len - cache_size);
                cache->len = cache_size;
        }

        return;

ring_enqueue:
#endif /* RTE_MEMPOOL_CACHE_MAX_SIZE > 0 */

        /* push remaining objects in ring */
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
        if (is_mp) {
                if (rte_ring_mp_enqueue_bulk(mp->ring, obj_table, n) < 0)
                        rte_panic("cannot put objects in mempool\n");
        }
        else {
                if (rte_ring_sp_enqueue_bulk(mp->ring, obj_table, n) < 0)
                        rte_panic("cannot put objects in mempool\n");
        }
#else
        if (is_mp)
                rte_ring_mp_enqueue_bulk(mp->ring, obj_table, n);
        else
                rte_ring_sp_enqueue_bulk(mp->ring, obj_table, n);
#endif
}


static inline void __attribute__((always_inline))
rte_mempool_mp_put_bulk(struct rte_mempool *mp, void * const *obj_table,
                        unsigned n)
{
        __mempool_check_cookies(mp, obj_table, n, 0);
        __mempool_put_bulk(mp, obj_table, n, 1);
}

static inline void
rte_mempool_sp_put_bulk(struct rte_mempool *mp, void * const *obj_table,
                        unsigned n)
{
        __mempool_check_cookies(mp, obj_table, n, 0);
        __mempool_put_bulk(mp, obj_table, n, 0);
}

static inline void __attribute__((always_inline))
rte_mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,
                     unsigned n)
{
        __mempool_check_cookies(mp, obj_table, n, 0);
        __mempool_put_bulk(mp, obj_table, n, !(mp->flags & MEMPOOL_F_SP_PUT));
}

static inline void __attribute__((always_inline))
rte_mempool_mp_put(struct rte_mempool *mp, void *obj)
{
        rte_mempool_mp_put_bulk(mp, &obj, 1);
}

static inline void __attribute__((always_inline))
rte_mempool_sp_put(struct rte_mempool *mp, void *obj)
{
        rte_mempool_sp_put_bulk(mp, &obj, 1);
}

static inline void __attribute__((always_inline))
rte_mempool_put(struct rte_mempool *mp, void *obj)
{
        rte_mempool_put_bulk(mp, &obj, 1);
}

static inline int __attribute__((always_inline))
__mempool_get_bulk(struct rte_mempool *mp, void **obj_table,
                   unsigned n, int is_mc)
{
        int ret;
#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0
        struct rte_mempool_cache *cache;
        uint32_t index, len;
        void **cache_objs;
        unsigned lcore_id = rte_lcore_id();
        uint32_t cache_size = mp->cache_size;

        /* cache is not enabled or single consumer */
        if (unlikely(cache_size == 0 || is_mc == 0 ||
                     n >= cache_size || lcore_id >= RTE_MAX_LCORE))
                goto ring_dequeue;

        cache = &mp->local_cache[lcore_id];
        cache_objs = cache->objs;

        /* Can this be satisfied from the cache? */
        if (cache->len < n) {
                /* No. Backfill the cache first, and then fill from it */
                uint32_t req = n + (cache_size - cache->len);

                /* How many do we require i.e. number to fill the cache + the request */
                ret = rte_ring_mc_dequeue_bulk(mp->ring, &cache->objs[cache->len], req);
                if (unlikely(ret < 0)) {
                        /*
                         * In the offchance that we are buffer constrained,
                         * where we are not able to allocate cache + n, go to
                         * the ring directly. If that fails, we are truly out of
                         * buffers.
                         */
                        goto ring_dequeue;
                }

                cache->len += req;
        }

        /* Now fill in the response ... */
        for (index = 0, len = cache->len - 1; index < n; ++index, len--, obj_table++)
                *obj_table = cache_objs[len];

        cache->len -= n;

        __MEMPOOL_STAT_ADD(mp, get_success, n);

        return 0;

ring_dequeue:
#endif /* RTE_MEMPOOL_CACHE_MAX_SIZE > 0 */

        /* get remaining objects from ring */
        if (is_mc)
                ret = rte_ring_mc_dequeue_bulk(mp->ring, obj_table, n);
        else
                ret = rte_ring_sc_dequeue_bulk(mp->ring, obj_table, n);

        if (ret < 0)
                __MEMPOOL_STAT_ADD(mp, get_fail, n);
        else
                __MEMPOOL_STAT_ADD(mp, get_success, n);

        return ret;
}

static inline int __attribute__((always_inline))
rte_mempool_mc_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned n)
{
        int ret;
        ret = __mempool_get_bulk(mp, obj_table, n, 1);
        if (ret == 0)
                __mempool_check_cookies(mp, obj_table, n, 1);
        return ret;
}

static inline int __attribute__((always_inline))
rte_mempool_sc_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned n)
{
        int ret;
        ret = __mempool_get_bulk(mp, obj_table, n, 0);
        if (ret == 0)
                __mempool_check_cookies(mp, obj_table, n, 1);
        return ret;
}

static inline int __attribute__((always_inline))
rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned n)
{
        int ret;
        ret = __mempool_get_bulk(mp, obj_table, n,
                                 !(mp->flags & MEMPOOL_F_SC_GET));
        if (ret == 0)
                __mempool_check_cookies(mp, obj_table, n, 1);
        return ret;
}

static inline int __attribute__((always_inline))
rte_mempool_mc_get(struct rte_mempool *mp, void **obj_p)
{
        return rte_mempool_mc_get_bulk(mp, obj_p, 1);
}

static inline int __attribute__((always_inline))
rte_mempool_sc_get(struct rte_mempool *mp, void **obj_p)
{
        return rte_mempool_sc_get_bulk(mp, obj_p, 1);
}

static inline int __attribute__((always_inline))
rte_mempool_get(struct rte_mempool *mp, void **obj_p)
{
        return rte_mempool_get_bulk(mp, obj_p, 1);
}

unsigned rte_mempool_count(const struct rte_mempool *mp);

static inline unsigned
rte_mempool_free_count(const struct rte_mempool *mp)
{
        return mp->size - rte_mempool_count(mp);
}

static inline int
rte_mempool_full(const struct rte_mempool *mp)
{
        return !!(rte_mempool_count(mp) == mp->size);
}

static inline int
rte_mempool_empty(const struct rte_mempool *mp)
{
        return !!(rte_mempool_count(mp) == 0);
}

static inline phys_addr_t
rte_mempool_virt2phy(const struct rte_mempool *mp, const void *elt)
{
        if (rte_eal_has_hugepages()) {
                uintptr_t off;

                off = (const char *)elt - (const char *)mp->elt_va_start;
                return mp->elt_pa[off >> mp->pg_shift] + (off & mp->pg_mask);
        } else {
                /*
                 * If huge pages are disabled, we cannot assume the
                 * memory region to be physically contiguous.
                 * Lookup for each element.
                 */
                return rte_mem_virt2phy(elt);
        }
}

void rte_mempool_audit(const struct rte_mempool *mp);

static inline void *rte_mempool_get_priv(struct rte_mempool *mp)
{
        return (char *)mp + MEMPOOL_HEADER_SIZE(mp, mp->pg_num);
}

void rte_mempool_list_dump(FILE *f);

struct rte_mempool *rte_mempool_lookup(const char *name);

uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
        struct rte_mempool_objsz *sz);

size_t rte_mempool_xmem_size(uint32_t elt_num, size_t elt_sz,
        uint32_t pg_shift);

ssize_t rte_mempool_xmem_usage(void *vaddr, uint32_t elt_num, size_t elt_sz,
        const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift);

void rte_mempool_walk(void (*func)(const struct rte_mempool *, void *arg),
                      void *arg);

#ifdef __cplusplus
}
#endif

#endif /* _RTE_MEMPOOL_H_ */