rte_bitops.h

Go to the documentation of this file.

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2020 Arm Limited
 * Copyright(c) 2010-2019 Intel Corporation
 * Copyright(c) 2023 Microsoft Corporation
 */

#ifndef _RTE_BITOPS_H_
#define _RTE_BITOPS_H_

#include <stdint.h>

#include <rte_debug.h>

#ifdef __cplusplus
extern "C" {
#endif

#define RTE_BIT64(nr) (UINT64_C(1) << (nr))

#define RTE_BIT32(nr) (UINT32_C(1) << (nr))

/*------------------------ 32-bit relaxed operations ------------------------*/

static inline uint32_t
rte_bit_relaxed_get32(unsigned int nr, volatile uint32_t *addr)
{
    RTE_ASSERT(nr < 32);

    uint32_t mask = UINT32_C(1) << nr;
    return (*addr) & mask;
}

static inline void
rte_bit_relaxed_set32(unsigned int nr, volatile uint32_t *addr)
{
    RTE_ASSERT(nr < 32);

    uint32_t mask = RTE_BIT32(nr);
    *addr = (*addr) | mask;
}

static inline void
rte_bit_relaxed_clear32(unsigned int nr, volatile uint32_t *addr)
{
    RTE_ASSERT(nr < 32);

    uint32_t mask = RTE_BIT32(nr);
    *addr = (*addr) & (~mask);
}

static inline uint32_t
rte_bit_relaxed_test_and_set32(unsigned int nr, volatile uint32_t *addr)
{
    RTE_ASSERT(nr < 32);

    uint32_t mask = RTE_BIT32(nr);
    uint32_t val = *addr;
    *addr = val | mask;
    return val & mask;
}

static inline uint32_t
rte_bit_relaxed_test_and_clear32(unsigned int nr, volatile uint32_t *addr)
{
    RTE_ASSERT(nr < 32);

    uint32_t mask = RTE_BIT32(nr);
    uint32_t val = *addr;
    *addr = val & (~mask);
    return val & mask;
}

/*------------------------ 64-bit relaxed operations ------------------------*/

static inline uint64_t
rte_bit_relaxed_get64(unsigned int nr, volatile uint64_t *addr)
{
    RTE_ASSERT(nr < 64);

    uint64_t mask = RTE_BIT64(nr);
    return (*addr) & mask;
}

static inline void
rte_bit_relaxed_set64(unsigned int nr, volatile uint64_t *addr)
{
    RTE_ASSERT(nr < 64);

    uint64_t mask = RTE_BIT64(nr);
    (*addr) = (*addr) | mask;
}

static inline void
rte_bit_relaxed_clear64(unsigned int nr, volatile uint64_t *addr)
{
    RTE_ASSERT(nr < 64);

    uint64_t mask = RTE_BIT64(nr);
    *addr = (*addr) & (~mask);
}

static inline uint64_t
rte_bit_relaxed_test_and_set64(unsigned int nr, volatile uint64_t *addr)
{
    RTE_ASSERT(nr < 64);

    uint64_t mask = RTE_BIT64(nr);
    uint64_t val = *addr;
    *addr = val | mask;
    return val;
}

static inline uint64_t
rte_bit_relaxed_test_and_clear64(unsigned int nr, volatile uint64_t *addr)
{
    RTE_ASSERT(nr < 64);

    uint64_t mask = RTE_BIT64(nr);
    uint64_t val = *addr;
    *addr = val & (~mask);
    return val & mask;
}

#ifdef RTE_TOOLCHAIN_MSVC

static inline unsigned int
rte_clz32(uint32_t v)
{
    unsigned long rv;

    (void)_BitScanReverse(&rv, v);

    return (unsigned int)(sizeof(v) * CHAR_BIT - 1 - rv);
}

static inline unsigned int
rte_clz64(uint64_t v)
{
    unsigned long rv;

    (void)_BitScanReverse64(&rv, v);

    return (unsigned int)(sizeof(v) * CHAR_BIT - 1 - rv);
}

static inline unsigned int
rte_ctz32(uint32_t v)
{
    unsigned long rv;

    (void)_BitScanForward(&rv, v);

    return (unsigned int)rv;
}

static inline unsigned int
rte_ctz64(uint64_t v)
{
    unsigned long rv;

    (void)_BitScanForward64(&rv, v);

    return (unsigned int)rv;
}

static inline unsigned int
rte_popcount32(uint32_t v)
{
    return (unsigned int)__popcnt(v);
}

static inline unsigned int
rte_popcount64(uint64_t v)
{
    return (unsigned int)__popcnt64(v);
}

#else

static inline unsigned int
rte_clz32(uint32_t v)
{
    return (unsigned int)__builtin_clz(v);
}

static inline unsigned int
rte_clz64(uint64_t v)
{
    return (unsigned int)__builtin_clzll(v);
}

static inline unsigned int
rte_ctz32(uint32_t v)
{
    return (unsigned int)__builtin_ctz(v);
}

static inline unsigned int
rte_ctz64(uint64_t v)
{
    return (unsigned int)__builtin_ctzll(v);
}

static inline unsigned int
rte_popcount32(uint32_t v)
{
    return (unsigned int)__builtin_popcount(v);
}

static inline unsigned int
rte_popcount64(uint64_t v)
{
    return (unsigned int)__builtin_popcountll(v);
}

#endif

static inline uint32_t
rte_combine32ms1b(uint32_t x)
{
    x |= x >> 1;
    x |= x >> 2;
    x |= x >> 4;
    x |= x >> 8;
    x |= x >> 16;

    return x;
}

static inline uint64_t
rte_combine64ms1b(uint64_t v)
{
    v |= v >> 1;
    v |= v >> 2;
    v |= v >> 4;
    v |= v >> 8;
    v |= v >> 16;
    v |= v >> 32;

    return v;
}

static inline uint32_t
rte_bsf32(uint32_t v)
{
    return (uint32_t)rte_ctz32(v);
}

static inline int
rte_bsf32_safe(uint32_t v, uint32_t *pos)
{
    if (v == 0)
        return 0;

    *pos = rte_bsf32(v);
    return 1;
}

static inline uint32_t
rte_bsf64(uint64_t v)
{
    return (uint32_t)rte_ctz64(v);
}

static inline int
rte_bsf64_safe(uint64_t v, uint32_t *pos)
{
    if (v == 0)
        return 0;

    *pos = rte_bsf64(v);
    return 1;
}

static inline uint32_t
rte_fls_u32(uint32_t x)
{
    return (x == 0) ? 0 : 32 - rte_clz32(x);
}

static inline uint32_t
rte_fls_u64(uint64_t x)
{
    return (x == 0) ? 0 : 64 - rte_clz64(x);
}

/*********** Macros to work with powers of 2 ********/

#define RTE_IS_POWER_OF_2(n) ((n) && !(((n) - 1) & (n)))

static inline int
rte_is_power_of_2(uint32_t n)
{
    return n && !(n & (n - 1));
}

static inline uint32_t
rte_align32pow2(uint32_t x)
{
    x--;
    x = rte_combine32ms1b(x);

    return x + 1;
}

static inline uint32_t
rte_align32prevpow2(uint32_t x)
{
    x = rte_combine32ms1b(x);

    return x - (x >> 1);
}

static inline uint64_t
rte_align64pow2(uint64_t v)
{
    v--;
    v = rte_combine64ms1b(v);

    return v + 1;
}

static inline uint64_t
rte_align64prevpow2(uint64_t v)
{
    v = rte_combine64ms1b(v);

    return v - (v >> 1);
}

static inline uint32_t
rte_log2_u32(uint32_t v)
{
    if (v == 0)
        return 0;
    v = rte_align32pow2(v);
    return rte_bsf32(v);
}

static inline uint32_t
rte_log2_u64(uint64_t v)
{
    if (v == 0)
        return 0;
    v = rte_align64pow2(v);
    /* we checked for v being 0 already, so no undefined behavior */
    return rte_bsf64(v);
}

#ifdef __cplusplus
}
#endif

#endif /* _RTE_BITOPS_H_ */