rte_ptr_compress.h

Go to the documentation of this file.

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2024 Arm Limited
 */

#ifndef RTE_PTR_COMPRESS_H
#define RTE_PTR_COMPRESS_H

#include <stdint.h>
#include <inttypes.h>

#include <rte_bitops.h>
#include <rte_branch_prediction.h>
#include <rte_common.h>
#include <rte_debug.h>
#include <rte_vect.h>

#ifdef __cplusplus
extern "C" {
#endif

#define RTE_PTR_COMPRESS_BITS_NEEDED_FOR_POINTER_WITHIN_RANGE(mem_length) \
    (((uint64_t)mem_length) < 2 ? 1 : \
        (sizeof(uint64_t) * CHAR_BIT - \
         rte_clz64((uint64_t)mem_length - 1)))

#define RTE_PTR_COMPRESS_BIT_SHIFT_FROM_ALIGNMENT(alignment) \
    ((alignment) == 0 ? 0 : rte_ctz64((uint64_t)alignment))

#define RTE_PTR_COMPRESS_CAN_COMPRESS_16_SHIFT(mem_length, obj_alignment) \
    ((RTE_PTR_COMPRESS_BITS_NEEDED_FOR_POINTER_WITHIN_RANGE(mem_length) - \
    RTE_PTR_COMPRESS_BIT_SHIFT_FROM_ALIGNMENT(obj_alignment)) <= 16 ? 1 : 0)

#define RTE_PTR_COMPRESS_CAN_COMPRESS_32_SHIFT(mem_length, obj_alignment) \
    ((RTE_PTR_COMPRESS_BITS_NEEDED_FOR_POINTER_WITHIN_RANGE(mem_length) - \
    RTE_PTR_COMPRESS_BIT_SHIFT_FROM_ALIGNMENT(obj_alignment)) <= 32 ? 1 : 0)

static __rte_always_inline void
rte_ptr_compress_32_shift(void *ptr_base, void * const *src_table,
        uint32_t *dest_table, size_t n, uint8_t bit_shift)
{
    size_t i = 0;
#if defined RTE_HAS_SVE_ACLE && !defined RTE_ARCH_ARMv8_AARCH32
    svuint64_t v_ptr_table;
    do {
        svbool_t pg = svwhilelt_b64(i, n);
        v_ptr_table = svld1_u64(pg, (uint64_t *)src_table + i);
        v_ptr_table = svsub_x(pg, v_ptr_table, (uint64_t)ptr_base);
        v_ptr_table = svlsr_x(pg, v_ptr_table, bit_shift);
        svst1w(pg, &dest_table[i], v_ptr_table);
        i += svcntd();
    } while (i < n);
#elif defined __ARM_NEON && !defined RTE_ARCH_ARMv8_AARCH32
    uintptr_t ptr_diff;
    uint64x2_t v_ptr_table;
    /* right shift is done by left shifting by negative int */
    int64x2_t v_shift = vdupq_n_s64(-bit_shift);
    uint64x2_t v_ptr_base = vdupq_n_u64((uint64_t)ptr_base);
    const size_t n_even = n & ~0x1;
    for (; i < n_even; i += 2) {
        v_ptr_table = vld1q_u64((const uint64_t *)src_table + i);
        v_ptr_table = vsubq_u64(v_ptr_table, v_ptr_base);
        v_ptr_table = vshlq_u64(v_ptr_table, v_shift);
        vst1_u32(dest_table + i, vqmovn_u64(v_ptr_table));
    }
    /* process leftover single item in case of odd number of n */
    if (unlikely(n & 0x1)) {
        ptr_diff = RTE_PTR_DIFF(src_table[i], ptr_base);
        dest_table[i] = (uint32_t) (ptr_diff >> bit_shift);
    }
#else
    uintptr_t ptr_diff;
    for (; i < n; i++) {
        ptr_diff = RTE_PTR_DIFF(src_table[i], ptr_base);
        ptr_diff = ptr_diff >> bit_shift;
        RTE_ASSERT(ptr_diff <= UINT32_MAX);
        dest_table[i] = (uint32_t) ptr_diff;
    }
#endif
}

static __rte_always_inline void
rte_ptr_decompress_32_shift(void *ptr_base, uint32_t const *src_table,
        void **dest_table, size_t n, uint8_t bit_shift)
{
    size_t i = 0;
#if defined RTE_HAS_SVE_ACLE && !defined RTE_ARCH_ARMv8_AARCH32
    svuint64_t v_ptr_table;
    do {
        svbool_t pg = svwhilelt_b64(i, n);
        v_ptr_table = svld1uw_u64(pg, &src_table[i]);
        v_ptr_table = svlsl_x(pg, v_ptr_table, bit_shift);
        v_ptr_table = svadd_x(pg, v_ptr_table, (uint64_t)ptr_base);
        svst1(pg, (uint64_t *)dest_table + i, v_ptr_table);
        i += svcntd();
    } while (i < n);
#elif defined __ARM_NEON && !defined RTE_ARCH_ARMv8_AARCH32
    uintptr_t ptr_diff;
    uint64x2_t v_ptr_table;
    int64x2_t v_shift = vdupq_n_s64(bit_shift);
    uint64x2_t v_ptr_base = vdupq_n_u64((uint64_t)ptr_base);
    const size_t n_even = n & ~0x1;
    for (; i < n_even; i += 2) {
        v_ptr_table = vmovl_u32(vld1_u32(src_table + i));
        v_ptr_table = vshlq_u64(v_ptr_table, v_shift);
        v_ptr_table = vaddq_u64(v_ptr_table, v_ptr_base);
        vst1q_u64((uint64_t *)dest_table + i, v_ptr_table);
    }
    /* process leftover single item in case of odd number of n */
    if (unlikely(n & 0x1)) {
        ptr_diff = ((uintptr_t) src_table[i]) << bit_shift;
        dest_table[i] = RTE_PTR_ADD(ptr_base, ptr_diff);
    }
#else
    uintptr_t ptr_diff;
    for (; i < n; i++) {
        ptr_diff = ((uintptr_t) src_table[i]) << bit_shift;
        dest_table[i] = RTE_PTR_ADD(ptr_base, ptr_diff);
    }
#endif
}

static __rte_always_inline void
rte_ptr_compress_16_shift(void *ptr_base, void * const *src_table,
        uint16_t *dest_table, size_t n, uint8_t bit_shift)
{

    size_t i = 0;
#if defined RTE_HAS_SVE_ACLE && !defined RTE_ARCH_ARMv8_AARCH32
    svuint64_t v_ptr_table;
    do {
        svbool_t pg = svwhilelt_b64(i, n);
        v_ptr_table = svld1_u64(pg, (uint64_t *)src_table + i);
        v_ptr_table = svsub_x(pg, v_ptr_table, (uint64_t)ptr_base);
        v_ptr_table = svlsr_x(pg, v_ptr_table, bit_shift);
        svst1h(pg, &dest_table[i], v_ptr_table);
        i += svcntd();
    } while (i < n);
#else
    uintptr_t ptr_diff;
    for (; i < n; i++) {
        ptr_diff = RTE_PTR_DIFF(src_table[i], ptr_base);
        ptr_diff = ptr_diff >> bit_shift;
        RTE_ASSERT(ptr_diff <= UINT16_MAX);
        dest_table[i] = (uint16_t) ptr_diff;
    }
#endif
}

static __rte_always_inline void
rte_ptr_decompress_16_shift(void *ptr_base, uint16_t const *src_table,
        void **dest_table, size_t n, uint8_t bit_shift)
{
    size_t i = 0;
#if defined RTE_HAS_SVE_ACLE && !defined RTE_ARCH_ARMv8_AARCH32
    svuint64_t v_ptr_table;
    do {
        svbool_t pg = svwhilelt_b64(i, n);
        v_ptr_table = svld1uh_u64(pg, &src_table[i]);
        v_ptr_table = svlsl_x(pg, v_ptr_table, bit_shift);
        v_ptr_table = svadd_x(pg, v_ptr_table, (uint64_t)ptr_base);
        svst1(pg, (uint64_t *)dest_table + i, v_ptr_table);
        i += svcntd();
    } while (i < n);
#else
    uintptr_t ptr_diff;
    for (; i < n; i++) {
        ptr_diff = ((uintptr_t) src_table[i]) << bit_shift;
        dest_table[i] = RTE_PTR_ADD(ptr_base, ptr_diff);
    }
#endif
}

#ifdef __cplusplus
}
#endif

#endif /* RTE_PTR_COMPRESS_H */