21#include <rte_compat.h>
22#include <rte_config.h>
31#ifndef RTE_TOOLCHAIN_MSVC
33#define typeof __typeof__
43#ifdef RTE_TOOLCHAIN_MSVC
81#ifdef RTE_TOOLCHAIN_MSVC
82#define __rte_constant(e) _Generic((1 ? (void *) ((e) * 0ll) : (int *) 0), int * : 1, void * : 0)
84#define __rte_constant(e) __extension__(__builtin_constant_p(e))
93#define RTE_CC_IS_GNU 0
99#define RTE_CC_IS_GNU 1
102#define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + \
118#ifdef RTE_TOOLCHAIN_MSVC
119#define __rte_aligned(a) __declspec(align(a))
121#define __rte_aligned(a) __attribute__((__aligned__(a)))
124#ifdef RTE_ARCH_STRICT_ALIGN
129typedef uint64_t unaligned_uint64_t;
130typedef uint32_t unaligned_uint32_t;
131typedef uint16_t unaligned_uint16_t;
141#ifdef RTE_TOOLCHAIN_MSVC
142#define __rte_packed RTE_DEPRECATED(__rte_packed)
144#define __rte_packed (RTE_DEPRECATED(__rte_packed) __attribute__((__packed__)))
154#ifdef RTE_TOOLCHAIN_MSVC
155#define __rte_packed_begin __pragma(pack(push, 1))
156#define __rte_packed_end __pragma(pack(pop))
158#define __rte_packed_begin
159#define __rte_packed_end __attribute__((__packed__))
165#ifdef RTE_TOOLCHAIN_MSVC
166#define __rte_may_alias
168#define __rte_may_alias __attribute__((__may_alias__))
172#ifdef RTE_TOOLCHAIN_MSVC
173#define __rte_deprecated
174#define __rte_deprecated_msg(msg)
176#define __rte_deprecated __attribute__((__deprecated__))
177#define __rte_deprecated_msg(msg) __attribute__((__deprecated__(msg)))
183#if defined(RTE_CC_GCC) || defined(RTE_CC_CLANG)
184#define RTE_PRAGMA(x) _Pragma(#x)
185#define RTE_PRAGMA_WARNING(w) RTE_PRAGMA(GCC warning #w)
186#define RTE_DEPRECATED(x) RTE_PRAGMA_WARNING(#x is deprecated)
188#define RTE_DEPRECATED(x)
195#if !defined(RTE_TOOLCHAIN_MSVC)
196#define __rte_diagnostic_push _Pragma("GCC diagnostic push")
197#define __rte_diagnostic_pop _Pragma("GCC diagnostic pop")
199#define __rte_diagnostic_push
200#define __rte_diagnostic_pop
207#if !defined(RTE_TOOLCHAIN_MSVC)
208#define __rte_diagnostic_ignored_wcast_qual _Pragma("GCC diagnostic ignored \"-Wcast-qual\"")
210#define __rte_diagnostic_ignored_wcast_qual
216#ifdef RTE_TOOLCHAIN_MSVC
217#define __rte_weak RTE_DEPRECATED(__rte_weak)
219#define __rte_weak RTE_DEPRECATED(__rte_weak) __attribute__((__weak__))
225#ifdef RTE_TOOLCHAIN_MSVC
228#define __rte_pure __attribute__((pure))
234#ifdef RTE_TOOLCHAIN_MSVC
237#define __rte_used __attribute__((used))
245#ifdef RTE_TOOLCHAIN_MSVC
248#define __rte_unused __attribute__((__unused__))
254#if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L
255#define __rte_restrict __restrict
257#define __rte_restrict restrict
264#define RTE_SET_USED(x) (void)(x)
273#ifdef RTE_TOOLCHAIN_MSVC
274#define __rte_format_printf(format_index, first_arg)
277#define __rte_format_printf(format_index, first_arg) \
278 __attribute__((format(gnu_printf, format_index, first_arg)))
280#define __rte_format_printf(format_index, first_arg) \
281 __attribute__((format(printf, format_index, first_arg)))
288#ifdef RTE_TOOLCHAIN_MSVC
289#define __rte_section(name) \
290 __pragma(data_seg(name)) __declspec(allocate(name))
292#define __rte_section(name) \
293 __attribute__((section(name)))
301#if defined(RTE_CC_GCC) || defined(RTE_CC_CLANG)
302#define __rte_alloc_size(...) \
303 __attribute__((alloc_size(__VA_ARGS__)))
305#define __rte_alloc_size(...)
314#if defined(RTE_CC_GCC)
315#define __rte_alloc_align(argno) \
316 __attribute__((alloc_align(argno)))
318#define __rte_alloc_align(argno)
325#if defined(RTE_CC_GCC) || defined(RTE_CC_CLANG)
326#define __rte_malloc __attribute__((__malloc__))
335#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION >= 110000)
336#define __rte_dealloc(dealloc, argno) \
337 __attribute__((malloc(dealloc, argno)))
339#define __rte_dealloc(dealloc, argno)
342#define RTE_PRIORITY_LOG 101
343#define RTE_PRIORITY_BUS 110
344#define RTE_PRIORITY_CLASS 120
345#define RTE_PRIORITY_LAST 65535
347#define RTE_PRIO(prio) \
348 RTE_PRIORITY_ ## prio
360#ifndef RTE_TOOLCHAIN_MSVC
361#define RTE_INIT_PRIO(func, prio) \
362static void __attribute__((constructor(RTE_PRIO(prio)), used)) func(void)
365typedef int(__cdecl *_PIFV)(void);
367#define CTOR_SECTION_LOG ".CRT$XIB"
368#define CTOR_SECTION_BUS ".CRT$XIC"
369#define CTOR_SECTION_CLASS ".CRT$XID"
370#define CTOR_SECTION_LAST ".CRT$XIY"
372#define CTOR_PRIORITY_TO_SECTION(priority) CTOR_SECTION_ ## priority
374#define RTE_INIT_PRIO(name, priority) \
375 static void name(void); \
376 static int __cdecl name ## _thunk(void) { name(); return 0; } \
377 __pragma(const_seg(CTOR_PRIORITY_TO_SECTION(priority))) \
378 __declspec(allocate(CTOR_PRIORITY_TO_SECTION(priority))) \
379 _PIFV name ## _pointer = &name ## _thunk; \
380 __pragma(const_seg()) \
381 static void name(void)
393#define RTE_INIT(func) \
394 RTE_INIT_PRIO(func, LAST)
406#ifndef RTE_TOOLCHAIN_MSVC
407#define RTE_FINI_PRIO(func, prio) \
408static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
410#define DTOR_SECTION_LOG "mydtor$B"
411#define DTOR_SECTION_BUS "mydtor$C"
412#define DTOR_SECTION_CLASS "mydtor$D"
413#define DTOR_SECTION_LAST "mydtor$Y"
415#define DTOR_PRIORITY_TO_SECTION(priority) DTOR_SECTION_ ## priority
417#define RTE_FINI_PRIO(name, priority) \
418 static void name(void); \
419 __pragma(const_seg(DTOR_PRIORITY_TO_SECTION(priority))) \
420 __declspec(allocate(DTOR_PRIORITY_TO_SECTION(priority))) void *name ## _pointer = &name; \
421 __pragma(const_seg()) \
422 static void name(void)
434#define RTE_FINI(func) \
435 RTE_FINI_PRIO(func, LAST)
440#ifdef RTE_TOOLCHAIN_MSVC
441#define __rte_noreturn
443#define __rte_noreturn __attribute__((noreturn))
449#if defined(RTE_TOOLCHAIN_GCC) || defined(RTE_TOOLCHAIN_CLANG)
450#define __rte_unreachable() __extension__(__builtin_unreachable())
452#define __rte_unreachable() __assume(0)
478#ifdef RTE_TOOLCHAIN_MSVC
479#define __rte_warn_unused_result
481#define __rte_warn_unused_result __attribute__((warn_unused_result))
487#ifdef RTE_TOOLCHAIN_MSVC
488#define __rte_always_inline __forceinline
490#define __rte_always_inline inline __attribute__((always_inline))
496#ifdef RTE_TOOLCHAIN_MSVC
497#define __rte_noinline __declspec(noinline)
499#define __rte_noinline __attribute__((noinline))
505#ifdef RTE_TOOLCHAIN_MSVC
508#define __rte_hot __attribute__((hot))
514#ifdef RTE_TOOLCHAIN_MSVC
517#define __rte_cold __attribute__((cold))
526#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION >= 130000)
527#define __rte_assume(condition) __attribute__((assume(condition)))
528#elif defined(RTE_TOOLCHAIN_GCC)
529#define __rte_assume(condition) do { if (!(condition)) __rte_unreachable(); } while (0)
530#elif defined(RTE_TOOLCHAIN_CLANG)
531#define __rte_assume(condition) __extension__(__builtin_assume(condition))
533#define __rte_assume(condition) __assume(condition)
539#ifdef RTE_MALLOC_ASAN
541#define __rte_no_asan __attribute__((no_sanitize("address", "hwaddress")))
543#define __rte_no_asan __attribute__((no_sanitize_address))
554#define RTE_PTR_ADD(ptr, x) ((void*)((uintptr_t)(ptr) + (x)))
559#define RTE_PTR_SUB(ptr, x) ((void *)((uintptr_t)(ptr) - (x)))
566#define RTE_PTR_DIFF(ptr1, ptr2) ((uintptr_t)(ptr1) - (uintptr_t)(ptr2))
574#define RTE_PTR_UNQUAL(X) ((void *)(uintptr_t)(X))
593#define RTE_CAST_PTR(type, ptr) ((type)(uintptr_t)(ptr))
598#define RTE_CAST_FIELD(var, field, type) \
599 (*(type *)((uintptr_t)(var) + offsetof(typeof(*(var)), field)))
610#define RTE_PTR_ALIGN_FLOOR(ptr, align) \
611 ((typeof(ptr))RTE_ALIGN_FLOOR((uintptr_t)(ptr), align))
619#define RTE_ALIGN_FLOOR(val, align) \
620 (typeof(val))((val) & (~((typeof(val))((align) - 1))))
628#define RTE_PTR_ALIGN_CEIL(ptr, align) \
629 RTE_PTR_ALIGN_FLOOR((typeof(ptr))RTE_PTR_ADD(ptr, (align) - 1), align)
637#define RTE_ALIGN_CEIL(val, align) \
638 RTE_ALIGN_FLOOR(((val) + ((typeof(val)) (align) - 1)), align)
647#define RTE_PTR_ALIGN(ptr, align) RTE_PTR_ALIGN_CEIL(ptr, align)
656#define RTE_ALIGN(val, align) RTE_ALIGN_CEIL(val, align)
663#define RTE_ALIGN_MUL_CEIL(v, mul) \
664 ((((v) + (typeof(v))(mul) - 1) / ((typeof(v))(mul))) * (typeof(v))(mul))
671#define RTE_ALIGN_MUL_FLOOR(v, mul) \
672 (((v) / ((typeof(v))(mul))) * (typeof(v))(mul))
679#define RTE_ALIGN_MUL_NEAR(v, mul) \
681 typeof(v) ceil = RTE_ALIGN_MUL_CEIL(v, mul); \
682 typeof(v) floor = RTE_ALIGN_MUL_FLOOR(v, mul); \
683 (ceil - (v)) > ((v) - floor) ? floor : ceil; \
700 return ((uintptr_t)ptr & (align - 1)) == 0;
706#if !defined(static_assert) && !defined(__cplusplus)
707#define static_assert _Static_assert
716#define RTE_BUILD_BUG_ON(condition) do { static_assert(!(condition), #condition); } while (0)
721#define RTE_CACHE_LINE_MASK (RTE_CACHE_LINE_SIZE-1)
724#define RTE_CACHE_LINE_ROUNDUP(size) RTE_ALIGN_CEIL(size, RTE_CACHE_LINE_SIZE)
727#if RTE_CACHE_LINE_SIZE == 64
728#define RTE_CACHE_LINE_SIZE_LOG2 6
729#elif RTE_CACHE_LINE_SIZE == 128
730#define RTE_CACHE_LINE_SIZE_LOG2 7
732#error "Unsupported cache line size"
736#define RTE_CACHE_LINE_MIN_SIZE 64
739#define __rte_cache_aligned __rte_aligned(RTE_CACHE_LINE_SIZE)
742#define __rte_cache_min_aligned __rte_aligned(RTE_CACHE_LINE_MIN_SIZE)
744#define _RTE_CACHE_GUARD_HELPER2(unique) \
745 alignas(RTE_CACHE_LINE_SIZE) \
746 char cache_guard_ ## unique[RTE_CACHE_LINE_SIZE * RTE_CACHE_GUARD_LINES]
747#define _RTE_CACHE_GUARD_HELPER1(unique) _RTE_CACHE_GUARD_HELPER2(unique)
755#define RTE_CACHE_GUARD _RTE_CACHE_GUARD_HELPER1(__COUNTER__)
761#define RTE_BAD_PHYS_ADDR ((phys_addr_t)-1)
771#define RTE_BAD_IOVA ((rte_iova_t)-1)
775#ifndef RTE_TOOLCHAIN_MSVC
795#define RTE_MIN(a, b) \
797 typeof (a) _a = (a); \
798 typeof (b) _b = (b); \
809#define RTE_MIN_T(a, b, t) \
810 ((t)(a) < (t)(b) ? (t)(a) : (t)(b))
815#define RTE_MAX(a, b) \
817 typeof (a) _a = (a); \
818 typeof (b) _b = (b); \
829#define RTE_MAX_T(a, b, t) \
830 ((t)(a) > (t)(b) ? (t)(a) : (t)(b))
836#define offsetof(TYPE, MEMBER) __builtin_offsetof (TYPE, MEMBER)
854#ifdef RTE_TOOLCHAIN_MSVC
855#define container_of(ptr, type, member) \
856 ((type *)((uintptr_t)(ptr) - offsetof(type, member)))
858#define container_of(ptr, type, member) __extension__ ({ \
859 const typeof(((type *)0)->member) *_ptr = (ptr); \
860 __rte_unused type *_target_ptr = \
862 (type *)(((uintptr_t)_ptr) - offsetof(type, member)); \
868#define RTE_SWAP(a, b) \
885#define RTE_SIZEOF_FIELD(type, field) (sizeof(((type *)0)->field))
887#define _RTE_STR(x) #x
889#define RTE_STR(x) _RTE_STR(x)
896#define RTE_FMT(fmt, ...) fmt "%.0s", __VA_ARGS__ ""
897#define RTE_FMT_HEAD(fmt, ...) fmt
898#define RTE_FMT_TAIL(fmt, ...) __VA_ARGS__
901#define RTE_LEN2MASK(ln, tp) \
902 ((tp)((uint64_t)-1 >> (sizeof(uint64_t) * CHAR_BIT - (ln))))
905#define RTE_DIM(a) (sizeof (a) / sizeof ((a)[0]))
957rte_size_to_str(
char *buf,
int buf_size, uint64_t count,
bool use_iec,
const char *unit);
uint64_t rte_str_to_size(const char *str)
__extension__ typedef uint64_t RTE_MARKER64[0]
__extension__ typedef uint8_t RTE_MARKER8[0]
__extension__ typedef uint32_t RTE_MARKER32[0]
__extension__ typedef uint16_t RTE_MARKER16[0]
#define __rte_format_printf(format_index, first_arg)
__rte_experimental char * rte_size_to_str(char *buf, int buf_size, uint64_t count, bool use_iec, const char *unit)
static int rte_is_aligned(const void *const __rte_restrict ptr, const unsigned int align)
__rte_noreturn void rte_exit(int exit_code, const char *format,...) __rte_format_printf(2
__extension__ typedef void * RTE_MARKER[0]