DPDK  24.07.0
rte_bitops.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2020 Arm Limited
3  * Copyright(c) 2010-2019 Intel Corporation
4  * Copyright(c) 2023 Microsoft Corporation
5  */
6 
7 #ifndef _RTE_BITOPS_H_
8 #define _RTE_BITOPS_H_
9 
18 #include <stdint.h>
19 
20 #include <rte_debug.h>
21 
22 #ifdef __cplusplus
23 extern "C" {
24 #endif
25 
32 #define RTE_BIT64(nr) (UINT64_C(1) << (nr))
33 
40 #define RTE_BIT32(nr) (UINT32_C(1) << (nr))
41 
50 #define RTE_SHIFT_VAL32(val, nr) (UINT32_C(val) << (nr))
51 
60 #define RTE_SHIFT_VAL64(val, nr) (UINT64_C(val) << (nr))
61 
71 #define RTE_GENMASK32(high, low) \
72  (((~UINT32_C(0)) << (low)) & (~UINT32_C(0) >> (31u - (high))))
73 
83 #define RTE_GENMASK64(high, low) \
84  (((~UINT64_C(0)) << (low)) & (~UINT64_C(0) >> (63u - (high))))
85 
94 #define RTE_FIELD_GET32(mask, reg) \
95  ((typeof(mask))(((reg) & (mask)) >> rte_ctz32(mask)))
96 
105 #define RTE_FIELD_GET64(mask, reg) \
106  ((typeof(mask))(((reg) & (mask)) >> rte_ctz64(mask)))
107 
108 /*------------------------ 32-bit relaxed operations ------------------------*/
109 
120 static inline uint32_t
121 rte_bit_relaxed_get32(unsigned int nr, volatile uint32_t *addr)
122 {
123  RTE_ASSERT(nr < 32);
124 
125  uint32_t mask = UINT32_C(1) << nr;
126  return (*addr) & mask;
127 }
128 
137 static inline void
138 rte_bit_relaxed_set32(unsigned int nr, volatile uint32_t *addr)
139 {
140  RTE_ASSERT(nr < 32);
141 
142  uint32_t mask = RTE_BIT32(nr);
143  *addr = (*addr) | mask;
144 }
145 
154 static inline void
155 rte_bit_relaxed_clear32(unsigned int nr, volatile uint32_t *addr)
156 {
157  RTE_ASSERT(nr < 32);
158 
159  uint32_t mask = RTE_BIT32(nr);
160  *addr = (*addr) & (~mask);
161 }
162 
174 static inline uint32_t
175 rte_bit_relaxed_test_and_set32(unsigned int nr, volatile uint32_t *addr)
176 {
177  RTE_ASSERT(nr < 32);
178 
179  uint32_t mask = RTE_BIT32(nr);
180  uint32_t val = *addr;
181  *addr = val | mask;
182  return val & mask;
183 }
184 
196 static inline uint32_t
197 rte_bit_relaxed_test_and_clear32(unsigned int nr, volatile uint32_t *addr)
198 {
199  RTE_ASSERT(nr < 32);
200 
201  uint32_t mask = RTE_BIT32(nr);
202  uint32_t val = *addr;
203  *addr = val & (~mask);
204  return val & mask;
205 }
206 
207 /*------------------------ 64-bit relaxed operations ------------------------*/
208 
219 static inline uint64_t
220 rte_bit_relaxed_get64(unsigned int nr, volatile uint64_t *addr)
221 {
222  RTE_ASSERT(nr < 64);
223 
224  uint64_t mask = RTE_BIT64(nr);
225  return (*addr) & mask;
226 }
227 
236 static inline void
237 rte_bit_relaxed_set64(unsigned int nr, volatile uint64_t *addr)
238 {
239  RTE_ASSERT(nr < 64);
240 
241  uint64_t mask = RTE_BIT64(nr);
242  (*addr) = (*addr) | mask;
243 }
244 
253 static inline void
254 rte_bit_relaxed_clear64(unsigned int nr, volatile uint64_t *addr)
255 {
256  RTE_ASSERT(nr < 64);
257 
258  uint64_t mask = RTE_BIT64(nr);
259  *addr = (*addr) & (~mask);
260 }
261 
273 static inline uint64_t
274 rte_bit_relaxed_test_and_set64(unsigned int nr, volatile uint64_t *addr)
275 {
276  RTE_ASSERT(nr < 64);
277 
278  uint64_t mask = RTE_BIT64(nr);
279  uint64_t val = *addr;
280  *addr = val | mask;
281  return val;
282 }
283 
295 static inline uint64_t
296 rte_bit_relaxed_test_and_clear64(unsigned int nr, volatile uint64_t *addr)
297 {
298  RTE_ASSERT(nr < 64);
299 
300  uint64_t mask = RTE_BIT64(nr);
301  uint64_t val = *addr;
302  *addr = val & (~mask);
303  return val & mask;
304 }
305 
306 #ifdef RTE_TOOLCHAIN_MSVC
307 
316 static inline unsigned int
317 rte_clz32(uint32_t v)
318 {
319  unsigned long rv;
320 
321  (void)_BitScanReverse(&rv, v);
322 
323  return (unsigned int)(sizeof(v) * CHAR_BIT - 1 - rv);
324 }
325 
334 static inline unsigned int
335 rte_clz64(uint64_t v)
336 {
337  unsigned long rv;
338 
339  (void)_BitScanReverse64(&rv, v);
340 
341  return (unsigned int)(sizeof(v) * CHAR_BIT - 1 - rv);
342 }
343 
352 static inline unsigned int
353 rte_ctz32(uint32_t v)
354 {
355  unsigned long rv;
356 
357  (void)_BitScanForward(&rv, v);
358 
359  return (unsigned int)rv;
360 }
361 
370 static inline unsigned int
371 rte_ctz64(uint64_t v)
372 {
373  unsigned long rv;
374 
375  (void)_BitScanForward64(&rv, v);
376 
377  return (unsigned int)rv;
378 }
379 
388 static inline unsigned int
389 rte_popcount32(uint32_t v)
390 {
391  return (unsigned int)__popcnt(v);
392 }
393 
402 static inline unsigned int
403 rte_popcount64(uint64_t v)
404 {
405  return (unsigned int)__popcnt64(v);
406 }
407 
408 #else
409 
418 static inline unsigned int
419 rte_clz32(uint32_t v)
420 {
421  return (unsigned int)__builtin_clz(v);
422 }
423 
432 static inline unsigned int
433 rte_clz64(uint64_t v)
434 {
435  return (unsigned int)__builtin_clzll(v);
436 }
437 
446 static inline unsigned int
447 rte_ctz32(uint32_t v)
448 {
449  return (unsigned int)__builtin_ctz(v);
450 }
451 
460 static inline unsigned int
461 rte_ctz64(uint64_t v)
462 {
463  return (unsigned int)__builtin_ctzll(v);
464 }
465 
474 static inline unsigned int
475 rte_popcount32(uint32_t v)
476 {
477  return (unsigned int)__builtin_popcount(v);
478 }
479 
488 static inline unsigned int
489 rte_popcount64(uint64_t v)
490 {
491  return (unsigned int)__builtin_popcountll(v);
492 }
493 
494 #endif
495 
506 static inline uint32_t
507 rte_combine32ms1b(uint32_t x)
508 {
509  x |= x >> 1;
510  x |= x >> 2;
511  x |= x >> 4;
512  x |= x >> 8;
513  x |= x >> 16;
514 
515  return x;
516 }
517 
528 static inline uint64_t
529 rte_combine64ms1b(uint64_t v)
530 {
531  v |= v >> 1;
532  v |= v >> 2;
533  v |= v >> 4;
534  v |= v >> 8;
535  v |= v >> 16;
536  v |= v >> 32;
537 
538  return v;
539 }
540 
552 static inline uint32_t
553 rte_bsf32(uint32_t v)
554 {
555  return (uint32_t)rte_ctz32(v);
556 }
557 
572 static inline int
573 rte_bsf32_safe(uint32_t v, uint32_t *pos)
574 {
575  if (v == 0)
576  return 0;
577 
578  *pos = rte_bsf32(v);
579  return 1;
580 }
581 
593 static inline uint32_t
594 rte_bsf64(uint64_t v)
595 {
596  return (uint32_t)rte_ctz64(v);
597 }
598 
613 static inline int
614 rte_bsf64_safe(uint64_t v, uint32_t *pos)
615 {
616  if (v == 0)
617  return 0;
618 
619  *pos = rte_bsf64(v);
620  return 1;
621 }
622 
634 static inline uint32_t
635 rte_fls_u32(uint32_t x)
636 {
637  return (x == 0) ? 0 : 32 - rte_clz32(x);
638 }
639 
652 static inline uint32_t
653 rte_fls_u64(uint64_t x)
654 {
655  return (x == 0) ? 0 : 64 - rte_clz64(x);
656 }
657 
658 /*********** Macros to work with powers of 2 ********/
659 
663 #define RTE_IS_POWER_OF_2(n) ((n) && !(((n) - 1) & (n)))
664 
671 static inline int
672 rte_is_power_of_2(uint32_t n)
673 {
674  return n && !(n & (n - 1));
675 }
676 
686 static inline uint32_t
687 rte_align32pow2(uint32_t x)
688 {
689  x--;
690  x = rte_combine32ms1b(x);
691 
692  return x + 1;
693 }
694 
704 static inline uint32_t
706 {
707  x = rte_combine32ms1b(x);
708 
709  return x - (x >> 1);
710 }
711 
721 static inline uint64_t
722 rte_align64pow2(uint64_t v)
723 {
724  v--;
725  v = rte_combine64ms1b(v);
726 
727  return v + 1;
728 }
729 
739 static inline uint64_t
741 {
742  v = rte_combine64ms1b(v);
743 
744  return v - (v >> 1);
745 }
746 
758 static inline uint32_t
759 rte_log2_u32(uint32_t v)
760 {
761  if (v == 0)
762  return 0;
763  v = rte_align32pow2(v);
764  return rte_bsf32(v);
765 }
766 
778 static inline uint32_t
779 rte_log2_u64(uint64_t v)
780 {
781  if (v == 0)
782  return 0;
783  v = rte_align64pow2(v);
784  /* we checked for v being 0 already, so no undefined behavior */
785  return rte_bsf64(v);
786 }
787 
788 #ifdef __cplusplus
789 }
790 #endif
791 
792 #endif /* _RTE_BITOPS_H_ */
static int rte_bsf32_safe(uint32_t v, uint32_t *pos)
Definition: rte_bitops.h:573
static void rte_bit_relaxed_clear64(unsigned int nr, volatile uint64_t *addr)
Definition: rte_bitops.h:254
static void rte_bit_relaxed_clear32(unsigned int nr, volatile uint32_t *addr)
Definition: rte_bitops.h:155
static unsigned int rte_popcount32(uint32_t v)
Definition: rte_bitops.h:475
static uint32_t rte_bit_relaxed_test_and_set32(unsigned int nr, volatile uint32_t *addr)
Definition: rte_bitops.h:175
static uint32_t rte_bit_relaxed_test_and_clear32(unsigned int nr, volatile uint32_t *addr)
Definition: rte_bitops.h:197
static unsigned int rte_clz32(uint32_t v)
Definition: rte_bitops.h:419
static uint32_t rte_bsf32(uint32_t v)
Definition: rte_bitops.h:553
static void rte_bit_relaxed_set32(unsigned int nr, volatile uint32_t *addr)
Definition: rte_bitops.h:138
#define RTE_BIT32(nr)
Definition: rte_bitops.h:40
static uint32_t rte_align32prevpow2(uint32_t x)
Definition: rte_bitops.h:705
static uint32_t rte_fls_u64(uint64_t x)
Definition: rte_bitops.h:653
static uint32_t rte_bsf64(uint64_t v)
Definition: rte_bitops.h:594
static uint32_t rte_align32pow2(uint32_t x)
Definition: rte_bitops.h:687
static uint32_t rte_fls_u32(uint32_t x)
Definition: rte_bitops.h:635
static int rte_is_power_of_2(uint32_t n)
Definition: rte_bitops.h:672
static int rte_bsf64_safe(uint64_t v, uint32_t *pos)
Definition: rte_bitops.h:614
static uint64_t rte_align64prevpow2(uint64_t v)
Definition: rte_bitops.h:740
static uint32_t rte_log2_u32(uint32_t v)
Definition: rte_bitops.h:759
static unsigned int rte_ctz64(uint64_t v)
Definition: rte_bitops.h:461
static uint64_t rte_bit_relaxed_test_and_set64(unsigned int nr, volatile uint64_t *addr)
Definition: rte_bitops.h:274
static unsigned int rte_ctz32(uint32_t v)
Definition: rte_bitops.h:447
static uint64_t rte_align64pow2(uint64_t v)
Definition: rte_bitops.h:722
static uint64_t rte_bit_relaxed_get64(unsigned int nr, volatile uint64_t *addr)
Definition: rte_bitops.h:220
static uint64_t rte_bit_relaxed_test_and_clear64(unsigned int nr, volatile uint64_t *addr)
Definition: rte_bitops.h:296
static uint32_t rte_combine32ms1b(uint32_t x)
Definition: rte_bitops.h:507
static uint64_t rte_combine64ms1b(uint64_t v)
Definition: rte_bitops.h:529
static void rte_bit_relaxed_set64(unsigned int nr, volatile uint64_t *addr)
Definition: rte_bitops.h:237
static unsigned int rte_popcount64(uint64_t v)
Definition: rte_bitops.h:489
static uint32_t rte_bit_relaxed_get32(unsigned int nr, volatile uint32_t *addr)
Definition: rte_bitops.h:121
static unsigned int rte_clz64(uint64_t v)
Definition: rte_bitops.h:433
#define RTE_BIT64(nr)
Definition: rte_bitops.h:32
static uint32_t rte_log2_u64(uint64_t v)
Definition: rte_bitops.h:779