DPDK  17.05.2
rte_atomic.h
Go to the documentation of this file.
1 /*-
2  * BSD LICENSE
3  *
4  * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * * Redistributions of source code must retain the above copyright
12  * notice, this list of conditions and the following disclaimer.
13  * * Redistributions in binary form must reproduce the above copyright
14  * notice, this list of conditions and the following disclaimer in
15  * the documentation and/or other materials provided with the
16  * distribution.
17  * * Neither the name of Intel Corporation nor the names of its
18  * contributors may be used to endorse or promote products derived
19  * from this software without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 #ifndef _RTE_ATOMIC_H_
35 #define _RTE_ATOMIC_H_
36 
44 #include <stdint.h>
45 #include <rte_common.h>
46 
47 #ifdef __DOXYGEN__
48 
56 static inline void rte_mb(void);
57 
65 static inline void rte_wmb(void);
66 
74 static inline void rte_rmb(void);
75 
83 static inline void rte_smp_mb(void);
84 
92 static inline void rte_smp_wmb(void);
93 
101 static inline void rte_smp_rmb(void);
102 
110 static inline void rte_io_mb(void);
111 
119 static inline void rte_io_wmb(void);
120 
128 static inline void rte_io_rmb(void);
129 
130 #endif /* __DOXYGEN__ */
131 
138 #define rte_compiler_barrier() do { \
139  asm volatile ("" : : : "memory"); \
140 } while(0)
141 
142 /*------------------------- 16 bit atomic operations -------------------------*/
143 
160 static inline int
161 rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src);
162 
163 #ifdef RTE_FORCE_INTRINSICS
164 static inline int
165 rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
166 {
167  return __sync_bool_compare_and_swap(dst, exp, src);
168 }
169 #endif
170 
174 typedef struct {
175  volatile int16_t cnt;
177 
181 #define RTE_ATOMIC16_INIT(val) { (val) }
182 
189 static inline void
191 {
192  v->cnt = 0;
193 }
194 
203 static inline int16_t
205 {
206  return v->cnt;
207 }
208 
217 static inline void
218 rte_atomic16_set(rte_atomic16_t *v, int16_t new_value)
219 {
220  v->cnt = new_value;
221 }
222 
231 static inline void
233 {
234  __sync_fetch_and_add(&v->cnt, inc);
235 }
236 
245 static inline void
247 {
248  __sync_fetch_and_sub(&v->cnt, dec);
249 }
250 
257 static inline void
259 
260 #ifdef RTE_FORCE_INTRINSICS
261 static inline void
263 {
264  rte_atomic16_add(v, 1);
265 }
266 #endif
267 
274 static inline void
276 
277 #ifdef RTE_FORCE_INTRINSICS
278 static inline void
280 {
281  rte_atomic16_sub(v, 1);
282 }
283 #endif
284 
298 static inline int16_t
300 {
301  return __sync_add_and_fetch(&v->cnt, inc);
302 }
303 
318 static inline int16_t
320 {
321  return __sync_sub_and_fetch(&v->cnt, dec);
322 }
323 
335 static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v);
336 
337 #ifdef RTE_FORCE_INTRINSICS
338 static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
339 {
340  return __sync_add_and_fetch(&v->cnt, 1) == 0;
341 }
342 #endif
343 
355 static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v);
356 
357 #ifdef RTE_FORCE_INTRINSICS
358 static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
359 {
360  return __sync_sub_and_fetch(&v->cnt, 1) == 0;
361 }
362 #endif
363 
375 static inline int rte_atomic16_test_and_set(rte_atomic16_t *v);
376 
377 #ifdef RTE_FORCE_INTRINSICS
378 static inline int rte_atomic16_test_and_set(rte_atomic16_t *v)
379 {
380  return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1);
381 }
382 #endif
383 
390 static inline void rte_atomic16_clear(rte_atomic16_t *v)
391 {
392  v->cnt = 0;
393 }
394 
395 /*------------------------- 32 bit atomic operations -------------------------*/
396 
413 static inline int
414 rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src);
415 
416 #ifdef RTE_FORCE_INTRINSICS
417 static inline int
418 rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
419 {
420  return __sync_bool_compare_and_swap(dst, exp, src);
421 }
422 #endif
423 
427 typedef struct {
428  volatile int32_t cnt;
430 
434 #define RTE_ATOMIC32_INIT(val) { (val) }
435 
442 static inline void
444 {
445  v->cnt = 0;
446 }
447 
456 static inline int32_t
458 {
459  return v->cnt;
460 }
461 
470 static inline void
471 rte_atomic32_set(rte_atomic32_t *v, int32_t new_value)
472 {
473  v->cnt = new_value;
474 }
475 
484 static inline void
486 {
487  __sync_fetch_and_add(&v->cnt, inc);
488 }
489 
498 static inline void
500 {
501  __sync_fetch_and_sub(&v->cnt, dec);
502 }
503 
510 static inline void
512 
513 #ifdef RTE_FORCE_INTRINSICS
514 static inline void
516 {
517  rte_atomic32_add(v, 1);
518 }
519 #endif
520 
527 static inline void
529 
530 #ifdef RTE_FORCE_INTRINSICS
531 static inline void
533 {
534  rte_atomic32_sub(v,1);
535 }
536 #endif
537 
551 static inline int32_t
553 {
554  return __sync_add_and_fetch(&v->cnt, inc);
555 }
556 
571 static inline int32_t
573 {
574  return __sync_sub_and_fetch(&v->cnt, dec);
575 }
576 
588 static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v);
589 
590 #ifdef RTE_FORCE_INTRINSICS
591 static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
592 {
593  return __sync_add_and_fetch(&v->cnt, 1) == 0;
594 }
595 #endif
596 
608 static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v);
609 
610 #ifdef RTE_FORCE_INTRINSICS
611 static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
612 {
613  return __sync_sub_and_fetch(&v->cnt, 1) == 0;
614 }
615 #endif
616 
628 static inline int rte_atomic32_test_and_set(rte_atomic32_t *v);
629 
630 #ifdef RTE_FORCE_INTRINSICS
631 static inline int rte_atomic32_test_and_set(rte_atomic32_t *v)
632 {
633  return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1);
634 }
635 #endif
636 
643 static inline void rte_atomic32_clear(rte_atomic32_t *v)
644 {
645  v->cnt = 0;
646 }
647 
648 /*------------------------- 64 bit atomic operations -------------------------*/
649 
665 static inline int
666 rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src);
667 
668 #ifdef RTE_FORCE_INTRINSICS
669 static inline int
670 rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
671 {
672  return __sync_bool_compare_and_swap(dst, exp, src);
673 }
674 #endif
675 
679 typedef struct {
680  volatile int64_t cnt;
682 
686 #define RTE_ATOMIC64_INIT(val) { (val) }
687 
694 static inline void
696 
697 #ifdef RTE_FORCE_INTRINSICS
698 static inline void
700 {
701 #ifdef __LP64__
702  v->cnt = 0;
703 #else
704  int success = 0;
705  uint64_t tmp;
706 
707  while (success == 0) {
708  tmp = v->cnt;
709  success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
710  tmp, 0);
711  }
712 #endif
713 }
714 #endif
715 
724 static inline int64_t
726 
727 #ifdef RTE_FORCE_INTRINSICS
728 static inline int64_t
730 {
731 #ifdef __LP64__
732  return v->cnt;
733 #else
734  int success = 0;
735  uint64_t tmp;
736 
737  while (success == 0) {
738  tmp = v->cnt;
739  /* replace the value by itself */
740  success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
741  tmp, tmp);
742  }
743  return tmp;
744 #endif
745 }
746 #endif
747 
756 static inline void
757 rte_atomic64_set(rte_atomic64_t *v, int64_t new_value);
758 
759 #ifdef RTE_FORCE_INTRINSICS
760 static inline void
761 rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
762 {
763 #ifdef __LP64__
764  v->cnt = new_value;
765 #else
766  int success = 0;
767  uint64_t tmp;
768 
769  while (success == 0) {
770  tmp = v->cnt;
771  success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
772  tmp, new_value);
773  }
774 #endif
775 }
776 #endif
777 
786 static inline void
787 rte_atomic64_add(rte_atomic64_t *v, int64_t inc);
788 
789 #ifdef RTE_FORCE_INTRINSICS
790 static inline void
791 rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
792 {
793  __sync_fetch_and_add(&v->cnt, inc);
794 }
795 #endif
796 
805 static inline void
806 rte_atomic64_sub(rte_atomic64_t *v, int64_t dec);
807 
808 #ifdef RTE_FORCE_INTRINSICS
809 static inline void
810 rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
811 {
812  __sync_fetch_and_sub(&v->cnt, dec);
813 }
814 #endif
815 
822 static inline void
824 
825 #ifdef RTE_FORCE_INTRINSICS
826 static inline void
828 {
829  rte_atomic64_add(v, 1);
830 }
831 #endif
832 
839 static inline void
841 
842 #ifdef RTE_FORCE_INTRINSICS
843 static inline void
845 {
846  rte_atomic64_sub(v, 1);
847 }
848 #endif
849 
863 static inline int64_t
864 rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc);
865 
866 #ifdef RTE_FORCE_INTRINSICS
867 static inline int64_t
869 {
870  return __sync_add_and_fetch(&v->cnt, inc);
871 }
872 #endif
873 
887 static inline int64_t
888 rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec);
889 
890 #ifdef RTE_FORCE_INTRINSICS
891 static inline int64_t
893 {
894  return __sync_sub_and_fetch(&v->cnt, dec);
895 }
896 #endif
897 
909 static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v);
910 
911 #ifdef RTE_FORCE_INTRINSICS
912 static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
913 {
914  return rte_atomic64_add_return(v, 1) == 0;
915 }
916 #endif
917 
929 static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v);
930 
931 #ifdef RTE_FORCE_INTRINSICS
932 static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
933 {
934  return rte_atomic64_sub_return(v, 1) == 0;
935 }
936 #endif
937 
949 static inline int rte_atomic64_test_and_set(rte_atomic64_t *v);
950 
951 #ifdef RTE_FORCE_INTRINSICS
952 static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
953 {
954  return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
955 }
956 #endif
957 
964 static inline void rte_atomic64_clear(rte_atomic64_t *v);
965 
966 #ifdef RTE_FORCE_INTRINSICS
967 static inline void rte_atomic64_clear(rte_atomic64_t *v)
968 {
969  rte_atomic64_set(v, 0);
970 }
971 #endif
972 
973 #endif /* _RTE_ATOMIC_H_ */