DPDK  22.11.0
rte_rwlock.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2014 Intel Corporation
3  */
4 
5 #ifndef _RTE_RWLOCK_H_
6 #define _RTE_RWLOCK_H_
7 
25 #ifdef __cplusplus
26 extern "C" {
27 #endif
28 
29 #include <errno.h>
30 
31 #include <rte_branch_prediction.h>
32 #include <rte_common.h>
33 #include <rte_pause.h>
34 
52 #define RTE_RWLOCK_WAIT 0x1 /* Writer is waiting */
53 #define RTE_RWLOCK_WRITE 0x2 /* Writer has the lock */
54 #define RTE_RWLOCK_MASK (RTE_RWLOCK_WAIT | RTE_RWLOCK_WRITE)
55  /* Writer is waiting or has lock */
56 #define RTE_RWLOCK_READ 0x4 /* Reader increment */
57 
58 typedef struct {
59  int32_t cnt;
60 } rte_rwlock_t;
61 
65 #define RTE_RWLOCK_INITIALIZER { 0 }
66 
73 static inline void
74 rte_rwlock_init(rte_rwlock_t *rwl)
75 {
76  rwl->cnt = 0;
77 }
78 
85 static inline void
86 rte_rwlock_read_lock(rte_rwlock_t *rwl)
87 {
88  int32_t x;
89 
90  while (1) {
91  /* Wait while writer is present or pending */
92  while (__atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED)
93  & RTE_RWLOCK_MASK)
94  rte_pause();
95 
96  /* Try to get read lock */
97  x = __atomic_add_fetch(&rwl->cnt, RTE_RWLOCK_READ,
98  __ATOMIC_ACQUIRE);
99 
100  /* If no writer, then acquire was successful */
101  if (likely(!(x & RTE_RWLOCK_MASK)))
102  return;
103 
104  /* Lost race with writer, backout the change. */
105  __atomic_fetch_sub(&rwl->cnt, RTE_RWLOCK_READ,
106  __ATOMIC_RELAXED);
107  }
108 }
109 
120 static inline int
121 rte_rwlock_read_trylock(rte_rwlock_t *rwl)
122 {
123  int32_t x;
124 
125  x = __atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED);
126 
127  /* fail if write lock is held or writer is pending */
128  if (x & RTE_RWLOCK_MASK)
129  return -EBUSY;
130 
131  /* Try to get read lock */
132  x = __atomic_add_fetch(&rwl->cnt, RTE_RWLOCK_READ,
133  __ATOMIC_ACQUIRE);
134 
135  /* Back out if writer raced in */
136  if (unlikely(x & RTE_RWLOCK_MASK)) {
137  __atomic_fetch_sub(&rwl->cnt, RTE_RWLOCK_READ,
138  __ATOMIC_RELEASE);
139 
140  return -EBUSY;
141  }
142  return 0;
143 }
144 
151 static inline void
152 rte_rwlock_read_unlock(rte_rwlock_t *rwl)
153 {
154  __atomic_fetch_sub(&rwl->cnt, RTE_RWLOCK_READ, __ATOMIC_RELEASE);
155 }
156 
167 static inline int
168 rte_rwlock_write_trylock(rte_rwlock_t *rwl)
169 {
170  int32_t x;
171 
172  x = __atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED);
173  if (x < RTE_RWLOCK_WRITE &&
174  __atomic_compare_exchange_n(&rwl->cnt, &x, x + RTE_RWLOCK_WRITE,
175  1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))
176  return 0;
177  else
178  return -EBUSY;
179 }
180 
187 static inline void
188 rte_rwlock_write_lock(rte_rwlock_t *rwl)
189 {
190  int32_t x;
191 
192  while (1) {
193  x = __atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED);
194 
195  /* No readers or writers? */
196  if (likely(x < RTE_RWLOCK_WRITE)) {
197  /* Turn off RTE_RWLOCK_WAIT, turn on RTE_RWLOCK_WRITE */
198  if (__atomic_compare_exchange_n(&rwl->cnt, &x, RTE_RWLOCK_WRITE, 1,
199  __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))
200  return;
201  }
202 
203  /* Turn on writer wait bit */
204  if (!(x & RTE_RWLOCK_WAIT))
205  __atomic_fetch_or(&rwl->cnt, RTE_RWLOCK_WAIT, __ATOMIC_RELAXED);
206 
207  /* Wait until no readers before trying again */
208  while (__atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED) > RTE_RWLOCK_WAIT)
209  rte_pause();
210 
211  }
212 }
213 
220 static inline void
221 rte_rwlock_write_unlock(rte_rwlock_t *rwl)
222 {
223  __atomic_fetch_sub(&rwl->cnt, RTE_RWLOCK_WRITE, __ATOMIC_RELEASE);
224 }
225 
239 static inline void
240 rte_rwlock_read_lock_tm(rte_rwlock_t *rwl);
241 
248 static inline void
249 rte_rwlock_read_unlock_tm(rte_rwlock_t *rwl);
250 
264 static inline void
265 rte_rwlock_write_lock_tm(rte_rwlock_t *rwl);
266 
273 static inline void
274 rte_rwlock_write_unlock_tm(rte_rwlock_t *rwl);
275 
276 #ifdef __cplusplus
277 }
278 #endif
279 
280 #endif /* _RTE_RWLOCK_H_ */
#define likely(x)
static void rte_rwlock_write_lock_tm(rte_rwlock_t *rwl)
static void rte_rwlock_read_lock(rte_rwlock_t *rwl)
Definition: rte_rwlock.h:86
static int rte_rwlock_read_trylock(rte_rwlock_t *rwl)
Definition: rte_rwlock.h:121
#define unlikely(x)
static void rte_rwlock_read_lock_tm(rte_rwlock_t *rwl)
static void rte_rwlock_read_unlock(rte_rwlock_t *rwl)
Definition: rte_rwlock.h:152
static void rte_pause(void)
#define RTE_RWLOCK_WAIT
Definition: rte_rwlock.h:52
static void rte_rwlock_write_unlock(rte_rwlock_t *rwl)
Definition: rte_rwlock.h:221
static void rte_rwlock_read_unlock_tm(rte_rwlock_t *rwl)
static void rte_rwlock_write_lock(rte_rwlock_t *rwl)
Definition: rte_rwlock.h:188
static void rte_rwlock_init(rte_rwlock_t *rwl)
Definition: rte_rwlock.h:74
static void rte_rwlock_write_unlock_tm(rte_rwlock_t *rwl)
static int rte_rwlock_write_trylock(rte_rwlock_t *rwl)
Definition: rte_rwlock.h:168