DPDK  19.11.10
rte_mcslock.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2019 Arm Limited
3  */
4 
5 #ifndef _RTE_MCSLOCK_H_
6 #define _RTE_MCSLOCK_H_
7 
22 #include <rte_lcore.h>
23 #include <rte_common.h>
24 #include <rte_pause.h>
25 #include <rte_branch_prediction.h>
26 
30 typedef struct rte_mcslock {
31  struct rte_mcslock *next;
32  int locked; /* 1 if the queue locked, 0 otherwise */
34 
49 __rte_experimental
50 static inline void
52 {
53  rte_mcslock_t *prev;
54 
55  /* Init me node */
56  __atomic_store_n(&me->locked, 1, __ATOMIC_RELAXED);
57  __atomic_store_n(&me->next, NULL, __ATOMIC_RELAXED);
58 
59  /* If the queue is empty, the exchange operation is enough to acquire
60  * the lock. Hence, the exchange operation requires acquire semantics.
61  * The store to me->next above should complete before the node is
62  * visible to other CPUs/threads. Hence, the exchange operation requires
63  * release semantics as well.
64  */
65  prev = __atomic_exchange_n(msl, me, __ATOMIC_ACQ_REL);
66  if (likely(prev == NULL)) {
67  /* Queue was empty, no further action required,
68  * proceed with lock taken.
69  */
70  return;
71  }
72  /* The store to me->next above should also complete before the node is
73  * visible to predecessor thread releasing the lock. Hence, the store
74  * prev->next also requires release semantics. Note that, for example,
75  * on ARM, the release semantics in the exchange operation is not
76  * strong as a release fence and is not sufficient to enforce the
77  * desired order here.
78  */
79  __atomic_store_n(&prev->next, me, __ATOMIC_RELEASE);
80 
81  /* The while-load of me->locked should not move above the previous
82  * store to prev->next. Otherwise it will cause a deadlock. Need a
83  * store-load barrier.
84  */
85  __atomic_thread_fence(__ATOMIC_ACQ_REL);
86  /* If the lock has already been acquired, it first atomically
87  * places the node at the end of the queue and then proceeds
88  * to spin on me->locked until the previous lock holder resets
89  * the me->locked using mcslock_unlock().
90  */
91  while (__atomic_load_n(&me->locked, __ATOMIC_ACQUIRE))
92  rte_pause();
93 }
94 
106 __rte_experimental
107 static inline void
109 {
110  /* Check if there are more nodes in the queue. */
111  if (likely(__atomic_load_n(&me->next, __ATOMIC_RELAXED) == NULL)) {
112  /* No, last member in the queue. */
113  rte_mcslock_t *save_me = __atomic_load_n(&me, __ATOMIC_RELAXED);
114 
115  /* Release the lock by setting it to NULL */
116  if (likely(__atomic_compare_exchange_n(msl, &save_me, NULL, 0,
117  __ATOMIC_RELEASE, __ATOMIC_RELAXED)))
118  return;
119 
120  /* Speculative execution would be allowed to read in the
121  * while-loop first. This has the potential to cause a
122  * deadlock. Need a load barrier.
123  */
124  __atomic_thread_fence(__ATOMIC_ACQUIRE);
125  /* More nodes added to the queue by other CPUs.
126  * Wait until the next pointer is set.
127  */
128  while (__atomic_load_n(&me->next, __ATOMIC_RELAXED) == NULL)
129  rte_pause();
130  }
131 
132  /* Pass lock to next waiter. */
133  __atomic_store_n(&me->next->locked, 0, __ATOMIC_RELEASE);
134 }
135 
149 __rte_experimental
150 static inline int
152 {
153  /* Init me node */
154  __atomic_store_n(&me->next, NULL, __ATOMIC_RELAXED);
155 
156  /* Try to lock */
157  rte_mcslock_t *expected = NULL;
158 
159  /* The lock can be taken only when the queue is empty. Hence,
160  * the compare-exchange operation requires acquire semantics.
161  * The store to me->next above should complete before the node
162  * is visible to other CPUs/threads. Hence, the compare-exchange
163  * operation requires release semantics as well.
164  */
165  return __atomic_compare_exchange_n(msl, &expected, me, 0,
166  __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
167 }
168 
180 __rte_experimental
181 static inline int
183 {
184  return (__atomic_load_n(&msl, __ATOMIC_RELAXED) != NULL);
185 }
186 
187 #endif /* _RTE_MCSLOCK_H_ */
#define likely(x)
static __rte_experimental int rte_mcslock_is_locked(rte_mcslock_t *msl)
Definition: rte_mcslock.h:182
static __rte_experimental void rte_mcslock_unlock(rte_mcslock_t **msl, rte_mcslock_t *me)
Definition: rte_mcslock.h:108
struct rte_mcslock rte_mcslock_t
static __rte_experimental int rte_mcslock_trylock(rte_mcslock_t **msl, rte_mcslock_t *me)
Definition: rte_mcslock.h:151
static void rte_pause(void)
static __rte_experimental void rte_mcslock_lock(rte_mcslock_t **msl, rte_mcslock_t *me)
Definition: rte_mcslock.h:51