rte_rwlock.h

Go to the documentation of this file.

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2010-2014 Intel Corporation
 */

#ifndef _RTE_RWLOCK_H_
#define _RTE_RWLOCK_H_

#ifdef __cplusplus
extern "C" {
#endif

#include <errno.h>

#include <rte_branch_prediction.h>
#include <rte_common.h>
#include <rte_lock_annotations.h>
#include <rte_pause.h>
#include <rte_stdatomic.h>

#define RTE_RWLOCK_WAIT  0x1    /* Writer is waiting */
#define RTE_RWLOCK_WRITE 0x2    /* Writer has the lock */
#define RTE_RWLOCK_MASK  (RTE_RWLOCK_WAIT | RTE_RWLOCK_WRITE)
                /* Writer is waiting or has lock */
#define RTE_RWLOCK_READ  0x4    /* Reader increment */

typedef struct __rte_lockable {
    RTE_ATOMIC(int32_t) cnt;
} rte_rwlock_t;

#define RTE_RWLOCK_INITIALIZER { 0 }

static inline void
rte_rwlock_init(rte_rwlock_t *rwl)
{
    rwl->cnt = 0;
}

static inline void
rte_rwlock_read_lock(rte_rwlock_t *rwl)
    __rte_shared_lock_function(rwl)
    __rte_no_thread_safety_analysis
{
    int32_t x;

    while (1) {
        /* Wait while writer is present or pending */
        while (rte_atomic_load_explicit(&rwl->cnt, rte_memory_order_relaxed)
               & RTE_RWLOCK_MASK)
            rte_pause();

        /* Try to get read lock */
        x = rte_atomic_fetch_add_explicit(&rwl->cnt, RTE_RWLOCK_READ,
                       rte_memory_order_acquire) + RTE_RWLOCK_READ;

        /* If no writer, then acquire was successful */
        if (likely(!(x & RTE_RWLOCK_MASK)))
            return;

        /* Lost race with writer, backout the change. */
        rte_atomic_fetch_sub_explicit(&rwl->cnt, RTE_RWLOCK_READ,
                   rte_memory_order_relaxed);
    }
}

static inline int
rte_rwlock_read_trylock(rte_rwlock_t *rwl)
    __rte_shared_trylock_function(0, rwl)
    __rte_no_thread_safety_analysis
{
    int32_t x;

    x = rte_atomic_load_explicit(&rwl->cnt, rte_memory_order_relaxed);

    /* fail if write lock is held or writer is pending */
    if (x & RTE_RWLOCK_MASK)
        return -EBUSY;

    /* Try to get read lock */
    x = rte_atomic_fetch_add_explicit(&rwl->cnt, RTE_RWLOCK_READ,
                   rte_memory_order_acquire) + RTE_RWLOCK_READ;

    /* Back out if writer raced in */
    if (unlikely(x & RTE_RWLOCK_MASK)) {
        rte_atomic_fetch_sub_explicit(&rwl->cnt, RTE_RWLOCK_READ,
                   rte_memory_order_release);

        return -EBUSY;
    }
    return 0;
}

static inline void
rte_rwlock_read_unlock(rte_rwlock_t *rwl)
    __rte_unlock_function(rwl)
    __rte_no_thread_safety_analysis
{
    rte_atomic_fetch_sub_explicit(&rwl->cnt, RTE_RWLOCK_READ, rte_memory_order_release);
}

static inline int
rte_rwlock_write_trylock(rte_rwlock_t *rwl)
    __rte_exclusive_trylock_function(0, rwl)
    __rte_no_thread_safety_analysis
{
    int32_t x;

    x = rte_atomic_load_explicit(&rwl->cnt, rte_memory_order_relaxed);
    if (x < RTE_RWLOCK_WRITE &&
        rte_atomic_compare_exchange_weak_explicit(&rwl->cnt, &x, x + RTE_RWLOCK_WRITE,
                    rte_memory_order_acquire, rte_memory_order_relaxed))
        return 0;
    else
        return -EBUSY;
}

static inline void
rte_rwlock_write_lock(rte_rwlock_t *rwl)
    __rte_exclusive_lock_function(rwl)
    __rte_no_thread_safety_analysis
{
    int32_t x;

    while (1) {
        x = rte_atomic_load_explicit(&rwl->cnt, rte_memory_order_relaxed);

        /* No readers or writers? */
        if (likely(x < RTE_RWLOCK_WRITE)) {
            /* Turn off RTE_RWLOCK_WAIT, turn on RTE_RWLOCK_WRITE */
            if (rte_atomic_compare_exchange_weak_explicit(&rwl->cnt, &x,
                    RTE_RWLOCK_WRITE, rte_memory_order_acquire,
                    rte_memory_order_relaxed))
                return;
        }

        /* Turn on writer wait bit */
        if (!(x & RTE_RWLOCK_WAIT))
            rte_atomic_fetch_or_explicit(&rwl->cnt, RTE_RWLOCK_WAIT,
                rte_memory_order_relaxed);

        /* Wait until no readers before trying again */
        while (rte_atomic_load_explicit(&rwl->cnt, rte_memory_order_relaxed)
                > RTE_RWLOCK_WAIT)
            rte_pause();

    }
}

static inline void
rte_rwlock_write_unlock(rte_rwlock_t *rwl)
    __rte_unlock_function(rwl)
    __rte_no_thread_safety_analysis
{
    rte_atomic_fetch_sub_explicit(&rwl->cnt, RTE_RWLOCK_WRITE, rte_memory_order_release);
}

static inline int
rte_rwlock_write_is_locked(rte_rwlock_t *rwl)
{
    if (rte_atomic_load_explicit(&rwl->cnt, rte_memory_order_relaxed) & RTE_RWLOCK_WRITE)
        return 1;

    return 0;
}

static inline void
rte_rwlock_read_lock_tm(rte_rwlock_t *rwl)
    __rte_shared_lock_function(rwl);

static inline void
rte_rwlock_read_unlock_tm(rte_rwlock_t *rwl)
    __rte_unlock_function(rwl);

static inline void
rte_rwlock_write_lock_tm(rte_rwlock_t *rwl)
    __rte_exclusive_lock_function(rwl);

static inline void
rte_rwlock_write_unlock_tm(rte_rwlock_t *rwl)
    __rte_unlock_function(rwl);

#ifdef __cplusplus
}
#endif

#endif /* _RTE_RWLOCK_H_ */