rte_rwlock.h

Go to the documentation of this file.

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2010-2014 Intel Corporation
 */

#ifndef _RTE_RWLOCK_H_
#define _RTE_RWLOCK_H_

#ifdef __cplusplus
extern "C" {
#endif

#include <errno.h>

#include <rte_branch_prediction.h>
#include <rte_common.h>
#include <rte_lock_annotations.h>
#include <rte_pause.h>

#define RTE_RWLOCK_WAIT  0x1    /* Writer is waiting */
#define RTE_RWLOCK_WRITE 0x2    /* Writer has the lock */
#define RTE_RWLOCK_MASK  (RTE_RWLOCK_WAIT | RTE_RWLOCK_WRITE)
                /* Writer is waiting or has lock */
#define RTE_RWLOCK_READ  0x4    /* Reader increment */

typedef struct __rte_lockable {
    int32_t cnt;
} rte_rwlock_t;

#define RTE_RWLOCK_INITIALIZER { 0 }

static inline void
rte_rwlock_init(rte_rwlock_t *rwl)
{
    rwl->cnt = 0;
}

static inline void
rte_rwlock_read_lock(rte_rwlock_t *rwl)
    __rte_shared_lock_function(rwl)
    __rte_no_thread_safety_analysis
{
    int32_t x;

    while (1) {
        /* Wait while writer is present or pending */
        while (__atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED)
               & RTE_RWLOCK_MASK)
            rte_pause();

        /* Try to get read lock */
        x = __atomic_fetch_add(&rwl->cnt, RTE_RWLOCK_READ,
                       __ATOMIC_ACQUIRE) + RTE_RWLOCK_READ;

        /* If no writer, then acquire was successful */
        if (likely(!(x & RTE_RWLOCK_MASK)))
            return;

        /* Lost race with writer, backout the change. */
        __atomic_fetch_sub(&rwl->cnt, RTE_RWLOCK_READ,
                   __ATOMIC_RELAXED);
    }
}

static inline int
rte_rwlock_read_trylock(rte_rwlock_t *rwl)
    __rte_shared_trylock_function(0, rwl)
    __rte_no_thread_safety_analysis
{
    int32_t x;

    x = __atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED);

    /* fail if write lock is held or writer is pending */
    if (x & RTE_RWLOCK_MASK)
        return -EBUSY;

    /* Try to get read lock */
    x = __atomic_fetch_add(&rwl->cnt, RTE_RWLOCK_READ,
                   __ATOMIC_ACQUIRE) + RTE_RWLOCK_READ;

    /* Back out if writer raced in */
    if (unlikely(x & RTE_RWLOCK_MASK)) {
        __atomic_fetch_sub(&rwl->cnt, RTE_RWLOCK_READ,
                   __ATOMIC_RELEASE);

        return -EBUSY;
    }
    return 0;
}

static inline void
rte_rwlock_read_unlock(rte_rwlock_t *rwl)
    __rte_unlock_function(rwl)
    __rte_no_thread_safety_analysis
{
    __atomic_fetch_sub(&rwl->cnt, RTE_RWLOCK_READ, __ATOMIC_RELEASE);
}

static inline int
rte_rwlock_write_trylock(rte_rwlock_t *rwl)
    __rte_exclusive_trylock_function(0, rwl)
    __rte_no_thread_safety_analysis
{
    int32_t x;

    x = __atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED);
    if (x < RTE_RWLOCK_WRITE &&
        __atomic_compare_exchange_n(&rwl->cnt, &x, x + RTE_RWLOCK_WRITE,
                    1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))
        return 0;
    else
        return -EBUSY;
}

static inline void
rte_rwlock_write_lock(rte_rwlock_t *rwl)
    __rte_exclusive_lock_function(rwl)
    __rte_no_thread_safety_analysis
{
    int32_t x;

    while (1) {
        x = __atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED);

        /* No readers or writers? */
        if (likely(x < RTE_RWLOCK_WRITE)) {
            /* Turn off RTE_RWLOCK_WAIT, turn on RTE_RWLOCK_WRITE */
            if (__atomic_compare_exchange_n(&rwl->cnt, &x, RTE_RWLOCK_WRITE, 1,
                            __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))
                return;
        }

        /* Turn on writer wait bit */
        if (!(x & RTE_RWLOCK_WAIT))
            __atomic_fetch_or(&rwl->cnt, RTE_RWLOCK_WAIT, __ATOMIC_RELAXED);

        /* Wait until no readers before trying again */
        while (__atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED) > RTE_RWLOCK_WAIT)
            rte_pause();

    }
}

static inline void
rte_rwlock_write_unlock(rte_rwlock_t *rwl)
    __rte_unlock_function(rwl)
    __rte_no_thread_safety_analysis
{
    __atomic_fetch_sub(&rwl->cnt, RTE_RWLOCK_WRITE, __ATOMIC_RELEASE);
}

static inline int
rte_rwlock_write_is_locked(rte_rwlock_t *rwl)
{
    if (__atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED) & RTE_RWLOCK_WRITE)
        return 1;

    return 0;
}

static inline void
rte_rwlock_read_lock_tm(rte_rwlock_t *rwl)
    __rte_shared_lock_function(rwl);

static inline void
rte_rwlock_read_unlock_tm(rte_rwlock_t *rwl)
    __rte_unlock_function(rwl);

static inline void
rte_rwlock_write_lock_tm(rte_rwlock_t *rwl)
    __rte_exclusive_lock_function(rwl);

static inline void
rte_rwlock_write_unlock_tm(rte_rwlock_t *rwl)
    __rte_unlock_function(rwl);

#ifdef __cplusplus
}
#endif

#endif /* _RTE_RWLOCK_H_ */