DPDK  18.11.11
Data Structures | Macros | Functions
rte_atomic.h File Reference
#include <stdint.h>
#include <rte_common.h>

Go to the source code of this file.

Data Structures

struct  rte_atomic16_t
 
struct  rte_atomic32_t
 
struct  rte_atomic64_t
 

Macros

#define rte_compiler_barrier()
 
#define RTE_ATOMIC16_INIT(val)   { (val) }
 
#define RTE_ATOMIC32_INIT(val)   { (val) }
 
#define RTE_ATOMIC64_INIT(val)   { (val) }
 

Functions

static int rte_atomic16_cmpset (volatile uint16_t *dst, uint16_t exp, uint16_t src)
 
static uint16_t rte_atomic16_exchange (volatile uint16_t *dst, uint16_t val)
 
static void rte_atomic16_init (rte_atomic16_t *v)
 
static int16_t rte_atomic16_read (const rte_atomic16_t *v)
 
static void rte_atomic16_set (rte_atomic16_t *v, int16_t new_value)
 
static void rte_atomic16_add (rte_atomic16_t *v, int16_t inc)
 
static void rte_atomic16_sub (rte_atomic16_t *v, int16_t dec)
 
static void rte_atomic16_inc (rte_atomic16_t *v)
 
static void rte_atomic16_dec (rte_atomic16_t *v)
 
static int16_t rte_atomic16_add_return (rte_atomic16_t *v, int16_t inc)
 
static int16_t rte_atomic16_sub_return (rte_atomic16_t *v, int16_t dec)
 
static int rte_atomic16_inc_and_test (rte_atomic16_t *v)
 
static int rte_atomic16_dec_and_test (rte_atomic16_t *v)
 
static int rte_atomic16_test_and_set (rte_atomic16_t *v)
 
static void rte_atomic16_clear (rte_atomic16_t *v)
 
static int rte_atomic32_cmpset (volatile uint32_t *dst, uint32_t exp, uint32_t src)
 
static uint32_t rte_atomic32_exchange (volatile uint32_t *dst, uint32_t val)
 
static void rte_atomic32_init (rte_atomic32_t *v)
 
static int32_t rte_atomic32_read (const rte_atomic32_t *v)
 
static void rte_atomic32_set (rte_atomic32_t *v, int32_t new_value)
 
static void rte_atomic32_add (rte_atomic32_t *v, int32_t inc)
 
static void rte_atomic32_sub (rte_atomic32_t *v, int32_t dec)
 
static void rte_atomic32_inc (rte_atomic32_t *v)
 
static void rte_atomic32_dec (rte_atomic32_t *v)
 
static int32_t rte_atomic32_add_return (rte_atomic32_t *v, int32_t inc)
 
static int32_t rte_atomic32_sub_return (rte_atomic32_t *v, int32_t dec)
 
static int rte_atomic32_inc_and_test (rte_atomic32_t *v)
 
static int rte_atomic32_dec_and_test (rte_atomic32_t *v)
 
static int rte_atomic32_test_and_set (rte_atomic32_t *v)
 
static void rte_atomic32_clear (rte_atomic32_t *v)
 
static int rte_atomic64_cmpset (volatile uint64_t *dst, uint64_t exp, uint64_t src)
 
static uint64_t rte_atomic64_exchange (volatile uint64_t *dst, uint64_t val)
 
static void rte_atomic64_init (rte_atomic64_t *v)
 
static int64_t rte_atomic64_read (rte_atomic64_t *v)
 
static void rte_atomic64_set (rte_atomic64_t *v, int64_t new_value)
 
static void rte_atomic64_add (rte_atomic64_t *v, int64_t inc)
 
static void rte_atomic64_sub (rte_atomic64_t *v, int64_t dec)
 
static void rte_atomic64_inc (rte_atomic64_t *v)
 
static void rte_atomic64_dec (rte_atomic64_t *v)
 
static int64_t rte_atomic64_add_return (rte_atomic64_t *v, int64_t inc)
 
static int64_t rte_atomic64_sub_return (rte_atomic64_t *v, int64_t dec)
 
static int rte_atomic64_inc_and_test (rte_atomic64_t *v)
 
static int rte_atomic64_dec_and_test (rte_atomic64_t *v)
 
static int rte_atomic64_test_and_set (rte_atomic64_t *v)
 
static void rte_atomic64_clear (rte_atomic64_t *v)
 
Memory Barrier
static void rte_mb (void)
 
static void rte_wmb (void)
 
static void rte_rmb (void)
 
SMP Memory Barrier
static void rte_smp_mb (void)
 
static void rte_smp_wmb (void)
 
static void rte_smp_rmb (void)
 
I/O Memory Barrier
static void rte_io_mb (void)
 
static void rte_io_wmb (void)
 
static void rte_io_rmb (void)
 
Coherent I/O Memory Barrier

Coherent I/O memory barrier is a lightweight version of I/O memory barriers which are system-wide data synchronization barriers. This is for only coherent memory domain between lcore and I/O device but it is same as the I/O memory barriers in most of architectures. However, some architecture provides even lighter barriers which are somewhere in between I/O memory barriers and SMP memory barriers. For example, in case of ARMv8, DMB(data memory barrier) instruction can have different shareability domains - inner-shareable and outer-shareable. And inner-shareable DMB fits for SMP memory barriers and outer-shareable DMB for coherent I/O memory barriers, which acts on coherent memory.

In most cases, I/O memory barriers are safer but if operations are on coherent memory instead of incoherent MMIO region of a device, then coherent I/O memory barriers can be used and this could bring performance gain depending on architectures.

static void rte_cio_wmb (void)
 
static void rte_cio_rmb (void)
 

Detailed Description

Atomic Operations

This file defines a generic API for atomic operations.

Definition in file rte_atomic.h.

Macro Definition Documentation

#define rte_compiler_barrier ( )
Value:
do { \
asm volatile ("" : : : "memory"); \
} while(0)

Compiler barrier.

Guarantees that operation reordering does not occur at compile time for operations directly before and after the barrier.

Examples:
examples/performance-thread/common/lthread_tls.c, and examples/tep_termination/main.c.

Definition at line 160 of file rte_atomic.h.

#define RTE_ATOMIC16_INIT (   val)    { (val) }

Static initializer for an atomic counter.

Definition at line 233 of file rte_atomic.h.

#define RTE_ATOMIC32_INIT (   val)    { (val) }

Static initializer for an atomic counter.

Examples:
examples/kni/main.c.

Definition at line 516 of file rte_atomic.h.

#define RTE_ATOMIC64_INIT (   val)    { (val) }

Static initializer for an atomic counter.

Definition at line 798 of file rte_atomic.h.

Function Documentation

static void rte_mb ( void  )
inlinestatic

General memory barrier.

Guarantees that the LOAD and STORE operations generated before the barrier occur before the LOAD and STORE operations generated after. This function is architecture dependent.

static void rte_wmb ( void  )
inlinestatic

Write memory barrier.

Guarantees that the STORE operations generated before the barrier occur before the STORE operations generated after. This function is architecture dependent.

Examples:
examples/performance-thread/common/lthread.c, examples/performance-thread/common/lthread_sched.c, and examples/vhost_crypto/main.c.
static void rte_rmb ( void  )
inlinestatic

Read memory barrier.

Guarantees that the LOAD operations generated before the barrier occur before the LOAD operations generated after. This function is architecture dependent.

static void rte_smp_mb ( void  )
inlinestatic

General memory barrier between lcores

Guarantees that the LOAD and STORE operations that precede the rte_smp_mb() call are globally visible across the lcores before the LOAD and STORE operations that follows it.

static void rte_smp_wmb ( void  )
inlinestatic

Write memory barrier between lcores

Guarantees that the STORE operations that precede the rte_smp_wmb() call are globally visible across the lcores before the STORE operations that follows it.

Examples:
examples/eventdev_pipeline/main.c, and examples/vhost/virtio_net.c.
static void rte_smp_rmb ( void  )
inlinestatic

Read memory barrier between lcores

Guarantees that the LOAD operations that precede the rte_smp_rmb() call are globally visible across the lcores before the LOAD operations that follows it.

Examples:
examples/vhost/virtio_net.c.
static void rte_io_mb ( void  )
inlinestatic

General memory barrier for I/O device

Guarantees that the LOAD and STORE operations that precede the rte_io_mb() call are visible to I/O device or CPU before the LOAD and STORE operations that follow it.

static void rte_io_wmb ( void  )
inlinestatic

Write memory barrier for I/O device

Guarantees that the STORE operations that precede the rte_io_wmb() call are visible to I/O device before the STORE operations that follow it.

static void rte_io_rmb ( void  )
inlinestatic

Read memory barrier for IO device

Guarantees that the LOAD operations on I/O device that precede the rte_io_rmb() call are visible to CPU before the LOAD operations that follow it.

static void rte_cio_wmb ( void  )
inlinestatic

Write memory barrier for coherent memory between lcore and I/O device

Guarantees that the STORE operations on coherent memory that precede the rte_cio_wmb() call are visible to I/O device before the STORE operations that follow it.

static void rte_cio_rmb ( void  )
inlinestatic

Read memory barrier for coherent memory between lcore and I/O device

Guarantees that the LOAD operations on coherent memory updated by I/O device that precede the rte_cio_rmb() call are visible to CPU before the LOAD operations that follow it.

static int rte_atomic16_cmpset ( volatile uint16_t *  dst,
uint16_t  exp,
uint16_t  src 
)
inlinestatic

Atomic compare and set.

(atomic) equivalent to: if (*dst == exp) *dst = src (all 16-bit words)

Parameters
dstThe destination location into which the value will be written.
expThe expected value.
srcThe new value.
Returns
Non-zero on success; 0 on failure.
static uint16_t rte_atomic16_exchange ( volatile uint16_t *  dst,
uint16_t  val 
)
inlinestatic

Atomic exchange.

(atomic) equivalent to: ret = *dst *dst = val; return ret;

Parameters
dstThe destination location into which the value will be written.
valThe new value.
Returns
The original value at that location
static void rte_atomic16_init ( rte_atomic16_t v)
inlinestatic

Initialize an atomic counter.

Parameters
vA pointer to the atomic counter.
Examples:
examples/bbdev_app/main.c, and examples/performance-thread/common/lthread_sched.c.

Definition at line 242 of file rte_atomic.h.

static int16_t rte_atomic16_read ( const rte_atomic16_t v)
inlinestatic

Atomically read a 16-bit value from a counter.

Parameters
vA pointer to the atomic counter.
Returns
The value of the counter.
Examples:
examples/bbdev_app/main.c, examples/l2fwd-jobstats/main.c, examples/performance-thread/common/lthread_sched.c, and examples/performance-thread/l3fwd-thread/main.c.

Definition at line 256 of file rte_atomic.h.

static void rte_atomic16_set ( rte_atomic16_t v,
int16_t  new_value 
)
inlinestatic

Atomically set a counter to a 16-bit value.

Parameters
vA pointer to the atomic counter.
new_valueThe new value for the counter.
Examples:
examples/bbdev_app/main.c, examples/l2fwd-jobstats/main.c, and examples/performance-thread/common/lthread_sched.c.

Definition at line 270 of file rte_atomic.h.

static void rte_atomic16_add ( rte_atomic16_t v,
int16_t  inc 
)
inlinestatic

Atomically add a 16-bit value to an atomic counter.

Parameters
vA pointer to the atomic counter.
incThe value to be added to the counter.

Definition at line 284 of file rte_atomic.h.

static void rte_atomic16_sub ( rte_atomic16_t v,
int16_t  dec 
)
inlinestatic

Atomically subtract a 16-bit value from an atomic counter.

Parameters
vA pointer to the atomic counter.
decThe value to be subtracted from the counter.

Definition at line 298 of file rte_atomic.h.

static void rte_atomic16_inc ( rte_atomic16_t v)
inlinestatic

Atomically increment a counter by one.

Parameters
vA pointer to the atomic counter.
Examples:
examples/performance-thread/common/lthread_sched.c, and examples/performance-thread/l3fwd-thread/main.c.
static void rte_atomic16_dec ( rte_atomic16_t v)
inlinestatic

Atomically decrement a counter by one.

Parameters
vA pointer to the atomic counter.
Examples:
examples/performance-thread/common/lthread_sched.c.
static int16_t rte_atomic16_add_return ( rte_atomic16_t v,
int16_t  inc 
)
inlinestatic

Atomically add a 16-bit value to a counter and return the result.

Atomically adds the 16-bits value (inc) to the atomic counter (v) and returns the value of v after addition.

Parameters
vA pointer to the atomic counter.
incThe value to be added to the counter.
Returns
The value of v after the addition.

Definition at line 351 of file rte_atomic.h.

static int16_t rte_atomic16_sub_return ( rte_atomic16_t v,
int16_t  dec 
)
inlinestatic

Atomically subtract a 16-bit value from a counter and return the result.

Atomically subtracts the 16-bit value (inc) from the atomic counter (v) and returns the value of v after the subtraction.

Parameters
vA pointer to the atomic counter.
decThe value to be subtracted from the counter.
Returns
The value of v after the subtraction.

Definition at line 371 of file rte_atomic.h.

static int rte_atomic16_inc_and_test ( rte_atomic16_t v)
inlinestatic

Atomically increment a 16-bit counter by one and test.

Atomically increments the atomic counter (v) by one and returns true if the result is 0, or false in all other cases.

Parameters
vA pointer to the atomic counter.
Returns
True if the result after the increment operation is 0; false otherwise.
static int rte_atomic16_dec_and_test ( rte_atomic16_t v)
inlinestatic

Atomically decrement a 16-bit counter by one and test.

Atomically decrements the atomic counter (v) by one and returns true if the result is 0, or false in all other cases.

Parameters
vA pointer to the atomic counter.
Returns
True if the result after the decrement operation is 0; false otherwise.
static int rte_atomic16_test_and_set ( rte_atomic16_t v)
inlinestatic

Atomically test and set a 16-bit atomic counter.

If the counter value is already set, return 0 (failed). Otherwise, set the counter value to 1 and return 1 (success).

Parameters
vA pointer to the atomic counter.
Returns
0 if failed; else 1, success.
static void rte_atomic16_clear ( rte_atomic16_t v)
inlinestatic

Atomically set a 16-bit counter to 0.

Parameters
vA pointer to the atomic counter.

Definition at line 442 of file rte_atomic.h.

static int rte_atomic32_cmpset ( volatile uint32_t *  dst,
uint32_t  exp,
uint32_t  src 
)
inlinestatic

Atomic compare and set.

(atomic) equivalent to: if (*dst == exp) *dst = src (all 32-bit words)

Parameters
dstThe destination location into which the value will be written.
expThe expected value.
srcThe new value.
Returns
Non-zero on success; 0 on failure.
Examples:
examples/vm_power_manager/channel_monitor.c.
static uint32_t rte_atomic32_exchange ( volatile uint32_t *  dst,
uint32_t  val 
)
inlinestatic

Atomic exchange.

(atomic) equivalent to: ret = *dst *dst = val; return ret;

Parameters
dstThe destination location into which the value will be written.
valThe new value.
Returns
The original value at that location
static void rte_atomic32_init ( rte_atomic32_t v)
inlinestatic

Initialize an atomic counter.

Parameters
vA pointer to the atomic counter.

Definition at line 525 of file rte_atomic.h.

static int32_t rte_atomic32_read ( const rte_atomic32_t v)
inlinestatic

Atomically read a 32-bit value from a counter.

Parameters
vA pointer to the atomic counter.
Returns
The value of the counter.
Examples:
examples/kni/main.c.

Definition at line 539 of file rte_atomic.h.

static void rte_atomic32_set ( rte_atomic32_t v,
int32_t  new_value 
)
inlinestatic

Atomically set a counter to a 32-bit value.

Parameters
vA pointer to the atomic counter.
new_valueThe new value for the counter.

Definition at line 553 of file rte_atomic.h.

static void rte_atomic32_add ( rte_atomic32_t v,
int32_t  inc 
)
inlinestatic

Atomically add a 32-bit value to an atomic counter.

Parameters
vA pointer to the atomic counter.
incThe value to be added to the counter.

Definition at line 567 of file rte_atomic.h.

static void rte_atomic32_sub ( rte_atomic32_t v,
int32_t  dec 
)
inlinestatic

Atomically subtract a 32-bit value from an atomic counter.

Parameters
vA pointer to the atomic counter.
decThe value to be subtracted from the counter.

Definition at line 581 of file rte_atomic.h.

static void rte_atomic32_inc ( rte_atomic32_t v)
inlinestatic

Atomically increment a counter by one.

Parameters
vA pointer to the atomic counter.
Examples:
examples/kni/main.c.
static void rte_atomic32_dec ( rte_atomic32_t v)
inlinestatic

Atomically decrement a counter by one.

Parameters
vA pointer to the atomic counter.
Examples:
examples/kni/main.c.
static int32_t rte_atomic32_add_return ( rte_atomic32_t v,
int32_t  inc 
)
inlinestatic

Atomically add a 32-bit value to a counter and return the result.

Atomically adds the 32-bits value (inc) to the atomic counter (v) and returns the value of v after addition.

Parameters
vA pointer to the atomic counter.
incThe value to be added to the counter.
Returns
The value of v after the addition.

Definition at line 634 of file rte_atomic.h.

static int32_t rte_atomic32_sub_return ( rte_atomic32_t v,
int32_t  dec 
)
inlinestatic

Atomically subtract a 32-bit value from a counter and return the result.

Atomically subtracts the 32-bit value (inc) from the atomic counter (v) and returns the value of v after the subtraction.

Parameters
vA pointer to the atomic counter.
decThe value to be subtracted from the counter.
Returns
The value of v after the subtraction.

Definition at line 654 of file rte_atomic.h.

static int rte_atomic32_inc_and_test ( rte_atomic32_t v)
inlinestatic

Atomically increment a 32-bit counter by one and test.

Atomically increments the atomic counter (v) by one and returns true if the result is 0, or false in all other cases.

Parameters
vA pointer to the atomic counter.
Returns
True if the result after the increment operation is 0; false otherwise.
static int rte_atomic32_dec_and_test ( rte_atomic32_t v)
inlinestatic

Atomically decrement a 32-bit counter by one and test.

Atomically decrements the atomic counter (v) by one and returns true if the result is 0, or false in all other cases.

Parameters
vA pointer to the atomic counter.
Returns
True if the result after the decrement operation is 0; false otherwise.
static int rte_atomic32_test_and_set ( rte_atomic32_t v)
inlinestatic

Atomically test and set a 32-bit atomic counter.

If the counter value is already set, return 0 (failed). Otherwise, set the counter value to 1 and return 1 (success).

Parameters
vA pointer to the atomic counter.
Returns
0 if failed; else 1, success.
Examples:
examples/multi_process/client_server_mp/mp_server/main.c, and examples/server_node_efd/server/main.c.
static void rte_atomic32_clear ( rte_atomic32_t v)
inlinestatic

Atomically set a 32-bit counter to 0.

Parameters
vA pointer to the atomic counter.

Definition at line 725 of file rte_atomic.h.

static int rte_atomic64_cmpset ( volatile uint64_t *  dst,
uint64_t  exp,
uint64_t  src 
)
inlinestatic

An atomic compare and set function used by the mutex functions. (atomic) equivalent to: if (*dst == exp) *dst = src (all 64-bit words)

Parameters
dstThe destination into which the value will be written.
expThe expected value.
srcThe new value.
Returns
Non-zero on success; 0 on failure.
Examples:
examples/performance-thread/common/lthread.c, examples/performance-thread/common/lthread_mutex.c, and examples/performance-thread/common/lthread_tls.c.
static uint64_t rte_atomic64_exchange ( volatile uint64_t *  dst,
uint64_t  val 
)
inlinestatic

Atomic exchange.

(atomic) equivalent to: ret = *dst *dst = val; return ret;

Parameters
dstThe destination location into which the value will be written.
valThe new value.
Returns
The original value at that location
static void rte_atomic64_init ( rte_atomic64_t v)
inlinestatic

Initialize the atomic counter.

Parameters
vA pointer to the atomic counter.
Examples:
examples/performance-thread/common/lthread_mutex.c, and examples/vm_power_manager/channel_manager.c.
static int64_t rte_atomic64_read ( rte_atomic64_t v)
inlinestatic

Atomically read a 64-bit counter.

Parameters
vA pointer to the atomic counter.
Returns
The value of the counter.
Examples:
examples/performance-thread/common/lthread_mutex.c, examples/tep_termination/main.c, examples/vhost/main.c, and examples/vm_power_manager/channel_manager.c.
static void rte_atomic64_set ( rte_atomic64_t v,
int64_t  new_value 
)
inlinestatic

Atomically set a 64-bit counter.

Parameters
vA pointer to the atomic counter.
new_valueThe new value of the counter.
Examples:
examples/vm_power_manager/channel_manager.c.
static void rte_atomic64_add ( rte_atomic64_t v,
int64_t  inc 
)
inlinestatic

Atomically add a 64-bit value to a counter.

Parameters
vA pointer to the atomic counter.
incThe value to be added to the counter.
Examples:
examples/tep_termination/main.c, examples/tep_termination/vxlan_setup.c, and examples/vhost/main.c.
static void rte_atomic64_sub ( rte_atomic64_t v,
int64_t  dec 
)
inlinestatic

Atomically subtract a 64-bit value from a counter.

Parameters
vA pointer to the atomic counter.
decThe value to be subtracted from the counter.
static void rte_atomic64_inc ( rte_atomic64_t v)
inlinestatic

Atomically increment a 64-bit counter by one and test.

Parameters
vA pointer to the atomic counter.
Examples:
examples/performance-thread/common/lthread_mutex.c, and examples/vhost/main.c.
static void rte_atomic64_dec ( rte_atomic64_t v)
inlinestatic

Atomically decrement a 64-bit counter by one and test.

Parameters
vA pointer to the atomic counter.
Examples:
examples/performance-thread/common/lthread_mutex.c.
static int64_t rte_atomic64_add_return ( rte_atomic64_t v,
int64_t  inc 
)
inlinestatic

Add a 64-bit value to an atomic counter and return the result.

Atomically adds the 64-bit value (inc) to the atomic counter (v) and returns the value of v after the addition.

Parameters
vA pointer to the atomic counter.
incThe value to be added to the counter.
Returns
The value of v after the addition.
static int64_t rte_atomic64_sub_return ( rte_atomic64_t v,
int64_t  dec 
)
inlinestatic

Subtract a 64-bit value from an atomic counter and return the result.

Atomically subtracts the 64-bit value (dec) from the atomic counter (v) and returns the value of v after the subtraction.

Parameters
vA pointer to the atomic counter.
decThe value to be subtracted from the counter.
Returns
The value of v after the subtraction.
static int rte_atomic64_inc_and_test ( rte_atomic64_t v)
inlinestatic

Atomically increment a 64-bit counter by one and test.

Atomically increments the atomic counter (v) by one and returns true if the result is 0, or false in all other cases.

Parameters
vA pointer to the atomic counter.
Returns
True if the result after the addition is 0; false otherwise.
static int rte_atomic64_dec_and_test ( rte_atomic64_t v)
inlinestatic

Atomically decrement a 64-bit counter by one and test.

Atomically decrements the atomic counter (v) by one and returns true if the result is 0, or false in all other cases.

Parameters
vA pointer to the atomic counter.
Returns
True if the result after subtraction is 0; false otherwise.
static int rte_atomic64_test_and_set ( rte_atomic64_t v)
inlinestatic

Atomically test and set a 64-bit atomic counter.

If the counter value is already set, return 0 (failed). Otherwise, set the counter value to 1 and return 1 (success).

Parameters
vA pointer to the atomic counter.
Returns
0 if failed; else 1, success.
static void rte_atomic64_clear ( rte_atomic64_t v)
inlinestatic

Atomically set a 64-bit counter to 0.

Parameters
vA pointer to the atomic counter.