35#include <rte_compat.h>
37#include <rte_config.h>
130#if RTE_CACHE_LINE_SIZE == 64
151 return mb->buf_iova + mb->data_off;
169 return mb->buf_iova + RTE_PKTMBUF_HEADROOM;
278static inline uint32_t
284 return mbp_priv->
flags;
293#define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF (1 << 0)
302#define RTE_MBUF_HAS_PINNED_EXTBUF(mb) \
303 (rte_pktmbuf_priv_flags(mb->pool) & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF)
305#ifdef RTE_LIBRTE_MBUF_DEBUG
308#define __rte_mbuf_sanity_check(m, is_h) rte_mbuf_sanity_check(m, is_h)
313#define __rte_mbuf_sanity_check(m, is_h) do { } while (0)
317#ifdef RTE_MBUF_REFCNT_ATOMIC
326static inline uint16_t
329 return __atomic_load_n(&m->
refcnt, __ATOMIC_RELAXED);
342 __atomic_store_n(&m->
refcnt, new_value, __ATOMIC_RELAXED);
346static inline uint16_t
347__rte_mbuf_refcnt_update(
struct rte_mbuf *m, int16_t value)
349 return __atomic_add_fetch(&m->
refcnt, (uint16_t)value,
362static inline uint16_t
375 return (uint16_t)value;
378 return __rte_mbuf_refcnt_update(m, value);
384static inline uint16_t
385__rte_mbuf_refcnt_update(
struct rte_mbuf *m, int16_t value)
394static inline uint16_t
397 return __rte_mbuf_refcnt_update(m, value);
403static inline uint16_t
428static inline uint16_t
431 return __atomic_load_n(&shinfo->refcnt, __ATOMIC_RELAXED);
446 __atomic_store_n(&shinfo->refcnt, new_value, __ATOMIC_RELAXED);
460static inline uint16_t
467 return (uint16_t)value;
470 return __atomic_add_fetch(&shinfo->refcnt, (uint16_t)value,
475#define RTE_MBUF_PREFETCH_TO_FREE(m) do { \
516 const char **reason);
534 RTE_ASSERT(m->next == NULL);
535 RTE_ASSERT(m->nb_segs == 1);
540#define MBUF_RAW_ALLOC_CHECK(m) __rte_mbuf_raw_sanity_check(m)
617 void *m,
unsigned i);
676 unsigned cache_size, uint16_t priv_size, uint16_t data_room_size,
717 unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size,
774 unsigned int ext_num);
787static inline uint16_t
808static inline uint16_t
827 m->data_off = (uint16_t)
RTE_MIN((uint16_t)RTE_PKTMBUF_HEADROOM,
893 struct rte_mbuf **mbufs,
unsigned count)
909 while (idx != count) {
976 if (addr <= buf_addr)
1055 RTE_ASSERT(shinfo->
free_cb != NULL);
1058 m->buf_iova = buf_iova;
1075#define rte_pktmbuf_detach_extbuf(m) rte_pktmbuf_detach(m)
1099 mdst->hash = msrc->hash;
1141 __rte_pktmbuf_copy_hdr(mi, m);
1143 mi->data_off = m->data_off;
1145 mi->buf_iova = m->buf_iova;
1165__rte_pktmbuf_free_extbuf(
struct rte_mbuf *m)
1168 RTE_ASSERT(m->
shinfo != NULL);
1181__rte_pktmbuf_free_direct(
struct rte_mbuf *m)
1218 uint32_t mbuf_size, buf_len;
1237 __rte_pktmbuf_free_extbuf(m);
1239 __rte_pktmbuf_free_direct(m);
1242 mbuf_size = (uint32_t)(
sizeof(
struct rte_mbuf) + priv_size);
1246 m->
buf_addr = (
char *)m + mbuf_size;
1248 m->
buf_len = (uint16_t)buf_len;
1267static inline int __rte_pktmbuf_pinned_extbuf_decref(
struct rte_mbuf *m)
1283 if (
likely(__atomic_add_fetch(&shinfo->refcnt, (uint16_t)-1,
1317 __rte_pktmbuf_pinned_extbuf_decref(m))
1321 if (m->
next != NULL)
1328 }
else if (__rte_mbuf_refcnt_update(m, -1) == 0) {
1334 __rte_pktmbuf_pinned_extbuf_decref(m))
1338 if (m->
next != NULL)
1446 uint32_t offset, uint32_t length);
1465 }
while ((m = m->
next) != NULL);
1508 while (m->
next != NULL)
1521#define rte_pktmbuf_pkt_len(m) ((m)->pkt_len)
1531#define rte_pktmbuf_data_len(m) ((m)->data_len)
1559 m->data_off = (uint16_t)(m->data_off - len);
1563 return (
char *)m->
buf_addr + m->data_off;
1595 return (
char*) tail;
1623 m->data_off = (uint16_t)(m->data_off + len);
1625 return (
char *)m->
buf_addr + m->data_off;
1675const void *__rte_pktmbuf_read(
const struct rte_mbuf *m, uint32_t off,
1676 uint32_t len,
void *buf);
1699 uint32_t off, uint32_t len,
void *buf)
1704 return __rte_pktmbuf_read(m, off, len, buf);
1733 cur_tail->
next = tail;
1771 uint64_t ol3, uint64_t ol2, uint64_t unused)
1773 return il2 << RTE_MBUF_L2_LEN_OFS |
1774 il3 << RTE_MBUF_L3_LEN_OFS |
1775 il4 << RTE_MBUF_L4_LEN_OFS |
1776 tso << RTE_MBUF_TSO_SEGSZ_OFS |
1777 ol3 << RTE_MBUF_OUTL3_LEN_OFS |
1778 ol2 << RTE_MBUF_OUTL2_LEN_OFS |
1779 unused << RTE_MBUF_TXOFLD_UNUSED_OFS;
1828int __rte_pktmbuf_linearize(
struct rte_mbuf *mbuf);
1847 return __rte_pktmbuf_linearize(mbuf);
1869static inline uint32_t
1872 return m->hash.sched.queue_id;
1878static inline uint8_t
1881 return m->hash.sched.traffic_class;
1887static inline uint8_t
1890 return m->hash.sched.color;
1907 uint8_t *traffic_class,
1941 m->hash.sched.color =
color;
#define RTE_PTR_ALIGN_FLOOR(ptr, align)
#define RTE_PTR_DIFF(ptr1, ptr2)
#define RTE_PTR_SUB(ptr, x)
#define RTE_PTR_ADD(ptr, x)
#define __rte_always_inline
static __rte_always_inline void rte_pktmbuf_free_seg(struct rte_mbuf *m)
int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen)
static void rte_mbuf_prefetch_part2(struct rte_mbuf *m)
static __rte_always_inline uint64_t rte_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso, uint64_t ol3, uint64_t ol2, uint64_t unused)
static void rte_pktmbuf_free(struct rte_mbuf *m)
static rte_iova_t rte_mbuf_data_iova_default(const struct rte_mbuf *mb)
static rte_iova_t rte_mbuf_data_iova(const struct rte_mbuf *mb)
static uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen)
struct rte_mbuf * rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp)
#define rte_pktmbuf_data_len(m)
static uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
static uint32_t rte_mbuf_sched_queue_get(const struct rte_mbuf *m)
static void rte_mbuf_prefetch_part1(struct rte_mbuf *m)
static char * rte_pktmbuf_prepend(struct rte_mbuf *m, uint16_t len)
static char * rte_mbuf_data_addr_default(struct rte_mbuf *mb)
struct rte_mempool * rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n, unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id, const char *ops_name)
static void rte_mbuf_sched_set(struct rte_mbuf *m, uint32_t queue_id, uint8_t traffic_class, uint8_t color)
static void rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
static char * rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
static void rte_mbuf_sched_color_set(struct rte_mbuf *m, uint8_t color)
static int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
int rte_mbuf_check(const struct rte_mbuf *m, int is_header, const char **reason)
static char * rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
const char * rte_get_tx_ol_flag_name(uint64_t mask)
static void rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id, uint8_t *traffic_class, uint8_t *color)
static void rte_pktmbuf_detach(struct rte_mbuf *m)
static char * rte_mbuf_to_baddr(struct rte_mbuf *md)
static struct rte_mbuf_ext_shared_info * rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len, rte_mbuf_extbuf_free_callback_t free_cb, void *fcb_opaque)
static uint32_t rte_pktmbuf_priv_flags(struct rte_mempool *mp)
static void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg, void *m, unsigned i)
static void rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
static struct rte_mbuf * rte_mbuf_raw_alloc(struct rte_mempool *mp)
static uint16_t rte_mbuf_ext_refcnt_update(struct rte_mbuf_ext_shared_info *shinfo, int16_t value)
struct rte_mempool * rte_pktmbuf_pool_create(const char *name, unsigned n, unsigned cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id)
void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count)
void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len)
#define __rte_mbuf_sanity_check(m, is_h)
static const void * rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off, uint32_t len, void *buf)
static uint16_t rte_mbuf_refcnt_read(const struct rte_mbuf *m)
static int rte_pktmbuf_linearize(struct rte_mbuf *mbuf)
__rte_experimental struct rte_mempool * rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n, unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id, const struct rte_pktmbuf_extmem *ext_mem, unsigned int ext_num)
static __rte_always_inline void rte_mbuf_raw_free(struct rte_mbuf *m)
struct rte_mbuf * rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp, uint32_t offset, uint32_t length)
static struct rte_mbuf * rte_pktmbuf_alloc(struct rte_mempool *mp)
static char * rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
static void rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo, uint16_t new_value)
void rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header)
static uint8_t rte_mbuf_sched_traffic_class_get(const struct rte_mbuf *m)
static uint16_t rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
static uint16_t rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
static int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
#define RTE_MBUF_HAS_PINNED_EXTBUF(mb)
static void rte_mbuf_sched_queue_set(struct rte_mbuf *m, uint32_t queue_id)
static uint16_t rte_pktmbuf_data_room_size(struct rte_mempool *mp)
static struct rte_mbuf * rte_pktmbuf_lastseg(struct rte_mbuf *m)
static void rte_mbuf_sched_traffic_class_set(struct rte_mbuf *m, uint8_t traffic_class)
#define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF
static __rte_always_inline struct rte_mbuf * rte_pktmbuf_prefree_seg(struct rte_mbuf *m)
static int rte_validate_tx_offload(const struct rte_mbuf *m)
static void rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr, rte_iova_t buf_iova, uint16_t buf_len, struct rte_mbuf_ext_shared_info *shinfo)
static struct rte_mbuf * rte_mbuf_from_indirect(struct rte_mbuf *mi)
static int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, struct rte_mbuf **mbufs, unsigned count)
static void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
static void * rte_mbuf_to_priv(struct rte_mbuf *m)
static uint8_t rte_mbuf_sched_color_get(const struct rte_mbuf *m)
static int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
static __rte_always_inline void __rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m)
const char * rte_get_rx_ol_flag_name(uint64_t mask)
void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg)
static uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp)
static void rte_pktmbuf_reset(struct rte_mbuf *m)
static void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
#define RTE_MBUF_F_TX_OUTER_IP_CKSUM
#define RTE_MBUF_F_EXTERNAL
#define RTE_MBUF_F_TX_IP_CKSUM
#define RTE_MBUF_MAX_NB_SEGS
#define RTE_MBUF_F_TX_TCP_SEG
#define RTE_MBUF_F_TX_L4_MASK
#define RTE_MBUF_F_TX_OUTER_IPV4
#define RTE_MBUF_HAS_EXTBUF(mb)
#define RTE_MBUF_F_TX_OFFLOAD_MASK
#define RTE_MBUF_CLONED(mb)
#define RTE_MBUF_DIRECT(mb)
#define RTE_MBUF_F_TX_IPV4
void(* rte_mbuf_extbuf_free_callback_t)(void *addr, void *opaque)
#define rte_pktmbuf_mtod_offset(m, t, o)
#define RTE_MBUF_F_TX_IPV6
#define RTE_MBUF_F_INDIRECT
#define RTE_MBUF_PORT_INVALID
static __rte_always_inline int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
static rte_iova_t rte_mempool_virt2iova(const void *elt)
static __rte_always_inline int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
static __rte_always_inline void rte_mempool_put(struct rte_mempool *mp, void *obj)
static void * rte_mempool_get_priv(struct rte_mempool *mp)
static void rte_prefetch0(const volatile void *p)
rte_mbuf_extbuf_free_callback_t free_cb
struct rte_mbuf_ext_shared_info * shinfo
struct rte_mempool * pool
char name[RTE_MEMPOOL_NAMESIZE]
uint16_t mbuf_data_room_size