35 #include <rte_compat.h> 37 #include <rte_config.h> 128 #if RTE_CACHE_LINE_SIZE == 64 149 return mb->buf_iova + mb->data_off;
167 return mb->buf_iova + RTE_PKTMBUF_HEADROOM;
276 static inline uint32_t
282 return mbp_priv->
flags;
291 #define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF (1 << 0) 300 #define RTE_MBUF_HAS_PINNED_EXTBUF(mb) \ 301 (rte_pktmbuf_priv_flags(mb->pool) & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) 303 #ifdef RTE_LIBRTE_MBUF_DEBUG 306 #define __rte_mbuf_sanity_check(m, is_h) rte_mbuf_sanity_check(m, is_h) 311 #define __rte_mbuf_sanity_check(m, is_h) do { } while (0) 315 #ifdef RTE_MBUF_REFCNT_ATOMIC 324 static inline uint16_t
327 return __atomic_load_n(&m->
refcnt, __ATOMIC_RELAXED);
340 __atomic_store_n(&m->
refcnt, new_value, __ATOMIC_RELAXED);
344 static inline uint16_t
345 __rte_mbuf_refcnt_update(
struct rte_mbuf *m, int16_t value)
347 return __atomic_add_fetch(&m->
refcnt, (uint16_t)value,
360 static inline uint16_t
373 return (uint16_t)value;
376 return __rte_mbuf_refcnt_update(m, value);
382 static inline uint16_t
383 __rte_mbuf_refcnt_update(
struct rte_mbuf *m, int16_t value)
392 static inline uint16_t
395 return __rte_mbuf_refcnt_update(m, value);
401 static inline uint16_t
426 static inline uint16_t
429 return __atomic_load_n(&shinfo->refcnt, __ATOMIC_RELAXED);
444 __atomic_store_n(&shinfo->refcnt, new_value, __ATOMIC_RELAXED);
458 static inline uint16_t
465 return (uint16_t)value;
468 return __atomic_add_fetch(&shinfo->refcnt, (uint16_t)value,
473 #define RTE_MBUF_PREFETCH_TO_FREE(m) do { \ 514 const char **reason);
532 RTE_ASSERT(m->next == NULL);
533 RTE_ASSERT(m->nb_segs == 1);
538 #define MBUF_RAW_ALLOC_CHECK(m) __rte_mbuf_raw_sanity_check(m) 615 void *m,
unsigned i);
675 unsigned cache_size, uint16_t priv_size, uint16_t data_room_size,
717 unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size,
718 int socket_id,
const char *ops_name);
772 unsigned int cache_size, uint16_t priv_size,
773 uint16_t data_room_size,
int socket_id,
775 unsigned int ext_num);
788 static inline uint16_t
809 static inline uint16_t
828 m->data_off = (uint16_t)
RTE_MIN((uint16_t)RTE_PKTMBUF_HEADROOM,
894 struct rte_mbuf **mbufs,
unsigned count)
910 while (idx != count) {
977 if (addr <= buf_addr)
1056 RTE_ASSERT(shinfo->
free_cb != NULL);
1059 m->buf_iova = buf_iova;
1076 #define rte_pktmbuf_detach_extbuf(m) rte_pktmbuf_detach(m) 1100 mdst->hash = msrc->hash;
1142 __rte_pktmbuf_copy_hdr(mi, m);
1144 mi->data_off = m->data_off;
1146 mi->buf_iova = m->buf_iova;
1166 __rte_pktmbuf_free_extbuf(
struct rte_mbuf *m)
1169 RTE_ASSERT(m->
shinfo != NULL);
1182 __rte_pktmbuf_free_direct(
struct rte_mbuf *m)
1219 uint32_t mbuf_size, buf_len;
1238 __rte_pktmbuf_free_extbuf(m);
1240 __rte_pktmbuf_free_direct(m);
1243 mbuf_size = (uint32_t)(
sizeof(
struct rte_mbuf) + priv_size);
1247 m->
buf_addr = (
char *)m + mbuf_size;
1249 m->
buf_len = (uint16_t)buf_len;
1268 static inline int __rte_pktmbuf_pinned_extbuf_decref(
struct rte_mbuf *m)
1284 if (
likely(__atomic_add_fetch(&shinfo->refcnt, (uint16_t)-1,
1318 __rte_pktmbuf_pinned_extbuf_decref(m))
1322 if (m->
next != NULL)
1329 }
else if (__rte_mbuf_refcnt_update(m, -1) == 0) {
1335 __rte_pktmbuf_pinned_extbuf_decref(m))
1339 if (m->
next != NULL)
1447 uint32_t offset, uint32_t length);
1466 }
while ((m = m->
next) != NULL);
1509 while (m->
next != NULL)
1522 #define rte_pktmbuf_pkt_len(m) ((m)->pkt_len) 1532 #define rte_pktmbuf_data_len(m) ((m)->data_len) 1560 m->data_off = (uint16_t)(m->data_off - len);
1564 return (
char *)m->
buf_addr + m->data_off;
1596 return (
char*) tail;
1624 m->data_off = (uint16_t)(m->data_off + len);
1626 return (
char *)m->
buf_addr + m->data_off;
1676 const void *__rte_pktmbuf_read(
const struct rte_mbuf *m, uint32_t off,
1677 uint32_t len,
void *buf);
1700 uint32_t off, uint32_t len,
void *buf)
1705 return __rte_pktmbuf_read(m, off, len, buf);
1734 cur_tail->
next = tail;
1772 uint64_t ol3, uint64_t ol2, uint64_t unused)
1774 return il2 << RTE_MBUF_L2_LEN_OFS |
1775 il3 << RTE_MBUF_L3_LEN_OFS |
1776 il4 << RTE_MBUF_L4_LEN_OFS |
1777 tso << RTE_MBUF_TSO_SEGSZ_OFS |
1778 ol3 << RTE_MBUF_OUTL3_LEN_OFS |
1779 ol2 << RTE_MBUF_OUTL2_LEN_OFS |
1780 unused << RTE_MBUF_TXOFLD_UNUSED_OFS;
1815 !(ol_flags & RTE_MBUF_F_TX_IP_CKSUM)))
1829 int __rte_pktmbuf_linearize(
struct rte_mbuf *mbuf);
1848 return __rte_pktmbuf_linearize(mbuf);
1870 static inline uint32_t
1873 return m->hash.sched.queue_id;
1879 static inline uint8_t
1882 return m->hash.sched.traffic_class;
1888 static inline uint8_t
1891 return m->hash.sched.color;
1908 uint8_t *traffic_class,
1915 *color = sched.
color;
1942 m->hash.sched.color =
color;
1963 .queue_id = queue_id,
1964 .traffic_class = traffic_class,
static void rte_pktmbuf_reset(struct rte_mbuf *m)
struct rte_mbuf_ext_shared_info * shinfo
static rte_iova_t rte_mbuf_data_iova(const struct rte_mbuf *mb)
uint16_t mbuf_data_room_size
void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count)
#define __rte_always_inline
#define RTE_MBUF_F_TX_OFFLOAD_MASK
static struct rte_mbuf * rte_pktmbuf_alloc(struct rte_mempool *mp)
static void rte_mbuf_sched_traffic_class_set(struct rte_mbuf *m, uint8_t traffic_class)
static uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp)
static int rte_validate_tx_offload(const struct rte_mbuf *m)
static void rte_pktmbuf_free(struct rte_mbuf *m)
static uint32_t rte_mbuf_sched_queue_get(const struct rte_mbuf *m)
#define RTE_PTR_ALIGN_FLOOR(ptr, align)
void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg)
static __rte_always_inline void rte_pktmbuf_free_seg(struct rte_mbuf *m)
static struct rte_mbuf * rte_mbuf_from_indirect(struct rte_mbuf *mi)
rte_mbuf_extbuf_free_callback_t free_cb
static int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
static uint8_t rte_mbuf_sched_traffic_class_get(const struct rte_mbuf *m)
#define RTE_MBUF_F_TX_IPV6
struct rte_mbuf * rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp, uint32_t offset, uint32_t length)
static __rte_always_inline void __rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m)
void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len)
static uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
static int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, struct rte_mbuf **mbufs, unsigned count)
static void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
static void * rte_mbuf_to_priv(struct rte_mbuf *m)
static void rte_mbuf_prefetch_part2(struct rte_mbuf *m)
char name[RTE_MEMPOOL_NAMESIZE]
static uint16_t rte_mbuf_ext_refcnt_update(struct rte_mbuf_ext_shared_info *shinfo, int16_t value)
int rte_mbuf_check(const struct rte_mbuf *m, int is_header, const char **reason)
#define rte_pktmbuf_mtod_offset(m, t, o)
static __rte_always_inline struct rte_mbuf * rte_pktmbuf_prefree_seg(struct rte_mbuf *m)
static int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
#define RTE_PTR_ADD(ptr, x)
int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen)
#define RTE_MBUF_MAX_NB_SEGS
static void rte_mbuf_sched_set(struct rte_mbuf *m, uint32_t queue_id, uint8_t traffic_class, uint8_t color)
static uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
static __rte_always_inline void rte_mbuf_raw_free(struct rte_mbuf *m)
static void rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id, uint8_t *traffic_class, uint8_t *color)
static __rte_always_inline uint64_t rte_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso, uint64_t ol3, uint64_t ol2, uint64_t unused)
static void rte_mbuf_sched_queue_set(struct rte_mbuf *m, uint32_t queue_id)
static char * rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
void rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header)
static void rte_mbuf_sched_color_set(struct rte_mbuf *m, uint8_t color)
void(* rte_mbuf_extbuf_free_callback_t)(void *addr, void *opaque)
#define __rte_mbuf_sanity_check(m, is_h)
const char * rte_get_tx_ol_flag_name(uint64_t mask)
static uint16_t rte_mbuf_refcnt_read(const struct rte_mbuf *m)
static int rte_pktmbuf_linearize(struct rte_mbuf *mbuf)
static __rte_always_inline int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
static uint32_t rte_pktmbuf_priv_flags(struct rte_mempool *mp)
static char * rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
#define RTE_MBUF_DIRECT(mb)
#define RTE_MBUF_CLONED(mb)
static void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
static __rte_always_inline int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
#define RTE_MBUF_F_TX_OUTER_IP_CKSUM
static void rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
static void rte_pktmbuf_detach(struct rte_mbuf *m)
static uint16_t rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
#define rte_pktmbuf_data_len(m)
#define RTE_MBUF_F_TX_L4_MASK
static uint16_t rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
#define RTE_MBUF_F_TX_IP_CKSUM
static uint16_t rte_pktmbuf_data_room_size(struct rte_mempool *mp)
#define RTE_MBUF_F_TX_TCP_SEG
#define RTE_MBUF_F_INDIRECT
#define RTE_MBUF_PORT_INVALID
const char * rte_get_rx_ol_flag_name(uint64_t mask)
static struct rte_mbuf_ext_shared_info * rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len, rte_mbuf_extbuf_free_callback_t free_cb, void *fcb_opaque)
#define RTE_MBUF_F_TX_OUTER_IPV4
void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg, void *m, unsigned i)
#define RTE_MBUF_HAS_EXTBUF(mb)
struct rte_mempool * pool
static void rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo, uint16_t new_value)
static char * rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
static void rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
struct rte_mempool * rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n, unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id, const char *ops_name)
static rte_iova_t rte_mbuf_data_iova_default(const struct rte_mbuf *mb)
static int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
static char * rte_mbuf_to_baddr(struct rte_mbuf *md)
static const void * rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off, uint32_t len, void *buf)
static char * rte_pktmbuf_prepend(struct rte_mbuf *m, uint16_t len)
static struct rte_mbuf * rte_mbuf_raw_alloc(struct rte_mempool *mp)
static void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
#define RTE_PTR_SUB(ptr, x)
static struct rte_mbuf * rte_pktmbuf_lastseg(struct rte_mbuf *m)
__rte_experimental struct rte_mempool * rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n, unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id, const struct rte_pktmbuf_extmem *ext_mem, unsigned int ext_num)
static uint8_t rte_mbuf_sched_color_get(const struct rte_mbuf *m)
#define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF
static char * rte_mbuf_data_addr_default(struct rte_mbuf *mb)
static rte_iova_t rte_mempool_virt2iova(const void *elt)
static __rte_always_inline void rte_mempool_put(struct rte_mempool *mp, void *obj)
#define RTE_MBUF_F_TX_IPV4
#define RTE_PTR_DIFF(ptr1, ptr2)
static void rte_mbuf_prefetch_part1(struct rte_mbuf *m)
#define RTE_MBUF_F_EXTERNAL
static void * rte_mempool_get_priv(struct rte_mempool *mp)
#define RTE_MBUF_HAS_PINNED_EXTBUF(mb)
struct rte_mbuf * rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp)
int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen)
static void rte_prefetch0(const volatile void *p)
struct rte_mempool * rte_pktmbuf_pool_create(const char *name, unsigned n, unsigned cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id)
static void rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr, rte_iova_t buf_iova, uint16_t buf_len, struct rte_mbuf_ext_shared_info *shinfo)