DPDK  22.11.7-rc1
examples/ipsec-secgw/ipsec_process.c
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2016-2020 Intel Corporation
*/
#include <sys/types.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <rte_log.h>
#include <rte_cryptodev.h>
#include <rte_ethdev.h>
#include <rte_mbuf.h>
#include "ipsec.h"
#include "ipsec-secgw.h"
#include "ipsec_worker.h"
/* helper routine to free bulk of crypto-ops and related packets */
static inline void
free_cops(struct rte_crypto_op *cop[], uint32_t n)
{
uint32_t i;
for (i = 0; i != n; i++)
rte_pktmbuf_free(cop[i]->sym->m_src);
}
/* helper routine to enqueue bulk of crypto ops */
static inline void
enqueue_cop_bulk(struct cdev_qp *cqp, struct rte_crypto_op *cop[], uint32_t num)
{
uint32_t i, k, len, n;
len = cqp->len;
/*
* if cqp is empty and we have enough ops,
* then queue them to the PMD straightway.
*/
if (num >= RTE_DIM(cqp->buf) * 3 / 4 && len == 0) {
n = rte_cryptodev_enqueue_burst(cqp->id, cqp->qp, cop, num);
cqp->in_flight += n;
free_cops(cop + n, num - n);
return;
}
k = 0;
do {
n = RTE_DIM(cqp->buf) - len;
n = RTE_MIN(num - k, n);
/* put packets into cqp */
for (i = 0; i != n; i++)
cqp->buf[len + i] = cop[k + i];
len += n;
k += n;
/* if cqp is full then, enqueue crypto-ops to PMD */
if (len == RTE_DIM(cqp->buf)) {
n = rte_cryptodev_enqueue_burst(cqp->id, cqp->qp,
cqp->buf, len);
cqp->in_flight += n;
free_cops(cqp->buf + n, len - n);
len = 0;
}
} while (k != num);
cqp->len = len;
}
static inline int
check_ipsec_session(const struct rte_ipsec_session *ss)
{
if (ss->crypto.ses == NULL)
return -ENOENT;
if (ss->security.ses == NULL)
return -ENOENT;
} else
RTE_ASSERT(0);
return 0;
}
/*
* group input packets byt the SA they belong to.
*/
static uint32_t
sa_group(void *sa_ptr[], struct rte_mbuf *pkts[],
struct rte_ipsec_group grp[], uint32_t num)
{
uint32_t i, n, spi;
void *sa;
void * const nosa = &spi;
sa = nosa;
grp[0].m = pkts;
for (i = 0, n = 0; i != num; i++) {
if (sa != sa_ptr[i]) {
grp[n].cnt = pkts + i - grp[n].m;
n += (sa != nosa);
grp[n].id.ptr = sa_ptr[i];
grp[n].m = pkts + i;
sa = sa_ptr[i];
}
}
/* terminate last group */
if (sa != nosa) {
grp[n].cnt = pkts + i - grp[n].m;
n++;
}
return n;
}
/*
* helper function, splits processed packets into ipv4/ipv6 traffic.
*/
static inline void
copy_to_trf(struct ipsec_traffic *trf, uint64_t satp, struct rte_mbuf *mb[],
uint32_t num)
{
uint32_t j, ofs, s;
struct traffic_type *out;
/*
* determine traffic type(ipv4/ipv6) and offset for ACL classify
* based on SA type
*/
if ((satp & RTE_IPSEC_SATP_DIR_MASK) == RTE_IPSEC_SATP_DIR_IB) {
if ((satp & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4) {
out = &trf->ip4;
ofs = offsetof(struct ip, ip_p);
} else {
out = &trf->ip6;
ofs = offsetof(struct ip6_hdr, ip6_nxt);
}
} else if (SATP_OUT_IPV4(satp)) {
out = &trf->ip4;
ofs = offsetof(struct ip, ip_p);
} else {
out = &trf->ip6;
ofs = offsetof(struct ip6_hdr, ip6_nxt);
}
for (j = 0, s = out->num; j != num; j++) {
out->data[s + j] = rte_pktmbuf_mtod_offset(mb[j],
void *, ofs);
out->pkts[s + j] = mb[j];
}
out->num += num;
}
static uint32_t
ipsec_prepare_crypto_group(struct ipsec_ctx *ctx, struct ipsec_sa *sa,
struct rte_ipsec_session *ips, struct rte_mbuf **m,
unsigned int cnt)
{
struct cdev_qp *cqp;
struct rte_crypto_op *cop[cnt];
uint32_t j, k;
struct ipsec_mbuf_metadata *priv;
cqp = sa->cqp[ctx->lcore_id];
/* for that app each mbuf has it's own crypto op */
for (j = 0; j != cnt; j++) {
priv = get_priv(m[j]);
cop[j] = &priv->cop;
/*
* this is just to satisfy inbound_sa_check()
* should be removed in future.
*/
priv->sa = sa;
}
/* prepare and enqueue crypto ops */
k = rte_ipsec_pkt_crypto_prepare(ips, m, cop, cnt);
if (k != 0)
enqueue_cop_bulk(cqp, cop, k);
return k;
}
/*
* finish processing of packets successfully decrypted by an inline processor
*/
static uint32_t
ipsec_process_inline_group(struct rte_ipsec_session *ips, void *sa,
struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t cnt)
{
uint64_t satp;
uint32_t k;
/* get SA type */
satp = rte_ipsec_sa_type(ips->sa);
prep_process_group(sa, mb, cnt);
k = rte_ipsec_pkt_process(ips, mb, cnt);
copy_to_trf(trf, satp, mb, k);
return k;
}
/*
* process packets synchronously
*/
static uint32_t
ipsec_process_cpu_group(struct rte_ipsec_session *ips, void *sa,
struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t cnt)
{
uint64_t satp;
uint32_t k;
/* get SA type */
satp = rte_ipsec_sa_type(ips->sa);
prep_process_group(sa, mb, cnt);
k = rte_ipsec_pkt_cpu_prepare(ips, mb, cnt);
k = rte_ipsec_pkt_process(ips, mb, k);
copy_to_trf(trf, satp, mb, k);
return k;
}
/*
* Process ipsec packets.
* If packet belong to SA that is subject of inline-crypto,
* then process it immediately.
* Otherwise do necessary preparations and queue it to related
* crypto-dev queue.
*/
void
ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
{
uint32_t i, k, n;
struct ipsec_sa *sa;
struct rte_ipsec_group *pg;
struct rte_ipsec_session *ips;
struct rte_ipsec_group grp[RTE_DIM(trf->ipsec.pkts)];
n = sa_group(trf->ipsec.saptr, trf->ipsec.pkts, grp, trf->ipsec.num);
for (i = 0; i != n; i++) {
pg = grp + i;
sa = ipsec_mask_saptr(pg->id.ptr);
/* fallback to cryptodev with RX packets which inline
* processor was unable to process
*/
if (sa != NULL)
ips = (pg->id.val & IPSEC_SA_OFFLOAD_FALLBACK_FLAG) ?
ipsec_get_fallback_session(sa) :
ipsec_get_primary_session(sa);
/* no valid HW session for that SA */
if (sa == NULL || unlikely(check_ipsec_session(ips) != 0))
k = 0;
/* process packets inline */
else {
switch (ips->type) {
/* enqueue packets to crypto dev */
k = ipsec_prepare_crypto_group(ctx, sa, ips,
pg->m, pg->cnt);
break;
k = ipsec_process_inline_group(ips, sa,
trf, pg->m, pg->cnt);
break;
k = ipsec_process_cpu_group(ips, sa,
trf, pg->m, pg->cnt);
break;
default:
k = 0;
}
}
/* drop packets that cannot be enqueued/processed */
if (k != pg->cnt)
free_pkts(pg->m + k, pg->cnt - k);
}
}
static inline uint32_t
cqp_dequeue(struct cdev_qp *cqp, struct rte_crypto_op *cop[], uint32_t num)
{
uint32_t n;
if (cqp->in_flight == 0)
return 0;
n = rte_cryptodev_dequeue_burst(cqp->id, cqp->qp, cop, num);
RTE_ASSERT(cqp->in_flight >= n);
cqp->in_flight -= n;
return n;
}
static inline uint32_t
ctx_dequeue(struct ipsec_ctx *ctx, struct rte_crypto_op *cop[], uint32_t num)
{
uint32_t i, n;
n = 0;
for (i = ctx->last_qp; n != num && i != ctx->nb_qps; i++)
n += cqp_dequeue(ctx->tbl + i, cop + n, num - n);
for (i = 0; n != num && i != ctx->last_qp; i++)
n += cqp_dequeue(ctx->tbl + i, cop + n, num - n);
ctx->last_qp = i;
return n;
}
/*
* dequeue packets from crypto-queues and finalize processing.
*/
void
ipsec_cqp_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
{
uint64_t satp;
uint32_t i, k, n, ng;
struct rte_ipsec_session *ss;
struct traffic_type *out;
struct rte_ipsec_group *pg;
const int nb_cops = RTE_DIM(trf->ipsec.pkts);
struct rte_crypto_op *cop[RTE_DIM(trf->ipsec.pkts)];
struct rte_ipsec_group grp[RTE_DIM(trf->ipsec.pkts)];
trf->ip4.num = 0;
trf->ip6.num = 0;
out = &trf->ipsec;
/* dequeue completed crypto-ops */
n = ctx_dequeue(ctx, cop, RTE_MIN(MAX_PKT_BURST, nb_cops));
if (n == 0)
return;
/* group them by ipsec session */
(uintptr_t)cop, out->pkts, grp, n);
/* process each group of packets */
for (i = 0; i != ng; i++) {
pg = grp + i;
ss = pg->id.ptr;
satp = rte_ipsec_sa_type(ss->sa);
k = rte_ipsec_pkt_process(ss, pg->m, pg->cnt);
copy_to_trf(trf, satp, pg->m, k);
/* free bad packets, if any */
free_pkts(pg->m + k, pg->cnt - k);
n -= pg->cnt;
}
/* we should never have packet with unknown SA here */
RTE_VERIFY(n == 0);
}