examples/ipsec-secgw/ipsec_process.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2016-2020 Intel Corporation
 */
#include <sys/types.h>
#include <netinet/in.h>
#include <netinet/ip.h>

#include <rte_branch_prediction.h>
#include <rte_log.h>
#include <rte_cryptodev.h>
#include <rte_ethdev.h>
#include <rte_mbuf.h>

#include "ipsec.h"
#include "ipsec-secgw.h"
#include "ipsec_worker.h"

/* helper routine to free bulk of crypto-ops and related packets */
static inline void
free_cops(struct rte_crypto_op *cop[], uint32_t n)
{
    uint32_t i;

    for (i = 0; i != n; i++)
        free_pkts(&cop[i]->sym->m_src, 1);
}

/* helper routine to enqueue bulk of crypto ops */
static inline void
enqueue_cop_bulk(struct cdev_qp *cqp, struct rte_crypto_op *cop[], uint32_t num)
{
    uint32_t i, k, len, n;

    len = cqp->len;

    /*
     * if cqp is empty and we have enough ops,
     * then queue them to the PMD straightway.
     */
    if (num >= RTE_DIM(cqp->buf) * 3 / 4 && len == 0) {
        n = rte_cryptodev_enqueue_burst(cqp->id, cqp->qp, cop, num);
        cqp->in_flight += n;
        free_cops(cop + n, num - n);
        return;
    }

    k = 0;

    do {
        n = RTE_DIM(cqp->buf) - len;
        n = RTE_MIN(num - k, n);

        /* put packets into cqp */
        for (i = 0; i != n; i++)
            cqp->buf[len + i] = cop[k + i];

        len += n;
        k += n;

        /* if cqp is full then, enqueue crypto-ops to PMD */
        if (len == RTE_DIM(cqp->buf)) {
            n = rte_cryptodev_enqueue_burst(cqp->id, cqp->qp,
                    cqp->buf, len);
            cqp->in_flight += n;
            free_cops(cqp->buf + n, len - n);
            len = 0;
        }


    } while (k != num);

    cqp->len = len;
}

static inline int
check_ipsec_session(const struct rte_ipsec_session *ss)
{
    if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE ||
            ss->type == RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO) {
        if (ss->crypto.ses == NULL)
            return -ENOENT;
    } else if (ss->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
        if (ss->security.ses == NULL)
            return -ENOENT;
    } else
        RTE_ASSERT(0);
    return 0;
}

/*
 * group input packets byt the SA they belong to.
 */
static uint32_t
sa_group(void *sa_ptr[], struct rte_mbuf *pkts[],
    struct rte_ipsec_group grp[], uint32_t num)
{
    uint32_t i, n, spi;
    void *sa;
    void * const nosa = &spi;

    sa = nosa;
    grp[0].m = pkts;
    for (i = 0, n = 0; i != num; i++) {

        if (sa != sa_ptr[i]) {
            grp[n].cnt = pkts + i - grp[n].m;
            n += (sa != nosa);
            grp[n].id.ptr = sa_ptr[i];
            grp[n].m = pkts + i;
            sa = sa_ptr[i];
        }
    }

    /* terminate last group */
    if (sa != nosa) {
        grp[n].cnt = pkts + i - grp[n].m;
        n++;
    }

    return n;
}

/*
 * helper function, splits processed packets into ipv4/ipv6 traffic.
 */
static inline void
copy_to_trf(struct ipsec_traffic *trf, uint64_t satp, struct rte_mbuf *mb[],
    uint32_t num)
{
    uint32_t j, ofs, s;
    struct traffic_type *out;

    /*
     * determine traffic type(ipv4/ipv6) and offset for ACL classify
     * based on SA type
     */
    if ((satp & RTE_IPSEC_SATP_DIR_MASK) == RTE_IPSEC_SATP_DIR_IB) {
        if ((satp & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4) {
            out = &trf->ip4;
            ofs = offsetof(struct ip, ip_p);
        } else {
            out = &trf->ip6;
            ofs = offsetof(struct ip6_hdr, ip6_nxt);
        }
    } else if (SATP_OUT_IPV4(satp)) {
        out = &trf->ip4;
        ofs = offsetof(struct ip, ip_p);
    } else {
        out = &trf->ip6;
        ofs = offsetof(struct ip6_hdr, ip6_nxt);
    }

    for (j = 0, s = out->num; j != num; j++) {
        out->data[s + j] = rte_pktmbuf_mtod_offset(mb[j],
                void *, ofs);
        out->pkts[s + j] = mb[j];
    }

    out->num += num;
}

static uint32_t
ipsec_prepare_crypto_group(struct ipsec_ctx *ctx, struct ipsec_sa *sa,
        struct rte_ipsec_session *ips, struct rte_mbuf **m,
        unsigned int cnt)
{
    struct cdev_qp *cqp;
    struct rte_crypto_op *cop[cnt];
    uint32_t j, k;
    struct ipsec_mbuf_metadata *priv;

    cqp = sa->cqp[ctx->lcore_id];

    /* for that app each mbuf has it's own crypto op */
    for (j = 0; j != cnt; j++) {
        priv = get_priv(m[j]);
        cop[j] = &priv->cop;
        /*
         * this is just to satisfy inbound_sa_check()
         * should be removed in future.
         */
        priv->sa = sa;
    }

    /* prepare and enqueue crypto ops */
    k = rte_ipsec_pkt_crypto_prepare(ips, m, cop, cnt);
    if (k != 0)
        enqueue_cop_bulk(cqp, cop, k);

    return k;
}

/*
 * finish processing of packets successfully decrypted by an inline processor
 */
static uint32_t
ipsec_process_inline_group(struct rte_ipsec_session *ips, void *sa,
    struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t cnt)
{
    uint64_t satp;
    uint32_t k;

    /* get SA type */
    satp = rte_ipsec_sa_type(ips->sa);
    prep_process_group(sa, mb, cnt);

    k = rte_ipsec_pkt_process(ips, mb, cnt);
    copy_to_trf(trf, satp, mb, k);
    return k;
}

/*
 * process packets synchronously
 */
static uint32_t
ipsec_process_cpu_group(struct rte_ipsec_session *ips, void *sa,
    struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t cnt)
{
    uint64_t satp;
    uint32_t k;

    /* get SA type */
    satp = rte_ipsec_sa_type(ips->sa);
    prep_process_group(sa, mb, cnt);

    k = rte_ipsec_pkt_cpu_prepare(ips, mb, cnt);
    k = rte_ipsec_pkt_process(ips, mb, k);
    copy_to_trf(trf, satp, mb, k);
    return k;
}

/*
 * Process ipsec packets.
 * If packet belong to SA that is subject of inline-crypto,
 * then process it immediately.
 * Otherwise do necessary preparations and queue it to related
 * crypto-dev queue.
 */
void
ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
{
    uint32_t i, k, n;
    struct ipsec_sa *sa;
    struct rte_ipsec_group *pg;
    struct rte_ipsec_session *ips;
    struct rte_ipsec_group grp[RTE_DIM(trf->ipsec.pkts)];

    n = sa_group(trf->ipsec.saptr, trf->ipsec.pkts, grp, trf->ipsec.num);

    for (i = 0; i != n; i++) {

        pg = grp + i;
        sa = ipsec_mask_saptr(pg->id.ptr);

        /* fallback to cryptodev with RX packets which inline
         * processor was unable to process
         */
        if (sa != NULL)
            ips = (pg->id.val & IPSEC_SA_OFFLOAD_FALLBACK_FLAG) ?
                ipsec_get_fallback_session(sa) :
                ipsec_get_primary_session(sa);

        /* no valid HW session for that SA */
        if (sa == NULL || unlikely(check_ipsec_session(ips) != 0))
            k = 0;

        /* process packets inline */
        else {
            switch (ips->type) {
            /* enqueue packets to crypto dev */
            case RTE_SECURITY_ACTION_TYPE_NONE:
            case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
                k = ipsec_prepare_crypto_group(ctx, sa, ips,
                    pg->m, pg->cnt);
                break;
            case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
            case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
                k = ipsec_process_inline_group(ips, sa,
                    trf, pg->m, pg->cnt);
                break;
            case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO:
                k = ipsec_process_cpu_group(ips, sa,
                    trf, pg->m, pg->cnt);
                break;
            default:
                k = 0;
            }
        }

        /* drop packets that cannot be enqueued/processed */
        if (k != pg->cnt)
            free_pkts(pg->m + k, pg->cnt - k);
    }
}

static inline uint32_t
cqp_dequeue(struct cdev_qp *cqp, struct rte_crypto_op *cop[], uint32_t num)
{
    uint32_t n;

    if (cqp->in_flight == 0)
        return 0;

    n = rte_cryptodev_dequeue_burst(cqp->id, cqp->qp, cop, num);
    RTE_ASSERT(cqp->in_flight >= n);
    cqp->in_flight -= n;

    return n;
}

static inline uint32_t
ctx_dequeue(struct ipsec_ctx *ctx, struct rte_crypto_op *cop[], uint32_t num)
{
    uint32_t i, n;

    n = 0;

    for (i = ctx->last_qp; n != num && i != ctx->nb_qps; i++)
        n += cqp_dequeue(ctx->tbl + i, cop + n, num - n);

    for (i = 0; n != num && i != ctx->last_qp; i++)
        n += cqp_dequeue(ctx->tbl + i, cop + n, num - n);

    ctx->last_qp = i;
    return n;
}

/*
 * dequeue packets from crypto-queues and finalize processing.
 */
void
ipsec_cqp_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
{
    uint64_t satp;
    uint32_t i, k, n, ng;
    struct rte_ipsec_session *ss;
    struct traffic_type *out;
    struct rte_ipsec_group *pg;
    const int nb_cops = RTE_DIM(trf->ipsec.pkts);
    struct rte_crypto_op *cop[RTE_DIM(trf->ipsec.pkts)];
    struct rte_ipsec_group grp[RTE_DIM(trf->ipsec.pkts)];

    trf->ip4.num = 0;
    trf->ip6.num = 0;

    out = &trf->ipsec;

    /* dequeue completed crypto-ops */
    n = ctx_dequeue(ctx, cop, RTE_MIN(MAX_PKT_BURST, nb_cops));
    if (n == 0)
        return;

    /* group them by ipsec session */
    ng = rte_ipsec_pkt_crypto_group((const struct rte_crypto_op **)
        (uintptr_t)cop, out->pkts, grp, n);

    /* process each group of packets */
    for (i = 0; i != ng; i++) {

        pg = grp + i;
        ss = pg->id.ptr;
        satp = rte_ipsec_sa_type(ss->sa);

        k = rte_ipsec_pkt_process(ss, pg->m, pg->cnt);
        copy_to_trf(trf, satp, pg->m, k);

        /* free bad packets, if any */
        free_pkts(pg->m + k, pg->cnt - k);

        n -= pg->cnt;
    }

    /* we should never have packet with unknown SA here */
    RTE_VERIFY(n == 0);
}