DPDK 22.11.7
rte_swx_pipeline_internal.h
1/* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2021 Intel Corporation
3 */
4#ifndef __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
5#define __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
6
7#include <inttypes.h>
8#include <string.h>
9#include <sys/queue.h>
10
11#include <rte_byteorder.h>
12#include <rte_common.h>
13#include <rte_cycles.h>
14#include <rte_prefetch.h>
15#include <rte_meter.h>
16
17#include <rte_swx_table_selector.h>
18#include <rte_swx_table_learner.h>
19#include <rte_swx_pipeline.h>
20#include <rte_swx_ctl.h>
21
22#ifndef TRACE_LEVEL
23#define TRACE_LEVEL 0
24#endif
25
26#if TRACE_LEVEL
27#define TRACE(...) printf(__VA_ARGS__)
28#else
29#define TRACE(...)
30#endif
31
32/*
33 * Environment.
34 */
35#define ntoh64(x) rte_be_to_cpu_64(x)
36#define hton64(x) rte_cpu_to_be_64(x)
37
38/*
39 * Struct.
40 */
41struct field {
42 char name[RTE_SWX_NAME_SIZE];
43 uint32_t n_bits;
44 uint32_t offset;
45 int var_size;
46};
47
48struct struct_type {
49 TAILQ_ENTRY(struct_type) node;
50 char name[RTE_SWX_NAME_SIZE];
51 struct field *fields;
52 uint32_t n_fields;
53 uint32_t n_bits;
54 uint32_t n_bits_min;
55 int var_size;
56};
57
58TAILQ_HEAD(struct_type_tailq, struct_type);
59
60/*
61 * Input port.
62 */
63struct port_in_type {
64 TAILQ_ENTRY(port_in_type) node;
65 char name[RTE_SWX_NAME_SIZE];
66 struct rte_swx_port_in_ops ops;
67};
68
69TAILQ_HEAD(port_in_type_tailq, port_in_type);
70
71struct port_in {
72 TAILQ_ENTRY(port_in) node;
73 struct port_in_type *type;
74 void *obj;
75 uint32_t id;
76};
77
78TAILQ_HEAD(port_in_tailq, port_in);
79
80struct port_in_runtime {
81 rte_swx_port_in_pkt_rx_t pkt_rx;
82 void *obj;
83};
84
85/*
86 * Output port.
87 */
88struct port_out_type {
89 TAILQ_ENTRY(port_out_type) node;
90 char name[RTE_SWX_NAME_SIZE];
91 struct rte_swx_port_out_ops ops;
92};
93
94TAILQ_HEAD(port_out_type_tailq, port_out_type);
95
96struct port_out {
97 TAILQ_ENTRY(port_out) node;
98 struct port_out_type *type;
99 void *obj;
100 uint32_t id;
101};
102
103TAILQ_HEAD(port_out_tailq, port_out);
104
105struct port_out_runtime {
106 rte_swx_port_out_pkt_tx_t pkt_tx;
107 rte_swx_port_out_pkt_fast_clone_tx_t pkt_fast_clone_tx;
108 rte_swx_port_out_pkt_clone_tx_t pkt_clone_tx;
109 rte_swx_port_out_flush_t flush;
110 void *obj;
111};
112
113/*
114 * Packet mirroring.
115 */
116struct mirroring_session {
117 uint32_t port_id;
118 int fast_clone;
119 uint32_t truncation_length;
120};
121
122/*
123 * Extern object.
124 */
125struct extern_type_member_func {
126 TAILQ_ENTRY(extern_type_member_func) node;
127 char name[RTE_SWX_NAME_SIZE];
128 rte_swx_extern_type_member_func_t func;
129 uint32_t id;
130};
131
132TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
133
134struct extern_type {
135 TAILQ_ENTRY(extern_type) node;
136 char name[RTE_SWX_NAME_SIZE];
137 struct struct_type *mailbox_struct_type;
138 rte_swx_extern_type_constructor_t constructor;
139 rte_swx_extern_type_destructor_t destructor;
140 struct extern_type_member_func_tailq funcs;
141 uint32_t n_funcs;
142};
143
144TAILQ_HEAD(extern_type_tailq, extern_type);
145
146struct extern_obj {
147 TAILQ_ENTRY(extern_obj) node;
148 char name[RTE_SWX_NAME_SIZE];
149 struct extern_type *type;
150 void *obj;
151 uint32_t struct_id;
152 uint32_t id;
153};
154
155TAILQ_HEAD(extern_obj_tailq, extern_obj);
156
157#ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
158#define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
159#endif
160
161struct extern_obj_runtime {
162 void *obj;
163 uint8_t *mailbox;
164 rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
165};
166
167/*
168 * Extern function.
169 */
170struct extern_func {
171 TAILQ_ENTRY(extern_func) node;
172 char name[RTE_SWX_NAME_SIZE];
173 struct struct_type *mailbox_struct_type;
174 rte_swx_extern_func_t func;
175 uint32_t struct_id;
176 uint32_t id;
177};
178
179TAILQ_HEAD(extern_func_tailq, extern_func);
180
181struct extern_func_runtime {
182 uint8_t *mailbox;
183 rte_swx_extern_func_t func;
184};
185
186/*
187 * Hash function.
188 */
189struct hash_func {
190 TAILQ_ENTRY(hash_func) node;
191 char name[RTE_SWX_NAME_SIZE];
192 rte_swx_hash_func_t func;
193 uint32_t id;
194};
195
196TAILQ_HEAD(hash_func_tailq, hash_func);
197
198struct hash_func_runtime {
199 rte_swx_hash_func_t func;
200};
201
202/*
203 * Header.
204 */
205struct header {
206 TAILQ_ENTRY(header) node;
207 char name[RTE_SWX_NAME_SIZE];
208 struct struct_type *st;
209 uint32_t struct_id;
210 uint32_t id;
211};
212
213TAILQ_HEAD(header_tailq, header);
214
215struct header_runtime {
216 uint8_t *ptr0;
217 uint32_t n_bytes;
218};
219
220struct header_out_runtime {
221 uint8_t *ptr0;
222 uint8_t *ptr;
223 uint32_t n_bytes;
224};
225
226/*
227 * Instruction.
228 */
229
230/* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
231 * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
232 * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
233 * when transferred to packet meta-data and in NBO when transferred to packet
234 * headers.
235 */
236
237/* Notation conventions:
238 * -Header field: H = h.header.field (dst/src)
239 * -Meta-data field: M = m.field (dst/src)
240 * -Extern object mailbox field: E = e.field (dst/src)
241 * -Extern function mailbox field: F = f.field (dst/src)
242 * -Table action data field: T = t.field (src only)
243 * -Immediate value: I = 32-bit unsigned value (src only)
244 */
245
246enum instruction_type {
247 /* rx m.port_in */
248 INSTR_RX,
249
250 /* tx port_out
251 * port_out = MI
252 */
253 INSTR_TX, /* port_out = M */
254 INSTR_TX_I, /* port_out = I */
255 INSTR_DROP,
256
257 /*
258 * mirror slot_id session_id
259 * slot_id = MEFT
260 * session_id = MEFT
261 */
262 INSTR_MIRROR,
263
264 /* recirculate
265 */
266 INSTR_RECIRCULATE,
267
268 /* recircid m.recirc_pass_id
269 * Read the internal recirculation pass ID into the specified meta-data field.
270 */
271 INSTR_RECIRCID,
272
273 /* extract h.header */
274 INSTR_HDR_EXTRACT,
275 INSTR_HDR_EXTRACT2,
276 INSTR_HDR_EXTRACT3,
277 INSTR_HDR_EXTRACT4,
278 INSTR_HDR_EXTRACT5,
279 INSTR_HDR_EXTRACT6,
280 INSTR_HDR_EXTRACT7,
281 INSTR_HDR_EXTRACT8,
282
283 /* extract h.header m.last_field_size */
284 INSTR_HDR_EXTRACT_M,
285
286 /* lookahead h.header */
287 INSTR_HDR_LOOKAHEAD,
288
289 /* emit h.header */
290 INSTR_HDR_EMIT,
291 INSTR_HDR_EMIT_TX,
292 INSTR_HDR_EMIT2_TX,
293 INSTR_HDR_EMIT3_TX,
294 INSTR_HDR_EMIT4_TX,
295 INSTR_HDR_EMIT5_TX,
296 INSTR_HDR_EMIT6_TX,
297 INSTR_HDR_EMIT7_TX,
298 INSTR_HDR_EMIT8_TX,
299
300 /* validate h.header */
301 INSTR_HDR_VALIDATE,
302
303 /* invalidate h.header */
304 INSTR_HDR_INVALIDATE,
305
306 /* mov dst src
307 * dst = src
308 * dst = HMEF, src = HMEFTI
309 */
310 INSTR_MOV, /* dst = MEF, src = MEFT; size(dst) <= 64 bits, size(src) <= 64 bits. */
311 INSTR_MOV_MH, /* dst = MEF, src = H; size(dst) <= 64 bits, size(src) <= 64 bits. */
312 INSTR_MOV_HM, /* dst = H, src = MEFT; size(dst) <= 64 bits, size(src) <= 64 bits. */
313 INSTR_MOV_HH, /* dst = H, src = H; size(dst) <= 64 bits, size(src) <= 64 bits. */
314 INSTR_MOV_DMA, /* dst = HMEF, src = HMEF; size(dst) = size(src) > 64 bits, NBO format. */
315 INSTR_MOV_128, /* dst = HMEF, src = HMEF; size(dst) = size(src) = 128 bits, NBO format. */
316 INSTR_MOV_I, /* dst = HMEF, src = I; size(dst) <= 64 bits. */
317
318 /* dma h.header t.field
319 * memcpy(h.header, t.field, sizeof(h.header))
320 */
321 INSTR_DMA_HT,
322 INSTR_DMA_HT2,
323 INSTR_DMA_HT3,
324 INSTR_DMA_HT4,
325 INSTR_DMA_HT5,
326 INSTR_DMA_HT6,
327 INSTR_DMA_HT7,
328 INSTR_DMA_HT8,
329
330 /* add dst src
331 * dst += src
332 * dst = HMEF, src = HMEFTI
333 */
334 INSTR_ALU_ADD, /* dst = MEF, src = MEF */
335 INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
336 INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
337 INSTR_ALU_ADD_HH, /* dst = H, src = H */
338 INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
339 INSTR_ALU_ADD_HI, /* dst = H, src = I */
340
341 /* sub dst src
342 * dst -= src
343 * dst = HMEF, src = HMEFTI
344 */
345 INSTR_ALU_SUB, /* dst = MEF, src = MEF */
346 INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
347 INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
348 INSTR_ALU_SUB_HH, /* dst = H, src = H */
349 INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
350 INSTR_ALU_SUB_HI, /* dst = H, src = I */
351
352 /* ckadd dst src
353 * dst = dst '+ src[0:1] '+ src[2:3] '+ ...
354 * dst = H, src = {H, h.header}, '+ = 1's complement addition operator
355 */
356 INSTR_ALU_CKADD_FIELD, /* src = H */
357 INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 bytes. */
358 INSTR_ALU_CKADD_STRUCT, /* src = h.header, with sizeof(header) any 4-byte multiple. */
359
360 /* cksub dst src
361 * dst = dst '- src
362 * dst = H, src = H, '- = 1's complement subtraction operator
363 */
364 INSTR_ALU_CKSUB_FIELD,
365
366 /* and dst src
367 * dst &= src
368 * dst = HMEF, src = HMEFTI
369 */
370 INSTR_ALU_AND, /* dst = MEF, src = MEFT */
371 INSTR_ALU_AND_MH, /* dst = MEF, src = H */
372 INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
373 INSTR_ALU_AND_HH, /* dst = H, src = H */
374 INSTR_ALU_AND_I, /* dst = HMEF, src = I */
375
376 /* or dst src
377 * dst |= src
378 * dst = HMEF, src = HMEFTI
379 */
380 INSTR_ALU_OR, /* dst = MEF, src = MEFT */
381 INSTR_ALU_OR_MH, /* dst = MEF, src = H */
382 INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
383 INSTR_ALU_OR_HH, /* dst = H, src = H */
384 INSTR_ALU_OR_I, /* dst = HMEF, src = I */
385
386 /* xor dst src
387 * dst ^= src
388 * dst = HMEF, src = HMEFTI
389 */
390 INSTR_ALU_XOR, /* dst = MEF, src = MEFT */
391 INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
392 INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
393 INSTR_ALU_XOR_HH, /* dst = H, src = H */
394 INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
395
396 /* shl dst src
397 * dst <<= src
398 * dst = HMEF, src = HMEFTI
399 */
400 INSTR_ALU_SHL, /* dst = MEF, src = MEF */
401 INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
402 INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
403 INSTR_ALU_SHL_HH, /* dst = H, src = H */
404 INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
405 INSTR_ALU_SHL_HI, /* dst = H, src = I */
406
407 /* shr dst src
408 * dst >>= src
409 * dst = HMEF, src = HMEFTI
410 */
411 INSTR_ALU_SHR, /* dst = MEF, src = MEF */
412 INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
413 INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
414 INSTR_ALU_SHR_HH, /* dst = H, src = H */
415 INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
416 INSTR_ALU_SHR_HI, /* dst = H, src = I */
417
418 /* regprefetch REGARRAY index
419 * prefetch REGARRAY[index]
420 * index = HMEFTI
421 */
422 INSTR_REGPREFETCH_RH, /* index = H */
423 INSTR_REGPREFETCH_RM, /* index = MEFT */
424 INSTR_REGPREFETCH_RI, /* index = I */
425
426 /* regrd dst REGARRAY index
427 * dst = REGARRAY[index]
428 * dst = HMEF, index = HMEFTI
429 */
430 INSTR_REGRD_HRH, /* dst = H, index = H */
431 INSTR_REGRD_HRM, /* dst = H, index = MEFT */
432 INSTR_REGRD_HRI, /* dst = H, index = I */
433 INSTR_REGRD_MRH, /* dst = MEF, index = H */
434 INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
435 INSTR_REGRD_MRI, /* dst = MEF, index = I */
436
437 /* regwr REGARRAY index src
438 * REGARRAY[index] = src
439 * index = HMEFTI, src = HMEFTI
440 */
441 INSTR_REGWR_RHH, /* index = H, src = H */
442 INSTR_REGWR_RHM, /* index = H, src = MEFT */
443 INSTR_REGWR_RHI, /* index = H, src = I */
444 INSTR_REGWR_RMH, /* index = MEFT, src = H */
445 INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
446 INSTR_REGWR_RMI, /* index = MEFT, src = I */
447 INSTR_REGWR_RIH, /* index = I, src = H */
448 INSTR_REGWR_RIM, /* index = I, src = MEFT */
449 INSTR_REGWR_RII, /* index = I, src = I */
450
451 /* regadd REGARRAY index src
452 * REGARRAY[index] += src
453 * index = HMEFTI, src = HMEFTI
454 */
455 INSTR_REGADD_RHH, /* index = H, src = H */
456 INSTR_REGADD_RHM, /* index = H, src = MEFT */
457 INSTR_REGADD_RHI, /* index = H, src = I */
458 INSTR_REGADD_RMH, /* index = MEFT, src = H */
459 INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
460 INSTR_REGADD_RMI, /* index = MEFT, src = I */
461 INSTR_REGADD_RIH, /* index = I, src = H */
462 INSTR_REGADD_RIM, /* index = I, src = MEFT */
463 INSTR_REGADD_RII, /* index = I, src = I */
464
465 /* metprefetch METARRAY index
466 * prefetch METARRAY[index]
467 * index = HMEFTI
468 */
469 INSTR_METPREFETCH_H, /* index = H */
470 INSTR_METPREFETCH_M, /* index = MEFT */
471 INSTR_METPREFETCH_I, /* index = I */
472
473 /* meter METARRAY index length color_in color_out
474 * color_out = meter(METARRAY[index], length, color_in)
475 * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
476 */
477 INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
478 INSTR_METER_HHI, /* index = H, length = H, color_in = I */
479 INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
480 INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
481 INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
482 INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
483 INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
484 INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
485 INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
486 INSTR_METER_IHI, /* index = I, length = H, color_in = I */
487 INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
488 INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
489
490 /* table TABLE */
491 INSTR_TABLE,
492 INSTR_TABLE_AF,
493 INSTR_SELECTOR,
494 INSTR_LEARNER,
495 INSTR_LEARNER_AF,
496
497 /* learn ACTION_NAME [ m.action_first_arg ] m.timeout_id */
498 INSTR_LEARNER_LEARN,
499
500 /* rearm [ m.timeout_id ] */
501 INSTR_LEARNER_REARM,
502 INSTR_LEARNER_REARM_NEW,
503
504 /* forget */
505 INSTR_LEARNER_FORGET,
506
507 /* entryid m.table_entry_id
508 * Read the internal table entry ID into the specified meta-data field.
509 */
510 INSTR_ENTRYID,
511
512 /* extern e.obj.func */
513 INSTR_EXTERN_OBJ,
514
515 /* extern f.func */
516 INSTR_EXTERN_FUNC,
517
518 /* hash HASH_FUNC_NAME dst src_first src_last
519 * Compute hash value over range of struct fields.
520 * dst = M
521 * src_first = HMEFT
522 * src_last = HMEFT
523 * src_first and src_last must be fields within the same struct
524 */
525 INSTR_HASH_FUNC,
526
527 /* jmp LABEL
528 * Unconditional jump
529 */
530 INSTR_JMP,
531
532 /* jmpv LABEL h.header
533 * Jump if header is valid
534 */
535 INSTR_JMP_VALID,
536
537 /* jmpnv LABEL h.header
538 * Jump if header is invalid
539 */
540 INSTR_JMP_INVALID,
541
542 /* jmph LABEL
543 * Jump if table lookup hit
544 */
545 INSTR_JMP_HIT,
546
547 /* jmpnh LABEL
548 * Jump if table lookup miss
549 */
550 INSTR_JMP_MISS,
551
552 /* jmpa LABEL ACTION
553 * Jump if action run
554 */
555 INSTR_JMP_ACTION_HIT,
556
557 /* jmpna LABEL ACTION
558 * Jump if action not run
559 */
560 INSTR_JMP_ACTION_MISS,
561
562 /* jmpeq LABEL a b
563 * Jump if a is equal to b
564 * a = HMEFT, b = HMEFTI
565 */
566 INSTR_JMP_EQ, /* a = MEFT, b = MEFT */
567 INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
568 INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
569 INSTR_JMP_EQ_HH, /* a = H, b = H */
570 INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
571
572 /* jmpneq LABEL a b
573 * Jump if a is not equal to b
574 * a = HMEFT, b = HMEFTI
575 */
576 INSTR_JMP_NEQ, /* a = MEFT, b = MEFT */
577 INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
578 INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
579 INSTR_JMP_NEQ_HH, /* a = H, b = H */
580 INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
581
582 /* jmplt LABEL a b
583 * Jump if a is less than b
584 * a = HMEFT, b = HMEFTI
585 */
586 INSTR_JMP_LT, /* a = MEFT, b = MEFT */
587 INSTR_JMP_LT_MH, /* a = MEFT, b = H */
588 INSTR_JMP_LT_HM, /* a = H, b = MEFT */
589 INSTR_JMP_LT_HH, /* a = H, b = H */
590 INSTR_JMP_LT_MI, /* a = MEFT, b = I */
591 INSTR_JMP_LT_HI, /* a = H, b = I */
592
593 /* jmpgt LABEL a b
594 * Jump if a is greater than b
595 * a = HMEFT, b = HMEFTI
596 */
597 INSTR_JMP_GT, /* a = MEFT, b = MEFT */
598 INSTR_JMP_GT_MH, /* a = MEFT, b = H */
599 INSTR_JMP_GT_HM, /* a = H, b = MEFT */
600 INSTR_JMP_GT_HH, /* a = H, b = H */
601 INSTR_JMP_GT_MI, /* a = MEFT, b = I */
602 INSTR_JMP_GT_HI, /* a = H, b = I */
603
604 /* return
605 * Return from action
606 */
607 INSTR_RETURN,
608
609 /* Start of custom instructions. */
610 INSTR_CUSTOM_0,
611};
612
613struct instr_operand {
614 uint8_t struct_id;
615 uint8_t n_bits;
616 uint8_t offset;
617 uint8_t pad;
618};
619
620struct instr_io {
621 struct {
622 union {
623 struct {
624 uint8_t offset;
625 uint8_t n_bits;
626 uint8_t pad[2];
627 };
628
629 uint32_t val;
630 };
631 } io;
632
633 struct {
634 uint8_t header_id[8];
635 uint8_t struct_id[8];
636 uint8_t n_bytes[8];
637 } hdr;
638};
639
640struct instr_hdr_validity {
641 uint8_t header_id;
642 uint8_t struct_id;
643};
644
645struct instr_table {
646 uint8_t table_id;
647};
648
649struct instr_learn {
650 uint8_t action_id;
651 uint8_t mf_first_arg_offset;
652 uint8_t mf_timeout_id_offset;
653 uint8_t mf_timeout_id_n_bits;
654};
655
656struct instr_extern_obj {
657 uint8_t ext_obj_id;
658 uint8_t func_id;
659};
660
661struct instr_extern_func {
662 uint8_t ext_func_id;
663};
664
665struct instr_hash_func {
666 uint8_t hash_func_id;
667
668 struct {
669 uint8_t offset;
670 uint8_t n_bits;
671 } dst;
672
673 struct {
674 uint8_t struct_id;
675 uint16_t offset;
676 uint16_t n_bytes;
677 } src;
678};
679
680struct instr_dst_src {
681 struct instr_operand dst;
682 union {
683 struct instr_operand src;
684 uint64_t src_val;
685 };
686};
687
688struct instr_regarray {
689 uint8_t regarray_id;
690 uint8_t pad[3];
691
692 union {
693 struct instr_operand idx;
694 uint32_t idx_val;
695 };
696
697 union {
698 struct instr_operand dstsrc;
699 uint64_t dstsrc_val;
700 };
701};
702
703struct instr_meter {
704 uint8_t metarray_id;
705 uint8_t pad[3];
706
707 union {
708 struct instr_operand idx;
709 uint32_t idx_val;
710 };
711
712 struct instr_operand length;
713
714 union {
715 struct instr_operand color_in;
716 uint32_t color_in_val;
717 };
718
719 struct instr_operand color_out;
720};
721
722struct instr_dma {
723 struct {
724 uint8_t header_id[8];
725 uint8_t struct_id[8];
726 } dst;
727
728 struct {
729 uint8_t offset[8];
730 } src;
731
732 uint16_t n_bytes[8];
733};
734
735struct instr_jmp {
736 struct instruction *ip;
737
738 union {
739 struct instr_operand a;
740 uint8_t header_id;
741 uint8_t action_id;
742 };
743
744 union {
745 struct instr_operand b;
746 uint64_t b_val;
747 };
748};
749
750struct instruction {
751 enum instruction_type type;
752 union {
753 struct instr_io io;
754 struct instr_dst_src mirror;
755 struct instr_hdr_validity valid;
756 struct instr_dst_src mov;
757 struct instr_regarray regarray;
758 struct instr_meter meter;
759 struct instr_dma dma;
760 struct instr_dst_src alu;
761 struct instr_table table;
762 struct instr_learn learn;
763 struct instr_extern_obj ext_obj;
764 struct instr_extern_func ext_func;
765 struct instr_hash_func hash_func;
766 struct instr_jmp jmp;
767 };
768};
769
770struct instruction_data {
771 char label[RTE_SWX_NAME_SIZE];
772 char jmp_label[RTE_SWX_NAME_SIZE];
773 uint32_t n_users; /* user = jmp instruction to this instruction. */
774 int invalid;
775};
776
777typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
778
779/*
780 * Action.
781 */
782typedef void
783(*action_func_t)(struct rte_swx_pipeline *p);
784
785struct action {
786 TAILQ_ENTRY(action) node;
787 char name[RTE_SWX_NAME_SIZE];
788 struct struct_type *st;
789 int *args_endianness; /* 0 = Host Byte Order (HBO); 1 = Network Byte Order (NBO). */
790 struct instruction *instructions;
791 struct instruction_data *instruction_data;
792 uint32_t n_instructions;
793 uint32_t id;
794};
795
796TAILQ_HEAD(action_tailq, action);
797
798/*
799 * Table.
800 */
801struct table_type {
802 TAILQ_ENTRY(table_type) node;
803 char name[RTE_SWX_NAME_SIZE];
804 enum rte_swx_table_match_type match_type;
805 struct rte_swx_table_ops ops;
806};
807
808TAILQ_HEAD(table_type_tailq, table_type);
809
810struct match_field {
811 enum rte_swx_table_match_type match_type;
812 struct field *field;
813};
814
815struct table {
816 TAILQ_ENTRY(table) node;
817 char name[RTE_SWX_NAME_SIZE];
818 char args[RTE_SWX_NAME_SIZE];
819 struct table_type *type; /* NULL when n_fields == 0. */
820
821 /* Match. */
822 struct match_field *fields;
823 uint32_t n_fields;
824 struct header *header; /* Only valid when n_fields > 0. */
825
826 /* Action. */
827 struct action **actions;
828 struct action *default_action;
829 uint8_t *default_action_data;
830 uint32_t n_actions;
831 int default_action_is_const;
832 uint32_t action_data_size_max;
833 int *action_is_for_table_entries;
834 int *action_is_for_default_entry;
835
836 struct hash_func *hf;
837 uint32_t size;
838 uint32_t id;
839};
840
841TAILQ_HEAD(table_tailq, table);
842
843struct table_runtime {
844 rte_swx_table_lookup_t func;
845 void *mailbox;
846 uint8_t **key;
847};
848
849struct table_statistics {
850 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
851 uint64_t *n_pkts_action;
852};
853
854/*
855 * Selector.
856 */
857struct selector {
858 TAILQ_ENTRY(selector) node;
859 char name[RTE_SWX_NAME_SIZE];
860
861 struct field *group_id_field;
862 struct field **selector_fields;
863 uint32_t n_selector_fields;
864 struct header *selector_header;
865 struct field *member_id_field;
866
867 uint32_t n_groups_max;
868 uint32_t n_members_per_group_max;
869
870 uint32_t id;
871};
872
873TAILQ_HEAD(selector_tailq, selector);
874
875struct selector_runtime {
876 void *mailbox;
877 uint8_t **group_id_buffer;
878 uint8_t **selector_buffer;
879 uint8_t **member_id_buffer;
880};
881
882struct selector_statistics {
883 uint64_t n_pkts;
884};
885
886/*
887 * Learner table.
888 */
889struct learner {
890 TAILQ_ENTRY(learner) node;
891 char name[RTE_SWX_NAME_SIZE];
892
893 /* Match. */
894 struct field **fields;
895 uint32_t n_fields;
896 struct header *header;
897
898 /* Action. */
899 struct action **actions;
900 struct action *default_action;
901 uint8_t *default_action_data;
902 uint32_t n_actions;
903 int default_action_is_const;
904 uint32_t action_data_size_max;
905 int *action_is_for_table_entries;
906 int *action_is_for_default_entry;
907
908 struct hash_func *hf;
909 uint32_t size;
910 uint32_t timeout[RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX];
911 uint32_t n_timeouts;
912 uint32_t id;
913};
914
915TAILQ_HEAD(learner_tailq, learner);
916
917struct learner_runtime {
918 void *mailbox;
919 uint8_t **key;
920};
921
922struct learner_statistics {
923 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
924 uint64_t n_pkts_learn[2]; /* 0 = Learn OK, 1 = Learn error. */
925 uint64_t n_pkts_rearm;
926 uint64_t n_pkts_forget;
927 uint64_t *n_pkts_action;
928};
929
930/*
931 * Register array.
932 */
933struct regarray {
934 TAILQ_ENTRY(regarray) node;
935 char name[RTE_SWX_NAME_SIZE];
936 uint64_t init_val;
937 uint32_t size;
938 uint32_t id;
939};
940
941TAILQ_HEAD(regarray_tailq, regarray);
942
943struct regarray_runtime {
944 uint64_t *regarray;
945 uint32_t size_mask;
946};
947
948/*
949 * Meter array.
950 */
951struct meter_profile {
952 TAILQ_ENTRY(meter_profile) node;
953 char name[RTE_SWX_NAME_SIZE];
954 struct rte_meter_trtcm_params params;
955 struct rte_meter_trtcm_profile profile;
956 uint32_t n_users;
957};
958
959TAILQ_HEAD(meter_profile_tailq, meter_profile);
960
961struct metarray {
962 TAILQ_ENTRY(metarray) node;
963 char name[RTE_SWX_NAME_SIZE];
964 uint32_t size;
965 uint32_t id;
966};
967
968TAILQ_HEAD(metarray_tailq, metarray);
969
970struct meter {
971 struct rte_meter_trtcm m;
972 struct meter_profile *profile;
973 enum rte_color color_mask;
974 uint8_t pad[20];
975
976 uint64_t n_pkts[RTE_COLORS];
977 uint64_t n_bytes[RTE_COLORS];
978};
979
980struct metarray_runtime {
981 struct meter *metarray;
982 uint32_t size_mask;
983};
984
985/*
986 * Pipeline.
987 */
988struct thread {
989 /* Packet. */
990 struct rte_swx_pkt pkt;
991 uint8_t *ptr;
992 uint32_t *mirroring_slots;
993 uint64_t mirroring_slots_mask;
994 int recirculate;
995 uint32_t recirc_pass_id;
996
997 /* Structures. */
998 uint8_t **structs;
999
1000 /* Packet headers. */
1001 struct header_runtime *headers; /* Extracted or generated headers. */
1002 struct header_out_runtime *headers_out; /* Emitted headers. */
1003 uint8_t *header_storage;
1004 uint8_t *header_out_storage;
1005 uint64_t valid_headers;
1006 uint32_t n_headers_out;
1007
1008 /* Packet meta-data. */
1009 uint8_t *metadata;
1010
1011 /* Tables. */
1012 struct table_runtime *tables;
1013 struct selector_runtime *selectors;
1014 struct learner_runtime *learners;
1015 struct rte_swx_table_state *table_state;
1016 uint64_t action_id;
1017 size_t entry_id;
1018 int hit; /* 0 = Miss, 1 = Hit. */
1019 uint32_t learner_id;
1020 uint64_t time;
1021
1022 /* Extern objects and functions. */
1023 struct extern_obj_runtime *extern_objs;
1024 struct extern_func_runtime *extern_funcs;
1025
1026 /* Instructions. */
1027 struct instruction *ip;
1028 struct instruction *ret;
1029};
1030
1031#define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
1032#define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
1033#define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
1034
1035#define HEADER_VALID(thread, header_id) \
1036 MASK64_BIT_GET((thread)->valid_headers, header_id)
1037
1038static inline uint64_t
1039instr_operand_hbo(struct thread *t, const struct instr_operand *x)
1040{
1041 uint8_t *x_struct = t->structs[x->struct_id];
1042 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
1043 uint64_t x64 = *x64_ptr;
1044 uint64_t x64_mask = UINT64_MAX >> (64 - x->n_bits);
1045
1046 return x64 & x64_mask;
1047}
1048
1049#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1050
1051static inline uint64_t
1052instr_operand_nbo(struct thread *t, const struct instr_operand *x)
1053{
1054 uint8_t *x_struct = t->structs[x->struct_id];
1055 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
1056 uint64_t x64 = *x64_ptr;
1057
1058 return ntoh64(x64) >> (64 - x->n_bits);
1059}
1060
1061#else
1062
1063#define instr_operand_nbo instr_operand_hbo
1064
1065#endif
1066
1067#define ALU(thread, ip, operator) \
1068{ \
1069 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1070 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1071 uint64_t dst64 = *dst64_ptr; \
1072 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1073 uint64_t dst = dst64 & dst64_mask; \
1074 \
1075 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1076 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1077 uint64_t src64 = *src64_ptr; \
1078 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1079 uint64_t src = src64 & src64_mask; \
1080 \
1081 uint64_t result = dst operator src; \
1082 \
1083 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1084}
1085
1086#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1087
1088#define ALU_MH(thread, ip, operator) \
1089{ \
1090 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1091 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1092 uint64_t dst64 = *dst64_ptr; \
1093 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1094 uint64_t dst = dst64 & dst64_mask; \
1095 \
1096 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1097 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1098 uint64_t src64 = *src64_ptr; \
1099 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1100 \
1101 uint64_t result = dst operator src; \
1102 \
1103 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1104}
1105
1106#define ALU_HM(thread, ip, operator) \
1107{ \
1108 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1109 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1110 uint64_t dst64 = *dst64_ptr; \
1111 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1112 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1113 \
1114 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1115 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1116 uint64_t src64 = *src64_ptr; \
1117 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1118 uint64_t src = src64 & src64_mask; \
1119 \
1120 uint64_t result = dst operator src; \
1121 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1122 \
1123 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1124}
1125
1126#define ALU_HM_FAST(thread, ip, operator) \
1127{ \
1128 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1129 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1130 uint64_t dst64 = *dst64_ptr; \
1131 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1132 uint64_t dst = dst64 & dst64_mask; \
1133 \
1134 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1135 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1136 uint64_t src64 = *src64_ptr; \
1137 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1138 uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
1139 \
1140 uint64_t result = dst operator src; \
1141 \
1142 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1143}
1144
1145#define ALU_HH(thread, ip, operator) \
1146{ \
1147 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1148 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1149 uint64_t dst64 = *dst64_ptr; \
1150 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1151 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1152 \
1153 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1154 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1155 uint64_t src64 = *src64_ptr; \
1156 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1157 \
1158 uint64_t result = dst operator src; \
1159 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1160 \
1161 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1162}
1163
1164#define ALU_HH_FAST(thread, ip, operator) \
1165{ \
1166 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1167 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1168 uint64_t dst64 = *dst64_ptr; \
1169 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1170 uint64_t dst = dst64 & dst64_mask; \
1171 \
1172 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1173 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1174 uint64_t src64 = *src64_ptr; \
1175 uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
1176 \
1177 uint64_t result = dst operator src; \
1178 \
1179 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1180}
1181
1182#else
1183
1184#define ALU_MH ALU
1185#define ALU_HM ALU
1186#define ALU_HM_FAST ALU
1187#define ALU_HH ALU
1188#define ALU_HH_FAST ALU
1189
1190#endif
1191
1192#define ALU_I(thread, ip, operator) \
1193{ \
1194 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1195 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1196 uint64_t dst64 = *dst64_ptr; \
1197 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1198 uint64_t dst = dst64 & dst64_mask; \
1199 \
1200 uint64_t src = (ip)->alu.src_val; \
1201 \
1202 uint64_t result = dst operator src; \
1203 \
1204 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1205}
1206
1207#define ALU_MI ALU_I
1208
1209#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1210
1211#define ALU_HI(thread, ip, operator) \
1212{ \
1213 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1214 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1215 uint64_t dst64 = *dst64_ptr; \
1216 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1217 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1218 \
1219 uint64_t src = (ip)->alu.src_val; \
1220 \
1221 uint64_t result = dst operator src; \
1222 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1223 \
1224 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1225}
1226
1227#else
1228
1229#define ALU_HI ALU_I
1230
1231#endif
1232
1233#define MOV(thread, ip) \
1234{ \
1235 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1236 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1237 uint64_t dst64 = *dst64_ptr; \
1238 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1239 \
1240 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1241 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1242 uint64_t src64 = *src64_ptr; \
1243 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1244 uint64_t src = src64 & src64_mask; \
1245 \
1246 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1247}
1248
1249#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1250
1251#define MOV_MH(thread, ip) \
1252{ \
1253 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1254 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1255 uint64_t dst64 = *dst64_ptr; \
1256 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1257 \
1258 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1259 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1260 uint64_t src64 = *src64_ptr; \
1261 uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits); \
1262 \
1263 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1264}
1265
1266#define MOV_HM(thread, ip) \
1267{ \
1268 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1269 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1270 uint64_t dst64 = *dst64_ptr; \
1271 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1272 \
1273 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1274 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1275 uint64_t src64 = *src64_ptr; \
1276 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1277 uint64_t src = src64 & src64_mask; \
1278 \
1279 src = hton64(src) >> (64 - (ip)->mov.dst.n_bits); \
1280 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1281}
1282
1283#define MOV_HH(thread, ip) \
1284{ \
1285 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1286 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1287 uint64_t dst64 = *dst64_ptr; \
1288 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1289 \
1290 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1291 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1292 uint64_t src64 = *src64_ptr; \
1293 \
1294 uint64_t src = src64 << (64 - (ip)->mov.src.n_bits); \
1295 src = src >> (64 - (ip)->mov.dst.n_bits); \
1296 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1297}
1298
1299#else
1300
1301#define MOV_MH MOV
1302#define MOV_HM MOV
1303#define MOV_HH MOV
1304
1305#endif
1306
1307#define MOV_I(thread, ip) \
1308{ \
1309 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1310 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1311 uint64_t dst64 = *dst64_ptr; \
1312 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1313 \
1314 uint64_t src = (ip)->mov.src_val; \
1315 \
1316 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1317}
1318
1319#define JMP_CMP(thread, ip, operator) \
1320{ \
1321 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1322 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1323 uint64_t a64 = *a64_ptr; \
1324 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1325 uint64_t a = a64 & a64_mask; \
1326 \
1327 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1328 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1329 uint64_t b64 = *b64_ptr; \
1330 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1331 uint64_t b = b64 & b64_mask; \
1332 \
1333 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1334}
1335
1336#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1337
1338#define JMP_CMP_MH(thread, ip, operator) \
1339{ \
1340 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1341 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1342 uint64_t a64 = *a64_ptr; \
1343 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1344 uint64_t a = a64 & a64_mask; \
1345 \
1346 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1347 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1348 uint64_t b64 = *b64_ptr; \
1349 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1350 \
1351 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1352}
1353
1354#define JMP_CMP_HM(thread, ip, operator) \
1355{ \
1356 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1357 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1358 uint64_t a64 = *a64_ptr; \
1359 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1360 \
1361 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1362 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1363 uint64_t b64 = *b64_ptr; \
1364 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1365 uint64_t b = b64 & b64_mask; \
1366 \
1367 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1368}
1369
1370#define JMP_CMP_HH(thread, ip, operator) \
1371{ \
1372 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1373 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1374 uint64_t a64 = *a64_ptr; \
1375 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1376 \
1377 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1378 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1379 uint64_t b64 = *b64_ptr; \
1380 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1381 \
1382 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1383}
1384
1385#define JMP_CMP_HH_FAST(thread, ip, operator) \
1386{ \
1387 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1388 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1389 uint64_t a64 = *a64_ptr; \
1390 uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits); \
1391 \
1392 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1393 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1394 uint64_t b64 = *b64_ptr; \
1395 uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits); \
1396 \
1397 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1398}
1399
1400#else
1401
1402#define JMP_CMP_MH JMP_CMP
1403#define JMP_CMP_HM JMP_CMP
1404#define JMP_CMP_HH JMP_CMP
1405#define JMP_CMP_HH_FAST JMP_CMP
1406
1407#endif
1408
1409#define JMP_CMP_I(thread, ip, operator) \
1410{ \
1411 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1412 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1413 uint64_t a64 = *a64_ptr; \
1414 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1415 uint64_t a = a64 & a64_mask; \
1416 \
1417 uint64_t b = (ip)->jmp.b_val; \
1418 \
1419 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1420}
1421
1422#define JMP_CMP_MI JMP_CMP_I
1423
1424#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1425
1426#define JMP_CMP_HI(thread, ip, operator) \
1427{ \
1428 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1429 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1430 uint64_t a64 = *a64_ptr; \
1431 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1432 \
1433 uint64_t b = (ip)->jmp.b_val; \
1434 \
1435 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1436}
1437
1438#else
1439
1440#define JMP_CMP_HI JMP_CMP_I
1441
1442#endif
1443
1444#define METADATA_READ(thread, offset, n_bits) \
1445({ \
1446 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1447 uint64_t m64 = *m64_ptr; \
1448 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1449 (m64 & m64_mask); \
1450})
1451
1452#define METADATA_WRITE(thread, offset, n_bits, value) \
1453{ \
1454 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1455 uint64_t m64 = *m64_ptr; \
1456 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1457 \
1458 uint64_t m_new = value; \
1459 \
1460 *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask); \
1461}
1462
1463#ifndef RTE_SWX_PIPELINE_THREADS_MAX
1464#define RTE_SWX_PIPELINE_THREADS_MAX 16
1465#endif
1466
1467#ifndef RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX
1468#define RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX 256
1469#endif
1470
1471struct rte_swx_pipeline {
1472 char name[RTE_SWX_NAME_SIZE];
1473
1474 struct struct_type_tailq struct_types;
1475 struct port_in_type_tailq port_in_types;
1476 struct port_in_tailq ports_in;
1477 struct port_out_type_tailq port_out_types;
1478 struct port_out_tailq ports_out;
1479 struct extern_type_tailq extern_types;
1480 struct extern_obj_tailq extern_objs;
1481 struct extern_func_tailq extern_funcs;
1482 struct hash_func_tailq hash_funcs;
1483 struct header_tailq headers;
1484 struct struct_type *metadata_st;
1485 uint32_t metadata_struct_id;
1486 struct action_tailq actions;
1487 struct table_type_tailq table_types;
1488 struct table_tailq tables;
1489 struct selector_tailq selectors;
1490 struct learner_tailq learners;
1491 struct regarray_tailq regarrays;
1492 struct meter_profile_tailq meter_profiles;
1493 struct metarray_tailq metarrays;
1494
1495 struct port_in_runtime *in;
1496 struct port_out_runtime *out;
1497 struct mirroring_session *mirroring_sessions;
1498 struct instruction **action_instructions;
1499 action_func_t *action_funcs;
1500 struct rte_swx_table_state *table_state;
1501 struct table_statistics *table_stats;
1502 struct selector_statistics *selector_stats;
1503 struct learner_statistics *learner_stats;
1504 struct hash_func_runtime *hash_func_runtime;
1505 struct regarray_runtime *regarray_runtime;
1506 struct metarray_runtime *metarray_runtime;
1507 struct instruction *instructions;
1508 struct instruction_data *instruction_data;
1509 instr_exec_t *instruction_table;
1510 struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1511 void *lib;
1512
1513 uint32_t n_structs;
1514 uint32_t n_ports_in;
1515 uint32_t n_ports_out;
1516 uint32_t n_mirroring_slots;
1517 uint32_t n_mirroring_sessions;
1518 uint32_t n_extern_objs;
1519 uint32_t n_extern_funcs;
1520 uint32_t n_hash_funcs;
1521 uint32_t n_actions;
1522 uint32_t n_tables;
1523 uint32_t n_selectors;
1524 uint32_t n_learners;
1525 uint32_t n_regarrays;
1526 uint32_t n_metarrays;
1527 uint32_t n_headers;
1528 uint32_t thread_id;
1529 uint32_t port_id;
1530 uint32_t n_instructions;
1531 int build_done;
1532 int numa_node;
1533};
1534
1535/*
1536 * Instruction.
1537 */
1538static inline void
1539pipeline_port_inc(struct rte_swx_pipeline *p)
1540{
1541 p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
1542}
1543
1544static inline void
1545thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
1546{
1547 t->ip = p->instructions;
1548}
1549
1550static inline void
1551thread_ip_set(struct thread *t, struct instruction *ip)
1552{
1553 t->ip = ip;
1554}
1555
1556static inline void
1557thread_ip_action_call(struct rte_swx_pipeline *p,
1558 struct thread *t,
1559 uint32_t action_id)
1560{
1561 t->ret = t->ip + 1;
1562 t->ip = p->action_instructions[action_id];
1563}
1564
1565static inline void
1566thread_ip_inc(struct rte_swx_pipeline *p);
1567
1568static inline void
1569thread_ip_inc(struct rte_swx_pipeline *p)
1570{
1571 struct thread *t = &p->threads[p->thread_id];
1572
1573 t->ip++;
1574}
1575
1576static inline void
1577thread_ip_inc_cond(struct thread *t, int cond)
1578{
1579 t->ip += cond;
1580}
1581
1582static inline void
1583thread_yield(struct rte_swx_pipeline *p)
1584{
1585 p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1586}
1587
1588static inline void
1589thread_yield_cond(struct rte_swx_pipeline *p, int cond)
1590{
1591 p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1592}
1593
1594/*
1595 * rx.
1596 */
1597static inline int
1598__instr_rx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1599{
1600 struct port_in_runtime *port = &p->in[p->port_id];
1601 struct rte_swx_pkt *pkt = &t->pkt;
1602 int pkt_received;
1603
1604 /* Recirculation: keep the current packet. */
1605 if (t->recirculate) {
1606 TRACE("[Thread %2u] rx - recirculate (pass %u)\n",
1607 p->thread_id,
1608 t->recirc_pass_id + 1);
1609
1610 /* Packet. */
1611 t->ptr = &pkt->pkt[pkt->offset];
1612 t->mirroring_slots_mask = 0;
1613 t->recirculate = 0;
1614 t->recirc_pass_id++;
1615
1616 /* Headers. */
1617 t->valid_headers = 0;
1618 t->n_headers_out = 0;
1619
1620 /* Tables. */
1621 t->table_state = p->table_state;
1622
1623 return 1;
1624 }
1625
1626 /* Packet. */
1627 pkt_received = port->pkt_rx(port->obj, pkt);
1628 t->ptr = &pkt->pkt[pkt->offset];
1629 rte_prefetch0(t->ptr);
1630
1631 TRACE("[Thread %2u] rx %s from port %u\n",
1632 p->thread_id,
1633 pkt_received ? "1 pkt" : "0 pkts",
1634 p->port_id);
1635
1636 t->mirroring_slots_mask = 0;
1637 t->recirc_pass_id = 0;
1638
1639 /* Headers. */
1640 t->valid_headers = 0;
1641 t->n_headers_out = 0;
1642
1643 /* Meta-data. */
1644 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
1645
1646 /* Tables. */
1647 t->table_state = p->table_state;
1648
1649 /* Thread. */
1650 pipeline_port_inc(p);
1651
1652 return pkt_received;
1653}
1654
1655static inline void
1656instr_rx_exec(struct rte_swx_pipeline *p)
1657{
1658 struct thread *t = &p->threads[p->thread_id];
1659 struct instruction *ip = t->ip;
1660 int pkt_received;
1661
1662 /* Packet. */
1663 pkt_received = __instr_rx_exec(p, t, ip);
1664
1665 /* Thread. */
1666 thread_ip_inc_cond(t, pkt_received);
1667 thread_yield(p);
1668}
1669
1670/*
1671 * tx.
1672 */
1673static inline void
1674emit_handler(struct thread *t)
1675{
1676 struct header_out_runtime *h0 = &t->headers_out[0];
1677 struct header_out_runtime *h1 = &t->headers_out[1];
1678 uint32_t offset = 0, i;
1679
1680 /* No header change or header decapsulation. */
1681 if ((t->n_headers_out == 1) &&
1682 (h0->ptr + h0->n_bytes == t->ptr)) {
1683 TRACE("Emit handler: no header change or header decap.\n");
1684
1685 t->pkt.offset -= h0->n_bytes;
1686 t->pkt.length += h0->n_bytes;
1687
1688 return;
1689 }
1690
1691 /* Header encapsulation (optionally, with prior header decapsulation). */
1692 if ((t->n_headers_out == 2) &&
1693 (h1->ptr + h1->n_bytes == t->ptr) &&
1694 (h0->ptr == h0->ptr0)) {
1695 uint32_t offset;
1696
1697 TRACE("Emit handler: header encapsulation.\n");
1698
1699 offset = h0->n_bytes + h1->n_bytes;
1700 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
1701 t->pkt.offset -= offset;
1702 t->pkt.length += offset;
1703
1704 return;
1705 }
1706
1707 /* For any other case. */
1708 TRACE("Emit handler: complex case.\n");
1709
1710 for (i = 0; i < t->n_headers_out; i++) {
1711 struct header_out_runtime *h = &t->headers_out[i];
1712
1713 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
1714 offset += h->n_bytes;
1715 }
1716
1717 if (offset) {
1718 memcpy(t->ptr - offset, t->header_out_storage, offset);
1719 t->pkt.offset -= offset;
1720 t->pkt.length += offset;
1721 }
1722}
1723
1724static inline void
1725mirroring_handler(struct rte_swx_pipeline *p, struct thread *t, struct rte_swx_pkt *pkt)
1726{
1727 uint64_t slots_mask = t->mirroring_slots_mask, slot_mask;
1728 uint32_t slot_id;
1729
1730 for (slot_id = 0, slot_mask = 1LLU ; slots_mask; slot_id++, slot_mask <<= 1)
1731 if (slot_mask & slots_mask) {
1732 struct port_out_runtime *port;
1733 struct mirroring_session *session;
1734 uint32_t port_id, session_id;
1735
1736 session_id = t->mirroring_slots[slot_id];
1737 session = &p->mirroring_sessions[session_id];
1738
1739 port_id = session->port_id;
1740 port = &p->out[port_id];
1741
1742 if (session->fast_clone)
1743 port->pkt_fast_clone_tx(port->obj, pkt);
1744 else
1745 port->pkt_clone_tx(port->obj, pkt, session->truncation_length);
1746
1747 slots_mask &= ~slot_mask;
1748 }
1749}
1750
1751static inline void
1752__instr_tx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1753{
1754 uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1755 struct port_out_runtime *port = &p->out[port_id];
1756 struct rte_swx_pkt *pkt = &t->pkt;
1757
1758 /* Recirculation: keep the current packet. */
1759 if (t->recirculate) {
1760 TRACE("[Thread %2u]: tx 1 pkt - recirculate\n",
1761 p->thread_id);
1762
1763 /* Headers. */
1764 emit_handler(t);
1765
1766 /* Packet. */
1767 mirroring_handler(p, t, pkt);
1768
1769 return;
1770 }
1771
1772 TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
1773 p->thread_id,
1774 (uint32_t)port_id);
1775
1776 /* Headers. */
1777 emit_handler(t);
1778
1779 /* Packet. */
1780 mirroring_handler(p, t, pkt);
1781 port->pkt_tx(port->obj, pkt);
1782}
1783
1784static inline void
1785__instr_tx_i_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1786{
1787 uint64_t port_id = ip->io.io.val;
1788 struct port_out_runtime *port = &p->out[port_id];
1789 struct rte_swx_pkt *pkt = &t->pkt;
1790
1791 /* Recirculation: keep the current packet. */
1792 if (t->recirculate) {
1793 TRACE("[Thread %2u]: tx (i) 1 pkt - recirculate\n",
1794 p->thread_id);
1795
1796 /* Headers. */
1797 emit_handler(t);
1798
1799 /* Packet. */
1800 mirroring_handler(p, t, pkt);
1801
1802 return;
1803 }
1804
1805 TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
1806 p->thread_id,
1807 (uint32_t)port_id);
1808
1809 /* Headers. */
1810 emit_handler(t);
1811
1812 /* Packet. */
1813 mirroring_handler(p, t, pkt);
1814 port->pkt_tx(port->obj, pkt);
1815}
1816
1817static inline void
1818__instr_drop_exec(struct rte_swx_pipeline *p,
1819 struct thread *t,
1820 const struct instruction *ip __rte_unused)
1821{
1822 uint64_t port_id = p->n_ports_out - 1;
1823 struct port_out_runtime *port = &p->out[port_id];
1824 struct rte_swx_pkt *pkt = &t->pkt;
1825
1826 TRACE("[Thread %2u]: drop 1 pkt\n",
1827 p->thread_id);
1828
1829 /* Headers. */
1830 emit_handler(t);
1831
1832 /* Packet. */
1833 mirroring_handler(p, t, pkt);
1834 port->pkt_tx(port->obj, pkt);
1835}
1836
1837static inline void
1838__instr_mirror_exec(struct rte_swx_pipeline *p,
1839 struct thread *t,
1840 const struct instruction *ip)
1841{
1842 uint64_t slot_id = instr_operand_hbo(t, &ip->mirror.dst);
1843 uint64_t session_id = instr_operand_hbo(t, &ip->mirror.src);
1844
1845 slot_id &= p->n_mirroring_slots - 1;
1846 session_id &= p->n_mirroring_sessions - 1;
1847
1848 TRACE("[Thread %2u]: mirror pkt (slot = %u, session = %u)\n",
1849 p->thread_id,
1850 (uint32_t)slot_id,
1851 (uint32_t)session_id);
1852
1853 t->mirroring_slots[slot_id] = session_id;
1854 t->mirroring_slots_mask |= 1LLU << slot_id;
1855}
1856
1857static inline void
1858__instr_recirculate_exec(struct rte_swx_pipeline *p __rte_unused,
1859 struct thread *t,
1860 const struct instruction *ip __rte_unused)
1861{
1862 TRACE("[Thread %2u]: recirculate\n",
1863 p->thread_id);
1864
1865 t->recirculate = 1;
1866}
1867
1868static inline void
1869__instr_recircid_exec(struct rte_swx_pipeline *p __rte_unused,
1870 struct thread *t,
1871 const struct instruction *ip)
1872{
1873 TRACE("[Thread %2u]: recircid (pass %u)\n",
1874 p->thread_id,
1875 t->recirc_pass_id);
1876
1877 /* Meta-data. */
1878 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, t->recirc_pass_id);
1879}
1880
1881/*
1882 * extract.
1883 */
1884static inline void
1885__instr_hdr_extract_many_exec(struct rte_swx_pipeline *p __rte_unused,
1886 struct thread *t,
1887 const struct instruction *ip,
1888 uint32_t n_extract)
1889{
1890 uint64_t valid_headers = t->valid_headers;
1891 uint8_t *ptr = t->ptr;
1892 uint32_t offset = t->pkt.offset;
1893 uint32_t length = t->pkt.length;
1894 uint32_t i;
1895
1896 for (i = 0; i < n_extract; i++) {
1897 uint32_t header_id = ip->io.hdr.header_id[i];
1898 uint32_t struct_id = ip->io.hdr.struct_id[i];
1899 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
1900
1901 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
1902 p->thread_id,
1903 header_id,
1904 n_bytes);
1905
1906 /* Headers. */
1907 t->structs[struct_id] = ptr;
1908 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1909
1910 /* Packet. */
1911 offset += n_bytes;
1912 length -= n_bytes;
1913 ptr += n_bytes;
1914 }
1915
1916 /* Headers. */
1917 t->valid_headers = valid_headers;
1918
1919 /* Packet. */
1920 t->pkt.offset = offset;
1921 t->pkt.length = length;
1922 t->ptr = ptr;
1923}
1924
1925static inline void
1926__instr_hdr_extract_exec(struct rte_swx_pipeline *p,
1927 struct thread *t,
1928 const struct instruction *ip)
1929{
1930 __instr_hdr_extract_many_exec(p, t, ip, 1);
1931}
1932
1933static inline void
1934__instr_hdr_extract2_exec(struct rte_swx_pipeline *p,
1935 struct thread *t,
1936 const struct instruction *ip)
1937{
1938 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
1939
1940 __instr_hdr_extract_many_exec(p, t, ip, 2);
1941}
1942
1943static inline void
1944__instr_hdr_extract3_exec(struct rte_swx_pipeline *p,
1945 struct thread *t,
1946 const struct instruction *ip)
1947{
1948 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
1949
1950 __instr_hdr_extract_many_exec(p, t, ip, 3);
1951}
1952
1953static inline void
1954__instr_hdr_extract4_exec(struct rte_swx_pipeline *p,
1955 struct thread *t,
1956 const struct instruction *ip)
1957{
1958 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
1959
1960 __instr_hdr_extract_many_exec(p, t, ip, 4);
1961}
1962
1963static inline void
1964__instr_hdr_extract5_exec(struct rte_swx_pipeline *p,
1965 struct thread *t,
1966 const struct instruction *ip)
1967{
1968 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
1969
1970 __instr_hdr_extract_many_exec(p, t, ip, 5);
1971}
1972
1973static inline void
1974__instr_hdr_extract6_exec(struct rte_swx_pipeline *p,
1975 struct thread *t,
1976 const struct instruction *ip)
1977{
1978 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
1979
1980 __instr_hdr_extract_many_exec(p, t, ip, 6);
1981}
1982
1983static inline void
1984__instr_hdr_extract7_exec(struct rte_swx_pipeline *p,
1985 struct thread *t,
1986 const struct instruction *ip)
1987{
1988 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
1989
1990 __instr_hdr_extract_many_exec(p, t, ip, 7);
1991}
1992
1993static inline void
1994__instr_hdr_extract8_exec(struct rte_swx_pipeline *p,
1995 struct thread *t,
1996 const struct instruction *ip)
1997{
1998 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
1999
2000 __instr_hdr_extract_many_exec(p, t, ip, 8);
2001}
2002
2003static inline void
2004__instr_hdr_extract_m_exec(struct rte_swx_pipeline *p __rte_unused,
2005 struct thread *t,
2006 const struct instruction *ip)
2007{
2008 uint64_t valid_headers = t->valid_headers;
2009 uint8_t *ptr = t->ptr;
2010 uint32_t offset = t->pkt.offset;
2011 uint32_t length = t->pkt.length;
2012
2013 uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2014 uint32_t header_id = ip->io.hdr.header_id[0];
2015 uint32_t struct_id = ip->io.hdr.struct_id[0];
2016 uint32_t n_bytes = ip->io.hdr.n_bytes[0];
2017
2018 struct header_runtime *h = &t->headers[header_id];
2019
2020 TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n",
2021 p->thread_id,
2022 header_id,
2023 n_bytes,
2024 n_bytes_last);
2025
2026 n_bytes += n_bytes_last;
2027
2028 /* Headers. */
2029 t->structs[struct_id] = ptr;
2030 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2031 h->n_bytes = n_bytes;
2032
2033 /* Packet. */
2034 t->pkt.offset = offset + n_bytes;
2035 t->pkt.length = length - n_bytes;
2036 t->ptr = ptr + n_bytes;
2037}
2038
2039static inline void
2040__instr_hdr_lookahead_exec(struct rte_swx_pipeline *p __rte_unused,
2041 struct thread *t,
2042 const struct instruction *ip)
2043{
2044 uint64_t valid_headers = t->valid_headers;
2045 uint8_t *ptr = t->ptr;
2046
2047 uint32_t header_id = ip->io.hdr.header_id[0];
2048 uint32_t struct_id = ip->io.hdr.struct_id[0];
2049
2050 TRACE("[Thread %2u]: lookahead header %u\n",
2051 p->thread_id,
2052 header_id);
2053
2054 /* Headers. */
2055 t->structs[struct_id] = ptr;
2056 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2057}
2058
2059/*
2060 * emit.
2061 */
2062static inline void
2063__instr_hdr_emit_many_exec(struct rte_swx_pipeline *p __rte_unused,
2064 struct thread *t,
2065 const struct instruction *ip,
2066 uint32_t n_emit)
2067{
2068 uint64_t valid_headers = t->valid_headers;
2069 uint32_t n_headers_out = t->n_headers_out;
2070 struct header_out_runtime *ho = NULL;
2071 uint8_t *ho_ptr = NULL;
2072 uint32_t ho_nbytes = 0, i;
2073
2074 for (i = 0; i < n_emit; i++) {
2075 uint32_t header_id = ip->io.hdr.header_id[i];
2076 uint32_t struct_id = ip->io.hdr.struct_id[i];
2077
2078 struct header_runtime *hi = &t->headers[header_id];
2079 uint8_t *hi_ptr0 = hi->ptr0;
2080 uint32_t n_bytes = hi->n_bytes;
2081
2082 uint8_t *hi_ptr = t->structs[struct_id];
2083
2084 if (!MASK64_BIT_GET(valid_headers, header_id)) {
2085 TRACE("[Thread %2u]: emit header %u (invalid)\n",
2086 p->thread_id,
2087 header_id);
2088
2089 continue;
2090 }
2091
2092 TRACE("[Thread %2u]: emit header %u (valid)\n",
2093 p->thread_id,
2094 header_id);
2095
2096 /* Headers. */
2097 if (!ho) {
2098 if (!n_headers_out) {
2099 ho = &t->headers_out[0];
2100
2101 ho->ptr0 = hi_ptr0;
2102 ho->ptr = hi_ptr;
2103
2104 ho_ptr = hi_ptr;
2105 ho_nbytes = n_bytes;
2106
2107 n_headers_out = 1;
2108
2109 continue;
2110 } else {
2111 ho = &t->headers_out[n_headers_out - 1];
2112
2113 ho_ptr = ho->ptr;
2114 ho_nbytes = ho->n_bytes;
2115 }
2116 }
2117
2118 if (ho_ptr + ho_nbytes == hi_ptr) {
2119 ho_nbytes += n_bytes;
2120 } else {
2121 ho->n_bytes = ho_nbytes;
2122
2123 ho++;
2124 ho->ptr0 = hi_ptr0;
2125 ho->ptr = hi_ptr;
2126
2127 ho_ptr = hi_ptr;
2128 ho_nbytes = n_bytes;
2129
2130 n_headers_out++;
2131 }
2132 }
2133
2134 if (ho)
2135 ho->n_bytes = ho_nbytes;
2136 t->n_headers_out = n_headers_out;
2137}
2138
2139static inline void
2140__instr_hdr_emit_exec(struct rte_swx_pipeline *p,
2141 struct thread *t,
2142 const struct instruction *ip)
2143{
2144 __instr_hdr_emit_many_exec(p, t, ip, 1);
2145}
2146
2147static inline void
2148__instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p,
2149 struct thread *t,
2150 const struct instruction *ip)
2151{
2152 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
2153
2154 __instr_hdr_emit_many_exec(p, t, ip, 1);
2155 __instr_tx_exec(p, t, ip);
2156}
2157
2158static inline void
2159__instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p,
2160 struct thread *t,
2161 const struct instruction *ip)
2162{
2163 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
2164
2165 __instr_hdr_emit_many_exec(p, t, ip, 2);
2166 __instr_tx_exec(p, t, ip);
2167}
2168
2169static inline void
2170__instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p,
2171 struct thread *t,
2172 const struct instruction *ip)
2173{
2174 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2175
2176 __instr_hdr_emit_many_exec(p, t, ip, 3);
2177 __instr_tx_exec(p, t, ip);
2178}
2179
2180static inline void
2181__instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p,
2182 struct thread *t,
2183 const struct instruction *ip)
2184{
2185 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2186
2187 __instr_hdr_emit_many_exec(p, t, ip, 4);
2188 __instr_tx_exec(p, t, ip);
2189}
2190
2191static inline void
2192__instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p,
2193 struct thread *t,
2194 const struct instruction *ip)
2195{
2196 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2197
2198 __instr_hdr_emit_many_exec(p, t, ip, 5);
2199 __instr_tx_exec(p, t, ip);
2200}
2201
2202static inline void
2203__instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p,
2204 struct thread *t,
2205 const struct instruction *ip)
2206{
2207 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2208
2209 __instr_hdr_emit_many_exec(p, t, ip, 6);
2210 __instr_tx_exec(p, t, ip);
2211}
2212
2213static inline void
2214__instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p,
2215 struct thread *t,
2216 const struct instruction *ip)
2217{
2218 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2219
2220 __instr_hdr_emit_many_exec(p, t, ip, 7);
2221 __instr_tx_exec(p, t, ip);
2222}
2223
2224static inline void
2225__instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p,
2226 struct thread *t,
2227 const struct instruction *ip)
2228{
2229 TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n", p->thread_id);
2230
2231 __instr_hdr_emit_many_exec(p, t, ip, 8);
2232 __instr_tx_exec(p, t, ip);
2233}
2234
2235/*
2236 * validate.
2237 */
2238static inline void
2239__instr_hdr_validate_exec(struct rte_swx_pipeline *p __rte_unused,
2240 struct thread *t,
2241 const struct instruction *ip)
2242{
2243 uint32_t header_id = ip->valid.header_id;
2244 uint32_t struct_id = ip->valid.struct_id;
2245 uint64_t valid_headers = t->valid_headers;
2246 struct header_runtime *h = &t->headers[header_id];
2247
2248 TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2249
2250 /* If this header is already valid, then its associated t->structs[] element is also valid
2251 * and therefore it should not be modified. It could point to the packet buffer (in case of
2252 * extracted header) and setting it to the default location (h->ptr0) would be incorrect.
2253 */
2254 if (MASK64_BIT_GET(valid_headers, header_id))
2255 return;
2256
2257 /* Headers. */
2258 t->structs[struct_id] = h->ptr0;
2259 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2260}
2261
2262/*
2263 * invalidate.
2264 */
2265static inline void
2266__instr_hdr_invalidate_exec(struct rte_swx_pipeline *p __rte_unused,
2267 struct thread *t,
2268 const struct instruction *ip)
2269{
2270 uint32_t header_id = ip->valid.header_id;
2271
2272 TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2273
2274 /* Headers. */
2275 t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2276}
2277
2278/*
2279 * learn.
2280 */
2281static inline void
2282__instr_learn_exec(struct rte_swx_pipeline *p,
2283 struct thread *t,
2284 const struct instruction *ip)
2285{
2286 uint64_t action_id = ip->learn.action_id;
2287 uint32_t mf_first_arg_offset = ip->learn.mf_first_arg_offset;
2288 uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset,
2289 ip->learn.mf_timeout_id_n_bits);
2290 uint32_t learner_id = t->learner_id;
2291 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2292 p->n_selectors + learner_id];
2293 struct learner_runtime *l = &t->learners[learner_id];
2294 struct learner_statistics *stats = &p->learner_stats[learner_id];
2295 uint32_t status;
2296
2297 /* Table. */
2298 status = rte_swx_table_learner_add(ts->obj,
2299 l->mailbox,
2300 t->time,
2301 action_id,
2302 &t->metadata[mf_first_arg_offset],
2303 timeout_id);
2304
2305 TRACE("[Thread %2u] learner %u learn %s\n",
2306 p->thread_id,
2307 learner_id,
2308 status ? "ok" : "error");
2309
2310 stats->n_pkts_learn[status] += 1;
2311}
2312
2313/*
2314 * rearm.
2315 */
2316static inline void
2317__instr_rearm_exec(struct rte_swx_pipeline *p,
2318 struct thread *t,
2319 const struct instruction *ip __rte_unused)
2320{
2321 uint32_t learner_id = t->learner_id;
2322 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2323 p->n_selectors + learner_id];
2324 struct learner_runtime *l = &t->learners[learner_id];
2325 struct learner_statistics *stats = &p->learner_stats[learner_id];
2326
2327 /* Table. */
2328 rte_swx_table_learner_rearm(ts->obj, l->mailbox, t->time);
2329
2330 TRACE("[Thread %2u] learner %u rearm\n",
2331 p->thread_id,
2332 learner_id);
2333
2334 stats->n_pkts_rearm += 1;
2335}
2336
2337static inline void
2338__instr_rearm_new_exec(struct rte_swx_pipeline *p,
2339 struct thread *t,
2340 const struct instruction *ip)
2341{
2342 uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset,
2343 ip->learn.mf_timeout_id_n_bits);
2344 uint32_t learner_id = t->learner_id;
2345 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2346 p->n_selectors + learner_id];
2347 struct learner_runtime *l = &t->learners[learner_id];
2348 struct learner_statistics *stats = &p->learner_stats[learner_id];
2349
2350 /* Table. */
2351 rte_swx_table_learner_rearm_new(ts->obj, l->mailbox, t->time, timeout_id);
2352
2353 TRACE("[Thread %2u] learner %u rearm with timeout ID %u\n",
2354 p->thread_id,
2355 learner_id,
2356 timeout_id);
2357
2358 stats->n_pkts_rearm += 1;
2359}
2360
2361/*
2362 * forget.
2363 */
2364static inline void
2365__instr_forget_exec(struct rte_swx_pipeline *p,
2366 struct thread *t,
2367 const struct instruction *ip __rte_unused)
2368{
2369 uint32_t learner_id = t->learner_id;
2370 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2371 p->n_selectors + learner_id];
2372 struct learner_runtime *l = &t->learners[learner_id];
2373 struct learner_statistics *stats = &p->learner_stats[learner_id];
2374
2375 /* Table. */
2376 rte_swx_table_learner_delete(ts->obj, l->mailbox);
2377
2378 TRACE("[Thread %2u] learner %u forget\n",
2379 p->thread_id,
2380 learner_id);
2381
2382 stats->n_pkts_forget += 1;
2383}
2384
2385/*
2386 * entryid.
2387 */
2388static inline void
2389__instr_entryid_exec(struct rte_swx_pipeline *p __rte_unused,
2390 struct thread *t,
2391 const struct instruction *ip)
2392{
2393 TRACE("[Thread %2u]: entryid\n",
2394 p->thread_id);
2395
2396 /* Meta-data. */
2397 METADATA_WRITE(t, ip->mov.dst.offset, ip->mov.dst.n_bits, t->entry_id);
2398}
2399
2400/*
2401 * extern.
2402 */
2403static inline uint32_t
2404__instr_extern_obj_exec(struct rte_swx_pipeline *p __rte_unused,
2405 struct thread *t,
2406 const struct instruction *ip)
2407{
2408 uint32_t obj_id = ip->ext_obj.ext_obj_id;
2409 uint32_t func_id = ip->ext_obj.func_id;
2410 struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
2411 rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
2412 uint32_t done;
2413
2414 TRACE("[Thread %2u] extern obj %u member func %u\n",
2415 p->thread_id,
2416 obj_id,
2417 func_id);
2418
2419 done = func(obj->obj, obj->mailbox);
2420
2421 return done;
2422}
2423
2424static inline uint32_t
2425__instr_extern_func_exec(struct rte_swx_pipeline *p __rte_unused,
2426 struct thread *t,
2427 const struct instruction *ip)
2428{
2429 uint32_t ext_func_id = ip->ext_func.ext_func_id;
2430 struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
2431 rte_swx_extern_func_t func = ext_func->func;
2432 uint32_t done;
2433
2434 TRACE("[Thread %2u] extern func %u\n",
2435 p->thread_id,
2436 ext_func_id);
2437
2438 done = func(ext_func->mailbox);
2439
2440 return done;
2441}
2442
2443/*
2444 * hash.
2445 */
2446static inline void
2447__instr_hash_func_exec(struct rte_swx_pipeline *p,
2448 struct thread *t,
2449 const struct instruction *ip)
2450{
2451 uint32_t hash_func_id = ip->hash_func.hash_func_id;
2452 uint32_t dst_offset = ip->hash_func.dst.offset;
2453 uint32_t n_dst_bits = ip->hash_func.dst.n_bits;
2454 uint32_t src_struct_id = ip->hash_func.src.struct_id;
2455 uint32_t src_offset = ip->hash_func.src.offset;
2456 uint32_t n_src_bytes = ip->hash_func.src.n_bytes;
2457
2458 struct hash_func_runtime *func = &p->hash_func_runtime[hash_func_id];
2459 uint8_t *src_ptr = t->structs[src_struct_id];
2460 uint32_t result;
2461
2462 TRACE("[Thread %2u] hash %u\n",
2463 p->thread_id,
2464 hash_func_id);
2465
2466 result = func->func(&src_ptr[src_offset], n_src_bytes, 0);
2467 METADATA_WRITE(t, dst_offset, n_dst_bits, result);
2468}
2469
2470/*
2471 * mov.
2472 */
2473static inline void
2474__instr_mov_exec(struct rte_swx_pipeline *p __rte_unused,
2475 struct thread *t,
2476 const struct instruction *ip)
2477{
2478 TRACE("[Thread %2u] mov\n", p->thread_id);
2479
2480 MOV(t, ip);
2481}
2482
2483static inline void
2484__instr_mov_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2485 struct thread *t,
2486 const struct instruction *ip)
2487{
2488 TRACE("[Thread %2u] mov (mh)\n", p->thread_id);
2489
2490 MOV_MH(t, ip);
2491}
2492
2493static inline void
2494__instr_mov_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2495 struct thread *t,
2496 const struct instruction *ip)
2497{
2498 TRACE("[Thread %2u] mov (hm)\n", p->thread_id);
2499
2500 MOV_HM(t, ip);
2501}
2502
2503static inline void
2504__instr_mov_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2505 struct thread *t,
2506 const struct instruction *ip)
2507{
2508 TRACE("[Thread %2u] mov (hh)\n", p->thread_id);
2509
2510 MOV_HH(t, ip);
2511}
2512
2513static inline void
2514__instr_mov_dma_exec(struct rte_swx_pipeline *p __rte_unused,
2515 struct thread *t,
2516 const struct instruction *ip)
2517{
2518 uint8_t *dst_struct = t->structs[ip->mov.dst.struct_id];
2519 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->mov.dst.offset];
2520 uint32_t *dst32_ptr;
2521 uint16_t *dst16_ptr;
2522 uint8_t *dst8_ptr;
2523
2524 uint8_t *src_struct = t->structs[ip->mov.src.struct_id];
2525 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->mov.src.offset];
2526 uint32_t *src32_ptr;
2527 uint16_t *src16_ptr;
2528 uint8_t *src8_ptr;
2529
2530 uint32_t n = ip->mov.dst.n_bits >> 3, i;
2531
2532 TRACE("[Thread %2u] mov (dma) %u bytes\n", p->thread_id, n);
2533
2534 /* 8-byte transfers. */
2535 for (i = 0; i < n >> 3; i++)
2536 *dst64_ptr++ = *src64_ptr++;
2537
2538 /* 4-byte transfers. */
2539 n &= 7;
2540 dst32_ptr = (uint32_t *)dst64_ptr;
2541 src32_ptr = (uint32_t *)src64_ptr;
2542
2543 for (i = 0; i < n >> 2; i++)
2544 *dst32_ptr++ = *src32_ptr++;
2545
2546 /* 2-byte transfers. */
2547 n &= 3;
2548 dst16_ptr = (uint16_t *)dst32_ptr;
2549 src16_ptr = (uint16_t *)src32_ptr;
2550
2551 for (i = 0; i < n >> 1; i++)
2552 *dst16_ptr++ = *src16_ptr++;
2553
2554 /* 1-byte transfer. */
2555 n &= 1;
2556 dst8_ptr = (uint8_t *)dst16_ptr;
2557 src8_ptr = (uint8_t *)src16_ptr;
2558 if (n)
2559 *dst8_ptr = *src8_ptr;
2560}
2561
2562static inline void
2563__instr_mov_128_exec(struct rte_swx_pipeline *p __rte_unused,
2564 struct thread *t,
2565 const struct instruction *ip)
2566{
2567 uint8_t *dst_struct = t->structs[ip->mov.dst.struct_id];
2568 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->mov.dst.offset];
2569
2570 uint8_t *src_struct = t->structs[ip->mov.src.struct_id];
2571 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->mov.src.offset];
2572
2573 TRACE("[Thread %2u] mov (128)\n", p->thread_id);
2574
2575 dst64_ptr[0] = src64_ptr[0];
2576 dst64_ptr[1] = src64_ptr[1];
2577}
2578
2579static inline void
2580__instr_mov_i_exec(struct rte_swx_pipeline *p __rte_unused,
2581 struct thread *t,
2582 const struct instruction *ip)
2583{
2584 TRACE("[Thread %2u] mov m.f %" PRIx64 "\n", p->thread_id, ip->mov.src_val);
2585
2586 MOV_I(t, ip);
2587}
2588
2589/*
2590 * dma.
2591 */
2592static inline void
2593__instr_dma_ht_many_exec(struct rte_swx_pipeline *p __rte_unused,
2594 struct thread *t,
2595 const struct instruction *ip,
2596 uint32_t n_dma)
2597{
2598 uint8_t *action_data = t->structs[0];
2599 uint64_t valid_headers = t->valid_headers;
2600 uint32_t i;
2601
2602 for (i = 0; i < n_dma; i++) {
2603 uint32_t header_id = ip->dma.dst.header_id[i];
2604 uint32_t struct_id = ip->dma.dst.struct_id[i];
2605 uint32_t offset = ip->dma.src.offset[i];
2606 uint32_t n_bytes = ip->dma.n_bytes[i];
2607
2608 struct header_runtime *h = &t->headers[header_id];
2609 uint8_t *h_ptr0 = h->ptr0;
2610 uint8_t *h_ptr = t->structs[struct_id];
2611
2612 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2613 h_ptr : h_ptr0;
2614 void *src = &action_data[offset];
2615
2616 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2617
2618 /* Headers. */
2619 memcpy(dst, src, n_bytes);
2620 t->structs[struct_id] = dst;
2621 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2622 }
2623
2624 t->valid_headers = valid_headers;
2625}
2626
2627static inline void
2628__instr_dma_ht_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2629{
2630 __instr_dma_ht_many_exec(p, t, ip, 1);
2631}
2632
2633static inline void
2634__instr_dma_ht2_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2635{
2636 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
2637
2638 __instr_dma_ht_many_exec(p, t, ip, 2);
2639}
2640
2641static inline void
2642__instr_dma_ht3_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2643{
2644 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
2645
2646 __instr_dma_ht_many_exec(p, t, ip, 3);
2647}
2648
2649static inline void
2650__instr_dma_ht4_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2651{
2652 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2653
2654 __instr_dma_ht_many_exec(p, t, ip, 4);
2655}
2656
2657static inline void
2658__instr_dma_ht5_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2659{
2660 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2661
2662 __instr_dma_ht_many_exec(p, t, ip, 5);
2663}
2664
2665static inline void
2666__instr_dma_ht6_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2667{
2668 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2669
2670 __instr_dma_ht_many_exec(p, t, ip, 6);
2671}
2672
2673static inline void
2674__instr_dma_ht7_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2675{
2676 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2677
2678 __instr_dma_ht_many_exec(p, t, ip, 7);
2679}
2680
2681static inline void
2682__instr_dma_ht8_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2683{
2684 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2685
2686 __instr_dma_ht_many_exec(p, t, ip, 8);
2687}
2688
2689/*
2690 * alu.
2691 */
2692static inline void
2693__instr_alu_add_exec(struct rte_swx_pipeline *p __rte_unused,
2694 struct thread *t,
2695 const struct instruction *ip)
2696{
2697 TRACE("[Thread %2u] add\n", p->thread_id);
2698
2699 ALU(t, ip, +);
2700}
2701
2702static inline void
2703__instr_alu_add_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2704 struct thread *t,
2705 const struct instruction *ip)
2706{
2707 TRACE("[Thread %2u] add (mh)\n", p->thread_id);
2708
2709 ALU_MH(t, ip, +);
2710}
2711
2712static inline void
2713__instr_alu_add_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2714 struct thread *t,
2715 const struct instruction *ip)
2716{
2717 TRACE("[Thread %2u] add (hm)\n", p->thread_id);
2718
2719 ALU_HM(t, ip, +);
2720}
2721
2722static inline void
2723__instr_alu_add_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2724 struct thread *t,
2725 const struct instruction *ip)
2726{
2727 TRACE("[Thread %2u] add (hh)\n", p->thread_id);
2728
2729 ALU_HH(t, ip, +);
2730}
2731
2732static inline void
2733__instr_alu_add_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2734 struct thread *t,
2735 const struct instruction *ip)
2736{
2737 TRACE("[Thread %2u] add (mi)\n", p->thread_id);
2738
2739 ALU_MI(t, ip, +);
2740}
2741
2742static inline void
2743__instr_alu_add_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2744 struct thread *t,
2745 const struct instruction *ip)
2746{
2747 TRACE("[Thread %2u] add (hi)\n", p->thread_id);
2748
2749 ALU_HI(t, ip, +);
2750}
2751
2752static inline void
2753__instr_alu_sub_exec(struct rte_swx_pipeline *p __rte_unused,
2754 struct thread *t,
2755 const struct instruction *ip)
2756{
2757 TRACE("[Thread %2u] sub\n", p->thread_id);
2758
2759 ALU(t, ip, -);
2760}
2761
2762static inline void
2763__instr_alu_sub_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2764 struct thread *t,
2765 const struct instruction *ip)
2766{
2767 TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
2768
2769 ALU_MH(t, ip, -);
2770}
2771
2772static inline void
2773__instr_alu_sub_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2774 struct thread *t,
2775 const struct instruction *ip)
2776{
2777 TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
2778
2779 ALU_HM(t, ip, -);
2780}
2781
2782static inline void
2783__instr_alu_sub_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2784 struct thread *t,
2785 const struct instruction *ip)
2786{
2787 TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
2788
2789 ALU_HH(t, ip, -);
2790}
2791
2792static inline void
2793__instr_alu_sub_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2794 struct thread *t,
2795 const struct instruction *ip)
2796{
2797 TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
2798
2799 ALU_MI(t, ip, -);
2800}
2801
2802static inline void
2803__instr_alu_sub_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2804 struct thread *t,
2805 const struct instruction *ip)
2806{
2807 TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
2808
2809 ALU_HI(t, ip, -);
2810}
2811
2812static inline void
2813__instr_alu_shl_exec(struct rte_swx_pipeline *p __rte_unused,
2814 struct thread *t,
2815 const struct instruction *ip)
2816{
2817 TRACE("[Thread %2u] shl\n", p->thread_id);
2818
2819 ALU(t, ip, <<);
2820}
2821
2822static inline void
2823__instr_alu_shl_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2824 struct thread *t,
2825 const struct instruction *ip)
2826{
2827 TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
2828
2829 ALU_MH(t, ip, <<);
2830}
2831
2832static inline void
2833__instr_alu_shl_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2834 struct thread *t,
2835 const struct instruction *ip)
2836{
2837 TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
2838
2839 ALU_HM(t, ip, <<);
2840}
2841
2842static inline void
2843__instr_alu_shl_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2844 struct thread *t,
2845 const struct instruction *ip)
2846{
2847 TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
2848
2849 ALU_HH(t, ip, <<);
2850}
2851
2852static inline void
2853__instr_alu_shl_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2854 struct thread *t,
2855 const struct instruction *ip)
2856{
2857 TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
2858
2859 ALU_MI(t, ip, <<);
2860}
2861
2862static inline void
2863__instr_alu_shl_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2864 struct thread *t,
2865 const struct instruction *ip)
2866{
2867 TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
2868
2869 ALU_HI(t, ip, <<);
2870}
2871
2872static inline void
2873__instr_alu_shr_exec(struct rte_swx_pipeline *p __rte_unused,
2874 struct thread *t,
2875 const struct instruction *ip)
2876{
2877 TRACE("[Thread %2u] shr\n", p->thread_id);
2878
2879 ALU(t, ip, >>);
2880}
2881
2882static inline void
2883__instr_alu_shr_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2884 struct thread *t,
2885 const struct instruction *ip)
2886{
2887 TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
2888
2889 ALU_MH(t, ip, >>);
2890}
2891
2892static inline void
2893__instr_alu_shr_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2894 struct thread *t,
2895 const struct instruction *ip)
2896{
2897 TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
2898
2899 ALU_HM(t, ip, >>);
2900}
2901
2902static inline void
2903__instr_alu_shr_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2904 struct thread *t,
2905 const struct instruction *ip)
2906{
2907 TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
2908
2909 ALU_HH(t, ip, >>);
2910}
2911
2912static inline void
2913__instr_alu_shr_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2914 struct thread *t,
2915 const struct instruction *ip)
2916{
2917 TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
2918
2919 /* Structs. */
2920 ALU_MI(t, ip, >>);
2921}
2922
2923static inline void
2924__instr_alu_shr_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2925 struct thread *t,
2926 const struct instruction *ip)
2927{
2928 TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
2929
2930 ALU_HI(t, ip, >>);
2931}
2932
2933static inline void
2934__instr_alu_and_exec(struct rte_swx_pipeline *p __rte_unused,
2935 struct thread *t,
2936 const struct instruction *ip)
2937{
2938 TRACE("[Thread %2u] and\n", p->thread_id);
2939
2940 ALU(t, ip, &);
2941}
2942
2943static inline void
2944__instr_alu_and_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2945 struct thread *t,
2946 const struct instruction *ip)
2947{
2948 TRACE("[Thread %2u] and (mh)\n", p->thread_id);
2949
2950 ALU_MH(t, ip, &);
2951}
2952
2953static inline void
2954__instr_alu_and_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2955 struct thread *t,
2956 const struct instruction *ip)
2957{
2958 TRACE("[Thread %2u] and (hm)\n", p->thread_id);
2959
2960 ALU_HM_FAST(t, ip, &);
2961}
2962
2963static inline void
2964__instr_alu_and_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2965 struct thread *t,
2966 const struct instruction *ip)
2967{
2968 TRACE("[Thread %2u] and (hh)\n", p->thread_id);
2969
2970 ALU_HH_FAST(t, ip, &);
2971}
2972
2973static inline void
2974__instr_alu_and_i_exec(struct rte_swx_pipeline *p __rte_unused,
2975 struct thread *t,
2976 const struct instruction *ip)
2977{
2978 TRACE("[Thread %2u] and (i)\n", p->thread_id);
2979
2980 ALU_I(t, ip, &);
2981}
2982
2983static inline void
2984__instr_alu_or_exec(struct rte_swx_pipeline *p __rte_unused,
2985 struct thread *t,
2986 const struct instruction *ip)
2987{
2988 TRACE("[Thread %2u] or\n", p->thread_id);
2989
2990 ALU(t, ip, |);
2991}
2992
2993static inline void
2994__instr_alu_or_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2995 struct thread *t,
2996 const struct instruction *ip)
2997{
2998 TRACE("[Thread %2u] or (mh)\n", p->thread_id);
2999
3000 ALU_MH(t, ip, |);
3001}
3002
3003static inline void
3004__instr_alu_or_hm_exec(struct rte_swx_pipeline *p __rte_unused,
3005 struct thread *t,
3006 const struct instruction *ip)
3007{
3008 TRACE("[Thread %2u] or (hm)\n", p->thread_id);
3009
3010 ALU_HM_FAST(t, ip, |);
3011}
3012
3013static inline void
3014__instr_alu_or_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3015 struct thread *t,
3016 const struct instruction *ip)
3017{
3018 TRACE("[Thread %2u] or (hh)\n", p->thread_id);
3019
3020 ALU_HH_FAST(t, ip, |);
3021}
3022
3023static inline void
3024__instr_alu_or_i_exec(struct rte_swx_pipeline *p __rte_unused,
3025 struct thread *t,
3026 const struct instruction *ip)
3027{
3028 TRACE("[Thread %2u] or (i)\n", p->thread_id);
3029
3030 ALU_I(t, ip, |);
3031}
3032
3033static inline void
3034__instr_alu_xor_exec(struct rte_swx_pipeline *p __rte_unused,
3035 struct thread *t,
3036 const struct instruction *ip)
3037{
3038 TRACE("[Thread %2u] xor\n", p->thread_id);
3039
3040 ALU(t, ip, ^);
3041}
3042
3043static inline void
3044__instr_alu_xor_mh_exec(struct rte_swx_pipeline *p __rte_unused,
3045 struct thread *t,
3046 const struct instruction *ip)
3047{
3048 TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
3049
3050 ALU_MH(t, ip, ^);
3051}
3052
3053static inline void
3054__instr_alu_xor_hm_exec(struct rte_swx_pipeline *p __rte_unused,
3055 struct thread *t,
3056 const struct instruction *ip)
3057{
3058 TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
3059
3060 ALU_HM_FAST(t, ip, ^);
3061}
3062
3063static inline void
3064__instr_alu_xor_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3065 struct thread *t,
3066 const struct instruction *ip)
3067{
3068 TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
3069
3070 ALU_HH_FAST(t, ip, ^);
3071}
3072
3073static inline void
3074__instr_alu_xor_i_exec(struct rte_swx_pipeline *p __rte_unused,
3075 struct thread *t,
3076 const struct instruction *ip)
3077{
3078 TRACE("[Thread %2u] xor (i)\n", p->thread_id);
3079
3080 ALU_I(t, ip, ^);
3081}
3082
3083static inline void
3084__instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p __rte_unused,
3085 struct thread *t,
3086 const struct instruction *ip)
3087{
3088 uint8_t *dst_struct, *src_struct;
3089 uint16_t *dst16_ptr, dst;
3090 uint64_t *src64_ptr, src64, src64_mask, src;
3091 uint64_t r;
3092
3093 TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3094
3095 /* Structs. */
3096 dst_struct = t->structs[ip->alu.dst.struct_id];
3097 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3098 dst = *dst16_ptr;
3099
3100 src_struct = t->structs[ip->alu.src.struct_id];
3101 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3102 src64 = *src64_ptr;
3103 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3104 src = src64 & src64_mask;
3105
3106 /* Initialize the result with destination 1's complement. */
3107 r = dst;
3108 r = ~r & 0xFFFF;
3109
3110 /* The first input (r) is a 16-bit number. The second and the third
3111 * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3112 * three numbers (output r) is a 34-bit number.
3113 */
3114 r += (src >> 32) + (src & 0xFFFFFFFF);
3115
3116 /* The first input is a 16-bit number. The second input is an 18-bit
3117 * number. In the worst case scenario, the sum of the two numbers is a
3118 * 19-bit number.
3119 */
3120 r = (r & 0xFFFF) + (r >> 16);
3121
3122 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3123 * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3124 */
3125 r = (r & 0xFFFF) + (r >> 16);
3126
3127 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3128 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3129 * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3130 * therefore the output r is always a 16-bit number.
3131 */
3132 r = (r & 0xFFFF) + (r >> 16);
3133
3134 /* Apply 1's complement to the result. */
3135 r = ~r & 0xFFFF;
3136 r = r ? r : 0xFFFF;
3137
3138 *dst16_ptr = (uint16_t)r;
3139}
3140
3141static inline void
3142__instr_alu_cksub_field_exec(struct rte_swx_pipeline *p __rte_unused,
3143 struct thread *t,
3144 const struct instruction *ip)
3145{
3146 uint8_t *dst_struct, *src_struct;
3147 uint16_t *dst16_ptr, dst;
3148 uint64_t *src64_ptr, src64, src64_mask, src;
3149 uint64_t r;
3150
3151 TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
3152
3153 /* Structs. */
3154 dst_struct = t->structs[ip->alu.dst.struct_id];
3155 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3156 dst = *dst16_ptr;
3157
3158 src_struct = t->structs[ip->alu.src.struct_id];
3159 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3160 src64 = *src64_ptr;
3161 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3162 src = src64 & src64_mask;
3163
3164 /* Initialize the result with destination 1's complement. */
3165 r = dst;
3166 r = ~r & 0xFFFF;
3167
3168 /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
3169 * the following sequence of operations in 2's complement arithmetic:
3170 * a '- b = (a - b) % 0xFFFF.
3171 *
3172 * In order to prevent an underflow for the below subtraction, in which
3173 * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
3174 * minuend), we first add a multiple of the 0xFFFF modulus to the
3175 * minuend. The number we add to the minuend needs to be a 34-bit number
3176 * or higher, so for readability reasons we picked the 36-bit multiple.
3177 * We are effectively turning the 16-bit minuend into a 36-bit number:
3178 * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
3179 */
3180 r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
3181
3182 /* A 33-bit number is subtracted from a 36-bit number (the input r). The
3183 * result (the output r) is a 36-bit number.
3184 */
3185 r -= (src >> 32) + (src & 0xFFFFFFFF);
3186
3187 /* The first input is a 16-bit number. The second input is a 20-bit
3188 * number. Their sum is a 21-bit number.
3189 */
3190 r = (r & 0xFFFF) + (r >> 16);
3191
3192 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3193 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
3194 */
3195 r = (r & 0xFFFF) + (r >> 16);
3196
3197 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3198 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3199 * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3200 * generated, therefore the output r is always a 16-bit number.
3201 */
3202 r = (r & 0xFFFF) + (r >> 16);
3203
3204 /* Apply 1's complement to the result. */
3205 r = ~r & 0xFFFF;
3206 r = r ? r : 0xFFFF;
3207
3208 *dst16_ptr = (uint16_t)r;
3209}
3210
3211static inline void
3212__instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p __rte_unused,
3213 struct thread *t,
3214 const struct instruction *ip)
3215{
3216 uint8_t *dst_struct, *src_struct;
3217 uint16_t *dst16_ptr, dst;
3218 uint32_t *src32_ptr;
3219 uint64_t r0, r1;
3220
3221 TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
3222
3223 /* Structs. */
3224 dst_struct = t->structs[ip->alu.dst.struct_id];
3225 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3226 dst = *dst16_ptr;
3227
3228 src_struct = t->structs[ip->alu.src.struct_id];
3229 src32_ptr = (uint32_t *)&src_struct[0];
3230
3231 /* Initialize the result with destination 1's complement. */
3232 r0 = dst;
3233 r0 = ~r0 & 0xFFFF;
3234
3235 r0 += src32_ptr[0]; /* The output r0 is a 33-bit number. */
3236 r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
3237 r0 += src32_ptr[2]; /* The output r0 is a 34-bit number. */
3238 r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
3239 r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
3240
3241 /* The first input is a 16-bit number. The second input is a 19-bit
3242 * number. Their sum is a 20-bit number.
3243 */
3244 r0 = (r0 & 0xFFFF) + (r0 >> 16);
3245
3246 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3247 * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
3248 */
3249 r0 = (r0 & 0xFFFF) + (r0 >> 16);
3250
3251 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3252 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3253 * 0x1000E), the output r is (0 .. 15). So no carry bit can be
3254 * generated, therefore the output r is always a 16-bit number.
3255 */
3256 r0 = (r0 & 0xFFFF) + (r0 >> 16);
3257
3258 /* Apply 1's complement to the result. */
3259 r0 = ~r0 & 0xFFFF;
3260 r0 = r0 ? r0 : 0xFFFF;
3261
3262 *dst16_ptr = (uint16_t)r0;
3263}
3264
3265static inline void
3266__instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p __rte_unused,
3267 struct thread *t,
3268 const struct instruction *ip)
3269{
3270 uint32_t src_header_id = ip->alu.src.n_bits; /* The src header ID is stored here. */
3271 uint32_t n_src_header_bytes = t->headers[src_header_id].n_bytes;
3272 uint8_t *dst_struct, *src_struct;
3273 uint16_t *dst16_ptr, dst;
3274 uint32_t *src32_ptr;
3275 uint64_t r;
3276 uint32_t i;
3277
3278 if (n_src_header_bytes == 20) {
3279 __instr_alu_ckadd_struct20_exec(p, t, ip);
3280 return;
3281 }
3282
3283 TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
3284
3285 /* Structs. */
3286 dst_struct = t->structs[ip->alu.dst.struct_id];
3287 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3288 dst = *dst16_ptr;
3289
3290 src_struct = t->structs[ip->alu.src.struct_id];
3291 src32_ptr = (uint32_t *)&src_struct[0];
3292
3293 /* Initialize the result with destination 1's complement. */
3294 r = dst;
3295 r = ~r & 0xFFFF;
3296
3297 /* The max number of 32-bit words in a 32K-byte header is 2^13.
3298 * Therefore, in the worst case scenario, a 45-bit number is added to a
3299 * 16-bit number (the input r), so the output r is 46-bit number.
3300 */
3301 for (i = 0; i < n_src_header_bytes / 4; i++, src32_ptr++)
3302 r += *src32_ptr;
3303
3304 /* The first input is a 16-bit number. The second input is a 30-bit
3305 * number. Their sum is a 31-bit number.
3306 */
3307 r = (r & 0xFFFF) + (r >> 16);
3308
3309 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3310 * a 15-bit number (0 .. 0x7FFF). The sum is a 17-bit number (0 .. 0x17FFE).
3311 */
3312 r = (r & 0xFFFF) + (r >> 16);
3313
3314 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3315 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3316 * 0x17FFE), the output r is (0 .. 0x7FFF). So no carry bit can be
3317 * generated, therefore the output r is always a 16-bit number.
3318 */
3319 r = (r & 0xFFFF) + (r >> 16);
3320
3321 /* Apply 1's complement to the result. */
3322 r = ~r & 0xFFFF;
3323 r = r ? r : 0xFFFF;
3324
3325 *dst16_ptr = (uint16_t)r;
3326}
3327
3328/*
3329 * Register array.
3330 */
3331static inline uint64_t *
3332instr_regarray_regarray(struct rte_swx_pipeline *p, const struct instruction *ip)
3333{
3334 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3335 return r->regarray;
3336}
3337
3338static inline uint64_t
3339instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3340{
3341 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3342
3343 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
3344 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
3345 uint64_t idx64 = *idx64_ptr;
3346 uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
3347 uint64_t idx = idx64 & idx64_mask & r->size_mask;
3348
3349 return idx;
3350}
3351
3352#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3353
3354static inline uint64_t
3355instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3356{
3357 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3358
3359 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
3360 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
3361 uint64_t idx64 = *idx64_ptr;
3362 uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
3363
3364 return idx;
3365}
3366
3367#else
3368
3369#define instr_regarray_idx_nbo instr_regarray_idx_hbo
3370
3371#endif
3372
3373static inline uint64_t
3374instr_regarray_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
3375{
3376 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3377
3378 uint64_t idx = ip->regarray.idx_val & r->size_mask;
3379
3380 return idx;
3381}
3382
3383static inline uint64_t
3384instr_regarray_src_hbo(struct thread *t, const struct instruction *ip)
3385{
3386 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
3387 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
3388 uint64_t src64 = *src64_ptr;
3389 uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3390 uint64_t src = src64 & src64_mask;
3391
3392 return src;
3393}
3394
3395#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3396
3397static inline uint64_t
3398instr_regarray_src_nbo(struct thread *t, const struct instruction *ip)
3399{
3400 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
3401 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
3402 uint64_t src64 = *src64_ptr;
3403 uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
3404
3405 return src;
3406}
3407
3408#else
3409
3410#define instr_regarray_src_nbo instr_regarray_src_hbo
3411
3412#endif
3413
3414static inline void
3415instr_regarray_dst_hbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
3416{
3417 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
3418 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
3419 uint64_t dst64 = *dst64_ptr;
3420 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3421
3422 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3423
3424}
3425
3426#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3427
3428static inline void
3429instr_regarray_dst_nbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
3430{
3431 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
3432 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
3433 uint64_t dst64 = *dst64_ptr;
3434 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3435
3436 src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
3437 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3438}
3439
3440#else
3441
3442#define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
3443
3444#endif
3445
3446static inline void
3447__instr_regprefetch_rh_exec(struct rte_swx_pipeline *p,
3448 struct thread *t,
3449 const struct instruction *ip)
3450{
3451 uint64_t *regarray, idx;
3452
3453 TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
3454
3455 regarray = instr_regarray_regarray(p, ip);
3456 idx = instr_regarray_idx_nbo(p, t, ip);
3457 rte_prefetch0(&regarray[idx]);
3458}
3459
3460static inline void
3461__instr_regprefetch_rm_exec(struct rte_swx_pipeline *p,
3462 struct thread *t,
3463 const struct instruction *ip)
3464{
3465 uint64_t *regarray, idx;
3466
3467 TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
3468
3469 regarray = instr_regarray_regarray(p, ip);
3470 idx = instr_regarray_idx_hbo(p, t, ip);
3471 rte_prefetch0(&regarray[idx]);
3472}
3473
3474static inline void
3475__instr_regprefetch_ri_exec(struct rte_swx_pipeline *p,
3476 struct thread *t __rte_unused,
3477 const struct instruction *ip)
3478{
3479 uint64_t *regarray, idx;
3480
3481 TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
3482
3483 regarray = instr_regarray_regarray(p, ip);
3484 idx = instr_regarray_idx_imm(p, ip);
3485 rte_prefetch0(&regarray[idx]);
3486}
3487
3488static inline void
3489__instr_regrd_hrh_exec(struct rte_swx_pipeline *p,
3490 struct thread *t,
3491 const struct instruction *ip)
3492{
3493 uint64_t *regarray, idx;
3494
3495 TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
3496
3497 regarray = instr_regarray_regarray(p, ip);
3498 idx = instr_regarray_idx_nbo(p, t, ip);
3499 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3500}
3501
3502static inline void
3503__instr_regrd_hrm_exec(struct rte_swx_pipeline *p,
3504 struct thread *t,
3505 const struct instruction *ip)
3506{
3507 uint64_t *regarray, idx;
3508
3509 TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
3510
3511 /* Structs. */
3512 regarray = instr_regarray_regarray(p, ip);
3513 idx = instr_regarray_idx_hbo(p, t, ip);
3514 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3515}
3516
3517static inline void
3518__instr_regrd_mrh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3519{
3520 uint64_t *regarray, idx;
3521
3522 TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
3523
3524 regarray = instr_regarray_regarray(p, ip);
3525 idx = instr_regarray_idx_nbo(p, t, ip);
3526 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3527}
3528
3529static inline void
3530__instr_regrd_mrm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3531{
3532 uint64_t *regarray, idx;
3533
3534 TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id);
3535
3536 regarray = instr_regarray_regarray(p, ip);
3537 idx = instr_regarray_idx_hbo(p, t, ip);
3538 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3539}
3540
3541static inline void
3542__instr_regrd_hri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3543{
3544 uint64_t *regarray, idx;
3545
3546 TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
3547
3548 regarray = instr_regarray_regarray(p, ip);
3549 idx = instr_regarray_idx_imm(p, ip);
3550 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3551}
3552
3553static inline void
3554__instr_regrd_mri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3555{
3556 uint64_t *regarray, idx;
3557
3558 TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
3559
3560 regarray = instr_regarray_regarray(p, ip);
3561 idx = instr_regarray_idx_imm(p, ip);
3562 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3563}
3564
3565static inline void
3566__instr_regwr_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3567{
3568 uint64_t *regarray, idx, src;
3569
3570 TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
3571
3572 regarray = instr_regarray_regarray(p, ip);
3573 idx = instr_regarray_idx_nbo(p, t, ip);
3574 src = instr_regarray_src_nbo(t, ip);
3575 regarray[idx] = src;
3576}
3577
3578static inline void
3579__instr_regwr_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3580{
3581 uint64_t *regarray, idx, src;
3582
3583 TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
3584
3585 regarray = instr_regarray_regarray(p, ip);
3586 idx = instr_regarray_idx_nbo(p, t, ip);
3587 src = instr_regarray_src_hbo(t, ip);
3588 regarray[idx] = src;
3589}
3590
3591static inline void
3592__instr_regwr_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3593{
3594 uint64_t *regarray, idx, src;
3595
3596 TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
3597
3598 regarray = instr_regarray_regarray(p, ip);
3599 idx = instr_regarray_idx_hbo(p, t, ip);
3600 src = instr_regarray_src_nbo(t, ip);
3601 regarray[idx] = src;
3602}
3603
3604static inline void
3605__instr_regwr_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3606{
3607 uint64_t *regarray, idx, src;
3608
3609 TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
3610
3611 regarray = instr_regarray_regarray(p, ip);
3612 idx = instr_regarray_idx_hbo(p, t, ip);
3613 src = instr_regarray_src_hbo(t, ip);
3614 regarray[idx] = src;
3615}
3616
3617static inline void
3618__instr_regwr_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3619{
3620 uint64_t *regarray, idx, src;
3621
3622 TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
3623
3624 regarray = instr_regarray_regarray(p, ip);
3625 idx = instr_regarray_idx_nbo(p, t, ip);
3626 src = ip->regarray.dstsrc_val;
3627 regarray[idx] = src;
3628}
3629
3630static inline void
3631__instr_regwr_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3632{
3633 uint64_t *regarray, idx, src;
3634
3635 TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
3636
3637 regarray = instr_regarray_regarray(p, ip);
3638 idx = instr_regarray_idx_hbo(p, t, ip);
3639 src = ip->regarray.dstsrc_val;
3640 regarray[idx] = src;
3641}
3642
3643static inline void
3644__instr_regwr_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3645{
3646 uint64_t *regarray, idx, src;
3647
3648 TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
3649
3650 regarray = instr_regarray_regarray(p, ip);
3651 idx = instr_regarray_idx_imm(p, ip);
3652 src = instr_regarray_src_nbo(t, ip);
3653 regarray[idx] = src;
3654}
3655
3656static inline void
3657__instr_regwr_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3658{
3659 uint64_t *regarray, idx, src;
3660
3661 TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
3662
3663 regarray = instr_regarray_regarray(p, ip);
3664 idx = instr_regarray_idx_imm(p, ip);
3665 src = instr_regarray_src_hbo(t, ip);
3666 regarray[idx] = src;
3667}
3668
3669static inline void
3670__instr_regwr_rii_exec(struct rte_swx_pipeline *p,
3671 struct thread *t __rte_unused,
3672 const struct instruction *ip)
3673{
3674 uint64_t *regarray, idx, src;
3675
3676 TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
3677
3678 regarray = instr_regarray_regarray(p, ip);
3679 idx = instr_regarray_idx_imm(p, ip);
3680 src = ip->regarray.dstsrc_val;
3681 regarray[idx] = src;
3682}
3683
3684static inline void
3685__instr_regadd_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3686{
3687 uint64_t *regarray, idx, src;
3688
3689 TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
3690
3691 regarray = instr_regarray_regarray(p, ip);
3692 idx = instr_regarray_idx_nbo(p, t, ip);
3693 src = instr_regarray_src_nbo(t, ip);
3694 regarray[idx] += src;
3695}
3696
3697static inline void
3698__instr_regadd_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3699{
3700 uint64_t *regarray, idx, src;
3701
3702 TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
3703
3704 regarray = instr_regarray_regarray(p, ip);
3705 idx = instr_regarray_idx_nbo(p, t, ip);
3706 src = instr_regarray_src_hbo(t, ip);
3707 regarray[idx] += src;
3708}
3709
3710static inline void
3711__instr_regadd_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3712{
3713 uint64_t *regarray, idx, src;
3714
3715 TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
3716
3717 regarray = instr_regarray_regarray(p, ip);
3718 idx = instr_regarray_idx_hbo(p, t, ip);
3719 src = instr_regarray_src_nbo(t, ip);
3720 regarray[idx] += src;
3721}
3722
3723static inline void
3724__instr_regadd_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3725{
3726 uint64_t *regarray, idx, src;
3727
3728 TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
3729
3730 regarray = instr_regarray_regarray(p, ip);
3731 idx = instr_regarray_idx_hbo(p, t, ip);
3732 src = instr_regarray_src_hbo(t, ip);
3733 regarray[idx] += src;
3734}
3735
3736static inline void
3737__instr_regadd_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3738{
3739 uint64_t *regarray, idx, src;
3740
3741 TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
3742
3743 regarray = instr_regarray_regarray(p, ip);
3744 idx = instr_regarray_idx_nbo(p, t, ip);
3745 src = ip->regarray.dstsrc_val;
3746 regarray[idx] += src;
3747}
3748
3749static inline void
3750__instr_regadd_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3751{
3752 uint64_t *regarray, idx, src;
3753
3754 TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
3755
3756 regarray = instr_regarray_regarray(p, ip);
3757 idx = instr_regarray_idx_hbo(p, t, ip);
3758 src = ip->regarray.dstsrc_val;
3759 regarray[idx] += src;
3760}
3761
3762static inline void
3763__instr_regadd_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3764{
3765 uint64_t *regarray, idx, src;
3766
3767 TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
3768
3769 regarray = instr_regarray_regarray(p, ip);
3770 idx = instr_regarray_idx_imm(p, ip);
3771 src = instr_regarray_src_nbo(t, ip);
3772 regarray[idx] += src;
3773}
3774
3775static inline void
3776__instr_regadd_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3777{
3778 uint64_t *regarray, idx, src;
3779
3780 TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
3781
3782 regarray = instr_regarray_regarray(p, ip);
3783 idx = instr_regarray_idx_imm(p, ip);
3784 src = instr_regarray_src_hbo(t, ip);
3785 regarray[idx] += src;
3786}
3787
3788static inline void
3789__instr_regadd_rii_exec(struct rte_swx_pipeline *p,
3790 struct thread *t __rte_unused,
3791 const struct instruction *ip)
3792{
3793 uint64_t *regarray, idx, src;
3794
3795 TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
3796
3797 regarray = instr_regarray_regarray(p, ip);
3798 idx = instr_regarray_idx_imm(p, ip);
3799 src = ip->regarray.dstsrc_val;
3800 regarray[idx] += src;
3801}
3802
3803/*
3804 * metarray.
3805 */
3806static inline struct meter *
3807instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3808{
3809 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3810
3811 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
3812 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
3813 uint64_t idx64 = *idx64_ptr;
3814 uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
3815 uint64_t idx = idx64 & idx64_mask & r->size_mask;
3816
3817 return &r->metarray[idx];
3818}
3819
3820#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3821
3822static inline struct meter *
3823instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3824{
3825 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3826
3827 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
3828 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
3829 uint64_t idx64 = *idx64_ptr;
3830 uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
3831
3832 return &r->metarray[idx];
3833}
3834
3835#else
3836
3837#define instr_meter_idx_nbo instr_meter_idx_hbo
3838
3839#endif
3840
3841static inline struct meter *
3842instr_meter_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
3843{
3844 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3845
3846 uint64_t idx = ip->meter.idx_val & r->size_mask;
3847
3848 return &r->metarray[idx];
3849}
3850
3851static inline uint32_t
3852instr_meter_length_hbo(struct thread *t, const struct instruction *ip)
3853{
3854 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
3855 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
3856 uint64_t src64 = *src64_ptr;
3857 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
3858 uint64_t src = src64 & src64_mask;
3859
3860 return (uint32_t)src;
3861}
3862
3863#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3864
3865static inline uint32_t
3866instr_meter_length_nbo(struct thread *t, const struct instruction *ip)
3867{
3868 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
3869 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
3870 uint64_t src64 = *src64_ptr;
3871 uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
3872
3873 return (uint32_t)src;
3874}
3875
3876#else
3877
3878#define instr_meter_length_nbo instr_meter_length_hbo
3879
3880#endif
3881
3882static inline enum rte_color
3883instr_meter_color_in_hbo(struct thread *t, const struct instruction *ip)
3884{
3885 uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
3886 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
3887 uint64_t src64 = *src64_ptr;
3888 uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
3889 uint64_t src = src64 & src64_mask;
3890
3891 return (enum rte_color)src;
3892}
3893
3894static inline void
3895instr_meter_color_out_hbo_set(struct thread *t,
3896 const struct instruction *ip,
3897 enum rte_color color_out)
3898{
3899 uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
3900 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
3901 uint64_t dst64 = *dst64_ptr;
3902 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
3903
3904 uint64_t src = (uint64_t)color_out;
3905
3906 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3907}
3908
3909static inline void
3910__instr_metprefetch_h_exec(struct rte_swx_pipeline *p,
3911 struct thread *t,
3912 const struct instruction *ip)
3913{
3914 struct meter *m;
3915
3916 TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
3917
3918 m = instr_meter_idx_nbo(p, t, ip);
3919 rte_prefetch0(m);
3920}
3921
3922static inline void
3923__instr_metprefetch_m_exec(struct rte_swx_pipeline *p,
3924 struct thread *t,
3925 const struct instruction *ip)
3926{
3927 struct meter *m;
3928
3929 TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
3930
3931 m = instr_meter_idx_hbo(p, t, ip);
3932 rte_prefetch0(m);
3933}
3934
3935static inline void
3936__instr_metprefetch_i_exec(struct rte_swx_pipeline *p,
3937 struct thread *t __rte_unused,
3938 const struct instruction *ip)
3939{
3940 struct meter *m;
3941
3942 TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
3943
3944 m = instr_meter_idx_imm(p, ip);
3945 rte_prefetch0(m);
3946}
3947
3948static inline void
3949__instr_meter_hhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3950{
3951 struct meter *m;
3952 uint64_t time, n_pkts, n_bytes;
3953 uint32_t length;
3954 enum rte_color color_in, color_out;
3955
3956 TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
3957
3958 m = instr_meter_idx_nbo(p, t, ip);
3959 rte_prefetch0(m->n_pkts);
3960 time = rte_get_tsc_cycles();
3961 length = instr_meter_length_nbo(t, ip);
3962 color_in = instr_meter_color_in_hbo(t, ip);
3963
3964 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3965 &m->profile->profile,
3966 time,
3967 length,
3968 color_in);
3969
3970 color_out &= m->color_mask;
3971
3972 n_pkts = m->n_pkts[color_out];
3973 n_bytes = m->n_bytes[color_out];
3974
3975 instr_meter_color_out_hbo_set(t, ip, color_out);
3976
3977 m->n_pkts[color_out] = n_pkts + 1;
3978 m->n_bytes[color_out] = n_bytes + length;
3979}
3980
3981static inline void
3982__instr_meter_hhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3983{
3984 struct meter *m;
3985 uint64_t time, n_pkts, n_bytes;
3986 uint32_t length;
3987 enum rte_color color_in, color_out;
3988
3989 TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
3990
3991 m = instr_meter_idx_nbo(p, t, ip);
3992 rte_prefetch0(m->n_pkts);
3993 time = rte_get_tsc_cycles();
3994 length = instr_meter_length_nbo(t, ip);
3995 color_in = (enum rte_color)ip->meter.color_in_val;
3996
3997 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3998 &m->profile->profile,
3999 time,
4000 length,
4001 color_in);
4002
4003 color_out &= m->color_mask;
4004
4005 n_pkts = m->n_pkts[color_out];
4006 n_bytes = m->n_bytes[color_out];
4007
4008 instr_meter_color_out_hbo_set(t, ip, color_out);
4009
4010 m->n_pkts[color_out] = n_pkts + 1;
4011 m->n_bytes[color_out] = n_bytes + length;
4012}
4013
4014static inline void
4015__instr_meter_hmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4016{
4017 struct meter *m;
4018 uint64_t time, n_pkts, n_bytes;
4019 uint32_t length;
4020 enum rte_color color_in, color_out;
4021
4022 TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
4023
4024 m = instr_meter_idx_nbo(p, t, ip);
4025 rte_prefetch0(m->n_pkts);
4026 time = rte_get_tsc_cycles();
4027 length = instr_meter_length_hbo(t, ip);
4028 color_in = instr_meter_color_in_hbo(t, ip);
4029
4030 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4031 &m->profile->profile,
4032 time,
4033 length,
4034 color_in);
4035
4036 color_out &= m->color_mask;
4037
4038 n_pkts = m->n_pkts[color_out];
4039 n_bytes = m->n_bytes[color_out];
4040
4041 instr_meter_color_out_hbo_set(t, ip, color_out);
4042
4043 m->n_pkts[color_out] = n_pkts + 1;
4044 m->n_bytes[color_out] = n_bytes + length;
4045}
4046
4047static inline void
4048__instr_meter_hmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4049{
4050 struct meter *m;
4051 uint64_t time, n_pkts, n_bytes;
4052 uint32_t length;
4053 enum rte_color color_in, color_out;
4054
4055 TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
4056
4057 m = instr_meter_idx_nbo(p, t, ip);
4058 rte_prefetch0(m->n_pkts);
4059 time = rte_get_tsc_cycles();
4060 length = instr_meter_length_hbo(t, ip);
4061 color_in = (enum rte_color)ip->meter.color_in_val;
4062
4063 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4064 &m->profile->profile,
4065 time,
4066 length,
4067 color_in);
4068
4069 color_out &= m->color_mask;
4070
4071 n_pkts = m->n_pkts[color_out];
4072 n_bytes = m->n_bytes[color_out];
4073
4074 instr_meter_color_out_hbo_set(t, ip, color_out);
4075
4076 m->n_pkts[color_out] = n_pkts + 1;
4077 m->n_bytes[color_out] = n_bytes + length;
4078}
4079
4080static inline void
4081__instr_meter_mhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4082{
4083 struct meter *m;
4084 uint64_t time, n_pkts, n_bytes;
4085 uint32_t length;
4086 enum rte_color color_in, color_out;
4087
4088 TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
4089
4090 m = instr_meter_idx_hbo(p, t, ip);
4091 rte_prefetch0(m->n_pkts);
4092 time = rte_get_tsc_cycles();
4093 length = instr_meter_length_nbo(t, ip);
4094 color_in = instr_meter_color_in_hbo(t, ip);
4095
4096 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4097 &m->profile->profile,
4098 time,
4099 length,
4100 color_in);
4101
4102 color_out &= m->color_mask;
4103
4104 n_pkts = m->n_pkts[color_out];
4105 n_bytes = m->n_bytes[color_out];
4106
4107 instr_meter_color_out_hbo_set(t, ip, color_out);
4108
4109 m->n_pkts[color_out] = n_pkts + 1;
4110 m->n_bytes[color_out] = n_bytes + length;
4111}
4112
4113static inline void
4114__instr_meter_mhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4115{
4116 struct meter *m;
4117 uint64_t time, n_pkts, n_bytes;
4118 uint32_t length;
4119 enum rte_color color_in, color_out;
4120
4121 TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
4122
4123 m = instr_meter_idx_hbo(p, t, ip);
4124 rte_prefetch0(m->n_pkts);
4125 time = rte_get_tsc_cycles();
4126 length = instr_meter_length_nbo(t, ip);
4127 color_in = (enum rte_color)ip->meter.color_in_val;
4128
4129 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4130 &m->profile->profile,
4131 time,
4132 length,
4133 color_in);
4134
4135 color_out &= m->color_mask;
4136
4137 n_pkts = m->n_pkts[color_out];
4138 n_bytes = m->n_bytes[color_out];
4139
4140 instr_meter_color_out_hbo_set(t, ip, color_out);
4141
4142 m->n_pkts[color_out] = n_pkts + 1;
4143 m->n_bytes[color_out] = n_bytes + length;
4144}
4145
4146static inline void
4147__instr_meter_mmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4148{
4149 struct meter *m;
4150 uint64_t time, n_pkts, n_bytes;
4151 uint32_t length;
4152 enum rte_color color_in, color_out;
4153
4154 TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
4155
4156 m = instr_meter_idx_hbo(p, t, ip);
4157 rte_prefetch0(m->n_pkts);
4158 time = rte_get_tsc_cycles();
4159 length = instr_meter_length_hbo(t, ip);
4160 color_in = instr_meter_color_in_hbo(t, ip);
4161
4162 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4163 &m->profile->profile,
4164 time,
4165 length,
4166 color_in);
4167
4168 color_out &= m->color_mask;
4169
4170 n_pkts = m->n_pkts[color_out];
4171 n_bytes = m->n_bytes[color_out];
4172
4173 instr_meter_color_out_hbo_set(t, ip, color_out);
4174
4175 m->n_pkts[color_out] = n_pkts + 1;
4176 m->n_bytes[color_out] = n_bytes + length;
4177}
4178
4179static inline void
4180__instr_meter_mmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4181{
4182 struct meter *m;
4183 uint64_t time, n_pkts, n_bytes;
4184 uint32_t length;
4185 enum rte_color color_in, color_out;
4186
4187 TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
4188
4189 m = instr_meter_idx_hbo(p, t, ip);
4190 rte_prefetch0(m->n_pkts);
4191 time = rte_get_tsc_cycles();
4192 length = instr_meter_length_hbo(t, ip);
4193 color_in = (enum rte_color)ip->meter.color_in_val;
4194
4195 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4196 &m->profile->profile,
4197 time,
4198 length,
4199 color_in);
4200
4201 color_out &= m->color_mask;
4202
4203 n_pkts = m->n_pkts[color_out];
4204 n_bytes = m->n_bytes[color_out];
4205
4206 instr_meter_color_out_hbo_set(t, ip, color_out);
4207
4208 m->n_pkts[color_out] = n_pkts + 1;
4209 m->n_bytes[color_out] = n_bytes + length;
4210}
4211
4212static inline void
4213__instr_meter_ihm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4214{
4215 struct meter *m;
4216 uint64_t time, n_pkts, n_bytes;
4217 uint32_t length;
4218 enum rte_color color_in, color_out;
4219
4220 TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
4221
4222 m = instr_meter_idx_imm(p, ip);
4223 rte_prefetch0(m->n_pkts);
4224 time = rte_get_tsc_cycles();
4225 length = instr_meter_length_nbo(t, ip);
4226 color_in = instr_meter_color_in_hbo(t, ip);
4227
4228 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4229 &m->profile->profile,
4230 time,
4231 length,
4232 color_in);
4233
4234 color_out &= m->color_mask;
4235
4236 n_pkts = m->n_pkts[color_out];
4237 n_bytes = m->n_bytes[color_out];
4238
4239 instr_meter_color_out_hbo_set(t, ip, color_out);
4240
4241 m->n_pkts[color_out] = n_pkts + 1;
4242 m->n_bytes[color_out] = n_bytes + length;
4243}
4244
4245static inline void
4246__instr_meter_ihi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4247{
4248 struct meter *m;
4249 uint64_t time, n_pkts, n_bytes;
4250 uint32_t length;
4251 enum rte_color color_in, color_out;
4252
4253 TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
4254
4255 m = instr_meter_idx_imm(p, ip);
4256 rte_prefetch0(m->n_pkts);
4257 time = rte_get_tsc_cycles();
4258 length = instr_meter_length_nbo(t, ip);
4259 color_in = (enum rte_color)ip->meter.color_in_val;
4260
4261 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4262 &m->profile->profile,
4263 time,
4264 length,
4265 color_in);
4266
4267 color_out &= m->color_mask;
4268
4269 n_pkts = m->n_pkts[color_out];
4270 n_bytes = m->n_bytes[color_out];
4271
4272 instr_meter_color_out_hbo_set(t, ip, color_out);
4273
4274 m->n_pkts[color_out] = n_pkts + 1;
4275 m->n_bytes[color_out] = n_bytes + length;
4276}
4277
4278static inline void
4279__instr_meter_imm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4280{
4281 struct meter *m;
4282 uint64_t time, n_pkts, n_bytes;
4283 uint32_t length;
4284 enum rte_color color_in, color_out;
4285
4286 TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
4287
4288 m = instr_meter_idx_imm(p, ip);
4289 rte_prefetch0(m->n_pkts);
4290 time = rte_get_tsc_cycles();
4291 length = instr_meter_length_hbo(t, ip);
4292 color_in = instr_meter_color_in_hbo(t, ip);
4293
4294 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4295 &m->profile->profile,
4296 time,
4297 length,
4298 color_in);
4299
4300 color_out &= m->color_mask;
4301
4302 n_pkts = m->n_pkts[color_out];
4303 n_bytes = m->n_bytes[color_out];
4304
4305 instr_meter_color_out_hbo_set(t, ip, color_out);
4306
4307 m->n_pkts[color_out] = n_pkts + 1;
4308 m->n_bytes[color_out] = n_bytes + length;
4309}
4310
4311static inline void
4312__instr_meter_imi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4313{
4314 struct meter *m;
4315 uint64_t time, n_pkts, n_bytes;
4316 uint32_t length;
4317 enum rte_color color_in, color_out;
4318
4319 TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
4320
4321 m = instr_meter_idx_imm(p, ip);
4322 rte_prefetch0(m->n_pkts);
4323 time = rte_get_tsc_cycles();
4324 length = instr_meter_length_hbo(t, ip);
4325 color_in = (enum rte_color)ip->meter.color_in_val;
4326
4327 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4328 &m->profile->profile,
4329 time,
4330 length,
4331 color_in);
4332
4333 color_out &= m->color_mask;
4334
4335 n_pkts = m->n_pkts[color_out];
4336 n_bytes = m->n_bytes[color_out];
4337
4338 instr_meter_color_out_hbo_set(t, ip, color_out);
4339
4340 m->n_pkts[color_out] = n_pkts + 1;
4341 m->n_bytes[color_out] = n_bytes + length;
4342}
4343
4344#endif
rte_byteorder.h
rte_common.h
__rte_unused
#define __rte_unused
Definition: rte_common.h:120
rte_cycles.h
rte_get_tsc_cycles
static uint64_t rte_get_tsc_cycles(void)
rte_meter.h
rte_meter_trtcm_color_aware_check
static enum rte_color rte_meter_trtcm_color_aware_check(struct rte_meter_trtcm *m, struct rte_meter_trtcm_profile *p, uint64_t time, uint32_t pkt_len, enum rte_color pkt_color)
Definition: rte_meter.h:526
rte_color
rte_color
Definition: rte_meter.h:35
RTE_COLORS
@ RTE_COLORS
Definition: rte_meter.h:39
rte_prefetch.h
rte_prefetch0
static void rte_prefetch0(const volatile void *p)
rte_swx_ctl.h
rte_swx_extern_type_destructor_t
void(* rte_swx_extern_type_destructor_t)(void *object)
Definition: rte_swx_extern.h:44
rte_swx_extern_func_t
int(* rte_swx_extern_func_t)(void *mailbox)
Definition: rte_swx_extern.h:92
rte_swx_extern_type_member_func_t
int(* rte_swx_extern_type_member_func_t)(void *object, void *mailbox)
Definition: rte_swx_extern.h:69
rte_swx_extern_type_constructor_t
void *(* rte_swx_extern_type_constructor_t)(const char *args)
Definition: rte_swx_extern.h:35
rte_swx_hash_func_t
uint32_t(* rte_swx_hash_func_t)(const void *key, uint32_t length, uint32_t seed)
Definition: rte_swx_hash_func.h:31
rte_swx_pipeline.h
RTE_SWX_NAME_SIZE
#define RTE_SWX_NAME_SIZE
Definition: rte_swx_pipeline.h:27
rte_swx_port_in_pkt_rx_t
int(* rte_swx_port_in_pkt_rx_t)(void *port, struct rte_swx_pkt *pkt)
Definition: rte_swx_port.h:72
rte_swx_port_out_flush_t
void(* rte_swx_port_out_flush_t)(void *port)
Definition: rte_swx_port.h:184
rte_swx_port_out_pkt_clone_tx_t
void(* rte_swx_port_out_pkt_clone_tx_t)(void *port, struct rte_swx_pkt *pkt, uint32_t truncation_length)
Definition: rte_swx_port.h:173
rte_swx_port_out_pkt_tx_t
void(* rte_swx_port_out_pkt_tx_t)(void *port, struct rte_swx_pkt *pkt)
Definition: rte_swx_port.h:147
rte_swx_port_out_pkt_fast_clone_tx_t
void(* rte_swx_port_out_pkt_fast_clone_tx_t)(void *port, struct rte_swx_pkt *pkt)
Definition: rte_swx_port.h:159
rte_swx_table_match_type
rte_swx_table_match_type
Definition: rte_swx_table.h:25
rte_swx_table_lookup_t
int(* rte_swx_table_lookup_t)(void *table, void *mailbox, uint8_t **key, uint64_t *action_id, uint8_t **action_data, size_t *entry_id, int *hit)
Definition: rte_swx_table.h:270
rte_swx_table_learner.h
rte_swx_table_learner_add
__rte_experimental uint32_t rte_swx_table_learner_add(void *table, void *mailbox, uint64_t time, uint64_t action_id, uint8_t *action_data, uint32_t key_timeout_id)
rte_swx_table_learner_rearm_new
__rte_experimental void rte_swx_table_learner_rearm_new(void *table, void *mailbox, uint64_t time, uint32_t key_timeout_id)
rte_swx_table_learner_delete
__rte_experimental void rte_swx_table_learner_delete(void *table, void *mailbox)
rte_swx_table_learner_rearm
__rte_experimental void rte_swx_table_learner_rearm(void *table, void *mailbox, uint64_t time)
RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX
#define RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX
Definition: rte_swx_table_learner.h:58
rte_swx_table_selector.h
rte_meter_trtcm_params
Definition: rte_meter.h:54
rte_meter_trtcm
Definition: rte_meter.h:359
rte_swx_pkt
Definition: rte_swx_port.h:21
rte_swx_pkt::pkt
uint8_t * pkt
Definition: rte_swx_port.h:26
rte_swx_pkt::offset
uint32_t offset
Definition: rte_swx_port.h:29
rte_swx_pkt::length
uint32_t length
Definition: rte_swx_port.h:32
rte_swx_port_in_ops
Definition: rte_swx_port.h:100
rte_swx_port_out_ops
Definition: rte_swx_port.h:220
rte_swx_table_ops
Definition: rte_swx_table.h:288
rte_swx_table_state
Definition: rte_swx_ctl.h:761
rte_swx_table_state::obj
void * obj
Definition: rte_swx_ctl.h:763
Generated by doxygen 1.9.4