DPDK 25.03.0-rc0
rte_swx_pipeline_internal.h
1/* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2021 Intel Corporation
3 */
4#ifndef __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
5#define __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
6
7#include <inttypes.h>
8#include <string.h>
9#include <sys/queue.h>
10
11#include <rte_bitops.h>
12#include <rte_byteorder.h>
13#include <rte_common.h>
14#include <rte_cycles.h>
15#include <rte_prefetch.h>
16#include <rte_meter.h>
17
18#include <rte_swx_table_selector.h>
19#include <rte_swx_table_learner.h>
20#include <rte_swx_pipeline.h>
21#include <rte_swx_ctl.h>
22
23#ifndef TRACE_LEVEL
24#define TRACE_LEVEL 0
25#endif
26
27#if TRACE_LEVEL
28#define TRACE(...) printf(__VA_ARGS__)
29#else
30#define TRACE(...)
31#endif
32
33/*
34 * Environment.
35 */
36#define ntoh64(x) rte_be_to_cpu_64(x)
37#define hton64(x) rte_cpu_to_be_64(x)
38
39/*
40 * Struct.
41 */
42struct field {
43 char name[RTE_SWX_NAME_SIZE];
44 uint32_t n_bits;
45 uint32_t offset;
46 int var_size;
47};
48
49struct struct_type {
50 TAILQ_ENTRY(struct_type) node;
51 char name[RTE_SWX_NAME_SIZE];
52 struct field *fields;
53 uint32_t n_fields;
54 uint32_t n_bits;
55 uint32_t n_bits_min;
56 int var_size;
57};
58
59TAILQ_HEAD(struct_type_tailq, struct_type);
60
61/*
62 * Input port.
63 */
64struct port_in_type {
65 TAILQ_ENTRY(port_in_type) node;
66 char name[RTE_SWX_NAME_SIZE];
67 struct rte_swx_port_in_ops ops;
68};
69
70TAILQ_HEAD(port_in_type_tailq, port_in_type);
71
72struct port_in {
73 TAILQ_ENTRY(port_in) node;
74 struct port_in_type *type;
75 void *obj;
76 uint32_t id;
77};
78
79TAILQ_HEAD(port_in_tailq, port_in);
80
81struct port_in_runtime {
82 rte_swx_port_in_pkt_rx_t pkt_rx;
83 void *obj;
84};
85
86/*
87 * Output port.
88 */
89struct port_out_type {
90 TAILQ_ENTRY(port_out_type) node;
91 char name[RTE_SWX_NAME_SIZE];
92 struct rte_swx_port_out_ops ops;
93};
94
95TAILQ_HEAD(port_out_type_tailq, port_out_type);
96
97struct port_out {
98 TAILQ_ENTRY(port_out) node;
99 struct port_out_type *type;
100 void *obj;
101 uint32_t id;
102};
103
104TAILQ_HEAD(port_out_tailq, port_out);
105
106struct port_out_runtime {
107 rte_swx_port_out_pkt_tx_t pkt_tx;
108 rte_swx_port_out_pkt_fast_clone_tx_t pkt_fast_clone_tx;
109 rte_swx_port_out_pkt_clone_tx_t pkt_clone_tx;
110 rte_swx_port_out_flush_t flush;
111 void *obj;
112};
113
114/*
115 * Packet mirroring.
116 */
117struct mirroring_session {
118 uint32_t port_id;
119 int fast_clone;
120 uint32_t truncation_length;
121};
122
123/*
124 * Extern object.
125 */
126struct extern_type_member_func {
127 TAILQ_ENTRY(extern_type_member_func) node;
128 char name[RTE_SWX_NAME_SIZE];
129 rte_swx_extern_type_member_func_t func;
130 uint32_t id;
131};
132
133TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
134
135struct extern_type {
136 TAILQ_ENTRY(extern_type) node;
137 char name[RTE_SWX_NAME_SIZE];
138 struct struct_type *mailbox_struct_type;
139 rte_swx_extern_type_constructor_t constructor;
140 rte_swx_extern_type_destructor_t destructor;
141 struct extern_type_member_func_tailq funcs;
142 uint32_t n_funcs;
143};
144
145TAILQ_HEAD(extern_type_tailq, extern_type);
146
147struct extern_obj {
148 TAILQ_ENTRY(extern_obj) node;
149 char name[RTE_SWX_NAME_SIZE];
150 struct extern_type *type;
151 void *obj;
152 uint32_t struct_id;
153 uint32_t id;
154};
155
156TAILQ_HEAD(extern_obj_tailq, extern_obj);
157
158#ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
159#define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
160#endif
161
162struct extern_obj_runtime {
163 void *obj;
164 uint8_t *mailbox;
165 rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
166};
167
168/*
169 * Extern function.
170 */
171struct extern_func {
172 TAILQ_ENTRY(extern_func) node;
173 char name[RTE_SWX_NAME_SIZE];
174 struct struct_type *mailbox_struct_type;
175 rte_swx_extern_func_t func;
176 uint32_t struct_id;
177 uint32_t id;
178};
179
180TAILQ_HEAD(extern_func_tailq, extern_func);
181
182struct extern_func_runtime {
183 uint8_t *mailbox;
184 rte_swx_extern_func_t func;
185};
186
187/*
188 * Hash function.
189 */
190struct hash_func {
191 TAILQ_ENTRY(hash_func) node;
192 char name[RTE_SWX_NAME_SIZE];
193 rte_swx_hash_func_t func;
194 uint32_t id;
195};
196
197TAILQ_HEAD(hash_func_tailq, hash_func);
198
199struct hash_func_runtime {
200 rte_swx_hash_func_t func;
201};
202
203/*
204 * RSS.
205 */
206struct rss {
207 TAILQ_ENTRY(rss) node;
208 char name[RTE_SWX_NAME_SIZE];
209 uint32_t id;
210};
211
212TAILQ_HEAD(rss_tailq, rss);
213
214struct rss_runtime {
215 uint32_t key_size; /* key size in bytes. */
216 uint8_t key[]; /* key. */
217};
218
219/*
220 * Header.
221 */
222struct header {
223 TAILQ_ENTRY(header) node;
224 char name[RTE_SWX_NAME_SIZE];
225 struct struct_type *st;
226 uint32_t struct_id;
227 uint32_t id;
228};
229
230TAILQ_HEAD(header_tailq, header);
231
232struct header_runtime {
233 uint8_t *ptr0;
234 uint32_t n_bytes;
235};
236
237struct header_out_runtime {
238 uint8_t *ptr0;
239 uint8_t *ptr;
240 uint32_t n_bytes;
241};
242
243/*
244 * Instruction.
245 */
246
247/* Operand endianness conventions:
248 *
249 * Case 1: Small fields (i.e. fields with size <= 64 bits)
250 *
251 * Packet headers are always in Network Byte Order (NBO), i.e. big endian.
252 * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
253 * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
254 * when transferred to packet meta-data and in NBO when transferred to packet
255 * headers.
256 *
257 * Notation conventions:
258 * -Header field: H = h.header.field (dst/src)
259 * -Meta-data field: M = m.field (dst/src)
260 * -Extern object mailbox field: E = e.field (dst/src)
261 * -Extern function mailbox field: F = f.field (dst/src)
262 * -Table action data field: T = t.field (src only)
263 * -Immediate value: I = 32-bit unsigned value (src only)
264 *
265 * Case 2: Big fields (i.e. fields with size > 64 bits)
266 *
267 * The big fields are allowed in both headers and meta-data, but they are always
268 * stored in NBO. This is why the few instructions that accept a big field
269 * operand require that the other operand, in case it is a small operand, be
270 * stored in NBO as well, i.e. the small operand must be a header field
271 * (i.e. meta-data field not allowed in this case).
272 *
273 * Notation conventions:
274 * -Header or meta-data big field: HM-NBO.
275 */
276
277enum instruction_type {
278 /* rx m.port_in */
279 INSTR_RX,
280
281 /* tx port_out
282 * port_out = MI
283 */
284 INSTR_TX, /* port_out = M */
285 INSTR_TX_I, /* port_out = I */
286 INSTR_DROP,
287
288 /*
289 * mirror slot_id session_id
290 * slot_id = MEFT
291 * session_id = MEFT
292 */
293 INSTR_MIRROR,
294
295 /* recirculate
296 */
297 INSTR_RECIRCULATE,
298
299 /* recircid m.recirc_pass_id
300 * Read the internal recirculation pass ID into the specified meta-data field.
301 */
302 INSTR_RECIRCID,
303
304 /* extract h.header */
305 INSTR_HDR_EXTRACT,
306 INSTR_HDR_EXTRACT2,
307 INSTR_HDR_EXTRACT3,
308 INSTR_HDR_EXTRACT4,
309 INSTR_HDR_EXTRACT5,
310 INSTR_HDR_EXTRACT6,
311 INSTR_HDR_EXTRACT7,
312 INSTR_HDR_EXTRACT8,
313
314 /* extract h.header m.last_field_size */
315 INSTR_HDR_EXTRACT_M,
316
317 /* lookahead h.header */
318 INSTR_HDR_LOOKAHEAD,
319
320 /* emit h.header */
321 INSTR_HDR_EMIT,
322 INSTR_HDR_EMIT_TX,
323 INSTR_HDR_EMIT2_TX,
324 INSTR_HDR_EMIT3_TX,
325 INSTR_HDR_EMIT4_TX,
326 INSTR_HDR_EMIT5_TX,
327 INSTR_HDR_EMIT6_TX,
328 INSTR_HDR_EMIT7_TX,
329 INSTR_HDR_EMIT8_TX,
330
331 /* validate h.header */
332 INSTR_HDR_VALIDATE,
333
334 /* invalidate h.header */
335 INSTR_HDR_INVALIDATE,
336
337 /* mov dst src
338 * dst = src
339 * dst = HMEF, src = HMEFTI
340 */
341 INSTR_MOV, /* dst = MEF, src = MEFT; size(dst) <= 64 bits, size(src) <= 64 bits. */
342 INSTR_MOV_MH, /* dst = MEF, src = H; size(dst) <= 64 bits, size(src) <= 64 bits. */
343 INSTR_MOV_HM, /* dst = H, src = MEFT; size(dst) <= 64 bits, size(src) <= 64 bits. */
344 INSTR_MOV_HH, /* dst = H, src = H; size(dst) <= 64 bits, size(src) <= 64 bits. */
345 INSTR_MOV_DMA, /* dst and src in NBO format. */
346 INSTR_MOV_128, /* dst and src in NBO format, size(dst) = size(src) = 128 bits. */
347 INSTR_MOV_128_64, /* dst and src in NBO format, size(dst) = 128 bits, size(src) = 64 b. */
348 INSTR_MOV_64_128, /* dst and src in NBO format, size(dst) = 64 bits, size(src) = 128 b. */
349 INSTR_MOV_128_32, /* dst and src in NBO format, size(dst) = 128 bits, size(src) = 32 b. */
350 INSTR_MOV_32_128, /* dst and src in NBO format, size(dst) = 32 bits, size(src) = 128 b. */
351 INSTR_MOV_I, /* dst = HMEF, src = I; size(dst) <= 64 bits. */
352
353 /* movh dst src
354 * Read/write the upper half (i.e. bits 127 .. 64) of a 128-bit field into/from a 64-bit
355 * header field:
356 *
357 * dst64 = src128[127:64], where: dst64 = H, src128 = HM-NBO.
358 * dst128[127:64] = src64, where: dst128 = HM-NBO, src64 = H.
359 *
360 * Typically required for operations involving IPv6 addresses.
361 */
362 INSTR_MOVH,
363
364 /* dma h.header t.field
365 * memcpy(h.header, t.field, sizeof(h.header))
366 */
367 INSTR_DMA_HT,
368 INSTR_DMA_HT2,
369 INSTR_DMA_HT3,
370 INSTR_DMA_HT4,
371 INSTR_DMA_HT5,
372 INSTR_DMA_HT6,
373 INSTR_DMA_HT7,
374 INSTR_DMA_HT8,
375
376 /* add dst src
377 * dst += src
378 * dst = HMEF, src = HMEFTI
379 */
380 INSTR_ALU_ADD, /* dst = MEF, src = MEF */
381 INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
382 INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
383 INSTR_ALU_ADD_HH, /* dst = H, src = H */
384 INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
385 INSTR_ALU_ADD_HI, /* dst = H, src = I */
386
387 /* sub dst src
388 * dst -= src
389 * dst = HMEF, src = HMEFTI
390 */
391 INSTR_ALU_SUB, /* dst = MEF, src = MEF */
392 INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
393 INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
394 INSTR_ALU_SUB_HH, /* dst = H, src = H */
395 INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
396 INSTR_ALU_SUB_HI, /* dst = H, src = I */
397
398 /* ckadd dst src
399 * dst = dst '+ src[0:1] '+ src[2:3] '+ ...
400 * dst = H, src = {H, h.header}, '+ = 1's complement addition operator
401 */
402 INSTR_ALU_CKADD_FIELD, /* src = H */
403 INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 bytes. */
404 INSTR_ALU_CKADD_STRUCT, /* src = h.header, with sizeof(header) any 4-byte multiple. */
405
406 /* cksub dst src
407 * dst = dst '- src
408 * dst = H, src = H, '- = 1's complement subtraction operator
409 */
410 INSTR_ALU_CKSUB_FIELD,
411
412 /* and dst src
413 * dst &= src
414 * dst = HMEF, src = HMEFTI
415 */
416 INSTR_ALU_AND, /* dst = MEF, src = MEFT */
417 INSTR_ALU_AND_MH, /* dst = MEF, src = H */
418 INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
419 INSTR_ALU_AND_HH, /* dst = H, src = H */
420 INSTR_ALU_AND_I, /* dst = HMEF, src = I */
421
422 /* or dst src
423 * dst |= src
424 * dst = HMEF, src = HMEFTI
425 */
426 INSTR_ALU_OR, /* dst = MEF, src = MEFT */
427 INSTR_ALU_OR_MH, /* dst = MEF, src = H */
428 INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
429 INSTR_ALU_OR_HH, /* dst = H, src = H */
430 INSTR_ALU_OR_I, /* dst = HMEF, src = I */
431
432 /* xor dst src
433 * dst ^= src
434 * dst = HMEF, src = HMEFTI
435 */
436 INSTR_ALU_XOR, /* dst = MEF, src = MEFT */
437 INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
438 INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
439 INSTR_ALU_XOR_HH, /* dst = H, src = H */
440 INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
441
442 /* shl dst src
443 * dst <<= src
444 * dst = HMEF, src = HMEFTI
445 */
446 INSTR_ALU_SHL, /* dst = MEF, src = MEF */
447 INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
448 INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
449 INSTR_ALU_SHL_HH, /* dst = H, src = H */
450 INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
451 INSTR_ALU_SHL_HI, /* dst = H, src = I */
452
453 /* shr dst src
454 * dst >>= src
455 * dst = HMEF, src = HMEFTI
456 */
457 INSTR_ALU_SHR, /* dst = MEF, src = MEF */
458 INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
459 INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
460 INSTR_ALU_SHR_HH, /* dst = H, src = H */
461 INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
462 INSTR_ALU_SHR_HI, /* dst = H, src = I */
463
464 /* regprefetch REGARRAY index
465 * prefetch REGARRAY[index]
466 * index = HMEFTI
467 */
468 INSTR_REGPREFETCH_RH, /* index = H */
469 INSTR_REGPREFETCH_RM, /* index = MEFT */
470 INSTR_REGPREFETCH_RI, /* index = I */
471
472 /* regrd dst REGARRAY index
473 * dst = REGARRAY[index]
474 * dst = HMEF, index = HMEFTI
475 */
476 INSTR_REGRD_HRH, /* dst = H, index = H */
477 INSTR_REGRD_HRM, /* dst = H, index = MEFT */
478 INSTR_REGRD_HRI, /* dst = H, index = I */
479 INSTR_REGRD_MRH, /* dst = MEF, index = H */
480 INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
481 INSTR_REGRD_MRI, /* dst = MEF, index = I */
482
483 /* regwr REGARRAY index src
484 * REGARRAY[index] = src
485 * index = HMEFTI, src = HMEFTI
486 */
487 INSTR_REGWR_RHH, /* index = H, src = H */
488 INSTR_REGWR_RHM, /* index = H, src = MEFT */
489 INSTR_REGWR_RHI, /* index = H, src = I */
490 INSTR_REGWR_RMH, /* index = MEFT, src = H */
491 INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
492 INSTR_REGWR_RMI, /* index = MEFT, src = I */
493 INSTR_REGWR_RIH, /* index = I, src = H */
494 INSTR_REGWR_RIM, /* index = I, src = MEFT */
495 INSTR_REGWR_RII, /* index = I, src = I */
496
497 /* regadd REGARRAY index src
498 * REGARRAY[index] += src
499 * index = HMEFTI, src = HMEFTI
500 */
501 INSTR_REGADD_RHH, /* index = H, src = H */
502 INSTR_REGADD_RHM, /* index = H, src = MEFT */
503 INSTR_REGADD_RHI, /* index = H, src = I */
504 INSTR_REGADD_RMH, /* index = MEFT, src = H */
505 INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
506 INSTR_REGADD_RMI, /* index = MEFT, src = I */
507 INSTR_REGADD_RIH, /* index = I, src = H */
508 INSTR_REGADD_RIM, /* index = I, src = MEFT */
509 INSTR_REGADD_RII, /* index = I, src = I */
510
511 /* metprefetch METARRAY index
512 * prefetch METARRAY[index]
513 * index = HMEFTI
514 */
515 INSTR_METPREFETCH_H, /* index = H */
516 INSTR_METPREFETCH_M, /* index = MEFT */
517 INSTR_METPREFETCH_I, /* index = I */
518
519 /* meter METARRAY index length color_in color_out
520 * color_out = meter(METARRAY[index], length, color_in)
521 * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
522 */
523 INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
524 INSTR_METER_HHI, /* index = H, length = H, color_in = I */
525 INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
526 INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
527 INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
528 INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
529 INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
530 INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
531 INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
532 INSTR_METER_IHI, /* index = I, length = H, color_in = I */
533 INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
534 INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
535
536 /* table TABLE */
537 INSTR_TABLE,
538 INSTR_TABLE_AF,
539 INSTR_SELECTOR,
540 INSTR_LEARNER,
541 INSTR_LEARNER_AF,
542
543 /* learn ACTION_NAME [ m.action_first_arg ] m.timeout_id */
544 INSTR_LEARNER_LEARN,
545
546 /* rearm [ m.timeout_id ] */
547 INSTR_LEARNER_REARM,
548 INSTR_LEARNER_REARM_NEW,
549
550 /* forget */
551 INSTR_LEARNER_FORGET,
552
553 /* entryid m.table_entry_id
554 * Read the internal table entry ID into the specified meta-data field.
555 */
556 INSTR_ENTRYID,
557
558 /* extern e.obj.func */
559 INSTR_EXTERN_OBJ,
560
561 /* extern f.func */
562 INSTR_EXTERN_FUNC,
563
564 /* hash HASH_FUNC_NAME dst src_first src_last
565 * Compute hash value over range of struct fields.
566 * dst = M
567 * src_first = HMEFT
568 * src_last = HMEFT
569 * src_first and src_last must be fields within the same struct
570 */
571 INSTR_HASH_FUNC,
572
573 /* rss RSS_OBJ_NAME dst src_first src_last
574 * Compute the RSS hash value over range of struct fields.
575 * dst = M
576 * src_first = HMEFT
577 * src_last = HMEFT
578 * src_first and src_last must be fields within the same struct
579 */
580 INSTR_RSS,
581
582 /* jmp LABEL
583 * Unconditional jump
584 */
585 INSTR_JMP,
586
587 /* jmpv LABEL h.header
588 * Jump if header is valid
589 */
590 INSTR_JMP_VALID,
591
592 /* jmpnv LABEL h.header
593 * Jump if header is invalid
594 */
595 INSTR_JMP_INVALID,
596
597 /* jmph LABEL
598 * Jump if table lookup hit
599 */
600 INSTR_JMP_HIT,
601
602 /* jmpnh LABEL
603 * Jump if table lookup miss
604 */
605 INSTR_JMP_MISS,
606
607 /* jmpa LABEL ACTION
608 * Jump if action run
609 */
610 INSTR_JMP_ACTION_HIT,
611
612 /* jmpna LABEL ACTION
613 * Jump if action not run
614 */
615 INSTR_JMP_ACTION_MISS,
616
617 /* jmpeq LABEL a b
618 * Jump if a is equal to b
619 * a = HMEFT, b = HMEFTI
620 */
621 INSTR_JMP_EQ, /* a = MEFT, b = MEFT */
622 INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
623 INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
624 INSTR_JMP_EQ_HH, /* a = H, b = H */
625 INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
626
627 /* jmpneq LABEL a b
628 * Jump if a is not equal to b
629 * a = HMEFT, b = HMEFTI
630 */
631 INSTR_JMP_NEQ, /* a = MEFT, b = MEFT */
632 INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
633 INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
634 INSTR_JMP_NEQ_HH, /* a = H, b = H */
635 INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
636
637 /* jmplt LABEL a b
638 * Jump if a is less than b
639 * a = HMEFT, b = HMEFTI
640 */
641 INSTR_JMP_LT, /* a = MEFT, b = MEFT */
642 INSTR_JMP_LT_MH, /* a = MEFT, b = H */
643 INSTR_JMP_LT_HM, /* a = H, b = MEFT */
644 INSTR_JMP_LT_HH, /* a = H, b = H */
645 INSTR_JMP_LT_MI, /* a = MEFT, b = I */
646 INSTR_JMP_LT_HI, /* a = H, b = I */
647
648 /* jmpgt LABEL a b
649 * Jump if a is greater than b
650 * a = HMEFT, b = HMEFTI
651 */
652 INSTR_JMP_GT, /* a = MEFT, b = MEFT */
653 INSTR_JMP_GT_MH, /* a = MEFT, b = H */
654 INSTR_JMP_GT_HM, /* a = H, b = MEFT */
655 INSTR_JMP_GT_HH, /* a = H, b = H */
656 INSTR_JMP_GT_MI, /* a = MEFT, b = I */
657 INSTR_JMP_GT_HI, /* a = H, b = I */
658
659 /* return
660 * Return from action
661 */
662 INSTR_RETURN,
663
664 /* Start of custom instructions. */
665 INSTR_CUSTOM_0,
666};
667
668struct instr_operand {
669 uint8_t struct_id;
670 uint8_t n_bits;
671 uint8_t offset;
672 uint8_t pad;
673};
674
675struct instr_io {
676 struct {
677 union {
678 struct {
679 uint8_t offset;
680 uint8_t n_bits;
681 uint8_t pad[2];
682 };
683
684 uint32_t val;
685 };
686 } io;
687
688 struct {
689 uint8_t header_id[8];
690 uint8_t struct_id[8];
691 uint8_t n_bytes[8];
692 } hdr;
693};
694
695struct instr_hdr_validity {
696 uint8_t header_id;
697 uint8_t struct_id;
698};
699
700struct instr_table {
701 uint8_t table_id;
702};
703
704struct instr_learn {
705 uint8_t action_id;
706 uint8_t mf_first_arg_offset;
707 uint8_t mf_timeout_id_offset;
708 uint8_t mf_timeout_id_n_bits;
709};
710
711struct instr_extern_obj {
712 uint8_t ext_obj_id;
713 uint8_t func_id;
714};
715
716struct instr_extern_func {
717 uint8_t ext_func_id;
718};
719
720struct instr_hash_func {
721 uint8_t hash_func_id;
722
723 struct {
724 uint8_t offset;
725 uint8_t n_bits;
726 } dst;
727
728 struct {
729 uint8_t struct_id;
730 uint16_t offset;
731 uint16_t n_bytes;
732 } src;
733};
734
735struct instr_rss {
736 uint8_t rss_obj_id;
737
738 struct {
739 uint8_t offset;
740 uint8_t n_bits;
741 } dst;
742
743 struct {
744 uint8_t struct_id;
745 uint16_t offset;
746 uint16_t n_bytes;
747 } src;
748};
749
750struct instr_dst_src {
751 struct instr_operand dst;
752 union {
753 struct instr_operand src;
754 uint64_t src_val;
755 };
756};
757
758struct instr_regarray {
759 uint8_t regarray_id;
760 uint8_t pad[3];
761
762 union {
763 struct instr_operand idx;
764 uint32_t idx_val;
765 };
766
767 union {
768 struct instr_operand dstsrc;
769 uint64_t dstsrc_val;
770 };
771};
772
773struct instr_meter {
774 uint8_t metarray_id;
775 uint8_t pad[3];
776
777 union {
778 struct instr_operand idx;
779 uint32_t idx_val;
780 };
781
782 struct instr_operand length;
783
784 union {
785 struct instr_operand color_in;
786 uint32_t color_in_val;
787 };
788
789 struct instr_operand color_out;
790};
791
792struct instr_dma {
793 struct {
794 uint8_t header_id[8];
795 uint8_t struct_id[8];
796 } dst;
797
798 struct {
799 uint8_t offset[8];
800 } src;
801
802 uint16_t n_bytes[8];
803};
804
805struct instr_jmp {
806 struct instruction *ip;
807
808 union {
809 struct instr_operand a;
810 uint8_t header_id;
811 uint8_t action_id;
812 };
813
814 union {
815 struct instr_operand b;
816 uint64_t b_val;
817 };
818};
819
820struct instruction {
821 enum instruction_type type;
822 union {
823 struct instr_io io;
824 struct instr_dst_src mirror;
825 struct instr_hdr_validity valid;
826 struct instr_dst_src mov;
827 struct instr_regarray regarray;
828 struct instr_meter meter;
829 struct instr_dma dma;
830 struct instr_dst_src alu;
831 struct instr_table table;
832 struct instr_learn learn;
833 struct instr_extern_obj ext_obj;
834 struct instr_extern_func ext_func;
835 struct instr_hash_func hash_func;
836 struct instr_rss rss;
837 struct instr_jmp jmp;
838 };
839};
840
841struct instruction_data {
842 char label[RTE_SWX_NAME_SIZE];
843 char jmp_label[RTE_SWX_NAME_SIZE];
844 uint32_t n_users; /* user = jmp instruction to this instruction. */
845 int invalid;
846};
847
848typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
849
850/*
851 * Action.
852 */
853typedef void
854(*action_func_t)(struct rte_swx_pipeline *p);
855
856struct action {
857 TAILQ_ENTRY(action) node;
858 char name[RTE_SWX_NAME_SIZE];
859 struct struct_type *st;
860 int *args_endianness; /* 0 = Host Byte Order (HBO); 1 = Network Byte Order (NBO). */
861 struct instruction *instructions;
862 struct instruction_data *instruction_data;
863 uint32_t n_instructions;
864 uint32_t id;
865};
866
867TAILQ_HEAD(action_tailq, action);
868
869/*
870 * Table.
871 */
872struct table_type {
873 TAILQ_ENTRY(table_type) node;
874 char name[RTE_SWX_NAME_SIZE];
875 enum rte_swx_table_match_type match_type;
876 struct rte_swx_table_ops ops;
877};
878
879TAILQ_HEAD(table_type_tailq, table_type);
880
881struct match_field {
882 enum rte_swx_table_match_type match_type;
883 struct field *field;
884};
885
886struct table {
887 TAILQ_ENTRY(table) node;
888 char name[RTE_SWX_NAME_SIZE];
889 char args[RTE_SWX_NAME_SIZE];
890 struct table_type *type; /* NULL when n_fields == 0. */
891
892 /* Match. */
893 struct match_field *fields;
894 uint32_t n_fields;
895 struct header *header; /* Only valid when n_fields > 0. */
896
897 /* Action. */
898 struct action **actions;
899 struct action *default_action;
900 uint8_t *default_action_data;
901 uint32_t n_actions;
902 int default_action_is_const;
903 uint32_t action_data_size_max;
904 int *action_is_for_table_entries;
905 int *action_is_for_default_entry;
906
907 struct hash_func *hf;
908 uint32_t size;
909 uint32_t id;
910};
911
912TAILQ_HEAD(table_tailq, table);
913
914struct table_runtime {
915 rte_swx_table_lookup_t func;
916 void *mailbox;
917 uint8_t **key;
918};
919
920struct table_statistics {
921 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
922 uint64_t *n_pkts_action;
923};
924
925/*
926 * Selector.
927 */
928struct selector {
929 TAILQ_ENTRY(selector) node;
930 char name[RTE_SWX_NAME_SIZE];
931
932 struct field *group_id_field;
933 struct field **selector_fields;
934 uint32_t n_selector_fields;
935 struct header *selector_header;
936 struct field *member_id_field;
937
938 uint32_t n_groups_max;
939 uint32_t n_members_per_group_max;
940
941 uint32_t id;
942};
943
944TAILQ_HEAD(selector_tailq, selector);
945
946struct selector_runtime {
947 void *mailbox;
948 uint8_t **group_id_buffer;
949 uint8_t **selector_buffer;
950 uint8_t **member_id_buffer;
951};
952
953struct selector_statistics {
954 uint64_t n_pkts;
955};
956
957/*
958 * Learner table.
959 */
960struct learner {
961 TAILQ_ENTRY(learner) node;
962 char name[RTE_SWX_NAME_SIZE];
963
964 /* Match. */
965 struct field **fields;
966 uint32_t n_fields;
967 struct header *header;
968
969 /* Action. */
970 struct action **actions;
971 struct action *default_action;
972 uint8_t *default_action_data;
973 uint32_t n_actions;
974 int default_action_is_const;
975 uint32_t action_data_size_max;
976 int *action_is_for_table_entries;
977 int *action_is_for_default_entry;
978
979 struct hash_func *hf;
980 uint32_t size;
981 uint32_t timeout[RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX];
982 uint32_t n_timeouts;
983 uint32_t id;
984};
985
986TAILQ_HEAD(learner_tailq, learner);
987
988struct learner_runtime {
989 void *mailbox;
990 uint8_t **key;
991};
992
993struct learner_statistics {
994 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
995 uint64_t n_pkts_learn[2]; /* 0 = Learn OK, 1 = Learn error. */
996 uint64_t n_pkts_rearm;
997 uint64_t n_pkts_forget;
998 uint64_t *n_pkts_action;
999};
1000
1001/*
1002 * Register array.
1003 */
1004struct regarray {
1005 TAILQ_ENTRY(regarray) node;
1006 char name[RTE_SWX_NAME_SIZE];
1007 uint64_t init_val;
1008 uint32_t size;
1009 uint32_t id;
1010};
1011
1012TAILQ_HEAD(regarray_tailq, regarray);
1013
1014struct regarray_runtime {
1015 uint64_t *regarray;
1016 uint32_t size_mask;
1017};
1018
1019/*
1020 * Meter array.
1021 */
1022struct meter_profile {
1023 TAILQ_ENTRY(meter_profile) node;
1024 char name[RTE_SWX_NAME_SIZE];
1025 struct rte_meter_trtcm_params params;
1026 struct rte_meter_trtcm_profile profile;
1027 uint32_t n_users;
1028};
1029
1030TAILQ_HEAD(meter_profile_tailq, meter_profile);
1031
1032struct metarray {
1033 TAILQ_ENTRY(metarray) node;
1034 char name[RTE_SWX_NAME_SIZE];
1035 uint32_t size;
1036 uint32_t id;
1037};
1038
1039TAILQ_HEAD(metarray_tailq, metarray);
1040
1041struct meter {
1042 struct rte_meter_trtcm m;
1043 struct meter_profile *profile;
1044 enum rte_color color_mask;
1045 uint8_t pad[20];
1046
1047 uint64_t n_pkts[RTE_COLORS];
1048 uint64_t n_bytes[RTE_COLORS];
1049};
1050
1051struct metarray_runtime {
1052 struct meter *metarray;
1053 uint32_t size_mask;
1054};
1055
1056/*
1057 * Pipeline.
1058 */
1059struct thread {
1060 /* Packet. */
1061 struct rte_swx_pkt pkt;
1062 uint8_t *ptr;
1063 uint32_t *mirroring_slots;
1064 uint64_t mirroring_slots_mask;
1065 int recirculate;
1066 uint32_t recirc_pass_id;
1067
1068 /* Structures. */
1069 uint8_t **structs;
1070
1071 /* Packet headers. */
1072 struct header_runtime *headers; /* Extracted or generated headers. */
1073 struct header_out_runtime *headers_out; /* Emitted headers. */
1074 uint8_t *header_storage;
1075 uint8_t *header_out_storage;
1076 uint64_t valid_headers;
1077 uint32_t n_headers_out;
1078
1079 /* Packet meta-data. */
1080 uint8_t *metadata;
1081
1082 /* Tables. */
1083 struct table_runtime *tables;
1084 struct selector_runtime *selectors;
1085 struct learner_runtime *learners;
1086 struct rte_swx_table_state *table_state;
1087 uint64_t action_id;
1088 size_t entry_id;
1089 int hit; /* 0 = Miss, 1 = Hit. */
1090 uint32_t learner_id;
1091 uint64_t time;
1092
1093 /* Extern objects and functions. */
1094 struct extern_obj_runtime *extern_objs;
1095 struct extern_func_runtime *extern_funcs;
1096
1097 /* Instructions. */
1098 struct instruction *ip;
1099 struct instruction *ret;
1100};
1101
1102#define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
1103#define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
1104#define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
1105
1106#define HEADER_VALID(thread, header_id) \
1107 MASK64_BIT_GET((thread)->valid_headers, header_id)
1108
1109static inline uint64_t
1110instr_operand_hbo(struct thread *t, const struct instr_operand *x)
1111{
1112 uint8_t *x_struct = t->structs[x->struct_id];
1113 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
1114 uint64_t x64 = *x64_ptr;
1115 uint64_t x64_mask = UINT64_MAX >> (64 - x->n_bits);
1116
1117 return x64 & x64_mask;
1118}
1119
1120#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1121
1122static inline uint64_t
1123instr_operand_nbo(struct thread *t, const struct instr_operand *x)
1124{
1125 uint8_t *x_struct = t->structs[x->struct_id];
1126 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
1127 uint64_t x64 = *x64_ptr;
1128
1129 return ntoh64(x64) >> (64 - x->n_bits);
1130}
1131
1132#else
1133
1134#define instr_operand_nbo instr_operand_hbo
1135
1136#endif
1137
1138#define ALU(thread, ip, operator) \
1139{ \
1140 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1141 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1142 uint64_t dst64 = *dst64_ptr; \
1143 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1144 uint64_t dst = dst64 & dst64_mask; \
1145 \
1146 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1147 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1148 uint64_t src64 = *src64_ptr; \
1149 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1150 uint64_t src = src64 & src64_mask; \
1151 \
1152 uint64_t result = dst operator src; \
1153 \
1154 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1155}
1156
1157#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1158
1159#define ALU_MH(thread, ip, operator) \
1160{ \
1161 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1162 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1163 uint64_t dst64 = *dst64_ptr; \
1164 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1165 uint64_t dst = dst64 & dst64_mask; \
1166 \
1167 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1168 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1169 uint64_t src64 = *src64_ptr; \
1170 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1171 \
1172 uint64_t result = dst operator src; \
1173 \
1174 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1175}
1176
1177#define ALU_HM(thread, ip, operator) \
1178{ \
1179 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1180 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1181 uint64_t dst64 = *dst64_ptr; \
1182 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1183 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1184 \
1185 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1186 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1187 uint64_t src64 = *src64_ptr; \
1188 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1189 uint64_t src = src64 & src64_mask; \
1190 \
1191 uint64_t result = dst operator src; \
1192 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1193 \
1194 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1195}
1196
1197#define ALU_HM_FAST(thread, ip, operator) \
1198{ \
1199 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1200 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1201 uint64_t dst64 = *dst64_ptr; \
1202 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1203 uint64_t dst = dst64 & dst64_mask; \
1204 \
1205 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1206 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1207 uint64_t src64 = *src64_ptr; \
1208 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1209 uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
1210 \
1211 uint64_t result = dst operator src; \
1212 \
1213 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1214}
1215
1216#define ALU_HH(thread, ip, operator) \
1217{ \
1218 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1219 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1220 uint64_t dst64 = *dst64_ptr; \
1221 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1222 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1223 \
1224 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1225 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1226 uint64_t src64 = *src64_ptr; \
1227 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1228 \
1229 uint64_t result = dst operator src; \
1230 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1231 \
1232 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1233}
1234
1235#define ALU_HH_FAST(thread, ip, operator) \
1236{ \
1237 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1238 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1239 uint64_t dst64 = *dst64_ptr; \
1240 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1241 uint64_t dst = dst64 & dst64_mask; \
1242 \
1243 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1244 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1245 uint64_t src64 = *src64_ptr; \
1246 uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
1247 \
1248 uint64_t result = dst operator src; \
1249 \
1250 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1251}
1252
1253#else
1254
1255#define ALU_MH ALU
1256#define ALU_HM ALU
1257#define ALU_HM_FAST ALU
1258#define ALU_HH ALU
1259#define ALU_HH_FAST ALU
1260
1261#endif
1262
1263#define ALU_I(thread, ip, operator) \
1264{ \
1265 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1266 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1267 uint64_t dst64 = *dst64_ptr; \
1268 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1269 uint64_t dst = dst64 & dst64_mask; \
1270 \
1271 uint64_t src = (ip)->alu.src_val; \
1272 \
1273 uint64_t result = dst operator src; \
1274 \
1275 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1276}
1277
1278#define ALU_MI ALU_I
1279
1280#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1281
1282#define ALU_HI(thread, ip, operator) \
1283{ \
1284 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1285 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1286 uint64_t dst64 = *dst64_ptr; \
1287 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1288 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1289 \
1290 uint64_t src = (ip)->alu.src_val; \
1291 \
1292 uint64_t result = dst operator src; \
1293 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1294 \
1295 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1296}
1297
1298#else
1299
1300#define ALU_HI ALU_I
1301
1302#endif
1303
1304#define MOV(thread, ip) \
1305{ \
1306 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1307 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1308 uint64_t dst64 = *dst64_ptr; \
1309 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1310 \
1311 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1312 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1313 uint64_t src64 = *src64_ptr; \
1314 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1315 uint64_t src = src64 & src64_mask; \
1316 \
1317 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1318}
1319
1320#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1321
1322#define MOV_MH(thread, ip) \
1323{ \
1324 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1325 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1326 uint64_t dst64 = *dst64_ptr; \
1327 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1328 \
1329 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1330 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1331 uint64_t src64 = *src64_ptr; \
1332 uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits); \
1333 \
1334 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1335}
1336
1337#define MOV_HM(thread, ip) \
1338{ \
1339 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1340 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1341 uint64_t dst64 = *dst64_ptr; \
1342 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1343 \
1344 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1345 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1346 uint64_t src64 = *src64_ptr; \
1347 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1348 uint64_t src = src64 & src64_mask; \
1349 \
1350 src = hton64(src) >> (64 - (ip)->mov.dst.n_bits); \
1351 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1352}
1353
1354#define MOV_HH(thread, ip) \
1355{ \
1356 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1357 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1358 uint64_t dst64 = *dst64_ptr; \
1359 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1360 \
1361 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1362 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1363 uint64_t src64 = *src64_ptr; \
1364 \
1365 uint64_t src = src64 << (64 - (ip)->mov.src.n_bits); \
1366 src = src >> (64 - (ip)->mov.dst.n_bits); \
1367 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1368}
1369
1370#else
1371
1372#define MOV_MH MOV
1373#define MOV_HM MOV
1374#define MOV_HH MOV
1375
1376#endif
1377
1378#define MOV_I(thread, ip) \
1379{ \
1380 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1381 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1382 uint64_t dst64 = *dst64_ptr; \
1383 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1384 \
1385 uint64_t src = (ip)->mov.src_val; \
1386 \
1387 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1388}
1389
1390#define JMP_CMP(thread, ip, operator) \
1391{ \
1392 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1393 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1394 uint64_t a64 = *a64_ptr; \
1395 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1396 uint64_t a = a64 & a64_mask; \
1397 \
1398 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1399 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1400 uint64_t b64 = *b64_ptr; \
1401 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1402 uint64_t b = b64 & b64_mask; \
1403 \
1404 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1405}
1406
1407#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1408
1409#define JMP_CMP_MH(thread, ip, operator) \
1410{ \
1411 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1412 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1413 uint64_t a64 = *a64_ptr; \
1414 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1415 uint64_t a = a64 & a64_mask; \
1416 \
1417 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1418 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1419 uint64_t b64 = *b64_ptr; \
1420 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1421 \
1422 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1423}
1424
1425#define JMP_CMP_HM(thread, ip, operator) \
1426{ \
1427 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1428 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1429 uint64_t a64 = *a64_ptr; \
1430 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1431 \
1432 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1433 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1434 uint64_t b64 = *b64_ptr; \
1435 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1436 uint64_t b = b64 & b64_mask; \
1437 \
1438 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1439}
1440
1441#define JMP_CMP_HH(thread, ip, operator) \
1442{ \
1443 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1444 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1445 uint64_t a64 = *a64_ptr; \
1446 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1447 \
1448 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1449 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1450 uint64_t b64 = *b64_ptr; \
1451 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1452 \
1453 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1454}
1455
1456#define JMP_CMP_HH_FAST(thread, ip, operator) \
1457{ \
1458 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1459 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1460 uint64_t a64 = *a64_ptr; \
1461 uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits); \
1462 \
1463 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1464 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1465 uint64_t b64 = *b64_ptr; \
1466 uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits); \
1467 \
1468 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1469}
1470
1471#else
1472
1473#define JMP_CMP_MH JMP_CMP
1474#define JMP_CMP_HM JMP_CMP
1475#define JMP_CMP_HH JMP_CMP
1476#define JMP_CMP_HH_FAST JMP_CMP
1477
1478#endif
1479
1480#define JMP_CMP_I(thread, ip, operator) \
1481{ \
1482 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1483 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1484 uint64_t a64 = *a64_ptr; \
1485 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1486 uint64_t a = a64 & a64_mask; \
1487 \
1488 uint64_t b = (ip)->jmp.b_val; \
1489 \
1490 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1491}
1492
1493#define JMP_CMP_MI JMP_CMP_I
1494
1495#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1496
1497#define JMP_CMP_HI(thread, ip, operator) \
1498{ \
1499 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1500 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1501 uint64_t a64 = *a64_ptr; \
1502 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1503 \
1504 uint64_t b = (ip)->jmp.b_val; \
1505 \
1506 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1507}
1508
1509#else
1510
1511#define JMP_CMP_HI JMP_CMP_I
1512
1513#endif
1514
1515#define METADATA_READ(thread, offset, n_bits) \
1516__extension__ ({ \
1517 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1518 uint64_t m64 = *m64_ptr; \
1519 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1520 (m64 & m64_mask); \
1521})
1522
1523#define METADATA_WRITE(thread, offset, n_bits, value) \
1524{ \
1525 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1526 uint64_t m64 = *m64_ptr; \
1527 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1528 \
1529 uint64_t m_new = value; \
1530 \
1531 *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask); \
1532}
1533
1534#ifndef RTE_SWX_PIPELINE_THREADS_MAX
1535#define RTE_SWX_PIPELINE_THREADS_MAX 16
1536#endif
1537
1538#ifndef RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX
1539#define RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX 1024
1540#endif
1541
1542struct rte_swx_pipeline {
1543 char name[RTE_SWX_NAME_SIZE];
1544
1545 struct struct_type_tailq struct_types;
1546 struct port_in_type_tailq port_in_types;
1547 struct port_in_tailq ports_in;
1548 struct port_out_type_tailq port_out_types;
1549 struct port_out_tailq ports_out;
1550 struct extern_type_tailq extern_types;
1551 struct extern_obj_tailq extern_objs;
1552 struct extern_func_tailq extern_funcs;
1553 struct hash_func_tailq hash_funcs;
1554 struct rss_tailq rss;
1555 struct header_tailq headers;
1556 struct struct_type *metadata_st;
1557 uint32_t metadata_struct_id;
1558 struct action_tailq actions;
1559 struct table_type_tailq table_types;
1560 struct table_tailq tables;
1561 struct selector_tailq selectors;
1562 struct learner_tailq learners;
1563 struct regarray_tailq regarrays;
1564 struct meter_profile_tailq meter_profiles;
1565 struct metarray_tailq metarrays;
1566
1567 struct port_in_runtime *in;
1568 struct port_out_runtime *out;
1569 struct mirroring_session *mirroring_sessions;
1570 struct instruction **action_instructions;
1571 action_func_t *action_funcs;
1572 struct rte_swx_table_state *table_state;
1573 struct table_statistics *table_stats;
1574 struct selector_statistics *selector_stats;
1575 struct learner_statistics *learner_stats;
1576 struct hash_func_runtime *hash_func_runtime;
1577 struct rss_runtime **rss_runtime;
1578 struct regarray_runtime *regarray_runtime;
1579 struct metarray_runtime *metarray_runtime;
1580 struct instruction *instructions;
1581 struct instruction_data *instruction_data;
1582 instr_exec_t *instruction_table;
1583 struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1584 void *lib;
1585
1586 uint32_t n_structs;
1587 uint32_t n_ports_in;
1588 uint32_t n_ports_out;
1589 uint32_t n_mirroring_slots;
1590 uint32_t n_mirroring_sessions;
1591 uint32_t n_extern_objs;
1592 uint32_t n_extern_funcs;
1593 uint32_t n_hash_funcs;
1594 uint32_t n_rss;
1595 uint32_t n_actions;
1596 uint32_t n_tables;
1597 uint32_t n_selectors;
1598 uint32_t n_learners;
1599 uint32_t n_regarrays;
1600 uint32_t n_metarrays;
1601 uint32_t n_headers;
1602 uint32_t thread_id;
1603 uint32_t port_id;
1604 uint32_t n_instructions;
1605 int build_done;
1606 int numa_node;
1607};
1608
1609/*
1610 * Instruction.
1611 */
1612static inline void
1613pipeline_port_inc(struct rte_swx_pipeline *p)
1614{
1615 uint32_t port_id = p->port_id;
1616
1617 port_id++;
1618 if (port_id == p->n_ports_in)
1619 port_id = 0;
1620
1621 p->port_id = port_id;
1622}
1623
1624static inline void
1625thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
1626{
1627 t->ip = p->instructions;
1628}
1629
1630static inline void
1631thread_ip_set(struct thread *t, struct instruction *ip)
1632{
1633 t->ip = ip;
1634}
1635
1636static inline void
1637thread_ip_action_call(struct rte_swx_pipeline *p,
1638 struct thread *t,
1639 uint32_t action_id)
1640{
1641 t->ret = t->ip + 1;
1642 t->ip = p->action_instructions[action_id];
1643}
1644
1645static inline void
1646thread_ip_inc(struct rte_swx_pipeline *p);
1647
1648static inline void
1649thread_ip_inc(struct rte_swx_pipeline *p)
1650{
1651 struct thread *t = &p->threads[p->thread_id];
1652
1653 t->ip++;
1654}
1655
1656static inline void
1657thread_ip_inc_cond(struct thread *t, int cond)
1658{
1659 t->ip += cond;
1660}
1661
1662static inline void
1663thread_yield(struct rte_swx_pipeline *p)
1664{
1665 p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1666}
1667
1668static inline void
1669thread_yield_cond(struct rte_swx_pipeline *p, int cond)
1670{
1671 p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1672}
1673
1674/*
1675 * rx.
1676 */
1677static inline int
1678__instr_rx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1679{
1680 struct port_in_runtime *port = &p->in[p->port_id];
1681 struct rte_swx_pkt *pkt = &t->pkt;
1682 int pkt_received;
1683
1684 /* Recirculation: keep the current packet. */
1685 if (t->recirculate) {
1686 TRACE("[Thread %2u] rx - recirculate (pass %u)\n",
1687 p->thread_id,
1688 t->recirc_pass_id + 1);
1689
1690 /* Packet. */
1691 t->ptr = &pkt->pkt[pkt->offset];
1692 t->mirroring_slots_mask = 0;
1693 t->recirculate = 0;
1694 t->recirc_pass_id++;
1695
1696 /* Headers. */
1697 t->valid_headers = 0;
1698 t->n_headers_out = 0;
1699
1700 /* Tables. */
1701 t->table_state = p->table_state;
1702
1703 return 1;
1704 }
1705
1706 /* Packet. */
1707 pkt_received = port->pkt_rx(port->obj, pkt);
1708 t->ptr = &pkt->pkt[pkt->offset];
1709 rte_prefetch0(t->ptr);
1710
1711 TRACE("[Thread %2u] rx %s from port %u\n",
1712 p->thread_id,
1713 pkt_received ? "1 pkt" : "0 pkts",
1714 p->port_id);
1715
1716 t->mirroring_slots_mask = 0;
1717 t->recirc_pass_id = 0;
1718
1719 /* Headers. */
1720 t->valid_headers = 0;
1721 t->n_headers_out = 0;
1722
1723 /* Meta-data. */
1724 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
1725
1726 /* Tables. */
1727 t->table_state = p->table_state;
1728
1729 /* Thread. */
1730 pipeline_port_inc(p);
1731
1732 return pkt_received;
1733}
1734
1735static inline void
1736instr_rx_exec(struct rte_swx_pipeline *p)
1737{
1738 struct thread *t = &p->threads[p->thread_id];
1739 struct instruction *ip = t->ip;
1740 int pkt_received;
1741
1742 /* Packet. */
1743 pkt_received = __instr_rx_exec(p, t, ip);
1744
1745 /* Thread. */
1746 thread_ip_inc_cond(t, pkt_received);
1747 thread_yield(p);
1748}
1749
1750/*
1751 * tx.
1752 */
1753static inline void
1754emit_handler(struct thread *t)
1755{
1756 struct header_out_runtime *h0 = &t->headers_out[0];
1757 struct header_out_runtime *h1 = &t->headers_out[1];
1758 uint32_t offset = 0, i;
1759
1760 /* No header change or header decapsulation. */
1761 if ((t->n_headers_out == 1) &&
1762 (h0->ptr + h0->n_bytes == t->ptr)) {
1763 TRACE("Emit handler: no header change or header decap.\n");
1764
1765 t->pkt.offset -= h0->n_bytes;
1766 t->pkt.length += h0->n_bytes;
1767
1768 return;
1769 }
1770
1771 /* Header encapsulation (optionally, with prior header decapsulation). */
1772 if ((t->n_headers_out == 2) &&
1773 (h1->ptr + h1->n_bytes == t->ptr) &&
1774 (h0->ptr == h0->ptr0)) {
1775 uint32_t offset;
1776
1777 TRACE("Emit handler: header encapsulation.\n");
1778
1779 offset = h0->n_bytes + h1->n_bytes;
1780 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
1781 t->pkt.offset -= offset;
1782 t->pkt.length += offset;
1783
1784 return;
1785 }
1786
1787 /* For any other case. */
1788 TRACE("Emit handler: complex case.\n");
1789
1790 for (i = 0; i < t->n_headers_out; i++) {
1791 struct header_out_runtime *h = &t->headers_out[i];
1792
1793 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
1794 offset += h->n_bytes;
1795 }
1796
1797 if (offset) {
1798 memcpy(t->ptr - offset, t->header_out_storage, offset);
1799 t->pkt.offset -= offset;
1800 t->pkt.length += offset;
1801 }
1802}
1803
1804static inline void
1805mirroring_handler(struct rte_swx_pipeline *p, struct thread *t, struct rte_swx_pkt *pkt)
1806{
1807 uint64_t slots_mask = t->mirroring_slots_mask, slot_mask;
1808 uint32_t slot_id;
1809
1810 for (slot_id = 0, slot_mask = 1LLU ; slots_mask; slot_id++, slot_mask <<= 1)
1811 if (slot_mask & slots_mask) {
1812 struct port_out_runtime *port;
1813 struct mirroring_session *session;
1814 uint32_t port_id, session_id;
1815
1816 session_id = t->mirroring_slots[slot_id];
1817 session = &p->mirroring_sessions[session_id];
1818
1819 port_id = session->port_id;
1820 port = &p->out[port_id];
1821
1822 if (session->fast_clone)
1823 port->pkt_fast_clone_tx(port->obj, pkt);
1824 else
1825 port->pkt_clone_tx(port->obj, pkt, session->truncation_length);
1826
1827 slots_mask &= ~slot_mask;
1828 }
1829}
1830
1831static inline void
1832__instr_tx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1833{
1834 struct rte_swx_pkt *pkt = &t->pkt;
1835 struct port_out_runtime *port;
1836 uint64_t port_id;
1837
1838 /* Recirculation: keep the current packet. */
1839 if (t->recirculate) {
1840 TRACE("[Thread %2u]: tx 1 pkt - recirculate\n",
1841 p->thread_id);
1842
1843 /* Headers. */
1844 emit_handler(t);
1845
1846 /* Packet. */
1847 mirroring_handler(p, t, pkt);
1848
1849 return;
1850 }
1851
1852 /* If the output port ID is invalid, then set it to the drop output port that has been set
1853 * up internally by the pipeline for this purpose.
1854 */
1855 port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1856 if (port_id >= p->n_ports_out)
1857 port_id = p->n_ports_out - 1;
1858
1859 port = &p->out[port_id];
1860
1861 TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
1862 p->thread_id,
1863 (uint32_t)port_id);
1864
1865 /* Headers. */
1866 emit_handler(t);
1867
1868 /* Packet. */
1869 mirroring_handler(p, t, pkt);
1870 port->pkt_tx(port->obj, pkt);
1871}
1872
1873static inline void
1874__instr_tx_i_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1875{
1876 struct rte_swx_pkt *pkt = &t->pkt;
1877 struct port_out_runtime *port;
1878 uint64_t port_id;
1879
1880 /* Recirculation: keep the current packet. */
1881 if (t->recirculate) {
1882 TRACE("[Thread %2u]: tx (i) 1 pkt - recirculate\n",
1883 p->thread_id);
1884
1885 /* Headers. */
1886 emit_handler(t);
1887
1888 /* Packet. */
1889 mirroring_handler(p, t, pkt);
1890
1891 return;
1892 }
1893
1894 /* If the output port ID is invalid, then set it to the drop output port that has been set
1895 * up internally by the pipeline for this purpose.
1896 *
1897 * This test cannot be done earlier at instruction translation time, even though the output
1898 * port ID is an immediate value, as the number of output ports is only known later at the
1899 * pipeline build time.
1900 */
1901 port_id = ip->io.io.val;
1902 if (port_id >= p->n_ports_out)
1903 port_id = p->n_ports_out - 1;
1904
1905 port = &p->out[port_id];
1906
1907 TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
1908 p->thread_id,
1909 (uint32_t)port_id);
1910
1911 /* Headers. */
1912 emit_handler(t);
1913
1914 /* Packet. */
1915 mirroring_handler(p, t, pkt);
1916 port->pkt_tx(port->obj, pkt);
1917}
1918
1919static inline void
1920__instr_drop_exec(struct rte_swx_pipeline *p,
1921 struct thread *t,
1922 const struct instruction *ip __rte_unused)
1923{
1924 uint64_t port_id = p->n_ports_out - 1;
1925 struct port_out_runtime *port = &p->out[port_id];
1926 struct rte_swx_pkt *pkt = &t->pkt;
1927
1928 TRACE("[Thread %2u]: drop 1 pkt\n",
1929 p->thread_id);
1930
1931 /* Headers. */
1932 emit_handler(t);
1933
1934 /* Packet. */
1935 mirroring_handler(p, t, pkt);
1936 port->pkt_tx(port->obj, pkt);
1937}
1938
1939static inline void
1940__instr_mirror_exec(struct rte_swx_pipeline *p,
1941 struct thread *t,
1942 const struct instruction *ip)
1943{
1944 uint64_t slot_id = instr_operand_hbo(t, &ip->mirror.dst);
1945 uint64_t session_id = instr_operand_hbo(t, &ip->mirror.src);
1946
1947 slot_id &= p->n_mirroring_slots - 1;
1948 session_id &= p->n_mirroring_sessions - 1;
1949
1950 TRACE("[Thread %2u]: mirror pkt (slot = %u, session = %u)\n",
1951 p->thread_id,
1952 (uint32_t)slot_id,
1953 (uint32_t)session_id);
1954
1955 t->mirroring_slots[slot_id] = session_id;
1956 t->mirroring_slots_mask |= 1LLU << slot_id;
1957}
1958
1959static inline void
1960__instr_recirculate_exec(struct rte_swx_pipeline *p __rte_unused,
1961 struct thread *t,
1962 const struct instruction *ip __rte_unused)
1963{
1964 TRACE("[Thread %2u]: recirculate\n",
1965 p->thread_id);
1966
1967 t->recirculate = 1;
1968}
1969
1970static inline void
1971__instr_recircid_exec(struct rte_swx_pipeline *p __rte_unused,
1972 struct thread *t,
1973 const struct instruction *ip)
1974{
1975 TRACE("[Thread %2u]: recircid (pass %u)\n",
1976 p->thread_id,
1977 t->recirc_pass_id);
1978
1979 /* Meta-data. */
1980 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, t->recirc_pass_id);
1981}
1982
1983/*
1984 * extract.
1985 */
1986static inline void
1987__instr_hdr_extract_many_exec(struct rte_swx_pipeline *p __rte_unused,
1988 struct thread *t,
1989 const struct instruction *ip,
1990 uint32_t n_extract)
1991{
1992 uint64_t valid_headers = t->valid_headers;
1993 uint8_t *ptr = t->ptr;
1994 uint32_t offset = t->pkt.offset;
1995 uint32_t length = t->pkt.length;
1996 uint32_t i;
1997
1998 for (i = 0; i < n_extract; i++) {
1999 uint32_t header_id = ip->io.hdr.header_id[i];
2000 uint32_t struct_id = ip->io.hdr.struct_id[i];
2001 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2002
2003 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2004 p->thread_id,
2005 header_id,
2006 n_bytes);
2007
2008 /* Headers. */
2009 t->structs[struct_id] = ptr;
2010 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2011
2012 /* Packet. */
2013 offset += n_bytes;
2014 length -= n_bytes;
2015 ptr += n_bytes;
2016 }
2017
2018 /* Headers. */
2019 t->valid_headers = valid_headers;
2020
2021 /* Packet. */
2022 t->pkt.offset = offset;
2023 t->pkt.length = length;
2024 t->ptr = ptr;
2025}
2026
2027static inline void
2028__instr_hdr_extract_exec(struct rte_swx_pipeline *p,
2029 struct thread *t,
2030 const struct instruction *ip)
2031{
2032 __instr_hdr_extract_many_exec(p, t, ip, 1);
2033}
2034
2035static inline void
2036__instr_hdr_extract2_exec(struct rte_swx_pipeline *p,
2037 struct thread *t,
2038 const struct instruction *ip)
2039{
2040 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
2041
2042 __instr_hdr_extract_many_exec(p, t, ip, 2);
2043}
2044
2045static inline void
2046__instr_hdr_extract3_exec(struct rte_swx_pipeline *p,
2047 struct thread *t,
2048 const struct instruction *ip)
2049{
2050 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
2051
2052 __instr_hdr_extract_many_exec(p, t, ip, 3);
2053}
2054
2055static inline void
2056__instr_hdr_extract4_exec(struct rte_swx_pipeline *p,
2057 struct thread *t,
2058 const struct instruction *ip)
2059{
2060 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2061
2062 __instr_hdr_extract_many_exec(p, t, ip, 4);
2063}
2064
2065static inline void
2066__instr_hdr_extract5_exec(struct rte_swx_pipeline *p,
2067 struct thread *t,
2068 const struct instruction *ip)
2069{
2070 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2071
2072 __instr_hdr_extract_many_exec(p, t, ip, 5);
2073}
2074
2075static inline void
2076__instr_hdr_extract6_exec(struct rte_swx_pipeline *p,
2077 struct thread *t,
2078 const struct instruction *ip)
2079{
2080 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2081
2082 __instr_hdr_extract_many_exec(p, t, ip, 6);
2083}
2084
2085static inline void
2086__instr_hdr_extract7_exec(struct rte_swx_pipeline *p,
2087 struct thread *t,
2088 const struct instruction *ip)
2089{
2090 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2091
2092 __instr_hdr_extract_many_exec(p, t, ip, 7);
2093}
2094
2095static inline void
2096__instr_hdr_extract8_exec(struct rte_swx_pipeline *p,
2097 struct thread *t,
2098 const struct instruction *ip)
2099{
2100 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2101
2102 __instr_hdr_extract_many_exec(p, t, ip, 8);
2103}
2104
2105static inline void
2106__instr_hdr_extract_m_exec(struct rte_swx_pipeline *p __rte_unused,
2107 struct thread *t,
2108 const struct instruction *ip)
2109{
2110 uint64_t valid_headers = t->valid_headers;
2111 uint8_t *ptr = t->ptr;
2112 uint32_t offset = t->pkt.offset;
2113 uint32_t length = t->pkt.length;
2114
2115 uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2116 uint32_t header_id = ip->io.hdr.header_id[0];
2117 uint32_t struct_id = ip->io.hdr.struct_id[0];
2118 uint32_t n_bytes = ip->io.hdr.n_bytes[0];
2119
2120 struct header_runtime *h = &t->headers[header_id];
2121
2122 TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n",
2123 p->thread_id,
2124 header_id,
2125 n_bytes,
2126 n_bytes_last);
2127
2128 n_bytes += n_bytes_last;
2129
2130 /* Headers. */
2131 t->structs[struct_id] = ptr;
2132 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2133 h->n_bytes = n_bytes;
2134
2135 /* Packet. */
2136 t->pkt.offset = offset + n_bytes;
2137 t->pkt.length = length - n_bytes;
2138 t->ptr = ptr + n_bytes;
2139}
2140
2141static inline void
2142__instr_hdr_lookahead_exec(struct rte_swx_pipeline *p __rte_unused,
2143 struct thread *t,
2144 const struct instruction *ip)
2145{
2146 uint64_t valid_headers = t->valid_headers;
2147 uint8_t *ptr = t->ptr;
2148
2149 uint32_t header_id = ip->io.hdr.header_id[0];
2150 uint32_t struct_id = ip->io.hdr.struct_id[0];
2151
2152 TRACE("[Thread %2u]: lookahead header %u\n",
2153 p->thread_id,
2154 header_id);
2155
2156 /* Headers. */
2157 t->structs[struct_id] = ptr;
2158 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2159}
2160
2161/*
2162 * emit.
2163 */
2164static inline void
2165__instr_hdr_emit_many_exec(struct rte_swx_pipeline *p __rte_unused,
2166 struct thread *t,
2167 const struct instruction *ip,
2168 uint32_t n_emit)
2169{
2170 uint64_t valid_headers = t->valid_headers;
2171 uint32_t n_headers_out = t->n_headers_out;
2172 struct header_out_runtime *ho = NULL;
2173 uint8_t *ho_ptr = NULL;
2174 uint32_t ho_nbytes = 0, i;
2175
2176 for (i = 0; i < n_emit; i++) {
2177 uint32_t header_id = ip->io.hdr.header_id[i];
2178 uint32_t struct_id = ip->io.hdr.struct_id[i];
2179
2180 struct header_runtime *hi = &t->headers[header_id];
2181 uint8_t *hi_ptr0 = hi->ptr0;
2182 uint32_t n_bytes = hi->n_bytes;
2183
2184 uint8_t *hi_ptr = t->structs[struct_id];
2185
2186 if (!MASK64_BIT_GET(valid_headers, header_id)) {
2187 TRACE("[Thread %2u]: emit header %u (invalid)\n",
2188 p->thread_id,
2189 header_id);
2190
2191 continue;
2192 }
2193
2194 TRACE("[Thread %2u]: emit header %u (valid)\n",
2195 p->thread_id,
2196 header_id);
2197
2198 /* Headers. */
2199 if (!ho) {
2200 if (!n_headers_out) {
2201 ho = &t->headers_out[0];
2202
2203 ho->ptr0 = hi_ptr0;
2204 ho->ptr = hi_ptr;
2205
2206 ho_ptr = hi_ptr;
2207 ho_nbytes = n_bytes;
2208
2209 n_headers_out = 1;
2210
2211 continue;
2212 } else {
2213 ho = &t->headers_out[n_headers_out - 1];
2214
2215 ho_ptr = ho->ptr;
2216 ho_nbytes = ho->n_bytes;
2217 }
2218 }
2219
2220 if (ho_ptr + ho_nbytes == hi_ptr) {
2221 ho_nbytes += n_bytes;
2222 } else {
2223 ho->n_bytes = ho_nbytes;
2224
2225 ho++;
2226 ho->ptr0 = hi_ptr0;
2227 ho->ptr = hi_ptr;
2228
2229 ho_ptr = hi_ptr;
2230 ho_nbytes = n_bytes;
2231
2232 n_headers_out++;
2233 }
2234 }
2235
2236 if (ho)
2237 ho->n_bytes = ho_nbytes;
2238 t->n_headers_out = n_headers_out;
2239}
2240
2241static inline void
2242__instr_hdr_emit_exec(struct rte_swx_pipeline *p,
2243 struct thread *t,
2244 const struct instruction *ip)
2245{
2246 __instr_hdr_emit_many_exec(p, t, ip, 1);
2247}
2248
2249static inline void
2250__instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p,
2251 struct thread *t,
2252 const struct instruction *ip)
2253{
2254 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
2255
2256 __instr_hdr_emit_many_exec(p, t, ip, 1);
2257 __instr_tx_exec(p, t, ip);
2258}
2259
2260static inline void
2261__instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p,
2262 struct thread *t,
2263 const struct instruction *ip)
2264{
2265 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
2266
2267 __instr_hdr_emit_many_exec(p, t, ip, 2);
2268 __instr_tx_exec(p, t, ip);
2269}
2270
2271static inline void
2272__instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p,
2273 struct thread *t,
2274 const struct instruction *ip)
2275{
2276 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2277
2278 __instr_hdr_emit_many_exec(p, t, ip, 3);
2279 __instr_tx_exec(p, t, ip);
2280}
2281
2282static inline void
2283__instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p,
2284 struct thread *t,
2285 const struct instruction *ip)
2286{
2287 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2288
2289 __instr_hdr_emit_many_exec(p, t, ip, 4);
2290 __instr_tx_exec(p, t, ip);
2291}
2292
2293static inline void
2294__instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p,
2295 struct thread *t,
2296 const struct instruction *ip)
2297{
2298 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2299
2300 __instr_hdr_emit_many_exec(p, t, ip, 5);
2301 __instr_tx_exec(p, t, ip);
2302}
2303
2304static inline void
2305__instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p,
2306 struct thread *t,
2307 const struct instruction *ip)
2308{
2309 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2310
2311 __instr_hdr_emit_many_exec(p, t, ip, 6);
2312 __instr_tx_exec(p, t, ip);
2313}
2314
2315static inline void
2316__instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p,
2317 struct thread *t,
2318 const struct instruction *ip)
2319{
2320 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2321
2322 __instr_hdr_emit_many_exec(p, t, ip, 7);
2323 __instr_tx_exec(p, t, ip);
2324}
2325
2326static inline void
2327__instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p,
2328 struct thread *t,
2329 const struct instruction *ip)
2330{
2331 TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n", p->thread_id);
2332
2333 __instr_hdr_emit_many_exec(p, t, ip, 8);
2334 __instr_tx_exec(p, t, ip);
2335}
2336
2337/*
2338 * validate.
2339 */
2340static inline void
2341__instr_hdr_validate_exec(struct rte_swx_pipeline *p __rte_unused,
2342 struct thread *t,
2343 const struct instruction *ip)
2344{
2345 uint32_t header_id = ip->valid.header_id;
2346 uint32_t struct_id = ip->valid.struct_id;
2347 uint64_t valid_headers = t->valid_headers;
2348 struct header_runtime *h = &t->headers[header_id];
2349
2350 TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2351
2352 /* If this header is already valid, then its associated t->structs[] element is also valid
2353 * and therefore it should not be modified. It could point to the packet buffer (in case of
2354 * extracted header) and setting it to the default location (h->ptr0) would be incorrect.
2355 */
2356 if (MASK64_BIT_GET(valid_headers, header_id))
2357 return;
2358
2359 /* Headers. */
2360 t->structs[struct_id] = h->ptr0;
2361 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2362}
2363
2364/*
2365 * invalidate.
2366 */
2367static inline void
2368__instr_hdr_invalidate_exec(struct rte_swx_pipeline *p __rte_unused,
2369 struct thread *t,
2370 const struct instruction *ip)
2371{
2372 uint32_t header_id = ip->valid.header_id;
2373
2374 TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2375
2376 /* Headers. */
2377 t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2378}
2379
2380/*
2381 * learn.
2382 */
2383static inline void
2384__instr_learn_exec(struct rte_swx_pipeline *p,
2385 struct thread *t,
2386 const struct instruction *ip)
2387{
2388 uint64_t action_id = ip->learn.action_id;
2389 uint32_t mf_first_arg_offset = ip->learn.mf_first_arg_offset;
2390 uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset,
2391 ip->learn.mf_timeout_id_n_bits);
2392 uint32_t learner_id = t->learner_id;
2393 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2394 p->n_selectors + learner_id];
2395 struct learner_runtime *l = &t->learners[learner_id];
2396 struct learner_statistics *stats = &p->learner_stats[learner_id];
2397 uint32_t status;
2398
2399 /* Table. */
2400 status = rte_swx_table_learner_add(ts->obj,
2401 l->mailbox,
2402 t->time,
2403 action_id,
2404 &t->metadata[mf_first_arg_offset],
2405 timeout_id);
2406
2407 TRACE("[Thread %2u] learner %u learn %s\n",
2408 p->thread_id,
2409 learner_id,
2410 status ? "ok" : "error");
2411
2412 stats->n_pkts_learn[status] += 1;
2413}
2414
2415/*
2416 * rearm.
2417 */
2418static inline void
2419__instr_rearm_exec(struct rte_swx_pipeline *p,
2420 struct thread *t,
2421 const struct instruction *ip __rte_unused)
2422{
2423 uint32_t learner_id = t->learner_id;
2424 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2425 p->n_selectors + learner_id];
2426 struct learner_runtime *l = &t->learners[learner_id];
2427 struct learner_statistics *stats = &p->learner_stats[learner_id];
2428
2429 /* Table. */
2430 rte_swx_table_learner_rearm(ts->obj, l->mailbox, t->time);
2431
2432 TRACE("[Thread %2u] learner %u rearm\n",
2433 p->thread_id,
2434 learner_id);
2435
2436 stats->n_pkts_rearm += 1;
2437}
2438
2439static inline void
2440__instr_rearm_new_exec(struct rte_swx_pipeline *p,
2441 struct thread *t,
2442 const struct instruction *ip)
2443{
2444 uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset,
2445 ip->learn.mf_timeout_id_n_bits);
2446 uint32_t learner_id = t->learner_id;
2447 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2448 p->n_selectors + learner_id];
2449 struct learner_runtime *l = &t->learners[learner_id];
2450 struct learner_statistics *stats = &p->learner_stats[learner_id];
2451
2452 /* Table. */
2453 rte_swx_table_learner_rearm_new(ts->obj, l->mailbox, t->time, timeout_id);
2454
2455 TRACE("[Thread %2u] learner %u rearm with timeout ID %u\n",
2456 p->thread_id,
2457 learner_id,
2458 timeout_id);
2459
2460 stats->n_pkts_rearm += 1;
2461}
2462
2463/*
2464 * forget.
2465 */
2466static inline void
2467__instr_forget_exec(struct rte_swx_pipeline *p,
2468 struct thread *t,
2469 const struct instruction *ip __rte_unused)
2470{
2471 uint32_t learner_id = t->learner_id;
2472 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2473 p->n_selectors + learner_id];
2474 struct learner_runtime *l = &t->learners[learner_id];
2475 struct learner_statistics *stats = &p->learner_stats[learner_id];
2476
2477 /* Table. */
2478 rte_swx_table_learner_delete(ts->obj, l->mailbox);
2479
2480 TRACE("[Thread %2u] learner %u forget\n",
2481 p->thread_id,
2482 learner_id);
2483
2484 stats->n_pkts_forget += 1;
2485}
2486
2487/*
2488 * entryid.
2489 */
2490static inline void
2491__instr_entryid_exec(struct rte_swx_pipeline *p __rte_unused,
2492 struct thread *t,
2493 const struct instruction *ip)
2494{
2495 TRACE("[Thread %2u]: entryid\n",
2496 p->thread_id);
2497
2498 /* Meta-data. */
2499 METADATA_WRITE(t, ip->mov.dst.offset, ip->mov.dst.n_bits, t->entry_id);
2500}
2501
2502/*
2503 * extern.
2504 */
2505static inline uint32_t
2506__instr_extern_obj_exec(struct rte_swx_pipeline *p __rte_unused,
2507 struct thread *t,
2508 const struct instruction *ip)
2509{
2510 uint32_t obj_id = ip->ext_obj.ext_obj_id;
2511 uint32_t func_id = ip->ext_obj.func_id;
2512 struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
2513 rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
2514 uint32_t done;
2515
2516 TRACE("[Thread %2u] extern obj %u member func %u\n",
2517 p->thread_id,
2518 obj_id,
2519 func_id);
2520
2521 done = func(obj->obj, obj->mailbox);
2522
2523 return done;
2524}
2525
2526static inline uint32_t
2527__instr_extern_func_exec(struct rte_swx_pipeline *p __rte_unused,
2528 struct thread *t,
2529 const struct instruction *ip)
2530{
2531 uint32_t ext_func_id = ip->ext_func.ext_func_id;
2532 struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
2533 rte_swx_extern_func_t func = ext_func->func;
2534 uint32_t done;
2535
2536 TRACE("[Thread %2u] extern func %u\n",
2537 p->thread_id,
2538 ext_func_id);
2539
2540 done = func(ext_func->mailbox);
2541
2542 return done;
2543}
2544
2545/*
2546 * hash.
2547 */
2548static inline void
2549__instr_hash_func_exec(struct rte_swx_pipeline *p,
2550 struct thread *t,
2551 const struct instruction *ip)
2552{
2553 uint32_t hash_func_id = ip->hash_func.hash_func_id;
2554 uint32_t dst_offset = ip->hash_func.dst.offset;
2555 uint32_t n_dst_bits = ip->hash_func.dst.n_bits;
2556 uint32_t src_struct_id = ip->hash_func.src.struct_id;
2557 uint32_t src_offset = ip->hash_func.src.offset;
2558 uint32_t n_src_bytes = ip->hash_func.src.n_bytes;
2559
2560 struct hash_func_runtime *func = &p->hash_func_runtime[hash_func_id];
2561 uint8_t *src_ptr = t->structs[src_struct_id];
2562 uint32_t result;
2563
2564 TRACE("[Thread %2u] hash %u\n",
2565 p->thread_id,
2566 hash_func_id);
2567
2568 result = func->func(&src_ptr[src_offset], n_src_bytes, 0);
2569 METADATA_WRITE(t, dst_offset, n_dst_bits, result);
2570}
2571
2572/*
2573 * rss.
2574 */
2575static inline uint32_t
2576rss_func(void *rss_key, uint32_t rss_key_size, void *input_data, uint32_t input_data_size)
2577{
2578 uint32_t *key = (uint32_t *)rss_key;
2579 uint32_t *data = (uint32_t *)input_data;
2580 uint32_t key_size = rss_key_size >> 2;
2581 uint32_t data_size = input_data_size >> 2;
2582 uint32_t hash_val = 0, i;
2583
2584 for (i = 0; i < data_size; i++) {
2585 uint32_t d;
2586
2587 for (d = data[i]; d; d &= (d - 1)) {
2588 uint32_t key0, key1, pos;
2589
2590 pos = rte_bsf32(d);
2591 key0 = key[i % key_size] << (31 - pos);
2592 key1 = key[(i + 1) % key_size] >> (pos + 1);
2593 hash_val ^= key0 | key1;
2594 }
2595 }
2596
2597 return hash_val;
2598}
2599
2600static inline void
2601__instr_rss_exec(struct rte_swx_pipeline *p,
2602 struct thread *t,
2603 const struct instruction *ip)
2604{
2605 uint32_t rss_obj_id = ip->rss.rss_obj_id;
2606 uint32_t dst_offset = ip->rss.dst.offset;
2607 uint32_t n_dst_bits = ip->rss.dst.n_bits;
2608 uint32_t src_struct_id = ip->rss.src.struct_id;
2609 uint32_t src_offset = ip->rss.src.offset;
2610 uint32_t n_src_bytes = ip->rss.src.n_bytes;
2611
2612 struct rss_runtime *r = p->rss_runtime[rss_obj_id];
2613 uint8_t *src_ptr = t->structs[src_struct_id];
2614 uint32_t result;
2615
2616 TRACE("[Thread %2u] rss %u\n",
2617 p->thread_id,
2618 rss_obj_id);
2619
2620 result = rss_func(r->key, r->key_size, &src_ptr[src_offset], n_src_bytes);
2621 METADATA_WRITE(t, dst_offset, n_dst_bits, result);
2622}
2623
2624/*
2625 * mov.
2626 */
2627static inline void
2628__instr_mov_exec(struct rte_swx_pipeline *p __rte_unused,
2629 struct thread *t,
2630 const struct instruction *ip)
2631{
2632 TRACE("[Thread %2u] mov\n", p->thread_id);
2633
2634 MOV(t, ip);
2635}
2636
2637static inline void
2638__instr_mov_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2639 struct thread *t,
2640 const struct instruction *ip)
2641{
2642 TRACE("[Thread %2u] mov (mh)\n", p->thread_id);
2643
2644 MOV_MH(t, ip);
2645}
2646
2647static inline void
2648__instr_mov_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2649 struct thread *t,
2650 const struct instruction *ip)
2651{
2652 TRACE("[Thread %2u] mov (hm)\n", p->thread_id);
2653
2654 MOV_HM(t, ip);
2655}
2656
2657static inline void
2658__instr_mov_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2659 struct thread *t,
2660 const struct instruction *ip)
2661{
2662 TRACE("[Thread %2u] mov (hh)\n", p->thread_id);
2663
2664 MOV_HH(t, ip);
2665}
2666
2667static inline void
2668__instr_mov_dma_exec(struct rte_swx_pipeline *p __rte_unused,
2669 struct thread *t,
2670 const struct instruction *ip)
2671{
2672 uint8_t *dst = t->structs[ip->mov.dst.struct_id] + ip->mov.dst.offset;
2673 uint8_t *src = t->structs[ip->mov.src.struct_id] + ip->mov.src.offset;
2674
2675 uint32_t n_dst = ip->mov.dst.n_bits >> 3;
2676 uint32_t n_src = ip->mov.src.n_bits >> 3;
2677
2678 TRACE("[Thread %2u] mov (dma) %u bytes\n", p->thread_id, n);
2679
2680 /* Both dst and src are in NBO format. */
2681 if (n_dst > n_src) {
2682 uint32_t n_dst_zero = n_dst - n_src;
2683
2684 /* Zero padding the most significant bytes in dst. */
2685 memset(dst, 0, n_dst_zero);
2686 dst += n_dst_zero;
2687
2688 /* Copy src to dst. */
2689 memcpy(dst, src, n_src);
2690 } else {
2691 uint32_t n_src_skipped = n_src - n_dst;
2692
2693 /* Copy src to dst. */
2694 src += n_src_skipped;
2695 memcpy(dst, src, n_dst);
2696 }
2697}
2698
2699static inline void
2700__instr_mov_128_exec(struct rte_swx_pipeline *p __rte_unused,
2701 struct thread *t,
2702 const struct instruction *ip)
2703{
2704 uint8_t *dst_struct = t->structs[ip->mov.dst.struct_id];
2705 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->mov.dst.offset];
2706
2707 uint8_t *src_struct = t->structs[ip->mov.src.struct_id];
2708 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->mov.src.offset];
2709
2710 TRACE("[Thread %2u] mov (128)\n", p->thread_id);
2711
2712 dst64_ptr[0] = src64_ptr[0];
2713 dst64_ptr[1] = src64_ptr[1];
2714}
2715
2716static inline void
2717__instr_mov_128_64_exec(struct rte_swx_pipeline *p __rte_unused,
2718 struct thread *t,
2719 const struct instruction *ip)
2720{
2721 uint8_t *dst = t->structs[ip->mov.dst.struct_id] + ip->mov.dst.offset;
2722 uint8_t *src = t->structs[ip->mov.src.struct_id] + ip->mov.src.offset;
2723
2724 uint64_t *dst64 = (uint64_t *)dst;
2725 uint64_t *src64 = (uint64_t *)src;
2726
2727 TRACE("[Thread %2u] mov (128 <- 64)\n", p->thread_id);
2728
2729 dst64[0] = 0;
2730 dst64[1] = src64[0];
2731}
2732
2733static inline void
2734__instr_mov_64_128_exec(struct rte_swx_pipeline *p __rte_unused,
2735 struct thread *t,
2736 const struct instruction *ip)
2737{
2738 uint8_t *dst = t->structs[ip->mov.dst.struct_id] + ip->mov.dst.offset;
2739 uint8_t *src = t->structs[ip->mov.src.struct_id] + ip->mov.src.offset;
2740
2741 uint64_t *dst64 = (uint64_t *)dst;
2742 uint64_t *src64 = (uint64_t *)src;
2743
2744 TRACE("[Thread %2u] mov (64 <- 128)\n", p->thread_id);
2745
2746 dst64[0] = src64[1];
2747}
2748
2749static inline void
2750__instr_mov_128_32_exec(struct rte_swx_pipeline *p __rte_unused,
2751 struct thread *t,
2752 const struct instruction *ip)
2753{
2754 uint8_t *dst = t->structs[ip->mov.dst.struct_id] + ip->mov.dst.offset;
2755 uint8_t *src = t->structs[ip->mov.src.struct_id] + ip->mov.src.offset;
2756
2757 uint32_t *dst32 = (uint32_t *)dst;
2758 uint32_t *src32 = (uint32_t *)src;
2759
2760 TRACE("[Thread %2u] mov (128 <- 32)\n", p->thread_id);
2761
2762 dst32[0] = 0;
2763 dst32[1] = 0;
2764 dst32[2] = 0;
2765 dst32[3] = src32[0];
2766}
2767
2768static inline void
2769__instr_mov_32_128_exec(struct rte_swx_pipeline *p __rte_unused,
2770 struct thread *t,
2771 const struct instruction *ip)
2772{
2773 uint8_t *dst = t->structs[ip->mov.dst.struct_id] + ip->mov.dst.offset;
2774 uint8_t *src = t->structs[ip->mov.src.struct_id] + ip->mov.src.offset;
2775
2776 uint32_t *dst32 = (uint32_t *)dst;
2777 uint32_t *src32 = (uint32_t *)src;
2778
2779 TRACE("[Thread %2u] mov (32 <- 128)\n", p->thread_id);
2780
2781 dst32[0] = src32[3];
2782}
2783
2784static inline void
2785__instr_mov_i_exec(struct rte_swx_pipeline *p __rte_unused,
2786 struct thread *t,
2787 const struct instruction *ip)
2788{
2789 TRACE("[Thread %2u] mov m.f %" PRIx64 "\n", p->thread_id, ip->mov.src_val);
2790
2791 MOV_I(t, ip);
2792}
2793
2794/*
2795 * movh.
2796 */
2797static inline void
2798__instr_movh_exec(struct rte_swx_pipeline *p __rte_unused,
2799 struct thread *t,
2800 const struct instruction *ip)
2801{
2802 uint8_t *dst = t->structs[ip->mov.dst.struct_id] + ip->mov.dst.offset;
2803 uint8_t *src = t->structs[ip->mov.src.struct_id] + ip->mov.src.offset;
2804
2805 uint64_t *dst64 = (uint64_t *)dst;
2806 uint64_t *src64 = (uint64_t *)src;
2807
2808 TRACE("[Thread %2u] movh\n", p->thread_id);
2809
2810 dst64[0] = src64[0];
2811}
2812
2813/*
2814 * dma.
2815 */
2816static inline void
2817__instr_dma_ht_many_exec(struct rte_swx_pipeline *p __rte_unused,
2818 struct thread *t,
2819 const struct instruction *ip,
2820 uint32_t n_dma)
2821{
2822 uint8_t *action_data = t->structs[0];
2823 uint64_t valid_headers = t->valid_headers;
2824 uint32_t i;
2825
2826 for (i = 0; i < n_dma; i++) {
2827 uint32_t header_id = ip->dma.dst.header_id[i];
2828 uint32_t struct_id = ip->dma.dst.struct_id[i];
2829 uint32_t offset = ip->dma.src.offset[i];
2830 uint32_t n_bytes = ip->dma.n_bytes[i];
2831
2832 struct header_runtime *h = &t->headers[header_id];
2833 uint8_t *h_ptr0 = h->ptr0;
2834 uint8_t *h_ptr = t->structs[struct_id];
2835
2836 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2837 h_ptr : h_ptr0;
2838 void *src = &action_data[offset];
2839
2840 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2841
2842 /* Headers. */
2843 memcpy(dst, src, n_bytes);
2844 t->structs[struct_id] = dst;
2845 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2846 }
2847
2848 t->valid_headers = valid_headers;
2849}
2850
2851static inline void
2852__instr_dma_ht_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2853{
2854 __instr_dma_ht_many_exec(p, t, ip, 1);
2855}
2856
2857static inline void
2858__instr_dma_ht2_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2859{
2860 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
2861
2862 __instr_dma_ht_many_exec(p, t, ip, 2);
2863}
2864
2865static inline void
2866__instr_dma_ht3_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2867{
2868 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
2869
2870 __instr_dma_ht_many_exec(p, t, ip, 3);
2871}
2872
2873static inline void
2874__instr_dma_ht4_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2875{
2876 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2877
2878 __instr_dma_ht_many_exec(p, t, ip, 4);
2879}
2880
2881static inline void
2882__instr_dma_ht5_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2883{
2884 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2885
2886 __instr_dma_ht_many_exec(p, t, ip, 5);
2887}
2888
2889static inline void
2890__instr_dma_ht6_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2891{
2892 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2893
2894 __instr_dma_ht_many_exec(p, t, ip, 6);
2895}
2896
2897static inline void
2898__instr_dma_ht7_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2899{
2900 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2901
2902 __instr_dma_ht_many_exec(p, t, ip, 7);
2903}
2904
2905static inline void
2906__instr_dma_ht8_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2907{
2908 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2909
2910 __instr_dma_ht_many_exec(p, t, ip, 8);
2911}
2912
2913/*
2914 * alu.
2915 */
2916static inline void
2917__instr_alu_add_exec(struct rte_swx_pipeline *p __rte_unused,
2918 struct thread *t,
2919 const struct instruction *ip)
2920{
2921 TRACE("[Thread %2u] add\n", p->thread_id);
2922
2923 ALU(t, ip, +);
2924}
2925
2926static inline void
2927__instr_alu_add_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2928 struct thread *t,
2929 const struct instruction *ip)
2930{
2931 TRACE("[Thread %2u] add (mh)\n", p->thread_id);
2932
2933 ALU_MH(t, ip, +);
2934}
2935
2936static inline void
2937__instr_alu_add_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2938 struct thread *t,
2939 const struct instruction *ip)
2940{
2941 TRACE("[Thread %2u] add (hm)\n", p->thread_id);
2942
2943 ALU_HM(t, ip, +);
2944}
2945
2946static inline void
2947__instr_alu_add_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2948 struct thread *t,
2949 const struct instruction *ip)
2950{
2951 TRACE("[Thread %2u] add (hh)\n", p->thread_id);
2952
2953 ALU_HH(t, ip, +);
2954}
2955
2956static inline void
2957__instr_alu_add_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2958 struct thread *t,
2959 const struct instruction *ip)
2960{
2961 TRACE("[Thread %2u] add (mi)\n", p->thread_id);
2962
2963 ALU_MI(t, ip, +);
2964}
2965
2966static inline void
2967__instr_alu_add_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2968 struct thread *t,
2969 const struct instruction *ip)
2970{
2971 TRACE("[Thread %2u] add (hi)\n", p->thread_id);
2972
2973 ALU_HI(t, ip, +);
2974}
2975
2976static inline void
2977__instr_alu_sub_exec(struct rte_swx_pipeline *p __rte_unused,
2978 struct thread *t,
2979 const struct instruction *ip)
2980{
2981 TRACE("[Thread %2u] sub\n", p->thread_id);
2982
2983 ALU(t, ip, -);
2984}
2985
2986static inline void
2987__instr_alu_sub_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2988 struct thread *t,
2989 const struct instruction *ip)
2990{
2991 TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
2992
2993 ALU_MH(t, ip, -);
2994}
2995
2996static inline void
2997__instr_alu_sub_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2998 struct thread *t,
2999 const struct instruction *ip)
3000{
3001 TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
3002
3003 ALU_HM(t, ip, -);
3004}
3005
3006static inline void
3007__instr_alu_sub_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3008 struct thread *t,
3009 const struct instruction *ip)
3010{
3011 TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
3012
3013 ALU_HH(t, ip, -);
3014}
3015
3016static inline void
3017__instr_alu_sub_mi_exec(struct rte_swx_pipeline *p __rte_unused,
3018 struct thread *t,
3019 const struct instruction *ip)
3020{
3021 TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
3022
3023 ALU_MI(t, ip, -);
3024}
3025
3026static inline void
3027__instr_alu_sub_hi_exec(struct rte_swx_pipeline *p __rte_unused,
3028 struct thread *t,
3029 const struct instruction *ip)
3030{
3031 TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
3032
3033 ALU_HI(t, ip, -);
3034}
3035
3036static inline void
3037__instr_alu_shl_exec(struct rte_swx_pipeline *p __rte_unused,
3038 struct thread *t,
3039 const struct instruction *ip)
3040{
3041 TRACE("[Thread %2u] shl\n", p->thread_id);
3042
3043 ALU(t, ip, <<);
3044}
3045
3046static inline void
3047__instr_alu_shl_mh_exec(struct rte_swx_pipeline *p __rte_unused,
3048 struct thread *t,
3049 const struct instruction *ip)
3050{
3051 TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
3052
3053 ALU_MH(t, ip, <<);
3054}
3055
3056static inline void
3057__instr_alu_shl_hm_exec(struct rte_swx_pipeline *p __rte_unused,
3058 struct thread *t,
3059 const struct instruction *ip)
3060{
3061 TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
3062
3063 ALU_HM(t, ip, <<);
3064}
3065
3066static inline void
3067__instr_alu_shl_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3068 struct thread *t,
3069 const struct instruction *ip)
3070{
3071 TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
3072
3073 ALU_HH(t, ip, <<);
3074}
3075
3076static inline void
3077__instr_alu_shl_mi_exec(struct rte_swx_pipeline *p __rte_unused,
3078 struct thread *t,
3079 const struct instruction *ip)
3080{
3081 TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
3082
3083 ALU_MI(t, ip, <<);
3084}
3085
3086static inline void
3087__instr_alu_shl_hi_exec(struct rte_swx_pipeline *p __rte_unused,
3088 struct thread *t,
3089 const struct instruction *ip)
3090{
3091 TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
3092
3093 ALU_HI(t, ip, <<);
3094}
3095
3096static inline void
3097__instr_alu_shr_exec(struct rte_swx_pipeline *p __rte_unused,
3098 struct thread *t,
3099 const struct instruction *ip)
3100{
3101 TRACE("[Thread %2u] shr\n", p->thread_id);
3102
3103 ALU(t, ip, >>);
3104}
3105
3106static inline void
3107__instr_alu_shr_mh_exec(struct rte_swx_pipeline *p __rte_unused,
3108 struct thread *t,
3109 const struct instruction *ip)
3110{
3111 TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
3112
3113 ALU_MH(t, ip, >>);
3114}
3115
3116static inline void
3117__instr_alu_shr_hm_exec(struct rte_swx_pipeline *p __rte_unused,
3118 struct thread *t,
3119 const struct instruction *ip)
3120{
3121 TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
3122
3123 ALU_HM(t, ip, >>);
3124}
3125
3126static inline void
3127__instr_alu_shr_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3128 struct thread *t,
3129 const struct instruction *ip)
3130{
3131 TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
3132
3133 ALU_HH(t, ip, >>);
3134}
3135
3136static inline void
3137__instr_alu_shr_mi_exec(struct rte_swx_pipeline *p __rte_unused,
3138 struct thread *t,
3139 const struct instruction *ip)
3140{
3141 TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
3142
3143 /* Structs. */
3144 ALU_MI(t, ip, >>);
3145}
3146
3147static inline void
3148__instr_alu_shr_hi_exec(struct rte_swx_pipeline *p __rte_unused,
3149 struct thread *t,
3150 const struct instruction *ip)
3151{
3152 TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
3153
3154 ALU_HI(t, ip, >>);
3155}
3156
3157static inline void
3158__instr_alu_and_exec(struct rte_swx_pipeline *p __rte_unused,
3159 struct thread *t,
3160 const struct instruction *ip)
3161{
3162 TRACE("[Thread %2u] and\n", p->thread_id);
3163
3164 ALU(t, ip, &);
3165}
3166
3167static inline void
3168__instr_alu_and_mh_exec(struct rte_swx_pipeline *p __rte_unused,
3169 struct thread *t,
3170 const struct instruction *ip)
3171{
3172 TRACE("[Thread %2u] and (mh)\n", p->thread_id);
3173
3174 ALU_MH(t, ip, &);
3175}
3176
3177static inline void
3178__instr_alu_and_hm_exec(struct rte_swx_pipeline *p __rte_unused,
3179 struct thread *t,
3180 const struct instruction *ip)
3181{
3182 TRACE("[Thread %2u] and (hm)\n", p->thread_id);
3183
3184 ALU_HM_FAST(t, ip, &);
3185}
3186
3187static inline void
3188__instr_alu_and_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3189 struct thread *t,
3190 const struct instruction *ip)
3191{
3192 TRACE("[Thread %2u] and (hh)\n", p->thread_id);
3193
3194 ALU_HH_FAST(t, ip, &);
3195}
3196
3197static inline void
3198__instr_alu_and_i_exec(struct rte_swx_pipeline *p __rte_unused,
3199 struct thread *t,
3200 const struct instruction *ip)
3201{
3202 TRACE("[Thread %2u] and (i)\n", p->thread_id);
3203
3204 ALU_I(t, ip, &);
3205}
3206
3207static inline void
3208__instr_alu_or_exec(struct rte_swx_pipeline *p __rte_unused,
3209 struct thread *t,
3210 const struct instruction *ip)
3211{
3212 TRACE("[Thread %2u] or\n", p->thread_id);
3213
3214 ALU(t, ip, |);
3215}
3216
3217static inline void
3218__instr_alu_or_mh_exec(struct rte_swx_pipeline *p __rte_unused,
3219 struct thread *t,
3220 const struct instruction *ip)
3221{
3222 TRACE("[Thread %2u] or (mh)\n", p->thread_id);
3223
3224 ALU_MH(t, ip, |);
3225}
3226
3227static inline void
3228__instr_alu_or_hm_exec(struct rte_swx_pipeline *p __rte_unused,
3229 struct thread *t,
3230 const struct instruction *ip)
3231{
3232 TRACE("[Thread %2u] or (hm)\n", p->thread_id);
3233
3234 ALU_HM_FAST(t, ip, |);
3235}
3236
3237static inline void
3238__instr_alu_or_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3239 struct thread *t,
3240 const struct instruction *ip)
3241{
3242 TRACE("[Thread %2u] or (hh)\n", p->thread_id);
3243
3244 ALU_HH_FAST(t, ip, |);
3245}
3246
3247static inline void
3248__instr_alu_or_i_exec(struct rte_swx_pipeline *p __rte_unused,
3249 struct thread *t,
3250 const struct instruction *ip)
3251{
3252 TRACE("[Thread %2u] or (i)\n", p->thread_id);
3253
3254 ALU_I(t, ip, |);
3255}
3256
3257static inline void
3258__instr_alu_xor_exec(struct rte_swx_pipeline *p __rte_unused,
3259 struct thread *t,
3260 const struct instruction *ip)
3261{
3262 TRACE("[Thread %2u] xor\n", p->thread_id);
3263
3264 ALU(t, ip, ^);
3265}
3266
3267static inline void
3268__instr_alu_xor_mh_exec(struct rte_swx_pipeline *p __rte_unused,
3269 struct thread *t,
3270 const struct instruction *ip)
3271{
3272 TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
3273
3274 ALU_MH(t, ip, ^);
3275}
3276
3277static inline void
3278__instr_alu_xor_hm_exec(struct rte_swx_pipeline *p __rte_unused,
3279 struct thread *t,
3280 const struct instruction *ip)
3281{
3282 TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
3283
3284 ALU_HM_FAST(t, ip, ^);
3285}
3286
3287static inline void
3288__instr_alu_xor_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3289 struct thread *t,
3290 const struct instruction *ip)
3291{
3292 TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
3293
3294 ALU_HH_FAST(t, ip, ^);
3295}
3296
3297static inline void
3298__instr_alu_xor_i_exec(struct rte_swx_pipeline *p __rte_unused,
3299 struct thread *t,
3300 const struct instruction *ip)
3301{
3302 TRACE("[Thread %2u] xor (i)\n", p->thread_id);
3303
3304 ALU_I(t, ip, ^);
3305}
3306
3307static inline void
3308__instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p __rte_unused,
3309 struct thread *t,
3310 const struct instruction *ip)
3311{
3312 uint8_t *dst_struct, *src_struct;
3313 uint16_t *dst16_ptr, dst;
3314 uint64_t *src64_ptr, src64, src64_mask, src;
3315 uint64_t r;
3316
3317 TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3318
3319 /* Structs. */
3320 dst_struct = t->structs[ip->alu.dst.struct_id];
3321 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3322 dst = *dst16_ptr;
3323
3324 src_struct = t->structs[ip->alu.src.struct_id];
3325 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3326 src64 = *src64_ptr;
3327 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3328 src = src64 & src64_mask;
3329
3330 /* Initialize the result with destination 1's complement. */
3331 r = dst;
3332 r = ~r & 0xFFFF;
3333
3334 /* The first input (r) is a 16-bit number. The second and the third
3335 * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3336 * three numbers (output r) is a 34-bit number.
3337 */
3338 r += (src >> 32) + (src & 0xFFFFFFFF);
3339
3340 /* The first input is a 16-bit number. The second input is an 18-bit
3341 * number. In the worst case scenario, the sum of the two numbers is a
3342 * 19-bit number.
3343 */
3344 r = (r & 0xFFFF) + (r >> 16);
3345
3346 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3347 * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3348 */
3349 r = (r & 0xFFFF) + (r >> 16);
3350
3351 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3352 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3353 * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3354 * therefore the output r is always a 16-bit number.
3355 */
3356 r = (r & 0xFFFF) + (r >> 16);
3357
3358 /* Apply 1's complement to the result. */
3359 r = ~r & 0xFFFF;
3360 r = r ? r : 0xFFFF;
3361
3362 *dst16_ptr = (uint16_t)r;
3363}
3364
3365static inline void
3366__instr_alu_cksub_field_exec(struct rte_swx_pipeline *p __rte_unused,
3367 struct thread *t,
3368 const struct instruction *ip)
3369{
3370 uint8_t *dst_struct, *src_struct;
3371 uint16_t *dst16_ptr, dst;
3372 uint64_t *src64_ptr, src64, src64_mask, src;
3373 uint64_t r;
3374
3375 TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
3376
3377 /* Structs. */
3378 dst_struct = t->structs[ip->alu.dst.struct_id];
3379 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3380 dst = *dst16_ptr;
3381
3382 src_struct = t->structs[ip->alu.src.struct_id];
3383 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3384 src64 = *src64_ptr;
3385 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3386 src = src64 & src64_mask;
3387
3388 /* Initialize the result with destination 1's complement. */
3389 r = dst;
3390 r = ~r & 0xFFFF;
3391
3392 /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
3393 * the following sequence of operations in 2's complement arithmetic:
3394 * a '- b = (a - b) % 0xFFFF.
3395 *
3396 * In order to prevent an underflow for the below subtraction, in which
3397 * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
3398 * minuend), we first add a multiple of the 0xFFFF modulus to the
3399 * minuend. The number we add to the minuend needs to be a 34-bit number
3400 * or higher, so for readability reasons we picked the 36-bit multiple.
3401 * We are effectively turning the 16-bit minuend into a 36-bit number:
3402 * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
3403 */
3404 r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
3405
3406 /* A 33-bit number is subtracted from a 36-bit number (the input r). The
3407 * result (the output r) is a 36-bit number.
3408 */
3409 r -= (src >> 32) + (src & 0xFFFFFFFF);
3410
3411 /* The first input is a 16-bit number. The second input is a 20-bit
3412 * number. Their sum is a 21-bit number.
3413 */
3414 r = (r & 0xFFFF) + (r >> 16);
3415
3416 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3417 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
3418 */
3419 r = (r & 0xFFFF) + (r >> 16);
3420
3421 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3422 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3423 * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3424 * generated, therefore the output r is always a 16-bit number.
3425 */
3426 r = (r & 0xFFFF) + (r >> 16);
3427
3428 /* Apply 1's complement to the result. */
3429 r = ~r & 0xFFFF;
3430 r = r ? r : 0xFFFF;
3431
3432 *dst16_ptr = (uint16_t)r;
3433}
3434
3435static inline void
3436__instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p __rte_unused,
3437 struct thread *t,
3438 const struct instruction *ip)
3439{
3440 uint8_t *dst_struct, *src_struct;
3441 uint16_t *dst16_ptr, dst;
3442 uint32_t *src32_ptr;
3443 uint64_t r0, r1;
3444
3445 TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
3446
3447 /* Structs. */
3448 dst_struct = t->structs[ip->alu.dst.struct_id];
3449 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3450 dst = *dst16_ptr;
3451
3452 src_struct = t->structs[ip->alu.src.struct_id];
3453 src32_ptr = (uint32_t *)&src_struct[0];
3454
3455 /* Initialize the result with destination 1's complement. */
3456 r0 = dst;
3457 r0 = ~r0 & 0xFFFF;
3458
3459 r0 += src32_ptr[0]; /* The output r0 is a 33-bit number. */
3460 r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
3461 r0 += src32_ptr[2]; /* The output r0 is a 34-bit number. */
3462 r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
3463 r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
3464
3465 /* The first input is a 16-bit number. The second input is a 19-bit
3466 * number. Their sum is a 20-bit number.
3467 */
3468 r0 = (r0 & 0xFFFF) + (r0 >> 16);
3469
3470 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3471 * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
3472 */
3473 r0 = (r0 & 0xFFFF) + (r0 >> 16);
3474
3475 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3476 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3477 * 0x1000E), the output r is (0 .. 15). So no carry bit can be
3478 * generated, therefore the output r is always a 16-bit number.
3479 */
3480 r0 = (r0 & 0xFFFF) + (r0 >> 16);
3481
3482 /* Apply 1's complement to the result. */
3483 r0 = ~r0 & 0xFFFF;
3484 r0 = r0 ? r0 : 0xFFFF;
3485
3486 *dst16_ptr = (uint16_t)r0;
3487}
3488
3489static inline void
3490__instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p __rte_unused,
3491 struct thread *t,
3492 const struct instruction *ip)
3493{
3494 uint32_t src_header_id = ip->alu.src.n_bits; /* The src header ID is stored here. */
3495 uint32_t n_src_header_bytes = t->headers[src_header_id].n_bytes;
3496 uint8_t *dst_struct, *src_struct;
3497 uint16_t *dst16_ptr, dst;
3498 uint32_t *src32_ptr;
3499 uint64_t r;
3500 uint32_t i;
3501
3502 if (n_src_header_bytes == 20) {
3503 __instr_alu_ckadd_struct20_exec(p, t, ip);
3504 return;
3505 }
3506
3507 TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
3508
3509 /* Structs. */
3510 dst_struct = t->structs[ip->alu.dst.struct_id];
3511 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3512 dst = *dst16_ptr;
3513
3514 src_struct = t->structs[ip->alu.src.struct_id];
3515 src32_ptr = (uint32_t *)&src_struct[0];
3516
3517 /* Initialize the result with destination 1's complement. */
3518 r = dst;
3519 r = ~r & 0xFFFF;
3520
3521 /* The max number of 32-bit words in a 32K-byte header is 2^13.
3522 * Therefore, in the worst case scenario, a 45-bit number is added to a
3523 * 16-bit number (the input r), so the output r is 46-bit number.
3524 */
3525 for (i = 0; i < n_src_header_bytes / 4; i++, src32_ptr++)
3526 r += *src32_ptr;
3527
3528 /* The first input is a 16-bit number. The second input is a 30-bit
3529 * number. Their sum is a 31-bit number.
3530 */
3531 r = (r & 0xFFFF) + (r >> 16);
3532
3533 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3534 * a 15-bit number (0 .. 0x7FFF). The sum is a 17-bit number (0 .. 0x17FFE).
3535 */
3536 r = (r & 0xFFFF) + (r >> 16);
3537
3538 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3539 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3540 * 0x17FFE), the output r is (0 .. 0x7FFF). So no carry bit can be
3541 * generated, therefore the output r is always a 16-bit number.
3542 */
3543 r = (r & 0xFFFF) + (r >> 16);
3544
3545 /* Apply 1's complement to the result. */
3546 r = ~r & 0xFFFF;
3547 r = r ? r : 0xFFFF;
3548
3549 *dst16_ptr = (uint16_t)r;
3550}
3551
3552/*
3553 * Register array.
3554 */
3555static inline uint64_t *
3556instr_regarray_regarray(struct rte_swx_pipeline *p, const struct instruction *ip)
3557{
3558 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3559 return r->regarray;
3560}
3561
3562static inline uint64_t
3563instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3564{
3565 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3566
3567 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
3568 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
3569 uint64_t idx64 = *idx64_ptr;
3570 uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
3571 uint64_t idx = idx64 & idx64_mask & r->size_mask;
3572
3573 return idx;
3574}
3575
3576#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3577
3578static inline uint64_t
3579instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3580{
3581 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3582
3583 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
3584 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
3585 uint64_t idx64 = *idx64_ptr;
3586 uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
3587
3588 return idx;
3589}
3590
3591#else
3592
3593#define instr_regarray_idx_nbo instr_regarray_idx_hbo
3594
3595#endif
3596
3597static inline uint64_t
3598instr_regarray_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
3599{
3600 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3601
3602 uint64_t idx = ip->regarray.idx_val & r->size_mask;
3603
3604 return idx;
3605}
3606
3607static inline uint64_t
3608instr_regarray_src_hbo(struct thread *t, const struct instruction *ip)
3609{
3610 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
3611 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
3612 uint64_t src64 = *src64_ptr;
3613 uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3614 uint64_t src = src64 & src64_mask;
3615
3616 return src;
3617}
3618
3619#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3620
3621static inline uint64_t
3622instr_regarray_src_nbo(struct thread *t, const struct instruction *ip)
3623{
3624 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
3625 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
3626 uint64_t src64 = *src64_ptr;
3627 uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
3628
3629 return src;
3630}
3631
3632#else
3633
3634#define instr_regarray_src_nbo instr_regarray_src_hbo
3635
3636#endif
3637
3638static inline void
3639instr_regarray_dst_hbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
3640{
3641 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
3642 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
3643 uint64_t dst64 = *dst64_ptr;
3644 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3645
3646 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3647
3648}
3649
3650#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3651
3652static inline void
3653instr_regarray_dst_nbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
3654{
3655 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
3656 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
3657 uint64_t dst64 = *dst64_ptr;
3658 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3659
3660 src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
3661 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3662}
3663
3664#else
3665
3666#define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
3667
3668#endif
3669
3670static inline void
3671__instr_regprefetch_rh_exec(struct rte_swx_pipeline *p,
3672 struct thread *t,
3673 const struct instruction *ip)
3674{
3675 uint64_t *regarray, idx;
3676
3677 TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
3678
3679 regarray = instr_regarray_regarray(p, ip);
3680 idx = instr_regarray_idx_nbo(p, t, ip);
3681 rte_prefetch0(&regarray[idx]);
3682}
3683
3684static inline void
3685__instr_regprefetch_rm_exec(struct rte_swx_pipeline *p,
3686 struct thread *t,
3687 const struct instruction *ip)
3688{
3689 uint64_t *regarray, idx;
3690
3691 TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
3692
3693 regarray = instr_regarray_regarray(p, ip);
3694 idx = instr_regarray_idx_hbo(p, t, ip);
3695 rte_prefetch0(&regarray[idx]);
3696}
3697
3698static inline void
3699__instr_regprefetch_ri_exec(struct rte_swx_pipeline *p,
3700 struct thread *t __rte_unused,
3701 const struct instruction *ip)
3702{
3703 uint64_t *regarray, idx;
3704
3705 TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
3706
3707 regarray = instr_regarray_regarray(p, ip);
3708 idx = instr_regarray_idx_imm(p, ip);
3709 rte_prefetch0(&regarray[idx]);
3710}
3711
3712static inline void
3713__instr_regrd_hrh_exec(struct rte_swx_pipeline *p,
3714 struct thread *t,
3715 const struct instruction *ip)
3716{
3717 uint64_t *regarray, idx;
3718
3719 TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
3720
3721 regarray = instr_regarray_regarray(p, ip);
3722 idx = instr_regarray_idx_nbo(p, t, ip);
3723 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3724}
3725
3726static inline void
3727__instr_regrd_hrm_exec(struct rte_swx_pipeline *p,
3728 struct thread *t,
3729 const struct instruction *ip)
3730{
3731 uint64_t *regarray, idx;
3732
3733 TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
3734
3735 /* Structs. */
3736 regarray = instr_regarray_regarray(p, ip);
3737 idx = instr_regarray_idx_hbo(p, t, ip);
3738 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3739}
3740
3741static inline void
3742__instr_regrd_mrh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3743{
3744 uint64_t *regarray, idx;
3745
3746 TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
3747
3748 regarray = instr_regarray_regarray(p, ip);
3749 idx = instr_regarray_idx_nbo(p, t, ip);
3750 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3751}
3752
3753static inline void
3754__instr_regrd_mrm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3755{
3756 uint64_t *regarray, idx;
3757
3758 TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id);
3759
3760 regarray = instr_regarray_regarray(p, ip);
3761 idx = instr_regarray_idx_hbo(p, t, ip);
3762 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3763}
3764
3765static inline void
3766__instr_regrd_hri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3767{
3768 uint64_t *regarray, idx;
3769
3770 TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
3771
3772 regarray = instr_regarray_regarray(p, ip);
3773 idx = instr_regarray_idx_imm(p, ip);
3774 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3775}
3776
3777static inline void
3778__instr_regrd_mri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3779{
3780 uint64_t *regarray, idx;
3781
3782 TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
3783
3784 regarray = instr_regarray_regarray(p, ip);
3785 idx = instr_regarray_idx_imm(p, ip);
3786 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3787}
3788
3789static inline void
3790__instr_regwr_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3791{
3792 uint64_t *regarray, idx, src;
3793
3794 TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
3795
3796 regarray = instr_regarray_regarray(p, ip);
3797 idx = instr_regarray_idx_nbo(p, t, ip);
3798 src = instr_regarray_src_nbo(t, ip);
3799 regarray[idx] = src;
3800}
3801
3802static inline void
3803__instr_regwr_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3804{
3805 uint64_t *regarray, idx, src;
3806
3807 TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
3808
3809 regarray = instr_regarray_regarray(p, ip);
3810 idx = instr_regarray_idx_nbo(p, t, ip);
3811 src = instr_regarray_src_hbo(t, ip);
3812 regarray[idx] = src;
3813}
3814
3815static inline void
3816__instr_regwr_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3817{
3818 uint64_t *regarray, idx, src;
3819
3820 TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
3821
3822 regarray = instr_regarray_regarray(p, ip);
3823 idx = instr_regarray_idx_hbo(p, t, ip);
3824 src = instr_regarray_src_nbo(t, ip);
3825 regarray[idx] = src;
3826}
3827
3828static inline void
3829__instr_regwr_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3830{
3831 uint64_t *regarray, idx, src;
3832
3833 TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
3834
3835 regarray = instr_regarray_regarray(p, ip);
3836 idx = instr_regarray_idx_hbo(p, t, ip);
3837 src = instr_regarray_src_hbo(t, ip);
3838 regarray[idx] = src;
3839}
3840
3841static inline void
3842__instr_regwr_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3843{
3844 uint64_t *regarray, idx, src;
3845
3846 TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
3847
3848 regarray = instr_regarray_regarray(p, ip);
3849 idx = instr_regarray_idx_nbo(p, t, ip);
3850 src = ip->regarray.dstsrc_val;
3851 regarray[idx] = src;
3852}
3853
3854static inline void
3855__instr_regwr_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3856{
3857 uint64_t *regarray, idx, src;
3858
3859 TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
3860
3861 regarray = instr_regarray_regarray(p, ip);
3862 idx = instr_regarray_idx_hbo(p, t, ip);
3863 src = ip->regarray.dstsrc_val;
3864 regarray[idx] = src;
3865}
3866
3867static inline void
3868__instr_regwr_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3869{
3870 uint64_t *regarray, idx, src;
3871
3872 TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
3873
3874 regarray = instr_regarray_regarray(p, ip);
3875 idx = instr_regarray_idx_imm(p, ip);
3876 src = instr_regarray_src_nbo(t, ip);
3877 regarray[idx] = src;
3878}
3879
3880static inline void
3881__instr_regwr_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3882{
3883 uint64_t *regarray, idx, src;
3884
3885 TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
3886
3887 regarray = instr_regarray_regarray(p, ip);
3888 idx = instr_regarray_idx_imm(p, ip);
3889 src = instr_regarray_src_hbo(t, ip);
3890 regarray[idx] = src;
3891}
3892
3893static inline void
3894__instr_regwr_rii_exec(struct rte_swx_pipeline *p,
3895 struct thread *t __rte_unused,
3896 const struct instruction *ip)
3897{
3898 uint64_t *regarray, idx, src;
3899
3900 TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
3901
3902 regarray = instr_regarray_regarray(p, ip);
3903 idx = instr_regarray_idx_imm(p, ip);
3904 src = ip->regarray.dstsrc_val;
3905 regarray[idx] = src;
3906}
3907
3908static inline void
3909__instr_regadd_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3910{
3911 uint64_t *regarray, idx, src;
3912
3913 TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
3914
3915 regarray = instr_regarray_regarray(p, ip);
3916 idx = instr_regarray_idx_nbo(p, t, ip);
3917 src = instr_regarray_src_nbo(t, ip);
3918 regarray[idx] += src;
3919}
3920
3921static inline void
3922__instr_regadd_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3923{
3924 uint64_t *regarray, idx, src;
3925
3926 TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
3927
3928 regarray = instr_regarray_regarray(p, ip);
3929 idx = instr_regarray_idx_nbo(p, t, ip);
3930 src = instr_regarray_src_hbo(t, ip);
3931 regarray[idx] += src;
3932}
3933
3934static inline void
3935__instr_regadd_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3936{
3937 uint64_t *regarray, idx, src;
3938
3939 TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
3940
3941 regarray = instr_regarray_regarray(p, ip);
3942 idx = instr_regarray_idx_hbo(p, t, ip);
3943 src = instr_regarray_src_nbo(t, ip);
3944 regarray[idx] += src;
3945}
3946
3947static inline void
3948__instr_regadd_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3949{
3950 uint64_t *regarray, idx, src;
3951
3952 TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
3953
3954 regarray = instr_regarray_regarray(p, ip);
3955 idx = instr_regarray_idx_hbo(p, t, ip);
3956 src = instr_regarray_src_hbo(t, ip);
3957 regarray[idx] += src;
3958}
3959
3960static inline void
3961__instr_regadd_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3962{
3963 uint64_t *regarray, idx, src;
3964
3965 TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
3966
3967 regarray = instr_regarray_regarray(p, ip);
3968 idx = instr_regarray_idx_nbo(p, t, ip);
3969 src = ip->regarray.dstsrc_val;
3970 regarray[idx] += src;
3971}
3972
3973static inline void
3974__instr_regadd_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3975{
3976 uint64_t *regarray, idx, src;
3977
3978 TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
3979
3980 regarray = instr_regarray_regarray(p, ip);
3981 idx = instr_regarray_idx_hbo(p, t, ip);
3982 src = ip->regarray.dstsrc_val;
3983 regarray[idx] += src;
3984}
3985
3986static inline void
3987__instr_regadd_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3988{
3989 uint64_t *regarray, idx, src;
3990
3991 TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
3992
3993 regarray = instr_regarray_regarray(p, ip);
3994 idx = instr_regarray_idx_imm(p, ip);
3995 src = instr_regarray_src_nbo(t, ip);
3996 regarray[idx] += src;
3997}
3998
3999static inline void
4000__instr_regadd_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4001{
4002 uint64_t *regarray, idx, src;
4003
4004 TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
4005
4006 regarray = instr_regarray_regarray(p, ip);
4007 idx = instr_regarray_idx_imm(p, ip);
4008 src = instr_regarray_src_hbo(t, ip);
4009 regarray[idx] += src;
4010}
4011
4012static inline void
4013__instr_regadd_rii_exec(struct rte_swx_pipeline *p,
4014 struct thread *t __rte_unused,
4015 const struct instruction *ip)
4016{
4017 uint64_t *regarray, idx, src;
4018
4019 TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
4020
4021 regarray = instr_regarray_regarray(p, ip);
4022 idx = instr_regarray_idx_imm(p, ip);
4023 src = ip->regarray.dstsrc_val;
4024 regarray[idx] += src;
4025}
4026
4027/*
4028 * metarray.
4029 */
4030static inline struct meter *
4031instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4032{
4033 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
4034
4035 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
4036 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
4037 uint64_t idx64 = *idx64_ptr;
4038 uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
4039 uint64_t idx = idx64 & idx64_mask & r->size_mask;
4040
4041 return &r->metarray[idx];
4042}
4043
4044#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4045
4046static inline struct meter *
4047instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4048{
4049 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
4050
4051 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
4052 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
4053 uint64_t idx64 = *idx64_ptr;
4054 uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
4055
4056 return &r->metarray[idx];
4057}
4058
4059#else
4060
4061#define instr_meter_idx_nbo instr_meter_idx_hbo
4062
4063#endif
4064
4065static inline struct meter *
4066instr_meter_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
4067{
4068 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
4069
4070 uint64_t idx = ip->meter.idx_val & r->size_mask;
4071
4072 return &r->metarray[idx];
4073}
4074
4075static inline uint32_t
4076instr_meter_length_hbo(struct thread *t, const struct instruction *ip)
4077{
4078 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
4079 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
4080 uint64_t src64 = *src64_ptr;
4081 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
4082 uint64_t src = src64 & src64_mask;
4083
4084 return (uint32_t)src;
4085}
4086
4087#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4088
4089static inline uint32_t
4090instr_meter_length_nbo(struct thread *t, const struct instruction *ip)
4091{
4092 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
4093 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
4094 uint64_t src64 = *src64_ptr;
4095 uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
4096
4097 return (uint32_t)src;
4098}
4099
4100#else
4101
4102#define instr_meter_length_nbo instr_meter_length_hbo
4103
4104#endif
4105
4106static inline enum rte_color
4107instr_meter_color_in_hbo(struct thread *t, const struct instruction *ip)
4108{
4109 uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
4110 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
4111 uint64_t src64 = *src64_ptr;
4112 uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
4113 uint64_t src = src64 & src64_mask;
4114
4115 return (enum rte_color)src;
4116}
4117
4118static inline void
4119instr_meter_color_out_hbo_set(struct thread *t,
4120 const struct instruction *ip,
4121 enum rte_color color_out)
4122{
4123 uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
4124 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
4125 uint64_t dst64 = *dst64_ptr;
4126 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
4127
4128 uint64_t src = (uint64_t)color_out;
4129
4130 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4131}
4132
4133static inline void
4134__instr_metprefetch_h_exec(struct rte_swx_pipeline *p,
4135 struct thread *t,
4136 const struct instruction *ip)
4137{
4138 struct meter *m;
4139
4140 TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
4141
4142 m = instr_meter_idx_nbo(p, t, ip);
4143 rte_prefetch0(m);
4144}
4145
4146static inline void
4147__instr_metprefetch_m_exec(struct rte_swx_pipeline *p,
4148 struct thread *t,
4149 const struct instruction *ip)
4150{
4151 struct meter *m;
4152
4153 TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
4154
4155 m = instr_meter_idx_hbo(p, t, ip);
4156 rte_prefetch0(m);
4157}
4158
4159static inline void
4160__instr_metprefetch_i_exec(struct rte_swx_pipeline *p,
4161 struct thread *t __rte_unused,
4162 const struct instruction *ip)
4163{
4164 struct meter *m;
4165
4166 TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
4167
4168 m = instr_meter_idx_imm(p, ip);
4169 rte_prefetch0(m);
4170}
4171
4172static inline void
4173__instr_meter_hhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4174{
4175 struct meter *m;
4176 uint64_t time, n_pkts, n_bytes;
4177 uint32_t length;
4178 enum rte_color color_in, color_out;
4179
4180 TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
4181
4182 m = instr_meter_idx_nbo(p, t, ip);
4183 rte_prefetch0(m->n_pkts);
4184 time = rte_get_tsc_cycles();
4185 length = instr_meter_length_nbo(t, ip);
4186 color_in = instr_meter_color_in_hbo(t, ip);
4187
4188 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4189 &m->profile->profile,
4190 time,
4191 length,
4192 color_in);
4193
4194 color_out &= m->color_mask;
4195
4196 n_pkts = m->n_pkts[color_out];
4197 n_bytes = m->n_bytes[color_out];
4198
4199 instr_meter_color_out_hbo_set(t, ip, color_out);
4200
4201 m->n_pkts[color_out] = n_pkts + 1;
4202 m->n_bytes[color_out] = n_bytes + length;
4203}
4204
4205static inline void
4206__instr_meter_hhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4207{
4208 struct meter *m;
4209 uint64_t time, n_pkts, n_bytes;
4210 uint32_t length;
4211 enum rte_color color_in, color_out;
4212
4213 TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
4214
4215 m = instr_meter_idx_nbo(p, t, ip);
4216 rte_prefetch0(m->n_pkts);
4217 time = rte_get_tsc_cycles();
4218 length = instr_meter_length_nbo(t, ip);
4219 color_in = (enum rte_color)ip->meter.color_in_val;
4220
4221 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4222 &m->profile->profile,
4223 time,
4224 length,
4225 color_in);
4226
4227 color_out &= m->color_mask;
4228
4229 n_pkts = m->n_pkts[color_out];
4230 n_bytes = m->n_bytes[color_out];
4231
4232 instr_meter_color_out_hbo_set(t, ip, color_out);
4233
4234 m->n_pkts[color_out] = n_pkts + 1;
4235 m->n_bytes[color_out] = n_bytes + length;
4236}
4237
4238static inline void
4239__instr_meter_hmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4240{
4241 struct meter *m;
4242 uint64_t time, n_pkts, n_bytes;
4243 uint32_t length;
4244 enum rte_color color_in, color_out;
4245
4246 TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
4247
4248 m = instr_meter_idx_nbo(p, t, ip);
4249 rte_prefetch0(m->n_pkts);
4250 time = rte_get_tsc_cycles();
4251 length = instr_meter_length_hbo(t, ip);
4252 color_in = instr_meter_color_in_hbo(t, ip);
4253
4254 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4255 &m->profile->profile,
4256 time,
4257 length,
4258 color_in);
4259
4260 color_out &= m->color_mask;
4261
4262 n_pkts = m->n_pkts[color_out];
4263 n_bytes = m->n_bytes[color_out];
4264
4265 instr_meter_color_out_hbo_set(t, ip, color_out);
4266
4267 m->n_pkts[color_out] = n_pkts + 1;
4268 m->n_bytes[color_out] = n_bytes + length;
4269}
4270
4271static inline void
4272__instr_meter_hmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4273{
4274 struct meter *m;
4275 uint64_t time, n_pkts, n_bytes;
4276 uint32_t length;
4277 enum rte_color color_in, color_out;
4278
4279 TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
4280
4281 m = instr_meter_idx_nbo(p, t, ip);
4282 rte_prefetch0(m->n_pkts);
4283 time = rte_get_tsc_cycles();
4284 length = instr_meter_length_hbo(t, ip);
4285 color_in = (enum rte_color)ip->meter.color_in_val;
4286
4287 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4288 &m->profile->profile,
4289 time,
4290 length,
4291 color_in);
4292
4293 color_out &= m->color_mask;
4294
4295 n_pkts = m->n_pkts[color_out];
4296 n_bytes = m->n_bytes[color_out];
4297
4298 instr_meter_color_out_hbo_set(t, ip, color_out);
4299
4300 m->n_pkts[color_out] = n_pkts + 1;
4301 m->n_bytes[color_out] = n_bytes + length;
4302}
4303
4304static inline void
4305__instr_meter_mhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4306{
4307 struct meter *m;
4308 uint64_t time, n_pkts, n_bytes;
4309 uint32_t length;
4310 enum rte_color color_in, color_out;
4311
4312 TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
4313
4314 m = instr_meter_idx_hbo(p, t, ip);
4315 rte_prefetch0(m->n_pkts);
4316 time = rte_get_tsc_cycles();
4317 length = instr_meter_length_nbo(t, ip);
4318 color_in = instr_meter_color_in_hbo(t, ip);
4319
4320 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4321 &m->profile->profile,
4322 time,
4323 length,
4324 color_in);
4325
4326 color_out &= m->color_mask;
4327
4328 n_pkts = m->n_pkts[color_out];
4329 n_bytes = m->n_bytes[color_out];
4330
4331 instr_meter_color_out_hbo_set(t, ip, color_out);
4332
4333 m->n_pkts[color_out] = n_pkts + 1;
4334 m->n_bytes[color_out] = n_bytes + length;
4335}
4336
4337static inline void
4338__instr_meter_mhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4339{
4340 struct meter *m;
4341 uint64_t time, n_pkts, n_bytes;
4342 uint32_t length;
4343 enum rte_color color_in, color_out;
4344
4345 TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
4346
4347 m = instr_meter_idx_hbo(p, t, ip);
4348 rte_prefetch0(m->n_pkts);
4349 time = rte_get_tsc_cycles();
4350 length = instr_meter_length_nbo(t, ip);
4351 color_in = (enum rte_color)ip->meter.color_in_val;
4352
4353 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4354 &m->profile->profile,
4355 time,
4356 length,
4357 color_in);
4358
4359 color_out &= m->color_mask;
4360
4361 n_pkts = m->n_pkts[color_out];
4362 n_bytes = m->n_bytes[color_out];
4363
4364 instr_meter_color_out_hbo_set(t, ip, color_out);
4365
4366 m->n_pkts[color_out] = n_pkts + 1;
4367 m->n_bytes[color_out] = n_bytes + length;
4368}
4369
4370static inline void
4371__instr_meter_mmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4372{
4373 struct meter *m;
4374 uint64_t time, n_pkts, n_bytes;
4375 uint32_t length;
4376 enum rte_color color_in, color_out;
4377
4378 TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
4379
4380 m = instr_meter_idx_hbo(p, t, ip);
4381 rte_prefetch0(m->n_pkts);
4382 time = rte_get_tsc_cycles();
4383 length = instr_meter_length_hbo(t, ip);
4384 color_in = instr_meter_color_in_hbo(t, ip);
4385
4386 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4387 &m->profile->profile,
4388 time,
4389 length,
4390 color_in);
4391
4392 color_out &= m->color_mask;
4393
4394 n_pkts = m->n_pkts[color_out];
4395 n_bytes = m->n_bytes[color_out];
4396
4397 instr_meter_color_out_hbo_set(t, ip, color_out);
4398
4399 m->n_pkts[color_out] = n_pkts + 1;
4400 m->n_bytes[color_out] = n_bytes + length;
4401}
4402
4403static inline void
4404__instr_meter_mmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4405{
4406 struct meter *m;
4407 uint64_t time, n_pkts, n_bytes;
4408 uint32_t length;
4409 enum rte_color color_in, color_out;
4410
4411 TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
4412
4413 m = instr_meter_idx_hbo(p, t, ip);
4414 rte_prefetch0(m->n_pkts);
4415 time = rte_get_tsc_cycles();
4416 length = instr_meter_length_hbo(t, ip);
4417 color_in = (enum rte_color)ip->meter.color_in_val;
4418
4419 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4420 &m->profile->profile,
4421 time,
4422 length,
4423 color_in);
4424
4425 color_out &= m->color_mask;
4426
4427 n_pkts = m->n_pkts[color_out];
4428 n_bytes = m->n_bytes[color_out];
4429
4430 instr_meter_color_out_hbo_set(t, ip, color_out);
4431
4432 m->n_pkts[color_out] = n_pkts + 1;
4433 m->n_bytes[color_out] = n_bytes + length;
4434}
4435
4436static inline void
4437__instr_meter_ihm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4438{
4439 struct meter *m;
4440 uint64_t time, n_pkts, n_bytes;
4441 uint32_t length;
4442 enum rte_color color_in, color_out;
4443
4444 TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
4445
4446 m = instr_meter_idx_imm(p, ip);
4447 rte_prefetch0(m->n_pkts);
4448 time = rte_get_tsc_cycles();
4449 length = instr_meter_length_nbo(t, ip);
4450 color_in = instr_meter_color_in_hbo(t, ip);
4451
4452 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4453 &m->profile->profile,
4454 time,
4455 length,
4456 color_in);
4457
4458 color_out &= m->color_mask;
4459
4460 n_pkts = m->n_pkts[color_out];
4461 n_bytes = m->n_bytes[color_out];
4462
4463 instr_meter_color_out_hbo_set(t, ip, color_out);
4464
4465 m->n_pkts[color_out] = n_pkts + 1;
4466 m->n_bytes[color_out] = n_bytes + length;
4467}
4468
4469static inline void
4470__instr_meter_ihi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4471{
4472 struct meter *m;
4473 uint64_t time, n_pkts, n_bytes;
4474 uint32_t length;
4475 enum rte_color color_in, color_out;
4476
4477 TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
4478
4479 m = instr_meter_idx_imm(p, ip);
4480 rte_prefetch0(m->n_pkts);
4481 time = rte_get_tsc_cycles();
4482 length = instr_meter_length_nbo(t, ip);
4483 color_in = (enum rte_color)ip->meter.color_in_val;
4484
4485 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4486 &m->profile->profile,
4487 time,
4488 length,
4489 color_in);
4490
4491 color_out &= m->color_mask;
4492
4493 n_pkts = m->n_pkts[color_out];
4494 n_bytes = m->n_bytes[color_out];
4495
4496 instr_meter_color_out_hbo_set(t, ip, color_out);
4497
4498 m->n_pkts[color_out] = n_pkts + 1;
4499 m->n_bytes[color_out] = n_bytes + length;
4500}
4501
4502static inline void
4503__instr_meter_imm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4504{
4505 struct meter *m;
4506 uint64_t time, n_pkts, n_bytes;
4507 uint32_t length;
4508 enum rte_color color_in, color_out;
4509
4510 TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
4511
4512 m = instr_meter_idx_imm(p, ip);
4513 rte_prefetch0(m->n_pkts);
4514 time = rte_get_tsc_cycles();
4515 length = instr_meter_length_hbo(t, ip);
4516 color_in = instr_meter_color_in_hbo(t, ip);
4517
4518 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4519 &m->profile->profile,
4520 time,
4521 length,
4522 color_in);
4523
4524 color_out &= m->color_mask;
4525
4526 n_pkts = m->n_pkts[color_out];
4527 n_bytes = m->n_bytes[color_out];
4528
4529 instr_meter_color_out_hbo_set(t, ip, color_out);
4530
4531 m->n_pkts[color_out] = n_pkts + 1;
4532 m->n_bytes[color_out] = n_bytes + length;
4533}
4534
4535static inline void
4536__instr_meter_imi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4537{
4538 struct meter *m;
4539 uint64_t time, n_pkts, n_bytes;
4540 uint32_t length;
4541 enum rte_color color_in, color_out;
4542
4543 TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
4544
4545 m = instr_meter_idx_imm(p, ip);
4546 rte_prefetch0(m->n_pkts);
4547 time = rte_get_tsc_cycles();
4548 length = instr_meter_length_hbo(t, ip);
4549 color_in = (enum rte_color)ip->meter.color_in_val;
4550
4551 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4552 &m->profile->profile,
4553 time,
4554 length,
4555 color_in);
4556
4557 color_out &= m->color_mask;
4558
4559 n_pkts = m->n_pkts[color_out];
4560 n_bytes = m->n_bytes[color_out];
4561
4562 instr_meter_color_out_hbo_set(t, ip, color_out);
4563
4564 m->n_pkts[color_out] = n_pkts + 1;
4565 m->n_bytes[color_out] = n_bytes + length;
4566}
4567
4568#endif
rte_bitops.h
rte_bsf32
static uint32_t rte_bsf32(uint32_t v)
Definition: rte_bitops.h:1106
rte_byteorder.h
rte_common.h
__rte_unused
#define __rte_unused
Definition: rte_common.h:171
rte_cycles.h
rte_get_tsc_cycles
static uint64_t rte_get_tsc_cycles(void)
rte_meter.h
rte_meter_trtcm_color_aware_check
static enum rte_color rte_meter_trtcm_color_aware_check(struct rte_meter_trtcm *m, struct rte_meter_trtcm_profile *p, uint64_t time, uint32_t pkt_len, enum rte_color pkt_color)
Definition: rte_meter.h:522
rte_color
rte_color
Definition: rte_meter.h:32
RTE_COLORS
@ RTE_COLORS
Definition: rte_meter.h:36
rte_prefetch.h
rte_prefetch0
static void rte_prefetch0(const volatile void *p)
rte_swx_ctl.h
rte_swx_extern_type_destructor_t
void(* rte_swx_extern_type_destructor_t)(void *object)
Definition: rte_swx_extern.h:40
rte_swx_extern_func_t
int(* rte_swx_extern_func_t)(void *mailbox)
Definition: rte_swx_extern.h:88
rte_swx_extern_type_member_func_t
int(* rte_swx_extern_type_member_func_t)(void *object, void *mailbox)
Definition: rte_swx_extern.h:65
rte_swx_extern_type_constructor_t
void *(* rte_swx_extern_type_constructor_t)(const char *args)
Definition: rte_swx_extern.h:31
rte_swx_hash_func_t
uint32_t(* rte_swx_hash_func_t)(const void *key, uint32_t length, uint32_t seed)
Definition: rte_swx_hash_func.h:27
rte_swx_pipeline.h
RTE_SWX_NAME_SIZE
#define RTE_SWX_NAME_SIZE
Definition: rte_swx_pipeline.h:27
rte_swx_port_in_pkt_rx_t
int(* rte_swx_port_in_pkt_rx_t)(void *port, struct rte_swx_pkt *pkt)
Definition: rte_swx_port.h:68
rte_swx_port_out_flush_t
void(* rte_swx_port_out_flush_t)(void *port)
Definition: rte_swx_port.h:180
rte_swx_port_out_pkt_clone_tx_t
void(* rte_swx_port_out_pkt_clone_tx_t)(void *port, struct rte_swx_pkt *pkt, uint32_t truncation_length)
Definition: rte_swx_port.h:169
rte_swx_port_out_pkt_tx_t
void(* rte_swx_port_out_pkt_tx_t)(void *port, struct rte_swx_pkt *pkt)
Definition: rte_swx_port.h:143
rte_swx_port_out_pkt_fast_clone_tx_t
void(* rte_swx_port_out_pkt_fast_clone_tx_t)(void *port, struct rte_swx_pkt *pkt)
Definition: rte_swx_port.h:155
rte_swx_table_match_type
rte_swx_table_match_type
Definition: rte_swx_table.h:21
rte_swx_table_lookup_t
int(* rte_swx_table_lookup_t)(void *table, void *mailbox, uint8_t **key, uint64_t *action_id, uint8_t **action_data, size_t *entry_id, int *hit)
Definition: rte_swx_table.h:266
rte_swx_table_learner.h
rte_swx_table_learner_add
__rte_experimental uint32_t rte_swx_table_learner_add(void *table, void *mailbox, uint64_t time, uint64_t action_id, uint8_t *action_data, uint32_t key_timeout_id)
rte_swx_table_learner_rearm_new
__rte_experimental void rte_swx_table_learner_rearm_new(void *table, void *mailbox, uint64_t time, uint32_t key_timeout_id)
rte_swx_table_learner_delete
__rte_experimental void rte_swx_table_learner_delete(void *table, void *mailbox)
rte_swx_table_learner_rearm
__rte_experimental void rte_swx_table_learner_rearm(void *table, void *mailbox, uint64_t time)
RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX
#define RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX
Definition: rte_swx_table_learner.h:58
rte_swx_table_selector.h
rte_meter_trtcm_params
Definition: rte_meter.h:51
rte_meter_trtcm
Definition: rte_meter.h:355
rte_swx_pkt
Definition: rte_swx_port.h:17
rte_swx_pkt::pkt
uint8_t * pkt
Definition: rte_swx_port.h:22
rte_swx_pkt::offset
uint32_t offset
Definition: rte_swx_port.h:25
rte_swx_pkt::length
uint32_t length
Definition: rte_swx_port.h:28
rte_swx_port_in_ops
Definition: rte_swx_port.h:96
rte_swx_port_out_ops
Definition: rte_swx_port.h:216
rte_swx_table_ops
Definition: rte_swx_table.h:284
rte_swx_table_state
Definition: rte_swx_ctl.h:764
rte_swx_table_state::obj
void * obj
Definition: rte_swx_ctl.h:766
Generated by doxygen 1.9.4