DPDK  22.07.0
rte_swx_pipeline_internal.h
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2021 Intel Corporation
3  */
4 #ifndef __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
5 #define __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
6 
7 #include <inttypes.h>
8 #include <string.h>
9 #include <sys/queue.h>
10 
11 #include <rte_byteorder.h>
12 #include <rte_common.h>
13 #include <rte_cycles.h>
14 #include <rte_prefetch.h>
15 #include <rte_meter.h>
16 
17 #include <rte_swx_table_selector.h>
18 #include <rte_swx_table_learner.h>
19 #include <rte_swx_pipeline.h>
20 #include <rte_swx_ctl.h>
21 
22 #ifndef TRACE_LEVEL
23 #define TRACE_LEVEL 0
24 #endif
25 
26 #if TRACE_LEVEL
27 #define TRACE(...) printf(__VA_ARGS__)
28 #else
29 #define TRACE(...)
30 #endif
31 
32 /*
33  * Environment.
34  */
35 #define ntoh64(x) rte_be_to_cpu_64(x)
36 #define hton64(x) rte_cpu_to_be_64(x)
37 
38 /*
39  * Struct.
40  */
41 struct field {
42  char name[RTE_SWX_NAME_SIZE];
43  uint32_t n_bits;
44  uint32_t offset;
45  int var_size;
46 };
47 
48 struct struct_type {
49  TAILQ_ENTRY(struct_type) node;
50  char name[RTE_SWX_NAME_SIZE];
51  struct field *fields;
52  uint32_t n_fields;
53  uint32_t n_bits;
54  uint32_t n_bits_min;
55  int var_size;
56 };
57 
58 TAILQ_HEAD(struct_type_tailq, struct_type);
59 
60 /*
61  * Input port.
62  */
63 struct port_in_type {
64  TAILQ_ENTRY(port_in_type) node;
65  char name[RTE_SWX_NAME_SIZE];
66  struct rte_swx_port_in_ops ops;
67 };
68 
69 TAILQ_HEAD(port_in_type_tailq, port_in_type);
70 
71 struct port_in {
72  TAILQ_ENTRY(port_in) node;
73  struct port_in_type *type;
74  void *obj;
75  uint32_t id;
76 };
77 
78 TAILQ_HEAD(port_in_tailq, port_in);
79 
80 struct port_in_runtime {
81  rte_swx_port_in_pkt_rx_t pkt_rx;
82  void *obj;
83 };
84 
85 /*
86  * Output port.
87  */
88 struct port_out_type {
89  TAILQ_ENTRY(port_out_type) node;
90  char name[RTE_SWX_NAME_SIZE];
91  struct rte_swx_port_out_ops ops;
92 };
93 
94 TAILQ_HEAD(port_out_type_tailq, port_out_type);
95 
96 struct port_out {
97  TAILQ_ENTRY(port_out) node;
98  struct port_out_type *type;
99  void *obj;
100  uint32_t id;
101 };
102 
103 TAILQ_HEAD(port_out_tailq, port_out);
104 
105 struct port_out_runtime {
106  rte_swx_port_out_pkt_tx_t pkt_tx;
107  rte_swx_port_out_pkt_fast_clone_tx_t pkt_fast_clone_tx;
108  rte_swx_port_out_pkt_clone_tx_t pkt_clone_tx;
109  rte_swx_port_out_flush_t flush;
110  void *obj;
111 };
112 
113 /*
114  * Packet mirroring.
115  */
116 struct mirroring_session {
117  uint32_t port_id;
118  int fast_clone;
119  uint32_t truncation_length;
120 };
121 
122 /*
123  * Extern object.
124  */
125 struct extern_type_member_func {
126  TAILQ_ENTRY(extern_type_member_func) node;
127  char name[RTE_SWX_NAME_SIZE];
128  rte_swx_extern_type_member_func_t func;
129  uint32_t id;
130 };
131 
132 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
133 
134 struct extern_type {
135  TAILQ_ENTRY(extern_type) node;
136  char name[RTE_SWX_NAME_SIZE];
137  struct struct_type *mailbox_struct_type;
138  rte_swx_extern_type_constructor_t constructor;
139  rte_swx_extern_type_destructor_t destructor;
140  struct extern_type_member_func_tailq funcs;
141  uint32_t n_funcs;
142 };
143 
144 TAILQ_HEAD(extern_type_tailq, extern_type);
145 
146 struct extern_obj {
147  TAILQ_ENTRY(extern_obj) node;
148  char name[RTE_SWX_NAME_SIZE];
149  struct extern_type *type;
150  void *obj;
151  uint32_t struct_id;
152  uint32_t id;
153 };
154 
155 TAILQ_HEAD(extern_obj_tailq, extern_obj);
156 
157 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
158 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
159 #endif
160 
161 struct extern_obj_runtime {
162  void *obj;
163  uint8_t *mailbox;
164  rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
165 };
166 
167 /*
168  * Extern function.
169  */
170 struct extern_func {
171  TAILQ_ENTRY(extern_func) node;
172  char name[RTE_SWX_NAME_SIZE];
173  struct struct_type *mailbox_struct_type;
174  rte_swx_extern_func_t func;
175  uint32_t struct_id;
176  uint32_t id;
177 };
178 
179 TAILQ_HEAD(extern_func_tailq, extern_func);
180 
181 struct extern_func_runtime {
182  uint8_t *mailbox;
183  rte_swx_extern_func_t func;
184 };
185 
186 /*
187  * Hash function.
188  */
189 struct hash_func {
190  TAILQ_ENTRY(hash_func) node;
191  char name[RTE_SWX_NAME_SIZE];
192  rte_swx_hash_func_t func;
193  uint32_t id;
194 };
195 
196 TAILQ_HEAD(hash_func_tailq, hash_func);
197 
198 struct hash_func_runtime {
199  rte_swx_hash_func_t func;
200 };
201 
202 /*
203  * Header.
204  */
205 struct header {
206  TAILQ_ENTRY(header) node;
207  char name[RTE_SWX_NAME_SIZE];
208  struct struct_type *st;
209  uint32_t struct_id;
210  uint32_t id;
211 };
212 
213 TAILQ_HEAD(header_tailq, header);
214 
215 struct header_runtime {
216  uint8_t *ptr0;
217  uint32_t n_bytes;
218 };
219 
220 struct header_out_runtime {
221  uint8_t *ptr0;
222  uint8_t *ptr;
223  uint32_t n_bytes;
224 };
225 
226 /*
227  * Instruction.
228  */
229 
230 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
231  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
232  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
233  * when transferred to packet meta-data and in NBO when transferred to packet
234  * headers.
235  */
236 
237 /* Notation conventions:
238  * -Header field: H = h.header.field (dst/src)
239  * -Meta-data field: M = m.field (dst/src)
240  * -Extern object mailbox field: E = e.field (dst/src)
241  * -Extern function mailbox field: F = f.field (dst/src)
242  * -Table action data field: T = t.field (src only)
243  * -Immediate value: I = 32-bit unsigned value (src only)
244  */
245 
246 enum instruction_type {
247  /* rx m.port_in */
248  INSTR_RX,
249 
250  /* tx port_out
251  * port_out = MI
252  */
253  INSTR_TX, /* port_out = M */
254  INSTR_TX_I, /* port_out = I */
255  INSTR_DROP,
256 
257  /*
258  * mirror slot_id session_id
259  * slot_id = MEFT
260  * session_id = MEFT
261  */
262  INSTR_MIRROR,
263 
264  /* recirculate
265  */
266  INSTR_RECIRCULATE,
267 
268  /* recircid m.recirc_pass_id
269  * Read the internal recirculation pass ID into the specified meta-data field.
270  */
271  INSTR_RECIRCID,
272 
273  /* extract h.header */
274  INSTR_HDR_EXTRACT,
275  INSTR_HDR_EXTRACT2,
276  INSTR_HDR_EXTRACT3,
277  INSTR_HDR_EXTRACT4,
278  INSTR_HDR_EXTRACT5,
279  INSTR_HDR_EXTRACT6,
280  INSTR_HDR_EXTRACT7,
281  INSTR_HDR_EXTRACT8,
282 
283  /* extract h.header m.last_field_size */
284  INSTR_HDR_EXTRACT_M,
285 
286  /* lookahead h.header */
287  INSTR_HDR_LOOKAHEAD,
288 
289  /* emit h.header */
290  INSTR_HDR_EMIT,
291  INSTR_HDR_EMIT_TX,
292  INSTR_HDR_EMIT2_TX,
293  INSTR_HDR_EMIT3_TX,
294  INSTR_HDR_EMIT4_TX,
295  INSTR_HDR_EMIT5_TX,
296  INSTR_HDR_EMIT6_TX,
297  INSTR_HDR_EMIT7_TX,
298  INSTR_HDR_EMIT8_TX,
299 
300  /* validate h.header */
301  INSTR_HDR_VALIDATE,
302 
303  /* invalidate h.header */
304  INSTR_HDR_INVALIDATE,
305 
306  /* mov dst src
307  * dst = src
308  * dst = HMEF, src = HMEFTI
309  */
310  INSTR_MOV, /* dst = MEF, src = MEFT */
311  INSTR_MOV_MH, /* dst = MEF, src = H */
312  INSTR_MOV_HM, /* dst = H, src = MEFT */
313  INSTR_MOV_HH, /* dst = H, src = H */
314  INSTR_MOV_I, /* dst = HMEF, src = I */
315 
316  /* dma h.header t.field
317  * memcpy(h.header, t.field, sizeof(h.header))
318  */
319  INSTR_DMA_HT,
320  INSTR_DMA_HT2,
321  INSTR_DMA_HT3,
322  INSTR_DMA_HT4,
323  INSTR_DMA_HT5,
324  INSTR_DMA_HT6,
325  INSTR_DMA_HT7,
326  INSTR_DMA_HT8,
327 
328  /* add dst src
329  * dst += src
330  * dst = HMEF, src = HMEFTI
331  */
332  INSTR_ALU_ADD, /* dst = MEF, src = MEF */
333  INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
334  INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
335  INSTR_ALU_ADD_HH, /* dst = H, src = H */
336  INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
337  INSTR_ALU_ADD_HI, /* dst = H, src = I */
338 
339  /* sub dst src
340  * dst -= src
341  * dst = HMEF, src = HMEFTI
342  */
343  INSTR_ALU_SUB, /* dst = MEF, src = MEF */
344  INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
345  INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
346  INSTR_ALU_SUB_HH, /* dst = H, src = H */
347  INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
348  INSTR_ALU_SUB_HI, /* dst = H, src = I */
349 
350  /* ckadd dst src
351  * dst = dst '+ src[0:1] '+ src[2:3] '+ ...
352  * dst = H, src = {H, h.header}, '+ = 1's complement addition operator
353  */
354  INSTR_ALU_CKADD_FIELD, /* src = H */
355  INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 bytes. */
356  INSTR_ALU_CKADD_STRUCT, /* src = h.header, with sizeof(header) any 4-byte multiple. */
357 
358  /* cksub dst src
359  * dst = dst '- src
360  * dst = H, src = H, '- = 1's complement subtraction operator
361  */
362  INSTR_ALU_CKSUB_FIELD,
363 
364  /* and dst src
365  * dst &= src
366  * dst = HMEF, src = HMEFTI
367  */
368  INSTR_ALU_AND, /* dst = MEF, src = MEFT */
369  INSTR_ALU_AND_MH, /* dst = MEF, src = H */
370  INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
371  INSTR_ALU_AND_HH, /* dst = H, src = H */
372  INSTR_ALU_AND_I, /* dst = HMEF, src = I */
373 
374  /* or dst src
375  * dst |= src
376  * dst = HMEF, src = HMEFTI
377  */
378  INSTR_ALU_OR, /* dst = MEF, src = MEFT */
379  INSTR_ALU_OR_MH, /* dst = MEF, src = H */
380  INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
381  INSTR_ALU_OR_HH, /* dst = H, src = H */
382  INSTR_ALU_OR_I, /* dst = HMEF, src = I */
383 
384  /* xor dst src
385  * dst ^= src
386  * dst = HMEF, src = HMEFTI
387  */
388  INSTR_ALU_XOR, /* dst = MEF, src = MEFT */
389  INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
390  INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
391  INSTR_ALU_XOR_HH, /* dst = H, src = H */
392  INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
393 
394  /* shl dst src
395  * dst <<= src
396  * dst = HMEF, src = HMEFTI
397  */
398  INSTR_ALU_SHL, /* dst = MEF, src = MEF */
399  INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
400  INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
401  INSTR_ALU_SHL_HH, /* dst = H, src = H */
402  INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
403  INSTR_ALU_SHL_HI, /* dst = H, src = I */
404 
405  /* shr dst src
406  * dst >>= src
407  * dst = HMEF, src = HMEFTI
408  */
409  INSTR_ALU_SHR, /* dst = MEF, src = MEF */
410  INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
411  INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
412  INSTR_ALU_SHR_HH, /* dst = H, src = H */
413  INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
414  INSTR_ALU_SHR_HI, /* dst = H, src = I */
415 
416  /* regprefetch REGARRAY index
417  * prefetch REGARRAY[index]
418  * index = HMEFTI
419  */
420  INSTR_REGPREFETCH_RH, /* index = H */
421  INSTR_REGPREFETCH_RM, /* index = MEFT */
422  INSTR_REGPREFETCH_RI, /* index = I */
423 
424  /* regrd dst REGARRAY index
425  * dst = REGARRAY[index]
426  * dst = HMEF, index = HMEFTI
427  */
428  INSTR_REGRD_HRH, /* dst = H, index = H */
429  INSTR_REGRD_HRM, /* dst = H, index = MEFT */
430  INSTR_REGRD_HRI, /* dst = H, index = I */
431  INSTR_REGRD_MRH, /* dst = MEF, index = H */
432  INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
433  INSTR_REGRD_MRI, /* dst = MEF, index = I */
434 
435  /* regwr REGARRAY index src
436  * REGARRAY[index] = src
437  * index = HMEFTI, src = HMEFTI
438  */
439  INSTR_REGWR_RHH, /* index = H, src = H */
440  INSTR_REGWR_RHM, /* index = H, src = MEFT */
441  INSTR_REGWR_RHI, /* index = H, src = I */
442  INSTR_REGWR_RMH, /* index = MEFT, src = H */
443  INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
444  INSTR_REGWR_RMI, /* index = MEFT, src = I */
445  INSTR_REGWR_RIH, /* index = I, src = H */
446  INSTR_REGWR_RIM, /* index = I, src = MEFT */
447  INSTR_REGWR_RII, /* index = I, src = I */
448 
449  /* regadd REGARRAY index src
450  * REGARRAY[index] += src
451  * index = HMEFTI, src = HMEFTI
452  */
453  INSTR_REGADD_RHH, /* index = H, src = H */
454  INSTR_REGADD_RHM, /* index = H, src = MEFT */
455  INSTR_REGADD_RHI, /* index = H, src = I */
456  INSTR_REGADD_RMH, /* index = MEFT, src = H */
457  INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
458  INSTR_REGADD_RMI, /* index = MEFT, src = I */
459  INSTR_REGADD_RIH, /* index = I, src = H */
460  INSTR_REGADD_RIM, /* index = I, src = MEFT */
461  INSTR_REGADD_RII, /* index = I, src = I */
462 
463  /* metprefetch METARRAY index
464  * prefetch METARRAY[index]
465  * index = HMEFTI
466  */
467  INSTR_METPREFETCH_H, /* index = H */
468  INSTR_METPREFETCH_M, /* index = MEFT */
469  INSTR_METPREFETCH_I, /* index = I */
470 
471  /* meter METARRAY index length color_in color_out
472  * color_out = meter(METARRAY[index], length, color_in)
473  * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
474  */
475  INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
476  INSTR_METER_HHI, /* index = H, length = H, color_in = I */
477  INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
478  INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
479  INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
480  INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
481  INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
482  INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
483  INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
484  INSTR_METER_IHI, /* index = I, length = H, color_in = I */
485  INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
486  INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
487 
488  /* table TABLE */
489  INSTR_TABLE,
490  INSTR_TABLE_AF,
491  INSTR_SELECTOR,
492  INSTR_LEARNER,
493  INSTR_LEARNER_AF,
494 
495  /* learn ACTION_NAME [ m.action_first_arg ] m.timeout_id */
496  INSTR_LEARNER_LEARN,
497 
498  /* rearm [ m.timeout_id ] */
499  INSTR_LEARNER_REARM,
500  INSTR_LEARNER_REARM_NEW,
501 
502  /* forget */
503  INSTR_LEARNER_FORGET,
504 
505  /* extern e.obj.func */
506  INSTR_EXTERN_OBJ,
507 
508  /* extern f.func */
509  INSTR_EXTERN_FUNC,
510 
511  /* hash HASH_FUNC_NAME dst src_first src_last
512  * Compute hash value over range of struct fields.
513  * dst = M
514  * src_first = HMEFT
515  * src_last = HMEFT
516  * src_first and src_last must be fields within the same struct
517  */
518  INSTR_HASH_FUNC,
519 
520  /* jmp LABEL
521  * Unconditional jump
522  */
523  INSTR_JMP,
524 
525  /* jmpv LABEL h.header
526  * Jump if header is valid
527  */
528  INSTR_JMP_VALID,
529 
530  /* jmpnv LABEL h.header
531  * Jump if header is invalid
532  */
533  INSTR_JMP_INVALID,
534 
535  /* jmph LABEL
536  * Jump if table lookup hit
537  */
538  INSTR_JMP_HIT,
539 
540  /* jmpnh LABEL
541  * Jump if table lookup miss
542  */
543  INSTR_JMP_MISS,
544 
545  /* jmpa LABEL ACTION
546  * Jump if action run
547  */
548  INSTR_JMP_ACTION_HIT,
549 
550  /* jmpna LABEL ACTION
551  * Jump if action not run
552  */
553  INSTR_JMP_ACTION_MISS,
554 
555  /* jmpeq LABEL a b
556  * Jump if a is equal to b
557  * a = HMEFT, b = HMEFTI
558  */
559  INSTR_JMP_EQ, /* a = MEFT, b = MEFT */
560  INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
561  INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
562  INSTR_JMP_EQ_HH, /* a = H, b = H */
563  INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
564 
565  /* jmpneq LABEL a b
566  * Jump if a is not equal to b
567  * a = HMEFT, b = HMEFTI
568  */
569  INSTR_JMP_NEQ, /* a = MEFT, b = MEFT */
570  INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
571  INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
572  INSTR_JMP_NEQ_HH, /* a = H, b = H */
573  INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
574 
575  /* jmplt LABEL a b
576  * Jump if a is less than b
577  * a = HMEFT, b = HMEFTI
578  */
579  INSTR_JMP_LT, /* a = MEFT, b = MEFT */
580  INSTR_JMP_LT_MH, /* a = MEFT, b = H */
581  INSTR_JMP_LT_HM, /* a = H, b = MEFT */
582  INSTR_JMP_LT_HH, /* a = H, b = H */
583  INSTR_JMP_LT_MI, /* a = MEFT, b = I */
584  INSTR_JMP_LT_HI, /* a = H, b = I */
585 
586  /* jmpgt LABEL a b
587  * Jump if a is greater than b
588  * a = HMEFT, b = HMEFTI
589  */
590  INSTR_JMP_GT, /* a = MEFT, b = MEFT */
591  INSTR_JMP_GT_MH, /* a = MEFT, b = H */
592  INSTR_JMP_GT_HM, /* a = H, b = MEFT */
593  INSTR_JMP_GT_HH, /* a = H, b = H */
594  INSTR_JMP_GT_MI, /* a = MEFT, b = I */
595  INSTR_JMP_GT_HI, /* a = H, b = I */
596 
597  /* return
598  * Return from action
599  */
600  INSTR_RETURN,
601 
602  /* Start of custom instructions. */
603  INSTR_CUSTOM_0,
604 };
605 
606 struct instr_operand {
607  uint8_t struct_id;
608  uint8_t n_bits;
609  uint8_t offset;
610  uint8_t pad;
611 };
612 
613 struct instr_io {
614  struct {
615  union {
616  struct {
617  uint8_t offset;
618  uint8_t n_bits;
619  uint8_t pad[2];
620  };
621 
622  uint32_t val;
623  };
624  } io;
625 
626  struct {
627  uint8_t header_id[8];
628  uint8_t struct_id[8];
629  uint8_t n_bytes[8];
630  } hdr;
631 };
632 
633 struct instr_hdr_validity {
634  uint8_t header_id;
635  uint8_t struct_id;
636 };
637 
638 struct instr_table {
639  uint8_t table_id;
640 };
641 
642 struct instr_learn {
643  uint8_t action_id;
644  uint8_t mf_first_arg_offset;
645  uint8_t mf_timeout_id_offset;
646  uint8_t mf_timeout_id_n_bits;
647 };
648 
649 struct instr_extern_obj {
650  uint8_t ext_obj_id;
651  uint8_t func_id;
652 };
653 
654 struct instr_extern_func {
655  uint8_t ext_func_id;
656 };
657 
658 struct instr_hash_func {
659  uint8_t hash_func_id;
660 
661  struct {
662  uint8_t offset;
663  uint8_t n_bits;
664  } dst;
665 
666  struct {
667  uint8_t struct_id;
668  uint16_t offset;
669  uint16_t n_bytes;
670  } src;
671 };
672 
673 struct instr_dst_src {
674  struct instr_operand dst;
675  union {
676  struct instr_operand src;
677  uint64_t src_val;
678  };
679 };
680 
681 struct instr_regarray {
682  uint8_t regarray_id;
683  uint8_t pad[3];
684 
685  union {
686  struct instr_operand idx;
687  uint32_t idx_val;
688  };
689 
690  union {
691  struct instr_operand dstsrc;
692  uint64_t dstsrc_val;
693  };
694 };
695 
696 struct instr_meter {
697  uint8_t metarray_id;
698  uint8_t pad[3];
699 
700  union {
701  struct instr_operand idx;
702  uint32_t idx_val;
703  };
704 
705  struct instr_operand length;
706 
707  union {
708  struct instr_operand color_in;
709  uint32_t color_in_val;
710  };
711 
712  struct instr_operand color_out;
713 };
714 
715 struct instr_dma {
716  struct {
717  uint8_t header_id[8];
718  uint8_t struct_id[8];
719  } dst;
720 
721  struct {
722  uint8_t offset[8];
723  } src;
724 
725  uint16_t n_bytes[8];
726 };
727 
728 struct instr_jmp {
729  struct instruction *ip;
730 
731  union {
732  struct instr_operand a;
733  uint8_t header_id;
734  uint8_t action_id;
735  };
736 
737  union {
738  struct instr_operand b;
739  uint64_t b_val;
740  };
741 };
742 
743 struct instruction {
744  enum instruction_type type;
745  union {
746  struct instr_io io;
747  struct instr_dst_src mirror;
748  struct instr_hdr_validity valid;
749  struct instr_dst_src mov;
750  struct instr_regarray regarray;
751  struct instr_meter meter;
752  struct instr_dma dma;
753  struct instr_dst_src alu;
754  struct instr_table table;
755  struct instr_learn learn;
756  struct instr_extern_obj ext_obj;
757  struct instr_extern_func ext_func;
758  struct instr_hash_func hash_func;
759  struct instr_jmp jmp;
760  };
761 };
762 
763 struct instruction_data {
764  char label[RTE_SWX_NAME_SIZE];
765  char jmp_label[RTE_SWX_NAME_SIZE];
766  uint32_t n_users; /* user = jmp instruction to this instruction. */
767  int invalid;
768 };
769 
770 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
771 
772 /*
773  * Action.
774  */
775 typedef void
776 (*action_func_t)(struct rte_swx_pipeline *p);
777 
778 struct action {
779  TAILQ_ENTRY(action) node;
780  char name[RTE_SWX_NAME_SIZE];
781  struct struct_type *st;
782  int *args_endianness; /* 0 = Host Byte Order (HBO); 1 = Network Byte Order (NBO). */
783  struct instruction *instructions;
784  struct instruction_data *instruction_data;
785  uint32_t n_instructions;
786  uint32_t id;
787 };
788 
789 TAILQ_HEAD(action_tailq, action);
790 
791 /*
792  * Table.
793  */
794 struct table_type {
795  TAILQ_ENTRY(table_type) node;
796  char name[RTE_SWX_NAME_SIZE];
797  enum rte_swx_table_match_type match_type;
798  struct rte_swx_table_ops ops;
799 };
800 
801 TAILQ_HEAD(table_type_tailq, table_type);
802 
803 struct match_field {
804  enum rte_swx_table_match_type match_type;
805  struct field *field;
806 };
807 
808 struct table {
809  TAILQ_ENTRY(table) node;
810  char name[RTE_SWX_NAME_SIZE];
811  char args[RTE_SWX_NAME_SIZE];
812  struct table_type *type; /* NULL when n_fields == 0. */
813 
814  /* Match. */
815  struct match_field *fields;
816  uint32_t n_fields;
817  struct header *header; /* Only valid when n_fields > 0. */
818 
819  /* Action. */
820  struct action **actions;
821  struct action *default_action;
822  uint8_t *default_action_data;
823  uint32_t n_actions;
824  int default_action_is_const;
825  uint32_t action_data_size_max;
826  int *action_is_for_table_entries;
827  int *action_is_for_default_entry;
828 
829  uint32_t size;
830  uint32_t id;
831 };
832 
833 TAILQ_HEAD(table_tailq, table);
834 
835 struct table_runtime {
836  rte_swx_table_lookup_t func;
837  void *mailbox;
838  uint8_t **key;
839 };
840 
841 struct table_statistics {
842  uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
843  uint64_t *n_pkts_action;
844 };
845 
846 /*
847  * Selector.
848  */
849 struct selector {
850  TAILQ_ENTRY(selector) node;
851  char name[RTE_SWX_NAME_SIZE];
852 
853  struct field *group_id_field;
854  struct field **selector_fields;
855  uint32_t n_selector_fields;
856  struct header *selector_header;
857  struct field *member_id_field;
858 
859  uint32_t n_groups_max;
860  uint32_t n_members_per_group_max;
861 
862  uint32_t id;
863 };
864 
865 TAILQ_HEAD(selector_tailq, selector);
866 
867 struct selector_runtime {
868  void *mailbox;
869  uint8_t **group_id_buffer;
870  uint8_t **selector_buffer;
871  uint8_t **member_id_buffer;
872 };
873 
874 struct selector_statistics {
875  uint64_t n_pkts;
876 };
877 
878 /*
879  * Learner table.
880  */
881 struct learner {
882  TAILQ_ENTRY(learner) node;
883  char name[RTE_SWX_NAME_SIZE];
884 
885  /* Match. */
886  struct field **fields;
887  uint32_t n_fields;
888  struct header *header;
889 
890  /* Action. */
891  struct action **actions;
892  struct action *default_action;
893  uint8_t *default_action_data;
894  uint32_t n_actions;
895  int default_action_is_const;
896  uint32_t action_data_size_max;
897  int *action_is_for_table_entries;
898  int *action_is_for_default_entry;
899 
900  uint32_t size;
901  uint32_t timeout[RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX];
902  uint32_t n_timeouts;
903  uint32_t id;
904 };
905 
906 TAILQ_HEAD(learner_tailq, learner);
907 
908 struct learner_runtime {
909  void *mailbox;
910  uint8_t **key;
911 };
912 
913 struct learner_statistics {
914  uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
915  uint64_t n_pkts_learn[2]; /* 0 = Learn OK, 1 = Learn error. */
916  uint64_t n_pkts_rearm;
917  uint64_t n_pkts_forget;
918  uint64_t *n_pkts_action;
919 };
920 
921 /*
922  * Register array.
923  */
924 struct regarray {
925  TAILQ_ENTRY(regarray) node;
926  char name[RTE_SWX_NAME_SIZE];
927  uint64_t init_val;
928  uint32_t size;
929  uint32_t id;
930 };
931 
932 TAILQ_HEAD(regarray_tailq, regarray);
933 
934 struct regarray_runtime {
935  uint64_t *regarray;
936  uint32_t size_mask;
937 };
938 
939 /*
940  * Meter array.
941  */
942 struct meter_profile {
943  TAILQ_ENTRY(meter_profile) node;
944  char name[RTE_SWX_NAME_SIZE];
945  struct rte_meter_trtcm_params params;
946  struct rte_meter_trtcm_profile profile;
947  uint32_t n_users;
948 };
949 
950 TAILQ_HEAD(meter_profile_tailq, meter_profile);
951 
952 struct metarray {
953  TAILQ_ENTRY(metarray) node;
954  char name[RTE_SWX_NAME_SIZE];
955  uint32_t size;
956  uint32_t id;
957 };
958 
959 TAILQ_HEAD(metarray_tailq, metarray);
960 
961 struct meter {
962  struct rte_meter_trtcm m;
963  struct meter_profile *profile;
964  enum rte_color color_mask;
965  uint8_t pad[20];
966 
967  uint64_t n_pkts[RTE_COLORS];
968  uint64_t n_bytes[RTE_COLORS];
969 };
970 
971 struct metarray_runtime {
972  struct meter *metarray;
973  uint32_t size_mask;
974 };
975 
976 /*
977  * Pipeline.
978  */
979 struct thread {
980  /* Packet. */
981  struct rte_swx_pkt pkt;
982  uint8_t *ptr;
983  uint32_t *mirroring_slots;
984  uint64_t mirroring_slots_mask;
985  int recirculate;
986  uint32_t recirc_pass_id;
987 
988  /* Structures. */
989  uint8_t **structs;
990 
991  /* Packet headers. */
992  struct header_runtime *headers; /* Extracted or generated headers. */
993  struct header_out_runtime *headers_out; /* Emitted headers. */
994  uint8_t *header_storage;
995  uint8_t *header_out_storage;
996  uint64_t valid_headers;
997  uint32_t n_headers_out;
998 
999  /* Packet meta-data. */
1000  uint8_t *metadata;
1001 
1002  /* Tables. */
1003  struct table_runtime *tables;
1004  struct selector_runtime *selectors;
1005  struct learner_runtime *learners;
1006  struct rte_swx_table_state *table_state;
1007  uint64_t action_id;
1008  int hit; /* 0 = Miss, 1 = Hit. */
1009  uint32_t learner_id;
1010  uint64_t time;
1011 
1012  /* Extern objects and functions. */
1013  struct extern_obj_runtime *extern_objs;
1014  struct extern_func_runtime *extern_funcs;
1015 
1016  /* Instructions. */
1017  struct instruction *ip;
1018  struct instruction *ret;
1019 };
1020 
1021 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
1022 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
1023 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
1024 
1025 #define HEADER_VALID(thread, header_id) \
1026  MASK64_BIT_GET((thread)->valid_headers, header_id)
1027 
1028 static inline uint64_t
1029 instr_operand_hbo(struct thread *t, const struct instr_operand *x)
1030 {
1031  uint8_t *x_struct = t->structs[x->struct_id];
1032  uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
1033  uint64_t x64 = *x64_ptr;
1034  uint64_t x64_mask = UINT64_MAX >> (64 - x->n_bits);
1035 
1036  return x64 & x64_mask;
1037 }
1038 
1039 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1040 
1041 static inline uint64_t
1042 instr_operand_nbo(struct thread *t, const struct instr_operand *x)
1043 {
1044  uint8_t *x_struct = t->structs[x->struct_id];
1045  uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
1046  uint64_t x64 = *x64_ptr;
1047 
1048  return ntoh64(x64) >> (64 - x->n_bits);
1049 }
1050 
1051 #else
1052 
1053 #define instr_operand_nbo instr_operand_hbo
1054 
1055 #endif
1056 
1057 #define ALU(thread, ip, operator) \
1058 { \
1059  uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1060  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1061  uint64_t dst64 = *dst64_ptr; \
1062  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1063  uint64_t dst = dst64 & dst64_mask; \
1064  \
1065  uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1066  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1067  uint64_t src64 = *src64_ptr; \
1068  uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1069  uint64_t src = src64 & src64_mask; \
1070  \
1071  uint64_t result = dst operator src; \
1072  \
1073  *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1074 }
1075 
1076 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1077 
1078 #define ALU_MH(thread, ip, operator) \
1079 { \
1080  uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1081  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1082  uint64_t dst64 = *dst64_ptr; \
1083  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1084  uint64_t dst = dst64 & dst64_mask; \
1085  \
1086  uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1087  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1088  uint64_t src64 = *src64_ptr; \
1089  uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1090  \
1091  uint64_t result = dst operator src; \
1092  \
1093  *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1094 }
1095 
1096 #define ALU_HM(thread, ip, operator) \
1097 { \
1098  uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1099  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1100  uint64_t dst64 = *dst64_ptr; \
1101  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1102  uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1103  \
1104  uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1105  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1106  uint64_t src64 = *src64_ptr; \
1107  uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1108  uint64_t src = src64 & src64_mask; \
1109  \
1110  uint64_t result = dst operator src; \
1111  result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1112  \
1113  *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1114 }
1115 
1116 #define ALU_HM_FAST(thread, ip, operator) \
1117 { \
1118  uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1119  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1120  uint64_t dst64 = *dst64_ptr; \
1121  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1122  uint64_t dst = dst64 & dst64_mask; \
1123  \
1124  uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1125  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1126  uint64_t src64 = *src64_ptr; \
1127  uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1128  uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
1129  \
1130  uint64_t result = dst operator src; \
1131  \
1132  *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1133 }
1134 
1135 #define ALU_HH(thread, ip, operator) \
1136 { \
1137  uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1138  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1139  uint64_t dst64 = *dst64_ptr; \
1140  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1141  uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1142  \
1143  uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1144  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1145  uint64_t src64 = *src64_ptr; \
1146  uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1147  \
1148  uint64_t result = dst operator src; \
1149  result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1150  \
1151  *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1152 }
1153 
1154 #define ALU_HH_FAST(thread, ip, operator) \
1155 { \
1156  uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1157  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1158  uint64_t dst64 = *dst64_ptr; \
1159  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1160  uint64_t dst = dst64 & dst64_mask; \
1161  \
1162  uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1163  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1164  uint64_t src64 = *src64_ptr; \
1165  uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
1166  \
1167  uint64_t result = dst operator src; \
1168  \
1169  *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1170 }
1171 
1172 #else
1173 
1174 #define ALU_MH ALU
1175 #define ALU_HM ALU
1176 #define ALU_HM_FAST ALU
1177 #define ALU_HH ALU
1178 #define ALU_HH_FAST ALU
1179 
1180 #endif
1181 
1182 #define ALU_I(thread, ip, operator) \
1183 { \
1184  uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1185  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1186  uint64_t dst64 = *dst64_ptr; \
1187  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1188  uint64_t dst = dst64 & dst64_mask; \
1189  \
1190  uint64_t src = (ip)->alu.src_val; \
1191  \
1192  uint64_t result = dst operator src; \
1193  \
1194  *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1195 }
1196 
1197 #define ALU_MI ALU_I
1198 
1199 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1200 
1201 #define ALU_HI(thread, ip, operator) \
1202 { \
1203  uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1204  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1205  uint64_t dst64 = *dst64_ptr; \
1206  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1207  uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1208  \
1209  uint64_t src = (ip)->alu.src_val; \
1210  \
1211  uint64_t result = dst operator src; \
1212  result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1213  \
1214  *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1215 }
1216 
1217 #else
1218 
1219 #define ALU_HI ALU_I
1220 
1221 #endif
1222 
1223 #define MOV(thread, ip) \
1224 { \
1225  uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1226  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1227  uint64_t dst64 = *dst64_ptr; \
1228  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1229  \
1230  uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1231  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1232  uint64_t src64 = *src64_ptr; \
1233  uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1234  uint64_t src = src64 & src64_mask; \
1235  \
1236  *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1237 }
1238 
1239 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1240 
1241 #define MOV_MH(thread, ip) \
1242 { \
1243  uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1244  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1245  uint64_t dst64 = *dst64_ptr; \
1246  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1247  \
1248  uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1249  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1250  uint64_t src64 = *src64_ptr; \
1251  uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits); \
1252  \
1253  *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1254 }
1255 
1256 #define MOV_HM(thread, ip) \
1257 { \
1258  uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1259  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1260  uint64_t dst64 = *dst64_ptr; \
1261  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1262  \
1263  uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1264  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1265  uint64_t src64 = *src64_ptr; \
1266  uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1267  uint64_t src = src64 & src64_mask; \
1268  \
1269  src = hton64(src) >> (64 - (ip)->mov.dst.n_bits); \
1270  *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1271 }
1272 
1273 #define MOV_HH(thread, ip) \
1274 { \
1275  uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1276  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1277  uint64_t dst64 = *dst64_ptr; \
1278  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1279  \
1280  uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1281  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1282  uint64_t src64 = *src64_ptr; \
1283  \
1284  uint64_t src = src64 << (64 - (ip)->mov.src.n_bits); \
1285  src = src >> (64 - (ip)->mov.dst.n_bits); \
1286  *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1287 }
1288 
1289 #else
1290 
1291 #define MOV_MH MOV
1292 #define MOV_HM MOV
1293 #define MOV_HH MOV
1294 
1295 #endif
1296 
1297 #define MOV_I(thread, ip) \
1298 { \
1299  uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1300  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1301  uint64_t dst64 = *dst64_ptr; \
1302  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1303  \
1304  uint64_t src = (ip)->mov.src_val; \
1305  \
1306  *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1307 }
1308 
1309 #define JMP_CMP(thread, ip, operator) \
1310 { \
1311  uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1312  uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1313  uint64_t a64 = *a64_ptr; \
1314  uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1315  uint64_t a = a64 & a64_mask; \
1316  \
1317  uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1318  uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1319  uint64_t b64 = *b64_ptr; \
1320  uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1321  uint64_t b = b64 & b64_mask; \
1322  \
1323  (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1324 }
1325 
1326 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1327 
1328 #define JMP_CMP_MH(thread, ip, operator) \
1329 { \
1330  uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1331  uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1332  uint64_t a64 = *a64_ptr; \
1333  uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1334  uint64_t a = a64 & a64_mask; \
1335  \
1336  uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1337  uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1338  uint64_t b64 = *b64_ptr; \
1339  uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1340  \
1341  (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1342 }
1343 
1344 #define JMP_CMP_HM(thread, ip, operator) \
1345 { \
1346  uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1347  uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1348  uint64_t a64 = *a64_ptr; \
1349  uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1350  \
1351  uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1352  uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1353  uint64_t b64 = *b64_ptr; \
1354  uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1355  uint64_t b = b64 & b64_mask; \
1356  \
1357  (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1358 }
1359 
1360 #define JMP_CMP_HH(thread, ip, operator) \
1361 { \
1362  uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1363  uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1364  uint64_t a64 = *a64_ptr; \
1365  uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1366  \
1367  uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1368  uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1369  uint64_t b64 = *b64_ptr; \
1370  uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1371  \
1372  (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1373 }
1374 
1375 #define JMP_CMP_HH_FAST(thread, ip, operator) \
1376 { \
1377  uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1378  uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1379  uint64_t a64 = *a64_ptr; \
1380  uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits); \
1381  \
1382  uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1383  uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1384  uint64_t b64 = *b64_ptr; \
1385  uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits); \
1386  \
1387  (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1388 }
1389 
1390 #else
1391 
1392 #define JMP_CMP_MH JMP_CMP
1393 #define JMP_CMP_HM JMP_CMP
1394 #define JMP_CMP_HH JMP_CMP
1395 #define JMP_CMP_HH_FAST JMP_CMP
1396 
1397 #endif
1398 
1399 #define JMP_CMP_I(thread, ip, operator) \
1400 { \
1401  uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1402  uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1403  uint64_t a64 = *a64_ptr; \
1404  uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1405  uint64_t a = a64 & a64_mask; \
1406  \
1407  uint64_t b = (ip)->jmp.b_val; \
1408  \
1409  (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1410 }
1411 
1412 #define JMP_CMP_MI JMP_CMP_I
1413 
1414 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1415 
1416 #define JMP_CMP_HI(thread, ip, operator) \
1417 { \
1418  uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1419  uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1420  uint64_t a64 = *a64_ptr; \
1421  uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1422  \
1423  uint64_t b = (ip)->jmp.b_val; \
1424  \
1425  (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1426 }
1427 
1428 #else
1429 
1430 #define JMP_CMP_HI JMP_CMP_I
1431 
1432 #endif
1433 
1434 #define METADATA_READ(thread, offset, n_bits) \
1435 ({ \
1436  uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1437  uint64_t m64 = *m64_ptr; \
1438  uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1439  (m64 & m64_mask); \
1440 })
1441 
1442 #define METADATA_WRITE(thread, offset, n_bits, value) \
1443 { \
1444  uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1445  uint64_t m64 = *m64_ptr; \
1446  uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1447  \
1448  uint64_t m_new = value; \
1449  \
1450  *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask); \
1451 }
1452 
1453 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1454 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1455 #endif
1456 
1457 #ifndef RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX
1458 #define RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX 256
1459 #endif
1460 
1461 struct rte_swx_pipeline {
1462  struct struct_type_tailq struct_types;
1463  struct port_in_type_tailq port_in_types;
1464  struct port_in_tailq ports_in;
1465  struct port_out_type_tailq port_out_types;
1466  struct port_out_tailq ports_out;
1467  struct extern_type_tailq extern_types;
1468  struct extern_obj_tailq extern_objs;
1469  struct extern_func_tailq extern_funcs;
1470  struct hash_func_tailq hash_funcs;
1471  struct header_tailq headers;
1472  struct struct_type *metadata_st;
1473  uint32_t metadata_struct_id;
1474  struct action_tailq actions;
1475  struct table_type_tailq table_types;
1476  struct table_tailq tables;
1477  struct selector_tailq selectors;
1478  struct learner_tailq learners;
1479  struct regarray_tailq regarrays;
1480  struct meter_profile_tailq meter_profiles;
1481  struct metarray_tailq metarrays;
1482 
1483  struct port_in_runtime *in;
1484  struct port_out_runtime *out;
1485  struct mirroring_session *mirroring_sessions;
1486  struct instruction **action_instructions;
1487  action_func_t *action_funcs;
1488  struct rte_swx_table_state *table_state;
1489  struct table_statistics *table_stats;
1490  struct selector_statistics *selector_stats;
1491  struct learner_statistics *learner_stats;
1492  struct hash_func_runtime *hash_func_runtime;
1493  struct regarray_runtime *regarray_runtime;
1494  struct metarray_runtime *metarray_runtime;
1495  struct instruction *instructions;
1496  struct instruction_data *instruction_data;
1497  instr_exec_t *instruction_table;
1498  struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1499  void *lib;
1500 
1501  uint32_t n_structs;
1502  uint32_t n_ports_in;
1503  uint32_t n_ports_out;
1504  uint32_t n_mirroring_slots;
1505  uint32_t n_mirroring_sessions;
1506  uint32_t n_extern_objs;
1507  uint32_t n_extern_funcs;
1508  uint32_t n_hash_funcs;
1509  uint32_t n_actions;
1510  uint32_t n_tables;
1511  uint32_t n_selectors;
1512  uint32_t n_learners;
1513  uint32_t n_regarrays;
1514  uint32_t n_metarrays;
1515  uint32_t n_headers;
1516  uint32_t thread_id;
1517  uint32_t port_id;
1518  uint32_t n_instructions;
1519  int build_done;
1520  int numa_node;
1521 };
1522 
1523 /*
1524  * Instruction.
1525  */
1526 static inline void
1527 pipeline_port_inc(struct rte_swx_pipeline *p)
1528 {
1529  p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
1530 }
1531 
1532 static inline void
1533 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
1534 {
1535  t->ip = p->instructions;
1536 }
1537 
1538 static inline void
1539 thread_ip_set(struct thread *t, struct instruction *ip)
1540 {
1541  t->ip = ip;
1542 }
1543 
1544 static inline void
1545 thread_ip_action_call(struct rte_swx_pipeline *p,
1546  struct thread *t,
1547  uint32_t action_id)
1548 {
1549  t->ret = t->ip + 1;
1550  t->ip = p->action_instructions[action_id];
1551 }
1552 
1553 static inline void
1554 thread_ip_inc(struct rte_swx_pipeline *p);
1555 
1556 static inline void
1557 thread_ip_inc(struct rte_swx_pipeline *p)
1558 {
1559  struct thread *t = &p->threads[p->thread_id];
1560 
1561  t->ip++;
1562 }
1563 
1564 static inline void
1565 thread_ip_inc_cond(struct thread *t, int cond)
1566 {
1567  t->ip += cond;
1568 }
1569 
1570 static inline void
1571 thread_yield(struct rte_swx_pipeline *p)
1572 {
1573  p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1574 }
1575 
1576 static inline void
1577 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
1578 {
1579  p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1580 }
1581 
1582 /*
1583  * rx.
1584  */
1585 static inline int
1586 __instr_rx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1587 {
1588  struct port_in_runtime *port = &p->in[p->port_id];
1589  struct rte_swx_pkt *pkt = &t->pkt;
1590  int pkt_received;
1591 
1592  /* Recirculation: keep the current packet. */
1593  if (t->recirculate) {
1594  TRACE("[Thread %2u] rx - recirculate (pass %u)\n",
1595  p->thread_id,
1596  t->recirc_pass_id + 1);
1597 
1598  /* Packet. */
1599  t->ptr = &pkt->pkt[pkt->offset];
1600  t->mirroring_slots_mask = 0;
1601  t->recirculate = 0;
1602  t->recirc_pass_id++;
1603 
1604  /* Headers. */
1605  t->valid_headers = 0;
1606  t->n_headers_out = 0;
1607 
1608  /* Tables. */
1609  t->table_state = p->table_state;
1610 
1611  return 1;
1612  }
1613 
1614  /* Packet. */
1615  pkt_received = port->pkt_rx(port->obj, pkt);
1616  t->ptr = &pkt->pkt[pkt->offset];
1617  rte_prefetch0(t->ptr);
1618 
1619  TRACE("[Thread %2u] rx %s from port %u\n",
1620  p->thread_id,
1621  pkt_received ? "1 pkt" : "0 pkts",
1622  p->port_id);
1623 
1624  t->mirroring_slots_mask = 0;
1625  t->recirc_pass_id = 0;
1626 
1627  /* Headers. */
1628  t->valid_headers = 0;
1629  t->n_headers_out = 0;
1630 
1631  /* Meta-data. */
1632  METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
1633 
1634  /* Tables. */
1635  t->table_state = p->table_state;
1636 
1637  /* Thread. */
1638  pipeline_port_inc(p);
1639 
1640  return pkt_received;
1641 }
1642 
1643 static inline void
1644 instr_rx_exec(struct rte_swx_pipeline *p)
1645 {
1646  struct thread *t = &p->threads[p->thread_id];
1647  struct instruction *ip = t->ip;
1648  int pkt_received;
1649 
1650  /* Packet. */
1651  pkt_received = __instr_rx_exec(p, t, ip);
1652 
1653  /* Thread. */
1654  thread_ip_inc_cond(t, pkt_received);
1655  thread_yield(p);
1656 }
1657 
1658 /*
1659  * tx.
1660  */
1661 static inline void
1662 emit_handler(struct thread *t)
1663 {
1664  struct header_out_runtime *h0 = &t->headers_out[0];
1665  struct header_out_runtime *h1 = &t->headers_out[1];
1666  uint32_t offset = 0, i;
1667 
1668  /* No header change or header decapsulation. */
1669  if ((t->n_headers_out == 1) &&
1670  (h0->ptr + h0->n_bytes == t->ptr)) {
1671  TRACE("Emit handler: no header change or header decap.\n");
1672 
1673  t->pkt.offset -= h0->n_bytes;
1674  t->pkt.length += h0->n_bytes;
1675 
1676  return;
1677  }
1678 
1679  /* Header encapsulation (optionally, with prior header decapsulation). */
1680  if ((t->n_headers_out == 2) &&
1681  (h1->ptr + h1->n_bytes == t->ptr) &&
1682  (h0->ptr == h0->ptr0)) {
1683  uint32_t offset;
1684 
1685  TRACE("Emit handler: header encapsulation.\n");
1686 
1687  offset = h0->n_bytes + h1->n_bytes;
1688  memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
1689  t->pkt.offset -= offset;
1690  t->pkt.length += offset;
1691 
1692  return;
1693  }
1694 
1695  /* For any other case. */
1696  TRACE("Emit handler: complex case.\n");
1697 
1698  for (i = 0; i < t->n_headers_out; i++) {
1699  struct header_out_runtime *h = &t->headers_out[i];
1700 
1701  memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
1702  offset += h->n_bytes;
1703  }
1704 
1705  if (offset) {
1706  memcpy(t->ptr - offset, t->header_out_storage, offset);
1707  t->pkt.offset -= offset;
1708  t->pkt.length += offset;
1709  }
1710 }
1711 
1712 static inline void
1713 mirroring_handler(struct rte_swx_pipeline *p, struct thread *t, struct rte_swx_pkt *pkt)
1714 {
1715  uint64_t slots_mask = t->mirroring_slots_mask, slot_mask;
1716  uint32_t slot_id;
1717 
1718  for (slot_id = 0, slot_mask = 1LLU ; slots_mask; slot_id++, slot_mask <<= 1)
1719  if (slot_mask & slots_mask) {
1720  struct port_out_runtime *port;
1721  struct mirroring_session *session;
1722  uint32_t port_id, session_id;
1723 
1724  session_id = t->mirroring_slots[slot_id];
1725  session = &p->mirroring_sessions[session_id];
1726 
1727  port_id = session->port_id;
1728  port = &p->out[port_id];
1729 
1730  if (session->fast_clone)
1731  port->pkt_fast_clone_tx(port->obj, pkt);
1732  else
1733  port->pkt_clone_tx(port->obj, pkt, session->truncation_length);
1734 
1735  slots_mask &= ~slot_mask;
1736  }
1737 }
1738 
1739 static inline void
1740 __instr_tx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1741 {
1742  uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1743  struct port_out_runtime *port = &p->out[port_id];
1744  struct rte_swx_pkt *pkt = &t->pkt;
1745 
1746  /* Recirculation: keep the current packet. */
1747  if (t->recirculate) {
1748  TRACE("[Thread %2u]: tx 1 pkt - recirculate\n",
1749  p->thread_id);
1750 
1751  /* Headers. */
1752  emit_handler(t);
1753 
1754  /* Packet. */
1755  mirroring_handler(p, t, pkt);
1756 
1757  return;
1758  }
1759 
1760  TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
1761  p->thread_id,
1762  (uint32_t)port_id);
1763 
1764  /* Headers. */
1765  emit_handler(t);
1766 
1767  /* Packet. */
1768  mirroring_handler(p, t, pkt);
1769  port->pkt_tx(port->obj, pkt);
1770 }
1771 
1772 static inline void
1773 __instr_tx_i_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1774 {
1775  uint64_t port_id = ip->io.io.val;
1776  struct port_out_runtime *port = &p->out[port_id];
1777  struct rte_swx_pkt *pkt = &t->pkt;
1778 
1779  /* Recirculation: keep the current packet. */
1780  if (t->recirculate) {
1781  TRACE("[Thread %2u]: tx (i) 1 pkt - recirculate\n",
1782  p->thread_id);
1783 
1784  /* Headers. */
1785  emit_handler(t);
1786 
1787  /* Packet. */
1788  mirroring_handler(p, t, pkt);
1789 
1790  return;
1791  }
1792 
1793  TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
1794  p->thread_id,
1795  (uint32_t)port_id);
1796 
1797  /* Headers. */
1798  emit_handler(t);
1799 
1800  /* Packet. */
1801  mirroring_handler(p, t, pkt);
1802  port->pkt_tx(port->obj, pkt);
1803 }
1804 
1805 static inline void
1806 __instr_drop_exec(struct rte_swx_pipeline *p,
1807  struct thread *t,
1808  const struct instruction *ip __rte_unused)
1809 {
1810  uint64_t port_id = p->n_ports_out - 1;
1811  struct port_out_runtime *port = &p->out[port_id];
1812  struct rte_swx_pkt *pkt = &t->pkt;
1813 
1814  TRACE("[Thread %2u]: drop 1 pkt\n",
1815  p->thread_id);
1816 
1817  /* Headers. */
1818  emit_handler(t);
1819 
1820  /* Packet. */
1821  mirroring_handler(p, t, pkt);
1822  port->pkt_tx(port->obj, pkt);
1823 }
1824 
1825 static inline void
1826 __instr_mirror_exec(struct rte_swx_pipeline *p,
1827  struct thread *t,
1828  const struct instruction *ip)
1829 {
1830  uint64_t slot_id = instr_operand_hbo(t, &ip->mirror.dst);
1831  uint64_t session_id = instr_operand_hbo(t, &ip->mirror.src);
1832 
1833  slot_id &= p->n_mirroring_slots - 1;
1834  session_id &= p->n_mirroring_sessions - 1;
1835 
1836  TRACE("[Thread %2u]: mirror pkt (slot = %u, session = %u)\n",
1837  p->thread_id,
1838  (uint32_t)slot_id,
1839  (uint32_t)session_id);
1840 
1841  t->mirroring_slots[slot_id] = session_id;
1842  t->mirroring_slots_mask |= 1LLU << slot_id;
1843 }
1844 
1845 static inline void
1846 __instr_recirculate_exec(struct rte_swx_pipeline *p __rte_unused,
1847  struct thread *t,
1848  const struct instruction *ip __rte_unused)
1849 {
1850  TRACE("[Thread %2u]: recirculate\n",
1851  p->thread_id);
1852 
1853  t->recirculate = 1;
1854 }
1855 
1856 static inline void
1857 __instr_recircid_exec(struct rte_swx_pipeline *p __rte_unused,
1858  struct thread *t,
1859  const struct instruction *ip)
1860 {
1861  TRACE("[Thread %2u]: recircid (pass %u)\n",
1862  p->thread_id,
1863  t->recirc_pass_id);
1864 
1865  /* Meta-data. */
1866  METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, t->recirc_pass_id);
1867 }
1868 
1869 /*
1870  * extract.
1871  */
1872 static inline void
1873 __instr_hdr_extract_many_exec(struct rte_swx_pipeline *p __rte_unused,
1874  struct thread *t,
1875  const struct instruction *ip,
1876  uint32_t n_extract)
1877 {
1878  uint64_t valid_headers = t->valid_headers;
1879  uint8_t *ptr = t->ptr;
1880  uint32_t offset = t->pkt.offset;
1881  uint32_t length = t->pkt.length;
1882  uint32_t i;
1883 
1884  for (i = 0; i < n_extract; i++) {
1885  uint32_t header_id = ip->io.hdr.header_id[i];
1886  uint32_t struct_id = ip->io.hdr.struct_id[i];
1887  uint32_t n_bytes = ip->io.hdr.n_bytes[i];
1888 
1889  TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
1890  p->thread_id,
1891  header_id,
1892  n_bytes);
1893 
1894  /* Headers. */
1895  t->structs[struct_id] = ptr;
1896  valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1897 
1898  /* Packet. */
1899  offset += n_bytes;
1900  length -= n_bytes;
1901  ptr += n_bytes;
1902  }
1903 
1904  /* Headers. */
1905  t->valid_headers = valid_headers;
1906 
1907  /* Packet. */
1908  t->pkt.offset = offset;
1909  t->pkt.length = length;
1910  t->ptr = ptr;
1911 }
1912 
1913 static inline void
1914 __instr_hdr_extract_exec(struct rte_swx_pipeline *p,
1915  struct thread *t,
1916  const struct instruction *ip)
1917 {
1918  __instr_hdr_extract_many_exec(p, t, ip, 1);
1919 }
1920 
1921 static inline void
1922 __instr_hdr_extract2_exec(struct rte_swx_pipeline *p,
1923  struct thread *t,
1924  const struct instruction *ip)
1925 {
1926  TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
1927 
1928  __instr_hdr_extract_many_exec(p, t, ip, 2);
1929 }
1930 
1931 static inline void
1932 __instr_hdr_extract3_exec(struct rte_swx_pipeline *p,
1933  struct thread *t,
1934  const struct instruction *ip)
1935 {
1936  TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
1937 
1938  __instr_hdr_extract_many_exec(p, t, ip, 3);
1939 }
1940 
1941 static inline void
1942 __instr_hdr_extract4_exec(struct rte_swx_pipeline *p,
1943  struct thread *t,
1944  const struct instruction *ip)
1945 {
1946  TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
1947 
1948  __instr_hdr_extract_many_exec(p, t, ip, 4);
1949 }
1950 
1951 static inline void
1952 __instr_hdr_extract5_exec(struct rte_swx_pipeline *p,
1953  struct thread *t,
1954  const struct instruction *ip)
1955 {
1956  TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
1957 
1958  __instr_hdr_extract_many_exec(p, t, ip, 5);
1959 }
1960 
1961 static inline void
1962 __instr_hdr_extract6_exec(struct rte_swx_pipeline *p,
1963  struct thread *t,
1964  const struct instruction *ip)
1965 {
1966  TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
1967 
1968  __instr_hdr_extract_many_exec(p, t, ip, 6);
1969 }
1970 
1971 static inline void
1972 __instr_hdr_extract7_exec(struct rte_swx_pipeline *p,
1973  struct thread *t,
1974  const struct instruction *ip)
1975 {
1976  TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
1977 
1978  __instr_hdr_extract_many_exec(p, t, ip, 7);
1979 }
1980 
1981 static inline void
1982 __instr_hdr_extract8_exec(struct rte_swx_pipeline *p,
1983  struct thread *t,
1984  const struct instruction *ip)
1985 {
1986  TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
1987 
1988  __instr_hdr_extract_many_exec(p, t, ip, 8);
1989 }
1990 
1991 static inline void
1992 __instr_hdr_extract_m_exec(struct rte_swx_pipeline *p __rte_unused,
1993  struct thread *t,
1994  const struct instruction *ip)
1995 {
1996  uint64_t valid_headers = t->valid_headers;
1997  uint8_t *ptr = t->ptr;
1998  uint32_t offset = t->pkt.offset;
1999  uint32_t length = t->pkt.length;
2000 
2001  uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2002  uint32_t header_id = ip->io.hdr.header_id[0];
2003  uint32_t struct_id = ip->io.hdr.struct_id[0];
2004  uint32_t n_bytes = ip->io.hdr.n_bytes[0];
2005 
2006  struct header_runtime *h = &t->headers[header_id];
2007 
2008  TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n",
2009  p->thread_id,
2010  header_id,
2011  n_bytes,
2012  n_bytes_last);
2013 
2014  n_bytes += n_bytes_last;
2015 
2016  /* Headers. */
2017  t->structs[struct_id] = ptr;
2018  t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2019  h->n_bytes = n_bytes;
2020 
2021  /* Packet. */
2022  t->pkt.offset = offset + n_bytes;
2023  t->pkt.length = length - n_bytes;
2024  t->ptr = ptr + n_bytes;
2025 }
2026 
2027 static inline void
2028 __instr_hdr_lookahead_exec(struct rte_swx_pipeline *p __rte_unused,
2029  struct thread *t,
2030  const struct instruction *ip)
2031 {
2032  uint64_t valid_headers = t->valid_headers;
2033  uint8_t *ptr = t->ptr;
2034 
2035  uint32_t header_id = ip->io.hdr.header_id[0];
2036  uint32_t struct_id = ip->io.hdr.struct_id[0];
2037 
2038  TRACE("[Thread %2u]: lookahead header %u\n",
2039  p->thread_id,
2040  header_id);
2041 
2042  /* Headers. */
2043  t->structs[struct_id] = ptr;
2044  t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2045 }
2046 
2047 /*
2048  * emit.
2049  */
2050 static inline void
2051 __instr_hdr_emit_many_exec(struct rte_swx_pipeline *p __rte_unused,
2052  struct thread *t,
2053  const struct instruction *ip,
2054  uint32_t n_emit)
2055 {
2056  uint64_t valid_headers = t->valid_headers;
2057  uint32_t n_headers_out = t->n_headers_out;
2058  struct header_out_runtime *ho = NULL;
2059  uint8_t *ho_ptr = NULL;
2060  uint32_t ho_nbytes = 0, i;
2061 
2062  for (i = 0; i < n_emit; i++) {
2063  uint32_t header_id = ip->io.hdr.header_id[i];
2064  uint32_t struct_id = ip->io.hdr.struct_id[i];
2065 
2066  struct header_runtime *hi = &t->headers[header_id];
2067  uint8_t *hi_ptr0 = hi->ptr0;
2068  uint32_t n_bytes = hi->n_bytes;
2069 
2070  uint8_t *hi_ptr = t->structs[struct_id];
2071 
2072  if (!MASK64_BIT_GET(valid_headers, header_id)) {
2073  TRACE("[Thread %2u]: emit header %u (invalid)\n",
2074  p->thread_id,
2075  header_id);
2076 
2077  continue;
2078  }
2079 
2080  TRACE("[Thread %2u]: emit header %u (valid)\n",
2081  p->thread_id,
2082  header_id);
2083 
2084  /* Headers. */
2085  if (!ho) {
2086  if (!n_headers_out) {
2087  ho = &t->headers_out[0];
2088 
2089  ho->ptr0 = hi_ptr0;
2090  ho->ptr = hi_ptr;
2091 
2092  ho_ptr = hi_ptr;
2093  ho_nbytes = n_bytes;
2094 
2095  n_headers_out = 1;
2096 
2097  continue;
2098  } else {
2099  ho = &t->headers_out[n_headers_out - 1];
2100 
2101  ho_ptr = ho->ptr;
2102  ho_nbytes = ho->n_bytes;
2103  }
2104  }
2105 
2106  if (ho_ptr + ho_nbytes == hi_ptr) {
2107  ho_nbytes += n_bytes;
2108  } else {
2109  ho->n_bytes = ho_nbytes;
2110 
2111  ho++;
2112  ho->ptr0 = hi_ptr0;
2113  ho->ptr = hi_ptr;
2114 
2115  ho_ptr = hi_ptr;
2116  ho_nbytes = n_bytes;
2117 
2118  n_headers_out++;
2119  }
2120  }
2121 
2122  if (ho)
2123  ho->n_bytes = ho_nbytes;
2124  t->n_headers_out = n_headers_out;
2125 }
2126 
2127 static inline void
2128 __instr_hdr_emit_exec(struct rte_swx_pipeline *p,
2129  struct thread *t,
2130  const struct instruction *ip)
2131 {
2132  __instr_hdr_emit_many_exec(p, t, ip, 1);
2133 }
2134 
2135 static inline void
2136 __instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p,
2137  struct thread *t,
2138  const struct instruction *ip)
2139 {
2140  TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
2141 
2142  __instr_hdr_emit_many_exec(p, t, ip, 1);
2143  __instr_tx_exec(p, t, ip);
2144 }
2145 
2146 static inline void
2147 __instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p,
2148  struct thread *t,
2149  const struct instruction *ip)
2150 {
2151  TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
2152 
2153  __instr_hdr_emit_many_exec(p, t, ip, 2);
2154  __instr_tx_exec(p, t, ip);
2155 }
2156 
2157 static inline void
2158 __instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p,
2159  struct thread *t,
2160  const struct instruction *ip)
2161 {
2162  TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2163 
2164  __instr_hdr_emit_many_exec(p, t, ip, 3);
2165  __instr_tx_exec(p, t, ip);
2166 }
2167 
2168 static inline void
2169 __instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p,
2170  struct thread *t,
2171  const struct instruction *ip)
2172 {
2173  TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2174 
2175  __instr_hdr_emit_many_exec(p, t, ip, 4);
2176  __instr_tx_exec(p, t, ip);
2177 }
2178 
2179 static inline void
2180 __instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p,
2181  struct thread *t,
2182  const struct instruction *ip)
2183 {
2184  TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2185 
2186  __instr_hdr_emit_many_exec(p, t, ip, 5);
2187  __instr_tx_exec(p, t, ip);
2188 }
2189 
2190 static inline void
2191 __instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p,
2192  struct thread *t,
2193  const struct instruction *ip)
2194 {
2195  TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2196 
2197  __instr_hdr_emit_many_exec(p, t, ip, 6);
2198  __instr_tx_exec(p, t, ip);
2199 }
2200 
2201 static inline void
2202 __instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p,
2203  struct thread *t,
2204  const struct instruction *ip)
2205 {
2206  TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2207 
2208  __instr_hdr_emit_many_exec(p, t, ip, 7);
2209  __instr_tx_exec(p, t, ip);
2210 }
2211 
2212 static inline void
2213 __instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p,
2214  struct thread *t,
2215  const struct instruction *ip)
2216 {
2217  TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n", p->thread_id);
2218 
2219  __instr_hdr_emit_many_exec(p, t, ip, 8);
2220  __instr_tx_exec(p, t, ip);
2221 }
2222 
2223 /*
2224  * validate.
2225  */
2226 static inline void
2227 __instr_hdr_validate_exec(struct rte_swx_pipeline *p __rte_unused,
2228  struct thread *t,
2229  const struct instruction *ip)
2230 {
2231  uint32_t header_id = ip->valid.header_id;
2232  uint32_t struct_id = ip->valid.struct_id;
2233  uint64_t valid_headers = t->valid_headers;
2234  struct header_runtime *h = &t->headers[header_id];
2235 
2236  TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2237 
2238  /* If this header is already valid, then its associated t->structs[] element is also valid
2239  * and therefore it should not be modified. It could point to the packet buffer (in case of
2240  * extracted header) and setting it to the default location (h->ptr0) would be incorrect.
2241  */
2242  if (MASK64_BIT_GET(valid_headers, header_id))
2243  return;
2244 
2245  /* Headers. */
2246  t->structs[struct_id] = h->ptr0;
2247  t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2248 }
2249 
2250 /*
2251  * invalidate.
2252  */
2253 static inline void
2254 __instr_hdr_invalidate_exec(struct rte_swx_pipeline *p __rte_unused,
2255  struct thread *t,
2256  const struct instruction *ip)
2257 {
2258  uint32_t header_id = ip->valid.header_id;
2259 
2260  TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2261 
2262  /* Headers. */
2263  t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2264 }
2265 
2266 /*
2267  * learn.
2268  */
2269 static inline void
2270 __instr_learn_exec(struct rte_swx_pipeline *p,
2271  struct thread *t,
2272  const struct instruction *ip)
2273 {
2274  uint64_t action_id = ip->learn.action_id;
2275  uint32_t mf_first_arg_offset = ip->learn.mf_first_arg_offset;
2276  uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset,
2277  ip->learn.mf_timeout_id_n_bits);
2278  uint32_t learner_id = t->learner_id;
2279  struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2280  p->n_selectors + learner_id];
2281  struct learner_runtime *l = &t->learners[learner_id];
2282  struct learner_statistics *stats = &p->learner_stats[learner_id];
2283  uint32_t status;
2284 
2285  /* Table. */
2286  status = rte_swx_table_learner_add(ts->obj,
2287  l->mailbox,
2288  t->time,
2289  action_id,
2290  &t->metadata[mf_first_arg_offset],
2291  timeout_id);
2292 
2293  TRACE("[Thread %2u] learner %u learn %s\n",
2294  p->thread_id,
2295  learner_id,
2296  status ? "ok" : "error");
2297 
2298  stats->n_pkts_learn[status] += 1;
2299 }
2300 
2301 /*
2302  * rearm.
2303  */
2304 static inline void
2305 __instr_rearm_exec(struct rte_swx_pipeline *p,
2306  struct thread *t,
2307  const struct instruction *ip __rte_unused)
2308 {
2309  uint32_t learner_id = t->learner_id;
2310  struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2311  p->n_selectors + learner_id];
2312  struct learner_runtime *l = &t->learners[learner_id];
2313  struct learner_statistics *stats = &p->learner_stats[learner_id];
2314 
2315  /* Table. */
2316  rte_swx_table_learner_rearm(ts->obj, l->mailbox, t->time);
2317 
2318  TRACE("[Thread %2u] learner %u rearm\n",
2319  p->thread_id,
2320  learner_id);
2321 
2322  stats->n_pkts_rearm += 1;
2323 }
2324 
2325 static inline void
2326 __instr_rearm_new_exec(struct rte_swx_pipeline *p,
2327  struct thread *t,
2328  const struct instruction *ip)
2329 {
2330  uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset,
2331  ip->learn.mf_timeout_id_n_bits);
2332  uint32_t learner_id = t->learner_id;
2333  struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2334  p->n_selectors + learner_id];
2335  struct learner_runtime *l = &t->learners[learner_id];
2336  struct learner_statistics *stats = &p->learner_stats[learner_id];
2337 
2338  /* Table. */
2339  rte_swx_table_learner_rearm_new(ts->obj, l->mailbox, t->time, timeout_id);
2340 
2341  TRACE("[Thread %2u] learner %u rearm with timeout ID %u\n",
2342  p->thread_id,
2343  learner_id,
2344  timeout_id);
2345 
2346  stats->n_pkts_rearm += 1;
2347 }
2348 
2349 /*
2350  * forget.
2351  */
2352 static inline void
2353 __instr_forget_exec(struct rte_swx_pipeline *p,
2354  struct thread *t,
2355  const struct instruction *ip __rte_unused)
2356 {
2357  uint32_t learner_id = t->learner_id;
2358  struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2359  p->n_selectors + learner_id];
2360  struct learner_runtime *l = &t->learners[learner_id];
2361  struct learner_statistics *stats = &p->learner_stats[learner_id];
2362 
2363  /* Table. */
2364  rte_swx_table_learner_delete(ts->obj, l->mailbox);
2365 
2366  TRACE("[Thread %2u] learner %u forget\n",
2367  p->thread_id,
2368  learner_id);
2369 
2370  stats->n_pkts_forget += 1;
2371 }
2372 
2373 /*
2374  * extern.
2375  */
2376 static inline uint32_t
2377 __instr_extern_obj_exec(struct rte_swx_pipeline *p __rte_unused,
2378  struct thread *t,
2379  const struct instruction *ip)
2380 {
2381  uint32_t obj_id = ip->ext_obj.ext_obj_id;
2382  uint32_t func_id = ip->ext_obj.func_id;
2383  struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
2384  rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
2385  uint32_t done;
2386 
2387  TRACE("[Thread %2u] extern obj %u member func %u\n",
2388  p->thread_id,
2389  obj_id,
2390  func_id);
2391 
2392  done = func(obj->obj, obj->mailbox);
2393 
2394  return done;
2395 }
2396 
2397 static inline uint32_t
2398 __instr_extern_func_exec(struct rte_swx_pipeline *p __rte_unused,
2399  struct thread *t,
2400  const struct instruction *ip)
2401 {
2402  uint32_t ext_func_id = ip->ext_func.ext_func_id;
2403  struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
2404  rte_swx_extern_func_t func = ext_func->func;
2405  uint32_t done;
2406 
2407  TRACE("[Thread %2u] extern func %u\n",
2408  p->thread_id,
2409  ext_func_id);
2410 
2411  done = func(ext_func->mailbox);
2412 
2413  return done;
2414 }
2415 
2416 /*
2417  * hash.
2418  */
2419 static inline void
2420 __instr_hash_func_exec(struct rte_swx_pipeline *p,
2421  struct thread *t,
2422  const struct instruction *ip)
2423 {
2424  uint32_t hash_func_id = ip->hash_func.hash_func_id;
2425  uint32_t dst_offset = ip->hash_func.dst.offset;
2426  uint32_t n_dst_bits = ip->hash_func.dst.n_bits;
2427  uint32_t src_struct_id = ip->hash_func.src.struct_id;
2428  uint32_t src_offset = ip->hash_func.src.offset;
2429  uint32_t n_src_bytes = ip->hash_func.src.n_bytes;
2430 
2431  struct hash_func_runtime *func = &p->hash_func_runtime[hash_func_id];
2432  uint8_t *src_ptr = t->structs[src_struct_id];
2433  uint32_t result;
2434 
2435  TRACE("[Thread %2u] hash %u\n",
2436  p->thread_id,
2437  hash_func_id);
2438 
2439  result = func->func(&src_ptr[src_offset], n_src_bytes, 0);
2440  METADATA_WRITE(t, dst_offset, n_dst_bits, result);
2441 }
2442 
2443 /*
2444  * mov.
2445  */
2446 static inline void
2447 __instr_mov_exec(struct rte_swx_pipeline *p __rte_unused,
2448  struct thread *t,
2449  const struct instruction *ip)
2450 {
2451  TRACE("[Thread %2u] mov\n", p->thread_id);
2452 
2453  MOV(t, ip);
2454 }
2455 
2456 static inline void
2457 __instr_mov_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2458  struct thread *t,
2459  const struct instruction *ip)
2460 {
2461  TRACE("[Thread %2u] mov (mh)\n", p->thread_id);
2462 
2463  MOV_MH(t, ip);
2464 }
2465 
2466 static inline void
2467 __instr_mov_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2468  struct thread *t,
2469  const struct instruction *ip)
2470 {
2471  TRACE("[Thread %2u] mov (hm)\n", p->thread_id);
2472 
2473  MOV_HM(t, ip);
2474 }
2475 
2476 static inline void
2477 __instr_mov_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2478  struct thread *t,
2479  const struct instruction *ip)
2480 {
2481  TRACE("[Thread %2u] mov (hh)\n", p->thread_id);
2482 
2483  MOV_HH(t, ip);
2484 }
2485 
2486 static inline void
2487 __instr_mov_i_exec(struct rte_swx_pipeline *p __rte_unused,
2488  struct thread *t,
2489  const struct instruction *ip)
2490 {
2491  TRACE("[Thread %2u] mov m.f %" PRIx64 "\n", p->thread_id, ip->mov.src_val);
2492 
2493  MOV_I(t, ip);
2494 }
2495 
2496 /*
2497  * dma.
2498  */
2499 static inline void
2500 __instr_dma_ht_many_exec(struct rte_swx_pipeline *p __rte_unused,
2501  struct thread *t,
2502  const struct instruction *ip,
2503  uint32_t n_dma)
2504 {
2505  uint8_t *action_data = t->structs[0];
2506  uint64_t valid_headers = t->valid_headers;
2507  uint32_t i;
2508 
2509  for (i = 0; i < n_dma; i++) {
2510  uint32_t header_id = ip->dma.dst.header_id[i];
2511  uint32_t struct_id = ip->dma.dst.struct_id[i];
2512  uint32_t offset = ip->dma.src.offset[i];
2513  uint32_t n_bytes = ip->dma.n_bytes[i];
2514 
2515  struct header_runtime *h = &t->headers[header_id];
2516  uint8_t *h_ptr0 = h->ptr0;
2517  uint8_t *h_ptr = t->structs[struct_id];
2518 
2519  void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2520  h_ptr : h_ptr0;
2521  void *src = &action_data[offset];
2522 
2523  TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2524 
2525  /* Headers. */
2526  memcpy(dst, src, n_bytes);
2527  t->structs[struct_id] = dst;
2528  valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2529  }
2530 
2531  t->valid_headers = valid_headers;
2532 }
2533 
2534 static inline void
2535 __instr_dma_ht_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2536 {
2537  __instr_dma_ht_many_exec(p, t, ip, 1);
2538 }
2539 
2540 static inline void
2541 __instr_dma_ht2_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2542 {
2543  TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
2544 
2545  __instr_dma_ht_many_exec(p, t, ip, 2);
2546 }
2547 
2548 static inline void
2549 __instr_dma_ht3_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2550 {
2551  TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
2552 
2553  __instr_dma_ht_many_exec(p, t, ip, 3);
2554 }
2555 
2556 static inline void
2557 __instr_dma_ht4_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2558 {
2559  TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2560 
2561  __instr_dma_ht_many_exec(p, t, ip, 4);
2562 }
2563 
2564 static inline void
2565 __instr_dma_ht5_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2566 {
2567  TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2568 
2569  __instr_dma_ht_many_exec(p, t, ip, 5);
2570 }
2571 
2572 static inline void
2573 __instr_dma_ht6_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2574 {
2575  TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2576 
2577  __instr_dma_ht_many_exec(p, t, ip, 6);
2578 }
2579 
2580 static inline void
2581 __instr_dma_ht7_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2582 {
2583  TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2584 
2585  __instr_dma_ht_many_exec(p, t, ip, 7);
2586 }
2587 
2588 static inline void
2589 __instr_dma_ht8_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2590 {
2591  TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2592 
2593  __instr_dma_ht_many_exec(p, t, ip, 8);
2594 }
2595 
2596 /*
2597  * alu.
2598  */
2599 static inline void
2600 __instr_alu_add_exec(struct rte_swx_pipeline *p __rte_unused,
2601  struct thread *t,
2602  const struct instruction *ip)
2603 {
2604  TRACE("[Thread %2u] add\n", p->thread_id);
2605 
2606  ALU(t, ip, +);
2607 }
2608 
2609 static inline void
2610 __instr_alu_add_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2611  struct thread *t,
2612  const struct instruction *ip)
2613 {
2614  TRACE("[Thread %2u] add (mh)\n", p->thread_id);
2615 
2616  ALU_MH(t, ip, +);
2617 }
2618 
2619 static inline void
2620 __instr_alu_add_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2621  struct thread *t,
2622  const struct instruction *ip)
2623 {
2624  TRACE("[Thread %2u] add (hm)\n", p->thread_id);
2625 
2626  ALU_HM(t, ip, +);
2627 }
2628 
2629 static inline void
2630 __instr_alu_add_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2631  struct thread *t,
2632  const struct instruction *ip)
2633 {
2634  TRACE("[Thread %2u] add (hh)\n", p->thread_id);
2635 
2636  ALU_HH(t, ip, +);
2637 }
2638 
2639 static inline void
2640 __instr_alu_add_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2641  struct thread *t,
2642  const struct instruction *ip)
2643 {
2644  TRACE("[Thread %2u] add (mi)\n", p->thread_id);
2645 
2646  ALU_MI(t, ip, +);
2647 }
2648 
2649 static inline void
2650 __instr_alu_add_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2651  struct thread *t,
2652  const struct instruction *ip)
2653 {
2654  TRACE("[Thread %2u] add (hi)\n", p->thread_id);
2655 
2656  ALU_HI(t, ip, +);
2657 }
2658 
2659 static inline void
2660 __instr_alu_sub_exec(struct rte_swx_pipeline *p __rte_unused,
2661  struct thread *t,
2662  const struct instruction *ip)
2663 {
2664  TRACE("[Thread %2u] sub\n", p->thread_id);
2665 
2666  ALU(t, ip, -);
2667 }
2668 
2669 static inline void
2670 __instr_alu_sub_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2671  struct thread *t,
2672  const struct instruction *ip)
2673 {
2674  TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
2675 
2676  ALU_MH(t, ip, -);
2677 }
2678 
2679 static inline void
2680 __instr_alu_sub_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2681  struct thread *t,
2682  const struct instruction *ip)
2683 {
2684  TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
2685 
2686  ALU_HM(t, ip, -);
2687 }
2688 
2689 static inline void
2690 __instr_alu_sub_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2691  struct thread *t,
2692  const struct instruction *ip)
2693 {
2694  TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
2695 
2696  ALU_HH(t, ip, -);
2697 }
2698 
2699 static inline void
2700 __instr_alu_sub_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2701  struct thread *t,
2702  const struct instruction *ip)
2703 {
2704  TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
2705 
2706  ALU_MI(t, ip, -);
2707 }
2708 
2709 static inline void
2710 __instr_alu_sub_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2711  struct thread *t,
2712  const struct instruction *ip)
2713 {
2714  TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
2715 
2716  ALU_HI(t, ip, -);
2717 }
2718 
2719 static inline void
2720 __instr_alu_shl_exec(struct rte_swx_pipeline *p __rte_unused,
2721  struct thread *t,
2722  const struct instruction *ip)
2723 {
2724  TRACE("[Thread %2u] shl\n", p->thread_id);
2725 
2726  ALU(t, ip, <<);
2727 }
2728 
2729 static inline void
2730 __instr_alu_shl_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2731  struct thread *t,
2732  const struct instruction *ip)
2733 {
2734  TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
2735 
2736  ALU_MH(t, ip, <<);
2737 }
2738 
2739 static inline void
2740 __instr_alu_shl_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2741  struct thread *t,
2742  const struct instruction *ip)
2743 {
2744  TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
2745 
2746  ALU_HM(t, ip, <<);
2747 }
2748 
2749 static inline void
2750 __instr_alu_shl_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2751  struct thread *t,
2752  const struct instruction *ip)
2753 {
2754  TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
2755 
2756  ALU_HH(t, ip, <<);
2757 }
2758 
2759 static inline void
2760 __instr_alu_shl_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2761  struct thread *t,
2762  const struct instruction *ip)
2763 {
2764  TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
2765 
2766  ALU_MI(t, ip, <<);
2767 }
2768 
2769 static inline void
2770 __instr_alu_shl_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2771  struct thread *t,
2772  const struct instruction *ip)
2773 {
2774  TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
2775 
2776  ALU_HI(t, ip, <<);
2777 }
2778 
2779 static inline void
2780 __instr_alu_shr_exec(struct rte_swx_pipeline *p __rte_unused,
2781  struct thread *t,
2782  const struct instruction *ip)
2783 {
2784  TRACE("[Thread %2u] shr\n", p->thread_id);
2785 
2786  ALU(t, ip, >>);
2787 }
2788 
2789 static inline void
2790 __instr_alu_shr_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2791  struct thread *t,
2792  const struct instruction *ip)
2793 {
2794  TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
2795 
2796  ALU_MH(t, ip, >>);
2797 }
2798 
2799 static inline void
2800 __instr_alu_shr_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2801  struct thread *t,
2802  const struct instruction *ip)
2803 {
2804  TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
2805 
2806  ALU_HM(t, ip, >>);
2807 }
2808 
2809 static inline void
2810 __instr_alu_shr_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2811  struct thread *t,
2812  const struct instruction *ip)
2813 {
2814  TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
2815 
2816  ALU_HH(t, ip, >>);
2817 }
2818 
2819 static inline void
2820 __instr_alu_shr_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2821  struct thread *t,
2822  const struct instruction *ip)
2823 {
2824  TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
2825 
2826  /* Structs. */
2827  ALU_MI(t, ip, >>);
2828 }
2829 
2830 static inline void
2831 __instr_alu_shr_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2832  struct thread *t,
2833  const struct instruction *ip)
2834 {
2835  TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
2836 
2837  ALU_HI(t, ip, >>);
2838 }
2839 
2840 static inline void
2841 __instr_alu_and_exec(struct rte_swx_pipeline *p __rte_unused,
2842  struct thread *t,
2843  const struct instruction *ip)
2844 {
2845  TRACE("[Thread %2u] and\n", p->thread_id);
2846 
2847  ALU(t, ip, &);
2848 }
2849 
2850 static inline void
2851 __instr_alu_and_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2852  struct thread *t,
2853  const struct instruction *ip)
2854 {
2855  TRACE("[Thread %2u] and (mh)\n", p->thread_id);
2856 
2857  ALU_MH(t, ip, &);
2858 }
2859 
2860 static inline void
2861 __instr_alu_and_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2862  struct thread *t,
2863  const struct instruction *ip)
2864 {
2865  TRACE("[Thread %2u] and (hm)\n", p->thread_id);
2866 
2867  ALU_HM_FAST(t, ip, &);
2868 }
2869 
2870 static inline void
2871 __instr_alu_and_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2872  struct thread *t,
2873  const struct instruction *ip)
2874 {
2875  TRACE("[Thread %2u] and (hh)\n", p->thread_id);
2876 
2877  ALU_HH_FAST(t, ip, &);
2878 }
2879 
2880 static inline void
2881 __instr_alu_and_i_exec(struct rte_swx_pipeline *p __rte_unused,
2882  struct thread *t,
2883  const struct instruction *ip)
2884 {
2885  TRACE("[Thread %2u] and (i)\n", p->thread_id);
2886 
2887  ALU_I(t, ip, &);
2888 }
2889 
2890 static inline void
2891 __instr_alu_or_exec(struct rte_swx_pipeline *p __rte_unused,
2892  struct thread *t,
2893  const struct instruction *ip)
2894 {
2895  TRACE("[Thread %2u] or\n", p->thread_id);
2896 
2897  ALU(t, ip, |);
2898 }
2899 
2900 static inline void
2901 __instr_alu_or_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2902  struct thread *t,
2903  const struct instruction *ip)
2904 {
2905  TRACE("[Thread %2u] or (mh)\n", p->thread_id);
2906 
2907  ALU_MH(t, ip, |);
2908 }
2909 
2910 static inline void
2911 __instr_alu_or_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2912  struct thread *t,
2913  const struct instruction *ip)
2914 {
2915  TRACE("[Thread %2u] or (hm)\n", p->thread_id);
2916 
2917  ALU_HM_FAST(t, ip, |);
2918 }
2919 
2920 static inline void
2921 __instr_alu_or_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2922  struct thread *t,
2923  const struct instruction *ip)
2924 {
2925  TRACE("[Thread %2u] or (hh)\n", p->thread_id);
2926 
2927  ALU_HH_FAST(t, ip, |);
2928 }
2929 
2930 static inline void
2931 __instr_alu_or_i_exec(struct rte_swx_pipeline *p __rte_unused,
2932  struct thread *t,
2933  const struct instruction *ip)
2934 {
2935  TRACE("[Thread %2u] or (i)\n", p->thread_id);
2936 
2937  ALU_I(t, ip, |);
2938 }
2939 
2940 static inline void
2941 __instr_alu_xor_exec(struct rte_swx_pipeline *p __rte_unused,
2942  struct thread *t,
2943  const struct instruction *ip)
2944 {
2945  TRACE("[Thread %2u] xor\n", p->thread_id);
2946 
2947  ALU(t, ip, ^);
2948 }
2949 
2950 static inline void
2951 __instr_alu_xor_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2952  struct thread *t,
2953  const struct instruction *ip)
2954 {
2955  TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
2956 
2957  ALU_MH(t, ip, ^);
2958 }
2959 
2960 static inline void
2961 __instr_alu_xor_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2962  struct thread *t,
2963  const struct instruction *ip)
2964 {
2965  TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
2966 
2967  ALU_HM_FAST(t, ip, ^);
2968 }
2969 
2970 static inline void
2971 __instr_alu_xor_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2972  struct thread *t,
2973  const struct instruction *ip)
2974 {
2975  TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
2976 
2977  ALU_HH_FAST(t, ip, ^);
2978 }
2979 
2980 static inline void
2981 __instr_alu_xor_i_exec(struct rte_swx_pipeline *p __rte_unused,
2982  struct thread *t,
2983  const struct instruction *ip)
2984 {
2985  TRACE("[Thread %2u] xor (i)\n", p->thread_id);
2986 
2987  ALU_I(t, ip, ^);
2988 }
2989 
2990 static inline void
2991 __instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p __rte_unused,
2992  struct thread *t,
2993  const struct instruction *ip)
2994 {
2995  uint8_t *dst_struct, *src_struct;
2996  uint16_t *dst16_ptr, dst;
2997  uint64_t *src64_ptr, src64, src64_mask, src;
2998  uint64_t r;
2999 
3000  TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3001 
3002  /* Structs. */
3003  dst_struct = t->structs[ip->alu.dst.struct_id];
3004  dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3005  dst = *dst16_ptr;
3006 
3007  src_struct = t->structs[ip->alu.src.struct_id];
3008  src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3009  src64 = *src64_ptr;
3010  src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3011  src = src64 & src64_mask;
3012 
3013  /* Initialize the result with destination 1's complement. */
3014  r = dst;
3015  r = ~r & 0xFFFF;
3016 
3017  /* The first input (r) is a 16-bit number. The second and the third
3018  * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3019  * three numbers (output r) is a 34-bit number.
3020  */
3021  r += (src >> 32) + (src & 0xFFFFFFFF);
3022 
3023  /* The first input is a 16-bit number. The second input is an 18-bit
3024  * number. In the worst case scenario, the sum of the two numbers is a
3025  * 19-bit number.
3026  */
3027  r = (r & 0xFFFF) + (r >> 16);
3028 
3029  /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3030  * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3031  */
3032  r = (r & 0xFFFF) + (r >> 16);
3033 
3034  /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3035  * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3036  * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3037  * therefore the output r is always a 16-bit number.
3038  */
3039  r = (r & 0xFFFF) + (r >> 16);
3040 
3041  /* Apply 1's complement to the result. */
3042  r = ~r & 0xFFFF;
3043  r = r ? r : 0xFFFF;
3044 
3045  *dst16_ptr = (uint16_t)r;
3046 }
3047 
3048 static inline void
3049 __instr_alu_cksub_field_exec(struct rte_swx_pipeline *p __rte_unused,
3050  struct thread *t,
3051  const struct instruction *ip)
3052 {
3053  uint8_t *dst_struct, *src_struct;
3054  uint16_t *dst16_ptr, dst;
3055  uint64_t *src64_ptr, src64, src64_mask, src;
3056  uint64_t r;
3057 
3058  TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
3059 
3060  /* Structs. */
3061  dst_struct = t->structs[ip->alu.dst.struct_id];
3062  dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3063  dst = *dst16_ptr;
3064 
3065  src_struct = t->structs[ip->alu.src.struct_id];
3066  src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3067  src64 = *src64_ptr;
3068  src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3069  src = src64 & src64_mask;
3070 
3071  /* Initialize the result with destination 1's complement. */
3072  r = dst;
3073  r = ~r & 0xFFFF;
3074 
3075  /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
3076  * the following sequence of operations in 2's complement arithmetic:
3077  * a '- b = (a - b) % 0xFFFF.
3078  *
3079  * In order to prevent an underflow for the below subtraction, in which
3080  * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
3081  * minuend), we first add a multiple of the 0xFFFF modulus to the
3082  * minuend. The number we add to the minuend needs to be a 34-bit number
3083  * or higher, so for readability reasons we picked the 36-bit multiple.
3084  * We are effectively turning the 16-bit minuend into a 36-bit number:
3085  * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
3086  */
3087  r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
3088 
3089  /* A 33-bit number is subtracted from a 36-bit number (the input r). The
3090  * result (the output r) is a 36-bit number.
3091  */
3092  r -= (src >> 32) + (src & 0xFFFFFFFF);
3093 
3094  /* The first input is a 16-bit number. The second input is a 20-bit
3095  * number. Their sum is a 21-bit number.
3096  */
3097  r = (r & 0xFFFF) + (r >> 16);
3098 
3099  /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3100  * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
3101  */
3102  r = (r & 0xFFFF) + (r >> 16);
3103 
3104  /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3105  * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3106  * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3107  * generated, therefore the output r is always a 16-bit number.
3108  */
3109  r = (r & 0xFFFF) + (r >> 16);
3110 
3111  /* Apply 1's complement to the result. */
3112  r = ~r & 0xFFFF;
3113  r = r ? r : 0xFFFF;
3114 
3115  *dst16_ptr = (uint16_t)r;
3116 }
3117 
3118 static inline void
3119 __instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p __rte_unused,
3120  struct thread *t,
3121  const struct instruction *ip)
3122 {
3123  uint8_t *dst_struct, *src_struct;
3124  uint16_t *dst16_ptr, dst;
3125  uint32_t *src32_ptr;
3126  uint64_t r0, r1;
3127 
3128  TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
3129 
3130  /* Structs. */
3131  dst_struct = t->structs[ip->alu.dst.struct_id];
3132  dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3133  dst = *dst16_ptr;
3134 
3135  src_struct = t->structs[ip->alu.src.struct_id];
3136  src32_ptr = (uint32_t *)&src_struct[0];
3137 
3138  /* Initialize the result with destination 1's complement. */
3139  r0 = dst;
3140  r0 = ~r0 & 0xFFFF;
3141 
3142  r0 += src32_ptr[0]; /* The output r0 is a 33-bit number. */
3143  r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
3144  r0 += src32_ptr[2]; /* The output r0 is a 34-bit number. */
3145  r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
3146  r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
3147 
3148  /* The first input is a 16-bit number. The second input is a 19-bit
3149  * number. Their sum is a 20-bit number.
3150  */
3151  r0 = (r0 & 0xFFFF) + (r0 >> 16);
3152 
3153  /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3154  * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
3155  */
3156  r0 = (r0 & 0xFFFF) + (r0 >> 16);
3157 
3158  /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3159  * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3160  * 0x1000E), the output r is (0 .. 15). So no carry bit can be
3161  * generated, therefore the output r is always a 16-bit number.
3162  */
3163  r0 = (r0 & 0xFFFF) + (r0 >> 16);
3164 
3165  /* Apply 1's complement to the result. */
3166  r0 = ~r0 & 0xFFFF;
3167  r0 = r0 ? r0 : 0xFFFF;
3168 
3169  *dst16_ptr = (uint16_t)r0;
3170 }
3171 
3172 static inline void
3173 __instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p __rte_unused,
3174  struct thread *t,
3175  const struct instruction *ip)
3176 {
3177  uint32_t src_header_id = ip->alu.src.n_bits; /* The src header ID is stored here. */
3178  uint32_t n_src_header_bytes = t->headers[src_header_id].n_bytes;
3179  uint8_t *dst_struct, *src_struct;
3180  uint16_t *dst16_ptr, dst;
3181  uint32_t *src32_ptr;
3182  uint64_t r;
3183  uint32_t i;
3184 
3185  if (n_src_header_bytes == 20) {
3186  __instr_alu_ckadd_struct20_exec(p, t, ip);
3187  return;
3188  }
3189 
3190  TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
3191 
3192  /* Structs. */
3193  dst_struct = t->structs[ip->alu.dst.struct_id];
3194  dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3195  dst = *dst16_ptr;
3196 
3197  src_struct = t->structs[ip->alu.src.struct_id];
3198  src32_ptr = (uint32_t *)&src_struct[0];
3199 
3200  /* Initialize the result with destination 1's complement. */
3201  r = dst;
3202  r = ~r & 0xFFFF;
3203 
3204  /* The max number of 32-bit words in a 32K-byte header is 2^13.
3205  * Therefore, in the worst case scenario, a 45-bit number is added to a
3206  * 16-bit number (the input r), so the output r is 46-bit number.
3207  */
3208  for (i = 0; i < n_src_header_bytes / 4; i++, src32_ptr++)
3209  r += *src32_ptr;
3210 
3211  /* The first input is a 16-bit number. The second input is a 30-bit
3212  * number. Their sum is a 31-bit number.
3213  */
3214  r = (r & 0xFFFF) + (r >> 16);
3215 
3216  /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3217  * a 15-bit number (0 .. 0x7FFF). The sum is a 17-bit number (0 .. 0x17FFE).
3218  */
3219  r = (r & 0xFFFF) + (r >> 16);
3220 
3221  /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3222  * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3223  * 0x17FFE), the output r is (0 .. 0x7FFF). So no carry bit can be
3224  * generated, therefore the output r is always a 16-bit number.
3225  */
3226  r = (r & 0xFFFF) + (r >> 16);
3227 
3228  /* Apply 1's complement to the result. */
3229  r = ~r & 0xFFFF;
3230  r = r ? r : 0xFFFF;
3231 
3232  *dst16_ptr = (uint16_t)r;
3233 }
3234 
3235 /*
3236  * Register array.
3237  */
3238 static inline uint64_t *
3239 instr_regarray_regarray(struct rte_swx_pipeline *p, const struct instruction *ip)
3240 {
3241  struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3242  return r->regarray;
3243 }
3244 
3245 static inline uint64_t
3246 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3247 {
3248  struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3249 
3250  uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
3251  uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
3252  uint64_t idx64 = *idx64_ptr;
3253  uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
3254  uint64_t idx = idx64 & idx64_mask & r->size_mask;
3255 
3256  return idx;
3257 }
3258 
3259 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3260 
3261 static inline uint64_t
3262 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3263 {
3264  struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3265 
3266  uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
3267  uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
3268  uint64_t idx64 = *idx64_ptr;
3269  uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
3270 
3271  return idx;
3272 }
3273 
3274 #else
3275 
3276 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
3277 
3278 #endif
3279 
3280 static inline uint64_t
3281 instr_regarray_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
3282 {
3283  struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3284 
3285  uint64_t idx = ip->regarray.idx_val & r->size_mask;
3286 
3287  return idx;
3288 }
3289 
3290 static inline uint64_t
3291 instr_regarray_src_hbo(struct thread *t, const struct instruction *ip)
3292 {
3293  uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
3294  uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
3295  uint64_t src64 = *src64_ptr;
3296  uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3297  uint64_t src = src64 & src64_mask;
3298 
3299  return src;
3300 }
3301 
3302 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3303 
3304 static inline uint64_t
3305 instr_regarray_src_nbo(struct thread *t, const struct instruction *ip)
3306 {
3307  uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
3308  uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
3309  uint64_t src64 = *src64_ptr;
3310  uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
3311 
3312  return src;
3313 }
3314 
3315 #else
3316 
3317 #define instr_regarray_src_nbo instr_regarray_src_hbo
3318 
3319 #endif
3320 
3321 static inline void
3322 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
3323 {
3324  uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
3325  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
3326  uint64_t dst64 = *dst64_ptr;
3327  uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3328 
3329  *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3330 
3331 }
3332 
3333 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3334 
3335 static inline void
3336 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
3337 {
3338  uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
3339  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
3340  uint64_t dst64 = *dst64_ptr;
3341  uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3342 
3343  src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
3344  *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3345 }
3346 
3347 #else
3348 
3349 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
3350 
3351 #endif
3352 
3353 static inline void
3354 __instr_regprefetch_rh_exec(struct rte_swx_pipeline *p,
3355  struct thread *t,
3356  const struct instruction *ip)
3357 {
3358  uint64_t *regarray, idx;
3359 
3360  TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
3361 
3362  regarray = instr_regarray_regarray(p, ip);
3363  idx = instr_regarray_idx_nbo(p, t, ip);
3364  rte_prefetch0(&regarray[idx]);
3365 }
3366 
3367 static inline void
3368 __instr_regprefetch_rm_exec(struct rte_swx_pipeline *p,
3369  struct thread *t,
3370  const struct instruction *ip)
3371 {
3372  uint64_t *regarray, idx;
3373 
3374  TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
3375 
3376  regarray = instr_regarray_regarray(p, ip);
3377  idx = instr_regarray_idx_hbo(p, t, ip);
3378  rte_prefetch0(&regarray[idx]);
3379 }
3380 
3381 static inline void
3382 __instr_regprefetch_ri_exec(struct rte_swx_pipeline *p,
3383  struct thread *t __rte_unused,
3384  const struct instruction *ip)
3385 {
3386  uint64_t *regarray, idx;
3387 
3388  TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
3389 
3390  regarray = instr_regarray_regarray(p, ip);
3391  idx = instr_regarray_idx_imm(p, ip);
3392  rte_prefetch0(&regarray[idx]);
3393 }
3394 
3395 static inline void
3396 __instr_regrd_hrh_exec(struct rte_swx_pipeline *p,
3397  struct thread *t,
3398  const struct instruction *ip)
3399 {
3400  uint64_t *regarray, idx;
3401 
3402  TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
3403 
3404  regarray = instr_regarray_regarray(p, ip);
3405  idx = instr_regarray_idx_nbo(p, t, ip);
3406  instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3407 }
3408 
3409 static inline void
3410 __instr_regrd_hrm_exec(struct rte_swx_pipeline *p,
3411  struct thread *t,
3412  const struct instruction *ip)
3413 {
3414  uint64_t *regarray, idx;
3415 
3416  TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
3417 
3418  /* Structs. */
3419  regarray = instr_regarray_regarray(p, ip);
3420  idx = instr_regarray_idx_hbo(p, t, ip);
3421  instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3422 }
3423 
3424 static inline void
3425 __instr_regrd_mrh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3426 {
3427  uint64_t *regarray, idx;
3428 
3429  TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
3430 
3431  regarray = instr_regarray_regarray(p, ip);
3432  idx = instr_regarray_idx_nbo(p, t, ip);
3433  instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3434 }
3435 
3436 static inline void
3437 __instr_regrd_mrm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3438 {
3439  uint64_t *regarray, idx;
3440 
3441  TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id);
3442 
3443  regarray = instr_regarray_regarray(p, ip);
3444  idx = instr_regarray_idx_hbo(p, t, ip);
3445  instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3446 }
3447 
3448 static inline void
3449 __instr_regrd_hri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3450 {
3451  uint64_t *regarray, idx;
3452 
3453  TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
3454 
3455  regarray = instr_regarray_regarray(p, ip);
3456  idx = instr_regarray_idx_imm(p, ip);
3457  instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3458 }
3459 
3460 static inline void
3461 __instr_regrd_mri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3462 {
3463  uint64_t *regarray, idx;
3464 
3465  TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
3466 
3467  regarray = instr_regarray_regarray(p, ip);
3468  idx = instr_regarray_idx_imm(p, ip);
3469  instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3470 }
3471 
3472 static inline void
3473 __instr_regwr_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3474 {
3475  uint64_t *regarray, idx, src;
3476 
3477  TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
3478 
3479  regarray = instr_regarray_regarray(p, ip);
3480  idx = instr_regarray_idx_nbo(p, t, ip);
3481  src = instr_regarray_src_nbo(t, ip);
3482  regarray[idx] = src;
3483 }
3484 
3485 static inline void
3486 __instr_regwr_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3487 {
3488  uint64_t *regarray, idx, src;
3489 
3490  TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
3491 
3492  regarray = instr_regarray_regarray(p, ip);
3493  idx = instr_regarray_idx_nbo(p, t, ip);
3494  src = instr_regarray_src_hbo(t, ip);
3495  regarray[idx] = src;
3496 }
3497 
3498 static inline void
3499 __instr_regwr_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3500 {
3501  uint64_t *regarray, idx, src;
3502 
3503  TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
3504 
3505  regarray = instr_regarray_regarray(p, ip);
3506  idx = instr_regarray_idx_hbo(p, t, ip);
3507  src = instr_regarray_src_nbo(t, ip);
3508  regarray[idx] = src;
3509 }
3510 
3511 static inline void
3512 __instr_regwr_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3513 {
3514  uint64_t *regarray, idx, src;
3515 
3516  TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
3517 
3518  regarray = instr_regarray_regarray(p, ip);
3519  idx = instr_regarray_idx_hbo(p, t, ip);
3520  src = instr_regarray_src_hbo(t, ip);
3521  regarray[idx] = src;
3522 }
3523 
3524 static inline void
3525 __instr_regwr_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3526 {
3527  uint64_t *regarray, idx, src;
3528 
3529  TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
3530 
3531  regarray = instr_regarray_regarray(p, ip);
3532  idx = instr_regarray_idx_nbo(p, t, ip);
3533  src = ip->regarray.dstsrc_val;
3534  regarray[idx] = src;
3535 }
3536 
3537 static inline void
3538 __instr_regwr_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3539 {
3540  uint64_t *regarray, idx, src;
3541 
3542  TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
3543 
3544  regarray = instr_regarray_regarray(p, ip);
3545  idx = instr_regarray_idx_hbo(p, t, ip);
3546  src = ip->regarray.dstsrc_val;
3547  regarray[idx] = src;
3548 }
3549 
3550 static inline void
3551 __instr_regwr_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3552 {
3553  uint64_t *regarray, idx, src;
3554 
3555  TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
3556 
3557  regarray = instr_regarray_regarray(p, ip);
3558  idx = instr_regarray_idx_imm(p, ip);
3559  src = instr_regarray_src_nbo(t, ip);
3560  regarray[idx] = src;
3561 }
3562 
3563 static inline void
3564 __instr_regwr_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3565 {
3566  uint64_t *regarray, idx, src;
3567 
3568  TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
3569 
3570  regarray = instr_regarray_regarray(p, ip);
3571  idx = instr_regarray_idx_imm(p, ip);
3572  src = instr_regarray_src_hbo(t, ip);
3573  regarray[idx] = src;
3574 }
3575 
3576 static inline void
3577 __instr_regwr_rii_exec(struct rte_swx_pipeline *p,
3578  struct thread *t __rte_unused,
3579  const struct instruction *ip)
3580 {
3581  uint64_t *regarray, idx, src;
3582 
3583  TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
3584 
3585  regarray = instr_regarray_regarray(p, ip);
3586  idx = instr_regarray_idx_imm(p, ip);
3587  src = ip->regarray.dstsrc_val;
3588  regarray[idx] = src;
3589 }
3590 
3591 static inline void
3592 __instr_regadd_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3593 {
3594  uint64_t *regarray, idx, src;
3595 
3596  TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
3597 
3598  regarray = instr_regarray_regarray(p, ip);
3599  idx = instr_regarray_idx_nbo(p, t, ip);
3600  src = instr_regarray_src_nbo(t, ip);
3601  regarray[idx] += src;
3602 }
3603 
3604 static inline void
3605 __instr_regadd_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3606 {
3607  uint64_t *regarray, idx, src;
3608 
3609  TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
3610 
3611  regarray = instr_regarray_regarray(p, ip);
3612  idx = instr_regarray_idx_nbo(p, t, ip);
3613  src = instr_regarray_src_hbo(t, ip);
3614  regarray[idx] += src;
3615 }
3616 
3617 static inline void
3618 __instr_regadd_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3619 {
3620  uint64_t *regarray, idx, src;
3621 
3622  TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
3623 
3624  regarray = instr_regarray_regarray(p, ip);
3625  idx = instr_regarray_idx_hbo(p, t, ip);
3626  src = instr_regarray_src_nbo(t, ip);
3627  regarray[idx] += src;
3628 }
3629 
3630 static inline void
3631 __instr_regadd_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3632 {
3633  uint64_t *regarray, idx, src;
3634 
3635  TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
3636 
3637  regarray = instr_regarray_regarray(p, ip);
3638  idx = instr_regarray_idx_hbo(p, t, ip);
3639  src = instr_regarray_src_hbo(t, ip);
3640  regarray[idx] += src;
3641 }
3642 
3643 static inline void
3644 __instr_regadd_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3645 {
3646  uint64_t *regarray, idx, src;
3647 
3648  TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
3649 
3650  regarray = instr_regarray_regarray(p, ip);
3651  idx = instr_regarray_idx_nbo(p, t, ip);
3652  src = ip->regarray.dstsrc_val;
3653  regarray[idx] += src;
3654 }
3655 
3656 static inline void
3657 __instr_regadd_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3658 {
3659  uint64_t *regarray, idx, src;
3660 
3661  TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
3662 
3663  regarray = instr_regarray_regarray(p, ip);
3664  idx = instr_regarray_idx_hbo(p, t, ip);
3665  src = ip->regarray.dstsrc_val;
3666  regarray[idx] += src;
3667 }
3668 
3669 static inline void
3670 __instr_regadd_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3671 {
3672  uint64_t *regarray, idx, src;
3673 
3674  TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
3675 
3676  regarray = instr_regarray_regarray(p, ip);
3677  idx = instr_regarray_idx_imm(p, ip);
3678  src = instr_regarray_src_nbo(t, ip);
3679  regarray[idx] += src;
3680 }
3681 
3682 static inline void
3683 __instr_regadd_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3684 {
3685  uint64_t *regarray, idx, src;
3686 
3687  TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
3688 
3689  regarray = instr_regarray_regarray(p, ip);
3690  idx = instr_regarray_idx_imm(p, ip);
3691  src = instr_regarray_src_hbo(t, ip);
3692  regarray[idx] += src;
3693 }
3694 
3695 static inline void
3696 __instr_regadd_rii_exec(struct rte_swx_pipeline *p,
3697  struct thread *t __rte_unused,
3698  const struct instruction *ip)
3699 {
3700  uint64_t *regarray, idx, src;
3701 
3702  TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
3703 
3704  regarray = instr_regarray_regarray(p, ip);
3705  idx = instr_regarray_idx_imm(p, ip);
3706  src = ip->regarray.dstsrc_val;
3707  regarray[idx] += src;
3708 }
3709 
3710 /*
3711  * metarray.
3712  */
3713 static inline struct meter *
3714 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3715 {
3716  struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3717 
3718  uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
3719  uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
3720  uint64_t idx64 = *idx64_ptr;
3721  uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
3722  uint64_t idx = idx64 & idx64_mask & r->size_mask;
3723 
3724  return &r->metarray[idx];
3725 }
3726 
3727 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3728 
3729 static inline struct meter *
3730 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3731 {
3732  struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3733 
3734  uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
3735  uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
3736  uint64_t idx64 = *idx64_ptr;
3737  uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
3738 
3739  return &r->metarray[idx];
3740 }
3741 
3742 #else
3743 
3744 #define instr_meter_idx_nbo instr_meter_idx_hbo
3745 
3746 #endif
3747 
3748 static inline struct meter *
3749 instr_meter_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
3750 {
3751  struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3752 
3753  uint64_t idx = ip->meter.idx_val & r->size_mask;
3754 
3755  return &r->metarray[idx];
3756 }
3757 
3758 static inline uint32_t
3759 instr_meter_length_hbo(struct thread *t, const struct instruction *ip)
3760 {
3761  uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
3762  uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
3763  uint64_t src64 = *src64_ptr;
3764  uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
3765  uint64_t src = src64 & src64_mask;
3766 
3767  return (uint32_t)src;
3768 }
3769 
3770 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3771 
3772 static inline uint32_t
3773 instr_meter_length_nbo(struct thread *t, const struct instruction *ip)
3774 {
3775  uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
3776  uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
3777  uint64_t src64 = *src64_ptr;
3778  uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
3779 
3780  return (uint32_t)src;
3781 }
3782 
3783 #else
3784 
3785 #define instr_meter_length_nbo instr_meter_length_hbo
3786 
3787 #endif
3788 
3789 static inline enum rte_color
3790 instr_meter_color_in_hbo(struct thread *t, const struct instruction *ip)
3791 {
3792  uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
3793  uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
3794  uint64_t src64 = *src64_ptr;
3795  uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
3796  uint64_t src = src64 & src64_mask;
3797 
3798  return (enum rte_color)src;
3799 }
3800 
3801 static inline void
3802 instr_meter_color_out_hbo_set(struct thread *t,
3803  const struct instruction *ip,
3804  enum rte_color color_out)
3805 {
3806  uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
3807  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
3808  uint64_t dst64 = *dst64_ptr;
3809  uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
3810 
3811  uint64_t src = (uint64_t)color_out;
3812 
3813  *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3814 }
3815 
3816 static inline void
3817 __instr_metprefetch_h_exec(struct rte_swx_pipeline *p,
3818  struct thread *t,
3819  const struct instruction *ip)
3820 {
3821  struct meter *m;
3822 
3823  TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
3824 
3825  m = instr_meter_idx_nbo(p, t, ip);
3826  rte_prefetch0(m);
3827 }
3828 
3829 static inline void
3830 __instr_metprefetch_m_exec(struct rte_swx_pipeline *p,
3831  struct thread *t,
3832  const struct instruction *ip)
3833 {
3834  struct meter *m;
3835 
3836  TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
3837 
3838  m = instr_meter_idx_hbo(p, t, ip);
3839  rte_prefetch0(m);
3840 }
3841 
3842 static inline void
3843 __instr_metprefetch_i_exec(struct rte_swx_pipeline *p,
3844  struct thread *t __rte_unused,
3845  const struct instruction *ip)
3846 {
3847  struct meter *m;
3848 
3849  TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
3850 
3851  m = instr_meter_idx_imm(p, ip);
3852  rte_prefetch0(m);
3853 }
3854 
3855 static inline void
3856 __instr_meter_hhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3857 {
3858  struct meter *m;
3859  uint64_t time, n_pkts, n_bytes;
3860  uint32_t length;
3861  enum rte_color color_in, color_out;
3862 
3863  TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
3864 
3865  m = instr_meter_idx_nbo(p, t, ip);
3866  rte_prefetch0(m->n_pkts);
3867  time = rte_get_tsc_cycles();
3868  length = instr_meter_length_nbo(t, ip);
3869  color_in = instr_meter_color_in_hbo(t, ip);
3870 
3871  color_out = rte_meter_trtcm_color_aware_check(&m->m,
3872  &m->profile->profile,
3873  time,
3874  length,
3875  color_in);
3876 
3877  color_out &= m->color_mask;
3878 
3879  n_pkts = m->n_pkts[color_out];
3880  n_bytes = m->n_bytes[color_out];
3881 
3882  instr_meter_color_out_hbo_set(t, ip, color_out);
3883 
3884  m->n_pkts[color_out] = n_pkts + 1;
3885  m->n_bytes[color_out] = n_bytes + length;
3886 }
3887 
3888 static inline void
3889 __instr_meter_hhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3890 {
3891  struct meter *m;
3892  uint64_t time, n_pkts, n_bytes;
3893  uint32_t length;
3894  enum rte_color color_in, color_out;
3895 
3896  TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
3897 
3898  m = instr_meter_idx_nbo(p, t, ip);
3899  rte_prefetch0(m->n_pkts);
3900  time = rte_get_tsc_cycles();
3901  length = instr_meter_length_nbo(t, ip);
3902  color_in = (enum rte_color)ip->meter.color_in_val;
3903 
3904  color_out = rte_meter_trtcm_color_aware_check(&m->m,
3905  &m->profile->profile,
3906  time,
3907  length,
3908  color_in);
3909 
3910  color_out &= m->color_mask;
3911 
3912  n_pkts = m->n_pkts[color_out];
3913  n_bytes = m->n_bytes[color_out];
3914 
3915  instr_meter_color_out_hbo_set(t, ip, color_out);
3916 
3917  m->n_pkts[color_out] = n_pkts + 1;
3918  m->n_bytes[color_out] = n_bytes + length;
3919 }
3920 
3921 static inline void
3922 __instr_meter_hmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3923 {
3924  struct meter *m;
3925  uint64_t time, n_pkts, n_bytes;
3926  uint32_t length;
3927  enum rte_color color_in, color_out;
3928 
3929  TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
3930 
3931  m = instr_meter_idx_nbo(p, t, ip);
3932  rte_prefetch0(m->n_pkts);
3933  time = rte_get_tsc_cycles();
3934  length = instr_meter_length_hbo(t, ip);
3935  color_in = instr_meter_color_in_hbo(t, ip);
3936 
3937  color_out = rte_meter_trtcm_color_aware_check(&m->m,
3938  &m->profile->profile,
3939  time,
3940  length,
3941  color_in);
3942 
3943  color_out &= m->color_mask;
3944 
3945  n_pkts = m->n_pkts[color_out];
3946  n_bytes = m->n_bytes[color_out];
3947 
3948  instr_meter_color_out_hbo_set(t, ip, color_out);
3949 
3950  m->n_pkts[color_out] = n_pkts + 1;
3951  m->n_bytes[color_out] = n_bytes + length;
3952 }
3953 
3954 static inline void
3955 __instr_meter_hmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3956 {
3957  struct meter *m;
3958  uint64_t time, n_pkts, n_bytes;
3959  uint32_t length;
3960  enum rte_color color_in, color_out;
3961 
3962  TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
3963 
3964  m = instr_meter_idx_nbo(p, t, ip);
3965  rte_prefetch0(m->n_pkts);
3966  time = rte_get_tsc_cycles();
3967  length = instr_meter_length_hbo(t, ip);
3968  color_in = (enum rte_color)ip->meter.color_in_val;
3969 
3970  color_out = rte_meter_trtcm_color_aware_check(&m->m,
3971  &m->profile->profile,
3972  time,
3973  length,
3974  color_in);
3975 
3976  color_out &= m->color_mask;
3977 
3978  n_pkts = m->n_pkts[color_out];
3979  n_bytes = m->n_bytes[color_out];
3980 
3981  instr_meter_color_out_hbo_set(t, ip, color_out);
3982 
3983  m->n_pkts[color_out] = n_pkts + 1;
3984  m->n_bytes[color_out] = n_bytes + length;
3985 }
3986 
3987 static inline void
3988 __instr_meter_mhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3989 {
3990  struct meter *m;
3991  uint64_t time, n_pkts, n_bytes;
3992  uint32_t length;
3993  enum rte_color color_in, color_out;
3994 
3995  TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
3996 
3997  m = instr_meter_idx_hbo(p, t, ip);
3998  rte_prefetch0(m->n_pkts);
3999  time = rte_get_tsc_cycles();
4000  length = instr_meter_length_nbo(t, ip);
4001  color_in = instr_meter_color_in_hbo(t, ip);
4002 
4003  color_out = rte_meter_trtcm_color_aware_check(&m->m,
4004  &m->profile->profile,
4005  time,
4006  length,
4007  color_in);
4008 
4009  color_out &= m->color_mask;
4010 
4011  n_pkts = m->n_pkts[color_out];
4012  n_bytes = m->n_bytes[color_out];
4013 
4014  instr_meter_color_out_hbo_set(t, ip, color_out);
4015 
4016  m->n_pkts[color_out] = n_pkts + 1;
4017  m->n_bytes[color_out] = n_bytes + length;
4018 }
4019 
4020 static inline void
4021 __instr_meter_mhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4022 {
4023  struct meter *m;
4024  uint64_t time, n_pkts, n_bytes;
4025  uint32_t length;
4026  enum rte_color color_in, color_out;
4027 
4028  TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
4029 
4030  m = instr_meter_idx_hbo(p, t, ip);
4031  rte_prefetch0(m->n_pkts);
4032  time = rte_get_tsc_cycles();
4033  length = instr_meter_length_nbo(t, ip);
4034  color_in = (enum rte_color)ip->meter.color_in_val;
4035 
4036  color_out = rte_meter_trtcm_color_aware_check(&m->m,
4037  &m->profile->profile,
4038  time,
4039  length,
4040  color_in);
4041 
4042  color_out &= m->color_mask;
4043 
4044  n_pkts = m->n_pkts[color_out];
4045  n_bytes = m->n_bytes[color_out];
4046 
4047  instr_meter_color_out_hbo_set(t, ip, color_out);
4048 
4049  m->n_pkts[color_out] = n_pkts + 1;
4050  m->n_bytes[color_out] = n_bytes + length;
4051 }
4052 
4053 static inline void
4054 __instr_meter_mmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4055 {
4056  struct meter *m;
4057  uint64_t time, n_pkts, n_bytes;
4058  uint32_t length;
4059  enum rte_color color_in, color_out;
4060 
4061  TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
4062 
4063  m = instr_meter_idx_hbo(p, t, ip);
4064  rte_prefetch0(m->n_pkts);
4065  time = rte_get_tsc_cycles();
4066  length = instr_meter_length_hbo(t, ip);
4067  color_in = instr_meter_color_in_hbo(t, ip);
4068 
4069  color_out = rte_meter_trtcm_color_aware_check(&m->m,
4070  &m->profile->profile,
4071  time,
4072  length,
4073  color_in);
4074 
4075  color_out &= m->color_mask;
4076 
4077  n_pkts = m->n_pkts[color_out];
4078  n_bytes = m->n_bytes[color_out];
4079 
4080  instr_meter_color_out_hbo_set(t, ip, color_out);
4081 
4082  m->n_pkts[color_out] = n_pkts + 1;
4083  m->n_bytes[color_out] = n_bytes + length;
4084 }
4085 
4086 static inline void
4087 __instr_meter_mmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4088 {
4089  struct meter *m;
4090  uint64_t time, n_pkts, n_bytes;
4091  uint32_t length;
4092  enum rte_color color_in, color_out;
4093 
4094  TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
4095 
4096  m = instr_meter_idx_hbo(p, t, ip);
4097  rte_prefetch0(m->n_pkts);
4098  time = rte_get_tsc_cycles();
4099  length = instr_meter_length_hbo(t, ip);
4100  color_in = (enum rte_color)ip->meter.color_in_val;
4101 
4102  color_out = rte_meter_trtcm_color_aware_check(&m->m,
4103  &m->profile->profile,
4104  time,
4105  length,
4106  color_in);
4107 
4108  color_out &= m->color_mask;
4109 
4110  n_pkts = m->n_pkts[color_out];
4111  n_bytes = m->n_bytes[color_out];
4112 
4113  instr_meter_color_out_hbo_set(t, ip, color_out);
4114 
4115  m->n_pkts[color_out] = n_pkts + 1;
4116  m->n_bytes[color_out] = n_bytes + length;
4117 }
4118 
4119 static inline void
4120 __instr_meter_ihm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4121 {
4122  struct meter *m;
4123  uint64_t time, n_pkts, n_bytes;
4124  uint32_t length;
4125  enum rte_color color_in, color_out;
4126 
4127  TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
4128 
4129  m = instr_meter_idx_imm(p, ip);
4130  rte_prefetch0(m->n_pkts);
4131  time = rte_get_tsc_cycles();
4132  length = instr_meter_length_nbo(t, ip);
4133  color_in = instr_meter_color_in_hbo(t, ip);
4134 
4135  color_out = rte_meter_trtcm_color_aware_check(&m->m,
4136  &m->profile->profile,
4137  time,
4138  length,
4139  color_in);
4140 
4141  color_out &= m->color_mask;
4142 
4143  n_pkts = m->n_pkts[color_out];
4144  n_bytes = m->n_bytes[color_out];
4145 
4146  instr_meter_color_out_hbo_set(t, ip, color_out);
4147 
4148  m->n_pkts[color_out] = n_pkts + 1;
4149  m->n_bytes[color_out] = n_bytes + length;
4150 }
4151 
4152 static inline void
4153 __instr_meter_ihi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4154 {
4155  struct meter *m;
4156  uint64_t time, n_pkts, n_bytes;
4157  uint32_t length;
4158  enum rte_color color_in, color_out;
4159 
4160  TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
4161 
4162  m = instr_meter_idx_imm(p, ip);
4163  rte_prefetch0(m->n_pkts);
4164  time = rte_get_tsc_cycles();
4165  length = instr_meter_length_nbo(t, ip);
4166  color_in = (enum rte_color)ip->meter.color_in_val;
4167 
4168  color_out = rte_meter_trtcm_color_aware_check(&m->m,
4169  &m->profile->profile,
4170  time,
4171  length,
4172  color_in);
4173 
4174  color_out &= m->color_mask;
4175 
4176  n_pkts = m->n_pkts[color_out];
4177  n_bytes = m->n_bytes[color_out];
4178 
4179  instr_meter_color_out_hbo_set(t, ip, color_out);
4180 
4181  m->n_pkts[color_out] = n_pkts + 1;
4182  m->n_bytes[color_out] = n_bytes + length;
4183 }
4184 
4185 static inline void
4186 __instr_meter_imm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4187 {
4188  struct meter *m;
4189  uint64_t time, n_pkts, n_bytes;
4190  uint32_t length;
4191  enum rte_color color_in, color_out;
4192 
4193  TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
4194 
4195  m = instr_meter_idx_imm(p, ip);
4196  rte_prefetch0(m->n_pkts);
4197  time = rte_get_tsc_cycles();
4198  length = instr_meter_length_hbo(t, ip);
4199  color_in = instr_meter_color_in_hbo(t, ip);
4200 
4201  color_out = rte_meter_trtcm_color_aware_check(&m->m,
4202  &m->profile->profile,
4203  time,
4204  length,
4205  color_in);
4206 
4207  color_out &= m->color_mask;
4208 
4209  n_pkts = m->n_pkts[color_out];
4210  n_bytes = m->n_bytes[color_out];
4211 
4212  instr_meter_color_out_hbo_set(t, ip, color_out);
4213 
4214  m->n_pkts[color_out] = n_pkts + 1;
4215  m->n_bytes[color_out] = n_bytes + length;
4216 }
4217 
4218 static inline void
4219 __instr_meter_imi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4220 {
4221  struct meter *m;
4222  uint64_t time, n_pkts, n_bytes;
4223  uint32_t length;
4224  enum rte_color color_in, color_out;
4225 
4226  TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
4227 
4228  m = instr_meter_idx_imm(p, ip);
4229  rte_prefetch0(m->n_pkts);
4230  time = rte_get_tsc_cycles();
4231  length = instr_meter_length_hbo(t, ip);
4232  color_in = (enum rte_color)ip->meter.color_in_val;
4233 
4234  color_out = rte_meter_trtcm_color_aware_check(&m->m,
4235  &m->profile->profile,
4236  time,
4237  length,
4238  color_in);
4239 
4240  color_out &= m->color_mask;
4241 
4242  n_pkts = m->n_pkts[color_out];
4243  n_bytes = m->n_bytes[color_out];
4244 
4245  instr_meter_color_out_hbo_set(t, ip, color_out);
4246 
4247  m->n_pkts[color_out] = n_pkts + 1;
4248  m->n_bytes[color_out] = n_bytes + length;
4249 }
4250 
4251 #endif
rte_swx_port_out_ops
Definition: rte_swx_port.h:214
rte_swx_extern_func_t
int(* rte_swx_extern_func_t)(void *mailbox)
Definition: rte_swx_extern.h:92
rte_swx_table_state
Definition: rte_swx_ctl.h:755
rte_swx_table_learner_delete
__rte_experimental void rte_swx_table_learner_delete(void *table, void *mailbox)
rte_byteorder.h
rte_swx_table_selector.h
RTE_COLORS
Definition: rte_meter.h:40
rte_swx_pkt
Definition: rte_swx_port.h:21
rte_swx_table_learner_rearm_new
__rte_experimental void rte_swx_table_learner_rearm_new(void *table, void *mailbox, uint64_t time, uint32_t key_timeout_id)
__rte_unused
#define __rte_unused
Definition: rte_common.h:123
rte_swx_pipeline.h
rte_swx_table_learner_add
__rte_experimental uint32_t rte_swx_table_learner_add(void *table, void *mailbox, uint64_t time, uint64_t action_id, uint8_t *action_data, uint32_t key_timeout_id)
rte_swx_table_state::obj
void * obj
Definition: rte_swx_ctl.h:757
rte_swx_pkt::pkt
uint8_t * pkt
Definition: rte_swx_port.h:26
rte_swx_table_match_type
rte_swx_table_match_type
Definition: rte_swx_table.h:23
rte_meter_trtcm_params
Definition: rte_meter.h:55
rte_cycles.h
rte_swx_port_out_flush_t
void(* rte_swx_port_out_flush_t)(void *port)
Definition: rte_swx_port.h:184
rte_meter_trtcm_color_aware_check
static enum rte_color rte_meter_trtcm_color_aware_check(struct rte_meter_trtcm *m, struct rte_meter_trtcm_profile *p, uint64_t time, uint32_t pkt_len, enum rte_color pkt_color)
Definition: rte_meter.h:539
rte_prefetch.h
rte_get_tsc_cycles
static uint64_t rte_get_tsc_cycles(void)
rte_swx_port_out_pkt_clone_tx_t
void(* rte_swx_port_out_pkt_clone_tx_t)(void *port, struct rte_swx_pkt *pkt, uint32_t truncation_length)
Definition: rte_swx_port.h:173
rte_swx_port_out_pkt_fast_clone_tx_t
void(* rte_swx_port_out_pkt_fast_clone_tx_t)(void *port, struct rte_swx_pkt *pkt)
Definition: rte_swx_port.h:159
rte_swx_extern_type_destructor_t
void(* rte_swx_extern_type_destructor_t)(void *object)
Definition: rte_swx_extern.h:44
rte_common.h
rte_swx_extern_type_constructor_t
void *(* rte_swx_extern_type_constructor_t)(const char *args)
Definition: rte_swx_extern.h:35
rte_swx_port_out_pkt_tx_t
void(* rte_swx_port_out_pkt_tx_t)(void *port, struct rte_swx_pkt *pkt)
Definition: rte_swx_port.h:147
rte_color
rte_color
Definition: rte_meter.h:36
rte_swx_hash_func_t
uint32_t(* rte_swx_hash_func_t)(const void *key, uint32_t length, uint32_t seed)
Definition: rte_swx_pipeline.h:331
rte_swx_table_ops
Definition: rte_swx_table.h:267
rte_swx_port_in_ops
Definition: rte_swx_port.h:100
rte_swx_table_learner.h
rte_swx_pkt::length
uint32_t length
Definition: rte_swx_port.h:32
rte_meter_trtcm
Definition: rte_meter.h:372
rte_meter.h
rte_swx_ctl.h
rte_swx_table_learner_rearm
__rte_experimental void rte_swx_table_learner_rearm(void *table, void *mailbox, uint64_t time)
RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX
#define RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX
Definition: rte_swx_table_learner.h:55
rte_swx_pkt::offset
uint32_t offset
Definition: rte_swx_port.h:29
RTE_SWX_NAME_SIZE
#define RTE_SWX_NAME_SIZE
Definition: rte_swx_pipeline.h:27
rte_swx_table_lookup_t
int(* rte_swx_table_lookup_t)(void *table, void *mailbox, uint8_t **key, uint64_t *action_id, uint8_t **action_data, int *hit)
Definition: rte_swx_table.h:250
rte_swx_extern_type_member_func_t
int(* rte_swx_extern_type_member_func_t)(void *object, void *mailbox)
Definition: rte_swx_extern.h:69
rte_prefetch0
static void rte_prefetch0(const volatile void *p)
rte_swx_port_in_pkt_rx_t
int(* rte_swx_port_in_pkt_rx_t)(void *port, struct rte_swx_pkt *pkt)
Definition: rte_swx_port.h:72
Generated by   doxygen 1.8.13