DPDK  21.02.0
rte_reciprocal.h
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2017 Cavium, Inc
3  */
4 /*
5  * Reciprocal divide
6  *
7  * Used with permission from original authors
8  * Hannes Frederic Sowa and Daniel Borkmann
9  *
10  * This algorithm is based on the paper "Division by Invariant
11  * Integers Using Multiplication" by Torbj√∂rn Granlund and Peter
12  * L. Montgomery.
13  *
14  * The assembler implementation from Agner Fog, which this code is
15  * based on, can be found here:
16  * http://www.agner.org/optimize/asmlib.zip
17  *
18  * This optimization for A/B is helpful if the divisor B is mostly
19  * runtime invariant. The reciprocal of B is calculated in the
20  * slow-path with reciprocal_value(). The fast-path can then just use
21  * a much faster multiplication operation with a variable dividend A
22  * to calculate the division A/B.
23  */
24 
25 #ifndef _RTE_RECIPROCAL_H_
26 #define _RTE_RECIPROCAL_H_
27 
28 #include <stdint.h>
29 
30 #include <rte_common.h>
31 
32 struct rte_reciprocal {
33  uint32_t m;
34  uint8_t sh1, sh2;
35 };
36 
37 struct rte_reciprocal_u64 {
38  uint64_t m;
39  uint8_t sh1, sh2;
40 };
41 
42 static inline uint32_t rte_reciprocal_divide(uint32_t a, struct rte_reciprocal R)
43 {
44  uint32_t t = (uint32_t)(((uint64_t)a * R.m) >> 32);
45 
46  return (t + ((a - t) >> R.sh1)) >> R.sh2;
47 }
48 
49 static __rte_always_inline uint64_t
50 mullhi_u64(uint64_t x, uint64_t y)
51 {
52 #ifdef __SIZEOF_INT128__
53  __uint128_t xl = x;
54  __uint128_t rl = xl * y;
55 
56  return (rl >> 64);
57 #else
58  uint64_t u0, u1, v0, v1, k, t;
59  uint64_t w1, w2;
60  uint64_t whi;
61 
62  u1 = x >> 32; u0 = x & 0xFFFFFFFF;
63  v1 = y >> 32; v0 = y & 0xFFFFFFFF;
64 
65  t = u0*v0;
66  k = t >> 32;
67 
68  t = u1*v0 + k;
69  w1 = t & 0xFFFFFFFF;
70  w2 = t >> 32;
71 
72  t = u0*v1 + w1;
73  k = t >> 32;
74 
75  whi = u1*v1 + w2 + k;
76 
77  return whi;
78 #endif
79 }
80 
81 static __rte_always_inline uint64_t
82 rte_reciprocal_divide_u64(uint64_t a, const struct rte_reciprocal_u64 *R)
83 {
84  uint64_t t = mullhi_u64(a, R->m);
85 
86  return (t + ((a - t) >> R->sh1)) >> R->sh2;
87 }
88 
89 struct rte_reciprocal rte_reciprocal_value(uint32_t d);
90 struct rte_reciprocal_u64 rte_reciprocal_value_u64(uint64_t d);
91 
92 #endif /* _RTE_RECIPROCAL_H_ */
#define __rte_always_inline
Definition: rte_common.h:226