projects/linux·blob master · refs

448 lines · 11.3 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_MATH64_H
3#define _LINUX_MATH64_H
4
5#include <linux/types.h>
6#include <linux/math.h>
7#include <asm/div64.h>
8#include <vdso/math64.h>
9
10#if BITS_PER_LONG == 64
11
12#define div64_long(x, y) div64_s64((x), (y))
13#define div64_ul(x, y) div64_u64((x), (y))
14
15/**
16 * div_u64_rem - unsigned 64bit divide with 32bit divisor with remainder
17 * @dividend: unsigned 64bit dividend
18 * @divisor: unsigned 32bit divisor
19 * @remainder: pointer to unsigned 32bit remainder
20 *
21 * Return: sets ``*remainder``, then returns dividend / divisor
22 *
23 * This is commonly provided by 32bit archs to provide an optimized 64bit
24 * divide.
25 */
26static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder)
27{
28 *remainder = dividend % divisor;
29 return dividend / divisor;
30}
31
32/**
33 * div_s64_rem - signed 64bit divide with 32bit divisor with remainder
34 * @dividend: signed 64bit dividend
35 * @divisor: signed 32bit divisor
36 * @remainder: pointer to signed 32bit remainder
37 *
38 * Return: sets ``*remainder``, then returns dividend / divisor
39 */
40static inline s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder)
41{
42 *remainder = dividend % divisor;
43 return dividend / divisor;
44}
45
46/**
47 * div64_u64_rem - unsigned 64bit divide with 64bit divisor and remainder
48 * @dividend: unsigned 64bit dividend
49 * @divisor: unsigned 64bit divisor
50 * @remainder: pointer to unsigned 64bit remainder
51 *
52 * Return: sets ``*remainder``, then returns dividend / divisor
53 */
54static inline u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder)
55{
56 *remainder = dividend % divisor;
57 return dividend / divisor;
58}
59
60/**
61 * div64_s64_rem - signed 64bit divide with 64bit divisor and remainder
62 * @dividend: signed 64bit dividend
63 * @divisor: signed 64bit divisor
64 * @remainder: pointer to signed 64bit remainder
65 *
66 * Return: sets ``*remainder``, then returns dividend / divisor
67 */
68static inline s64 div64_s64_rem(s64 dividend, s64 divisor, s64 *remainder)
69{
70 *remainder = dividend % divisor;
71 return dividend / divisor;
72}
73
74/**
75 * div64_u64 - unsigned 64bit divide with 64bit divisor
76 * @dividend: unsigned 64bit dividend
77 * @divisor: unsigned 64bit divisor
78 *
79 * Return: dividend / divisor
80 */
81static inline u64 div64_u64(u64 dividend, u64 divisor)
82{
83 return dividend / divisor;
84}
85
86/**
87 * div64_s64 - signed 64bit divide with 64bit divisor
88 * @dividend: signed 64bit dividend
89 * @divisor: signed 64bit divisor
90 *
91 * Return: dividend / divisor
92 */
93static inline s64 div64_s64(s64 dividend, s64 divisor)
94{
95 return dividend / divisor;
96}
97
98#elif BITS_PER_LONG == 32
99
100#define div64_long(x, y) div_s64((x), (y))
101#define div64_ul(x, y) div_u64((x), (y))
102
103#ifndef div_u64_rem
104static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder)
105{
106 *remainder = do_div(dividend, divisor);
107 return dividend;
108}
109#endif
110
111#ifndef div_s64_rem
112extern s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder);
113#endif
114
115#ifndef div64_u64_rem
116extern u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder);
117#endif
118
119#ifndef div64_s64_rem
120extern s64 div64_s64_rem(s64 dividend, s64 divisor, s64 *remainder);
121#endif
122
123#ifndef div64_u64
124extern u64 div64_u64(u64 dividend, u64 divisor);
125#endif
126
127#ifndef div64_s64
128extern s64 div64_s64(s64 dividend, s64 divisor);
129#endif
130
131#endif /* BITS_PER_LONG */
132
133/**
134 * div_u64 - unsigned 64bit divide with 32bit divisor
135 * @dividend: unsigned 64bit dividend
136 * @divisor: unsigned 32bit divisor
137 *
138 * This is the most common 64bit divide and should be used if possible,
139 * as many 32bit archs can optimize this variant better than a full 64bit
140 * divide.
141 *
142 * Return: dividend / divisor
143 */
144#ifndef div_u64
145static inline u64 div_u64(u64 dividend, u32 divisor)
146{
147 u32 remainder;
148 return div_u64_rem(dividend, divisor, &remainder);
149}
150#endif
151
152/**
153 * div_s64 - signed 64bit divide with 32bit divisor
154 * @dividend: signed 64bit dividend
155 * @divisor: signed 32bit divisor
156 *
157 * Return: dividend / divisor
158 */
159#ifndef div_s64
160static inline s64 div_s64(s64 dividend, s32 divisor)
161{
162 s32 remainder;
163 return div_s64_rem(dividend, divisor, &remainder);
164}
165#endif
166
167u32 iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder);
168
169#ifndef mul_u32_u32
170/*
171 * Many a GCC version messes this up and generates a 64x64 mult :-(
172 */
173static inline u64 mul_u32_u32(u32 a, u32 b)
174{
175 return (u64)a * b;
176}
177#endif
178
179#ifndef add_u64_u32
180/*
181 * Many a GCC version also messes this up.
182 * Zero extending b and then spilling everything to stack.
183 */
184static inline u64 add_u64_u32(u64 a, u32 b)
185{
186 return a + b;
187}
188#endif
189
190#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__)
191
192#ifndef mul_u64_u32_shr
193static __always_inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift)
194{
195 return (u64)(((unsigned __int128)a * mul) >> shift);
196}
197#endif /* mul_u64_u32_shr */
198
199#ifndef mul_u64_u64_shr
200static __always_inline u64 mul_u64_u64_shr(u64 a, u64 mul, unsigned int shift)
201{
202 return (u64)(((unsigned __int128)a * mul) >> shift);
203}
204#endif /* mul_u64_u64_shr */
205
206#else
207
208#ifndef mul_u64_u32_shr
209static __always_inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift)
210{
211 u32 ah = a >> 32, al = a;
212 u64 ret;
213
214 ret = mul_u32_u32(al, mul) >> shift;
215 if (ah)
216 ret += mul_u32_u32(ah, mul) << (32 - shift);
217 return ret;
218}
219#endif /* mul_u64_u32_shr */
220
221#ifndef mul_u64_u64_shr
222static inline u64 mul_u64_u64_shr(u64 a, u64 b, unsigned int shift)
223{
224 union {
225 u64 ll;
226 struct {
227#ifdef __BIG_ENDIAN
228 u32 high, low;
229#else
230 u32 low, high;
231#endif
232 } l;
233 } rl, rm, rn, rh, a0, b0;
234 u64 c;
235
236 a0.ll = a;
237 b0.ll = b;
238
239 rl.ll = mul_u32_u32(a0.l.low, b0.l.low);
240 rm.ll = mul_u32_u32(a0.l.low, b0.l.high);
241 rn.ll = mul_u32_u32(a0.l.high, b0.l.low);
242 rh.ll = mul_u32_u32(a0.l.high, b0.l.high);
243
244 /*
245 * Each of these lines computes a 64-bit intermediate result into "c",
246 * starting at bits 32-95. The low 32-bits go into the result of the
247 * multiplication, the high 32-bits are carried into the next step.
248 */
249 rl.l.high = c = (u64)rl.l.high + rm.l.low + rn.l.low;
250 rh.l.low = c = (c >> 32) + rm.l.high + rn.l.high + rh.l.low;
251 rh.l.high = (c >> 32) + rh.l.high;
252
253 /*
254 * The 128-bit result of the multiplication is in rl.ll and rh.ll,
255 * shift it right and throw away the high part of the result.
256 */
257 if (shift == 0)
258 return rl.ll;
259 if (shift < 64)
260 return (rl.ll >> shift) | (rh.ll << (64 - shift));
261 return rh.ll >> (shift & 63);
262}
263#endif /* mul_u64_u64_shr */
264
265#endif
266
267#ifndef mul_s64_u64_shr
268static inline u64 mul_s64_u64_shr(s64 a, u64 b, unsigned int shift)
269{
270 u64 ret;
271
272 /*
273 * Extract the sign before the multiplication and put it back
274 * afterwards if needed.
275 */
276 ret = mul_u64_u64_shr(abs(a), b, shift);
277
278 if (a < 0)
279 ret = -((s64) ret);
280
281 return ret;
282}
283#endif /* mul_s64_u64_shr */
284
285#ifndef mul_u64_u32_div
286static inline u64 mul_u64_u32_div(u64 a, u32 mul, u32 divisor)
287{
288 union {
289 u64 ll;
290 struct {
291#ifdef __BIG_ENDIAN
292 u32 high, low;
293#else
294 u32 low, high;
295#endif
296 } l;
297 } u, rl, rh;
298
299 u.ll = a;
300 rl.ll = mul_u32_u32(u.l.low, mul);
301 rh.ll = mul_u32_u32(u.l.high, mul) + rl.l.high;
302
303 /* Bits 32-63 of the result will be in rh.l.low. */
304 rl.l.high = do_div(rh.ll, divisor);
305
306 /* Bits 0-31 of the result will be in rl.l.low. */
307 do_div(rl.ll, divisor);
308
309 rl.l.high = rh.l.low;
310 return rl.ll;
311}
312#endif /* mul_u64_u32_div */
313
314/**
315 * mul_u64_add_u64_div_u64 - unsigned 64bit multiply, add, and divide
316 * @a: first unsigned 64bit multiplicand
317 * @b: second unsigned 64bit multiplicand
318 * @c: unsigned 64bit addend
319 * @d: unsigned 64bit divisor
320 *
321 * Multiply two 64bit values together to generate a 128bit product
322 * add a third value and then divide by a fourth.
323 * The Generic code divides by 0 if @d is zero and returns ~0 on overflow.
324 * Architecture specific code may trap on zero or overflow.
325 *
326 * Return: (@a * @b + @c) / @d
327 */
328u64 mul_u64_add_u64_div_u64(u64 a, u64 b, u64 c, u64 d);
329
330/**
331 * mul_u64_u64_div_u64 - unsigned 64bit multiply and divide
332 * @a: first unsigned 64bit multiplicand
333 * @b: second unsigned 64bit multiplicand
334 * @d: unsigned 64bit divisor
335 *
336 * Multiply two 64bit values together to generate a 128bit product
337 * and then divide by a third value.
338 * The Generic code divides by 0 if @d is zero and returns ~0 on overflow.
339 * Architecture specific code may trap on zero or overflow.
340 *
341 * Return: @a * @b / @d
342 */
343#define mul_u64_u64_div_u64(a, b, d) mul_u64_add_u64_div_u64(a, b, 0, d)
344
345/**
346 * mul_u64_u64_div_u64_roundup - unsigned 64bit multiply and divide rounded up
347 * @a: first unsigned 64bit multiplicand
348 * @b: second unsigned 64bit multiplicand
349 * @d: unsigned 64bit divisor
350 *
351 * Multiply two 64bit values together to generate a 128bit product
352 * and then divide and round up.
353 * The Generic code divides by 0 if @d is zero and returns ~0 on overflow.
354 * Architecture specific code may trap on zero or overflow.
355 *
356 * Return: (@a * @b + @d - 1) / @d
357 */
358#define mul_u64_u64_div_u64_roundup(a, b, d) \
359 ({ u64 _tmp = (d); mul_u64_add_u64_div_u64(a, b, _tmp - 1, _tmp); })
360
361
362/**
363 * DIV64_U64_ROUND_UP - unsigned 64bit divide with 64bit divisor rounded up
364 * @ll: unsigned 64bit dividend
365 * @d: unsigned 64bit divisor
366 *
367 * Divide unsigned 64bit dividend by unsigned 64bit divisor
368 * and round up.
369 *
370 * Return: dividend / divisor rounded up
371 */
372#define DIV64_U64_ROUND_UP(ll, d) \
373 ({ u64 _tmp = (d); div64_u64((ll) + _tmp - 1, _tmp); })
374
375/**
376 * DIV_U64_ROUND_UP - unsigned 64bit divide with 32bit divisor rounded up
377 * @ll: unsigned 64bit dividend
378 * @d: unsigned 32bit divisor
379 *
380 * Divide unsigned 64bit dividend by unsigned 32bit divisor
381 * and round up.
382 *
383 * Return: dividend / divisor rounded up
384 */
385#define DIV_U64_ROUND_UP(ll, d) \
386 ({ u32 _tmp = (d); div_u64((ll) + _tmp - 1, _tmp); })
387
388/**
389 * DIV64_U64_ROUND_CLOSEST - unsigned 64bit divide with 64bit divisor rounded to nearest integer
390 * @dividend: unsigned 64bit dividend
391 * @divisor: unsigned 64bit divisor
392 *
393 * Divide unsigned 64bit dividend by unsigned 64bit divisor
394 * and round to closest integer.
395 *
396 * Return: dividend / divisor rounded to nearest integer
397 */
398#define DIV64_U64_ROUND_CLOSEST(dividend, divisor) \
399 ({ u64 _tmp = (divisor); div64_u64((dividend) + _tmp / 2, _tmp); })
400
401/**
402 * DIV_U64_ROUND_CLOSEST - unsigned 64bit divide with 32bit divisor rounded to nearest integer
403 * @dividend: unsigned 64bit dividend
404 * @divisor: unsigned 32bit divisor
405 *
406 * Divide unsigned 64bit dividend by unsigned 32bit divisor
407 * and round to closest integer.
408 *
409 * Return: dividend / divisor rounded to nearest integer
410 */
411#define DIV_U64_ROUND_CLOSEST(dividend, divisor) \
412 ({ u32 _tmp = (divisor); div_u64((u64)(dividend) + _tmp / 2, _tmp); })
413
414/**
415 * DIV_S64_ROUND_CLOSEST - signed 64bit divide with 32bit divisor rounded to nearest integer
416 * @dividend: signed 64bit dividend
417 * @divisor: signed 32bit divisor
418 *
419 * Divide signed 64bit dividend by signed 32bit divisor
420 * and round to closest integer.
421 *
422 * Return: dividend / divisor rounded to nearest integer
423 */
424#define DIV_S64_ROUND_CLOSEST(dividend, divisor)( \
425{ \
426 s64 __x = (dividend); \
427 s32 __d = (divisor); \
428 ((__x > 0) == (__d > 0)) ? \
429 div_s64((__x + (__d / 2)), __d) : \
430 div_s64((__x - (__d / 2)), __d); \
431} \
432)
433
434/**
435 * roundup_u64 - Round up a 64bit value to the next specified 32bit multiple
436 * @x: the value to up
437 * @y: 32bit multiple to round up to
438 *
439 * Rounds @x to the next multiple of @y. For 32bit @x values, see roundup and
440 * the faster round_up() for powers of 2.
441 *
442 * Return: rounded up value.
443 */
444static inline u64 roundup_u64(u64 x, u32 y)
445{
446 return DIV_U64_ROUND_UP(x, y) * y;
447}
448#endif /* _LINUX_MATH64_H */