projects/linux·blob master · refs

387 lines · 11.1 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright 2002 H. Peter Anvin - All Rights Reserved
4 *
5 * Algorithm list and algorithm selection for RAID-6
6 */
7
8#include <linux/module.h>
9#include <linux/gfp.h>
10#include <linux/raid/pq.h>
11#include <linux/slab.h>
12#include <linux/static_call.h>
13#include <kunit/visibility.h>
14#include "algos.h"
15
16#define RAID6_MAX_ALGOS 16
17static const struct raid6_calls *raid6_algos[RAID6_MAX_ALGOS];
18static unsigned int raid6_nr_algos;
19static const struct raid6_recov_calls *raid6_recov_algo;
20
21/* Selected algorithm */
22DEFINE_STATIC_CALL_NULL(raid6_gen_syndrome_impl, *raid6_intx1.gen_syndrome);
23DEFINE_STATIC_CALL_NULL(raid6_xor_syndrome_impl, *raid6_intx1.xor_syndrome);
24DEFINE_STATIC_CALL_NULL(raid6_recov_2data_impl, *raid6_recov_intx1.data2);
25DEFINE_STATIC_CALL_NULL(raid6_recov_datap_impl, *raid6_recov_intx1.datap);
26
27/**
28 * raid6_gen_syndrome - generate RAID6 P/Q parity
29 * @disks: number of "disks" to operate on including parity
30 * @bytes: length in bytes of each vector
31 * @ptrs: @disks size array of memory pointers
32 *
33 * Generate @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and
34 * @ptrs[@disks - 1] respectively from the memory pointed to by @ptrs[0] to
35 * @ptrs[@disks - 3].
36 *
37 * @disks must be at least 4, and the memory pointed to by each member of @ptrs
38 * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of
39 * 512.
40 *
41 * See https://kernel.org/pub/linux/kernel/people/hpa/raid6.pdf for underlying
42 * algorithm.
43 */
44void raid6_gen_syndrome(int disks, size_t bytes, void **ptrs)
45{
46 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count());
47 WARN_ON_ONCE(bytes & 511);
48 WARN_ON_ONCE(disks < RAID6_MIN_DISKS);
49
50 static_call(raid6_gen_syndrome_impl)(disks, bytes, ptrs);
51}
52EXPORT_SYMBOL_GPL(raid6_gen_syndrome);
53
54/**
55 * raid6_xor_syndrome - update RAID6 P/Q parity
56 * @disks: number of "disks" to operate on including parity
57 * @start: first index into @disk to update
58 * @stop: last index into @disk to update
59 * @bytes: length in bytes of each vector
60 * @ptrs: @disks size array of memory pointers
61 *
62 * Update @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and
63 * @ptrs[@disks - 1] respectively for the memory pointed to by
64 * @ptrs[@start..@stop].
65 *
66 * This is used to update parity in place using the following sequence:
67 *
68 * 1) call raid6_xor_syndrome(disk, start, stop, ...) for the existing data.
69 * 2) update the the data in @ptrs[@start..@stop].
70 * 3) call raid6_xor_syndrome(disk, start, stop, ...) for the new data.
71 *
72 * Data between @start and @stop that is not changed should be filled
73 * with a pointer to the kernel zero page.
74 *
75 * @disks must be at least 4, and the memory pointed to by each member of @ptrs
76 * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of
77 * 512. @stop must be larger or equal to @start.
78 */
79void raid6_xor_syndrome(int disks, int start, int stop, size_t bytes,
80 void **ptrs)
81{
82 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count());
83 WARN_ON_ONCE(bytes & 511);
84 WARN_ON_ONCE(disks < RAID6_MIN_DISKS);
85 WARN_ON_ONCE(stop < start);
86
87 static_call(raid6_xor_syndrome_impl)(disks, start, stop, bytes, ptrs);
88}
89EXPORT_SYMBOL_GPL(raid6_xor_syndrome);
90
91/*
92 * raid6_can_xor_syndrome - check if raid6_xor_syndrome() can be used
93 *
94 * Returns %true if raid6_can_xor_syndrome() can be used, else %false.
95 */
96bool raid6_can_xor_syndrome(void)
97{
98 return !!static_call_query(raid6_xor_syndrome_impl);
99}
100EXPORT_SYMBOL_GPL(raid6_can_xor_syndrome);
101
102/**
103 * raid6_recov_2data - recover two missing data disks
104 * @disks: number of "disks" to operate on including parity
105 * @bytes: length in bytes of each vector
106 * @faila: first failed data disk index
107 * @failb: second failed data disk index
108 * @ptrs: @disks size array of memory pointers
109 *
110 * Rebuild @bytes of missing data in @ptrs[@faila] and @ptrs[@failb] from the
111 * data in the remaining disks and the two parities pointed to by the other
112 * indices between 0 and @disks - 1 in @ptrs. @disks includes the data disks
113 * and the two parities. @faila must be smaller than @failb.
114 *
115 * Memory pointed to by each pointer in @ptrs must be page aligned and is
116 * limited to %PAGE_SIZE.
117 */
118void raid6_recov_2data(int disks, size_t bytes, int faila, int failb,
119 void **ptrs)
120{
121 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count());
122 WARN_ON_ONCE(bytes & 511);
123 WARN_ON_ONCE(bytes > PAGE_SIZE);
124 WARN_ON_ONCE(failb <= faila);
125
126 static_call(raid6_recov_2data_impl)(disks, bytes, faila, failb, ptrs);
127}
128EXPORT_SYMBOL_GPL(raid6_recov_2data);
129
130/**
131 * raid6_recov_datap - recover a missing data disk and missing P-parity
132 * @disks: number of "disks" to operate on including parity
133 * @bytes: length in bytes of each vector
134 * @faila: failed data disk index
135 * @ptrs: @disks size array of memory pointers
136 *
137 * Rebuild @bytes of missing data in @ptrs[@faila] and the missing P-parity in
138 * @ptrs[@disks - 2] from the data in the remaining disks and the Q-parity
139 * pointed to by the other indices between 0 and @disks - 1 in @ptrs. @disks
140 * includes the data disks and the two parities.
141 *
142 * Memory pointed to by each pointer in @ptrs must be page aligned and is
143 * limited to %PAGE_SIZE.
144 */
145void raid6_recov_datap(int disks, size_t bytes, int faila, void **ptrs)
146{
147 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count());
148 WARN_ON_ONCE(bytes & 511);
149 WARN_ON_ONCE(bytes > PAGE_SIZE);
150
151 static_call(raid6_recov_datap_impl)(disks, bytes, faila, ptrs);
152}
153EXPORT_SYMBOL_GPL(raid6_recov_datap);
154
155#define BENCH_SIZE SZ_4K
156#define NR_SRCS 8
157#define NR_DISKS (NR_SRCS + 2)
158#define REPS 800U
159
160static int raid6_choose_gen(void *dptrs[NR_DISKS], const int disks)
161{
162 const struct raid6_calls *best = NULL;
163 unsigned long bestgenperf = 0;
164 unsigned int i;
165
166 for (i = 0; i < raid6_nr_algos; i++) {
167 const struct raid6_calls *algo = raid6_algos[i];
168 unsigned long perf = 0;
169 u64 t;
170 int i;
171
172 preempt_disable();
173 t = ktime_get_ns();
174 for (i = 0; i < REPS; i++)
175 algo->gen_syndrome(disks, BENCH_SIZE, dptrs);
176 t = max(ktime_get_ns() - t, 1);
177 preempt_enable();
178
179 /* bytes/ns == GB/s, multiply by 1000 to get MB/s [not MiB/s] */
180 perf = div64_u64((u64)BENCH_SIZE * REPS * NR_SRCS * 1000, t);
181 if (perf > bestgenperf) {
182 bestgenperf = perf;
183 best = algo;
184 }
185 pr_info("raid6: %-8s gen() %5lu MB/s\n", algo->name, perf);
186 }
187
188 if (!best) {
189 pr_err("raid6: Yikes! No algorithm found!\n");
190 return -EINVAL;
191 }
192
193 static_call_update(raid6_gen_syndrome_impl, best->gen_syndrome);
194 static_call_update(raid6_xor_syndrome_impl, best->xor_syndrome);
195
196 pr_info("raid6: using algorithm %s gen() %ld MB/s\n",
197 best->name, bestgenperf);
198
199 if (best->xor_syndrome) {
200 /* work on the second half of the disks */
201 int start = (disks / 2) - 1, stop = disks - 3;
202 u64 t;
203
204 preempt_disable();
205 t = ktime_get_ns();
206 for (i = 0; i < REPS; i++)
207 best->xor_syndrome(disks, start, stop, BENCH_SIZE,
208 dptrs);
209 t = max(ktime_get_ns() - t, 1);
210 preempt_enable();
211
212 pr_info("raid6: .... xor() %llu MB/s, rmw enabled\n",
213 div64_u64((u64)BENCH_SIZE * REPS * NR_SRCS / 2 * 1000,
214 t));
215 }
216
217 return 0;
218}
219
220
221/* Try to pick the best algorithm */
222/* This code uses the gfmul table as convenient data set to abuse */
223
224static int __init raid6_select_algo(void)
225{
226 const int disks = NR_DISKS;
227 void *dptrs[NR_DISKS];
228 char *disk_ptr, *p;
229 int i, cycle;
230 int error;
231
232 if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK) || raid6_nr_algos == 1) {
233 pr_info("raid6: skipped pq benchmark and selected %s\n",
234 raid6_algos[raid6_nr_algos - 1]->name);
235 return 0;
236 }
237
238 /* prepare the buffer and fill it circularly with gfmul table */
239 disk_ptr = kmalloc_array(NR_DISKS, BENCH_SIZE, GFP_KERNEL);
240 if (!disk_ptr) {
241 pr_err("raid6: Yikes! No memory available.\n");
242 return -ENOMEM;
243 }
244
245 p = disk_ptr;
246 for (i = 0; i < disks; i++)
247 dptrs[i] = p + BENCH_SIZE * i;
248
249 cycle = ((disks - 2) * BENCH_SIZE) / 65536;
250 for (i = 0; i < cycle; i++) {
251 memcpy(p, raid6_gfmul, 65536);
252 p += 65536;
253 }
254
255 if ((disks - 2) * BENCH_SIZE % 65536)
256 memcpy(p, raid6_gfmul, (disks - 2) * BENCH_SIZE % 65536);
257
258 /* select raid gen_syndrome function */
259 error = raid6_choose_gen(dptrs, disks);
260
261 kfree(disk_ptr);
262
263 return error;
264}
265
266/*
267 * Register a RAID6 P/Q generation algorithm. The most optimized/unrolled
268 * implementation should be registered last so it will be selected when the
269 * boot-time benchmark is disabled.
270 */
271void __init raid6_algo_add(const struct raid6_calls *algo)
272{
273 if (WARN_ON_ONCE(raid6_nr_algos == RAID6_MAX_ALGOS))
274 return;
275 raid6_algos[raid6_nr_algos++] = algo;
276}
277
278void __init raid6_algo_add_default(void)
279{
280 raid6_algo_add(&raid6_intx1);
281 raid6_algo_add(&raid6_intx2);
282 raid6_algo_add(&raid6_intx4);
283 raid6_algo_add(&raid6_intx8);
284}
285
286void __init raid6_recov_algo_add(const struct raid6_recov_calls *algo)
287{
288 if (WARN_ON_ONCE(raid6_recov_algo))
289 return;
290 raid6_recov_algo = algo;
291}
292
293#ifdef CONFIG_RAID6_PQ_ARCH
294#include "pq_arch.h"
295#else
296static inline void arch_raid6_init(void)
297{
298 raid6_algo_add_default();
299}
300#endif /* CONFIG_RAID6_PQ_ARCH */
301
302static int __init raid6_init(void)
303{
304 /*
305 * Architectures providing arch_raid6_init must add all PQ generation
306 * algorithms they want to consider in arch_raid6_init(), including
307 * the generic ones using raid6_algo_add_default() if wanted.
308 */
309 arch_raid6_init();
310
311 /*
312 * Architectures don't have to set a recovery algorithm, we'll just pick
313 * the generic integer one if none was set.
314 */
315 if (!raid6_recov_algo)
316 raid6_recov_algo = &raid6_recov_intx1;
317 static_call_update(raid6_recov_2data_impl, raid6_recov_algo->data2);
318 static_call_update(raid6_recov_datap_impl, raid6_recov_algo->datap);
319 pr_info("raid6: using %s recovery algorithm\n", raid6_recov_algo->name);
320
321 /*
322 * Pick the last registered implementation as the temporary default until
323 * calibration happens.
324 */
325 static_call_update(raid6_gen_syndrome_impl,
326 raid6_algos[raid6_nr_algos - 1]->gen_syndrome);
327 static_call_update(raid6_xor_syndrome_impl,
328 raid6_algos[raid6_nr_algos - 1]->xor_syndrome);
329
330#ifdef MODULE
331 return raid6_select_algo();
332#else
333 return 0;
334#endif
335}
336
337static void __exit raid6_exit(void)
338{
339}
340
341/*
342 * When built-in we must register the default implementation before md
343 * initializes, but we don't want calibration to run that early as that
344 * would delay the boot process.
345 */
346#ifndef MODULE
347device_initcall(raid6_select_algo);
348#endif
349subsys_initcall(raid6_init);
350module_exit(raid6_exit);
351MODULE_LICENSE("GPL");
352MODULE_DESCRIPTION("RAID6 Q-syndrome calculations");
353
354#if IS_ENABLED(CONFIG_RAID6_PQ_KUNIT_TEST)
355const struct raid6_calls *raid6_algo_find(unsigned int idx)
356{
357 if (idx >= raid6_nr_algos) {
358 /*
359 * Always include the simplest generic integer implementation in
360 * the unit tests as a baseline.
361 */
362 if (idx == raid6_nr_algos &&
363 raid6_algos[0] != &raid6_intx1)
364 return &raid6_intx1;
365 return NULL;
366 }
367 return raid6_algos[idx];
368}
369EXPORT_SYMBOL_IF_KUNIT(raid6_algo_find);
370
371const struct raid6_recov_calls *raid6_recov_algo_find(unsigned int idx)
372{
373 switch (idx) {
374 case 0:
375 /* always test the generic integer implementation */
376 return &raid6_recov_intx1;
377 case 1:
378 /* test the optimized implementation if there is one */
379 if (raid6_recov_algo != &raid6_recov_intx1)
380 return raid6_recov_algo;
381 return NULL;
382 default:
383 return NULL;
384 }
385}
386EXPORT_SYMBOL_IF_KUNIT(raid6_recov_algo_find);
387#endif /* CONFIG_RAID6_PQ_KUNIT_TEST */