projects/linux·blob master · refs

1450 lines · 39.4 KiB

1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * Module-based torture test facility for locking
4 *
5 * Copyright (C) IBM Corporation, 2014
6 *
7 * Authors: Paul E. McKenney <[email protected]>
8 * Davidlohr Bueso <[email protected]>
9 * Based on kernel/rcu/torture.c.
10 */
11
12#define pr_fmt(fmt) fmt
13
14#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/kthread.h>
17#include <linux/sched/rt.h>
18#include <linux/spinlock.h>
19#include <linux/mutex.h>
20#include <linux/rwsem.h>
21#include <linux/smp.h>
22#include <linux/interrupt.h>
23#include <linux/sched.h>
24#include <uapi/linux/sched/types.h>
25#include <linux/rtmutex.h>
26#include <linux/atomic.h>
27#include <linux/moduleparam.h>
28#include <linux/delay.h>
29#include <linux/slab.h>
30#include <linux/torture.h>
31#include <linux/reboot.h>
32
33MODULE_DESCRIPTION("torture test facility for locking");
34MODULE_LICENSE("GPL");
35MODULE_AUTHOR("Paul E. McKenney <[email protected]>");
36
37torture_param(int, acq_writer_lim, 0, "Write_acquisition time limit (jiffies).");
38torture_param(int, call_rcu_chains, 0, "Self-propagate call_rcu() chains during test (0=disable).");
39torture_param(int, long_hold, 100, "Do occasional long hold of lock (ms), 0=disable");
40torture_param(int, nested_locks, 0, "Number of nested locks (max = 8)");
41torture_param(int, nreaders_stress, -1, "Number of read-locking stress-test threads");
42torture_param(int, nwriters_stress, -1, "Number of write-locking stress-test threads");
43torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
44torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (s), 0=disable");
45torture_param(int, rt_boost, 2,
46 "Do periodic rt-boost. 0=Disable, 1=Only for rt_mutex, 2=For all lock types.");
47torture_param(int, rt_boost_factor, 50, "A factor determining how often rt-boost happens.");
48torture_param(int, shuffle_interval, 3, "Number of jiffies between shuffles, 0=disable");
49torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable.");
50torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s");
51torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable");
52torture_param(int, verbose, 1, "Enable verbose debugging printk()s");
53torture_param(int, writer_fifo, 0, "Run writers at sched_set_fifo() priority");
54/* Going much higher trips "BUG: MAX_LOCKDEP_CHAIN_HLOCKS too low!" errors */
55#define MAX_NESTED_LOCKS 8
56
57static char *torture_type = IS_ENABLED(CONFIG_PREEMPT_RT) ? "raw_spin_lock" : "spin_lock";
58module_param(torture_type, charp, 0444);
59MODULE_PARM_DESC(torture_type,
60 "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)");
61
62static cpumask_var_t bind_readers; // Bind the readers to the specified set of CPUs.
63static cpumask_var_t bind_writers; // Bind the writers to the specified set of CPUs.
64
65// Parse a cpumask kernel parameter. If there are more users later on,
66// this might need to got to a more central location.
67static int param_set_cpumask(const char *val, const struct kernel_param *kp)
68{
69 cpumask_var_t *cm_bind = kp->arg;
70 int ret;
71 char *s;
72
73 if (!alloc_cpumask_var(cm_bind, GFP_KERNEL)) {
74 s = "Out of memory";
75 ret = -ENOMEM;
76 goto out_err;
77 }
78 ret = cpulist_parse(val, *cm_bind);
79 if (!ret)
80 return ret;
81 s = "Bad CPU range";
82out_err:
83 pr_warn("%s: %s, all CPUs set\n", kp->name, s);
84 cpumask_setall(*cm_bind);
85 return ret;
86}
87
88// Output a cpumask kernel parameter.
89static int param_get_cpumask(char *buffer, const struct kernel_param *kp)
90{
91 cpumask_var_t *cm_bind = kp->arg;
92
93 return sprintf(buffer, "%*pbl", cpumask_pr_args(*cm_bind));
94}
95
96static bool cpumask_nonempty(cpumask_var_t mask)
97{
98 return cpumask_available(mask) && !cpumask_empty(mask);
99}
100
101static const struct kernel_param_ops lt_bind_ops = {
102 .set = param_set_cpumask,
103 .get = param_get_cpumask,
104};
105
106module_param_cb(bind_readers, &lt_bind_ops, &bind_readers, 0444);
107module_param_cb(bind_writers, &lt_bind_ops, &bind_writers, 0444);
108
109long torture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask, bool dowarn);
110
111static struct task_struct *stats_task;
112static struct task_struct **writer_tasks;
113static struct task_struct **reader_tasks;
114
115static bool lock_is_write_held;
116static atomic_t lock_is_read_held;
117static unsigned long last_lock_release;
118
119struct lock_stress_stats {
120 long n_lock_fail;
121 long n_lock_acquired;
122};
123
124struct call_rcu_chain {
125 struct rcu_head crc_rh;
126 bool crc_stop;
127};
128struct call_rcu_chain *call_rcu_chain_list;
129
130/* Forward reference. */
131static void lock_torture_cleanup(void);
132
133/*
134 * Operations vector for selecting different types of tests.
135 */
136struct lock_torture_ops {
137 void (*init)(void);
138 void (*exit)(void);
139 int (*nested_lock)(int tid, u32 lockset);
140 int (*writelock)(int tid);
141 void (*write_delay)(struct torture_random_state *trsp);
142 void (*task_boost)(struct torture_random_state *trsp);
143 void (*writeunlock)(int tid);
144 void (*nested_unlock)(int tid, u32 lockset);
145 int (*readlock)(int tid);
146 void (*read_delay)(struct torture_random_state *trsp);
147 void (*readunlock)(int tid);
148
149 unsigned long flags; /* for irq spinlocks */
150 const char *name;
151};
152
153struct lock_torture_cxt {
154 int nrealwriters_stress;
155 int nrealreaders_stress;
156 bool debug_lock;
157 bool init_called;
158 atomic_t n_lock_torture_errors;
159 struct lock_torture_ops *cur_ops;
160 struct lock_stress_stats *lwsa; /* writer statistics */
161 struct lock_stress_stats *lrsa; /* reader statistics */
162};
163static struct lock_torture_cxt cxt = { 0, 0, false, false,
164 ATOMIC_INIT(0),
165 NULL, NULL};
166/*
167 * Definitions for lock torture testing.
168 */
169
170static int torture_lock_busted_write_lock(int tid __maybe_unused)
171{
172 return 0; /* BUGGY, do not use in real life!!! */
173}
174
175static void torture_lock_busted_write_delay(struct torture_random_state *trsp)
176{
177 /* We want a long delay occasionally to force massive contention. */
178 if (long_hold && !(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * long_hold)))
179 mdelay(long_hold);
180 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
181 torture_preempt_schedule(); /* Allow test to be preempted. */
182}
183
184static void torture_lock_busted_write_unlock(int tid __maybe_unused)
185{
186 /* BUGGY, do not use in real life!!! */
187}
188
189static void __torture_rt_boost(struct torture_random_state *trsp)
190{
191 const unsigned int factor = rt_boost_factor;
192
193 if (!rt_task(current)) {
194 /*
195 * Boost priority once every rt_boost_factor operations. When
196 * the task tries to take the lock, the rtmutex it will account
197 * for the new priority, and do any corresponding pi-dance.
198 */
199 if (trsp && !(torture_random(trsp) %
200 (cxt.nrealwriters_stress * factor))) {
201 sched_set_fifo(current);
202 } else /* common case, do nothing */
203 return;
204 } else {
205 /*
206 * The task will remain boosted for another 10 * rt_boost_factor
207 * operations, then restored back to its original prio, and so
208 * forth.
209 *
210 * When @trsp is nil, we want to force-reset the task for
211 * stopping the kthread.
212 */
213 if (!trsp || !(torture_random(trsp) %
214 (cxt.nrealwriters_stress * factor * 2))) {
215 sched_set_normal(current, 0);
216 } else /* common case, do nothing */
217 return;
218 }
219}
220
221static void torture_rt_boost(struct torture_random_state *trsp)
222{
223 if (rt_boost != 2)
224 return;
225
226 __torture_rt_boost(trsp);
227}
228
229static struct lock_torture_ops lock_busted_ops = {
230 .writelock = torture_lock_busted_write_lock,
231 .write_delay = torture_lock_busted_write_delay,
232 .task_boost = torture_rt_boost,
233 .writeunlock = torture_lock_busted_write_unlock,
234 .readlock = NULL,
235 .read_delay = NULL,
236 .readunlock = NULL,
237 .name = "lock_busted"
238};
239
240static DEFINE_SPINLOCK(torture_spinlock);
241
242static int torture_spin_lock_write_lock(int tid __maybe_unused)
243__acquires(torture_spinlock)
244{
245 spin_lock(&torture_spinlock);
246 return 0;
247}
248
249static void torture_spin_lock_write_delay(struct torture_random_state *trsp)
250{
251 const unsigned long shortdelay_us = 2;
252 unsigned long j;
253
254 /* We want a short delay mostly to emulate likely code, and
255 * we want a long delay occasionally to force massive contention.
256 */
257 if (long_hold && !(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * long_hold))) {
258 j = jiffies;
259 mdelay(long_hold);
260 pr_alert("%s: delay = %lu jiffies.\n", __func__, jiffies - j);
261 }
262 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 200 * shortdelay_us)))
263 udelay(shortdelay_us);
264 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
265 torture_preempt_schedule(); /* Allow test to be preempted. */
266}
267
268static void torture_spin_lock_write_unlock(int tid __maybe_unused)
269__releases(torture_spinlock)
270{
271 spin_unlock(&torture_spinlock);
272}
273
274static struct lock_torture_ops spin_lock_ops = {
275 .writelock = torture_spin_lock_write_lock,
276 .write_delay = torture_spin_lock_write_delay,
277 .task_boost = torture_rt_boost,
278 .writeunlock = torture_spin_lock_write_unlock,
279 .readlock = NULL,
280 .read_delay = NULL,
281 .readunlock = NULL,
282 .name = "spin_lock"
283};
284
285static int torture_spin_lock_write_lock_irq(int tid __maybe_unused)
286__acquires(torture_spinlock)
287{
288 unsigned long flags;
289
290 spin_lock_irqsave(&torture_spinlock, flags);
291 cxt.cur_ops->flags = flags;
292 return 0;
293}
294
295static void torture_lock_spin_write_unlock_irq(int tid __maybe_unused)
296__releases(torture_spinlock)
297{
298 spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags);
299}
300
301static struct lock_torture_ops spin_lock_irq_ops = {
302 .writelock = torture_spin_lock_write_lock_irq,
303 .write_delay = torture_spin_lock_write_delay,
304 .task_boost = torture_rt_boost,
305 .writeunlock = torture_lock_spin_write_unlock_irq,
306 .readlock = NULL,
307 .read_delay = NULL,
308 .readunlock = NULL,
309 .name = "spin_lock_irq"
310};
311
312static DEFINE_RAW_SPINLOCK(torture_raw_spinlock);
313
314static int torture_raw_spin_lock_write_lock(int tid __maybe_unused)
315__acquires(torture_raw_spinlock)
316{
317 raw_spin_lock(&torture_raw_spinlock);
318 return 0;
319}
320
321static void torture_raw_spin_lock_write_unlock(int tid __maybe_unused)
322__releases(torture_raw_spinlock)
323{
324 raw_spin_unlock(&torture_raw_spinlock);
325}
326
327static struct lock_torture_ops raw_spin_lock_ops = {
328 .writelock = torture_raw_spin_lock_write_lock,
329 .write_delay = torture_spin_lock_write_delay,
330 .task_boost = torture_rt_boost,
331 .writeunlock = torture_raw_spin_lock_write_unlock,
332 .readlock = NULL,
333 .read_delay = NULL,
334 .readunlock = NULL,
335 .name = "raw_spin_lock"
336};
337
338static int torture_raw_spin_lock_write_lock_irq(int tid __maybe_unused)
339__acquires(torture_raw_spinlock)
340{
341 unsigned long flags;
342
343 raw_spin_lock_irqsave(&torture_raw_spinlock, flags);
344 cxt.cur_ops->flags = flags;
345 return 0;
346}
347
348static void torture_raw_spin_lock_write_unlock_irq(int tid __maybe_unused)
349__releases(torture_raw_spinlock)
350{
351 raw_spin_unlock_irqrestore(&torture_raw_spinlock, cxt.cur_ops->flags);
352}
353
354static struct lock_torture_ops raw_spin_lock_irq_ops = {
355 .writelock = torture_raw_spin_lock_write_lock_irq,
356 .write_delay = torture_spin_lock_write_delay,
357 .task_boost = torture_rt_boost,
358 .writeunlock = torture_raw_spin_lock_write_unlock_irq,
359 .readlock = NULL,
360 .read_delay = NULL,
361 .readunlock = NULL,
362 .name = "raw_spin_lock_irq"
363};
364
365#ifdef CONFIG_BPF_SYSCALL
366
367#include <asm/rqspinlock.h>
368static rqspinlock_t rqspinlock;
369
370static int torture_raw_res_spin_write_lock(int tid __maybe_unused)
371{
372 raw_res_spin_lock(&rqspinlock);
373 return 0;
374}
375
376static void torture_raw_res_spin_write_unlock(int tid __maybe_unused)
377{
378 raw_res_spin_unlock(&rqspinlock);
379}
380
381static struct lock_torture_ops raw_res_spin_lock_ops = {
382 .writelock = torture_raw_res_spin_write_lock,
383 .write_delay = torture_spin_lock_write_delay,
384 .task_boost = torture_rt_boost,
385 .writeunlock = torture_raw_res_spin_write_unlock,
386 .readlock = NULL,
387 .read_delay = NULL,
388 .readunlock = NULL,
389 .name = "raw_res_spin_lock"
390};
391
392static int torture_raw_res_spin_write_lock_irq(int tid __maybe_unused)
393{
394 unsigned long flags;
395
396 raw_res_spin_lock_irqsave(&rqspinlock, flags);
397 cxt.cur_ops->flags = flags;
398 return 0;
399}
400
401static void torture_raw_res_spin_write_unlock_irq(int tid __maybe_unused)
402{
403 raw_res_spin_unlock_irqrestore(&rqspinlock, cxt.cur_ops->flags);
404}
405
406static struct lock_torture_ops raw_res_spin_lock_irq_ops = {
407 .writelock = torture_raw_res_spin_write_lock_irq,
408 .write_delay = torture_spin_lock_write_delay,
409 .task_boost = torture_rt_boost,
410 .writeunlock = torture_raw_res_spin_write_unlock_irq,
411 .readlock = NULL,
412 .read_delay = NULL,
413 .readunlock = NULL,
414 .name = "raw_res_spin_lock_irq"
415};
416
417#endif
418
419static DEFINE_RWLOCK(torture_rwlock);
420
421static int torture_rwlock_write_lock(int tid __maybe_unused)
422__acquires(torture_rwlock)
423{
424 write_lock(&torture_rwlock);
425 return 0;
426}
427
428static void torture_rwlock_write_delay(struct torture_random_state *trsp)
429{
430 const unsigned long shortdelay_us = 2;
431
432 /* We want a short delay mostly to emulate likely code, and
433 * we want a long delay occasionally to force massive contention.
434 */
435 if (long_hold && !(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * long_hold)))
436 mdelay(long_hold);
437 else
438 udelay(shortdelay_us);
439}
440
441static void torture_rwlock_write_unlock(int tid __maybe_unused)
442__releases(torture_rwlock)
443{
444 write_unlock(&torture_rwlock);
445}
446
447static int torture_rwlock_read_lock(int tid __maybe_unused)
448__acquires(torture_rwlock)
449{
450 read_lock(&torture_rwlock);
451 return 0;
452}
453
454static void torture_rwlock_read_delay(struct torture_random_state *trsp)
455{
456 const unsigned long shortdelay_us = 10;
457
458 /* We want a short delay mostly to emulate likely code, and
459 * we want a long delay occasionally to force massive contention.
460 */
461 if (long_hold && !(torture_random(trsp) % (cxt.nrealreaders_stress * 2000 * long_hold)))
462 mdelay(long_hold);
463 else
464 udelay(shortdelay_us);
465}
466
467static void torture_rwlock_read_unlock(int tid __maybe_unused)
468__releases(torture_rwlock)
469{
470 read_unlock(&torture_rwlock);
471}
472
473static struct lock_torture_ops rw_lock_ops = {
474 .writelock = torture_rwlock_write_lock,
475 .write_delay = torture_rwlock_write_delay,
476 .task_boost = torture_rt_boost,
477 .writeunlock = torture_rwlock_write_unlock,
478 .readlock = torture_rwlock_read_lock,
479 .read_delay = torture_rwlock_read_delay,
480 .readunlock = torture_rwlock_read_unlock,
481 .name = "rw_lock"
482};
483
484static int torture_rwlock_write_lock_irq(int tid __maybe_unused)
485__acquires(torture_rwlock)
486{
487 unsigned long flags;
488
489 write_lock_irqsave(&torture_rwlock, flags);
490 cxt.cur_ops->flags = flags;
491 return 0;
492}
493
494static void torture_rwlock_write_unlock_irq(int tid __maybe_unused)
495__releases(torture_rwlock)
496{
497 write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
498}
499
500static int torture_rwlock_read_lock_irq(int tid __maybe_unused)
501__acquires(torture_rwlock)
502{
503 unsigned long flags;
504
505 read_lock_irqsave(&torture_rwlock, flags);
506 cxt.cur_ops->flags = flags;
507 return 0;
508}
509
510static void torture_rwlock_read_unlock_irq(int tid __maybe_unused)
511__releases(torture_rwlock)
512{
513 read_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
514}
515
516static struct lock_torture_ops rw_lock_irq_ops = {
517 .writelock = torture_rwlock_write_lock_irq,
518 .write_delay = torture_rwlock_write_delay,
519 .task_boost = torture_rt_boost,
520 .writeunlock = torture_rwlock_write_unlock_irq,
521 .readlock = torture_rwlock_read_lock_irq,
522 .read_delay = torture_rwlock_read_delay,
523 .readunlock = torture_rwlock_read_unlock_irq,
524 .name = "rw_lock_irq"
525};
526
527static DEFINE_MUTEX(torture_mutex);
528static struct mutex torture_nested_mutexes[MAX_NESTED_LOCKS];
529static struct lock_class_key nested_mutex_keys[MAX_NESTED_LOCKS];
530
531static void torture_mutex_init(void)
532{
533 int i;
534
535 for (i = 0; i < MAX_NESTED_LOCKS; i++)
536 __mutex_init(&torture_nested_mutexes[i], __func__,
537 &nested_mutex_keys[i]);
538}
539
540static int torture_mutex_nested_lock(int tid __maybe_unused,
541 u32 lockset)
542{
543 int i;
544
545 for (i = 0; i < nested_locks; i++)
546 if (lockset & (1 << i))
547 mutex_lock(&torture_nested_mutexes[i]);
548 return 0;
549}
550
551static int torture_mutex_lock(int tid __maybe_unused)
552__acquires(torture_mutex)
553{
554 mutex_lock(&torture_mutex);
555 return 0;
556}
557
558static void torture_mutex_delay(struct torture_random_state *trsp)
559{
560 /* We want a long delay occasionally to force massive contention. */
561 if (long_hold && !(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * long_hold)))
562 mdelay(long_hold * 5);
563 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
564 torture_preempt_schedule(); /* Allow test to be preempted. */
565}
566
567static void torture_mutex_unlock(int tid __maybe_unused)
568__releases(torture_mutex)
569{
570 mutex_unlock(&torture_mutex);
571}
572
573static void torture_mutex_nested_unlock(int tid __maybe_unused,
574 u32 lockset)
575{
576 int i;
577
578 for (i = nested_locks - 1; i >= 0; i--)
579 if (lockset & (1 << i))
580 mutex_unlock(&torture_nested_mutexes[i]);
581}
582
583static struct lock_torture_ops mutex_lock_ops = {
584 .init = torture_mutex_init,
585 .nested_lock = torture_mutex_nested_lock,
586 .writelock = torture_mutex_lock,
587 .write_delay = torture_mutex_delay,
588 .task_boost = torture_rt_boost,
589 .writeunlock = torture_mutex_unlock,
590 .nested_unlock = torture_mutex_nested_unlock,
591 .readlock = NULL,
592 .read_delay = NULL,
593 .readunlock = NULL,
594 .name = "mutex_lock"
595};
596
597#include <linux/ww_mutex.h>
598/*
599 * The torture ww_mutexes should belong to the same lock class as
600 * torture_ww_class to avoid lockdep problem. The ww_mutex_init()
601 * function is called for initialization to ensure that.
602 */
603static DEFINE_WD_CLASS(torture_ww_class);
604static struct ww_mutex torture_ww_mutex_0, torture_ww_mutex_1, torture_ww_mutex_2;
605static struct ww_acquire_ctx *ww_acquire_ctxs;
606
607static void torture_ww_mutex_init(void)
608{
609 ww_mutex_init(&torture_ww_mutex_0, &torture_ww_class);
610 ww_mutex_init(&torture_ww_mutex_1, &torture_ww_class);
611 ww_mutex_init(&torture_ww_mutex_2, &torture_ww_class);
612
613 ww_acquire_ctxs = kmalloc_objs(*ww_acquire_ctxs,
614 cxt.nrealwriters_stress);
615 if (!ww_acquire_ctxs)
616 VERBOSE_TOROUT_STRING("ww_acquire_ctx: Out of memory");
617}
618
619static void torture_ww_mutex_exit(void)
620{
621 kfree(ww_acquire_ctxs);
622}
623
624static int torture_ww_mutex_lock(int tid)
625__acquires(torture_ww_mutex_0)
626__acquires(torture_ww_mutex_1)
627__acquires(torture_ww_mutex_2)
628{
629 LIST_HEAD(list);
630 struct reorder_lock {
631 struct list_head link;
632 struct ww_mutex *lock;
633 } locks[3], *ll, *ln;
634 struct ww_acquire_ctx *ctx = &ww_acquire_ctxs[tid];
635
636 locks[0].lock = &torture_ww_mutex_0;
637 list_add(&locks[0].link, &list);
638
639 locks[1].lock = &torture_ww_mutex_1;
640 list_add(&locks[1].link, &list);
641
642 locks[2].lock = &torture_ww_mutex_2;
643 list_add(&locks[2].link, &list);
644
645 ww_acquire_init(ctx, &torture_ww_class);
646
647 list_for_each_entry(ll, &list, link) {
648 int err;
649
650 err = ww_mutex_lock(ll->lock, ctx);
651 if (!err)
652 continue;
653
654 ln = ll;
655 list_for_each_entry_continue_reverse(ln, &list, link)
656 ww_mutex_unlock(ln->lock);
657
658 if (err != -EDEADLK)
659 return err;
660
661 ww_mutex_lock_slow(ll->lock, ctx);
662 list_move(&ll->link, &list);
663 }
664
665 return 0;
666}
667
668static void torture_ww_mutex_unlock(int tid)
669__releases(torture_ww_mutex_0)
670__releases(torture_ww_mutex_1)
671__releases(torture_ww_mutex_2)
672{
673 struct ww_acquire_ctx *ctx = &ww_acquire_ctxs[tid];
674
675 ww_mutex_unlock(&torture_ww_mutex_0);
676 ww_mutex_unlock(&torture_ww_mutex_1);
677 ww_mutex_unlock(&torture_ww_mutex_2);
678 ww_acquire_fini(ctx);
679}
680
681static struct lock_torture_ops ww_mutex_lock_ops = {
682 .init = torture_ww_mutex_init,
683 .exit = torture_ww_mutex_exit,
684 .writelock = torture_ww_mutex_lock,
685 .write_delay = torture_mutex_delay,
686 .task_boost = torture_rt_boost,
687 .writeunlock = torture_ww_mutex_unlock,
688 .readlock = NULL,
689 .read_delay = NULL,
690 .readunlock = NULL,
691 .name = "ww_mutex_lock"
692};
693
694#ifdef CONFIG_RT_MUTEXES
695static DEFINE_RT_MUTEX(torture_rtmutex);
696static struct rt_mutex torture_nested_rtmutexes[MAX_NESTED_LOCKS];
697static struct lock_class_key nested_rtmutex_keys[MAX_NESTED_LOCKS];
698
699static void torture_rtmutex_init(void)
700{
701 int i;
702
703 for (i = 0; i < MAX_NESTED_LOCKS; i++)
704 __rt_mutex_init(&torture_nested_rtmutexes[i], __func__,
705 &nested_rtmutex_keys[i]);
706}
707
708static int torture_rtmutex_nested_lock(int tid __maybe_unused,
709 u32 lockset)
710{
711 int i;
712
713 for (i = 0; i < nested_locks; i++)
714 if (lockset & (1 << i))
715 rt_mutex_lock(&torture_nested_rtmutexes[i]);
716 return 0;
717}
718
719static int torture_rtmutex_lock(int tid __maybe_unused)
720__acquires(torture_rtmutex)
721{
722 rt_mutex_lock(&torture_rtmutex);
723 return 0;
724}
725
726static void torture_rtmutex_delay(struct torture_random_state *trsp)
727{
728 const unsigned long shortdelay_us = 2;
729
730 /*
731 * We want a short delay mostly to emulate likely code, and
732 * we want a long delay occasionally to force massive contention.
733 */
734 if (long_hold && !(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * long_hold)))
735 mdelay(long_hold);
736 if (!(torture_random(trsp) %
737 (cxt.nrealwriters_stress * 200 * shortdelay_us)))
738 udelay(shortdelay_us);
739 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
740 torture_preempt_schedule(); /* Allow test to be preempted. */
741}
742
743static void torture_rtmutex_unlock(int tid __maybe_unused)
744__releases(torture_rtmutex)
745{
746 rt_mutex_unlock(&torture_rtmutex);
747}
748
749static void torture_rt_boost_rtmutex(struct torture_random_state *trsp)
750{
751 if (!rt_boost)
752 return;
753
754 __torture_rt_boost(trsp);
755}
756
757static void torture_rtmutex_nested_unlock(int tid __maybe_unused,
758 u32 lockset)
759{
760 int i;
761
762 for (i = nested_locks - 1; i >= 0; i--)
763 if (lockset & (1 << i))
764 rt_mutex_unlock(&torture_nested_rtmutexes[i]);
765}
766
767static struct lock_torture_ops rtmutex_lock_ops = {
768 .init = torture_rtmutex_init,
769 .nested_lock = torture_rtmutex_nested_lock,
770 .writelock = torture_rtmutex_lock,
771 .write_delay = torture_rtmutex_delay,
772 .task_boost = torture_rt_boost_rtmutex,
773 .writeunlock = torture_rtmutex_unlock,
774 .nested_unlock = torture_rtmutex_nested_unlock,
775 .readlock = NULL,
776 .read_delay = NULL,
777 .readunlock = NULL,
778 .name = "rtmutex_lock"
779};
780#endif
781
782static DECLARE_RWSEM(torture_rwsem);
783static int torture_rwsem_down_write(int tid __maybe_unused)
784__acquires(torture_rwsem)
785{
786 down_write(&torture_rwsem);
787 return 0;
788}
789
790static void torture_rwsem_write_delay(struct torture_random_state *trsp)
791{
792 /* We want a long delay occasionally to force massive contention. */
793 if (long_hold && !(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * long_hold)))
794 mdelay(long_hold * 10);
795 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
796 torture_preempt_schedule(); /* Allow test to be preempted. */
797}
798
799static void torture_rwsem_up_write(int tid __maybe_unused)
800__releases(torture_rwsem)
801{
802 up_write(&torture_rwsem);
803}
804
805static int torture_rwsem_down_read(int tid __maybe_unused)
806__acquires(torture_rwsem)
807{
808 down_read(&torture_rwsem);
809 return 0;
810}
811
812static void torture_rwsem_read_delay(struct torture_random_state *trsp)
813{
814 /* We want a long delay occasionally to force massive contention. */
815 if (long_hold && !(torture_random(trsp) % (cxt.nrealreaders_stress * 2000 * long_hold)))
816 mdelay(long_hold * 2);
817 else
818 mdelay(long_hold / 2);
819 if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000)))
820 torture_preempt_schedule(); /* Allow test to be preempted. */
821}
822
823static void torture_rwsem_up_read(int tid __maybe_unused)
824__releases(torture_rwsem)
825{
826 up_read(&torture_rwsem);
827}
828
829static struct lock_torture_ops rwsem_lock_ops = {
830 .writelock = torture_rwsem_down_write,
831 .write_delay = torture_rwsem_write_delay,
832 .task_boost = torture_rt_boost,
833 .writeunlock = torture_rwsem_up_write,
834 .readlock = torture_rwsem_down_read,
835 .read_delay = torture_rwsem_read_delay,
836 .readunlock = torture_rwsem_up_read,
837 .name = "rwsem_lock"
838};
839
840#include <linux/percpu-rwsem.h>
841static struct percpu_rw_semaphore pcpu_rwsem;
842
843static void torture_percpu_rwsem_init(void)
844{
845 BUG_ON(percpu_init_rwsem(&pcpu_rwsem));
846}
847
848static void torture_percpu_rwsem_exit(void)
849{
850 percpu_free_rwsem(&pcpu_rwsem);
851}
852
853static int torture_percpu_rwsem_down_write(int tid __maybe_unused)
854__acquires(pcpu_rwsem)
855{
856 percpu_down_write(&pcpu_rwsem);
857 return 0;
858}
859
860static void torture_percpu_rwsem_up_write(int tid __maybe_unused)
861__releases(pcpu_rwsem)
862{
863 percpu_up_write(&pcpu_rwsem);
864}
865
866static int torture_percpu_rwsem_down_read(int tid __maybe_unused)
867__acquires(pcpu_rwsem)
868{
869 percpu_down_read(&pcpu_rwsem);
870 return 0;
871}
872
873static void torture_percpu_rwsem_up_read(int tid __maybe_unused)
874__releases(pcpu_rwsem)
875{
876 percpu_up_read(&pcpu_rwsem);
877}
878
879static struct lock_torture_ops percpu_rwsem_lock_ops = {
880 .init = torture_percpu_rwsem_init,
881 .exit = torture_percpu_rwsem_exit,
882 .writelock = torture_percpu_rwsem_down_write,
883 .write_delay = torture_rwsem_write_delay,
884 .task_boost = torture_rt_boost,
885 .writeunlock = torture_percpu_rwsem_up_write,
886 .readlock = torture_percpu_rwsem_down_read,
887 .read_delay = torture_rwsem_read_delay,
888 .readunlock = torture_percpu_rwsem_up_read,
889 .name = "percpu_rwsem_lock"
890};
891
892/*
893 * Lock torture writer kthread. Repeatedly acquires and releases
894 * the lock, checking for duplicate acquisitions.
895 */
896static int lock_torture_writer(void *arg)
897{
898 unsigned long j;
899 unsigned long j1;
900 u32 lockset_mask;
901 struct lock_stress_stats *lwsp = arg;
902 DEFINE_TORTURE_RANDOM(rand);
903 bool skip_main_lock;
904 int tid = lwsp - cxt.lwsa;
905
906 VERBOSE_TOROUT_STRING("lock_torture_writer task started");
907 if (!rt_task(current))
908 set_user_nice(current, MAX_NICE);
909
910 do {
911 if ((torture_random(&rand) & 0xfffff) == 0)
912 schedule_timeout_uninterruptible(1);
913
914 lockset_mask = torture_random(&rand);
915 /*
916 * When using nested_locks, we want to occasionally
917 * skip the main lock so we can avoid always serializing
918 * the lock chains on that central lock. By skipping the
919 * main lock occasionally, we can create different
920 * contention patterns (allowing for multiple disjoint
921 * blocked trees)
922 */
923 skip_main_lock = (nested_locks &&
924 !(torture_random(&rand) % 100));
925
926 cxt.cur_ops->task_boost(&rand);
927 if (cxt.cur_ops->nested_lock)
928 cxt.cur_ops->nested_lock(tid, lockset_mask);
929
930 if (!skip_main_lock) {
931 if (acq_writer_lim > 0)
932 j = jiffies;
933 cxt.cur_ops->writelock(tid);
934 if (WARN_ON_ONCE(lock_is_write_held))
935 lwsp->n_lock_fail++;
936 lock_is_write_held = true;
937 if (WARN_ON_ONCE(atomic_read(&lock_is_read_held)))
938 lwsp->n_lock_fail++; /* rare, but... */
939 if (acq_writer_lim > 0) {
940 j1 = jiffies;
941 WARN_ONCE(time_after(j1, j + acq_writer_lim),
942 "%s: Lock acquisition took %lu jiffies.\n",
943 __func__, j1 - j);
944 }
945 lwsp->n_lock_acquired++;
946
947 cxt.cur_ops->write_delay(&rand);
948
949 lock_is_write_held = false;
950 WRITE_ONCE(last_lock_release, jiffies);
951 cxt.cur_ops->writeunlock(tid);
952 }
953 if (cxt.cur_ops->nested_unlock)
954 cxt.cur_ops->nested_unlock(tid, lockset_mask);
955
956 stutter_wait("lock_torture_writer");
957 } while (!torture_must_stop());
958
959 cxt.cur_ops->task_boost(NULL); /* reset prio */
960 torture_kthread_stopping("lock_torture_writer");
961 return 0;
962}
963
964/*
965 * Lock torture reader kthread. Repeatedly acquires and releases
966 * the reader lock.
967 */
968static int lock_torture_reader(void *arg)
969{
970 struct lock_stress_stats *lrsp = arg;
971 int tid = lrsp - cxt.lrsa;
972 DEFINE_TORTURE_RANDOM(rand);
973
974 VERBOSE_TOROUT_STRING("lock_torture_reader task started");
975 set_user_nice(current, MAX_NICE);
976
977 do {
978 if ((torture_random(&rand) & 0xfffff) == 0)
979 schedule_timeout_uninterruptible(1);
980
981 cxt.cur_ops->readlock(tid);
982 atomic_inc(&lock_is_read_held);
983 if (WARN_ON_ONCE(lock_is_write_held))
984 lrsp->n_lock_fail++; /* rare, but... */
985
986 lrsp->n_lock_acquired++;
987 cxt.cur_ops->read_delay(&rand);
988 atomic_dec(&lock_is_read_held);
989 cxt.cur_ops->readunlock(tid);
990
991 stutter_wait("lock_torture_reader");
992 } while (!torture_must_stop());
993 torture_kthread_stopping("lock_torture_reader");
994 return 0;
995}
996
997/*
998 * Create an lock-torture-statistics message in the specified buffer.
999 */
1000static void __torture_print_stats(char *page,
1001 struct lock_stress_stats *statp, bool write)
1002{
1003 long cur;
1004 bool fail = false;
1005 int i, n_stress;
1006 long max = 0, min = statp ? data_race(statp[0].n_lock_acquired) : 0;
1007 long long sum = 0;
1008
1009 n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress;
1010 for (i = 0; i < n_stress; i++) {
1011 if (data_race(statp[i].n_lock_fail))
1012 fail = true;
1013 cur = data_race(statp[i].n_lock_acquired);
1014 sum += cur;
1015 if (max < cur)
1016 max = cur;
1017 if (min > cur)
1018 min = cur;
1019 }
1020 page += sprintf(page,
1021 "%s: Total: %lld Max/Min: %ld/%ld %s Fail: %d %s\n",
1022 write ? "Writes" : "Reads ",
1023 sum, max, min,
1024 !onoff_interval && max / 2 > min ? "???" : "",
1025 fail, fail ? "!!!" : "");
1026 if (fail)
1027 atomic_inc(&cxt.n_lock_torture_errors);
1028}
1029
1030/*
1031 * Print torture statistics. Caller must ensure that there is only one
1032 * call to this function at a given time!!! This is normally accomplished
1033 * by relying on the module system to only have one copy of the module
1034 * loaded, and then by giving the lock_torture_stats kthread full control
1035 * (or the init/cleanup functions when lock_torture_stats thread is not
1036 * running).
1037 */
1038static void lock_torture_stats_print(void)
1039{
1040 int size = cxt.nrealwriters_stress * 200 + 8192;
1041 char *buf;
1042
1043 if (cxt.cur_ops->readlock)
1044 size += cxt.nrealreaders_stress * 200 + 8192;
1045
1046 buf = kmalloc(size, GFP_KERNEL);
1047 if (!buf) {
1048 pr_err("lock_torture_stats_print: Out of memory, need: %d",
1049 size);
1050 return;
1051 }
1052
1053 __torture_print_stats(buf, cxt.lwsa, true);
1054 pr_alert("%s", buf);
1055 kfree(buf);
1056
1057 if (cxt.cur_ops->readlock) {
1058 buf = kmalloc(size, GFP_KERNEL);
1059 if (!buf) {
1060 pr_err("lock_torture_stats_print: Out of memory, need: %d",
1061 size);
1062 return;
1063 }
1064
1065 __torture_print_stats(buf, cxt.lrsa, false);
1066 pr_alert("%s", buf);
1067 kfree(buf);
1068 }
1069}
1070
1071/*
1072 * Periodically prints torture statistics, if periodic statistics printing
1073 * was specified via the stat_interval module parameter.
1074 *
1075 * No need to worry about fullstop here, since this one doesn't reference
1076 * volatile state or register callbacks.
1077 */
1078static int lock_torture_stats(void *arg)
1079{
1080 VERBOSE_TOROUT_STRING("lock_torture_stats task started");
1081 do {
1082 schedule_timeout_interruptible(stat_interval * HZ);
1083 lock_torture_stats_print();
1084 torture_shutdown_absorb("lock_torture_stats");
1085 } while (!torture_must_stop());
1086 torture_kthread_stopping("lock_torture_stats");
1087 return 0;
1088}
1089
1090
1091static inline void
1092lock_torture_print_module_parms(struct lock_torture_ops *cur_ops,
1093 const char *tag)
1094{
1095 static cpumask_t cpumask_all;
1096 cpumask_t *rcmp = cpumask_nonempty(bind_readers) ? bind_readers : &cpumask_all;
1097 cpumask_t *wcmp = cpumask_nonempty(bind_writers) ? bind_writers : &cpumask_all;
1098
1099 cpumask_setall(&cpumask_all);
1100 pr_alert("%s" TORTURE_FLAG
1101 "--- %s%s: acq_writer_lim=%d bind_readers=%*pbl bind_writers=%*pbl call_rcu_chains=%d long_hold=%d nested_locks=%d nreaders_stress=%d nwriters_stress=%d onoff_holdoff=%d onoff_interval=%d rt_boost=%d rt_boost_factor=%d shuffle_interval=%d shutdown_secs=%d stat_interval=%d stutter=%d verbose=%d writer_fifo=%d\n",
1102 torture_type, tag, cxt.debug_lock ? " [debug]": "",
1103 acq_writer_lim, cpumask_pr_args(rcmp), cpumask_pr_args(wcmp),
1104 call_rcu_chains, long_hold, nested_locks, cxt.nrealreaders_stress,
1105 cxt.nrealwriters_stress, onoff_holdoff, onoff_interval, rt_boost,
1106 rt_boost_factor, shuffle_interval, shutdown_secs, stat_interval, stutter,
1107 verbose, writer_fifo);
1108}
1109
1110// If requested, maintain call_rcu() chains to keep a grace period always
1111// in flight. These increase the probability of getting an RCU CPU stall
1112// warning and associated diagnostics when a locking primitive stalls.
1113
1114static void call_rcu_chain_cb(struct rcu_head *rhp)
1115{
1116 struct call_rcu_chain *crcp = container_of(rhp, struct call_rcu_chain, crc_rh);
1117
1118 if (!smp_load_acquire(&crcp->crc_stop)) {
1119 (void)start_poll_synchronize_rcu(); // Start one grace period...
1120 call_rcu(&crcp->crc_rh, call_rcu_chain_cb); // ... and later start another.
1121 }
1122}
1123
1124// Start the requested number of call_rcu() chains.
1125static int call_rcu_chain_init(void)
1126{
1127 int i;
1128
1129 if (call_rcu_chains <= 0)
1130 return 0;
1131 call_rcu_chain_list = kzalloc_objs(*call_rcu_chain_list,
1132 call_rcu_chains);
1133 if (!call_rcu_chain_list)
1134 return -ENOMEM;
1135 for (i = 0; i < call_rcu_chains; i++) {
1136 call_rcu_chain_list[i].crc_stop = false;
1137 call_rcu(&call_rcu_chain_list[i].crc_rh, call_rcu_chain_cb);
1138 }
1139 return 0;
1140}
1141
1142// Stop all of the call_rcu() chains.
1143static void call_rcu_chain_cleanup(void)
1144{
1145 int i;
1146
1147 if (!call_rcu_chain_list)
1148 return;
1149 for (i = 0; i < call_rcu_chains; i++)
1150 smp_store_release(&call_rcu_chain_list[i].crc_stop, true);
1151 rcu_barrier();
1152 kfree(call_rcu_chain_list);
1153 call_rcu_chain_list = NULL;
1154}
1155
1156static void lock_torture_cleanup(void)
1157{
1158 int i;
1159
1160 if (torture_cleanup_begin())
1161 return;
1162
1163 /*
1164 * Indicates early cleanup, meaning that the test has not run,
1165 * such as when passing bogus args when loading the module.
1166 * However cxt->cur_ops.init() may have been invoked, so beside
1167 * perform the underlying torture-specific cleanups, cur_ops.exit()
1168 * will be invoked if needed.
1169 */
1170 if (!cxt.lwsa && !cxt.lrsa)
1171 goto end;
1172
1173 if (writer_tasks) {
1174 for (i = 0; i < cxt.nrealwriters_stress; i++)
1175 torture_stop_kthread(lock_torture_writer, writer_tasks[i]);
1176 kfree(writer_tasks);
1177 writer_tasks = NULL;
1178 }
1179
1180 if (reader_tasks) {
1181 for (i = 0; i < cxt.nrealreaders_stress; i++)
1182 torture_stop_kthread(lock_torture_reader,
1183 reader_tasks[i]);
1184 kfree(reader_tasks);
1185 reader_tasks = NULL;
1186 }
1187
1188 torture_stop_kthread(lock_torture_stats, stats_task);
1189 lock_torture_stats_print(); /* -After- the stats thread is stopped! */
1190
1191 if (atomic_read(&cxt.n_lock_torture_errors))
1192 lock_torture_print_module_parms(cxt.cur_ops,
1193 "End of test: FAILURE");
1194 else if (torture_onoff_failures())
1195 lock_torture_print_module_parms(cxt.cur_ops,
1196 "End of test: LOCK_HOTPLUG");
1197 else
1198 lock_torture_print_module_parms(cxt.cur_ops,
1199 "End of test: SUCCESS");
1200
1201 kfree(cxt.lwsa);
1202 cxt.lwsa = NULL;
1203 kfree(cxt.lrsa);
1204 cxt.lrsa = NULL;
1205
1206 call_rcu_chain_cleanup();
1207
1208end:
1209 if (cxt.init_called) {
1210 if (cxt.cur_ops->exit)
1211 cxt.cur_ops->exit();
1212 cxt.init_called = false;
1213 }
1214
1215 free_cpumask_var(bind_readers);
1216 free_cpumask_var(bind_writers);
1217
1218 torture_cleanup_end();
1219}
1220
1221static int __init lock_torture_init(void)
1222{
1223 int i, j;
1224 int firsterr = 0;
1225 static struct lock_torture_ops *torture_ops[] = {
1226 &lock_busted_ops,
1227 &spin_lock_ops, &spin_lock_irq_ops,
1228 &raw_spin_lock_ops, &raw_spin_lock_irq_ops,
1229#ifdef CONFIG_BPF_SYSCALL
1230 &raw_res_spin_lock_ops, &raw_res_spin_lock_irq_ops,
1231#endif
1232 &rw_lock_ops, &rw_lock_irq_ops,
1233 &mutex_lock_ops,
1234 &ww_mutex_lock_ops,
1235#ifdef CONFIG_RT_MUTEXES
1236 &rtmutex_lock_ops,
1237#endif
1238 &rwsem_lock_ops,
1239 &percpu_rwsem_lock_ops,
1240 };
1241
1242 if (!torture_init_begin(torture_type, verbose))
1243 return -EBUSY;
1244
1245 /* Process args and tell the world that the torturer is on the job. */
1246 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
1247 cxt.cur_ops = torture_ops[i];
1248 if (strcmp(torture_type, cxt.cur_ops->name) == 0)
1249 break;
1250 }
1251 if (i == ARRAY_SIZE(torture_ops)) {
1252 pr_alert("lock-torture: invalid torture type: \"%s\"\n",
1253 torture_type);
1254 pr_alert("lock-torture types:");
1255 for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
1256 pr_alert(" %s", torture_ops[i]->name);
1257 pr_alert("\n");
1258 firsterr = -EINVAL;
1259 goto unwind;
1260 }
1261
1262 if (nwriters_stress == 0 &&
1263 (!cxt.cur_ops->readlock || nreaders_stress == 0)) {
1264 pr_alert("lock-torture: must run at least one locking thread\n");
1265 firsterr = -EINVAL;
1266 goto unwind;
1267 }
1268
1269 if (nwriters_stress >= 0)
1270 cxt.nrealwriters_stress = nwriters_stress;
1271 else
1272 cxt.nrealwriters_stress = 2 * num_online_cpus();
1273
1274 if (cxt.cur_ops->init) {
1275 cxt.cur_ops->init();
1276 cxt.init_called = true;
1277 }
1278
1279#ifdef CONFIG_DEBUG_MUTEXES
1280 if (str_has_prefix(torture_type, "mutex"))
1281 cxt.debug_lock = true;
1282#endif
1283#ifdef CONFIG_DEBUG_RT_MUTEXES
1284 if (str_has_prefix(torture_type, "rtmutex"))
1285 cxt.debug_lock = true;
1286#endif
1287#ifdef CONFIG_DEBUG_SPINLOCK
1288 if ((str_has_prefix(torture_type, "spin")) ||
1289 (str_has_prefix(torture_type, "rw_lock")))
1290 cxt.debug_lock = true;
1291#endif
1292
1293 /* Initialize the statistics so that each run gets its own numbers. */
1294 if (nwriters_stress) {
1295 lock_is_write_held = false;
1296 cxt.lwsa = kmalloc_objs(*cxt.lwsa, cxt.nrealwriters_stress);
1297 if (cxt.lwsa == NULL) {
1298 VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory");
1299 firsterr = -ENOMEM;
1300 goto unwind;
1301 }
1302
1303 for (i = 0; i < cxt.nrealwriters_stress; i++) {
1304 cxt.lwsa[i].n_lock_fail = 0;
1305 cxt.lwsa[i].n_lock_acquired = 0;
1306 }
1307 }
1308
1309 if (cxt.cur_ops->readlock) {
1310 if (nreaders_stress >= 0)
1311 cxt.nrealreaders_stress = nreaders_stress;
1312 else {
1313 /*
1314 * By default distribute evenly the number of
1315 * readers and writers. We still run the same number
1316 * of threads as the writer-only locks default.
1317 */
1318 if (nwriters_stress < 0) /* user doesn't care */
1319 cxt.nrealwriters_stress = num_online_cpus();
1320 cxt.nrealreaders_stress = cxt.nrealwriters_stress;
1321 }
1322
1323 if (nreaders_stress) {
1324 cxt.lrsa = kmalloc_objs(*cxt.lrsa,
1325 cxt.nrealreaders_stress);
1326 if (cxt.lrsa == NULL) {
1327 VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory");
1328 firsterr = -ENOMEM;
1329 kfree(cxt.lwsa);
1330 cxt.lwsa = NULL;
1331 goto unwind;
1332 }
1333
1334 for (i = 0; i < cxt.nrealreaders_stress; i++) {
1335 cxt.lrsa[i].n_lock_fail = 0;
1336 cxt.lrsa[i].n_lock_acquired = 0;
1337 }
1338 }
1339 }
1340
1341 firsterr = call_rcu_chain_init();
1342 if (torture_init_error(firsterr))
1343 goto unwind;
1344
1345 lock_torture_print_module_parms(cxt.cur_ops, "Start of test");
1346
1347 /* Prepare torture context. */
1348 if (onoff_interval > 0) {
1349 firsterr = torture_onoff_init(onoff_holdoff * HZ,
1350 onoff_interval * HZ, NULL);
1351 if (torture_init_error(firsterr))
1352 goto unwind;
1353 }
1354 if (shuffle_interval > 0) {
1355 firsterr = torture_shuffle_init(shuffle_interval);
1356 if (torture_init_error(firsterr))
1357 goto unwind;
1358 }
1359 if (shutdown_secs > 0) {
1360 firsterr = torture_shutdown_init(shutdown_secs,
1361 lock_torture_cleanup);
1362 if (torture_init_error(firsterr))
1363 goto unwind;
1364 }
1365 if (stutter > 0) {
1366 firsterr = torture_stutter_init(stutter, stutter);
1367 if (torture_init_error(firsterr))
1368 goto unwind;
1369 }
1370
1371 if (nwriters_stress) {
1372 writer_tasks = kzalloc_objs(writer_tasks[0],
1373 cxt.nrealwriters_stress);
1374 if (writer_tasks == NULL) {
1375 TOROUT_ERRSTRING("writer_tasks: Out of memory");
1376 firsterr = -ENOMEM;
1377 goto unwind;
1378 }
1379 }
1380
1381 /* cap nested_locks to MAX_NESTED_LOCKS */
1382 if (nested_locks > MAX_NESTED_LOCKS)
1383 nested_locks = MAX_NESTED_LOCKS;
1384
1385 if (cxt.cur_ops->readlock) {
1386 reader_tasks = kzalloc_objs(reader_tasks[0],
1387 cxt.nrealreaders_stress);
1388 if (reader_tasks == NULL) {
1389 TOROUT_ERRSTRING("reader_tasks: Out of memory");
1390 kfree(writer_tasks);
1391 writer_tasks = NULL;
1392 firsterr = -ENOMEM;
1393 goto unwind;
1394 }
1395 }
1396
1397 /*
1398 * Create the kthreads and start torturing (oh, those poor little locks).
1399 *
1400 * TODO: Note that we interleave writers with readers, giving writers a
1401 * slight advantage, by creating its kthread first. This can be modified
1402 * for very specific needs, or even let the user choose the policy, if
1403 * ever wanted.
1404 */
1405 for (i = 0, j = 0; i < cxt.nrealwriters_stress ||
1406 j < cxt.nrealreaders_stress; i++, j++) {
1407 if (i >= cxt.nrealwriters_stress)
1408 goto create_reader;
1409
1410 /* Create writer. */
1411 firsterr = torture_create_kthread_cb(lock_torture_writer, &cxt.lwsa[i],
1412 writer_tasks[i],
1413 writer_fifo ? sched_set_fifo : NULL);
1414 if (torture_init_error(firsterr))
1415 goto unwind;
1416 if (cpumask_nonempty(bind_writers))
1417 torture_sched_setaffinity(writer_tasks[i]->pid, bind_writers, true);
1418
1419 create_reader:
1420 if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress))
1421 continue;
1422 /* Create reader. */
1423 firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j],
1424 reader_tasks[j]);
1425 if (torture_init_error(firsterr))
1426 goto unwind;
1427 if (cpumask_nonempty(bind_readers))
1428 torture_sched_setaffinity(reader_tasks[j]->pid, bind_readers, true);
1429 }
1430 if (stat_interval > 0) {
1431 firsterr = torture_create_kthread(lock_torture_stats, NULL,
1432 stats_task);
1433 if (torture_init_error(firsterr))
1434 goto unwind;
1435 }
1436 torture_init_end();
1437 return 0;
1438
1439unwind:
1440 torture_init_end();
1441 lock_torture_cleanup();
1442 if (shutdown_secs) {
1443 WARN_ON(!IS_MODULE(CONFIG_LOCK_TORTURE_TEST));
1444 kernel_power_off();
1445 }
1446 return firsterr;
1447}
1448
1449module_init(lock_torture_init);
1450module_exit(lock_torture_cleanup);