linux - drivers/md/md.c

11121 lines · 285.6 KiB

1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	md.c : Multiple Devices driver for Linux
4	Copyright (C) 1998, 1999, 2000 Ingo Molnar
5
6	completely rewritten, based on the MD driver code from Marc Zyngier
7
8	Changes:
9
10	- RAID-1/RAID-5 extensions by Miguel de Icaza, Gadi Oxman, Ingo Molnar
11	- RAID-6 extensions by H. Peter Anvin <[email protected]>
12	- boot support for linear and striped mode by Harald Hoyer <[email protected]>
13	- kerneld support by Boris Tobotras <[email protected]>
14	- kmod support by: Cyrus Durgin
15	- RAID0 bugfixes: Mark Anthony Lisher <[email protected]>
16	- Devfs support by Richard Gooch <[email protected]>
17
18	- lots of fixes and improvements to the RAID1/RAID5 and generic
19	RAID code (such as request based resynchronization):
20
21	Neil Brown <[email protected]>.
22
23	- persistent bitmap code
24	Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.
25
26
27	Errors, Warnings, etc.
28	Please use:
29	pr_crit() for error conditions that risk data loss
30	pr_err() for error conditions that are unexpected, like an IO error
31	or internal inconsistency
32	pr_warn() for error conditions that could have been predicated, like
33	adding a device to an array when it has incompatible metadata
34	pr_info() for every interesting, very rare events, like an array starting
35	or stopping, or resync starting or stopping
36	pr_debug() for everything else.
37
38	*/
39
40	#include <linux/sched/mm.h>
41	#include <linux/sched/signal.h>
42	#include <linux/kthread.h>
43	#include <linux/blkdev.h>
44	#include <linux/blk-integrity.h>
45	#include <linux/badblocks.h>
46	#include <linux/sysctl.h>
47	#include <linux/seq_file.h>
48	#include <linux/fs.h>
49	#include <linux/poll.h>
50	#include <linux/ctype.h>
51	#include <linux/string.h>
52	#include <linux/hdreg.h>
53	#include <linux/proc_fs.h>
54	#include <linux/random.h>
55	#include <linux/major.h>
56	#include <linux/module.h>
57	#include <linux/reboot.h>
58	#include <linux/file.h>
59	#include <linux/compat.h>
60	#include <linux/delay.h>
61	#include <linux/raid/md_p.h>
62	#include <linux/raid/md_u.h>
63	#include <linux/raid/detect.h>
64	#include <linux/slab.h>
65	#include <linux/percpu-refcount.h>
66	#include <linux/part_stat.h>
67
68	#include "md.h"
69	#include "md-bitmap.h"
70	#include "md-cluster.h"
71
72	static const char *action_name[NR_SYNC_ACTIONS] = {
73	[ACTION_RESYNC] = "resync",
74	[ACTION_RECOVER] = "recover",
75	[ACTION_CHECK] = "check",
76	[ACTION_REPAIR] = "repair",
77	[ACTION_RESHAPE] = "reshape",
78	[ACTION_FROZEN] = "frozen",
79	[ACTION_IDLE] = "idle",
80	};
81
82	static DEFINE_XARRAY(md_submodule);
83
84	static const struct kobj_type md_ktype;
85
86	static DECLARE_WAIT_QUEUE_HEAD(resync_wait);
87
88	/*
89	* This workqueue is used for sync_work to register new sync_thread, and for
90	* del_work to remove rdev, and for event_work that is only set by dm-raid.
91	*
92	* Noted that sync_work will grab reconfig_mutex, hence never flush this
93	* workqueue whith reconfig_mutex grabbed.
94	*/
95	static struct workqueue_struct *md_misc_wq;
96
97	static int remove_and_add_spares(struct mddev *mddev,
98	struct md_rdev *this);
99	static void mddev_detach(struct mddev *mddev);
100	static void export_rdev(struct md_rdev *rdev);
101	static void md_wakeup_thread_directly(struct md_thread __rcu **thread);
102
103	/*
104	* Default number of read corrections we'll attempt on an rdev
105	* before ejecting it from the array. We divide the read error
106	* count by 2 for every hour elapsed between read errors.
107	*/
108	#define MD_DEFAULT_MAX_CORRECTED_READ_ERRORS 20
109	/* Default safemode delay: 200 msec */
110	#define DEFAULT_SAFEMODE_DELAY ((200 * HZ)/1000 +1)
111	/*
112	* Current RAID-1,4,5,6,10 parallel reconstruction 'guaranteed speed limit'
113	* is sysctl_speed_limit_min, 1000 KB/sec by default, so the extra system load
114	* does not show up that much. Increase it if you want to have more guaranteed
115	* speed. Note that the RAID driver will use the maximum bandwidth
116	* sysctl_speed_limit_max, 200 MB/sec by default, if the IO subsystem is idle.
117	*
118	* Background sync IO speed control:
119	*
120	* - below speed min:
121	* no limit;
122	* - above speed min and below speed max:
123	* a) if mddev is idle, then no limit;
124	* b) if mddev is busy handling normal IO, then limit inflight sync IO
125	* to sync_io_depth;
126	* - above speed max:
127	* sync IO can't be issued;
128	*
129	* Following configurations can be changed via /proc/sys/dev/raid/ for system
130	* or /sys/block/mdX/md/ for one array.
131	*/
132	static int sysctl_speed_limit_min = 1000;
133	static int sysctl_speed_limit_max = 200000;
134	static int sysctl_sync_io_depth = 32;
135
136	static int speed_min(struct mddev *mddev)
137	{
138	return mddev->sync_speed_min ?
139	mddev->sync_speed_min : sysctl_speed_limit_min;
140	}
141
142	static int speed_max(struct mddev *mddev)
143	{
144	return mddev->sync_speed_max ?
145	mddev->sync_speed_max : sysctl_speed_limit_max;
146	}
147
148	static int sync_io_depth(struct mddev *mddev)
149	{
150	return mddev->sync_io_depth ?
151	mddev->sync_io_depth : sysctl_sync_io_depth;
152	}
153
154	static void rdev_uninit_serial(struct md_rdev *rdev)
155	{
156	if (!test_and_clear_bit(CollisionCheck, &rdev->flags))
157	return;
158
159	kvfree(rdev->serial);
160	rdev->serial = NULL;
161	}
162
163	static void rdevs_uninit_serial(struct mddev *mddev)
164	{
165	struct md_rdev *rdev;
166
167	rdev_for_each(rdev, mddev)
168	rdev_uninit_serial(rdev);
169	}
170
171	static int rdev_init_serial(struct md_rdev *rdev)
172	{
173	/* serial_nums equals with BARRIER_BUCKETS_NR */
174	int i, serial_nums = 1 << ((PAGE_SHIFT - ilog2(sizeof(atomic_t))));
175	struct serial_in_rdev *serial = NULL;
176
177	if (test_bit(CollisionCheck, &rdev->flags))
178	return 0;
179
180	serial = kvmalloc(sizeof(struct serial_in_rdev) * serial_nums,
181	GFP_KERNEL);
182	if (!serial)
183	return -ENOMEM;
184
185	for (i = 0; i < serial_nums; i++) {
186	struct serial_in_rdev *serial_tmp = &serial[i];
187
188	spin_lock_init(&serial_tmp->serial_lock);
189	serial_tmp->serial_rb = RB_ROOT_CACHED;
190	}
191
192	rdev->serial = serial;
193	set_bit(CollisionCheck, &rdev->flags);
194
195	return 0;
196	}
197
198	static int rdevs_init_serial(struct mddev *mddev)
199	{
200	struct md_rdev *rdev;
201	int ret = 0;
202
203	rdev_for_each(rdev, mddev) {
204	ret = rdev_init_serial(rdev);
205	if (ret)
206	break;
207	}
208
209	/* Free all resources if pool is not existed */
210	if (ret && !mddev->serial_info_pool)
211	rdevs_uninit_serial(mddev);
212
213	return ret;
214	}
215
216	/*
217	* rdev needs to enable serial stuffs if it meets the conditions:
218	* 1. it is multi-queue device flaged with writemostly.
219	* 2. the write-behind mode is enabled.
220	*/
221	static int rdev_need_serial(struct md_rdev *rdev)
222	{
223	return (rdev && rdev->mddev->bitmap_info.max_write_behind > 0 &&
224	rdev->bdev->bd_disk->queue->nr_hw_queues != 1 &&
225	test_bit(WriteMostly, &rdev->flags));
226	}
227
228	/*
229	* Init resource for rdev(s), then create serial_info_pool if:
230	* 1. rdev is the first device which return true from rdev_enable_serial.
231	* 2. rdev is NULL, means we want to enable serialization for all rdevs.
232	*/
233	void mddev_create_serial_pool(struct mddev mddev, struct md_rdev rdev)
234	{
235	int ret = 0;
236	unsigned int noio_flags;
237
238	if (!test_bit(MD_SERIALIZE_POLICY, &mddev->flags) &&
239	rdev && !rdev_need_serial(rdev) &&
240	!test_bit(CollisionCheck, &rdev->flags))
241	return;
242
243	noio_flags = memalloc_noio_save();
244	if (!rdev)
245	ret = rdevs_init_serial(mddev);
246	else
247	ret = rdev_init_serial(rdev);
248	if (ret)
249	goto out;
250
251	if (mddev->serial_info_pool == NULL) {
252	mddev->serial_info_pool =
253	mempool_create_kmalloc_pool(NR_SERIAL_INFOS,
254	sizeof(struct serial_info));
255	if (!mddev->serial_info_pool) {
256	rdevs_uninit_serial(mddev);
257	pr_err("can't alloc memory pool for serialization\n");
258	}
259	}
260	out:
261	memalloc_noio_restore(noio_flags);
262	}
263
264	/*
265	* Free resource from rdev(s), and destroy serial_info_pool under conditions:
266	* 1. rdev is the last device flaged with CollisionCheck.
267	* 2. when bitmap is destroyed while policy is not enabled.
268	* 3. for disable policy, the pool is destroyed only when no rdev needs it.
269	*/
270	void mddev_destroy_serial_pool(struct mddev mddev, struct md_rdev rdev)
271	{
272	if (rdev && !test_bit(CollisionCheck, &rdev->flags))
273	return;
274
275	if (mddev->serial_info_pool) {
276	struct md_rdev *temp;
277	int num = 0; /* used to track if other rdevs need the pool */
278
279	rdev_for_each(temp, mddev) {
280	if (!rdev) {
281	if (!test_bit(MD_SERIALIZE_POLICY,
282	&mddev->flags) \|\|
283	!rdev_need_serial(temp))
284	rdev_uninit_serial(temp);
285	else
286	num++;
287	} else if (temp != rdev &&
288	test_bit(CollisionCheck, &temp->flags))
289	num++;
290	}
291
292	if (rdev)
293	rdev_uninit_serial(rdev);
294
295	if (num)
296	pr_info("The mempool could be used by other devices\n");
297	else {
298	mempool_destroy(mddev->serial_info_pool);
299	mddev->serial_info_pool = NULL;
300	}
301	}
302	}
303
304	static struct ctl_table_header *raid_table_header;
305
306	static const struct ctl_table raid_table[] = {
307	{
308	.procname = "speed_limit_min",
309	.data = &sysctl_speed_limit_min,
310	.maxlen = sizeof(int),
311	.mode = 0644,
312	.proc_handler = proc_dointvec,
313	},
314	{
315	.procname = "speed_limit_max",
316	.data = &sysctl_speed_limit_max,
317	.maxlen = sizeof(int),
318	.mode = 0644,
319	.proc_handler = proc_dointvec,
320	},
321	{
322	.procname = "sync_io_depth",
323	.data = &sysctl_sync_io_depth,
324	.maxlen = sizeof(int),
325	.mode = 0644,
326	.proc_handler = proc_dointvec,
327	},
328	};
329
330	static int start_readonly;
331
332	/*
333	* The original mechanism for creating an md device is to create
334	* a device node in /dev and to open it. This causes races with device-close.
335	* The preferred method is to write to the "new_array" module parameter.
336	* This can avoid races.
337	* Setting create_on_open to false disables the original mechanism
338	* so all the races disappear.
339	*/
340	static bool create_on_open = true;
341	static bool legacy_async_del_gendisk = true;
342	static bool check_new_feature = true;
343
344	/*
345	* We have a system wide 'event count' that is incremented
346	* on any 'interesting' event, and readers of /proc/mdstat
347	* can use 'poll' or 'select' to find out when the event
348	* count increases.
349	*
350	* Events are:
351	* start array, stop array, error, add device, remove device,
352	* start build, activate spare
353	*/
354	static DECLARE_WAIT_QUEUE_HEAD(md_event_waiters);
355	static atomic_t md_event_count;
356	void md_new_event(void)
357	{
358	atomic_inc(&md_event_count);
359	wake_up(&md_event_waiters);
360	}
361	EXPORT_SYMBOL_GPL(md_new_event);
362
363	/*
364	* Enables to iterate over all existing md arrays
365	* all_mddevs_lock protects this list.
366	*/
367	static LIST_HEAD(all_mddevs);
368	static DEFINE_SPINLOCK(all_mddevs_lock);
369
370	static bool is_md_suspended(struct mddev *mddev)
371	{
372	return percpu_ref_is_dying(&mddev->active_io);
373	}
374	/* Rather than calling directly into the personality make_request function,
375	* IO requests come here first so that we can check if the device is
376	* being suspended pending a reconfiguration.
377	* We hold a refcount over the call to ->make_request. By the time that
378	* call has finished, the bio has been linked into some internal structure
379	* and so is visible to ->quiesce(), so we don't need the refcount any more.
380	*/
381	static bool is_suspended(struct mddev mddev, struct bio bio)
382	{
383	if (is_md_suspended(mddev))
384	return true;
385	if (bio_data_dir(bio) != WRITE)
386	return false;
387	if (READ_ONCE(mddev->suspend_lo) >= READ_ONCE(mddev->suspend_hi))
388	return false;
389	if (bio->bi_iter.bi_sector >= READ_ONCE(mddev->suspend_hi))
390	return false;
391	if (bio_end_sector(bio) < READ_ONCE(mddev->suspend_lo))
392	return false;
393	return true;
394	}
395
396	bool md_handle_request(struct mddev mddev, struct bio bio)
397	{
398	check_suspended:
399	if (unlikely(md_cloned_bio(mddev, bio))) {
400	/*
401	* This bio is an MD cloned bio and already holds an
402	* active_io reference, so percpu_ref_get() is safe here.
403	*/
404	percpu_ref_get(&mddev->active_io);
405	} else {
406	if (is_suspended(mddev, bio)) {
407	/* Bail out if REQ_NOWAIT is set for the bio */
408	if (bio->bi_opf & REQ_NOWAIT) {
409	bio_wouldblock_error(bio);
410	return true;
411	}
412	wait_event(mddev->sb_wait, !is_suspended(mddev, bio));
413	}
414	if (!percpu_ref_tryget_live(&mddev->active_io))
415	goto check_suspended;
416	}
417	if (!mddev->pers->make_request(mddev, bio)) {
418	percpu_ref_put(&mddev->active_io);
419	if (mddev_is_dm(mddev) && mddev->pers->prepare_suspend)
420	return false;
421	goto check_suspended;
422	}
423
424	percpu_ref_put(&mddev->active_io);
425	return true;
426	}
427	EXPORT_SYMBOL(md_handle_request);
428
429	static void md_submit_bio(struct bio *bio)
430	{
431	const int rw = bio_data_dir(bio);
432	struct mddev *mddev = bio->bi_bdev->bd_disk->private_data;
433
434	if (mddev == NULL \|\| mddev->pers == NULL) {
435	bio_io_error(bio);
436	return;
437	}
438
439	if (unlikely(test_bit(MD_BROKEN, &mddev->flags)) && (rw == WRITE)) {
440	bio_io_error(bio);
441	return;
442	}
443
444	bio = bio_split_to_limits(bio);
445	if (!bio)
446	return;
447
448	if (mddev->ro == MD_RDONLY && unlikely(rw == WRITE)) {
449	if (bio_sectors(bio) != 0)
450	bio->bi_status = BLK_STS_IOERR;
451	bio_endio(bio);
452	return;
453	}
454
455	/* bio could be mergeable after passing to underlayer */
456	bio->bi_opf &= ~REQ_NOMERGE;
457
458	md_handle_request(mddev, bio);
459	}
460
461	/*
462	* Make sure no new requests are submitted to the device, and any requests that
463	* have been submitted are completely handled.
464	*/
465	int mddev_suspend(struct mddev *mddev, bool interruptible)
466	{
467	int err = 0;
468
469	/*
470	* hold reconfig_mutex to wait for normal io will deadlock, because
471	* other context can't update super_block, and normal io can rely on
472	* updating super_block.
473	*/
474	lockdep_assert_not_held(&mddev->reconfig_mutex);
475
476	if (interruptible)
477	err = mutex_lock_interruptible(&mddev->suspend_mutex);
478	else
479	mutex_lock(&mddev->suspend_mutex);
480	if (err)
481	return err;
482
483	if (mddev->suspended) {
484	WRITE_ONCE(mddev->suspended, mddev->suspended + 1);
485	mutex_unlock(&mddev->suspend_mutex);
486	return 0;
487	}
488
489	percpu_ref_kill(&mddev->active_io);
490
491	/*
492	* RAID456 IO can sleep in wait_for_reshape while still holding an
493	* active_io reference. If reshape is already interrupted or frozen,
494	* wake those waiters so they can abort and drop the reference instead
495	* of deadlocking suspend.
496	*/
497	if (mddev->pers && mddev->pers->prepare_suspend &&
498	reshape_interrupted(mddev))
499	mddev->pers->prepare_suspend(mddev);
500
501	if (interruptible)
502	err = wait_event_interruptible(mddev->sb_wait,
503	percpu_ref_is_zero(&mddev->active_io));
504	else
505	wait_event(mddev->sb_wait,
506	percpu_ref_is_zero(&mddev->active_io));
507	if (err) {
508	percpu_ref_resurrect(&mddev->active_io);
509	mutex_unlock(&mddev->suspend_mutex);
510	return err;
511	}
512
513	/*
514	* For raid456, io might be waiting for reshape to make progress,
515	* allow new reshape to start while waiting for io to be done to
516	* prevent deadlock.
517	*/
518	WRITE_ONCE(mddev->suspended, mddev->suspended + 1);
519
520	mutex_unlock(&mddev->suspend_mutex);
521	return 0;
522	}
523	EXPORT_SYMBOL_GPL(mddev_suspend);
524
525	static void __mddev_resume(struct mddev *mddev, bool recovery_needed)
526	{
527	lockdep_assert_not_held(&mddev->reconfig_mutex);
528
529	mutex_lock(&mddev->suspend_mutex);
530	WRITE_ONCE(mddev->suspended, mddev->suspended - 1);
531	if (mddev->suspended) {
532	mutex_unlock(&mddev->suspend_mutex);
533	return;
534	}
535
536	percpu_ref_resurrect(&mddev->active_io);
537	wake_up(&mddev->sb_wait);
538
539	if (recovery_needed)
540	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
541	md_wakeup_thread(mddev->thread);
542	md_wakeup_thread(mddev->sync_thread); /* possibly kick off a reshape */
543
544	mutex_unlock(&mddev->suspend_mutex);
545	}
546
547	void mddev_resume(struct mddev *mddev)
548	{
549	return __mddev_resume(mddev, true);
550	}
551	EXPORT_SYMBOL_GPL(mddev_resume);
552
553	/* sync bdev before setting device to readonly or stopping raid*/
554	static int mddev_set_closing_and_sync_blockdev(struct mddev *mddev, int opener_num)
555	{
556	mutex_lock(&mddev->open_mutex);
557	if (mddev->pers && atomic_read(&mddev->openers) > opener_num) {
558	mutex_unlock(&mddev->open_mutex);
559	return -EBUSY;
560	}
561	if (test_and_set_bit(MD_CLOSING, &mddev->flags)) {
562	mutex_unlock(&mddev->open_mutex);
563	return -EBUSY;
564	}
565	mutex_unlock(&mddev->open_mutex);
566
567	sync_blockdev(mddev->gendisk->part0);
568	return 0;
569	}
570
571	/*
572	* The only difference from bio_chain_endio() is that the current
573	* bi_status of bio does not affect the bi_status of parent.
574	*/
575	static void md_end_flush(struct bio *bio)
576	{
577	struct bio *parent = bio->bi_private;
578
579	/*
580	* If any flush io error before the power failure,
581	* disk data may be lost.
582	*/
583	if (bio->bi_status)
584	pr_err("md: %pg flush io error %d\n", bio->bi_bdev,
585	blk_status_to_errno(bio->bi_status));
586
587	bio_put(bio);
588	bio_endio(parent);
589	}
590
591	bool md_flush_request(struct mddev mddev, struct bio bio)
592	{
593	struct md_rdev *rdev;
594	struct bio *new;
595
596	/*
597	* md_flush_reqeust() should be called under md_handle_request() and
598	* 'active_io' is already grabbed. Hence it's safe to get rdev directly
599	* without rcu protection.
600	*/
601	WARN_ON(percpu_ref_is_zero(&mddev->active_io));
602
603	rdev_for_each(rdev, mddev) {
604	if (rdev->raid_disk < 0 \|\| test_bit(Faulty, &rdev->flags))
605	continue;
606
607	new = bio_alloc_bioset(rdev->bdev, 0,
608	REQ_OP_WRITE \| REQ_PREFLUSH, GFP_NOIO,
609	&mddev->bio_set);
610	new->bi_private = bio;
611	new->bi_end_io = md_end_flush;
612	bio_inc_remaining(bio);
613	submit_bio(new);
614	}
615
616	if (bio_sectors(bio) == 0) {
617	bio_endio(bio);
618	return true;
619	}
620
621	bio->bi_opf &= ~REQ_PREFLUSH;
622	return false;
623	}
624	EXPORT_SYMBOL(md_flush_request);
625
626	static inline struct mddev mddev_get(struct mddev mddev)
627	{
628	lockdep_assert_held(&all_mddevs_lock);
629
630	if (test_bit(MD_DELETED, &mddev->flags))
631	return NULL;
632	atomic_inc(&mddev->active);
633	return mddev;
634	}
635
636	static void mddev_delayed_delete(struct work_struct *ws);
637
638	static void __mddev_put(struct mddev *mddev)
639	{
640	if (mddev->raid_disks \|\| !list_empty(&mddev->disks) \|\|
641	mddev->ctime \|\| mddev->hold_active)
642	return;
643
644	/*
645	* If array is freed by stopping array, MD_DELETED is set by
646	* do_md_stop(), MD_DELETED is still set here in case mddev is freed
647	* directly by closing a mddev that is created by create_on_open.
648	*/
649	set_bit(MD_DELETED, &mddev->flags);
650	/*
651	* Call queue_work inside the spinlock so that flush_workqueue() after
652	* mddev_find will succeed in waiting for the work to be done.
653	*/
654	queue_work(md_misc_wq, &mddev->del_work);
655	}
656
657	static void mddev_put_locked(struct mddev *mddev)
658	{
659	if (atomic_dec_and_test(&mddev->active))
660	__mddev_put(mddev);
661	}
662
663	void mddev_put(struct mddev *mddev)
664	{
665	if (!atomic_dec_and_lock(&mddev->active, &all_mddevs_lock))
666	return;
667
668	__mddev_put(mddev);
669	spin_unlock(&all_mddevs_lock);
670	}
671
672	static void md_safemode_timeout(struct timer_list *t);
673	static void md_start_sync(struct work_struct *ws);
674
675	static void active_io_release(struct percpu_ref *ref)
676	{
677	struct mddev *mddev = container_of(ref, struct mddev, active_io);
678
679	wake_up(&mddev->sb_wait);
680	}
681
682	static void no_op(struct percpu_ref *r) {}
683
684	static void md_bitmap_sysfs_add(struct mddev *mddev)
685	{
686	if (sysfs_update_groups(&mddev->kobj, mddev->bitmap_ops->groups))
687	pr_warn("md: cannot register extra bitmap attributes for %s\n",
688	mdname(mddev));
689	else
690	/*
691	* Inform user with KOBJ_CHANGE about new bitmap
692	* attributes.
693	*/
694	kobject_uevent(&mddev->kobj, KOBJ_CHANGE);
695	}
696
697	static void md_bitmap_sysfs_del(struct mddev *mddev)
698	{
699	int nr_groups = 0;
700
701	for (nr_groups = 0; mddev->bitmap_ops->groups[nr_groups]; nr_groups++)
702	;
703
704	while (--nr_groups >= 1)
705	sysfs_unmerge_group(&mddev->kobj,
706	mddev->bitmap_ops->groups[nr_groups]);
707	sysfs_remove_group(&mddev->kobj, mddev->bitmap_ops->groups[0]);
708	}
709
710	bool mddev_set_bitmap_ops_nosysfs(struct mddev *mddev)
711	{
712	struct md_submodule_head *head;
713
714	if (mddev->bitmap_ops &&
715	mddev->bitmap_ops->head.id == mddev->bitmap_id)
716	return true;
717
718	xa_lock(&md_submodule);
719	head = xa_load(&md_submodule, mddev->bitmap_id);
720
721	if (!head) {
722	pr_warn("md: can't find bitmap id %d\n", mddev->bitmap_id);
723	goto err;
724	}
725
726	if (head->type != MD_BITMAP) {
727	pr_warn("md: invalid bitmap id %d\n", mddev->bitmap_id);
728	goto err;
729	}
730
731	mddev->bitmap_ops = (void *)head;
732	xa_unlock(&md_submodule);
733	return true;
734
735	err:
736	xa_unlock(&md_submodule);
737	return false;
738	}
739
740	int mddev_init(struct mddev *mddev)
741	{
742	int err = 0;
743
744	if (!IS_ENABLED(CONFIG_MD_BITMAP))
745	mddev->bitmap_id = ID_BITMAP_NONE;
746	else
747	mddev->bitmap_id = ID_BITMAP;
748
749	if (percpu_ref_init(&mddev->active_io, active_io_release,
750	PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
751	return -ENOMEM;
752
753	if (percpu_ref_init(&mddev->writes_pending, no_op,
754	PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) {
755	err = -ENOMEM;
756	goto exit_acitve_io;
757	}
758
759	err = bioset_init(&mddev->bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
760	if (err)
761	goto exit_writes_pending;
762
763	err = bioset_init(&mddev->sync_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
764	if (err)
765	goto exit_bio_set;
766
767	err = bioset_init(&mddev->io_clone_set, BIO_POOL_SIZE,
768	offsetof(struct md_io_clone, bio_clone), 0);
769	if (err)
770	goto exit_sync_set;
771
772	/* We want to start with the refcount at zero */
773	percpu_ref_put(&mddev->writes_pending);
774
775	mutex_init(&mddev->open_mutex);
776	mutex_init(&mddev->reconfig_mutex);
777	mutex_init(&mddev->suspend_mutex);
778	mutex_init(&mddev->bitmap_info.mutex);
779	INIT_LIST_HEAD(&mddev->disks);
780	INIT_LIST_HEAD(&mddev->all_mddevs);
781	INIT_LIST_HEAD(&mddev->deleting);
782	timer_setup(&mddev->safemode_timer, md_safemode_timeout, 0);
783	atomic_set(&mddev->active, 1);
784	atomic_set(&mddev->openers, 0);
785	atomic_set(&mddev->sync_seq, 0);
786	spin_lock_init(&mddev->lock);
787	init_waitqueue_head(&mddev->sb_wait);
788	init_waitqueue_head(&mddev->recovery_wait);
789	mddev->reshape_position = MaxSector;
790	mddev->reshape_backwards = 0;
791	mddev->last_sync_action = ACTION_IDLE;
792	mddev->resync_min = 0;
793	mddev->resync_max = MaxSector;
794	mddev->level = LEVEL_NONE;
795
796	INIT_WORK(&mddev->sync_work, md_start_sync);
797	INIT_WORK(&mddev->del_work, mddev_delayed_delete);
798
799	return 0;
800
801	exit_sync_set:
802	bioset_exit(&mddev->sync_set);
803	exit_bio_set:
804	bioset_exit(&mddev->bio_set);
805	exit_writes_pending:
806	percpu_ref_exit(&mddev->writes_pending);
807	exit_acitve_io:
808	percpu_ref_exit(&mddev->active_io);
809	return err;
810	}
811	EXPORT_SYMBOL_GPL(mddev_init);
812
813	void mddev_destroy(struct mddev *mddev)
814	{
815	bioset_exit(&mddev->bio_set);
816	bioset_exit(&mddev->sync_set);
817	bioset_exit(&mddev->io_clone_set);
818	percpu_ref_exit(&mddev->active_io);
819	percpu_ref_exit(&mddev->writes_pending);
820	}
821	EXPORT_SYMBOL_GPL(mddev_destroy);
822
823	static struct mddev *mddev_find_locked(dev_t unit)
824	{
825	struct mddev *mddev;
826
827	list_for_each_entry(mddev, &all_mddevs, all_mddevs)
828	if (mddev->unit == unit)
829	return mddev;
830
831	return NULL;
832	}
833
834	/* find an unused unit number */
835	static dev_t mddev_alloc_unit(void)
836	{
837	static int next_minor = 512;
838	int start = next_minor;
839	bool is_free = 0;
840	dev_t dev = 0;
841
842	while (!is_free) {
843	dev = MKDEV(MD_MAJOR, next_minor);
844	next_minor++;
845	if (next_minor > MINORMASK)
846	next_minor = 0;
847	if (next_minor == start)
848	return 0; /* Oh dear, all in use. */
849	is_free = !mddev_find_locked(dev);
850	}
851
852	return dev;
853	}
854
855	static struct mddev *mddev_alloc(dev_t unit)
856	{
857	struct mddev *new;
858	int error;
859
860	if (unit && MAJOR(unit) != MD_MAJOR)
861	unit &= ~((1 << MdpMinorShift) - 1);
862
863	new = kzalloc_obj(*new);
864	if (!new)
865	return ERR_PTR(-ENOMEM);
866
867	error = mddev_init(new);
868	if (error)
869	goto out_free_new;
870
871	spin_lock(&all_mddevs_lock);
872	if (unit) {
873	error = -EEXIST;
874	if (mddev_find_locked(unit))
875	goto out_destroy_new;
876	new->unit = unit;
877	if (MAJOR(unit) == MD_MAJOR)
878	new->md_minor = MINOR(unit);
879	else
880	new->md_minor = MINOR(unit) >> MdpMinorShift;
881	new->hold_active = UNTIL_IOCTL;
882	} else {
883	error = -ENODEV;
884	new->unit = mddev_alloc_unit();
885	if (!new->unit)
886	goto out_destroy_new;
887	new->md_minor = MINOR(new->unit);
888	new->hold_active = UNTIL_STOP;
889	}
890
891	list_add(&new->all_mddevs, &all_mddevs);
892	spin_unlock(&all_mddevs_lock);
893	return new;
894
895	out_destroy_new:
896	spin_unlock(&all_mddevs_lock);
897	mddev_destroy(new);
898	out_free_new:
899	kfree(new);
900	return ERR_PTR(error);
901	}
902
903	static void mddev_free(struct mddev *mddev)
904	{
905	spin_lock(&all_mddevs_lock);
906	list_del(&mddev->all_mddevs);
907	spin_unlock(&all_mddevs_lock);
908
909	mddev_destroy(mddev);
910	kfree(mddev);
911	}
912
913	static const struct attribute_group md_redundancy_group;
914
915	void mddev_unlock(struct mddev *mddev)
916	{
917	struct md_rdev *rdev;
918	struct md_rdev *tmp;
919	LIST_HEAD(delete);
920
921	if (!list_empty(&mddev->deleting))
922	list_splice_init(&mddev->deleting, &delete);
923
924	if (mddev->to_remove) {
925	/* These cannot be removed under reconfig_mutex as
926	* an access to the files will try to take reconfig_mutex
927	* while holding the file unremovable, which leads to
928	* a deadlock.
929	* So hold set sysfs_active while the remove in happeing,
930	* and anything else which might set ->to_remove or my
931	* otherwise change the sysfs namespace will fail with
932	* -EBUSY if sysfs_active is still set.
933	* We set sysfs_active under reconfig_mutex and elsewhere
934	* test it under the same mutex to ensure its correct value
935	* is seen.
936	*/
937	const struct attribute_group *to_remove = mddev->to_remove;
938	mddev->to_remove = NULL;
939	mddev->sysfs_active = 1;
940	mutex_unlock(&mddev->reconfig_mutex);
941
942	if (mddev->kobj.sd) {
943	if (to_remove != &md_redundancy_group)
944	sysfs_remove_group(&mddev->kobj, to_remove);
945	if (mddev->pers == NULL \|\|
946	mddev->pers->sync_request == NULL) {
947	sysfs_remove_group(&mddev->kobj, &md_redundancy_group);
948	if (mddev->sysfs_action)
949	sysfs_put(mddev->sysfs_action);
950	if (mddev->sysfs_completed)
951	sysfs_put(mddev->sysfs_completed);
952	if (mddev->sysfs_degraded)
953	sysfs_put(mddev->sysfs_degraded);
954	mddev->sysfs_action = NULL;
955	mddev->sysfs_completed = NULL;
956	mddev->sysfs_degraded = NULL;
957	}
958	}
959	mddev->sysfs_active = 0;
960	} else
961	mutex_unlock(&mddev->reconfig_mutex);
962
963	md_wakeup_thread(mddev->thread);
964	wake_up(&mddev->sb_wait);
965
966	list_for_each_entry_safe(rdev, tmp, &delete, same_set) {
967	list_del_init(&rdev->same_set);
968	kobject_del(&rdev->kobj);
969	export_rdev(rdev);
970	}
971
972	if (!legacy_async_del_gendisk) {
973	/*
974	* Call del_gendisk after release reconfig_mutex to avoid
975	* deadlock (e.g. call del_gendisk under the lock and an
976	* access to sysfs files waits the lock)
977	* And MD_DELETED is only used for md raid which is set in
978	* do_md_stop. dm raid only uses md_stop to stop. So dm raid
979	* doesn't need to check MD_DELETED when getting reconfig lock
980	*/
981	if (test_bit(MD_DELETED, &mddev->flags) &&
982	!test_and_set_bit(MD_DO_DELETE, &mddev->flags)) {
983	kobject_del(&mddev->kobj);
984	del_gendisk(mddev->gendisk);
985	}
986	}
987	}
988	EXPORT_SYMBOL_GPL(mddev_unlock);
989
990	struct md_rdev md_find_rdev_nr_rcu(struct mddev mddev, int nr)
991	{
992	struct md_rdev *rdev;
993
994	rdev_for_each_rcu(rdev, mddev)
995	if (rdev->desc_nr == nr)
996	return rdev;
997
998	return NULL;
999	}
1000	EXPORT_SYMBOL_GPL(md_find_rdev_nr_rcu);
1001
1002	static struct md_rdev find_rdev(struct mddev mddev, dev_t dev)
1003	{
1004	struct md_rdev *rdev;
1005
1006	rdev_for_each(rdev, mddev)
1007	if (rdev->bdev->bd_dev == dev)
1008	return rdev;
1009
1010	return NULL;
1011	}
1012
1013	struct md_rdev md_find_rdev_rcu(struct mddev mddev, dev_t dev)
1014	{
1015	struct md_rdev *rdev;
1016
1017	rdev_for_each_rcu(rdev, mddev)
1018	if (rdev->bdev->bd_dev == dev)
1019	return rdev;
1020
1021	return NULL;
1022	}
1023	EXPORT_SYMBOL_GPL(md_find_rdev_rcu);
1024
1025	static struct md_personality get_pers(int level, char clevel)
1026	{
1027	struct md_personality *ret = NULL;
1028	struct md_submodule_head *head;
1029	unsigned long i;
1030
1031	xa_lock(&md_submodule);
1032	xa_for_each(&md_submodule, i, head) {
1033	if (head->type != MD_PERSONALITY)
1034	continue;
1035	if ((level != LEVEL_NONE && head->id == level) \|\|
1036	!strcmp(head->name, clevel)) {
1037	if (try_module_get(head->owner))
1038	ret = (void *)head;
1039	break;
1040	}
1041	}
1042	xa_unlock(&md_submodule);
1043
1044	if (!ret) {
1045	if (level != LEVEL_NONE)
1046	pr_warn("md: personality for level %d is not loaded!\n",
1047	level);
1048	else
1049	pr_warn("md: personality for level %s is not loaded!\n",
1050	clevel);
1051	}
1052
1053	return ret;
1054	}
1055
1056	static void put_pers(struct md_personality *pers)
1057	{
1058	module_put(pers->head.owner);
1059	}
1060
1061	/* return the offset of the super block in 512byte sectors */
1062	static inline sector_t calc_dev_sboffset(struct md_rdev *rdev)
1063	{
1064	return MD_NEW_SIZE_SECTORS(bdev_nr_sectors(rdev->bdev));
1065	}
1066
1067	static int alloc_disk_sb(struct md_rdev *rdev)
1068	{
1069	rdev->sb_page = alloc_page(GFP_KERNEL);
1070	if (!rdev->sb_page)
1071	return -ENOMEM;
1072	return 0;
1073	}
1074
1075	void md_rdev_clear(struct md_rdev *rdev)
1076	{
1077	if (rdev->sb_page) {
1078	put_page(rdev->sb_page);
1079	rdev->sb_loaded = 0;
1080	rdev->sb_page = NULL;
1081	rdev->sb_start = 0;
1082	rdev->sectors = 0;
1083	}
1084	if (rdev->bb_page) {
1085	put_page(rdev->bb_page);
1086	rdev->bb_page = NULL;
1087	}
1088	badblocks_exit(&rdev->badblocks);
1089	}
1090	EXPORT_SYMBOL_GPL(md_rdev_clear);
1091
1092	static void super_written(struct bio *bio)
1093	{
1094	struct md_rdev *rdev = bio->bi_private;
1095	struct mddev *mddev = rdev->mddev;
1096
1097	if (bio->bi_status) {
1098	pr_err("md: %s gets error=%d\n", __func__,
1099	blk_status_to_errno(bio->bi_status));
1100	md_error(mddev, rdev);
1101	if (!test_bit(Faulty, &rdev->flags)
1102	&& (bio->bi_opf & MD_FAILFAST)) {
1103	set_bit(MD_SB_NEED_REWRITE, &mddev->sb_flags);
1104	set_bit(LastDev, &rdev->flags);
1105	}
1106	} else
1107	clear_bit(LastDev, &rdev->flags);
1108
1109	bio_put(bio);
1110
1111	rdev_dec_pending(rdev, mddev);
1112
1113	if (atomic_dec_and_test(&mddev->pending_writes))
1114	wake_up(&mddev->sb_wait);
1115	}
1116
1117	/**
1118	* md_write_metadata - write metadata to underlying disk, including
1119	* array superblock, badblocks, bitmap superblock and bitmap bits.
1120	* @mddev: the array to write
1121	* @rdev: the underlying disk to write
1122	* @sector: the offset to @rdev
1123	* @size: the length of the metadata
1124	* @page: the metadata
1125	* @offset: the offset to @page
1126	*
1127	* Write @size bytes of @page start from @offset, to @sector of @rdev, Increment
1128	* mddev->pending_writes before returning, and decrement it on completion,
1129	* waking up sb_wait. Caller must call md_super_wait() after issuing io to all
1130	* rdev. If an error occurred, md_error() will be called, and the @rdev will be
1131	* kicked out from @mddev.
1132	*/
1133	void md_write_metadata(struct mddev mddev, struct md_rdev rdev,
1134	sector_t sector, int size, struct page *page,
1135	unsigned int offset)
1136	{
1137	struct bio *bio;
1138
1139	if (!page)
1140	return;
1141
1142	if (test_bit(Faulty, &rdev->flags))
1143	return;
1144
1145	bio = bio_alloc_bioset(rdev->meta_bdev ? rdev->meta_bdev : rdev->bdev,
1146	1,
1147	REQ_OP_WRITE \| REQ_SYNC \| REQ_IDLE \| REQ_META
1148	\| REQ_PREFLUSH \| REQ_FUA,
1149	GFP_NOIO, &mddev->sync_set);
1150
1151	atomic_inc(&rdev->nr_pending);
1152
1153	bio->bi_iter.bi_sector = sector;
1154	__bio_add_page(bio, page, size, offset);
1155	bio->bi_private = rdev;
1156	bio->bi_end_io = super_written;
1157
1158	if (test_bit(MD_FAILFAST_SUPPORTED, &mddev->flags) &&
1159	test_bit(FailFast, &rdev->flags) &&
1160	!test_bit(LastDev, &rdev->flags))
1161	bio->bi_opf \|= MD_FAILFAST;
1162
1163	atomic_inc(&mddev->pending_writes);
1164	submit_bio(bio);
1165	}
1166
1167	int md_super_wait(struct mddev *mddev)
1168	{
1169	/* wait for all superblock writes that were scheduled to complete */
1170	wait_event(mddev->sb_wait, atomic_read(&mddev->pending_writes)==0);
1171	if (test_and_clear_bit(MD_SB_NEED_REWRITE, &mddev->sb_flags))
1172	return -EAGAIN;
1173	return 0;
1174	}
1175
1176	int sync_page_io(struct md_rdev *rdev, sector_t sector, int size,
1177	struct page *page, blk_opf_t opf, bool metadata_op)
1178	{
1179	struct bio bio;
1180	struct bio_vec bvec;
1181
1182	if (metadata_op && rdev->meta_bdev)
1183	bio_init(&bio, rdev->meta_bdev, &bvec, 1, opf);
1184	else
1185	bio_init(&bio, rdev->bdev, &bvec, 1, opf);
1186
1187	if (metadata_op)
1188	bio.bi_iter.bi_sector = sector + rdev->sb_start;
1189	else if (rdev->mddev->reshape_position != MaxSector &&
1190	(rdev->mddev->reshape_backwards ==
1191	(sector >= rdev->mddev->reshape_position)))
1192	bio.bi_iter.bi_sector = sector + rdev->new_data_offset;
1193	else
1194	bio.bi_iter.bi_sector = sector + rdev->data_offset;
1195	__bio_add_page(&bio, page, size, 0);
1196
1197	submit_bio_wait(&bio);
1198
1199	return !bio.bi_status;
1200	}
1201	EXPORT_SYMBOL_GPL(sync_page_io);
1202
1203	static int read_disk_sb(struct md_rdev *rdev, int size)
1204	{
1205	if (rdev->sb_loaded)
1206	return 0;
1207
1208	if (!sync_page_io(rdev, 0, size, rdev->sb_page, REQ_OP_READ, true))
1209	goto fail;
1210	rdev->sb_loaded = 1;
1211	return 0;
1212
1213	fail:
1214	pr_err("md: disabled device %pg, could not read superblock.\n",
1215	rdev->bdev);
1216	return -EINVAL;
1217	}
1218
1219	static int md_uuid_equal(mdp_super_t sb1, mdp_super_t sb2)
1220	{
1221	return sb1->set_uuid0 == sb2->set_uuid0 &&
1222	sb1->set_uuid1 == sb2->set_uuid1 &&
1223	sb1->set_uuid2 == sb2->set_uuid2 &&
1224	sb1->set_uuid3 == sb2->set_uuid3;
1225	}
1226
1227	static int md_sb_equal(mdp_super_t sb1, mdp_super_t sb2)
1228	{
1229	int ret;
1230	mdp_super_t tmp1, tmp2;
1231
1232	tmp1 = kmalloc_obj(*tmp1);
1233	tmp2 = kmalloc_obj(*tmp2);
1234
1235	if (!tmp1 \|\| !tmp2) {
1236	ret = 0;
1237	goto abort;
1238	}
1239
1240	tmp1 = sb1;
1241	tmp2 = sb2;
1242
1243	/*
1244	* nr_disks is not constant
1245	*/
1246	tmp1->nr_disks = 0;
1247	tmp2->nr_disks = 0;
1248
1249	ret = (memcmp(tmp1, tmp2, MD_SB_GENERIC_CONSTANT_WORDS * 4) == 0);
1250	abort:
1251	kfree(tmp1);
1252	kfree(tmp2);
1253	return ret;
1254	}
1255
1256	static u32 md_csum_fold(u32 csum)
1257	{
1258	csum = (csum & 0xffff) + (csum >> 16);
1259	return (csum & 0xffff) + (csum >> 16);
1260	}
1261
1262	static unsigned int calc_sb_csum(mdp_super_t *sb)
1263	{
1264	u64 newcsum = 0;
1265	u32 sb32 = (u32)sb;
1266	int i;
1267	unsigned int disk_csum, csum;
1268
1269	disk_csum = sb->sb_csum;
1270	sb->sb_csum = 0;
1271
1272	for (i = 0; i < MD_SB_BYTES/4 ; i++)
1273	newcsum += sb32[i];
1274	csum = (newcsum & 0xffffffff) + (newcsum>>32);
1275
1276	#ifdef CONFIG_ALPHA
1277	/* This used to use csum_partial, which was wrong for several
1278	* reasons including that different results are returned on
1279	* different architectures. It isn't critical that we get exactly
1280	* the same return value as before (we always csum_fold before
1281	* testing, and that removes any differences). However as we
1282	* know that csum_partial always returned a 16bit value on
1283	* alphas, do a fold to maximise conformity to previous behaviour.
1284	*/
1285	sb->sb_csum = md_csum_fold(disk_csum);
1286	#else
1287	sb->sb_csum = disk_csum;
1288	#endif
1289	return csum;
1290	}
1291
1292	/*
1293	* Handle superblock details.
1294	* We want to be able to handle multiple superblock formats
1295	* so we have a common interface to them all, and an array of
1296	* different handlers.
1297	* We rely on user-space to write the initial superblock, and support
1298	* reading and updating of superblocks.
1299	* Interface methods are:
1300	* int load_super(struct md_rdev dev, struct md_rdev refdev, int minor_version)
1301	* loads and validates a superblock on dev.
1302	* if refdev != NULL, compare superblocks on both devices
1303	* Return:
1304	* 0 - dev has a superblock that is compatible with refdev
1305	* 1 - dev has a superblock that is compatible and newer than refdev
1306	* so dev should be used as the refdev in future
1307	* -EINVAL superblock incompatible or invalid
1308	* -othererror e.g. -EIO
1309	*
1310	* int validate_super(struct mddev mddev, struct md_rdev dev)
1311	* Verify that dev is acceptable into mddev.
1312	* The first time, mddev->raid_disks will be 0, and data from
1313	* dev should be merged in. Subsequent calls check that dev
1314	* is new enough. Return 0 or -EINVAL
1315	*
1316	* void sync_super(struct mddev mddev, struct md_rdev dev)
1317	* Update the superblock for rdev with data in mddev
1318	* This does not write to disc.
1319	*
1320	*/
1321
1322	struct super_type {
1323	char *name;
1324	struct module *owner;
1325	int (load_super)(struct md_rdev rdev,
1326	struct md_rdev *refdev,
1327	int minor_version);
1328	int (validate_super)(struct mddev mddev,
1329	struct md_rdev *freshest,
1330	struct md_rdev *rdev);
1331	void (sync_super)(struct mddev mddev,
1332	struct md_rdev *rdev);
1333	unsigned long long (rdev_size_change)(struct md_rdev rdev,
1334	sector_t num_sectors);
1335	int (allow_new_offset)(struct md_rdev rdev,
1336	unsigned long long new_offset);
1337	};
1338
1339	/*
1340	* Check that the given mddev has no bitmap.
1341	*
1342	* This function is called from the run method of all personalities that do not
1343	* support bitmaps. It prints an error message and returns non-zero if mddev
1344	* has a bitmap. Otherwise, it returns 0.
1345	*
1346	*/
1347	int md_check_no_bitmap(struct mddev *mddev)
1348	{
1349	if (!mddev->bitmap_info.file && !mddev->bitmap_info.offset)
1350	return 0;
1351	pr_warn("%s: bitmaps are not supported for %s\n",
1352	mdname(mddev), mddev->pers->head.name);
1353	return 1;
1354	}
1355	EXPORT_SYMBOL(md_check_no_bitmap);
1356
1357	/*
1358	* load_super for 0.90.0
1359	*/
1360	static int super_90_load(struct md_rdev rdev, struct md_rdev refdev, int minor_version)
1361	{
1362	mdp_super_t *sb;
1363	int ret;
1364	bool spare_disk = true;
1365
1366	/*
1367	* Calculate the position of the superblock (512byte sectors),
1368	* it's at the end of the disk.
1369	*
1370	* It also happens to be a multiple of 4Kb.
1371	*/
1372	rdev->sb_start = calc_dev_sboffset(rdev);
1373
1374	ret = read_disk_sb(rdev, MD_SB_BYTES);
1375	if (ret)
1376	return ret;
1377
1378	ret = -EINVAL;
1379
1380	sb = page_address(rdev->sb_page);
1381
1382	if (sb->md_magic != MD_SB_MAGIC) {
1383	pr_warn("md: invalid raid superblock magic on %pg\n",
1384	rdev->bdev);
1385	goto abort;
1386	}
1387
1388	if (sb->major_version != 0 \|\|
1389	sb->minor_version < 90 \|\|
1390	sb->minor_version > 91) {
1391	pr_warn("Bad version number %d.%d on %pg\n",
1392	sb->major_version, sb->minor_version, rdev->bdev);
1393	goto abort;
1394	}
1395
1396	if (sb->raid_disks <= 0)
1397	goto abort;
1398
1399	if (md_csum_fold(calc_sb_csum(sb)) != md_csum_fold(sb->sb_csum)) {
1400	pr_warn("md: invalid superblock checksum on %pg\n", rdev->bdev);
1401	goto abort;
1402	}
1403
1404	rdev->preferred_minor = sb->md_minor;
1405	rdev->data_offset = 0;
1406	rdev->new_data_offset = 0;
1407	rdev->sb_size = MD_SB_BYTES;
1408	rdev->badblocks.shift = -1;
1409
1410	rdev->desc_nr = sb->this_disk.number;
1411
1412	/* not spare disk */
1413	if (rdev->desc_nr >= 0 && rdev->desc_nr < MD_SB_DISKS &&
1414	sb->disks[rdev->desc_nr].state & ((1<<MD_DISK_SYNC) \| (1 << MD_DISK_ACTIVE)))
1415	spare_disk = false;
1416
1417	if (!refdev) {
1418	if (!spare_disk)
1419	ret = 1;
1420	else
1421	ret = 0;
1422	} else {
1423	__u64 ev1, ev2;
1424	mdp_super_t *refsb = page_address(refdev->sb_page);
1425	if (!md_uuid_equal(refsb, sb)) {
1426	pr_warn("md: %pg has different UUID to %pg\n",
1427	rdev->bdev, refdev->bdev);
1428	goto abort;
1429	}
1430	if (!md_sb_equal(refsb, sb)) {
1431	pr_warn("md: %pg has same UUID but different superblock to %pg\n",
1432	rdev->bdev, refdev->bdev);
1433	goto abort;
1434	}
1435	ev1 = md_event(sb);
1436	ev2 = md_event(refsb);
1437
1438	if (!spare_disk && ev1 > ev2)
1439	ret = 1;
1440	else
1441	ret = 0;
1442	}
1443	rdev->sectors = rdev->sb_start;
1444	/* Limit to 4TB as metadata cannot record more than that.
1445	* (not needed for Linear and RAID0 as metadata doesn't
1446	* record this size)
1447	*/
1448	if ((u64)rdev->sectors >= (2ULL << 32) && sb->level >= 1)
1449	rdev->sectors = (sector_t)(2ULL << 32) - 2;
1450
1451	if (rdev->sectors < ((sector_t)sb->size) * 2 && sb->level >= 1)
1452	/* "this cannot possibly happen" ... */
1453	ret = -EINVAL;
1454
1455	abort:
1456	return ret;
1457	}
1458
1459	static u64 md_bitmap_events_cleared(struct mddev *mddev)
1460	{
1461	struct md_bitmap_stats stats;
1462	int err;
1463
1464	if (!md_bitmap_enabled(mddev, false))
1465	return 0;
1466
1467	err = mddev->bitmap_ops->get_stats(mddev->bitmap, &stats);
1468	if (err)
1469	return 0;
1470
1471	return stats.events_cleared;
1472	}
1473
1474	/*
1475	* validate_super for 0.90.0
1476	* note: we are not using "freshest" for 0.9 superblock
1477	*/
1478	static int super_90_validate(struct mddev mddev, struct md_rdev freshest, struct md_rdev *rdev)
1479	{
1480	mdp_disk_t *desc;
1481	mdp_super_t *sb = page_address(rdev->sb_page);
1482	__u64 ev1 = md_event(sb);
1483
1484	rdev->raid_disk = -1;
1485	clear_bit(Faulty, &rdev->flags);
1486	clear_bit(In_sync, &rdev->flags);
1487	clear_bit(Bitmap_sync, &rdev->flags);
1488	clear_bit(WriteMostly, &rdev->flags);
1489
1490	if (mddev->raid_disks == 0) {
1491	mddev->major_version = 0;
1492	mddev->minor_version = sb->minor_version;
1493	mddev->patch_version = sb->patch_version;
1494	mddev->external = 0;
1495	mddev->chunk_sectors = sb->chunk_size >> 9;
1496	mddev->ctime = sb->ctime;
1497	mddev->utime = sb->utime;
1498	mddev->level = sb->level;
1499	mddev->clevel[0] = 0;
1500	mddev->layout = sb->layout;
1501	mddev->raid_disks = sb->raid_disks;
1502	mddev->dev_sectors = ((sector_t)sb->size) * 2;
1503	mddev->events = ev1;
1504	mddev->bitmap_info.offset = 0;
1505	mddev->bitmap_info.space = 0;
1506	/* bitmap can use 60 K after the 4K superblocks */
1507	mddev->bitmap_info.default_offset = MD_SB_BYTES >> 9;
1508	mddev->bitmap_info.default_space = 64*2 - (MD_SB_BYTES >> 9);
1509	mddev->reshape_backwards = 0;
1510
1511	if (mddev->minor_version >= 91) {
1512	mddev->reshape_position = sb->reshape_position;
1513	mddev->delta_disks = sb->delta_disks;
1514	mddev->new_level = sb->new_level;
1515	mddev->new_layout = sb->new_layout;
1516	mddev->new_chunk_sectors = sb->new_chunk >> 9;
1517	if (mddev->delta_disks < 0)
1518	mddev->reshape_backwards = 1;
1519	} else {
1520	mddev->reshape_position = MaxSector;
1521	mddev->delta_disks = 0;
1522	mddev->new_level = mddev->level;
1523	mddev->new_layout = mddev->layout;
1524	mddev->new_chunk_sectors = mddev->chunk_sectors;
1525	}
1526	if (mddev->level == 0)
1527	mddev->layout = -1;
1528
1529	if (sb->state & (1<<MD_SB_CLEAN))
1530	mddev->resync_offset = MaxSector;
1531	else {
1532	if (sb->events_hi == sb->cp_events_hi &&
1533	sb->events_lo == sb->cp_events_lo) {
1534	mddev->resync_offset = sb->recovery_cp;
1535	} else
1536	mddev->resync_offset = 0;
1537	}
1538
1539	memcpy(mddev->uuid+0, &sb->set_uuid0, 4);
1540	memcpy(mddev->uuid+4, &sb->set_uuid1, 4);
1541	memcpy(mddev->uuid+8, &sb->set_uuid2, 4);
1542	memcpy(mddev->uuid+12,&sb->set_uuid3, 4);
1543
1544	mddev->max_disks = MD_SB_DISKS;
1545
1546	if (sb->state & (1<<MD_SB_BITMAP_PRESENT) &&
1547	mddev->bitmap_info.file == NULL) {
1548	mddev->bitmap_info.offset =
1549	mddev->bitmap_info.default_offset;
1550	mddev->bitmap_info.space =
1551	mddev->bitmap_info.default_space;
1552	}
1553
1554	} else if (mddev->pers == NULL) {
1555	/* Insist on good event counter while assembling, except
1556	* for spares (which don't need an event count) */
1557	++ev1;
1558	if (sb->disks[rdev->desc_nr].state & (
1559	(1<<MD_DISK_SYNC) \| (1 << MD_DISK_ACTIVE)))
1560	if (ev1 < mddev->events)
1561	return -EINVAL;
1562	} else if (mddev->bitmap) {
1563	/* if adding to array with a bitmap, then we can accept an
1564	* older device ... but not too old.
1565	*/
1566	if (ev1 < md_bitmap_events_cleared(mddev))
1567	return 0;
1568	if (ev1 < mddev->events)
1569	set_bit(Bitmap_sync, &rdev->flags);
1570	} else {
1571	if (ev1 < mddev->events)
1572	/* just a hot-add of a new device, leave raid_disk at -1 */
1573	return 0;
1574	}
1575
1576	desc = sb->disks + rdev->desc_nr;
1577
1578	if (desc->state & (1<<MD_DISK_FAULTY))
1579	set_bit(Faulty, &rdev->flags);
1580	else if (desc->state & (1<<MD_DISK_SYNC)) {
1581	set_bit(In_sync, &rdev->flags);
1582	rdev->raid_disk = desc->raid_disk;
1583	rdev->saved_raid_disk = desc->raid_disk;
1584	} else if (desc->state & (1<<MD_DISK_ACTIVE)) {
1585	/* active but not in sync implies recovery up to
1586	* reshape position. We don't know exactly where
1587	* that is, so set to zero for now
1588	*/
1589	if (mddev->minor_version >= 91) {
1590	rdev->recovery_offset = 0;
1591	rdev->raid_disk = desc->raid_disk;
1592	}
1593	}
1594	if (desc->state & (1<<MD_DISK_WRITEMOSTLY))
1595	set_bit(WriteMostly, &rdev->flags);
1596	if (desc->state & (1<<MD_DISK_FAILFAST))
1597	set_bit(FailFast, &rdev->flags);
1598	return 0;
1599	}
1600
1601	/*
1602	* sync_super for 0.90.0
1603	*/
1604	static void super_90_sync(struct mddev mddev, struct md_rdev rdev)
1605	{
1606	mdp_super_t *sb;
1607	struct md_rdev *rdev2;
1608	int next_spare = mddev->raid_disks;
1609
1610	/* make rdev->sb match mddev data..
1611	*
1612	* 1/ zero out disks
1613	* 2/ Add info for each disk, keeping track of highest desc_nr (next_spare);
1614	* 3/ any empty disks < next_spare become removed
1615	*
1616	* disks[0] gets initialised to REMOVED because
1617	* we cannot be sure from other fields if it has
1618	* been initialised or not.
1619	*/
1620	int i;
1621	int active=0, working=0,failed=0,spare=0,nr_disks=0;
1622
1623	rdev->sb_size = MD_SB_BYTES;
1624
1625	sb = page_address(rdev->sb_page);
1626
1627	memset(sb, 0, sizeof(*sb));
1628
1629	sb->md_magic = MD_SB_MAGIC;
1630	sb->major_version = mddev->major_version;
1631	sb->patch_version = mddev->patch_version;
1632	sb->gvalid_words = 0; /* ignored */
1633	memcpy(&sb->set_uuid0, mddev->uuid+0, 4);
1634	memcpy(&sb->set_uuid1, mddev->uuid+4, 4);
1635	memcpy(&sb->set_uuid2, mddev->uuid+8, 4);
1636	memcpy(&sb->set_uuid3, mddev->uuid+12,4);
1637
1638	sb->ctime = clamp_t(time64_t, mddev->ctime, 0, U32_MAX);
1639	sb->level = mddev->level;
1640	sb->size = mddev->dev_sectors / 2;
1641	sb->raid_disks = mddev->raid_disks;
1642	sb->md_minor = mddev->md_minor;
1643	sb->not_persistent = 0;
1644	sb->utime = clamp_t(time64_t, mddev->utime, 0, U32_MAX);
1645	sb->state = 0;
1646	sb->events_hi = (mddev->events>>32);
1647	sb->events_lo = (u32)mddev->events;
1648
1649	if (mddev->reshape_position == MaxSector)
1650	sb->minor_version = 90;
1651	else {
1652	sb->minor_version = 91;
1653	sb->reshape_position = mddev->reshape_position;
1654	sb->new_level = mddev->new_level;
1655	sb->delta_disks = mddev->delta_disks;
1656	sb->new_layout = mddev->new_layout;
1657	sb->new_chunk = mddev->new_chunk_sectors << 9;
1658	}
1659	mddev->minor_version = sb->minor_version;
1660	if (mddev->in_sync)
1661	{
1662	sb->recovery_cp = mddev->resync_offset;
1663	sb->cp_events_hi = (mddev->events>>32);
1664	sb->cp_events_lo = (u32)mddev->events;
1665	if (mddev->resync_offset == MaxSector)
1666	sb->state = (1<< MD_SB_CLEAN);
1667	} else
1668	sb->recovery_cp = 0;
1669
1670	sb->layout = mddev->layout;
1671	sb->chunk_size = mddev->chunk_sectors << 9;
1672
1673	if (mddev->bitmap && mddev->bitmap_info.file == NULL)
1674	sb->state \|= (1<<MD_SB_BITMAP_PRESENT);
1675
1676	sb->disks[0].state = (1<<MD_DISK_REMOVED);
1677	rdev_for_each(rdev2, mddev) {
1678	mdp_disk_t *d;
1679	int desc_nr;
1680	int is_active = test_bit(In_sync, &rdev2->flags);
1681
1682	if (rdev2->raid_disk >= 0 &&
1683	sb->minor_version >= 91)
1684	/* we have nowhere to store the recovery_offset,
1685	* but if it is not below the reshape_position,
1686	* we can piggy-back on that.
1687	*/
1688	is_active = 1;
1689	if (rdev2->raid_disk < 0 \|\|
1690	test_bit(Faulty, &rdev2->flags))
1691	is_active = 0;
1692	if (is_active)
1693	desc_nr = rdev2->raid_disk;
1694	else
1695	desc_nr = next_spare++;
1696	rdev2->desc_nr = desc_nr;
1697	d = &sb->disks[rdev2->desc_nr];
1698	nr_disks++;
1699	d->number = rdev2->desc_nr;
1700	d->major = MAJOR(rdev2->bdev->bd_dev);
1701	d->minor = MINOR(rdev2->bdev->bd_dev);
1702	if (is_active)
1703	d->raid_disk = rdev2->raid_disk;
1704	else
1705	d->raid_disk = rdev2->desc_nr; /* compatibility */
1706	if (test_bit(Faulty, &rdev2->flags))
1707	d->state = (1<<MD_DISK_FAULTY);
1708	else if (is_active) {
1709	d->state = (1<<MD_DISK_ACTIVE);
1710	if (test_bit(In_sync, &rdev2->flags))
1711	d->state \|= (1<<MD_DISK_SYNC);
1712	active++;
1713	working++;
1714	} else {
1715	d->state = 0;
1716	spare++;
1717	working++;
1718	}
1719	if (test_bit(WriteMostly, &rdev2->flags))
1720	d->state \|= (1<<MD_DISK_WRITEMOSTLY);
1721	if (test_bit(FailFast, &rdev2->flags))
1722	d->state \|= (1<<MD_DISK_FAILFAST);
1723	}
1724	/* now set the "removed" and "faulty" bits on any missing devices */
1725	for (i=0 ; i < mddev->raid_disks ; i++) {
1726	mdp_disk_t *d = &sb->disks[i];
1727	if (d->state == 0 && d->number == 0) {
1728	d->number = i;
1729	d->raid_disk = i;
1730	d->state = (1<<MD_DISK_REMOVED);
1731	d->state \|= (1<<MD_DISK_FAULTY);
1732	failed++;
1733	}
1734	}
1735	sb->nr_disks = nr_disks;
1736	sb->active_disks = active;
1737	sb->working_disks = working;
1738	sb->failed_disks = failed;
1739	sb->spare_disks = spare;
1740
1741	sb->this_disk = sb->disks[rdev->desc_nr];
1742	sb->sb_csum = calc_sb_csum(sb);
1743	}
1744
1745	/*
1746	* rdev_size_change for 0.90.0
1747	*/
1748	static unsigned long long
1749	super_90_rdev_size_change(struct md_rdev *rdev, sector_t num_sectors)
1750	{
1751	if (num_sectors && num_sectors < rdev->mddev->dev_sectors)
1752	return 0; /* component must fit device */
1753	if (rdev->mddev->bitmap_info.offset)
1754	return 0; /* can't move bitmap */
1755	rdev->sb_start = calc_dev_sboffset(rdev);
1756	if (!num_sectors \|\| num_sectors > rdev->sb_start)
1757	num_sectors = rdev->sb_start;
1758	/* Limit to 4TB as metadata cannot record more than that.
1759	* 4TB == 2^32 KB, or 2*2^32 sectors.
1760	*/
1761	if ((u64)num_sectors >= (2ULL << 32) && rdev->mddev->level >= 1)
1762	num_sectors = (sector_t)(2ULL << 32) - 2;
1763	do {
1764	md_write_metadata(rdev->mddev, rdev, rdev->sb_start,
1765	rdev->sb_size, rdev->sb_page, 0);
1766	} while (md_super_wait(rdev->mddev) < 0);
1767	return num_sectors;
1768	}
1769
1770	static int
1771	super_90_allow_new_offset(struct md_rdev *rdev, unsigned long long new_offset)
1772	{
1773	/* non-zero offset changes not possible with v0.90 */
1774	return new_offset == 0;
1775	}
1776
1777	/*
1778	* version 1 superblock
1779	*/
1780
1781	static __le32 calc_sb_1_csum(struct mdp_superblock_1 *sb)
1782	{
1783	__le32 disk_csum;
1784	u32 csum;
1785	unsigned long long newcsum;
1786	int size = 256 + le32_to_cpu(sb->max_dev)*2;
1787	__le32 isuper = (__le32)sb;
1788
1789	disk_csum = sb->sb_csum;
1790	sb->sb_csum = 0;
1791	newcsum = 0;
1792	for (; size >= 4; size -= 4)
1793	newcsum += le32_to_cpu(*isuper++);
1794
1795	if (size == 2)
1796	newcsum += le16_to_cpu((__le16) isuper);
1797
1798	csum = (newcsum & 0xffffffff) + (newcsum >> 32);
1799	sb->sb_csum = disk_csum;
1800	return cpu_to_le32(csum);
1801	}
1802
1803	static int super_1_load(struct md_rdev rdev, struct md_rdev refdev, int minor_version)
1804	{
1805	struct mdp_superblock_1 *sb;
1806	int ret;
1807	sector_t sb_start;
1808	sector_t sectors;
1809	int bmask;
1810	bool spare_disk = true;
1811
1812	/*
1813	* Calculate the position of the superblock in 512byte sectors.
1814	* It is always aligned to a 4K boundary and
1815	* depeding on minor_version, it can be:
1816	* 0: At least 8K, but less than 12K, from end of device
1817	* 1: At start of device
1818	* 2: 4K from start of device.
1819	*/
1820	switch(minor_version) {
1821	case 0:
1822	sb_start = bdev_nr_sectors(rdev->bdev) - 8 * 2;
1823	sb_start &= ~(sector_t)(4*2-1);
1824	break;
1825	case 1:
1826	sb_start = 0;
1827	break;
1828	case 2:
1829	sb_start = 8;
1830	break;
1831	default:
1832	return -EINVAL;
1833	}
1834	rdev->sb_start = sb_start;
1835
1836	/* superblock is rarely larger than 1K, but it can be larger,
1837	* and it is safe to read 4k, so we do that
1838	*/
1839	ret = read_disk_sb(rdev, 4096);
1840	if (ret) return ret;
1841
1842	sb = page_address(rdev->sb_page);
1843
1844	if (sb->magic != cpu_to_le32(MD_SB_MAGIC) \|\|
1845	sb->major_version != cpu_to_le32(1) \|\|
1846	le32_to_cpu(sb->max_dev) > (4096-256)/2 \|\|
1847	le64_to_cpu(sb->super_offset) != rdev->sb_start \|\|
1848	(le32_to_cpu(sb->feature_map) & ~MD_FEATURE_ALL) != 0)
1849	return -EINVAL;
1850
1851	if (calc_sb_1_csum(sb) != sb->sb_csum) {
1852	pr_warn("md: invalid superblock checksum on %pg\n",
1853	rdev->bdev);
1854	return -EINVAL;
1855	}
1856	if (le64_to_cpu(sb->data_size) < 10) {
1857	pr_warn("md: data_size too small on %pg\n",
1858	rdev->bdev);
1859	return -EINVAL;
1860	}
1861	if (sb->pad0 \|\|
1862	sb->pad3[0] \|\|
1863	memcmp(sb->pad3, sb->pad3+1, sizeof(sb->pad3) - sizeof(sb->pad3[1]))) {
1864	pr_warn("Some padding is non-zero on %pg, might be a new feature\n",
1865	rdev->bdev);
1866	if (check_new_feature)
1867	return -EINVAL;
1868	pr_warn("check_new_feature is disabled, data corruption possible\n");
1869	}
1870
1871	rdev->preferred_minor = 0xffff;
1872	rdev->data_offset = le64_to_cpu(sb->data_offset);
1873	rdev->new_data_offset = rdev->data_offset;
1874	if ((le32_to_cpu(sb->feature_map) & MD_FEATURE_RESHAPE_ACTIVE) &&
1875	(le32_to_cpu(sb->feature_map) & MD_FEATURE_NEW_OFFSET))
1876	rdev->new_data_offset += (s32)le32_to_cpu(sb->new_offset);
1877	atomic_set(&rdev->corrected_errors, le32_to_cpu(sb->cnt_corrected_read));
1878
1879	rdev->sb_size = le32_to_cpu(sb->max_dev) * 2 + 256;
1880	bmask = queue_logical_block_size(rdev->bdev->bd_disk->queue)-1;
1881	if (rdev->sb_size & bmask)
1882	rdev->sb_size = (rdev->sb_size \| bmask) + 1;
1883
1884	if (minor_version
1885	&& rdev->data_offset < sb_start + (rdev->sb_size/512))
1886	return -EINVAL;
1887	if (minor_version
1888	&& rdev->new_data_offset < sb_start + (rdev->sb_size/512))
1889	return -EINVAL;
1890
1891	rdev->desc_nr = le32_to_cpu(sb->dev_number);
1892
1893	if (!rdev->bb_page) {
1894	rdev->bb_page = alloc_page(GFP_KERNEL);
1895	if (!rdev->bb_page)
1896	return -ENOMEM;
1897	}
1898	if ((le32_to_cpu(sb->feature_map) & MD_FEATURE_BAD_BLOCKS) &&
1899	rdev->badblocks.count == 0) {
1900	/* need to load the bad block list.
1901	* Currently we limit it to one page.
1902	*/
1903	s32 offset;
1904	sector_t bb_sector;
1905	__le64 *bbp;
1906	int i;
1907	int sectors = le16_to_cpu(sb->bblog_size);
1908	if (sectors > (PAGE_SIZE / 512))
1909	return -EINVAL;
1910	offset = le32_to_cpu(sb->bblog_offset);
1911	if (offset == 0)
1912	return -EINVAL;
1913	bb_sector = (long long)offset;
1914	if (!sync_page_io(rdev, bb_sector, sectors << 9,
1915	rdev->bb_page, REQ_OP_READ, true))
1916	return -EIO;
1917	bbp = (__le64 *)page_address(rdev->bb_page);
1918
1919	/* check for badblocks api. */
1920	if (sb->bblog_shift >= BITS_PER_TYPE(sector_t)) {
1921	pr_err("md: %pg: bogus bblog_shift %u for badblocks.\n",
1922	rdev->bdev, sb->bblog_shift);
1923	return -EINVAL;
1924	}
1925	rdev->badblocks.shift = sb->bblog_shift;
1926	for (i = 0 ; i < (sectors << (9-3)) ; i++, bbp++) {
1927	u64 bb = le64_to_cpu(*bbp);
1928	int count = bb & (0x3ff);
1929	u64 sector = bb >> 10;
1930	sector <<= sb->bblog_shift;
1931	count <<= sb->bblog_shift;
1932	if (bb + 1 == 0)
1933	break;
1934	if (!badblocks_set(&rdev->badblocks, sector, count, 1))
1935	return -EINVAL;
1936	}
1937	} else if (sb->bblog_offset != 0)
1938	rdev->badblocks.shift = 0;
1939
1940	if ((le32_to_cpu(sb->feature_map) &
1941	(MD_FEATURE_PPL \| MD_FEATURE_MULTIPLE_PPLS))) {
1942	rdev->ppl.offset = (__s16)le16_to_cpu(sb->ppl.offset);
1943	rdev->ppl.size = le16_to_cpu(sb->ppl.size);
1944	rdev->ppl.sector = rdev->sb_start + rdev->ppl.offset;
1945	}
1946
1947	if ((le32_to_cpu(sb->feature_map) & MD_FEATURE_RAID0_LAYOUT) &&
1948	sb->level != 0)
1949	return -EINVAL;
1950
1951	/* not spare disk */
1952	if (rdev->desc_nr >= 0 && rdev->desc_nr < le32_to_cpu(sb->max_dev) &&
1953	(le16_to_cpu(sb->dev_roles[rdev->desc_nr]) < MD_DISK_ROLE_MAX \|\|
1954	le16_to_cpu(sb->dev_roles[rdev->desc_nr]) == MD_DISK_ROLE_JOURNAL))
1955	spare_disk = false;
1956
1957	if (!refdev) {
1958	if (!spare_disk)
1959	ret = 1;
1960	else
1961	ret = 0;
1962	} else {
1963	__u64 ev1, ev2;
1964	struct mdp_superblock_1 *refsb = page_address(refdev->sb_page);
1965
1966	if (memcmp(sb->set_uuid, refsb->set_uuid, 16) != 0 \|\|
1967	sb->level != refsb->level \|\|
1968	sb->layout != refsb->layout \|\|
1969	sb->chunksize != refsb->chunksize) {
1970	pr_warn("md: %pg has strangely different superblock to %pg\n",
1971	rdev->bdev,
1972	refdev->bdev);
1973	return -EINVAL;
1974	}
1975	ev1 = le64_to_cpu(sb->events);
1976	ev2 = le64_to_cpu(refsb->events);
1977
1978	if (!spare_disk && ev1 > ev2)
1979	ret = 1;
1980	else
1981	ret = 0;
1982	}
1983	if (minor_version)
1984	sectors = bdev_nr_sectors(rdev->bdev) - rdev->data_offset;
1985	else
1986	sectors = rdev->sb_start;
1987	if (sectors < le64_to_cpu(sb->data_size))
1988	return -EINVAL;
1989	rdev->sectors = le64_to_cpu(sb->data_size);
1990	return ret;
1991	}
1992
1993	static int super_1_validate(struct mddev mddev, struct md_rdev freshest, struct md_rdev *rdev)
1994	{
1995	struct mdp_superblock_1 *sb = page_address(rdev->sb_page);
1996	__u64 ev1 = le64_to_cpu(sb->events);
1997	int role;
1998
1999	rdev->raid_disk = -1;
2000	clear_bit(Faulty, &rdev->flags);
2001	clear_bit(In_sync, &rdev->flags);
2002	clear_bit(Bitmap_sync, &rdev->flags);
2003	clear_bit(WriteMostly, &rdev->flags);
2004
2005	if (mddev->raid_disks == 0) {
2006	mddev->major_version = 1;
2007	mddev->patch_version = 0;
2008	mddev->external = 0;
2009	mddev->chunk_sectors = le32_to_cpu(sb->chunksize);
2010	mddev->ctime = le64_to_cpu(sb->ctime);
2011	mddev->utime = le64_to_cpu(sb->utime);
2012	mddev->level = le32_to_cpu(sb->level);
2013	mddev->clevel[0] = 0;
2014	mddev->layout = le32_to_cpu(sb->layout);
2015	mddev->raid_disks = le32_to_cpu(sb->raid_disks);
2016	mddev->dev_sectors = le64_to_cpu(sb->size);
2017	mddev->events = ev1;
2018	mddev->bitmap_info.offset = 0;
2019	mddev->bitmap_info.space = 0;
2020	/* Default location for bitmap is 1K after superblock
2021	* using 3K - total of 4K
2022	*/
2023	mddev->bitmap_info.default_offset = 1024 >> 9;
2024	mddev->bitmap_info.default_space = (4096-1024) >> 9;
2025	mddev->reshape_backwards = 0;
2026
2027	mddev->resync_offset = le64_to_cpu(sb->resync_offset);
2028	memcpy(mddev->uuid, sb->set_uuid, 16);
2029
2030	mddev->max_disks = (4096-256)/2;
2031
2032	if (!mddev->logical_block_size)
2033	mddev->logical_block_size = le32_to_cpu(sb->logical_block_size);
2034
2035	if ((le32_to_cpu(sb->feature_map) & MD_FEATURE_BITMAP_OFFSET) &&
2036	mddev->bitmap_info.file == NULL) {
2037	mddev->bitmap_info.offset =
2038	(__s32)le32_to_cpu(sb->bitmap_offset);
2039	/* Metadata doesn't record how much space is available.
2040	* For 1.0, we assume we can use up to the superblock
2041	* if before, else to 4K beyond superblock.
2042	* For others, assume no change is possible.
2043	*/
2044	if (mddev->minor_version > 0)
2045	mddev->bitmap_info.space = 0;
2046	else if (mddev->bitmap_info.offset > 0)
2047	mddev->bitmap_info.space =
2048	8 - mddev->bitmap_info.offset;
2049	else
2050	mddev->bitmap_info.space =
2051	-mddev->bitmap_info.offset;
2052	}
2053
2054	if ((le32_to_cpu(sb->feature_map) & MD_FEATURE_RESHAPE_ACTIVE)) {
2055	mddev->reshape_position = le64_to_cpu(sb->reshape_position);
2056	mddev->delta_disks = le32_to_cpu(sb->delta_disks);
2057	mddev->new_level = le32_to_cpu(sb->new_level);
2058	mddev->new_layout = le32_to_cpu(sb->new_layout);
2059	mddev->new_chunk_sectors = le32_to_cpu(sb->new_chunk);
2060	if (mddev->delta_disks < 0 \|\|
2061	(mddev->delta_disks == 0 &&
2062	(le32_to_cpu(sb->feature_map)
2063	& MD_FEATURE_RESHAPE_BACKWARDS)))
2064	mddev->reshape_backwards = 1;
2065	} else {
2066	mddev->reshape_position = MaxSector;
2067	mddev->delta_disks = 0;
2068	mddev->new_level = mddev->level;
2069	mddev->new_layout = mddev->layout;
2070	mddev->new_chunk_sectors = mddev->chunk_sectors;
2071	}
2072
2073	if (mddev->level == 0 &&
2074	!(le32_to_cpu(sb->feature_map) & MD_FEATURE_RAID0_LAYOUT))
2075	mddev->layout = -1;
2076
2077	if (le32_to_cpu(sb->feature_map) & MD_FEATURE_JOURNAL)
2078	set_bit(MD_HAS_JOURNAL, &mddev->flags);
2079
2080	if (le32_to_cpu(sb->feature_map) &
2081	(MD_FEATURE_PPL \| MD_FEATURE_MULTIPLE_PPLS)) {
2082	if (le32_to_cpu(sb->feature_map) &
2083	(MD_FEATURE_BITMAP_OFFSET \| MD_FEATURE_JOURNAL))
2084	return -EINVAL;
2085	if ((le32_to_cpu(sb->feature_map) & MD_FEATURE_PPL) &&
2086	(le32_to_cpu(sb->feature_map) &
2087	MD_FEATURE_MULTIPLE_PPLS))
2088	return -EINVAL;
2089	set_bit(MD_HAS_PPL, &mddev->flags);
2090	}
2091	} else if (mddev->pers == NULL) {
2092	/* Insist of good event counter while assembling, except for
2093	* spares (which don't need an event count).
2094	* Similar to mdadm, we allow event counter difference of 1
2095	* from the freshest device.
2096	*/
2097	if (rdev->desc_nr >= 0 &&
2098	rdev->desc_nr < le32_to_cpu(sb->max_dev) &&
2099	(le16_to_cpu(sb->dev_roles[rdev->desc_nr]) < MD_DISK_ROLE_MAX \|\|
2100	le16_to_cpu(sb->dev_roles[rdev->desc_nr]) == MD_DISK_ROLE_JOURNAL))
2101	if (ev1 + 1 < mddev->events)
2102	return -EINVAL;
2103	} else if (mddev->bitmap) {
2104	/* If adding to array with a bitmap, then we can accept an
2105	* older device, but not too old.
2106	*/
2107	if (ev1 < md_bitmap_events_cleared(mddev))
2108	return 0;
2109	if (ev1 < mddev->events)
2110	set_bit(Bitmap_sync, &rdev->flags);
2111	} else {
2112	if (ev1 < mddev->events)
2113	/* just a hot-add of a new device, leave raid_disk at -1 */
2114	return 0;
2115	}
2116
2117	if (rdev->desc_nr < 0 \|\|
2118	rdev->desc_nr >= le32_to_cpu(sb->max_dev)) {
2119	role = MD_DISK_ROLE_SPARE;
2120	rdev->desc_nr = -1;
2121	} else if (mddev->pers == NULL && freshest && ev1 < mddev->events) {
2122	/*
2123	* If we are assembling, and our event counter is smaller than the
2124	* highest event counter, we cannot trust our superblock about the role.
2125	* It could happen that our rdev was marked as Faulty, and all other
2126	* superblocks were updated with +1 event counter.
2127	* Then, before the next superblock update, which typically happens when
2128	* remove_and_add_spares() removes the device from the array, there was
2129	* a crash or reboot.
2130	* If we allow current rdev without consulting the freshest superblock,
2131	* we could cause data corruption.
2132	* Note that in this case our event counter is smaller by 1 than the
2133	* highest, otherwise, this rdev would not be allowed into array;
2134	* both kernel and mdadm allow event counter difference of 1.
2135	*/
2136	struct mdp_superblock_1 *freshest_sb = page_address(freshest->sb_page);
2137	u32 freshest_max_dev = le32_to_cpu(freshest_sb->max_dev);
2138
2139	if (rdev->desc_nr >= freshest_max_dev) {
2140	/* this is unexpected, better not proceed */
2141	pr_warn("md: %s: rdev[%pg]: desc_nr(%d) >= freshest(%pg)->sb->max_dev(%u)\n",
2142	mdname(mddev), rdev->bdev, rdev->desc_nr,
2143	freshest->bdev, freshest_max_dev);
2144	return -EUCLEAN;
2145	}
2146
2147	role = le16_to_cpu(freshest_sb->dev_roles[rdev->desc_nr]);
2148	pr_debug("md: %s: rdev[%pg]: role=%d(0x%x) according to freshest %pg\n",
2149	mdname(mddev), rdev->bdev, role, role, freshest->bdev);
2150	} else {
2151	role = le16_to_cpu(sb->dev_roles[rdev->desc_nr]);
2152	}
2153	switch (role) {
2154	case MD_DISK_ROLE_SPARE: /* spare */
2155	break;
2156	case MD_DISK_ROLE_FAULTY: /* faulty */
2157	set_bit(Faulty, &rdev->flags);
2158	break;
2159	case MD_DISK_ROLE_JOURNAL: /* journal device */
2160	if (!(le32_to_cpu(sb->feature_map) & MD_FEATURE_JOURNAL)) {
2161	/* journal device without journal feature */
2162	pr_warn("md: journal device provided without journal feature, ignoring the device\n");
2163	return -EINVAL;
2164	}
2165	set_bit(Journal, &rdev->flags);
2166	rdev->journal_tail = le64_to_cpu(sb->journal_tail);
2167	rdev->raid_disk = 0;
2168	break;
2169	default:
2170	rdev->saved_raid_disk = role;
2171	if ((le32_to_cpu(sb->feature_map) &
2172	MD_FEATURE_RECOVERY_OFFSET)) {
2173	rdev->recovery_offset = le64_to_cpu(sb->recovery_offset);
2174	if (!(le32_to_cpu(sb->feature_map) &
2175	MD_FEATURE_RECOVERY_BITMAP))
2176	rdev->saved_raid_disk = -1;
2177	} else {
2178	/*
2179	* If the array is FROZEN, then the device can't
2180	* be in_sync with rest of array.
2181	*/
2182	if (!test_bit(MD_RECOVERY_FROZEN,
2183	&mddev->recovery))
2184	set_bit(In_sync, &rdev->flags);
2185	}
2186	rdev->raid_disk = role;
2187	break;
2188	}
2189	if (sb->devflags & WriteMostly1)
2190	set_bit(WriteMostly, &rdev->flags);
2191	if (sb->devflags & FailFast1)
2192	set_bit(FailFast, &rdev->flags);
2193	if (le32_to_cpu(sb->feature_map) & MD_FEATURE_REPLACEMENT)
2194	set_bit(Replacement, &rdev->flags);
2195
2196	return 0;
2197	}
2198
2199	static void super_1_sync(struct mddev mddev, struct md_rdev rdev)
2200	{
2201	struct mdp_superblock_1 *sb;
2202	struct md_rdev *rdev2;
2203	int max_dev, i;
2204	/* make rdev->sb match mddev and rdev data. */
2205
2206	sb = page_address(rdev->sb_page);
2207
2208	sb->feature_map = 0;
2209	sb->pad0 = 0;
2210	sb->recovery_offset = cpu_to_le64(0);
2211	memset(sb->pad3, 0, sizeof(sb->pad3));
2212
2213	sb->utime = cpu_to_le64((__u64)mddev->utime);
2214	sb->events = cpu_to_le64(mddev->events);
2215	if (mddev->in_sync)
2216	sb->resync_offset = cpu_to_le64(mddev->resync_offset);
2217	else if (test_bit(MD_JOURNAL_CLEAN, &mddev->flags))
2218	sb->resync_offset = cpu_to_le64(MaxSector);
2219	else
2220	sb->resync_offset = cpu_to_le64(0);
2221
2222	sb->cnt_corrected_read = cpu_to_le32(atomic_read(&rdev->corrected_errors));
2223
2224	sb->raid_disks = cpu_to_le32(mddev->raid_disks);
2225	sb->size = cpu_to_le64(mddev->dev_sectors);
2226	sb->chunksize = cpu_to_le32(mddev->chunk_sectors);
2227	sb->level = cpu_to_le32(mddev->level);
2228	sb->layout = cpu_to_le32(mddev->layout);
2229	sb->logical_block_size = cpu_to_le32(mddev->logical_block_size);
2230	if (test_bit(FailFast, &rdev->flags))
2231	sb->devflags \|= FailFast1;
2232	else
2233	sb->devflags &= ~FailFast1;
2234
2235	if (test_bit(WriteMostly, &rdev->flags))
2236	sb->devflags \|= WriteMostly1;
2237	else
2238	sb->devflags &= ~WriteMostly1;
2239	sb->data_offset = cpu_to_le64(rdev->data_offset);
2240	sb->data_size = cpu_to_le64(rdev->sectors);
2241
2242	if (mddev->bitmap && mddev->bitmap_info.file == NULL) {
2243	sb->bitmap_offset = cpu_to_le32((__u32)mddev->bitmap_info.offset);
2244	sb->feature_map = cpu_to_le32(MD_FEATURE_BITMAP_OFFSET);
2245	}
2246
2247	if (rdev->raid_disk >= 0 && !test_bit(Journal, &rdev->flags) &&
2248	!test_bit(In_sync, &rdev->flags)) {
2249	sb->feature_map \|=
2250	cpu_to_le32(MD_FEATURE_RECOVERY_OFFSET);
2251	sb->recovery_offset =
2252	cpu_to_le64(rdev->recovery_offset);
2253	if (rdev->saved_raid_disk >= 0 && mddev->bitmap)
2254	sb->feature_map \|=
2255	cpu_to_le32(MD_FEATURE_RECOVERY_BITMAP);
2256	}
2257	/* Note: recovery_offset and journal_tail share space */
2258	if (test_bit(Journal, &rdev->flags))
2259	sb->journal_tail = cpu_to_le64(rdev->journal_tail);
2260	if (test_bit(Replacement, &rdev->flags))
2261	sb->feature_map \|=
2262	cpu_to_le32(MD_FEATURE_REPLACEMENT);
2263
2264	if (mddev->reshape_position != MaxSector) {
2265	sb->feature_map \|= cpu_to_le32(MD_FEATURE_RESHAPE_ACTIVE);
2266	sb->reshape_position = cpu_to_le64(mddev->reshape_position);
2267	sb->new_layout = cpu_to_le32(mddev->new_layout);
2268	sb->delta_disks = cpu_to_le32(mddev->delta_disks);
2269	sb->new_level = cpu_to_le32(mddev->new_level);
2270	sb->new_chunk = cpu_to_le32(mddev->new_chunk_sectors);
2271	if (mddev->delta_disks == 0 &&
2272	mddev->reshape_backwards)
2273	sb->feature_map
2274	\|= cpu_to_le32(MD_FEATURE_RESHAPE_BACKWARDS);
2275	if (rdev->new_data_offset != rdev->data_offset) {
2276	sb->feature_map
2277	\|= cpu_to_le32(MD_FEATURE_NEW_OFFSET);
2278	sb->new_offset = cpu_to_le32((__u32)(rdev->new_data_offset
2279	- rdev->data_offset));
2280	}
2281	}
2282
2283	if (mddev_is_clustered(mddev))
2284	sb->feature_map \|= cpu_to_le32(MD_FEATURE_CLUSTERED);
2285
2286	if (rdev->badblocks.count == 0)
2287	/* Nothing to do for bad blocks*/ ;
2288	else if (sb->bblog_offset == 0)
2289	/* Cannot record bad blocks on this device */
2290	md_error(mddev, rdev);
2291	else {
2292	struct badblocks *bb = &rdev->badblocks;
2293	__le64 bbp = (__le64 )page_address(rdev->bb_page);
2294	u64 *p = bb->page;
2295	sb->feature_map \|= cpu_to_le32(MD_FEATURE_BAD_BLOCKS);
2296	if (bb->changed) {
2297	unsigned seq;
2298
2299	retry:
2300	seq = read_seqbegin(&bb->lock);
2301
2302	memset(bbp, 0xff, PAGE_SIZE);
2303
2304	for (i = 0 ; i < bb->count ; i++) {
2305	u64 internal_bb = p[i];
2306	u64 store_bb = ((BB_OFFSET(internal_bb) << 10)
2307	\| BB_LEN(internal_bb));
2308	bbp[i] = cpu_to_le64(store_bb);
2309	}
2310	bb->changed = 0;
2311	if (read_seqretry(&bb->lock, seq))
2312	goto retry;
2313
2314	bb->sector = (rdev->sb_start +
2315	(int)le32_to_cpu(sb->bblog_offset));
2316	bb->size = le16_to_cpu(sb->bblog_size);
2317	}
2318	}
2319
2320	max_dev = 0;
2321	rdev_for_each(rdev2, mddev)
2322	if (rdev2->desc_nr+1 > max_dev)
2323	max_dev = rdev2->desc_nr+1;
2324
2325	if (max_dev > le32_to_cpu(sb->max_dev)) {
2326	int bmask;
2327	sb->max_dev = cpu_to_le32(max_dev);
2328	rdev->sb_size = max_dev * 2 + 256;
2329	bmask = queue_logical_block_size(rdev->bdev->bd_disk->queue)-1;
2330	if (rdev->sb_size & bmask)
2331	rdev->sb_size = (rdev->sb_size \| bmask) + 1;
2332	} else
2333	max_dev = le32_to_cpu(sb->max_dev);
2334
2335	for (i=0; i<max_dev;i++)
2336	sb->dev_roles[i] = cpu_to_le16(MD_DISK_ROLE_SPARE);
2337
2338	if (test_bit(MD_HAS_JOURNAL, &mddev->flags))
2339	sb->feature_map \|= cpu_to_le32(MD_FEATURE_JOURNAL);
2340
2341	if (test_bit(MD_HAS_PPL, &mddev->flags)) {
2342	if (test_bit(MD_HAS_MULTIPLE_PPLS, &mddev->flags))
2343	sb->feature_map \|=
2344	cpu_to_le32(MD_FEATURE_MULTIPLE_PPLS);
2345	else
2346	sb->feature_map \|= cpu_to_le32(MD_FEATURE_PPL);
2347	sb->ppl.offset = cpu_to_le16(rdev->ppl.offset);
2348	sb->ppl.size = cpu_to_le16(rdev->ppl.size);
2349	}
2350
2351	rdev_for_each(rdev2, mddev) {
2352	i = rdev2->desc_nr;
2353	if (test_bit(Faulty, &rdev2->flags))
2354	sb->dev_roles[i] = cpu_to_le16(MD_DISK_ROLE_FAULTY);
2355	else if (test_bit(In_sync, &rdev2->flags))
2356	sb->dev_roles[i] = cpu_to_le16(rdev2->raid_disk);
2357	else if (test_bit(Journal, &rdev2->flags))
2358	sb->dev_roles[i] = cpu_to_le16(MD_DISK_ROLE_JOURNAL);
2359	else if (rdev2->raid_disk >= 0)
2360	sb->dev_roles[i] = cpu_to_le16(rdev2->raid_disk);
2361	else
2362	sb->dev_roles[i] = cpu_to_le16(MD_DISK_ROLE_SPARE);
2363	}
2364
2365	sb->sb_csum = calc_sb_1_csum(sb);
2366	}
2367
2368	static sector_t super_1_choose_bm_space(sector_t dev_size)
2369	{
2370	sector_t bm_space;
2371
2372	/* if the device is bigger than 8Gig, save 64k for bitmap
2373	* usage, if bigger than 200Gig, save 128k
2374	*/
2375	if (dev_size < 64*2)
2376	bm_space = 0;
2377	else if (dev_size - 642 >= 200102410242)
2378	bm_space = 128*2;
2379	else if (dev_size - 42 > 8102410242)
2380	bm_space = 64*2;
2381	else
2382	bm_space = 4*2;
2383	return bm_space;
2384	}
2385
2386	static unsigned long long
2387	super_1_rdev_size_change(struct md_rdev *rdev, sector_t num_sectors)
2388	{
2389	struct mdp_superblock_1 *sb;
2390	sector_t max_sectors;
2391	if (num_sectors && num_sectors < rdev->mddev->dev_sectors)
2392	return 0; /* component must fit device */
2393	if (rdev->data_offset != rdev->new_data_offset)
2394	return 0; /* too confusing */
2395	if (rdev->sb_start < rdev->data_offset) {
2396	/* minor versions 1 and 2; superblock before data */
2397	max_sectors = bdev_nr_sectors(rdev->bdev) - rdev->data_offset;
2398	if (!num_sectors \|\| num_sectors > max_sectors)
2399	num_sectors = max_sectors;
2400	} else if (rdev->mddev->bitmap_info.offset) {
2401	/* minor version 0 with bitmap we can't move */
2402	return 0;
2403	} else {
2404	/* minor version 0; superblock after data */
2405	sector_t sb_start, bm_space;
2406	sector_t dev_size = bdev_nr_sectors(rdev->bdev);
2407
2408	/* 8K is for superblock */
2409	sb_start = dev_size - 8*2;
2410	sb_start &= ~(sector_t)(4*2 - 1);
2411
2412	bm_space = super_1_choose_bm_space(dev_size);
2413
2414	/* Space that can be used to store date needs to decrease
2415	* superblock bitmap space and bad block space(4K)
2416	*/
2417	max_sectors = sb_start - bm_space - 4*2;
2418
2419	if (!num_sectors \|\| num_sectors > max_sectors)
2420	num_sectors = max_sectors;
2421	rdev->sb_start = sb_start;
2422	}
2423	sb = page_address(rdev->sb_page);
2424	sb->data_size = cpu_to_le64(num_sectors);
2425	sb->super_offset = cpu_to_le64(rdev->sb_start);
2426	sb->sb_csum = calc_sb_1_csum(sb);
2427	do {
2428	md_write_metadata(rdev->mddev, rdev, rdev->sb_start,
2429	rdev->sb_size, rdev->sb_page, 0);
2430	} while (md_super_wait(rdev->mddev) < 0);
2431	return num_sectors;
2432
2433	}
2434
2435	static int
2436	super_1_allow_new_offset(struct md_rdev *rdev,
2437	unsigned long long new_offset)
2438	{
2439	struct mddev *mddev = rdev->mddev;
2440
2441	/* All necessary checks on new >= old have been done */
2442	if (new_offset >= rdev->data_offset)
2443	return 1;
2444
2445	/* with 1.0 metadata, there is no metadata to tread on
2446	* so we can always move back */
2447	if (mddev->minor_version == 0)
2448	return 1;
2449
2450	/* otherwise we must be sure not to step on
2451	* any metadata, so stay:
2452	* 36K beyond start of superblock
2453	* beyond end of badblocks
2454	* beyond write-intent bitmap
2455	*/
2456	if (rdev->sb_start + (32+4)*2 > new_offset)
2457	return 0;
2458
2459	if (md_bitmap_registered(mddev) && !mddev->bitmap_info.file) {
2460	struct md_bitmap_stats stats;
2461	int err;
2462
2463	err = mddev->bitmap_ops->get_stats(mddev->bitmap, &stats);
2464	if (!err && rdev->sb_start + mddev->bitmap_info.offset +
2465	stats.file_pages * (PAGE_SIZE >> 9) > new_offset)
2466	return 0;
2467	}
2468
2469	if (rdev->badblocks.sector + rdev->badblocks.size > new_offset)
2470	return 0;
2471
2472	return 1;
2473	}
2474
2475	static struct super_type super_types[] = {
2476	[0] = {
2477	.name = "0.90.0",
2478	.owner = THIS_MODULE,
2479	.load_super = super_90_load,
2480	.validate_super = super_90_validate,
2481	.sync_super = super_90_sync,
2482	.rdev_size_change = super_90_rdev_size_change,
2483	.allow_new_offset = super_90_allow_new_offset,
2484	},
2485	[1] = {
2486	.name = "md-1",
2487	.owner = THIS_MODULE,
2488	.load_super = super_1_load,
2489	.validate_super = super_1_validate,
2490	.sync_super = super_1_sync,
2491	.rdev_size_change = super_1_rdev_size_change,
2492	.allow_new_offset = super_1_allow_new_offset,
2493	},
2494	};
2495
2496	static void sync_super(struct mddev mddev, struct md_rdev rdev)
2497	{
2498	if (mddev->sync_super) {
2499	mddev->sync_super(mddev, rdev);
2500	return;
2501	}
2502
2503	BUG_ON(mddev->major_version >= ARRAY_SIZE(super_types));
2504
2505	super_types[mddev->major_version].sync_super(mddev, rdev);
2506	}
2507
2508	static int match_mddev_units(struct mddev mddev1, struct mddev mddev2)
2509	{
2510	struct md_rdev rdev, rdev2;
2511
2512	rcu_read_lock();
2513	rdev_for_each_rcu(rdev, mddev1) {
2514	if (test_bit(Faulty, &rdev->flags) \|\|
2515	test_bit(Journal, &rdev->flags) \|\|
2516	rdev->raid_disk == -1)
2517	continue;
2518	rdev_for_each_rcu(rdev2, mddev2) {
2519	if (test_bit(Faulty, &rdev2->flags) \|\|
2520	test_bit(Journal, &rdev2->flags) \|\|
2521	rdev2->raid_disk == -1)
2522	continue;
2523	if (rdev->bdev->bd_disk == rdev2->bdev->bd_disk) {
2524	rcu_read_unlock();
2525	return 1;
2526	}
2527	}
2528	}
2529	rcu_read_unlock();
2530	return 0;
2531	}
2532
2533	static LIST_HEAD(pending_raid_disks);
2534
2535	/*
2536	* Try to register data integrity profile for an mddev
2537	*
2538	* This is called when an array is started and after a disk has been kicked
2539	* from the array. It only succeeds if all working and active component devices
2540	* are integrity capable with matching profiles.
2541	*/
2542	int md_integrity_register(struct mddev *mddev)
2543	{
2544	if (list_empty(&mddev->disks))
2545	return 0; /* nothing to do */
2546	if (mddev_is_dm(mddev) \|\| !blk_get_integrity(mddev->gendisk))
2547	return 0; /* shouldn't register */
2548
2549	pr_debug("md: data integrity enabled on %s\n", mdname(mddev));
2550	return 0;
2551	}
2552	EXPORT_SYMBOL(md_integrity_register);
2553
2554	static bool rdev_read_only(struct md_rdev *rdev)
2555	{
2556	return bdev_read_only(rdev->bdev) \|\|
2557	(rdev->meta_bdev && bdev_read_only(rdev->meta_bdev));
2558	}
2559
2560	static int bind_rdev_to_array(struct md_rdev rdev, struct mddev mddev)
2561	{
2562	char b[BDEVNAME_SIZE];
2563	int err;
2564
2565	/* prevent duplicates */
2566	if (find_rdev(mddev, rdev->bdev->bd_dev))
2567	return -EEXIST;
2568
2569	if (rdev_read_only(rdev) && mddev->pers)
2570	return -EROFS;
2571
2572	/* make sure rdev->sectors exceeds mddev->dev_sectors */
2573	if (!test_bit(Journal, &rdev->flags) &&
2574	rdev->sectors &&
2575	(mddev->dev_sectors == 0 \|\| rdev->sectors < mddev->dev_sectors)) {
2576	if (mddev->pers) {
2577	/* Cannot change size, so fail
2578	* If mddev->level <= 0, then we don't care
2579	* about aligning sizes (e.g. linear)
2580	*/
2581	if (mddev->level > 0)
2582	return -ENOSPC;
2583	} else
2584	mddev->dev_sectors = rdev->sectors;
2585	}
2586
2587	/* Verify rdev->desc_nr is unique.
2588	* If it is -1, assign a free number, else
2589	* check number is not in use
2590	*/
2591	rcu_read_lock();
2592	if (rdev->desc_nr < 0) {
2593	int choice = 0;
2594	if (mddev->pers)
2595	choice = mddev->raid_disks;
2596	while (md_find_rdev_nr_rcu(mddev, choice))
2597	choice++;
2598	rdev->desc_nr = choice;
2599	} else {
2600	if (md_find_rdev_nr_rcu(mddev, rdev->desc_nr)) {
2601	rcu_read_unlock();
2602	return -EBUSY;
2603	}
2604	}
2605	rcu_read_unlock();
2606	if (!test_bit(Journal, &rdev->flags) &&
2607	mddev->max_disks && rdev->desc_nr >= mddev->max_disks) {
2608	pr_warn("md: %s: array is limited to %d devices\n",
2609	mdname(mddev), mddev->max_disks);
2610	return -EBUSY;
2611	}
2612	snprintf(b, sizeof(b), "%pg", rdev->bdev);
2613	strreplace(b, '/', '!');
2614
2615	rdev->mddev = mddev;
2616	pr_debug("md: bind<%s>\n", b);
2617
2618	if (mddev->raid_disks)
2619	mddev_create_serial_pool(mddev, rdev);
2620
2621	if ((err = kobject_add(&rdev->kobj, &mddev->kobj, "dev-%s", b)))
2622	goto fail;
2623
2624	/* failure here is OK */
2625	err = sysfs_create_link(&rdev->kobj, bdev_kobj(rdev->bdev), "block");
2626	rdev->sysfs_state = sysfs_get_dirent_safe(rdev->kobj.sd, "state");
2627	rdev->sysfs_unack_badblocks =
2628	sysfs_get_dirent_safe(rdev->kobj.sd, "unacknowledged_bad_blocks");
2629	rdev->sysfs_badblocks =
2630	sysfs_get_dirent_safe(rdev->kobj.sd, "bad_blocks");
2631
2632	list_add_rcu(&rdev->same_set, &mddev->disks);
2633	bd_link_disk_holder(rdev->bdev, mddev->gendisk);
2634
2635	return 0;
2636
2637	fail:
2638	pr_warn("md: failed to register dev-%s for %s\n",
2639	b, mdname(mddev));
2640	mddev_destroy_serial_pool(mddev, rdev);
2641	return err;
2642	}
2643
2644	void md_autodetect_dev(dev_t dev);
2645
2646	/* just for claiming the bdev */
2647	static struct md_rdev claim_rdev;
2648
2649	static void export_rdev(struct md_rdev *rdev)
2650	{
2651	pr_debug("md: export_rdev(%pg)\n", rdev->bdev);
2652	md_rdev_clear(rdev);
2653	#ifndef MODULE
2654	if (test_bit(AutoDetected, &rdev->flags))
2655	md_autodetect_dev(rdev->bdev->bd_dev);
2656	#endif
2657	fput(rdev->bdev_file);
2658	rdev->bdev = NULL;
2659	kobject_put(&rdev->kobj);
2660	}
2661
2662	static void md_kick_rdev_from_array(struct md_rdev *rdev)
2663	{
2664	struct mddev *mddev = rdev->mddev;
2665
2666	bd_unlink_disk_holder(rdev->bdev, rdev->mddev->gendisk);
2667	list_del_rcu(&rdev->same_set);
2668	pr_debug("md: unbind<%pg>\n", rdev->bdev);
2669	mddev_destroy_serial_pool(rdev->mddev, rdev);
2670	WRITE_ONCE(rdev->mddev, NULL);
2671	sysfs_remove_link(&rdev->kobj, "block");
2672	sysfs_put(rdev->sysfs_state);
2673	sysfs_put(rdev->sysfs_unack_badblocks);
2674	sysfs_put(rdev->sysfs_badblocks);
2675	rdev->sysfs_state = NULL;
2676	rdev->sysfs_unack_badblocks = NULL;
2677	rdev->sysfs_badblocks = NULL;
2678	rdev->badblocks.count = 0;
2679
2680	synchronize_rcu();
2681
2682	/*
2683	* kobject_del() will wait for all in progress writers to be done, where
2684	* reconfig_mutex is held, hence it can't be called under
2685	* reconfig_mutex and it's delayed to mddev_unlock().
2686	*/
2687	list_add(&rdev->same_set, &mddev->deleting);
2688	}
2689
2690	static void export_array(struct mddev *mddev)
2691	{
2692	struct md_rdev *rdev;
2693
2694	while (!list_empty(&mddev->disks)) {
2695	rdev = list_first_entry(&mddev->disks, struct md_rdev,
2696	same_set);
2697	md_kick_rdev_from_array(rdev);
2698	}
2699	mddev->raid_disks = 0;
2700	mddev->major_version = 0;
2701	}
2702
2703	static bool set_in_sync(struct mddev *mddev)
2704	{
2705	lockdep_assert_held(&mddev->lock);
2706	if (!mddev->in_sync) {
2707	mddev->sync_checkers++;
2708	spin_unlock(&mddev->lock);
2709	percpu_ref_switch_to_atomic_sync(&mddev->writes_pending);
2710	spin_lock(&mddev->lock);
2711	if (!mddev->in_sync &&
2712	percpu_ref_is_zero(&mddev->writes_pending)) {
2713	mddev->in_sync = 1;
2714	/*
2715	* Ensure ->in_sync is visible before we clear
2716	* ->sync_checkers.
2717	*/
2718	smp_mb();
2719	set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
2720	sysfs_notify_dirent_safe(mddev->sysfs_state);
2721	}
2722	if (--mddev->sync_checkers == 0)
2723	percpu_ref_switch_to_percpu(&mddev->writes_pending);
2724	}
2725	if (mddev->safemode == 1)
2726	mddev->safemode = 0;
2727	return mddev->in_sync;
2728	}
2729
2730	static void sync_sbs(struct mddev *mddev, int nospares)
2731	{
2732	/* Update each superblock (in-memory image), but
2733	* if we are allowed to, skip spares which already
2734	* have the right event counter, or have one earlier
2735	* (which would mean they aren't being marked as dirty
2736	* with the rest of the array)
2737	*/
2738	struct md_rdev *rdev;
2739	rdev_for_each(rdev, mddev) {
2740	if (rdev->sb_events == mddev->events \|\|
2741	(nospares &&
2742	rdev->raid_disk < 0 &&
2743	rdev->sb_events+1 == mddev->events)) {
2744	/* Don't update this superblock */
2745	rdev->sb_loaded = 2;
2746	} else {
2747	sync_super(mddev, rdev);
2748	rdev->sb_loaded = 1;
2749	}
2750	}
2751	}
2752
2753	static bool does_sb_need_changing(struct mddev *mddev)
2754	{
2755	struct md_rdev rdev = NULL, iter;
2756	struct mdp_superblock_1 *sb;
2757	int role;
2758
2759	/* Find a good rdev */
2760	rdev_for_each(iter, mddev)
2761	if ((iter->raid_disk >= 0) && !test_bit(Faulty, &iter->flags)) {
2762	rdev = iter;
2763	break;
2764	}
2765
2766	/* No good device found. */
2767	if (!rdev)
2768	return false;
2769
2770	sb = page_address(rdev->sb_page);
2771	/* Check if a device has become faulty or a spare become active */
2772	rdev_for_each(rdev, mddev) {
2773	role = le16_to_cpu(sb->dev_roles[rdev->desc_nr]);
2774	/* Device activated? */
2775	if (role == MD_DISK_ROLE_SPARE && rdev->raid_disk >= 0 &&
2776	!test_bit(Faulty, &rdev->flags))
2777	return true;
2778	/* Device turned faulty? */
2779	if (test_bit(Faulty, &rdev->flags) && (role < MD_DISK_ROLE_MAX))
2780	return true;
2781	}
2782
2783	/* Check if any mddev parameters have changed */
2784	if ((mddev->dev_sectors != le64_to_cpu(sb->size)) \|\|
2785	(mddev->reshape_position != le64_to_cpu(sb->reshape_position)) \|\|
2786	(mddev->layout != le32_to_cpu(sb->layout)) \|\|
2787	(mddev->raid_disks != le32_to_cpu(sb->raid_disks)) \|\|
2788	(mddev->chunk_sectors != le32_to_cpu(sb->chunksize)))
2789	return true;
2790
2791	return false;
2792	}
2793
2794	void md_update_sb(struct mddev *mddev, int force_change)
2795	{
2796	struct md_rdev *rdev;
2797	int sync_req;
2798	int nospares = 0;
2799	int any_badblocks_changed = 0;
2800	int ret = -1;
2801
2802	if (!md_is_rdwr(mddev)) {
2803	if (force_change)
2804	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
2805	if (!mddev_is_dm(mddev))
2806	pr_err_ratelimited("%s: can't update sb for read-only array %s\n",
2807	__func__, mdname(mddev));
2808	return;
2809	}
2810
2811	repeat:
2812	if (mddev_is_clustered(mddev)) {
2813	if (test_and_clear_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags))
2814	force_change = 1;
2815	if (test_and_clear_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags))
2816	nospares = 1;
2817	ret = mddev->cluster_ops->metadata_update_start(mddev);
2818	/* Has someone else has updated the sb */
2819	if (!does_sb_need_changing(mddev)) {
2820	if (ret == 0)
2821	mddev->cluster_ops->metadata_update_cancel(mddev);
2822	bit_clear_unless(&mddev->sb_flags, BIT(MD_SB_CHANGE_PENDING),
2823	BIT(MD_SB_CHANGE_DEVS) \|
2824	BIT(MD_SB_CHANGE_CLEAN));
2825	return;
2826	}
2827	}
2828
2829	/*
2830	* First make sure individual recovery_offsets are correct
2831	* curr_resync_completed can only be used during recovery.
2832	* During reshape/resync it might use array-addresses rather
2833	* that device addresses.
2834	*/
2835	rdev_for_each(rdev, mddev) {
2836	if (rdev->raid_disk >= 0 &&
2837	mddev->delta_disks >= 0 &&
2838	test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) &&
2839	test_bit(MD_RECOVERY_RECOVER, &mddev->recovery) &&
2840	!test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
2841	!test_bit(Journal, &rdev->flags) &&
2842	!test_bit(In_sync, &rdev->flags) &&
2843	mddev->curr_resync_completed > rdev->recovery_offset)
2844	rdev->recovery_offset = mddev->curr_resync_completed;
2845
2846	}
2847	if (!mddev->persistent) {
2848	clear_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
2849	clear_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
2850	if (!mddev->external) {
2851	clear_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
2852	rdev_for_each(rdev, mddev) {
2853	if (rdev->badblocks.changed) {
2854	rdev->badblocks.changed = 0;
2855	ack_all_badblocks(&rdev->badblocks);
2856	md_error(mddev, rdev);
2857	}
2858	clear_bit(Blocked, &rdev->flags);
2859	clear_bit(BlockedBadBlocks, &rdev->flags);
2860	wake_up(&rdev->blocked_wait);
2861	}
2862	}
2863	wake_up(&mddev->sb_wait);
2864	return;
2865	}
2866
2867	spin_lock(&mddev->lock);
2868
2869	mddev->utime = ktime_get_real_seconds();
2870
2871	if (test_and_clear_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags))
2872	force_change = 1;
2873	if (test_and_clear_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags))
2874	/* just a clean<-> dirty transition, possibly leave spares alone,
2875	* though if events isn't the right even/odd, we will have to do
2876	* spares after all
2877	*/
2878	nospares = 1;
2879	if (force_change)
2880	nospares = 0;
2881	if (mddev->degraded)
2882	/* If the array is degraded, then skipping spares is both
2883	* dangerous and fairly pointless.
2884	* Dangerous because a device that was removed from the array
2885	* might have a event_count that still looks up-to-date,
2886	* so it can be re-added without a resync.
2887	* Pointless because if there are any spares to skip,
2888	* then a recovery will happen and soon that array won't
2889	* be degraded any more and the spare can go back to sleep then.
2890	*/
2891	nospares = 0;
2892
2893	sync_req = mddev->in_sync;
2894
2895	/* If this is just a dirty<->clean transition, and the array is clean
2896	* and 'events' is odd, we can roll back to the previous clean state */
2897	if (nospares
2898	&& (mddev->in_sync && mddev->resync_offset == MaxSector)
2899	&& mddev->can_decrease_events
2900	&& mddev->events != 1) {
2901	mddev->events--;
2902	mddev->can_decrease_events = 0;
2903	} else {
2904	/* otherwise we have to go forward and ... */
2905	mddev->events ++;
2906	mddev->can_decrease_events = nospares;
2907	}
2908
2909	/*
2910	* This 64-bit counter should never wrap.
2911	* Either we are in around ~1 trillion A.C., assuming
2912	* 1 reboot per second, or we have a bug...
2913	*/
2914	WARN_ON(mddev->events == 0);
2915
2916	rdev_for_each(rdev, mddev) {
2917	if (rdev->badblocks.changed)
2918	any_badblocks_changed++;
2919	if (test_bit(Faulty, &rdev->flags))
2920	set_bit(FaultRecorded, &rdev->flags);
2921	}
2922
2923	sync_sbs(mddev, nospares);
2924	spin_unlock(&mddev->lock);
2925
2926	pr_debug("md: updating %s RAID superblock on device (in sync %d)\n",
2927	mdname(mddev), mddev->in_sync);
2928
2929	mddev_add_trace_msg(mddev, "md md_update_sb");
2930	rewrite:
2931	if (md_bitmap_enabled(mddev, false))
2932	mddev->bitmap_ops->update_sb(mddev->bitmap);
2933	rdev_for_each(rdev, mddev) {
2934	if (rdev->sb_loaded != 1)
2935	continue; /* no noise on spare devices */
2936
2937	if (!test_bit(Faulty, &rdev->flags)) {
2938	md_write_metadata(mddev, rdev, rdev->sb_start,
2939	rdev->sb_size, rdev->sb_page, 0);
2940	pr_debug("md: (write) %pg's sb offset: %llu\n",
2941	rdev->bdev,
2942	(unsigned long long)rdev->sb_start);
2943	rdev->sb_events = mddev->events;
2944	if (rdev->badblocks.size) {
2945	md_write_metadata(mddev, rdev,
2946	rdev->badblocks.sector,
2947	rdev->badblocks.size << 9,
2948	rdev->bb_page, 0);
2949	rdev->badblocks.size = 0;
2950	}
2951
2952	} else
2953	pr_debug("md: %pg (skipping faulty)\n",
2954	rdev->bdev);
2955	}
2956	if (md_super_wait(mddev) < 0)
2957	goto rewrite;
2958	/* if there was a failure, MD_SB_CHANGE_DEVS was set, and we re-write super */
2959
2960	if (mddev_is_clustered(mddev) && ret == 0)
2961	mddev->cluster_ops->metadata_update_finish(mddev);
2962
2963	if (mddev->in_sync != sync_req \|\|
2964	!bit_clear_unless(&mddev->sb_flags, BIT(MD_SB_CHANGE_PENDING),
2965	BIT(MD_SB_CHANGE_DEVS) \| BIT(MD_SB_CHANGE_CLEAN)))
2966	/* have to write it out again */
2967	goto repeat;
2968	wake_up(&mddev->sb_wait);
2969	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
2970	sysfs_notify_dirent_safe(mddev->sysfs_completed);
2971
2972	rdev_for_each(rdev, mddev) {
2973	if (test_and_clear_bit(FaultRecorded, &rdev->flags))
2974	clear_bit(Blocked, &rdev->flags);
2975
2976	if (any_badblocks_changed)
2977	ack_all_badblocks(&rdev->badblocks);
2978	clear_bit(BlockedBadBlocks, &rdev->flags);
2979	wake_up(&rdev->blocked_wait);
2980	}
2981	}
2982	EXPORT_SYMBOL(md_update_sb);
2983
2984	static int add_bound_rdev(struct md_rdev *rdev)
2985	{
2986	struct mddev *mddev = rdev->mddev;
2987	int err = 0;
2988	bool add_journal = test_bit(Journal, &rdev->flags);
2989
2990	if (!mddev->pers->hot_remove_disk \|\| add_journal) {
2991	/* If there is hot_add_disk but no hot_remove_disk
2992	* then added disks for geometry changes,
2993	* and should be added immediately.
2994	*/
2995	super_types[mddev->major_version].
2996	validate_super(mddev, NULL/freshest/, rdev);
2997	err = mddev->pers->hot_add_disk(mddev, rdev);
2998	if (err) {
2999	md_kick_rdev_from_array(rdev);
3000	return err;
3001	}
3002	}
3003	sysfs_notify_dirent_safe(rdev->sysfs_state);
3004
3005	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
3006	if (mddev->degraded)
3007	set_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
3008	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
3009	md_new_event();
3010	return 0;
3011	}
3012
3013	/* words written to sysfs files may, or may not, be \n terminated.
3014	* We want to accept with case. For this we use cmd_match.
3015	*/
3016	static int cmd_match(const char cmd, const char str)
3017	{
3018	/* See if cmd, written into a sysfs file, matches
3019	* str. They must either be the same, or cmd can
3020	* have a trailing newline
3021	*/
3022	while (cmd && str && cmd == str) {
3023	cmd++;
3024	str++;
3025	}
3026	if (*cmd == '\n')
3027	cmd++;
3028	if (str \|\| cmd)
3029	return 0;
3030	return 1;
3031	}
3032
3033	struct rdev_sysfs_entry {
3034	struct attribute attr;
3035	ssize_t (show)(struct md_rdev , char *);
3036	ssize_t (store)(struct md_rdev , const char *, size_t);
3037	};
3038
3039	static ssize_t
3040	state_show(struct md_rdev rdev, char page)
3041	{
3042	char *sep = ",";
3043	size_t len = 0;
3044	unsigned long flags = READ_ONCE(rdev->flags);
3045
3046	if (test_bit(Faulty, &flags) \|\|
3047	(!test_bit(ExternalBbl, &flags) &&
3048	rdev->badblocks.unacked_exist))
3049	len += sprintf(page+len, "faulty%s", sep);
3050	if (test_bit(In_sync, &flags))
3051	len += sprintf(page+len, "in_sync%s", sep);
3052	if (test_bit(Journal, &flags))
3053	len += sprintf(page+len, "journal%s", sep);
3054	if (test_bit(WriteMostly, &flags))
3055	len += sprintf(page+len, "write_mostly%s", sep);
3056	if (test_bit(Blocked, &flags) \|\|
3057	(rdev->badblocks.unacked_exist
3058	&& !test_bit(Faulty, &flags)))
3059	len += sprintf(page+len, "blocked%s", sep);
3060	if (!test_bit(Faulty, &flags) &&
3061	!test_bit(Journal, &flags) &&
3062	!test_bit(In_sync, &flags))
3063	len += sprintf(page+len, "spare%s", sep);
3064	if (test_bit(WriteErrorSeen, &flags))
3065	len += sprintf(page+len, "write_error%s", sep);
3066	if (test_bit(WantReplacement, &flags))
3067	len += sprintf(page+len, "want_replacement%s", sep);
3068	if (test_bit(Replacement, &flags))
3069	len += sprintf(page+len, "replacement%s", sep);
3070	if (test_bit(ExternalBbl, &flags))
3071	len += sprintf(page+len, "external_bbl%s", sep);
3072	if (test_bit(FailFast, &flags))
3073	len += sprintf(page+len, "failfast%s", sep);
3074
3075	if (len)
3076	len -= strlen(sep);
3077
3078	return len+sprintf(page+len, "\n");
3079	}
3080
3081	static ssize_t
3082	state_store(struct md_rdev rdev, const char buf, size_t len)
3083	{
3084	/* can write
3085	* faulty - simulates an error
3086	* remove - disconnects the device
3087	* writemostly - sets write_mostly
3088	* -writemostly - clears write_mostly
3089	* blocked - sets the Blocked flags
3090	* -blocked - clears the Blocked and possibly simulates an error
3091	* insync - sets Insync providing device isn't active
3092	* -insync - clear Insync for a device with a slot assigned,
3093	* so that it gets rebuilt based on bitmap
3094	* write_error - sets WriteErrorSeen
3095	* -write_error - clears WriteErrorSeen
3096	* {,-}failfast - set/clear FailFast
3097	*/
3098
3099	struct mddev *mddev = rdev->mddev;
3100	int err = -EINVAL;
3101	bool need_update_sb = false;
3102
3103	if (cmd_match(buf, "faulty") && rdev->mddev->pers) {
3104	md_error(rdev->mddev, rdev);
3105
3106	if (test_bit(MD_BROKEN, &rdev->mddev->flags))
3107	err = -EBUSY;
3108	else
3109	err = 0;
3110	} else if (cmd_match(buf, "remove")) {
3111	if (rdev->mddev->pers) {
3112	clear_bit(Blocked, &rdev->flags);
3113	remove_and_add_spares(rdev->mddev, rdev);
3114	}
3115	if (rdev->raid_disk >= 0)
3116	err = -EBUSY;
3117	else {
3118	err = 0;
3119	if (mddev_is_clustered(mddev))
3120	err = mddev->cluster_ops->remove_disk(mddev, rdev);
3121
3122	if (err == 0) {
3123	md_kick_rdev_from_array(rdev);
3124	if (mddev->pers)
3125	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
3126	md_new_event();
3127	}
3128	}
3129	} else if (cmd_match(buf, "writemostly")) {
3130	set_bit(WriteMostly, &rdev->flags);
3131	mddev_create_serial_pool(rdev->mddev, rdev);
3132	need_update_sb = true;
3133	err = 0;
3134	} else if (cmd_match(buf, "-writemostly")) {
3135	mddev_destroy_serial_pool(rdev->mddev, rdev);
3136	clear_bit(WriteMostly, &rdev->flags);
3137	need_update_sb = true;
3138	err = 0;
3139	} else if (cmd_match(buf, "blocked")) {
3140	set_bit(Blocked, &rdev->flags);
3141	err = 0;
3142	} else if (cmd_match(buf, "-blocked")) {
3143	if (!test_bit(Faulty, &rdev->flags) &&
3144	!test_bit(ExternalBbl, &rdev->flags) &&
3145	rdev->badblocks.unacked_exist) {
3146	/* metadata handler doesn't understand badblocks,
3147	* so we need to fail the device
3148	*/
3149	md_error(rdev->mddev, rdev);
3150	}
3151	clear_bit(Blocked, &rdev->flags);
3152	clear_bit(BlockedBadBlocks, &rdev->flags);
3153	wake_up(&rdev->blocked_wait);
3154	set_bit(MD_RECOVERY_NEEDED, &rdev->mddev->recovery);
3155
3156	err = 0;
3157	} else if (cmd_match(buf, "insync") && rdev->raid_disk == -1) {
3158	set_bit(In_sync, &rdev->flags);
3159	err = 0;
3160	} else if (cmd_match(buf, "failfast")) {
3161	set_bit(FailFast, &rdev->flags);
3162	need_update_sb = true;
3163	err = 0;
3164	} else if (cmd_match(buf, "-failfast")) {
3165	clear_bit(FailFast, &rdev->flags);
3166	need_update_sb = true;
3167	err = 0;
3168	} else if (cmd_match(buf, "-insync") && rdev->raid_disk >= 0 &&
3169	!test_bit(Journal, &rdev->flags)) {
3170	if (rdev->mddev->pers == NULL) {
3171	clear_bit(In_sync, &rdev->flags);
3172	rdev->saved_raid_disk = rdev->raid_disk;
3173	rdev->raid_disk = -1;
3174	err = 0;
3175	}
3176	} else if (cmd_match(buf, "write_error")) {
3177	set_bit(WriteErrorSeen, &rdev->flags);
3178	err = 0;
3179	} else if (cmd_match(buf, "-write_error")) {
3180	clear_bit(WriteErrorSeen, &rdev->flags);
3181	err = 0;
3182	} else if (cmd_match(buf, "want_replacement")) {
3183	/* Any non-spare device that is not a replacement can
3184	* become want_replacement at any time, but we then need to
3185	* check if recovery is needed.
3186	*/
3187	if (rdev->raid_disk >= 0 &&
3188	!test_bit(Journal, &rdev->flags) &&
3189	!test_bit(Replacement, &rdev->flags))
3190	set_bit(WantReplacement, &rdev->flags);
3191	set_bit(MD_RECOVERY_NEEDED, &rdev->mddev->recovery);
3192	err = 0;
3193	} else if (cmd_match(buf, "-want_replacement")) {
3194	/* Clearing 'want_replacement' is always allowed.
3195	* Once replacements starts it is too late though.
3196	*/
3197	err = 0;
3198	clear_bit(WantReplacement, &rdev->flags);
3199	} else if (cmd_match(buf, "replacement")) {
3200	/* Can only set a device as a replacement when array has not
3201	* yet been started. Once running, replacement is automatic
3202	* from spares, or by assigning 'slot'.
3203	*/
3204	if (rdev->mddev->pers)
3205	err = -EBUSY;
3206	else {
3207	set_bit(Replacement, &rdev->flags);
3208	err = 0;
3209	}
3210	} else if (cmd_match(buf, "-replacement")) {
3211	/* Similarly, can only clear Replacement before start */
3212	if (rdev->mddev->pers)
3213	err = -EBUSY;
3214	else {
3215	clear_bit(Replacement, &rdev->flags);
3216	err = 0;
3217	}
3218	} else if (cmd_match(buf, "re-add")) {
3219	if (!rdev->mddev->pers)
3220	err = -EINVAL;
3221	else if (test_bit(Faulty, &rdev->flags) && (rdev->raid_disk == -1) &&
3222	rdev->saved_raid_disk >= 0) {
3223	/* clear_bit is performed _after_ all the devices
3224	* have their local Faulty bit cleared. If any writes
3225	* happen in the meantime in the local node, they
3226	* will land in the local bitmap, which will be synced
3227	* by this node eventually
3228	*/
3229	if (!mddev_is_clustered(rdev->mddev) \|\|
3230	(err = mddev->cluster_ops->gather_bitmaps(rdev)) == 0) {
3231	clear_bit(Faulty, &rdev->flags);
3232	err = add_bound_rdev(rdev);
3233	}
3234	} else
3235	err = -EBUSY;
3236	} else if (cmd_match(buf, "external_bbl") && (rdev->mddev->external)) {
3237	set_bit(ExternalBbl, &rdev->flags);
3238	rdev->badblocks.shift = 0;
3239	err = 0;
3240	} else if (cmd_match(buf, "-external_bbl") && (rdev->mddev->external)) {
3241	clear_bit(ExternalBbl, &rdev->flags);
3242	err = 0;
3243	}
3244	if (need_update_sb)
3245	md_update_sb(mddev, 1);
3246	if (!err)
3247	sysfs_notify_dirent_safe(rdev->sysfs_state);
3248	return err ? err : len;
3249	}
3250	static struct rdev_sysfs_entry rdev_state =
3251	__ATTR_PREALLOC(state, S_IRUGO\|S_IWUSR, state_show, state_store);
3252
3253	static ssize_t
3254	errors_show(struct md_rdev rdev, char page)
3255	{
3256	return sprintf(page, "%d\n", atomic_read(&rdev->corrected_errors));
3257	}
3258
3259	static ssize_t
3260	errors_store(struct md_rdev rdev, const char buf, size_t len)
3261	{
3262	unsigned int n;
3263	int rv;
3264
3265	rv = kstrtouint(buf, 10, &n);
3266	if (rv < 0)
3267	return rv;
3268	atomic_set(&rdev->corrected_errors, n);
3269	return len;
3270	}
3271	static struct rdev_sysfs_entry rdev_errors =
3272	__ATTR(errors, S_IRUGO\|S_IWUSR, errors_show, errors_store);
3273
3274	static ssize_t
3275	slot_show(struct md_rdev rdev, char page)
3276	{
3277	if (test_bit(Journal, &rdev->flags))
3278	return sprintf(page, "journal\n");
3279	else if (rdev->raid_disk < 0)
3280	return sprintf(page, "none\n");
3281	else
3282	return sprintf(page, "%d\n", rdev->raid_disk);
3283	}
3284
3285	static ssize_t
3286	slot_store(struct md_rdev rdev, const char buf, size_t len)
3287	{
3288	int slot;
3289	int err;
3290
3291	if (test_bit(Journal, &rdev->flags))
3292	return -EBUSY;
3293	if (strncmp(buf, "none", 4)==0)
3294	slot = -1;
3295	else {
3296	err = kstrtouint(buf, 10, (unsigned int *)&slot);
3297	if (err < 0)
3298	return err;
3299	if (slot < 0)
3300	/* overflow */
3301	return -ENOSPC;
3302	}
3303	if (rdev->mddev->pers && slot == -1) {
3304	/* Setting 'slot' on an active array requires also
3305	* updating the 'rd%d' link, and communicating
3306	* with the personality with ->hot_*_disk.
3307	* For now we only support removing
3308	* failed/spare devices. This normally happens automatically,
3309	* but not when the metadata is externally managed.
3310	*/
3311	if (rdev->raid_disk == -1)
3312	return -EEXIST;
3313	/* personality does all needed checks */
3314	if (rdev->mddev->pers->hot_remove_disk == NULL)
3315	return -EINVAL;
3316	clear_bit(Blocked, &rdev->flags);
3317	remove_and_add_spares(rdev->mddev, rdev);
3318	if (rdev->raid_disk >= 0)
3319	return -EBUSY;
3320	set_bit(MD_RECOVERY_NEEDED, &rdev->mddev->recovery);
3321	} else if (rdev->mddev->pers) {
3322	/* Activating a spare .. or possibly reactivating
3323	* if we ever get bitmaps working here.
3324	*/
3325	int err;
3326
3327	if (rdev->raid_disk != -1)
3328	return -EBUSY;
3329
3330	if (test_bit(MD_RECOVERY_RUNNING, &rdev->mddev->recovery))
3331	return -EBUSY;
3332
3333	if (rdev->mddev->pers->hot_add_disk == NULL)
3334	return -EINVAL;
3335
3336	if (slot >= rdev->mddev->raid_disks &&
3337	slot >= rdev->mddev->raid_disks + rdev->mddev->delta_disks)
3338	return -ENOSPC;
3339
3340	rdev->raid_disk = slot;
3341	if (test_bit(In_sync, &rdev->flags))
3342	rdev->saved_raid_disk = slot;
3343	else
3344	rdev->saved_raid_disk = -1;
3345	clear_bit(In_sync, &rdev->flags);
3346	clear_bit(Bitmap_sync, &rdev->flags);
3347	err = rdev->mddev->pers->hot_add_disk(rdev->mddev, rdev);
3348	if (err) {
3349	rdev->raid_disk = -1;
3350	return err;
3351	} else
3352	sysfs_notify_dirent_safe(rdev->sysfs_state);
3353	/* failure here is OK */;
3354	sysfs_link_rdev(rdev->mddev, rdev);
3355	/* don't wakeup anyone, leave that to userspace. */
3356	} else {
3357	if (slot >= rdev->mddev->raid_disks &&
3358	slot >= rdev->mddev->raid_disks + rdev->mddev->delta_disks)
3359	return -ENOSPC;
3360	rdev->raid_disk = slot;
3361	/* assume it is working */
3362	clear_bit(Faulty, &rdev->flags);
3363	clear_bit(WriteMostly, &rdev->flags);
3364	set_bit(In_sync, &rdev->flags);
3365	sysfs_notify_dirent_safe(rdev->sysfs_state);
3366	}
3367	return len;
3368	}
3369
3370	static struct rdev_sysfs_entry rdev_slot =
3371	__ATTR(slot, S_IRUGO\|S_IWUSR, slot_show, slot_store);
3372
3373	static ssize_t
3374	offset_show(struct md_rdev rdev, char page)
3375	{
3376	return sprintf(page, "%llu\n", (unsigned long long)rdev->data_offset);
3377	}
3378
3379	static ssize_t
3380	offset_store(struct md_rdev rdev, const char buf, size_t len)
3381	{
3382	unsigned long long offset;
3383	if (kstrtoull(buf, 10, &offset) < 0)
3384	return -EINVAL;
3385	if (rdev->mddev->pers && rdev->raid_disk >= 0)
3386	return -EBUSY;
3387	if (rdev->sectors && rdev->mddev->external)
3388	/* Must set offset before size, so overlap checks
3389	* can be sane */
3390	return -EBUSY;
3391	rdev->data_offset = offset;
3392	rdev->new_data_offset = offset;
3393	return len;
3394	}
3395
3396	static struct rdev_sysfs_entry rdev_offset =
3397	__ATTR(offset, S_IRUGO\|S_IWUSR, offset_show, offset_store);
3398
3399	static ssize_t new_offset_show(struct md_rdev rdev, char page)
3400	{
3401	return sprintf(page, "%llu\n",
3402	(unsigned long long)rdev->new_data_offset);
3403	}
3404
3405	static ssize_t new_offset_store(struct md_rdev *rdev,
3406	const char *buf, size_t len)
3407	{
3408	unsigned long long new_offset;
3409	struct mddev *mddev = rdev->mddev;
3410
3411	if (kstrtoull(buf, 10, &new_offset) < 0)
3412	return -EINVAL;
3413
3414	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
3415	return -EBUSY;
3416	if (new_offset == rdev->data_offset)
3417	/* reset is always permitted */
3418	;
3419	else if (new_offset > rdev->data_offset) {
3420	/* must not push array size beyond rdev_sectors */
3421	if (new_offset - rdev->data_offset
3422	+ mddev->dev_sectors > rdev->sectors)
3423	return -E2BIG;
3424	}
3425	/* Metadata worries about other space details. */
3426
3427	/* decreasing the offset is inconsistent with a backwards
3428	* reshape.
3429	*/
3430	if (new_offset < rdev->data_offset &&
3431	mddev->reshape_backwards)
3432	return -EINVAL;
3433	/* Increasing offset is inconsistent with forwards
3434	* reshape. reshape_direction should be set to
3435	* 'backwards' first.
3436	*/
3437	if (new_offset > rdev->data_offset &&
3438	!mddev->reshape_backwards)
3439	return -EINVAL;
3440
3441	if (mddev->pers && mddev->persistent &&
3442	!super_types[mddev->major_version]
3443	.allow_new_offset(rdev, new_offset))
3444	return -E2BIG;
3445	rdev->new_data_offset = new_offset;
3446	if (new_offset > rdev->data_offset)
3447	mddev->reshape_backwards = 1;
3448	else if (new_offset < rdev->data_offset)
3449	mddev->reshape_backwards = 0;
3450
3451	return len;
3452	}
3453	static struct rdev_sysfs_entry rdev_new_offset =
3454	__ATTR(new_offset, S_IRUGO\|S_IWUSR, new_offset_show, new_offset_store);
3455
3456	static ssize_t
3457	rdev_size_show(struct md_rdev rdev, char page)
3458	{
3459	return sprintf(page, "%llu\n", (unsigned long long)rdev->sectors / 2);
3460	}
3461
3462	static int md_rdevs_overlap(struct md_rdev a, struct md_rdev b)
3463	{
3464	/* check if two start/length pairs overlap */
3465	if (a->data_offset + a->sectors <= b->data_offset)
3466	return false;
3467	if (b->data_offset + b->sectors <= a->data_offset)
3468	return false;
3469	return true;
3470	}
3471
3472	static bool md_rdev_overlaps(struct md_rdev *rdev)
3473	{
3474	struct mddev *mddev;
3475	struct md_rdev *rdev2;
3476
3477	spin_lock(&all_mddevs_lock);
3478	list_for_each_entry(mddev, &all_mddevs, all_mddevs) {
3479	if (test_bit(MD_DELETED, &mddev->flags))
3480	continue;
3481	rdev_for_each(rdev2, mddev) {
3482	if (rdev != rdev2 && rdev->bdev == rdev2->bdev &&
3483	md_rdevs_overlap(rdev, rdev2)) {
3484	spin_unlock(&all_mddevs_lock);
3485	return true;
3486	}
3487	}
3488	}
3489	spin_unlock(&all_mddevs_lock);
3490	return false;
3491	}
3492
3493	static int strict_blocks_to_sectors(const char buf, sector_t sectors)
3494	{
3495	unsigned long long blocks;
3496	sector_t new;
3497
3498	if (kstrtoull(buf, 10, &blocks) < 0)
3499	return -EINVAL;
3500
3501	if (blocks & 1ULL << (8 * sizeof(blocks) - 1))
3502	return -EINVAL; /* sector conversion overflow */
3503
3504	new = blocks * 2;
3505	if (new != blocks * 2)
3506	return -EINVAL; /* unsigned long long to sector_t overflow */
3507
3508	*sectors = new;
3509	return 0;
3510	}
3511
3512	static ssize_t
3513	rdev_size_store(struct md_rdev rdev, const char buf, size_t len)
3514	{
3515	struct mddev *my_mddev = rdev->mddev;
3516	sector_t oldsectors = rdev->sectors;
3517	sector_t sectors;
3518
3519	if (test_bit(Journal, &rdev->flags))
3520	return -EBUSY;
3521	if (strict_blocks_to_sectors(buf, &sectors) < 0)
3522	return -EINVAL;
3523	if (rdev->data_offset != rdev->new_data_offset)
3524	return -EINVAL; /* too confusing */
3525	if (my_mddev->pers && rdev->raid_disk >= 0) {
3526	if (my_mddev->persistent) {
3527	sectors = super_types[my_mddev->major_version].
3528	rdev_size_change(rdev, sectors);
3529	if (!sectors)
3530	return -EBUSY;
3531	} else if (!sectors)
3532	sectors = bdev_nr_sectors(rdev->bdev) -
3533	rdev->data_offset;
3534	if (!my_mddev->pers->resize)
3535	/* Cannot change size for RAID0 or Linear etc */
3536	return -EINVAL;
3537	}
3538	if (sectors < my_mddev->dev_sectors)
3539	return -EINVAL; /* component must fit device */
3540
3541	rdev->sectors = sectors;
3542
3543	/*
3544	* Check that all other rdevs with the same bdev do not overlap. This
3545	* check does not provide a hard guarantee, it just helps avoid
3546	* dangerous mistakes.
3547	*/
3548	if (sectors > oldsectors && my_mddev->external &&
3549	md_rdev_overlaps(rdev)) {
3550	/*
3551	* Someone else could have slipped in a size change here, but
3552	* doing so is just silly. We put oldsectors back because we
3553	* know it is safe, and trust userspace not to race with itself.
3554	*/
3555	rdev->sectors = oldsectors;
3556	return -EBUSY;
3557	}
3558	return len;
3559	}
3560
3561	static struct rdev_sysfs_entry rdev_size =
3562	__ATTR(size, S_IRUGO\|S_IWUSR, rdev_size_show, rdev_size_store);
3563
3564	static ssize_t recovery_start_show(struct md_rdev rdev, char page)
3565	{
3566	unsigned long long recovery_start = rdev->recovery_offset;
3567
3568	if (test_bit(In_sync, &rdev->flags) \|\|
3569	recovery_start == MaxSector)
3570	return sprintf(page, "none\n");
3571
3572	return sprintf(page, "%llu\n", recovery_start);
3573	}
3574
3575	static ssize_t recovery_start_store(struct md_rdev rdev, const char buf, size_t len)
3576	{
3577	unsigned long long recovery_start;
3578
3579	if (cmd_match(buf, "none"))
3580	recovery_start = MaxSector;
3581	else if (kstrtoull(buf, 10, &recovery_start))
3582	return -EINVAL;
3583
3584	if (rdev->mddev->pers &&
3585	rdev->raid_disk >= 0)
3586	return -EBUSY;
3587
3588	rdev->recovery_offset = recovery_start;
3589	if (recovery_start == MaxSector)
3590	set_bit(In_sync, &rdev->flags);
3591	else
3592	clear_bit(In_sync, &rdev->flags);
3593	return len;
3594	}
3595
3596	static struct rdev_sysfs_entry rdev_recovery_start =
3597	__ATTR(recovery_start, S_IRUGO\|S_IWUSR, recovery_start_show, recovery_start_store);
3598
3599	/* sysfs access to bad-blocks list.
3600	* We present two files.
3601	* 'bad-blocks' lists sector numbers and lengths of ranges that
3602	* are recorded as bad. The list is truncated to fit within
3603	* the one-page limit of sysfs.
3604	* Writing "sector length" to this file adds an acknowledged
3605	* bad block list.
3606	* 'unacknowledged-bad-blocks' lists bad blocks that have not yet
3607	* been acknowledged. Writing to this file adds bad blocks
3608	* without acknowledging them. This is largely for testing.
3609	*/
3610	static ssize_t bb_show(struct md_rdev rdev, char page)
3611	{
3612	return badblocks_show(&rdev->badblocks, page, 0);
3613	}
3614	static ssize_t bb_store(struct md_rdev rdev, const char page, size_t len)
3615	{
3616	int rv = badblocks_store(&rdev->badblocks, page, len, 0);
3617	/* Maybe that ack was all we needed */
3618	if (test_and_clear_bit(BlockedBadBlocks, &rdev->flags))
3619	wake_up(&rdev->blocked_wait);
3620	return rv;
3621	}
3622	static struct rdev_sysfs_entry rdev_bad_blocks =
3623	__ATTR(bad_blocks, S_IRUGO\|S_IWUSR, bb_show, bb_store);
3624
3625	static ssize_t ubb_show(struct md_rdev rdev, char page)
3626	{
3627	return badblocks_show(&rdev->badblocks, page, 1);
3628	}
3629	static ssize_t ubb_store(struct md_rdev rdev, const char page, size_t len)
3630	{
3631	return badblocks_store(&rdev->badblocks, page, len, 1);
3632	}
3633	static struct rdev_sysfs_entry rdev_unack_bad_blocks =
3634	__ATTR(unacknowledged_bad_blocks, S_IRUGO\|S_IWUSR, ubb_show, ubb_store);
3635
3636	static ssize_t
3637	ppl_sector_show(struct md_rdev rdev, char page)
3638	{
3639	return sprintf(page, "%llu\n", (unsigned long long)rdev->ppl.sector);
3640	}
3641
3642	static ssize_t
3643	ppl_sector_store(struct md_rdev rdev, const char buf, size_t len)
3644	{
3645	unsigned long long sector;
3646
3647	if (kstrtoull(buf, 10, &sector) < 0)
3648	return -EINVAL;
3649	if (sector != (sector_t)sector)
3650	return -EINVAL;
3651
3652	if (rdev->mddev->pers && test_bit(MD_HAS_PPL, &rdev->mddev->flags) &&
3653	rdev->raid_disk >= 0)
3654	return -EBUSY;
3655
3656	if (rdev->mddev->persistent) {
3657	if (rdev->mddev->major_version == 0)
3658	return -EINVAL;
3659	if ((sector > rdev->sb_start &&
3660	sector - rdev->sb_start > S16_MAX) \|\|
3661	(sector < rdev->sb_start &&
3662	rdev->sb_start - sector > -S16_MIN))
3663	return -EINVAL;
3664	rdev->ppl.offset = sector - rdev->sb_start;
3665	} else if (!rdev->mddev->external) {
3666	return -EBUSY;
3667	}
3668	rdev->ppl.sector = sector;
3669	return len;
3670	}
3671
3672	static struct rdev_sysfs_entry rdev_ppl_sector =
3673	__ATTR(ppl_sector, S_IRUGO\|S_IWUSR, ppl_sector_show, ppl_sector_store);
3674
3675	static ssize_t
3676	ppl_size_show(struct md_rdev rdev, char page)
3677	{
3678	return sprintf(page, "%u\n", rdev->ppl.size);
3679	}
3680
3681	static ssize_t
3682	ppl_size_store(struct md_rdev rdev, const char buf, size_t len)
3683	{
3684	unsigned int size;
3685
3686	if (kstrtouint(buf, 10, &size) < 0)
3687	return -EINVAL;
3688
3689	if (rdev->mddev->pers && test_bit(MD_HAS_PPL, &rdev->mddev->flags) &&
3690	rdev->raid_disk >= 0)
3691	return -EBUSY;
3692
3693	if (rdev->mddev->persistent) {
3694	if (rdev->mddev->major_version == 0)
3695	return -EINVAL;
3696	if (size > U16_MAX)
3697	return -EINVAL;
3698	} else if (!rdev->mddev->external) {
3699	return -EBUSY;
3700	}
3701	rdev->ppl.size = size;
3702	return len;
3703	}
3704
3705	static struct rdev_sysfs_entry rdev_ppl_size =
3706	__ATTR(ppl_size, S_IRUGO\|S_IWUSR, ppl_size_show, ppl_size_store);
3707
3708	static struct attribute *rdev_default_attrs[] = {
3709	&rdev_state.attr,
3710	&rdev_errors.attr,
3711	&rdev_slot.attr,
3712	&rdev_offset.attr,
3713	&rdev_new_offset.attr,
3714	&rdev_size.attr,
3715	&rdev_recovery_start.attr,
3716	&rdev_bad_blocks.attr,
3717	&rdev_unack_bad_blocks.attr,
3718	&rdev_ppl_sector.attr,
3719	&rdev_ppl_size.attr,
3720	NULL,
3721	};
3722	ATTRIBUTE_GROUPS(rdev_default);
3723	static ssize_t
3724	rdev_attr_show(struct kobject kobj, struct attribute attr, char *page)
3725	{
3726	struct rdev_sysfs_entry *entry = container_of(attr, struct rdev_sysfs_entry, attr);
3727	struct md_rdev *rdev = container_of(kobj, struct md_rdev, kobj);
3728
3729	if (!entry->show)
3730	return -EIO;
3731	if (!rdev->mddev)
3732	return -ENODEV;
3733	return entry->show(rdev, page);
3734	}
3735
3736	static ssize_t
3737	rdev_attr_store(struct kobject kobj, struct attribute attr,
3738	const char *page, size_t length)
3739	{
3740	struct rdev_sysfs_entry *entry = container_of(attr, struct rdev_sysfs_entry, attr);
3741	struct md_rdev *rdev = container_of(kobj, struct md_rdev, kobj);
3742	struct kernfs_node *kn = NULL;
3743	bool suspend = false;
3744	ssize_t rv;
3745	struct mddev *mddev = READ_ONCE(rdev->mddev);
3746
3747	if (!entry->store)
3748	return -EIO;
3749	if (!capable(CAP_SYS_ADMIN))
3750	return -EACCES;
3751	if (!mddev)
3752	return -ENODEV;
3753
3754	if (entry->store == state_store) {
3755	if (cmd_match(page, "remove"))
3756	kn = sysfs_break_active_protection(kobj, attr);
3757	if (cmd_match(page, "remove") \|\| cmd_match(page, "re-add") \|\|
3758	cmd_match(page, "writemostly") \|\|
3759	cmd_match(page, "-writemostly"))
3760	suspend = true;
3761	}
3762
3763	rv = suspend ? mddev_suspend_and_lock(mddev) : mddev_lock(mddev);
3764	if (!rv) {
3765	if (rdev->mddev == NULL)
3766	rv = -ENODEV;
3767	else
3768	rv = entry->store(rdev, page, length);
3769	suspend ? mddev_unlock_and_resume(mddev) : mddev_unlock(mddev);
3770	}
3771
3772	if (kn)
3773	sysfs_unbreak_active_protection(kn);
3774
3775	return rv;
3776	}
3777
3778	static void rdev_free(struct kobject *ko)
3779	{
3780	struct md_rdev *rdev = container_of(ko, struct md_rdev, kobj);
3781	kfree(rdev);
3782	}
3783	static const struct sysfs_ops rdev_sysfs_ops = {
3784	.show = rdev_attr_show,
3785	.store = rdev_attr_store,
3786	};
3787	static const struct kobj_type rdev_ktype = {
3788	.release = rdev_free,
3789	.sysfs_ops = &rdev_sysfs_ops,
3790	.default_groups = rdev_default_groups,
3791	};
3792
3793	int md_rdev_init(struct md_rdev *rdev)
3794	{
3795	rdev->desc_nr = -1;
3796	rdev->saved_raid_disk = -1;
3797	rdev->raid_disk = -1;
3798	rdev->flags = 0;
3799	rdev->data_offset = 0;
3800	rdev->new_data_offset = 0;
3801	rdev->sb_events = 0;
3802	rdev->last_read_error = 0;
3803	rdev->sb_loaded = 0;
3804	rdev->bb_page = NULL;
3805	atomic_set(&rdev->nr_pending, 0);
3806	atomic_set(&rdev->read_errors, 0);
3807	atomic_set(&rdev->corrected_errors, 0);
3808
3809	INIT_LIST_HEAD(&rdev->same_set);
3810	init_waitqueue_head(&rdev->blocked_wait);
3811
3812	/* Add space to store bad block list.
3813	* This reserves the space even on arrays where it cannot
3814	* be used - I wonder if that matters
3815	*/
3816	return badblocks_init(&rdev->badblocks, 0);
3817	}
3818	EXPORT_SYMBOL_GPL(md_rdev_init);
3819
3820	/*
3821	* Import a device. If 'super_format' >= 0, then sanity check the superblock
3822	*
3823	* mark the device faulty if:
3824	*
3825	* - the device is nonexistent (zero size)
3826	* - the device has no valid superblock
3827	*
3828	* a faulty rdev _never_ has rdev->sb set.
3829	*/
3830	static struct md_rdev *md_import_device(dev_t newdev, int super_format, int super_minor)
3831	{
3832	struct md_rdev *rdev;
3833	sector_t size;
3834	int err;
3835
3836	rdev = kzalloc_obj(*rdev);
3837	if (!rdev)
3838	return ERR_PTR(-ENOMEM);
3839
3840	err = md_rdev_init(rdev);
3841	if (err)
3842	goto out_free_rdev;
3843	err = alloc_disk_sb(rdev);
3844	if (err)
3845	goto out_clear_rdev;
3846
3847	rdev->bdev_file = bdev_file_open_by_dev(newdev,
3848	BLK_OPEN_READ \| BLK_OPEN_WRITE,
3849	super_format == -2 ? &claim_rdev : rdev, NULL);
3850	if (IS_ERR(rdev->bdev_file)) {
3851	pr_warn("md: could not open device unknown-block(%u,%u).\n",
3852	MAJOR(newdev), MINOR(newdev));
3853	err = PTR_ERR(rdev->bdev_file);
3854	goto out_clear_rdev;
3855	}
3856	rdev->bdev = file_bdev(rdev->bdev_file);
3857
3858	kobject_init(&rdev->kobj, &rdev_ktype);
3859
3860	size = bdev_nr_bytes(rdev->bdev) >> BLOCK_SIZE_BITS;
3861	if (!size) {
3862	pr_warn("md: %pg has zero or unknown size, marking faulty!\n",
3863	rdev->bdev);
3864	err = -EINVAL;
3865	goto out_blkdev_put;
3866	}
3867
3868	if (super_format >= 0) {
3869	err = super_types[super_format].
3870	load_super(rdev, NULL, super_minor);
3871	if (err == -EINVAL) {
3872	pr_warn("md: %pg does not have a valid v%d.%d superblock, not importing!\n",
3873	rdev->bdev,
3874	super_format, super_minor);
3875	goto out_blkdev_put;
3876	}
3877	if (err < 0) {
3878	pr_warn("md: could not read %pg's sb, not importing!\n",
3879	rdev->bdev);
3880	goto out_blkdev_put;
3881	}
3882	}
3883
3884	return rdev;
3885
3886	out_blkdev_put:
3887	fput(rdev->bdev_file);
3888	out_clear_rdev:
3889	md_rdev_clear(rdev);
3890	out_free_rdev:
3891	kfree(rdev);
3892	return ERR_PTR(err);
3893	}
3894
3895	/*
3896	* Check a full RAID array for plausibility
3897	*/
3898
3899	static int analyze_sbs(struct mddev *mddev)
3900	{
3901	struct md_rdev rdev, freshest, *tmp;
3902
3903	freshest = NULL;
3904	rdev_for_each_safe(rdev, tmp, mddev)
3905	switch (super_types[mddev->major_version].
3906	load_super(rdev, freshest, mddev->minor_version)) {
3907	case 1:
3908	freshest = rdev;
3909	break;
3910	case 0:
3911	break;
3912	default:
3913	pr_warn("md: fatal superblock inconsistency in %pg -- removing from array\n",
3914	rdev->bdev);
3915	md_kick_rdev_from_array(rdev);
3916	}
3917
3918	/* Cannot find a valid fresh disk */
3919	if (!freshest) {
3920	pr_warn("md: cannot find a valid disk\n");
3921	return -EINVAL;
3922	}
3923
3924	super_types[mddev->major_version].
3925	validate_super(mddev, NULL/freshest/, freshest);
3926
3927	rdev_for_each_safe(rdev, tmp, mddev) {
3928	if (mddev->max_disks &&
3929	rdev->desc_nr >= mddev->max_disks) {
3930	pr_warn("md: %s: %pg: only %d devices permitted\n",
3931	mdname(mddev), rdev->bdev,
3932	mddev->max_disks);
3933	md_kick_rdev_from_array(rdev);
3934	continue;
3935	}
3936	if (rdev != freshest) {
3937	if (super_types[mddev->major_version].
3938	validate_super(mddev, freshest, rdev)) {
3939	pr_warn("md: kicking non-fresh %pg from array!\n",
3940	rdev->bdev);
3941	md_kick_rdev_from_array(rdev);
3942	continue;
3943	}
3944	}
3945	if (rdev->raid_disk >= (mddev->raid_disks - min(0, mddev->delta_disks)) &&
3946	!test_bit(Journal, &rdev->flags)) {
3947	rdev->raid_disk = -1;
3948	clear_bit(In_sync, &rdev->flags);
3949	}
3950	}
3951
3952	return 0;
3953	}
3954
3955	/* Read a fixed-point number.
3956	* Numbers in sysfs attributes should be in "standard" units where
3957	* possible, so time should be in seconds.
3958	* However we internally use a a much smaller unit such as
3959	* milliseconds or jiffies.
3960	* This function takes a decimal number with a possible fractional
3961	* component, and produces an integer which is the result of
3962	* multiplying that number by 10^'scale'.
3963	* all without any floating-point arithmetic.
3964	*/
3965	int strict_strtoul_scaled(const char cp, unsigned long res, int scale)
3966	{
3967	unsigned long result = 0;
3968	long decimals = -1;
3969	while (isdigit(cp) \|\| (cp == '.' && decimals < 0)) {
3970	if (*cp == '.')
3971	decimals = 0;
3972	else if (decimals < scale) {
3973	unsigned int value;
3974	value = *cp - '0';
3975	result = result * 10 + value;
3976	if (decimals >= 0)
3977	decimals++;
3978	}
3979	cp++;
3980	}
3981	if (*cp == '\n')
3982	cp++;
3983	if (*cp)
3984	return -EINVAL;
3985	if (decimals < 0)
3986	decimals = 0;
3987	res = result int_pow(10, scale - decimals);
3988	return 0;
3989	}
3990
3991	static ssize_t
3992	safe_delay_show(struct mddev mddev, char page)
3993	{
3994	unsigned int msec = ((unsigned long)mddev->safemode_delay*1000)/HZ;
3995
3996	return sprintf(page, "%u.%03u\n", msec/1000, msec%1000);
3997	}
3998	static ssize_t
3999	safe_delay_store(struct mddev mddev, const char cbuf, size_t len)
4000	{
4001	unsigned long msec;
4002
4003	if (mddev_is_clustered(mddev)) {
4004	pr_warn("md: Safemode is disabled for clustered mode\n");
4005	return -EINVAL;
4006	}
4007
4008	if (strict_strtoul_scaled(cbuf, &msec, 3) < 0 \|\| msec > UINT_MAX / HZ)
4009	return -EINVAL;
4010	if (msec == 0)
4011	mddev->safemode_delay = 0;
4012	else {
4013	unsigned long old_delay = mddev->safemode_delay;
4014	unsigned long new_delay = (msec*HZ)/1000;
4015
4016	if (new_delay == 0)
4017	new_delay = 1;
4018	mddev->safemode_delay = new_delay;
4019	if (new_delay < old_delay \|\| old_delay == 0)
4020	mod_timer(&mddev->safemode_timer, jiffies+1);
4021	}
4022	return len;
4023	}
4024	static struct md_sysfs_entry md_safe_delay =
4025	__ATTR(safe_mode_delay, S_IRUGO\|S_IWUSR,safe_delay_show, safe_delay_store);
4026
4027	static ssize_t
4028	level_show(struct mddev mddev, char page)
4029	{
4030	struct md_personality *p;
4031	int ret;
4032	spin_lock(&mddev->lock);
4033	p = mddev->pers;
4034	if (p)
4035	ret = sprintf(page, "%s\n", p->head.name);
4036	else if (mddev->clevel[0])
4037	ret = sprintf(page, "%s\n", mddev->clevel);
4038	else if (mddev->level != LEVEL_NONE)
4039	ret = sprintf(page, "%d\n", mddev->level);
4040	else
4041	ret = 0;
4042	spin_unlock(&mddev->lock);
4043	return ret;
4044	}
4045
4046	static ssize_t
4047	level_store(struct mddev mddev, const char buf, size_t len)
4048	{
4049	char clevel[16];
4050	ssize_t rv;
4051	size_t slen = len;
4052	unsigned int noio_flags;
4053	struct md_personality pers, oldpers;
4054	long level;
4055	void priv, oldpriv;
4056	struct md_rdev *rdev;
4057
4058	if (slen == 0 \|\| slen >= sizeof(clevel))
4059	return -EINVAL;
4060
4061	rv = mddev_suspend_and_lock(mddev);
4062	if (rv)
4063	return rv;
4064	noio_flags = memalloc_noio_save();
4065
4066	if (mddev->pers == NULL) {
4067	memcpy(mddev->clevel, buf, slen);
4068	if (mddev->clevel[slen-1] == '\n')
4069	slen--;
4070	mddev->clevel[slen] = 0;
4071	mddev->level = LEVEL_NONE;
4072	rv = len;
4073	goto out_unlock;
4074	}
4075	rv = -EROFS;
4076	if (!md_is_rdwr(mddev))
4077	goto out_unlock;
4078
4079	/* request to change the personality. Need to ensure:
4080	* - array is not engaged in resync/recovery/reshape
4081	* - old personality can be suspended
4082	* - new personality will access other array.
4083	*/
4084
4085	rv = -EBUSY;
4086	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) \|\|
4087	mddev->reshape_position != MaxSector \|\|
4088	mddev->sysfs_active)
4089	goto out_unlock;
4090
4091	rv = -EINVAL;
4092	if (!mddev->pers->quiesce) {
4093	pr_warn("md: %s: %s does not support online personality change\n",
4094	mdname(mddev), mddev->pers->head.name);
4095	goto out_unlock;
4096	}
4097
4098	/* Now find the new personality */
4099	memcpy(clevel, buf, slen);
4100	if (clevel[slen-1] == '\n')
4101	slen--;
4102	clevel[slen] = 0;
4103	if (kstrtol(clevel, 10, &level))
4104	level = LEVEL_NONE;
4105
4106	if (request_module("md-%s", clevel) != 0)
4107	request_module("md-level-%s", clevel);
4108	pers = get_pers(level, clevel);
4109	if (!pers) {
4110	rv = -EINVAL;
4111	goto out_unlock;
4112	}
4113
4114	if (pers == mddev->pers) {
4115	/* Nothing to do! */
4116	put_pers(pers);
4117	rv = len;
4118	goto out_unlock;
4119	}
4120	if (!pers->takeover) {
4121	put_pers(pers);
4122	pr_warn("md: %s: %s does not support personality takeover\n",
4123	mdname(mddev), clevel);
4124	rv = -EINVAL;
4125	goto out_unlock;
4126	}
4127
4128	rdev_for_each(rdev, mddev)
4129	rdev->new_raid_disk = rdev->raid_disk;
4130
4131	/* ->takeover must set new_* and/or delta_disks
4132	* if it succeeds, and may set them when it fails.
4133	*/
4134	priv = pers->takeover(mddev);
4135	if (IS_ERR(priv)) {
4136	mddev->new_level = mddev->level;
4137	mddev->new_layout = mddev->layout;
4138	mddev->new_chunk_sectors = mddev->chunk_sectors;
4139	mddev->raid_disks -= mddev->delta_disks;
4140	mddev->delta_disks = 0;
4141	mddev->reshape_backwards = 0;
4142	put_pers(pers);
4143	pr_warn("md: %s: %s would not accept array\n",
4144	mdname(mddev), clevel);
4145	rv = PTR_ERR(priv);
4146	goto out_unlock;
4147	}
4148
4149	/* Looks like we have a winner */
4150	mddev_detach(mddev);
4151
4152	spin_lock(&mddev->lock);
4153	oldpers = mddev->pers;
4154	oldpriv = mddev->private;
4155	mddev->pers = pers;
4156	mddev->private = priv;
4157	strscpy(mddev->clevel, pers->head.name, sizeof(mddev->clevel));
4158	mddev->level = mddev->new_level;
4159	mddev->layout = mddev->new_layout;
4160	mddev->chunk_sectors = mddev->new_chunk_sectors;
4161	mddev->delta_disks = 0;
4162	mddev->reshape_backwards = 0;
4163	mddev->degraded = 0;
4164	spin_unlock(&mddev->lock);
4165
4166	if (oldpers->sync_request == NULL &&
4167	mddev->external) {
4168	/* We are converting from a no-redundancy array
4169	* to a redundancy array and metadata is managed
4170	* externally so we need to be sure that writes
4171	* won't block due to a need to transition
4172	* clean->dirty
4173	* until external management is started.
4174	*/
4175	mddev->in_sync = 0;
4176	mddev->safemode_delay = 0;
4177	mddev->safemode = 0;
4178	}
4179
4180	oldpers->free(mddev, oldpriv);
4181
4182	if (oldpers->sync_request == NULL &&
4183	pers->sync_request != NULL) {
4184	/* need to add the md_redundancy_group */
4185	if (sysfs_create_group(&mddev->kobj, &md_redundancy_group))
4186	pr_warn("md: cannot register extra attributes for %s\n",
4187	mdname(mddev));
4188	mddev->sysfs_action = sysfs_get_dirent(mddev->kobj.sd, "sync_action");
4189	mddev->sysfs_completed = sysfs_get_dirent_safe(mddev->kobj.sd, "sync_completed");
4190	mddev->sysfs_degraded = sysfs_get_dirent_safe(mddev->kobj.sd, "degraded");
4191	}
4192	if (oldpers->sync_request != NULL &&
4193	pers->sync_request == NULL) {
4194	/* need to remove the md_redundancy_group */
4195	if (mddev->to_remove == NULL)
4196	mddev->to_remove = &md_redundancy_group;
4197	}
4198
4199	put_pers(oldpers);
4200
4201	rdev_for_each(rdev, mddev) {
4202	if (rdev->raid_disk < 0)
4203	continue;
4204	if (rdev->new_raid_disk >= mddev->raid_disks)
4205	rdev->new_raid_disk = -1;
4206	if (rdev->new_raid_disk == rdev->raid_disk)
4207	continue;
4208	sysfs_unlink_rdev(mddev, rdev);
4209	}
4210	rdev_for_each(rdev, mddev) {
4211	if (rdev->raid_disk < 0)
4212	continue;
4213	if (rdev->new_raid_disk == rdev->raid_disk)
4214	continue;
4215	rdev->raid_disk = rdev->new_raid_disk;
4216	if (rdev->raid_disk < 0)
4217	clear_bit(In_sync, &rdev->flags);
4218	else {
4219	if (sysfs_link_rdev(mddev, rdev))
4220	pr_warn("md: cannot register rd%d for %s after level change\n",
4221	rdev->raid_disk, mdname(mddev));
4222	}
4223	}
4224
4225	if (pers->sync_request == NULL) {
4226	/* this is now an array without redundancy, so
4227	* it must always be in_sync
4228	*/
4229	mddev->in_sync = 1;
4230	timer_delete_sync(&mddev->safemode_timer);
4231	}
4232	pers->run(mddev);
4233	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
4234	if (!mddev->thread)
4235	md_update_sb(mddev, 1);
4236	sysfs_notify_dirent_safe(mddev->sysfs_level);
4237	md_new_event();
4238	rv = len;
4239	out_unlock:
4240	memalloc_noio_restore(noio_flags);
4241	mddev_unlock_and_resume(mddev);
4242	return rv;
4243	}
4244
4245	static struct md_sysfs_entry md_level =
4246	__ATTR(level, S_IRUGO\|S_IWUSR, level_show, level_store);
4247
4248	static ssize_t
4249	new_level_show(struct mddev mddev, char page)
4250	{
4251	return sprintf(page, "%d\n", mddev->new_level);
4252	}
4253
4254	static ssize_t
4255	new_level_store(struct mddev mddev, const char buf, size_t len)
4256	{
4257	unsigned int n;
4258	int err;
4259
4260	err = kstrtouint(buf, 10, &n);
4261	if (err < 0)
4262	return err;
4263	err = mddev_lock(mddev);
4264	if (err)
4265	return err;
4266
4267	mddev->new_level = n;
4268	md_update_sb(mddev, 1);
4269
4270	mddev_unlock(mddev);
4271	return len;
4272	}
4273	static struct md_sysfs_entry md_new_level =
4274	__ATTR(new_level, 0664, new_level_show, new_level_store);
4275
4276	static ssize_t
4277	bitmap_type_show(struct mddev mddev, char page)
4278	{
4279	struct md_submodule_head *head;
4280	unsigned long i;
4281	ssize_t len = 0;
4282
4283	if (mddev->bitmap_id == ID_BITMAP_NONE)
4284	len += sprintf(page + len, "[none] ");
4285	else
4286	len += sprintf(page + len, "none ");
4287
4288	xa_lock(&md_submodule);
4289	xa_for_each(&md_submodule, i, head) {
4290	if (head->type != MD_BITMAP \|\| head->id == ID_BITMAP_NONE)
4291	continue;
4292
4293	if (mddev->bitmap_id == head->id)
4294	len += sprintf(page + len, "[%s] ", head->name);
4295	else
4296	len += sprintf(page + len, "%s ", head->name);
4297	}
4298	xa_unlock(&md_submodule);
4299
4300	len += sprintf(page + len, "\n");
4301	return len;
4302	}
4303
4304	static ssize_t
4305	bitmap_type_store(struct mddev mddev, const char buf, size_t len)
4306	{
4307	struct md_submodule_head *head;
4308	enum md_submodule_id id;
4309	unsigned long i;
4310	int err = 0;
4311
4312	xa_lock(&md_submodule);
4313
4314	if (mddev->bitmap_ops) {
4315	err = -EBUSY;
4316	goto out;
4317	}
4318
4319	if (cmd_match(buf, "none")) {
4320	mddev->bitmap_id = ID_BITMAP_NONE;
4321	goto out;
4322	}
4323
4324	xa_for_each(&md_submodule, i, head) {
4325	if (head->type == MD_BITMAP && cmd_match(buf, head->name)) {
4326	mddev->bitmap_id = head->id;
4327	goto out;
4328	}
4329	}
4330
4331	err = kstrtoint(buf, 10, &id);
4332	if (err)
4333	goto out;
4334
4335	if (id == ID_BITMAP_NONE) {
4336	mddev->bitmap_id = id;
4337	goto out;
4338	}
4339
4340	head = xa_load(&md_submodule, id);
4341	if (head && head->type == MD_BITMAP) {
4342	mddev->bitmap_id = id;
4343	goto out;
4344	}
4345
4346	err = -ENOENT;
4347
4348	out:
4349	xa_unlock(&md_submodule);
4350	return err ? err : len;
4351	}
4352
4353	static struct md_sysfs_entry md_bitmap_type =
4354	__ATTR(bitmap_type, 0664, bitmap_type_show, bitmap_type_store);
4355
4356	static ssize_t
4357	layout_show(struct mddev mddev, char page)
4358	{
4359	/* just a number, not meaningful for all levels */
4360	if (mddev->reshape_position != MaxSector &&
4361	mddev->layout != mddev->new_layout)
4362	return sprintf(page, "%d (%d)\n",
4363	mddev->new_layout, mddev->layout);
4364	return sprintf(page, "%d\n", mddev->layout);
4365	}
4366
4367	static ssize_t
4368	layout_store(struct mddev mddev, const char buf, size_t len)
4369	{
4370	unsigned int n;
4371	int err;
4372
4373	err = kstrtouint(buf, 10, &n);
4374	if (err < 0)
4375	return err;
4376	err = mddev_lock(mddev);
4377	if (err)
4378	return err;
4379
4380	if (mddev->pers) {
4381	if (mddev->pers->check_reshape == NULL)
4382	err = -EBUSY;
4383	else if (!md_is_rdwr(mddev))
4384	err = -EROFS;
4385	else {
4386	mddev->new_layout = n;
4387	err = mddev->pers->check_reshape(mddev);
4388	if (err)
4389	mddev->new_layout = mddev->layout;
4390	}
4391	} else {
4392	mddev->new_layout = n;
4393	if (mddev->reshape_position == MaxSector)
4394	mddev->layout = n;
4395	}
4396	mddev_unlock(mddev);
4397	return err ?: len;
4398	}
4399	static struct md_sysfs_entry md_layout =
4400	__ATTR(layout, S_IRUGO\|S_IWUSR, layout_show, layout_store);
4401
4402	static ssize_t
4403	raid_disks_show(struct mddev mddev, char page)
4404	{
4405	if (mddev->raid_disks == 0)
4406	return 0;
4407	if (mddev->reshape_position != MaxSector &&
4408	mddev->delta_disks != 0)
4409	return sprintf(page, "%d (%d)\n", mddev->raid_disks,
4410	mddev->raid_disks - mddev->delta_disks);
4411	return sprintf(page, "%d\n", mddev->raid_disks);
4412	}
4413
4414	static int update_raid_disks(struct mddev *mddev, int raid_disks);
4415
4416	static ssize_t
4417	raid_disks_store(struct mddev mddev, const char buf, size_t len)
4418	{
4419	unsigned int n;
4420	unsigned int noio_flags;
4421	int err;
4422
4423	err = kstrtouint(buf, 10, &n);
4424	if (err < 0)
4425	return err;
4426
4427	err = mddev_suspend_and_lock(mddev);
4428	if (err)
4429	return err;
4430	noio_flags = memalloc_noio_save();
4431	if (mddev->pers) {
4432	if (n != mddev->raid_disks)
4433	err = update_raid_disks(mddev, n);
4434	} else if (mddev->reshape_position != MaxSector) {
4435	struct md_rdev *rdev;
4436	int olddisks = mddev->raid_disks - mddev->delta_disks;
4437
4438	err = -EINVAL;
4439	rdev_for_each(rdev, mddev) {
4440	if (olddisks < n &&
4441	rdev->data_offset < rdev->new_data_offset)
4442	goto out_unlock;
4443	if (olddisks > n &&
4444	rdev->data_offset > rdev->new_data_offset)
4445	goto out_unlock;
4446	}
4447	err = 0;
4448	mddev->delta_disks = n - olddisks;
4449	mddev->raid_disks = n;
4450	mddev->reshape_backwards = (mddev->delta_disks < 0);
4451	} else
4452	mddev->raid_disks = n;
4453	out_unlock:
4454	memalloc_noio_restore(noio_flags);
4455	mddev_unlock_and_resume(mddev);
4456	return err ? err : len;
4457	}
4458	static struct md_sysfs_entry md_raid_disks =
4459	__ATTR(raid_disks, S_IRUGO\|S_IWUSR, raid_disks_show, raid_disks_store);
4460
4461	static ssize_t
4462	uuid_show(struct mddev mddev, char page)
4463	{
4464	return sprintf(page, "%pU\n", mddev->uuid);
4465	}
4466	static struct md_sysfs_entry md_uuid =
4467	__ATTR(uuid, S_IRUGO, uuid_show, NULL);
4468
4469	static ssize_t
4470	chunk_size_show(struct mddev mddev, char page)
4471	{
4472	if (mddev->reshape_position != MaxSector &&
4473	mddev->chunk_sectors != mddev->new_chunk_sectors)
4474	return sprintf(page, "%d (%d)\n",
4475	mddev->new_chunk_sectors << 9,
4476	mddev->chunk_sectors << 9);
4477	return sprintf(page, "%d\n", mddev->chunk_sectors << 9);
4478	}
4479
4480	static ssize_t
4481	chunk_size_store(struct mddev mddev, const char buf, size_t len)
4482	{
4483	unsigned long n;
4484	int err;
4485
4486	err = kstrtoul(buf, 10, &n);
4487	if (err < 0)
4488	return err;
4489
4490	err = mddev_lock(mddev);
4491	if (err)
4492	return err;
4493	if (mddev->pers) {
4494	if (mddev->pers->check_reshape == NULL)
4495	err = -EBUSY;
4496	else if (!md_is_rdwr(mddev))
4497	err = -EROFS;
4498	else {
4499	mddev->new_chunk_sectors = n >> 9;
4500	err = mddev->pers->check_reshape(mddev);
4501	if (err)
4502	mddev->new_chunk_sectors = mddev->chunk_sectors;
4503	}
4504	} else {
4505	mddev->new_chunk_sectors = n >> 9;
4506	if (mddev->reshape_position == MaxSector)
4507	mddev->chunk_sectors = n >> 9;
4508	}
4509	mddev_unlock(mddev);
4510	return err ?: len;
4511	}
4512	static struct md_sysfs_entry md_chunk_size =
4513	__ATTR(chunk_size, S_IRUGO\|S_IWUSR, chunk_size_show, chunk_size_store);
4514
4515	static ssize_t
4516	resync_start_show(struct mddev mddev, char page)
4517	{
4518	if (mddev->resync_offset == MaxSector)
4519	return sprintf(page, "none\n");
4520	return sprintf(page, "%llu\n", (unsigned long long)mddev->resync_offset);
4521	}
4522
4523	static ssize_t
4524	resync_start_store(struct mddev mddev, const char buf, size_t len)
4525	{
4526	unsigned long long n;
4527	int err;
4528
4529	if (cmd_match(buf, "none"))
4530	n = MaxSector;
4531	else {
4532	err = kstrtoull(buf, 10, &n);
4533	if (err < 0)
4534	return err;
4535	if (n != (sector_t)n)
4536	return -EINVAL;
4537	}
4538
4539	err = mddev_lock(mddev);
4540	if (err)
4541	return err;
4542	if (mddev->pers && !test_bit(MD_RECOVERY_FROZEN, &mddev->recovery))
4543	err = -EBUSY;
4544
4545	if (!err) {
4546	mddev->resync_offset = n;
4547	if (mddev->pers)
4548	set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
4549	}
4550	mddev_unlock(mddev);
4551	return err ?: len;
4552	}
4553	static struct md_sysfs_entry md_resync_start =
4554	__ATTR_PREALLOC(resync_start, S_IRUGO\|S_IWUSR,
4555	resync_start_show, resync_start_store);
4556
4557	/*
4558	* The array state can be:
4559	*
4560	* clear
4561	* No devices, no size, no level
4562	* Equivalent to STOP_ARRAY ioctl
4563	* inactive
4564	* May have some settings, but array is not active
4565	* all IO results in error
4566	* When written, doesn't tear down array, but just stops it
4567	* suspended (not supported yet)
4568	* All IO requests will block. The array can be reconfigured.
4569	* Writing this, if accepted, will block until array is quiescent
4570	* readonly
4571	* no resync can happen. no superblocks get written.
4572	* write requests fail
4573	* read-auto
4574	* like readonly, but behaves like 'clean' on a write request.
4575	*
4576	* clean - no pending writes, but otherwise active.
4577	* When written to inactive array, starts without resync
4578	* If a write request arrives then
4579	* if metadata is known, mark 'dirty' and switch to 'active'.
4580	* if not known, block and switch to write-pending
4581	* If written to an active array that has pending writes, then fails.
4582	* active
4583	* fully active: IO and resync can be happening.
4584	* When written to inactive array, starts with resync
4585	*
4586	* write-pending
4587	* clean, but writes are blocked waiting for 'active' to be written.
4588	*
4589	* active-idle
4590	* like active, but no writes have been seen for a while (100msec).
4591	*
4592	* broken
4593	* Array is failed. It's useful because mounted-arrays aren't stopped
4594	* when array is failed, so this state will at least alert the user that
4595	* something is wrong.
4596	*/
4597	enum array_state { clear, inactive, suspended, readonly, read_auto, clean, active,
4598	write_pending, active_idle, broken, bad_word};
4599	static char *array_states[] = {
4600	"clear", "inactive", "suspended", "readonly", "read-auto", "clean", "active",
4601	"write-pending", "active-idle", "broken", NULL };
4602
4603	static int match_word(const char word, char *list)
4604	{
4605	int n;
4606	for (n=0; list[n]; n++)
4607	if (cmd_match(word, list[n]))
4608	break;
4609	return n;
4610	}
4611
4612	static ssize_t
4613	array_state_show(struct mddev mddev, char page)
4614	{
4615	enum array_state st = inactive;
4616
4617	if (mddev->pers && !test_bit(MD_NOT_READY, &mddev->flags)) {
4618	switch(mddev->ro) {
4619	case MD_RDONLY:
4620	st = readonly;
4621	break;
4622	case MD_AUTO_READ:
4623	st = read_auto;
4624	break;
4625	case MD_RDWR:
4626	spin_lock(&mddev->lock);
4627	if (test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags))
4628	st = write_pending;
4629	else if (mddev->in_sync)
4630	st = clean;
4631	else if (mddev->safemode)
4632	st = active_idle;
4633	else
4634	st = active;
4635	spin_unlock(&mddev->lock);
4636	}
4637
4638	if (test_bit(MD_BROKEN, &mddev->flags) && st == clean)
4639	st = broken;
4640	} else {
4641	if (list_empty(&mddev->disks) &&
4642	mddev->raid_disks == 0 &&
4643	mddev->dev_sectors == 0)
4644	st = clear;
4645	else
4646	st = inactive;
4647	}
4648	return sprintf(page, "%s\n", array_states[st]);
4649	}
4650
4651	static int do_md_stop(struct mddev *mddev, int ro);
4652	static int md_set_readonly(struct mddev *mddev);
4653	static int restart_array(struct mddev *mddev);
4654
4655	static ssize_t
4656	array_state_store(struct mddev mddev, const char buf, size_t len)
4657	{
4658	int err = 0;
4659	enum array_state st = match_word(buf, array_states);
4660
4661	/* No lock dependent actions */
4662	switch (st) {
4663	case suspended: /* not supported yet */
4664	case write_pending: /* cannot be set */
4665	case active_idle: /* cannot be set */
4666	case broken: /* cannot be set */
4667	case bad_word:
4668	return -EINVAL;
4669	case clear:
4670	case readonly:
4671	case inactive:
4672	case read_auto:
4673	if (!mddev->pers \|\| !md_is_rdwr(mddev))
4674	break;
4675	/* write sysfs will not open mddev and opener should be 0 */
4676	err = mddev_set_closing_and_sync_blockdev(mddev, 0);
4677	if (err)
4678	return err;
4679	break;
4680	default:
4681	break;
4682	}
4683
4684	if (mddev->pers && (st == active \|\| st == clean) &&
4685	mddev->ro != MD_RDONLY) {
4686	/* don't take reconfig_mutex when toggling between
4687	* clean and active
4688	*/
4689	spin_lock(&mddev->lock);
4690	if (st == active) {
4691	restart_array(mddev);
4692	clear_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
4693	md_wakeup_thread(mddev->thread);
4694	wake_up(&mddev->sb_wait);
4695	} else /* st == clean */ {
4696	restart_array(mddev);
4697	if (!set_in_sync(mddev))
4698	err = -EBUSY;
4699	}
4700	if (!err)
4701	sysfs_notify_dirent_safe(mddev->sysfs_state);
4702	spin_unlock(&mddev->lock);
4703	return err ?: len;
4704	}
4705	err = mddev_lock(mddev);
4706	if (err)
4707	return err;
4708
4709	switch (st) {
4710	case inactive:
4711	/* stop an active array, return 0 otherwise */
4712	if (mddev->pers)
4713	err = do_md_stop(mddev, 2);
4714	break;
4715	case clear:
4716	err = do_md_stop(mddev, 0);
4717	break;
4718	case readonly:
4719	if (mddev->pers)
4720	err = md_set_readonly(mddev);
4721	else {
4722	mddev->ro = MD_RDONLY;
4723	set_disk_ro(mddev->gendisk, 1);
4724	err = do_md_run(mddev);
4725	}
4726	break;
4727	case read_auto:
4728	if (mddev->pers) {
4729	if (md_is_rdwr(mddev))
4730	err = md_set_readonly(mddev);
4731	else if (mddev->ro == MD_RDONLY)
4732	err = restart_array(mddev);
4733	if (err == 0) {
4734	mddev->ro = MD_AUTO_READ;
4735	set_disk_ro(mddev->gendisk, 0);
4736	}
4737	} else {
4738	mddev->ro = MD_AUTO_READ;
4739	err = do_md_run(mddev);
4740	}
4741	break;
4742	case clean:
4743	if (mddev->pers) {
4744	err = restart_array(mddev);
4745	if (err)
4746	break;
4747	spin_lock(&mddev->lock);
4748	if (!set_in_sync(mddev))
4749	err = -EBUSY;
4750	spin_unlock(&mddev->lock);
4751	} else
4752	err = -EINVAL;
4753	break;
4754	case active:
4755	if (mddev->pers) {
4756	err = restart_array(mddev);
4757	if (err)
4758	break;
4759	clear_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
4760	wake_up(&mddev->sb_wait);
4761	err = 0;
4762	} else {
4763	mddev->ro = MD_RDWR;
4764	set_disk_ro(mddev->gendisk, 0);
4765	err = do_md_run(mddev);
4766	}
4767	break;
4768	default:
4769	err = -EINVAL;
4770	break;
4771	}
4772
4773	if (!err) {
4774	if (mddev->hold_active == UNTIL_IOCTL)
4775	mddev->hold_active = 0;
4776	sysfs_notify_dirent_safe(mddev->sysfs_state);
4777	}
4778	mddev_unlock(mddev);
4779
4780	if (st == readonly \|\| st == read_auto \|\| st == inactive \|\|
4781	(err && st == clear))
4782	clear_bit(MD_CLOSING, &mddev->flags);
4783
4784	return err ?: len;
4785	}
4786	static struct md_sysfs_entry md_array_state =
4787	__ATTR_PREALLOC(array_state, S_IRUGO\|S_IWUSR, array_state_show, array_state_store);
4788
4789	static ssize_t
4790	max_corrected_read_errors_show(struct mddev mddev, char page) {
4791	return sprintf(page, "%d\n",
4792	atomic_read(&mddev->max_corr_read_errors));
4793	}
4794
4795	static ssize_t
4796	max_corrected_read_errors_store(struct mddev mddev, const char buf, size_t len)
4797	{
4798	unsigned int n;
4799	int rv;
4800
4801	rv = kstrtouint(buf, 10, &n);
4802	if (rv < 0)
4803	return rv;
4804	if (n > INT_MAX)
4805	return -EINVAL;
4806	atomic_set(&mddev->max_corr_read_errors, n);
4807	return len;
4808	}
4809
4810	static struct md_sysfs_entry max_corr_read_errors =
4811	__ATTR(max_read_errors, S_IRUGO\|S_IWUSR, max_corrected_read_errors_show,
4812	max_corrected_read_errors_store);
4813
4814	static ssize_t
4815	null_show(struct mddev mddev, char page)
4816	{
4817	return -EINVAL;
4818	}
4819
4820	static ssize_t
4821	new_dev_store(struct mddev mddev, const char buf, size_t len)
4822	{
4823	/* buf must be %d:%d\n? giving major and minor numbers */
4824	/* The new device is added to the array.
4825	* If the array has a persistent superblock, we read the
4826	* superblock to initialise info and check validity.
4827	* Otherwise, only checking done is that in bind_rdev_to_array,
4828	* which mainly checks size.
4829	*/
4830	char *e;
4831	int major = simple_strtoul(buf, &e, 10);
4832	int minor;
4833	dev_t dev;
4834	struct md_rdev *rdev;
4835	unsigned int noio_flags;
4836	int err;
4837
4838	if (!buf \|\| e != ':' \|\| !e[1] \|\| e[1] == '\n')
4839	return -EINVAL;
4840	minor = simple_strtoul(e+1, &e, 10);
4841	if (e && e != '\n')
4842	return -EINVAL;
4843	dev = MKDEV(major, minor);
4844	if (major != MAJOR(dev) \|\|
4845	minor != MINOR(dev))
4846	return -EOVERFLOW;
4847
4848	err = mddev_suspend_and_lock(mddev);
4849	if (err)
4850	return err;
4851	noio_flags = memalloc_noio_save();
4852	if (mddev->persistent) {
4853	rdev = md_import_device(dev, mddev->major_version,
4854	mddev->minor_version);
4855	if (!IS_ERR(rdev) && !list_empty(&mddev->disks)) {
4856	struct md_rdev *rdev0
4857	= list_entry(mddev->disks.next,
4858	struct md_rdev, same_set);
4859	err = super_types[mddev->major_version]
4860	.load_super(rdev, rdev0, mddev->minor_version);
4861	if (err < 0)
4862	goto out;
4863	}
4864	} else if (mddev->external)
4865	rdev = md_import_device(dev, -2, -1);
4866	else
4867	rdev = md_import_device(dev, -1, -1);
4868
4869	if (IS_ERR(rdev)) {
4870	memalloc_noio_restore(noio_flags);
4871	mddev_unlock_and_resume(mddev);
4872	return PTR_ERR(rdev);
4873	}
4874	err = bind_rdev_to_array(rdev, mddev);
4875	out:
4876	if (err)
4877	export_rdev(rdev);
4878	memalloc_noio_restore(noio_flags);
4879	mddev_unlock_and_resume(mddev);
4880	if (!err)
4881	md_new_event();
4882	return err ? err : len;
4883	}
4884
4885	static struct md_sysfs_entry md_new_device =
4886	__ATTR(new_dev, S_IWUSR, null_show, new_dev_store);
4887
4888	static ssize_t
4889	bitmap_store(struct mddev mddev, const char buf, size_t len)
4890	{
4891	char *end;
4892	unsigned long chunk, end_chunk;
4893	int err;
4894
4895	if (!md_bitmap_enabled(mddev, false))
4896	return len;
4897
4898	err = mddev_lock(mddev);
4899	if (err)
4900	return err;
4901	if (!mddev->bitmap)
4902	goto out;
4903	/* buf should be <chunk> <chunk> ... or <chunk>-<chunk> ... (range) */
4904	while (*buf) {
4905	chunk = end_chunk = simple_strtoul(buf, &end, 0);
4906	if (buf == end)
4907	break;
4908
4909	if (end == '-') { / range */
4910	buf = end + 1;
4911	end_chunk = simple_strtoul(buf, &end, 0);
4912	if (buf == end)
4913	break;
4914	}
4915
4916	if (end && !isspace(end))
4917	break;
4918
4919	mddev->bitmap_ops->dirty_bits(mddev, chunk, end_chunk);
4920	buf = skip_spaces(end);
4921	}
4922	mddev->bitmap_ops->unplug(mddev, true); /* flush the bits to disk */
4923	out:
4924	mddev_unlock(mddev);
4925	return len;
4926	}
4927
4928	static struct md_sysfs_entry md_bitmap =
4929	__ATTR(bitmap_set_bits, S_IWUSR, null_show, bitmap_store);
4930
4931	static ssize_t
4932	size_show(struct mddev mddev, char page)
4933	{
4934	return sprintf(page, "%llu\n",
4935	(unsigned long long)mddev->dev_sectors / 2);
4936	}
4937
4938	static int update_size(struct mddev *mddev, sector_t num_sectors);
4939
4940	static ssize_t
4941	size_store(struct mddev mddev, const char buf, size_t len)
4942	{
4943	/* If array is inactive, we can reduce the component size, but
4944	* not increase it (except from 0).
4945	* If array is active, we can try an on-line resize
4946	*/
4947	sector_t sectors;
4948	int err = strict_blocks_to_sectors(buf, &sectors);
4949
4950	if (err < 0)
4951	return err;
4952	err = mddev_lock(mddev);
4953	if (err)
4954	return err;
4955	if (mddev->pers) {
4956	err = update_size(mddev, sectors);
4957	if (err == 0)
4958	md_update_sb(mddev, 1);
4959	} else {
4960	if (mddev->dev_sectors == 0 \|\|
4961	mddev->dev_sectors > sectors)
4962	mddev->dev_sectors = sectors;
4963	else
4964	err = -ENOSPC;
4965	}
4966	mddev_unlock(mddev);
4967	return err ? err : len;
4968	}
4969
4970	static struct md_sysfs_entry md_size =
4971	__ATTR(component_size, S_IRUGO\|S_IWUSR, size_show, size_store);
4972
4973	/* Metadata version.
4974	* This is one of
4975	* 'none' for arrays with no metadata (good luck...)
4976	* 'external' for arrays with externally managed metadata,
4977	* or N.M for internally known formats
4978	*/
4979	static ssize_t
4980	metadata_show(struct mddev mddev, char page)
4981	{
4982	if (mddev->persistent)
4983	return sprintf(page, "%d.%d\n",
4984	mddev->major_version, mddev->minor_version);
4985	else if (mddev->external)
4986	return sprintf(page, "external:%s\n", mddev->metadata_type);
4987	else
4988	return sprintf(page, "none\n");
4989	}
4990
4991	static ssize_t
4992	metadata_store(struct mddev mddev, const char buf, size_t len)
4993	{
4994	int major, minor;
4995	char *e;
4996	int err;
4997	/* Changing the details of 'external' metadata is
4998	* always permitted. Otherwise there must be
4999	* no devices attached to the array.
5000	*/
5001
5002	err = mddev_lock(mddev);
5003	if (err)
5004	return err;
5005	err = -EBUSY;
5006	if (mddev->external && strncmp(buf, "external:", 9) == 0)
5007	;
5008	else if (!list_empty(&mddev->disks))
5009	goto out_unlock;
5010
5011	err = 0;
5012	if (cmd_match(buf, "none")) {
5013	mddev->persistent = 0;
5014	mddev->external = 0;
5015	mddev->major_version = 0;
5016	mddev->minor_version = 90;
5017	goto out_unlock;
5018	}
5019	if (strncmp(buf, "external:", 9) == 0) {
5020	size_t namelen = len-9;
5021	if (namelen >= sizeof(mddev->metadata_type))
5022	namelen = sizeof(mddev->metadata_type)-1;
5023	memcpy(mddev->metadata_type, buf+9, namelen);
5024	mddev->metadata_type[namelen] = 0;
5025	if (namelen && mddev->metadata_type[namelen-1] == '\n')
5026	mddev->metadata_type[--namelen] = 0;
5027	mddev->persistent = 0;
5028	mddev->external = 1;
5029	mddev->major_version = 0;
5030	mddev->minor_version = 90;
5031	goto out_unlock;
5032	}
5033	major = simple_strtoul(buf, &e, 10);
5034	err = -EINVAL;
5035	if (e==buf \|\| *e != '.')
5036	goto out_unlock;
5037	buf = e+1;
5038	minor = simple_strtoul(buf, &e, 10);
5039	if (e==buf \|\| (e && e != '\n') )
5040	goto out_unlock;
5041	err = -ENOENT;
5042	if (major >= ARRAY_SIZE(super_types) \|\| super_types[major].name == NULL)
5043	goto out_unlock;
5044	mddev->major_version = major;
5045	mddev->minor_version = minor;
5046	mddev->persistent = 1;
5047	mddev->external = 0;
5048	err = 0;
5049	out_unlock:
5050	mddev_unlock(mddev);
5051	return err ?: len;
5052	}
5053
5054	static struct md_sysfs_entry md_metadata =
5055	__ATTR_PREALLOC(metadata_version, S_IRUGO\|S_IWUSR, metadata_show, metadata_store);
5056
5057	static bool rdev_needs_recovery(struct md_rdev *rdev, sector_t sectors)
5058	{
5059	return rdev->raid_disk >= 0 &&
5060	!test_bit(Journal, &rdev->flags) &&
5061	!test_bit(Faulty, &rdev->flags) &&
5062	!test_bit(In_sync, &rdev->flags) &&
5063	rdev->recovery_offset < sectors;
5064	}
5065
5066	static enum sync_action md_get_active_sync_action(struct mddev *mddev)
5067	{
5068	struct md_rdev *rdev;
5069	bool is_recover = false;
5070
5071	if (mddev->resync_offset < MaxSector)
5072	return ACTION_RESYNC;
5073
5074	if (mddev->reshape_position != MaxSector)
5075	return ACTION_RESHAPE;
5076
5077	rcu_read_lock();
5078	rdev_for_each_rcu(rdev, mddev) {
5079	if (rdev_needs_recovery(rdev, MaxSector)) {
5080	is_recover = true;
5081	break;
5082	}
5083	}
5084	rcu_read_unlock();
5085
5086	return is_recover ? ACTION_RECOVER : ACTION_IDLE;
5087	}
5088
5089	enum sync_action md_sync_action(struct mddev *mddev)
5090	{
5091	unsigned long recovery = mddev->recovery;
5092	enum sync_action active_action;
5093
5094	/*
5095	* frozen has the highest priority, means running sync_thread will be
5096	* stopped immediately, and no new sync_thread can start.
5097	*/
5098	if (test_bit(MD_RECOVERY_FROZEN, &recovery))
5099	return ACTION_FROZEN;
5100
5101	/*
5102	* read-only array can't register sync_thread, and it can only
5103	* add/remove spares.
5104	*/
5105	if (!md_is_rdwr(mddev))
5106	return ACTION_IDLE;
5107
5108	/*
5109	* idle means no sync_thread is running, and no new sync_thread is
5110	* requested.
5111	*/
5112	if (!test_bit(MD_RECOVERY_RUNNING, &recovery) &&
5113	!test_bit(MD_RECOVERY_NEEDED, &recovery))
5114	return ACTION_IDLE;
5115
5116	/*
5117	* Check if any sync operation (resync/recover/reshape) is
5118	* currently active. This ensures that only one sync operation
5119	* can run at a time. Returns the type of active operation, or
5120	* ACTION_IDLE if none are active.
5121	*/
5122	active_action = md_get_active_sync_action(mddev);
5123	if (active_action != ACTION_IDLE)
5124	return active_action;
5125
5126	if (test_bit(MD_RECOVERY_RESHAPE, &recovery))
5127	return ACTION_RESHAPE;
5128
5129	if (test_bit(MD_RECOVERY_RECOVER, &recovery))
5130	return ACTION_RECOVER;
5131
5132	if (test_bit(MD_RECOVERY_SYNC, &recovery)) {
5133	/*
5134	* MD_RECOVERY_CHECK must be paired with
5135	* MD_RECOVERY_REQUESTED.
5136	*/
5137	if (test_bit(MD_RECOVERY_CHECK, &recovery))
5138	return ACTION_CHECK;
5139	if (test_bit(MD_RECOVERY_REQUESTED, &recovery))
5140	return ACTION_REPAIR;
5141	return ACTION_RESYNC;
5142	}
5143
5144	/*
5145	* MD_RECOVERY_NEEDED or MD_RECOVERY_RUNNING is set, however, no
5146	* sync_action is specified.
5147	*/
5148	return ACTION_IDLE;
5149	}
5150
5151	enum sync_action md_sync_action_by_name(const char *page)
5152	{
5153	enum sync_action action;
5154
5155	for (action = 0; action < NR_SYNC_ACTIONS; ++action) {
5156	if (cmd_match(page, action_name[action]))
5157	return action;
5158	}
5159
5160	return NR_SYNC_ACTIONS;
5161	}
5162
5163	const char *md_sync_action_name(enum sync_action action)
5164	{
5165	return action_name[action];
5166	}
5167
5168	static ssize_t
5169	action_show(struct mddev mddev, char page)
5170	{
5171	enum sync_action action = md_sync_action(mddev);
5172
5173	return sprintf(page, "%s\n", md_sync_action_name(action));
5174	}
5175
5176	/**
5177	* stop_sync_thread() - wait for sync_thread to stop if it's running.
5178	* @mddev: the array.
5179	* @locked: if set, reconfig_mutex will still be held after this function
5180	* return; if not set, reconfig_mutex will be released after this
5181	* function return.
5182	*/
5183	static void stop_sync_thread(struct mddev *mddev, bool locked)
5184	{
5185	int sync_seq = atomic_read(&mddev->sync_seq);
5186
5187	if (!test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) {
5188	if (!locked)
5189	mddev_unlock(mddev);
5190	return;
5191	}
5192
5193	mddev_unlock(mddev);
5194
5195	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
5196	/*
5197	* Thread might be blocked waiting for metadata update which will now
5198	* never happen
5199	*/
5200	md_wakeup_thread_directly(&mddev->sync_thread);
5201	if (work_pending(&mddev->sync_work))
5202	flush_work(&mddev->sync_work);
5203
5204	wait_event(resync_wait,
5205	!test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) \|\|
5206	(!test_bit(MD_RECOVERY_FROZEN, &mddev->recovery) &&
5207	sync_seq != atomic_read(&mddev->sync_seq)));
5208
5209	if (locked)
5210	mddev_lock_nointr(mddev);
5211	}
5212
5213	void md_idle_sync_thread(struct mddev *mddev)
5214	{
5215	lockdep_assert_held(&mddev->reconfig_mutex);
5216
5217	clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
5218	stop_sync_thread(mddev, true);
5219	}
5220	EXPORT_SYMBOL_GPL(md_idle_sync_thread);
5221
5222	void md_frozen_sync_thread(struct mddev *mddev)
5223	{
5224	lockdep_assert_held(&mddev->reconfig_mutex);
5225
5226	set_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
5227	stop_sync_thread(mddev, true);
5228	}
5229	EXPORT_SYMBOL_GPL(md_frozen_sync_thread);
5230
5231	void md_unfrozen_sync_thread(struct mddev *mddev)
5232	{
5233	lockdep_assert_held(&mddev->reconfig_mutex);
5234
5235	clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
5236	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
5237	md_wakeup_thread(mddev->thread);
5238	sysfs_notify_dirent_safe(mddev->sysfs_action);
5239	}
5240	EXPORT_SYMBOL_GPL(md_unfrozen_sync_thread);
5241
5242	static int mddev_start_reshape(struct mddev *mddev)
5243	{
5244	int ret;
5245
5246	if (mddev->pers->start_reshape == NULL)
5247	return -EINVAL;
5248
5249	if (mddev->reshape_position == MaxSector \|\|
5250	mddev->pers->check_reshape == NULL \|\|
5251	mddev->pers->check_reshape(mddev)) {
5252	clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
5253	ret = mddev->pers->start_reshape(mddev);
5254	if (ret)
5255	return ret;
5256	} else {
5257	/*
5258	* If reshape is still in progress, and md_check_recovery() can
5259	* continue to reshape, don't restart reshape because data can
5260	* be corrupted for raid456.
5261	*/
5262	clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
5263	}
5264
5265	sysfs_notify_dirent_safe(mddev->sysfs_degraded);
5266	return 0;
5267	}
5268
5269	static ssize_t
5270	action_store(struct mddev mddev, const char page, size_t len)
5271	{
5272	int ret;
5273	enum sync_action action;
5274
5275	if (!mddev->pers \|\| !mddev->pers->sync_request)
5276	return -EINVAL;
5277
5278	action = md_sync_action_by_name(page);
5279	if (action == ACTION_RESHAPE) {
5280	ret = mddev_suspend(mddev, true);
5281	if (ret)
5282	return ret;
5283	}
5284	retry:
5285	if (work_busy(&mddev->sync_work))
5286	flush_work(&mddev->sync_work);
5287
5288	ret = mddev_lock(mddev);
5289	if (ret) {
5290	if (action == ACTION_RESHAPE)
5291	mddev_resume(mddev);
5292	return ret;
5293	}
5294
5295	if (work_busy(&mddev->sync_work)) {
5296	mddev_unlock(mddev);
5297	goto retry;
5298	}
5299
5300	/* TODO: mdadm rely on "idle" to start sync_thread. */
5301	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) {
5302	switch (action) {
5303	case ACTION_FROZEN:
5304	md_frozen_sync_thread(mddev);
5305	ret = len;
5306	goto out;
5307	case ACTION_IDLE:
5308	md_idle_sync_thread(mddev);
5309	break;
5310	case ACTION_RESHAPE:
5311	case ACTION_RECOVER:
5312	case ACTION_CHECK:
5313	case ACTION_REPAIR:
5314	case ACTION_RESYNC:
5315	ret = -EBUSY;
5316	goto out;
5317	default:
5318	ret = -EINVAL;
5319	goto out;
5320	}
5321	} else {
5322	switch (action) {
5323	case ACTION_FROZEN:
5324	set_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
5325	ret = len;
5326	goto out;
5327	case ACTION_RESHAPE:
5328	clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
5329	ret = mddev_start_reshape(mddev);
5330	if (ret)
5331	goto out;
5332	break;
5333	case ACTION_RECOVER:
5334	clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
5335	set_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
5336	break;
5337	case ACTION_CHECK:
5338	set_bit(MD_RECOVERY_CHECK, &mddev->recovery);
5339	fallthrough;
5340	case ACTION_REPAIR:
5341	set_bit(MD_RECOVERY_REQUESTED, &mddev->recovery);
5342	set_bit(MD_RECOVERY_SYNC, &mddev->recovery);
5343	fallthrough;
5344	case ACTION_RESYNC:
5345	case ACTION_IDLE:
5346	clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
5347	break;
5348	default:
5349	ret = -EINVAL;
5350	goto out;
5351	}
5352	}
5353
5354	if (mddev->ro == MD_AUTO_READ) {
5355	/* A write to sync_action is enough to justify
5356	* canceling read-auto mode
5357	*/
5358	mddev->ro = MD_RDWR;
5359	md_wakeup_thread(mddev->sync_thread);
5360	}
5361
5362	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
5363	md_wakeup_thread(mddev->thread);
5364	sysfs_notify_dirent_safe(mddev->sysfs_action);
5365	ret = len;
5366
5367	out:
5368	mddev_unlock(mddev);
5369	if (action == ACTION_RESHAPE)
5370	mddev_resume(mddev);
5371	return ret;
5372	}
5373
5374	static struct md_sysfs_entry md_scan_mode =
5375	__ATTR_PREALLOC(sync_action, S_IRUGO\|S_IWUSR, action_show, action_store);
5376
5377	static ssize_t
5378	last_sync_action_show(struct mddev mddev, char page)
5379	{
5380	return sprintf(page, "%s\n",
5381	md_sync_action_name(mddev->last_sync_action));
5382	}
5383
5384	static struct md_sysfs_entry md_last_scan_mode = __ATTR_RO(last_sync_action);
5385
5386	static ssize_t
5387	mismatch_cnt_show(struct mddev mddev, char page)
5388	{
5389	return sprintf(page, "%llu\n",
5390	(unsigned long long)
5391	atomic64_read(&mddev->resync_mismatches));
5392	}
5393
5394	static struct md_sysfs_entry md_mismatches = __ATTR_RO(mismatch_cnt);
5395
5396	static ssize_t
5397	sync_min_show(struct mddev mddev, char page)
5398	{
5399	return sprintf(page, "%d (%s)\n", speed_min(mddev),
5400	mddev->sync_speed_min ? "local" : "system");
5401	}
5402
5403	static ssize_t
5404	sync_min_store(struct mddev mddev, const char buf, size_t len)
5405	{
5406	unsigned int min;
5407	int rv;
5408
5409	if (strncmp(buf, "system", 6) == 0) {
5410	min = 0;
5411	} else {
5412	rv = kstrtouint(buf, 10, &min);
5413	if (rv < 0)
5414	return rv;
5415	if (min == 0)
5416	return -EINVAL;
5417	}
5418	mddev->sync_speed_min = min;
5419	return len;
5420	}
5421
5422	static struct md_sysfs_entry md_sync_min =
5423	__ATTR(sync_speed_min, S_IRUGO\|S_IWUSR, sync_min_show, sync_min_store);
5424
5425	static ssize_t
5426	sync_max_show(struct mddev mddev, char page)
5427	{
5428	return sprintf(page, "%d (%s)\n", speed_max(mddev),
5429	mddev->sync_speed_max ? "local" : "system");
5430	}
5431
5432	static ssize_t
5433	sync_max_store(struct mddev mddev, const char buf, size_t len)
5434	{
5435	unsigned int max;
5436	int rv;
5437
5438	if (strncmp(buf, "system", 6) == 0) {
5439	max = 0;
5440	} else {
5441	rv = kstrtouint(buf, 10, &max);
5442	if (rv < 0)
5443	return rv;
5444	if (max == 0)
5445	return -EINVAL;
5446	}
5447	mddev->sync_speed_max = max;
5448	return len;
5449	}
5450
5451	static struct md_sysfs_entry md_sync_max =
5452	__ATTR(sync_speed_max, S_IRUGO\|S_IWUSR, sync_max_show, sync_max_store);
5453
5454	static ssize_t
5455	sync_io_depth_show(struct mddev mddev, char page)
5456	{
5457	return sprintf(page, "%d (%s)\n", sync_io_depth(mddev),
5458	mddev->sync_io_depth ? "local" : "system");
5459	}
5460
5461	static ssize_t
5462	sync_io_depth_store(struct mddev mddev, const char buf, size_t len)
5463	{
5464	unsigned int max;
5465	int rv;
5466
5467	if (strncmp(buf, "system", 6) == 0) {
5468	max = 0;
5469	} else {
5470	rv = kstrtouint(buf, 10, &max);
5471	if (rv < 0)
5472	return rv;
5473	if (max == 0)
5474	return -EINVAL;
5475	}
5476	mddev->sync_io_depth = max;
5477	return len;
5478	}
5479
5480	static struct md_sysfs_entry md_sync_io_depth =
5481	__ATTR_RW(sync_io_depth);
5482
5483	static ssize_t
5484	degraded_show(struct mddev mddev, char page)
5485	{
5486	return sprintf(page, "%d\n", mddev->degraded);
5487	}
5488	static struct md_sysfs_entry md_degraded = __ATTR_RO(degraded);
5489
5490	static ssize_t
5491	sync_force_parallel_show(struct mddev mddev, char page)
5492	{
5493	return sprintf(page, "%d\n", mddev->parallel_resync);
5494	}
5495
5496	static ssize_t
5497	sync_force_parallel_store(struct mddev mddev, const char buf, size_t len)
5498	{
5499	long n;
5500
5501	if (kstrtol(buf, 10, &n))
5502	return -EINVAL;
5503
5504	if (n != 0 && n != 1)
5505	return -EINVAL;
5506
5507	mddev->parallel_resync = n;
5508
5509	if (mddev->sync_thread)
5510	wake_up(&resync_wait);
5511
5512	return len;
5513	}
5514
5515	/* force parallel resync, even with shared block devices */
5516	static struct md_sysfs_entry md_sync_force_parallel =
5517	__ATTR(sync_force_parallel, S_IRUGO\|S_IWUSR,
5518	sync_force_parallel_show, sync_force_parallel_store);
5519
5520	static ssize_t
5521	sync_speed_show(struct mddev mddev, char page)
5522	{
5523	unsigned long resync, dt, db;
5524	if (mddev->curr_resync == MD_RESYNC_NONE)
5525	return sprintf(page, "none\n");
5526	resync = mddev->curr_mark_cnt - atomic_read(&mddev->recovery_active);
5527	dt = (jiffies - mddev->resync_mark) / HZ;
5528	if (!dt) dt++;
5529	db = resync - mddev->resync_mark_cnt;
5530	return sprintf(page, "%lu\n", db/dt/2); /* K/sec */
5531	}
5532
5533	static struct md_sysfs_entry md_sync_speed = __ATTR_RO(sync_speed);
5534
5535	static ssize_t
5536	sync_completed_show(struct mddev mddev, char page)
5537	{
5538	unsigned long long max_sectors, resync;
5539
5540	if (!test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
5541	return sprintf(page, "none\n");
5542
5543	if (mddev->curr_resync == MD_RESYNC_YIELDED \|\|
5544	mddev->curr_resync == MD_RESYNC_DELAYED)
5545	return sprintf(page, "delayed\n");
5546
5547	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) \|\|
5548	test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
5549	max_sectors = mddev->resync_max_sectors;
5550	else
5551	max_sectors = mddev->dev_sectors;
5552
5553	resync = mddev->curr_resync_completed;
5554	return sprintf(page, "%llu / %llu\n", resync, max_sectors);
5555	}
5556
5557	static struct md_sysfs_entry md_sync_completed =
5558	__ATTR_PREALLOC(sync_completed, S_IRUGO, sync_completed_show, NULL);
5559
5560	static ssize_t
5561	min_sync_show(struct mddev mddev, char page)
5562	{
5563	return sprintf(page, "%llu\n",
5564	(unsigned long long)mddev->resync_min);
5565	}
5566	static ssize_t
5567	min_sync_store(struct mddev mddev, const char buf, size_t len)
5568	{
5569	unsigned long long min;
5570	int err;
5571
5572	if (kstrtoull(buf, 10, &min))
5573	return -EINVAL;
5574
5575	spin_lock(&mddev->lock);
5576	err = -EINVAL;
5577	if (min > mddev->resync_max)
5578	goto out_unlock;
5579
5580	err = -EBUSY;
5581	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
5582	goto out_unlock;
5583
5584	/* Round down to multiple of 4K for safety */
5585	mddev->resync_min = round_down(min, 8);
5586	err = 0;
5587
5588	out_unlock:
5589	spin_unlock(&mddev->lock);
5590	return err ?: len;
5591	}
5592
5593	static struct md_sysfs_entry md_min_sync =
5594	__ATTR(sync_min, S_IRUGO\|S_IWUSR, min_sync_show, min_sync_store);
5595
5596	static ssize_t
5597	max_sync_show(struct mddev mddev, char page)
5598	{
5599	if (mddev->resync_max == MaxSector)
5600	return sprintf(page, "max\n");
5601	else
5602	return sprintf(page, "%llu\n",
5603	(unsigned long long)mddev->resync_max);
5604	}
5605	static ssize_t
5606	max_sync_store(struct mddev mddev, const char buf, size_t len)
5607	{
5608	int err;
5609	spin_lock(&mddev->lock);
5610	if (strncmp(buf, "max", 3) == 0)
5611	mddev->resync_max = MaxSector;
5612	else {
5613	unsigned long long max;
5614	int chunk;
5615
5616	err = -EINVAL;
5617	if (kstrtoull(buf, 10, &max))
5618	goto out_unlock;
5619	if (max < mddev->resync_min)
5620	goto out_unlock;
5621
5622	err = -EBUSY;
5623	if (max < mddev->resync_max && md_is_rdwr(mddev) &&
5624	test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
5625	goto out_unlock;
5626
5627	/* Must be a multiple of chunk_size */
5628	chunk = mddev->chunk_sectors;
5629	if (chunk) {
5630	sector_t temp = max;
5631
5632	err = -EINVAL;
5633	if (sector_div(temp, chunk))
5634	goto out_unlock;
5635	}
5636	mddev->resync_max = max;
5637	}
5638	wake_up(&mddev->recovery_wait);
5639	err = 0;
5640	out_unlock:
5641	spin_unlock(&mddev->lock);
5642	return err ?: len;
5643	}
5644
5645	static struct md_sysfs_entry md_max_sync =
5646	__ATTR(sync_max, S_IRUGO\|S_IWUSR, max_sync_show, max_sync_store);
5647
5648	static ssize_t
5649	suspend_lo_show(struct mddev mddev, char page)
5650	{
5651	return sprintf(page, "%llu\n",
5652	(unsigned long long)READ_ONCE(mddev->suspend_lo));
5653	}
5654
5655	static ssize_t
5656	suspend_lo_store(struct mddev mddev, const char buf, size_t len)
5657	{
5658	unsigned long long new;
5659	int err;
5660
5661	err = kstrtoull(buf, 10, &new);
5662	if (err < 0)
5663	return err;
5664	if (new != (sector_t)new)
5665	return -EINVAL;
5666
5667	err = mddev_suspend(mddev, true);
5668	if (err)
5669	return err;
5670
5671	WRITE_ONCE(mddev->suspend_lo, new);
5672	mddev_resume(mddev);
5673
5674	return len;
5675	}
5676	static struct md_sysfs_entry md_suspend_lo =
5677	__ATTR(suspend_lo, S_IRUGO\|S_IWUSR, suspend_lo_show, suspend_lo_store);
5678
5679	static ssize_t
5680	suspend_hi_show(struct mddev mddev, char page)
5681	{
5682	return sprintf(page, "%llu\n",
5683	(unsigned long long)READ_ONCE(mddev->suspend_hi));
5684	}
5685
5686	static ssize_t
5687	suspend_hi_store(struct mddev mddev, const char buf, size_t len)
5688	{
5689	unsigned long long new;
5690	int err;
5691
5692	err = kstrtoull(buf, 10, &new);
5693	if (err < 0)
5694	return err;
5695	if (new != (sector_t)new)
5696	return -EINVAL;
5697
5698	err = mddev_suspend(mddev, true);
5699	if (err)
5700	return err;
5701
5702	WRITE_ONCE(mddev->suspend_hi, new);
5703	mddev_resume(mddev);
5704
5705	return len;
5706	}
5707	static struct md_sysfs_entry md_suspend_hi =
5708	__ATTR(suspend_hi, S_IRUGO\|S_IWUSR, suspend_hi_show, suspend_hi_store);
5709
5710	static ssize_t
5711	reshape_position_show(struct mddev mddev, char page)
5712	{
5713	if (mddev->reshape_position != MaxSector)
5714	return sprintf(page, "%llu\n",
5715	(unsigned long long)mddev->reshape_position);
5716	strcpy(page, "none\n");
5717	return 5;
5718	}
5719
5720	static ssize_t
5721	reshape_position_store(struct mddev mddev, const char buf, size_t len)
5722	{
5723	struct md_rdev *rdev;
5724	unsigned long long new;
5725	int err;
5726
5727	err = kstrtoull(buf, 10, &new);
5728	if (err < 0)
5729	return err;
5730	if (new != (sector_t)new)
5731	return -EINVAL;
5732	err = mddev_lock(mddev);
5733	if (err)
5734	return err;
5735	err = -EBUSY;
5736	if (mddev->pers)
5737	goto unlock;
5738	mddev->reshape_position = new;
5739	mddev->delta_disks = 0;
5740	mddev->reshape_backwards = 0;
5741	mddev->new_level = mddev->level;
5742	mddev->new_layout = mddev->layout;
5743	mddev->new_chunk_sectors = mddev->chunk_sectors;
5744	rdev_for_each(rdev, mddev)
5745	rdev->new_data_offset = rdev->data_offset;
5746	err = 0;
5747	unlock:
5748	mddev_unlock(mddev);
5749	return err ?: len;
5750	}
5751
5752	static struct md_sysfs_entry md_reshape_position =
5753	__ATTR(reshape_position, S_IRUGO\|S_IWUSR, reshape_position_show,
5754	reshape_position_store);
5755
5756	static ssize_t
5757	reshape_direction_show(struct mddev mddev, char page)
5758	{
5759	return sprintf(page, "%s\n",
5760	mddev->reshape_backwards ? "backwards" : "forwards");
5761	}
5762
5763	static ssize_t
5764	reshape_direction_store(struct mddev mddev, const char buf, size_t len)
5765	{
5766	int backwards = 0;
5767	int err;
5768
5769	if (cmd_match(buf, "forwards"))
5770	backwards = 0;
5771	else if (cmd_match(buf, "backwards"))
5772	backwards = 1;
5773	else
5774	return -EINVAL;
5775	if (mddev->reshape_backwards == backwards)
5776	return len;
5777
5778	err = mddev_lock(mddev);
5779	if (err)
5780	return err;
5781	/* check if we are allowed to change */
5782	if (mddev->delta_disks)
5783	err = -EBUSY;
5784	else if (mddev->persistent &&
5785	mddev->major_version == 0)
5786	err = -EINVAL;
5787	else
5788	mddev->reshape_backwards = backwards;
5789	mddev_unlock(mddev);
5790	return err ?: len;
5791	}
5792
5793	static struct md_sysfs_entry md_reshape_direction =
5794	__ATTR(reshape_direction, S_IRUGO\|S_IWUSR, reshape_direction_show,
5795	reshape_direction_store);
5796
5797	static ssize_t
5798	array_size_show(struct mddev mddev, char page)
5799	{
5800	if (mddev->external_size)
5801	return sprintf(page, "%llu\n",
5802	(unsigned long long)mddev->array_sectors/2);
5803	else
5804	return sprintf(page, "default\n");
5805	}
5806
5807	static ssize_t
5808	array_size_store(struct mddev mddev, const char buf, size_t len)
5809	{
5810	sector_t sectors;
5811	int err;
5812
5813	err = mddev_lock(mddev);
5814	if (err)
5815	return err;
5816
5817	/* cluster raid doesn't support change array_sectors */
5818	if (mddev_is_clustered(mddev)) {
5819	mddev_unlock(mddev);
5820	return -EINVAL;
5821	}
5822
5823	if (strncmp(buf, "default", 7) == 0) {
5824	if (mddev->pers)
5825	sectors = mddev->pers->size(mddev, 0, 0);
5826	else
5827	sectors = mddev->array_sectors;
5828
5829	mddev->external_size = 0;
5830	} else {
5831	if (strict_blocks_to_sectors(buf, &sectors) < 0)
5832	err = -EINVAL;
5833	else if (mddev->pers && mddev->pers->size(mddev, 0, 0) < sectors)
5834	err = -E2BIG;
5835	else
5836	mddev->external_size = 1;
5837	}
5838
5839	if (!err) {
5840	mddev->array_sectors = sectors;
5841	if (mddev->pers)
5842	set_capacity_and_notify(mddev->gendisk,
5843	mddev->array_sectors);
5844	}
5845	mddev_unlock(mddev);
5846	return err ?: len;
5847	}
5848
5849	static struct md_sysfs_entry md_array_size =
5850	__ATTR(array_size, S_IRUGO\|S_IWUSR, array_size_show,
5851	array_size_store);
5852
5853	static ssize_t
5854	consistency_policy_show(struct mddev mddev, char page)
5855	{
5856	int ret;
5857
5858	if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
5859	ret = sprintf(page, "journal\n");
5860	} else if (test_bit(MD_HAS_PPL, &mddev->flags)) {
5861	ret = sprintf(page, "ppl\n");
5862	} else if (mddev->bitmap) {
5863	ret = sprintf(page, "bitmap\n");
5864	} else if (mddev->pers) {
5865	if (mddev->pers->sync_request)
5866	ret = sprintf(page, "resync\n");
5867	else
5868	ret = sprintf(page, "none\n");
5869	} else {
5870	ret = sprintf(page, "unknown\n");
5871	}
5872
5873	return ret;
5874	}
5875
5876	static ssize_t
5877	consistency_policy_store(struct mddev mddev, const char buf, size_t len)
5878	{
5879	int err = 0;
5880
5881	if (mddev->pers) {
5882	if (mddev->pers->change_consistency_policy)
5883	err = mddev->pers->change_consistency_policy(mddev, buf);
5884	else
5885	err = -EBUSY;
5886	} else if (mddev->external && strncmp(buf, "ppl", 3) == 0) {
5887	set_bit(MD_HAS_PPL, &mddev->flags);
5888	} else {
5889	err = -EINVAL;
5890	}
5891
5892	return err ? err : len;
5893	}
5894
5895	static struct md_sysfs_entry md_consistency_policy =
5896	__ATTR(consistency_policy, S_IRUGO \| S_IWUSR, consistency_policy_show,
5897	consistency_policy_store);
5898
5899	static ssize_t fail_last_dev_show(struct mddev mddev, char page)
5900	{
5901	return sprintf(page, "%d\n", test_bit(MD_FAILLAST_DEV, &mddev->flags));
5902	}
5903
5904	/*
5905	* Setting MD_FAILLAST_DEV to allow last device to be forcibly removed
5906	* from RAID1/RAID10.
5907	*/
5908	static ssize_t
5909	fail_last_dev_store(struct mddev mddev, const char buf, size_t len)
5910	{
5911	int ret;
5912	bool value;
5913
5914	ret = kstrtobool(buf, &value);
5915	if (ret)
5916	return ret;
5917
5918	if (value)
5919	set_bit(MD_FAILLAST_DEV, &mddev->flags);
5920	else
5921	clear_bit(MD_FAILLAST_DEV, &mddev->flags);
5922
5923	return len;
5924	}
5925	static struct md_sysfs_entry md_fail_last_dev =
5926	__ATTR(fail_last_dev, S_IRUGO \| S_IWUSR, fail_last_dev_show,
5927	fail_last_dev_store);
5928
5929	static ssize_t serialize_policy_show(struct mddev mddev, char page)
5930	{
5931	if (mddev->pers == NULL \|\| (mddev->pers->head.id != ID_RAID1))
5932	return sprintf(page, "n/a\n");
5933	else
5934	return sprintf(page, "%d\n",
5935	test_bit(MD_SERIALIZE_POLICY, &mddev->flags));
5936	}
5937
5938	/*
5939	* Setting MD_SERIALIZE_POLICY enforce write IO is not reordered
5940	* for raid1.
5941	*/
5942	static ssize_t
5943	serialize_policy_store(struct mddev mddev, const char buf, size_t len)
5944	{
5945	int err;
5946	bool value;
5947
5948	err = kstrtobool(buf, &value);
5949	if (err)
5950	return err;
5951
5952	if (value == test_bit(MD_SERIALIZE_POLICY, &mddev->flags))
5953	return len;
5954
5955	err = mddev_suspend_and_lock(mddev);
5956	if (err)
5957	return err;
5958	if (mddev->pers == NULL \|\| (mddev->pers->head.id != ID_RAID1)) {
5959	pr_err("md: serialize_policy is only effective for raid1\n");
5960	err = -EINVAL;
5961	goto unlock;
5962	}
5963
5964	if (value) {
5965	mddev_create_serial_pool(mddev, NULL);
5966	set_bit(MD_SERIALIZE_POLICY, &mddev->flags);
5967	} else {
5968	mddev_destroy_serial_pool(mddev, NULL);
5969	clear_bit(MD_SERIALIZE_POLICY, &mddev->flags);
5970	}
5971	unlock:
5972	mddev_unlock_and_resume(mddev);
5973	return err ?: len;
5974	}
5975
5976	static struct md_sysfs_entry md_serialize_policy =
5977	__ATTR(serialize_policy, S_IRUGO \| S_IWUSR, serialize_policy_show,
5978	serialize_policy_store);
5979
5980	static int mddev_set_logical_block_size(struct mddev *mddev,
5981	unsigned int lbs)
5982	{
5983	int err = 0;
5984	struct queue_limits lim;
5985
5986	if (queue_logical_block_size(mddev->gendisk->queue) >= lbs) {
5987	pr_err("%s: Cannot set LBS smaller than mddev LBS %u\n",
5988	mdname(mddev), lbs);
5989	return -EINVAL;
5990	}
5991
5992	lim = queue_limits_start_update(mddev->gendisk->queue);
5993	lim.logical_block_size = lbs;
5994	pr_info("%s: logical_block_size is changed, data may be lost\n",
5995	mdname(mddev));
5996	err = queue_limits_commit_update(mddev->gendisk->queue, &lim);
5997	if (err)
5998	return err;
5999
6000	mddev->logical_block_size = lbs;
6001	/* New lbs will be written to superblock after array is running */
6002	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
6003	return 0;
6004	}
6005
6006	static ssize_t
6007	lbs_show(struct mddev mddev, char page)
6008	{
6009	return sprintf(page, "%u\n", mddev->logical_block_size);
6010	}
6011
6012	static ssize_t
6013	lbs_store(struct mddev mddev, const char buf, size_t len)
6014	{
6015	unsigned int lbs;
6016	int err = -EBUSY;
6017
6018	/* Only 1.x meta supports configurable LBS */
6019	if (mddev->major_version == 0)
6020	return -EINVAL;
6021
6022	err = kstrtouint(buf, 10, &lbs);
6023	if (err < 0)
6024	return -EINVAL;
6025
6026	if (mddev->pers) {
6027	unsigned int curr_lbs;
6028
6029	if (mddev->logical_block_size)
6030	return -EBUSY;
6031	/*
6032	* To fix forward compatibility issues, LBS is not
6033	* configured for arrays from old kernels (<=6.18) by default.
6034	* If the user confirms no rollback to old kernels,
6035	* enable LBS by writing current LBS — to prevent data
6036	* loss from LBS changes.
6037	*/
6038	curr_lbs = queue_logical_block_size(mddev->gendisk->queue);
6039	if (lbs != curr_lbs)
6040	return -EINVAL;
6041
6042	mddev->logical_block_size = curr_lbs;
6043	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
6044	pr_info("%s: logical block size configured successfully, array will not be assembled in old kernels (<= 6.18)\n",
6045	mdname(mddev));
6046	return len;
6047	}
6048
6049	err = mddev_lock(mddev);
6050	if (err)
6051	goto unlock;
6052
6053	err = mddev_set_logical_block_size(mddev, lbs);
6054
6055	unlock:
6056	mddev_unlock(mddev);
6057	return err ?: len;
6058	}
6059
6060	static struct md_sysfs_entry md_logical_block_size =
6061	__ATTR(logical_block_size, 0644, lbs_show, lbs_store);
6062
6063	static struct attribute *md_default_attrs[] = {
6064	&md_level.attr,
6065	&md_new_level.attr,
6066	&md_bitmap_type.attr,
6067	&md_layout.attr,
6068	&md_raid_disks.attr,
6069	&md_uuid.attr,
6070	&md_chunk_size.attr,
6071	&md_size.attr,
6072	&md_resync_start.attr,
6073	&md_metadata.attr,
6074	&md_new_device.attr,
6075	&md_safe_delay.attr,
6076	&md_array_state.attr,
6077	&md_reshape_position.attr,
6078	&md_reshape_direction.attr,
6079	&md_array_size.attr,
6080	&max_corr_read_errors.attr,
6081	&md_consistency_policy.attr,
6082	&md_fail_last_dev.attr,
6083	&md_serialize_policy.attr,
6084	&md_logical_block_size.attr,
6085	NULL,
6086	};
6087	ATTRIBUTE_GROUPS(md_default);
6088
6089	static struct attribute *md_redundancy_attrs[] = {
6090	&md_scan_mode.attr,
6091	&md_last_scan_mode.attr,
6092	&md_mismatches.attr,
6093	&md_sync_min.attr,
6094	&md_sync_max.attr,
6095	&md_sync_io_depth.attr,
6096	&md_sync_speed.attr,
6097	&md_sync_force_parallel.attr,
6098	&md_sync_completed.attr,
6099	&md_min_sync.attr,
6100	&md_max_sync.attr,
6101	&md_suspend_lo.attr,
6102	&md_suspend_hi.attr,
6103	&md_bitmap.attr,
6104	&md_degraded.attr,
6105	NULL,
6106	};
6107	static const struct attribute_group md_redundancy_group = {
6108	.name = NULL,
6109	.attrs = md_redundancy_attrs,
6110	};
6111
6112	static ssize_t
6113	md_attr_show(struct kobject kobj, struct attribute attr, char *page)
6114	{
6115	struct md_sysfs_entry *entry = container_of(attr, struct md_sysfs_entry, attr);
6116	struct mddev *mddev = container_of(kobj, struct mddev, kobj);
6117	ssize_t rv;
6118
6119	if (!entry->show)
6120	return -EIO;
6121	spin_lock(&all_mddevs_lock);
6122	if (!mddev_get(mddev)) {
6123	spin_unlock(&all_mddevs_lock);
6124	return -EBUSY;
6125	}
6126	spin_unlock(&all_mddevs_lock);
6127
6128	rv = entry->show(mddev, page);
6129	mddev_put(mddev);
6130	return rv;
6131	}
6132
6133	static ssize_t
6134	md_attr_store(struct kobject kobj, struct attribute attr,
6135	const char *page, size_t length)
6136	{
6137	struct md_sysfs_entry *entry = container_of(attr, struct md_sysfs_entry, attr);
6138	struct mddev *mddev = container_of(kobj, struct mddev, kobj);
6139	ssize_t rv;
6140	struct kernfs_node *kn = NULL;
6141
6142	if (!entry->store)
6143	return -EIO;
6144	if (!capable(CAP_SYS_ADMIN))
6145	return -EACCES;
6146
6147	if (entry->store == array_state_store && cmd_match(page, "clear"))
6148	kn = sysfs_break_active_protection(kobj, attr);
6149
6150	spin_lock(&all_mddevs_lock);
6151	if (!mddev_get(mddev)) {
6152	spin_unlock(&all_mddevs_lock);
6153	if (kn)
6154	sysfs_unbreak_active_protection(kn);
6155	return -EBUSY;
6156	}
6157	spin_unlock(&all_mddevs_lock);
6158	rv = entry->store(mddev, page, length);
6159
6160	/*
6161	* For "array_state=clear", dropping the extra kobject reference from
6162	* sysfs_break_active_protection() can trigger md kobject deletion.
6163	* Restore active protection before mddev_put() so deletion happens
6164	* after the sysfs write path fully unwinds.
6165	*/
6166	if (kn)
6167	sysfs_unbreak_active_protection(kn);
6168	mddev_put(mddev);
6169
6170	return rv;
6171	}
6172
6173	static void md_kobj_release(struct kobject *ko)
6174	{
6175	struct mddev *mddev = container_of(ko, struct mddev, kobj);
6176
6177	if (legacy_async_del_gendisk) {
6178	if (mddev->sysfs_state)
6179	sysfs_put(mddev->sysfs_state);
6180	if (mddev->sysfs_level)
6181	sysfs_put(mddev->sysfs_level);
6182	del_gendisk(mddev->gendisk);
6183	}
6184	put_disk(mddev->gendisk);
6185	}
6186
6187	static const struct sysfs_ops md_sysfs_ops = {
6188	.show = md_attr_show,
6189	.store = md_attr_store,
6190	};
6191	static const struct kobj_type md_ktype = {
6192	.release = md_kobj_release,
6193	.sysfs_ops = &md_sysfs_ops,
6194	.default_groups = md_default_groups,
6195	};
6196
6197	int mdp_major = 0;
6198
6199	/* stack the limit for all rdevs into lim */
6200	int mddev_stack_rdev_limits(struct mddev mddev, struct queue_limits lim,
6201	unsigned int flags)
6202	{
6203	struct md_rdev *rdev;
6204
6205	rdev_for_each(rdev, mddev) {
6206	queue_limits_stack_bdev(lim, rdev->bdev, rdev->data_offset,
6207	mddev->gendisk->disk_name);
6208	if ((flags & MDDEV_STACK_INTEGRITY) &&
6209	!queue_limits_stack_integrity_bdev(lim, rdev->bdev))
6210	return -EINVAL;
6211	}
6212
6213	/*
6214	* Before RAID adding folio support, the logical_block_size
6215	* should be smaller than the page size.
6216	*/
6217	if (lim->logical_block_size > PAGE_SIZE) {
6218	pr_err("%s: logical_block_size must not larger than PAGE_SIZE\n",
6219	mdname(mddev));
6220	return -EINVAL;
6221	}
6222
6223	/* Only 1.x meta needs to set logical block size */
6224	if (mddev->major_version == 0)
6225	return 0;
6226
6227	/*
6228	* Fix forward compatibility issue. Only set LBS by default for
6229	* new arrays, mddev->events == 0 indicates the array was just
6230	* created. When assembling an array, read LBS from the superblock
6231	* instead — LBS is 0 in superblocks created by old kernels.
6232	*/
6233	if (!mddev->events) {
6234	pr_info("%s: array will not be assembled in old kernels that lack configurable LBS support (<= 6.18)\n",
6235	mdname(mddev));
6236	mddev->logical_block_size = lim->logical_block_size;
6237	}
6238
6239	if (!mddev->logical_block_size)
6240	pr_warn("%s: echo current LBS to md/logical_block_size to prevent data loss issues from LBS changes.\n"
6241	"\tNote: After setting, array will not be assembled in old kernels (<= 6.18)\n",
6242	mdname(mddev));
6243
6244	return 0;
6245	}
6246	EXPORT_SYMBOL_GPL(mddev_stack_rdev_limits);
6247
6248	/* apply the extra stacking limits from a new rdev into mddev */
6249	int mddev_stack_new_rdev(struct mddev mddev, struct md_rdev rdev)
6250	{
6251	struct queue_limits lim;
6252
6253	if (mddev_is_dm(mddev))
6254	return 0;
6255
6256	if (queue_logical_block_size(rdev->bdev->bd_disk->queue) >
6257	queue_logical_block_size(mddev->gendisk->queue)) {
6258	pr_err("%s: incompatible logical_block_size, can not add\n",
6259	mdname(mddev));
6260	return -EINVAL;
6261	}
6262
6263	lim = queue_limits_start_update(mddev->gendisk->queue);
6264	queue_limits_stack_bdev(&lim, rdev->bdev, rdev->data_offset,
6265	mddev->gendisk->disk_name);
6266
6267	if (!queue_limits_stack_integrity_bdev(&lim, rdev->bdev)) {
6268	pr_err("%s: incompatible integrity profile for %pg\n",
6269	mdname(mddev), rdev->bdev);
6270	queue_limits_cancel_update(mddev->gendisk->queue);
6271	return -ENXIO;
6272	}
6273
6274	return queue_limits_commit_update(mddev->gendisk->queue, &lim);
6275	}
6276	EXPORT_SYMBOL_GPL(mddev_stack_new_rdev);
6277
6278	/* update the optimal I/O size after a reshape */
6279	void mddev_update_io_opt(struct mddev *mddev, unsigned int nr_stripes)
6280	{
6281	struct queue_limits lim;
6282
6283	if (mddev_is_dm(mddev))
6284	return;
6285
6286	/* don't bother updating io_opt if we can't suspend the array */
6287	if (mddev_suspend(mddev, false) < 0)
6288	return;
6289	lim = queue_limits_start_update(mddev->gendisk->queue);
6290	lim.io_opt = lim.io_min * nr_stripes;
6291	queue_limits_commit_update(mddev->gendisk->queue, &lim);
6292	mddev_resume(mddev);
6293	}
6294	EXPORT_SYMBOL_GPL(mddev_update_io_opt);
6295
6296	static void mddev_delayed_delete(struct work_struct *ws)
6297	{
6298	struct mddev *mddev = container_of(ws, struct mddev, del_work);
6299
6300	kobject_put(&mddev->kobj);
6301	}
6302
6303	void md_init_stacking_limits(struct queue_limits *lim)
6304	{
6305	blk_set_stacking_limits(lim);
6306	lim->features = BLK_FEAT_WRITE_CACHE \| BLK_FEAT_FUA \|
6307	BLK_FEAT_IO_STAT;
6308	}
6309	EXPORT_SYMBOL_GPL(md_init_stacking_limits);
6310
6311	struct mddev md_alloc(dev_t dev, char name)
6312	{
6313	/*
6314	* If dev is zero, name is the name of a device to allocate with
6315	* an arbitrary minor number. It will be "md_???"
6316	* If dev is non-zero it must be a device number with a MAJOR of
6317	* MD_MAJOR or mdp_major. In this case, if "name" is NULL, then
6318	* the device is being created by opening a node in /dev.
6319	* If "name" is not NULL, the device is being created by
6320	* writing to /sys/module/md_mod/parameters/new_array.
6321	*/
6322	static DEFINE_MUTEX(disks_mutex);
6323	struct mddev *mddev;
6324	struct gendisk *disk;
6325	int partitioned;
6326	int shift;
6327	int unit;
6328	int error;
6329
6330	/*
6331	* Wait for any previous instance of this device to be completely
6332	* removed (mddev_delayed_delete).
6333	*/
6334	flush_workqueue(md_misc_wq);
6335
6336	mutex_lock(&disks_mutex);
6337	mddev = mddev_alloc(dev);
6338	if (IS_ERR(mddev)) {
6339	error = PTR_ERR(mddev);
6340	goto out_unlock;
6341	}
6342
6343	partitioned = (MAJOR(mddev->unit) != MD_MAJOR);
6344	shift = partitioned ? MdpMinorShift : 0;
6345	unit = MINOR(mddev->unit) >> shift;
6346
6347	if (name && !dev) {
6348	/* Need to ensure that 'name' is not a duplicate.
6349	*/
6350	struct mddev *mddev2;
6351	spin_lock(&all_mddevs_lock);
6352
6353	list_for_each_entry(mddev2, &all_mddevs, all_mddevs)
6354	if (mddev2->gendisk &&
6355	strcmp(mddev2->gendisk->disk_name, name) == 0) {
6356	spin_unlock(&all_mddevs_lock);
6357	error = -EEXIST;
6358	goto out_free_mddev;
6359	}
6360	spin_unlock(&all_mddevs_lock);
6361	}
6362	if (name && dev)
6363	/*
6364	* Creating /dev/mdNNN via "newarray", so adjust hold_active.
6365	*/
6366	mddev->hold_active = UNTIL_STOP;
6367
6368	disk = blk_alloc_disk(NULL, NUMA_NO_NODE);
6369	if (IS_ERR(disk)) {
6370	error = PTR_ERR(disk);
6371	goto out_free_mddev;
6372	}
6373
6374	disk->major = MAJOR(mddev->unit);
6375	disk->first_minor = unit << shift;
6376	disk->minors = 1 << shift;
6377	if (name)
6378	strcpy(disk->disk_name, name);
6379	else if (partitioned)
6380	sprintf(disk->disk_name, "md_d%d", unit);
6381	else
6382	sprintf(disk->disk_name, "md%d", unit);
6383	disk->fops = &md_fops;
6384	disk->private_data = mddev;
6385
6386	disk->events \|= DISK_EVENT_MEDIA_CHANGE;
6387	mddev->gendisk = disk;
6388	error = add_disk(disk);
6389	if (error)
6390	goto out_put_disk;
6391
6392	kobject_init(&mddev->kobj, &md_ktype);
6393	error = kobject_add(&mddev->kobj, &disk_to_dev(disk)->kobj, "%s", "md");
6394	if (error) {
6395	/*
6396	* The disk is already live at this point. Clear the hold flag
6397	* and let mddev_put take care of the deletion, as it isn't any
6398	* different from a normal close on last release now.
6399	*/
6400	mddev->hold_active = 0;
6401	mutex_unlock(&disks_mutex);
6402	mddev_put(mddev);
6403	return ERR_PTR(error);
6404	}
6405
6406	kobject_uevent(&mddev->kobj, KOBJ_ADD);
6407	mddev->sysfs_state = sysfs_get_dirent_safe(mddev->kobj.sd, "array_state");
6408	mddev->sysfs_level = sysfs_get_dirent_safe(mddev->kobj.sd, "level");
6409	mutex_unlock(&disks_mutex);
6410	return mddev;
6411
6412	out_put_disk:
6413	put_disk(disk);
6414	out_free_mddev:
6415	mddev_free(mddev);
6416	out_unlock:
6417	mutex_unlock(&disks_mutex);
6418	return ERR_PTR(error);
6419	}
6420
6421	static int md_alloc_and_put(dev_t dev, char *name)
6422	{
6423	struct mddev *mddev = md_alloc(dev, name);
6424
6425	if (legacy_async_del_gendisk)
6426	pr_warn("md: async del_gendisk mode will be removed in future, please upgrade to mdadm-4.5+\n");
6427
6428	if (IS_ERR(mddev))
6429	return PTR_ERR(mddev);
6430	mddev_put(mddev);
6431	return 0;
6432	}
6433
6434	static void md_probe(dev_t dev)
6435	{
6436	if (MAJOR(dev) == MD_MAJOR && MINOR(dev) >= 512)
6437	return;
6438	if (create_on_open)
6439	md_alloc_and_put(dev, NULL);
6440	}
6441
6442	static int add_named_array(const char val, const struct kernel_param kp)
6443	{
6444	/*
6445	* val must be "md_*" or "mdNNN".
6446	* For "md_*" we allocate an array with a large free minor number, and
6447	* set the name to val. val must not already be an active name.
6448	* For "mdNNN" we allocate an array with the minor number NNN
6449	* which must not already be in use.
6450	*/
6451	int len = strlen(val);
6452	char buf[DISK_NAME_LEN];
6453	unsigned long devnum;
6454
6455	while (len && val[len-1] == '\n')
6456	len--;
6457	if (len >= DISK_NAME_LEN)
6458	return -E2BIG;
6459	strscpy(buf, val, len+1);
6460	if (strncmp(buf, "md_", 3) == 0)
6461	return md_alloc_and_put(0, buf);
6462	if (strncmp(buf, "md", 2) == 0 &&
6463	isdigit(buf[2]) &&
6464	kstrtoul(buf+2, 10, &devnum) == 0 &&
6465	devnum <= MINORMASK)
6466	return md_alloc_and_put(MKDEV(MD_MAJOR, devnum), NULL);
6467
6468	return -EINVAL;
6469	}
6470
6471	static void md_safemode_timeout(struct timer_list *t)
6472	{
6473	struct mddev *mddev = timer_container_of(mddev, t, safemode_timer);
6474
6475	mddev->safemode = 1;
6476	if (mddev->external)
6477	sysfs_notify_dirent_safe(mddev->sysfs_state);
6478
6479	md_wakeup_thread(mddev->thread);
6480	}
6481
6482	static int start_dirty_degraded;
6483
6484	/*
6485	* Read bitmap superblock and return the bitmap_id based on disk version.
6486	* This is used as fallback when default bitmap version and on-disk version
6487	* doesn't match, and mdadm is not the latest version to set bitmap_type.
6488	*/
6489	static enum md_submodule_id md_bitmap_get_id_from_sb(struct mddev *mddev)
6490	{
6491	struct md_rdev *rdev;
6492	struct page *sb_page;
6493	bitmap_super_t *sb;
6494	enum md_submodule_id id = ID_BITMAP_NONE;
6495	sector_t sector;
6496	u32 version;
6497
6498	if (!mddev->bitmap_info.offset)
6499	return ID_BITMAP_NONE;
6500
6501	sb_page = alloc_page(GFP_KERNEL);
6502	if (!sb_page) {
6503	pr_warn("md: %s: failed to allocate memory for bitmap\n",
6504	mdname(mddev));
6505	return ID_BITMAP_NONE;
6506	}
6507
6508	sector = mddev->bitmap_info.offset;
6509
6510	rdev_for_each(rdev, mddev) {
6511	u32 iosize;
6512
6513	if (!test_bit(In_sync, &rdev->flags) \|\|
6514	test_bit(Faulty, &rdev->flags) \|\|
6515	test_bit(Bitmap_sync, &rdev->flags))
6516	continue;
6517
6518	iosize = roundup(sizeof(bitmap_super_t),
6519	bdev_logical_block_size(rdev->bdev));
6520	if (sync_page_io(rdev, sector, iosize, sb_page, REQ_OP_READ,
6521	true))
6522	goto read_ok;
6523	}
6524	pr_warn("md: %s: failed to read bitmap from any device\n",
6525	mdname(mddev));
6526	goto out;
6527
6528	read_ok:
6529	sb = kmap_local_page(sb_page);
6530	if (sb->magic != cpu_to_le32(BITMAP_MAGIC)) {
6531	pr_warn("md: %s: invalid bitmap magic 0x%x\n",
6532	mdname(mddev), le32_to_cpu(sb->magic));
6533	goto out_unmap;
6534	}
6535
6536	version = le32_to_cpu(sb->version);
6537	switch (version) {
6538	case BITMAP_MAJOR_LO:
6539	case BITMAP_MAJOR_HI:
6540	case BITMAP_MAJOR_CLUSTERED:
6541	id = ID_BITMAP;
6542	break;
6543	case BITMAP_MAJOR_LOCKLESS:
6544	id = ID_LLBITMAP;
6545	break;
6546	default:
6547	pr_warn("md: %s: unknown bitmap version %u\n",
6548	mdname(mddev), version);
6549	break;
6550	}
6551
6552	out_unmap:
6553	kunmap_local(sb);
6554	out:
6555	__free_page(sb_page);
6556	return id;
6557	}
6558
6559	int md_bitmap_create_nosysfs(struct mddev *mddev)
6560	{
6561	enum md_submodule_id orig_id = mddev->bitmap_id;
6562	enum md_submodule_id sb_id;
6563	int err;
6564
6565	if (mddev->bitmap_id == ID_BITMAP_NONE)
6566	return -EINVAL;
6567
6568	if (!mddev_set_bitmap_ops_nosysfs(mddev)) {
6569	mddev->bitmap_id = orig_id;
6570	return -ENOENT;
6571	}
6572
6573	err = mddev->bitmap_ops->create(mddev);
6574	if (!err)
6575	return 0;
6576
6577	/*
6578	* Create failed, if default bitmap version and on-disk version
6579	* doesn't match, and mdadm is not the latest version to set
6580	* bitmap_type, set bitmap_ops based on the disk version.
6581	*/
6582	mddev->bitmap_ops = NULL;
6583
6584	sb_id = md_bitmap_get_id_from_sb(mddev);
6585	if (sb_id == ID_BITMAP_NONE \|\| sb_id == orig_id) {
6586	mddev->bitmap_id = orig_id;
6587	return err;
6588	}
6589
6590	pr_info("md: %s: bitmap version mismatch, switching from %d to %d\n",
6591	mdname(mddev), orig_id, sb_id);
6592
6593	mddev->bitmap_id = sb_id;
6594	if (!mddev_set_bitmap_ops_nosysfs(mddev)) {
6595	mddev->bitmap_id = orig_id;
6596	return -ENOENT;
6597	}
6598
6599	err = mddev->bitmap_ops->create(mddev);
6600	if (err) {
6601	mddev->bitmap_ops = NULL;
6602	mddev->bitmap_id = orig_id;
6603	}
6604
6605	return err;
6606	}
6607
6608	static int md_bitmap_create(struct mddev *mddev)
6609	{
6610	int err;
6611
6612	err = md_bitmap_create_nosysfs(mddev);
6613	if (err)
6614	return err;
6615
6616	if (!mddev_is_dm(mddev) && mddev->bitmap_ops->groups)
6617	md_bitmap_sysfs_add(mddev);
6618
6619	return 0;
6620	}
6621
6622	void md_bitmap_destroy_nosysfs(struct mddev *mddev)
6623	{
6624	if (!md_bitmap_registered(mddev))
6625	return;
6626
6627	mddev->bitmap_ops->destroy(mddev);
6628	mddev->bitmap_ops = NULL;
6629	}
6630
6631	static void md_bitmap_destroy(struct mddev *mddev)
6632	{
6633	if (!mddev_is_dm(mddev) && mddev->bitmap_ops &&
6634	mddev->bitmap_ops->groups)
6635	md_bitmap_sysfs_del(mddev);
6636
6637	md_bitmap_destroy_nosysfs(mddev);
6638	}
6639
6640	static void md_bitmap_set_none(struct mddev *mddev)
6641	{
6642	mddev->bitmap_id = ID_BITMAP_NONE;
6643	if (!mddev_set_bitmap_ops_nosysfs(mddev))
6644	return;
6645
6646	if (!mddev_is_dm(mddev) && mddev->bitmap_ops->groups)
6647	md_bitmap_sysfs_add(mddev);
6648	}
6649
6650	int md_run(struct mddev *mddev)
6651	{
6652	int err;
6653	struct md_rdev *rdev;
6654	struct md_personality *pers;
6655
6656	if (list_empty(&mddev->disks))
6657	/* cannot run an array with no devices.. */
6658	return -EINVAL;
6659
6660	if (mddev->pers)
6661	return -EBUSY;
6662	/* Cannot run until previous stop completes properly */
6663	if (mddev->sysfs_active)
6664	return -EBUSY;
6665
6666	/*
6667	* Analyze all RAID superblock(s)
6668	*/
6669	if (!mddev->raid_disks) {
6670	if (!mddev->persistent)
6671	return -EINVAL;
6672	err = analyze_sbs(mddev);
6673	if (err)
6674	return -EINVAL;
6675	}
6676
6677	if (mddev->level != LEVEL_NONE)
6678	request_module("md-level-%d", mddev->level);
6679	else if (mddev->clevel[0])
6680	request_module("md-%s", mddev->clevel);
6681
6682	/*
6683	* Drop all container device buffers, from now on
6684	* the only valid external interface is through the md
6685	* device.
6686	*/
6687	clear_bit(MD_HAS_SUPERBLOCK, &mddev->flags);
6688	rdev_for_each(rdev, mddev) {
6689	if (test_bit(Faulty, &rdev->flags))
6690	continue;
6691	sync_blockdev(rdev->bdev);
6692	invalidate_bdev(rdev->bdev);
6693	if (mddev->ro != MD_RDONLY && rdev_read_only(rdev)) {
6694	mddev->ro = MD_RDONLY;
6695	if (!mddev_is_dm(mddev))
6696	set_disk_ro(mddev->gendisk, 1);
6697	}
6698
6699	if (rdev->sb_page)
6700	set_bit(MD_HAS_SUPERBLOCK, &mddev->flags);
6701
6702	/* perform some consistency tests on the device.
6703	* We don't want the data to overlap the metadata,
6704	* Internal Bitmap issues have been handled elsewhere.
6705	*/
6706	if (rdev->meta_bdev) {
6707	/* Nothing to check */;
6708	} else if (rdev->data_offset < rdev->sb_start) {
6709	if (mddev->dev_sectors &&
6710	rdev->data_offset + mddev->dev_sectors
6711	> rdev->sb_start) {
6712	pr_warn("md: %s: data overlaps metadata\n",
6713	mdname(mddev));
6714	return -EINVAL;
6715	}
6716	} else {
6717	if (rdev->sb_start + rdev->sb_size/512
6718	> rdev->data_offset) {
6719	pr_warn("md: %s: metadata overlaps data\n",
6720	mdname(mddev));
6721	return -EINVAL;
6722	}
6723	}
6724	sysfs_notify_dirent_safe(rdev->sysfs_state);
6725	}
6726
6727	pers = get_pers(mddev->level, mddev->clevel);
6728	if (!pers)
6729	return -EINVAL;
6730	if (mddev->level != pers->head.id) {
6731	mddev->level = pers->head.id;
6732	mddev->new_level = pers->head.id;
6733	}
6734	strscpy(mddev->clevel, pers->head.name, sizeof(mddev->clevel));
6735
6736	if (mddev->reshape_position != MaxSector &&
6737	pers->start_reshape == NULL) {
6738	/* This personality cannot handle reshaping... */
6739	put_pers(pers);
6740	return -EINVAL;
6741	}
6742
6743	if (pers->sync_request) {
6744	/* Warn if this is a potentially silly
6745	* configuration.
6746	*/
6747	struct md_rdev *rdev2;
6748	int warned = 0;
6749
6750	rdev_for_each(rdev, mddev)
6751	rdev_for_each(rdev2, mddev) {
6752	if (rdev < rdev2 &&
6753	rdev->bdev->bd_disk ==
6754	rdev2->bdev->bd_disk) {
6755	pr_warn("%s: WARNING: %pg appears to be on the same physical disk as %pg.\n",
6756	mdname(mddev),
6757	rdev->bdev,
6758	rdev2->bdev);
6759	warned = 1;
6760	}
6761	}
6762
6763	if (warned)
6764	pr_warn("True protection against single-disk failure might be compromised.\n");
6765	}
6766
6767	/* dm-raid expect sync_thread to be frozen until resume */
6768	if (!mddev_is_dm(mddev))
6769	mddev->recovery = 0;
6770
6771	/* may be over-ridden by personality */
6772	mddev->resync_max_sectors = mddev->dev_sectors;
6773
6774	mddev->ok_start_degraded = start_dirty_degraded;
6775
6776	if (start_readonly && md_is_rdwr(mddev))
6777	mddev->ro = MD_AUTO_READ; /* read-only, but switch on first write */
6778
6779	err = pers->run(mddev);
6780	if (err)
6781	pr_warn("md: pers->run() failed ...\n");
6782	else if (pers->size(mddev, 0, 0) < mddev->array_sectors) {
6783	WARN_ONCE(!mddev->external_size,
6784	"%s: default size too small, but 'external_size' not in effect?\n",
6785	__func__);
6786	pr_warn("md: invalid array_size %llu > default size %llu\n",
6787	(unsigned long long)mddev->array_sectors / 2,
6788	(unsigned long long)pers->size(mddev, 0, 0) / 2);
6789	err = -EINVAL;
6790	}
6791	if (err == 0 && pers->sync_request &&
6792	(mddev->bitmap_info.file \|\| mddev->bitmap_info.offset)) {
6793	err = md_bitmap_create(mddev);
6794	if (err)
6795	pr_warn("%s: failed to create bitmap (%d)\n",
6796	mdname(mddev), err);
6797	}
6798	if (err)
6799	goto bitmap_abort;
6800
6801	if (mddev->bitmap_info.max_write_behind > 0) {
6802	bool create_pool = false;
6803
6804	rdev_for_each(rdev, mddev) {
6805	if (test_bit(WriteMostly, &rdev->flags) &&
6806	rdev_init_serial(rdev))
6807	create_pool = true;
6808	}
6809	if (create_pool && mddev->serial_info_pool == NULL) {
6810	mddev->serial_info_pool =
6811	mempool_create_kmalloc_pool(NR_SERIAL_INFOS,
6812	sizeof(struct serial_info));
6813	if (!mddev->serial_info_pool) {
6814	err = -ENOMEM;
6815	goto bitmap_abort;
6816	}
6817	}
6818	}
6819
6820	if (pers->sync_request) {
6821	if (mddev->kobj.sd &&
6822	sysfs_create_group(&mddev->kobj, &md_redundancy_group))
6823	pr_warn("md: cannot register extra attributes for %s\n",
6824	mdname(mddev));
6825	mddev->sysfs_action = sysfs_get_dirent_safe(mddev->kobj.sd, "sync_action");
6826	mddev->sysfs_completed = sysfs_get_dirent_safe(mddev->kobj.sd, "sync_completed");
6827	mddev->sysfs_degraded = sysfs_get_dirent_safe(mddev->kobj.sd, "degraded");
6828	} else if (mddev->ro == MD_AUTO_READ)
6829	mddev->ro = MD_RDWR;
6830
6831	atomic_set(&mddev->max_corr_read_errors,
6832	MD_DEFAULT_MAX_CORRECTED_READ_ERRORS);
6833	mddev->safemode = 0;
6834	if (mddev_is_clustered(mddev))
6835	mddev->safemode_delay = 0;
6836	else
6837	mddev->safemode_delay = DEFAULT_SAFEMODE_DELAY;
6838	mddev->in_sync = 1;
6839	smp_wmb();
6840	spin_lock(&mddev->lock);
6841	mddev->pers = pers;
6842	spin_unlock(&mddev->lock);
6843	rdev_for_each(rdev, mddev)
6844	if (rdev->raid_disk >= 0)
6845	sysfs_link_rdev(mddev, rdev); /* failure here is OK */
6846
6847	if (mddev->degraded && md_is_rdwr(mddev))
6848	/* This ensures that recovering status is reported immediately
6849	* via sysfs - until a lack of spares is confirmed.
6850	*/
6851	set_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
6852	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
6853
6854	if (mddev->sb_flags)
6855	md_update_sb(mddev, 0);
6856
6857	if (IS_ENABLED(CONFIG_MD_BITMAP) && !mddev->bitmap_info.file &&
6858	!mddev->bitmap_info.offset)
6859	md_bitmap_set_none(mddev);
6860
6861	md_new_event();
6862	return 0;
6863
6864	bitmap_abort:
6865	mddev_detach(mddev);
6866	if (mddev->private)
6867	pers->free(mddev, mddev->private);
6868	mddev->private = NULL;
6869	put_pers(pers);
6870	md_bitmap_destroy(mddev);
6871	return err;
6872	}
6873	EXPORT_SYMBOL_GPL(md_run);
6874
6875	int do_md_run(struct mddev *mddev)
6876	{
6877	int err;
6878
6879	set_bit(MD_NOT_READY, &mddev->flags);
6880	err = md_run(mddev);
6881	if (err)
6882	goto out;
6883
6884	if (md_bitmap_registered(mddev)) {
6885	err = mddev->bitmap_ops->load(mddev);
6886	if (err) {
6887	md_bitmap_destroy(mddev);
6888	goto out;
6889	}
6890	}
6891
6892	if (mddev_is_clustered(mddev))
6893	md_allow_write(mddev);
6894
6895	/* run start up tasks that require md_thread */
6896	md_start(mddev);
6897
6898	md_wakeup_thread(mddev->sync_thread); /* possibly kick off a reshape */
6899
6900	set_capacity_and_notify(mddev->gendisk, mddev->array_sectors);
6901	clear_bit(MD_NOT_READY, &mddev->flags);
6902	mddev->changed = 1;
6903	kobject_uevent(&disk_to_dev(mddev->gendisk)->kobj, KOBJ_CHANGE);
6904	sysfs_notify_dirent_safe(mddev->sysfs_state);
6905	sysfs_notify_dirent_safe(mddev->sysfs_action);
6906	sysfs_notify_dirent_safe(mddev->sysfs_degraded);
6907	out:
6908	clear_bit(MD_NOT_READY, &mddev->flags);
6909	return err;
6910	}
6911
6912	int md_start(struct mddev *mddev)
6913	{
6914	int ret = 0;
6915
6916	if (mddev->pers->start) {
6917	set_bit(MD_RECOVERY_WAIT, &mddev->recovery);
6918	ret = mddev->pers->start(mddev);
6919	clear_bit(MD_RECOVERY_WAIT, &mddev->recovery);
6920	md_wakeup_thread(mddev->sync_thread);
6921	}
6922	return ret;
6923	}
6924	EXPORT_SYMBOL_GPL(md_start);
6925
6926	static int restart_array(struct mddev *mddev)
6927	{
6928	struct gendisk *disk = mddev->gendisk;
6929	struct md_rdev *rdev;
6930	bool has_journal = false;
6931	bool has_readonly = false;
6932
6933	/* Complain if it has no devices */
6934	if (list_empty(&mddev->disks))
6935	return -ENXIO;
6936	if (!mddev->pers)
6937	return -EINVAL;
6938	if (md_is_rdwr(mddev))
6939	return -EBUSY;
6940
6941	rcu_read_lock();
6942	rdev_for_each_rcu(rdev, mddev) {
6943	if (test_bit(Journal, &rdev->flags) &&
6944	!test_bit(Faulty, &rdev->flags))
6945	has_journal = true;
6946	if (rdev_read_only(rdev))
6947	has_readonly = true;
6948	}
6949	rcu_read_unlock();
6950	if (test_bit(MD_HAS_JOURNAL, &mddev->flags) && !has_journal)
6951	/* Don't restart rw with journal missing/faulty */
6952	return -EINVAL;
6953	if (has_readonly)
6954	return -EROFS;
6955
6956	mddev->safemode = 0;
6957	mddev->ro = MD_RDWR;
6958	set_disk_ro(disk, 0);
6959	pr_debug("md: %s switched to read-write mode.\n", mdname(mddev));
6960	/* Kick recovery or resync if necessary */
6961	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
6962	md_wakeup_thread(mddev->sync_thread);
6963	sysfs_notify_dirent_safe(mddev->sysfs_state);
6964	return 0;
6965	}
6966
6967	static void md_clean(struct mddev *mddev)
6968	{
6969	mddev->array_sectors = 0;
6970	mddev->external_size = 0;
6971	mddev->dev_sectors = 0;
6972	mddev->raid_disks = 0;
6973	mddev->resync_offset = 0;
6974	mddev->resync_min = 0;
6975	mddev->resync_max = MaxSector;
6976	mddev->reshape_position = MaxSector;
6977	/* we still need mddev->external in export_rdev, do not clear it yet */
6978	mddev->persistent = 0;
6979	mddev->level = LEVEL_NONE;
6980	mddev->clevel[0] = 0;
6981
6982	/*
6983	* For legacy_async_del_gendisk mode, it can stop the array in the
6984	* middle of assembling it, then it still can access the array. So
6985	* it needs to clear MD_CLOSING. If not legacy_async_del_gendisk,
6986	* it can't open the array again after stopping it. So it doesn't
6987	* clear MD_CLOSING.
6988	*/
6989	if (legacy_async_del_gendisk && mddev->hold_active) {
6990	clear_bit(MD_CLOSING, &mddev->flags);
6991	} else {
6992	/* if UNTIL_STOP is set, it's cleared here */
6993	mddev->hold_active = 0;
6994	/* Don't clear MD_CLOSING, or mddev can be opened again. */
6995	mddev->flags &= BIT_ULL_MASK(MD_CLOSING);
6996	}
6997	mddev->sb_flags = 0;
6998	mddev->ro = MD_RDWR;
6999	mddev->metadata_type[0] = 0;
7000	mddev->chunk_sectors = 0;
7001	mddev->ctime = mddev->utime = 0;
7002	mddev->layout = 0;
7003	mddev->logical_block_size = 0;
7004	mddev->max_disks = 0;
7005	mddev->events = 0;
7006	mddev->can_decrease_events = 0;
7007	mddev->delta_disks = 0;
7008	mddev->reshape_backwards = 0;
7009	mddev->new_level = LEVEL_NONE;
7010	mddev->new_layout = 0;
7011	mddev->new_chunk_sectors = 0;
7012	mddev->curr_resync = MD_RESYNC_NONE;
7013	atomic64_set(&mddev->resync_mismatches, 0);
7014	mddev->suspend_lo = mddev->suspend_hi = 0;
7015	mddev->sync_speed_min = mddev->sync_speed_max = 0;
7016	mddev->recovery = 0;
7017	mddev->in_sync = 0;
7018	mddev->changed = 0;
7019	mddev->degraded = 0;
7020	mddev->safemode = 0;
7021	mddev->private = NULL;
7022	mddev->cluster_info = NULL;
7023	mddev->bitmap_info.offset = 0;
7024	mddev->bitmap_info.default_offset = 0;
7025	mddev->bitmap_info.default_space = 0;
7026	mddev->bitmap_info.chunksize = 0;
7027	mddev->bitmap_info.daemon_sleep = 0;
7028	mddev->bitmap_info.max_write_behind = 0;
7029	mddev->bitmap_info.nodes = 0;
7030	}
7031
7032	static void __md_stop_writes(struct mddev *mddev)
7033	{
7034	timer_delete_sync(&mddev->safemode_timer);
7035
7036	if (md_is_rdwr(mddev) \|\| !mddev_is_dm(mddev)) {
7037	if (mddev->pers && mddev->pers->quiesce) {
7038	mddev->pers->quiesce(mddev, 1);
7039	mddev->pers->quiesce(mddev, 0);
7040	}
7041
7042	if (md_bitmap_enabled(mddev, true))
7043	mddev->bitmap_ops->flush(mddev);
7044	}
7045
7046	if (md_is_rdwr(mddev) &&
7047	((!mddev->in_sync && !mddev_is_clustered(mddev)) \|\|
7048	mddev->sb_flags)) {
7049	/* mark array as shutdown cleanly */
7050	if (!mddev_is_clustered(mddev))
7051	mddev->in_sync = 1;
7052	md_update_sb(mddev, 1);
7053	}
7054	/* disable policy to guarantee rdevs free resources for serialization */
7055	clear_bit(MD_SERIALIZE_POLICY, &mddev->flags);
7056	mddev_destroy_serial_pool(mddev, NULL);
7057	}
7058
7059	void md_stop_writes(struct mddev *mddev)
7060	{
7061	mddev_lock_nointr(mddev);
7062	set_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
7063	stop_sync_thread(mddev, true);
7064	__md_stop_writes(mddev);
7065	mddev_unlock(mddev);
7066	}
7067	EXPORT_SYMBOL_GPL(md_stop_writes);
7068
7069	static void mddev_detach(struct mddev *mddev)
7070	{
7071	if (md_bitmap_enabled(mddev, false))
7072	mddev->bitmap_ops->wait_behind_writes(mddev);
7073	if (mddev->pers && mddev->pers->quiesce && !is_md_suspended(mddev)) {
7074	mddev->pers->quiesce(mddev, 1);
7075	mddev->pers->quiesce(mddev, 0);
7076	}
7077	md_unregister_thread(mddev, &mddev->thread);
7078
7079	/* the unplug fn references 'conf' */
7080	if (!mddev_is_dm(mddev))
7081	blk_sync_queue(mddev->gendisk->queue);
7082	}
7083
7084	static void __md_stop(struct mddev *mddev)
7085	{
7086	struct md_personality *pers = mddev->pers;
7087
7088	mddev_detach(mddev);
7089	md_bitmap_destroy(mddev);
7090	spin_lock(&mddev->lock);
7091	mddev->pers = NULL;
7092	spin_unlock(&mddev->lock);
7093	if (mddev->private)
7094	pers->free(mddev, mddev->private);
7095	mddev->private = NULL;
7096	put_pers(pers);
7097	clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
7098	}
7099
7100	void md_stop(struct mddev *mddev)
7101	{
7102	lockdep_assert_held(&mddev->reconfig_mutex);
7103
7104	/* stop the array and free an attached data structures.
7105	* This is called from dm-raid
7106	*/
7107	__md_stop_writes(mddev);
7108	__md_stop(mddev);
7109	}
7110
7111	EXPORT_SYMBOL_GPL(md_stop);
7112
7113	/* ensure 'mddev->pers' exist before calling md_set_readonly() */
7114	static int md_set_readonly(struct mddev *mddev)
7115	{
7116	int err = 0;
7117	int did_freeze = 0;
7118
7119	if (mddev->external && test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags))
7120	return -EBUSY;
7121
7122	if (!test_bit(MD_RECOVERY_FROZEN, &mddev->recovery)) {
7123	did_freeze = 1;
7124	set_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
7125	}
7126
7127	stop_sync_thread(mddev, false);
7128	wait_event(mddev->sb_wait,
7129	!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
7130	mddev_lock_nointr(mddev);
7131
7132	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) {
7133	pr_warn("md: %s still in use.\n",mdname(mddev));
7134	err = -EBUSY;
7135	goto out;
7136	}
7137
7138	__md_stop_writes(mddev);
7139
7140	if (mddev->ro == MD_RDONLY) {
7141	err = -ENXIO;
7142	goto out;
7143	}
7144
7145	mddev->ro = MD_RDONLY;
7146	set_disk_ro(mddev->gendisk, 1);
7147
7148	out:
7149	if (!err \|\| did_freeze) {
7150	clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
7151	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
7152	sysfs_notify_dirent_safe(mddev->sysfs_state);
7153	}
7154
7155	return err;
7156	}
7157
7158	/* mode:
7159	* 0 - completely stop and dis-assemble array
7160	* 2 - stop but do not disassemble array
7161	*/
7162	static int do_md_stop(struct mddev *mddev, int mode)
7163	{
7164	struct gendisk *disk = mddev->gendisk;
7165	struct md_rdev *rdev;
7166	int did_freeze = 0;
7167
7168	if (!test_bit(MD_RECOVERY_FROZEN, &mddev->recovery)) {
7169	did_freeze = 1;
7170	set_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
7171	}
7172
7173	stop_sync_thread(mddev, true);
7174
7175	if (mddev->sysfs_active \|\|
7176	test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) {
7177	pr_warn("md: %s still in use.\n",mdname(mddev));
7178	if (did_freeze) {
7179	clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
7180	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
7181	}
7182	return -EBUSY;
7183	}
7184	if (mddev->pers) {
7185	if (!md_is_rdwr(mddev))
7186	set_disk_ro(disk, 0);
7187
7188	if (mode == 2 && mddev->pers->sync_request &&
7189	mddev->to_remove == NULL)
7190	mddev->to_remove = &md_redundancy_group;
7191
7192	__md_stop_writes(mddev);
7193	__md_stop(mddev);
7194
7195	/* tell userspace to handle 'inactive' */
7196	sysfs_notify_dirent_safe(mddev->sysfs_state);
7197
7198	rdev_for_each(rdev, mddev)
7199	if (rdev->raid_disk >= 0)
7200	sysfs_unlink_rdev(mddev, rdev);
7201
7202	set_capacity_and_notify(disk, 0);
7203	mddev->changed = 1;
7204
7205	if (!md_is_rdwr(mddev))
7206	mddev->ro = MD_RDWR;
7207	}
7208	/*
7209	* Free resources if final stop
7210	*/
7211	if (mode == 0) {
7212	pr_info("md: %s stopped.\n", mdname(mddev));
7213
7214	if (mddev->bitmap_info.file) {
7215	struct file *f = mddev->bitmap_info.file;
7216	spin_lock(&mddev->lock);
7217	mddev->bitmap_info.file = NULL;
7218	spin_unlock(&mddev->lock);
7219	fput(f);
7220	}
7221	mddev->bitmap_info.offset = 0;
7222
7223	export_array(mddev);
7224	md_clean(mddev);
7225	if (!legacy_async_del_gendisk)
7226	set_bit(MD_DELETED, &mddev->flags);
7227	}
7228	md_new_event();
7229	sysfs_notify_dirent_safe(mddev->sysfs_state);
7230	return 0;
7231	}
7232
7233	#ifndef MODULE
7234	static void autorun_array(struct mddev *mddev)
7235	{
7236	struct md_rdev *rdev;
7237	int err;
7238
7239	if (list_empty(&mddev->disks))
7240	return;
7241
7242	pr_info("md: running: ");
7243
7244	rdev_for_each(rdev, mddev) {
7245	pr_cont("<%pg>", rdev->bdev);
7246	}
7247	pr_cont("\n");
7248
7249	err = do_md_run(mddev);
7250	if (err) {
7251	pr_warn("md: do_md_run() returned %d\n", err);
7252	do_md_stop(mddev, 0);
7253	}
7254	}
7255
7256	/*
7257	* lets try to run arrays based on all disks that have arrived
7258	* until now. (those are in pending_raid_disks)
7259	*
7260	* the method: pick the first pending disk, collect all disks with
7261	* the same UUID, remove all from the pending list and put them into
7262	* the 'same_array' list. Then order this list based on superblock
7263	* update time (freshest comes first), kick out 'old' disks and
7264	* compare superblocks. If everything's fine then run it.
7265	*
7266	* If "unit" is allocated, then bump its reference count
7267	*/
7268	static void autorun_devices(int part)
7269	{
7270	struct md_rdev rdev0, rdev, *tmp;
7271	struct mddev *mddev;
7272
7273	pr_info("md: autorun ...\n");
7274	while (!list_empty(&pending_raid_disks)) {
7275	int unit;
7276	dev_t dev;
7277	LIST_HEAD(candidates);
7278	rdev0 = list_entry(pending_raid_disks.next,
7279	struct md_rdev, same_set);
7280
7281	pr_debug("md: considering %pg ...\n", rdev0->bdev);
7282	INIT_LIST_HEAD(&candidates);
7283	rdev_for_each_list(rdev, tmp, &pending_raid_disks)
7284	if (super_90_load(rdev, rdev0, 0) >= 0) {
7285	pr_debug("md: adding %pg ...\n",
7286	rdev->bdev);
7287	list_move(&rdev->same_set, &candidates);
7288	}
7289	/*
7290	* now we have a set of devices, with all of them having
7291	* mostly sane superblocks. It's time to allocate the
7292	* mddev.
7293	*/
7294	if (part) {
7295	dev = MKDEV(mdp_major,
7296	rdev0->preferred_minor << MdpMinorShift);
7297	unit = MINOR(dev) >> MdpMinorShift;
7298	} else {
7299	dev = MKDEV(MD_MAJOR, rdev0->preferred_minor);
7300	unit = MINOR(dev);
7301	}
7302	if (rdev0->preferred_minor != unit) {
7303	pr_warn("md: unit number in %pg is bad: %d\n",
7304	rdev0->bdev, rdev0->preferred_minor);
7305	break;
7306	}
7307
7308	mddev = md_alloc(dev, NULL);
7309	if (IS_ERR(mddev))
7310	break;
7311
7312	if (mddev_suspend_and_lock(mddev))
7313	pr_warn("md: %s locked, cannot run\n", mdname(mddev));
7314	else if (mddev->raid_disks \|\| mddev->major_version
7315	\|\| !list_empty(&mddev->disks)) {
7316	pr_warn("md: %s already running, cannot run %pg\n",
7317	mdname(mddev), rdev0->bdev);
7318	mddev_unlock_and_resume(mddev);
7319	} else {
7320	pr_debug("md: created %s\n", mdname(mddev));
7321	mddev->persistent = 1;
7322	rdev_for_each_list(rdev, tmp, &candidates) {
7323	list_del_init(&rdev->same_set);
7324	if (bind_rdev_to_array(rdev, mddev))
7325	export_rdev(rdev);
7326	}
7327	autorun_array(mddev);
7328	mddev_unlock_and_resume(mddev);
7329	}
7330	/* on success, candidates will be empty, on error
7331	* it won't...
7332	*/
7333	rdev_for_each_list(rdev, tmp, &candidates) {
7334	list_del_init(&rdev->same_set);
7335	export_rdev(rdev);
7336	}
7337	mddev_put(mddev);
7338	}
7339	pr_info("md: ... autorun DONE.\n");
7340	}
7341	#endif /* !MODULE */
7342
7343	static int get_version(void __user *arg)
7344	{
7345	mdu_version_t ver;
7346
7347	ver.major = MD_MAJOR_VERSION;
7348	ver.minor = MD_MINOR_VERSION;
7349	ver.patchlevel = MD_PATCHLEVEL_VERSION;
7350
7351	if (copy_to_user(arg, &ver, sizeof(ver)))
7352	return -EFAULT;
7353
7354	return 0;
7355	}
7356
7357	static int get_array_info(struct mddev mddev, void __user arg)
7358	{
7359	mdu_array_info_t info;
7360	int nr,working,insync,failed,spare;
7361	struct md_rdev *rdev;
7362
7363	nr = working = insync = failed = spare = 0;
7364	rcu_read_lock();
7365	rdev_for_each_rcu(rdev, mddev) {
7366	nr++;
7367	if (test_bit(Faulty, &rdev->flags))
7368	failed++;
7369	else {
7370	working++;
7371	if (test_bit(In_sync, &rdev->flags))
7372	insync++;
7373	else if (test_bit(Journal, &rdev->flags))
7374	/* TODO: add journal count to md_u.h */
7375	;
7376	else
7377	spare++;
7378	}
7379	}
7380	rcu_read_unlock();
7381
7382	info.major_version = mddev->major_version;
7383	info.minor_version = mddev->minor_version;
7384	info.patch_version = MD_PATCHLEVEL_VERSION;
7385	info.ctime = clamp_t(time64_t, mddev->ctime, 0, U32_MAX);
7386	info.level = mddev->level;
7387	info.size = mddev->dev_sectors / 2;
7388	if (info.size != mddev->dev_sectors / 2) /* overflow */
7389	info.size = -1;
7390	info.nr_disks = nr;
7391	info.raid_disks = mddev->raid_disks;
7392	info.md_minor = mddev->md_minor;
7393	info.not_persistent= !mddev->persistent;
7394
7395	info.utime = clamp_t(time64_t, mddev->utime, 0, U32_MAX);
7396	info.state = 0;
7397	if (mddev->in_sync)
7398	info.state = (1<<MD_SB_CLEAN);
7399	if (mddev->bitmap && mddev->bitmap_info.offset)
7400	info.state \|= (1<<MD_SB_BITMAP_PRESENT);
7401	if (mddev_is_clustered(mddev))
7402	info.state \|= (1<<MD_SB_CLUSTERED);
7403	info.active_disks = insync;
7404	info.working_disks = working;
7405	info.failed_disks = failed;
7406	info.spare_disks = spare;
7407
7408	info.layout = mddev->layout;
7409	info.chunk_size = mddev->chunk_sectors << 9;
7410
7411	if (copy_to_user(arg, &info, sizeof(info)))
7412	return -EFAULT;
7413
7414	return 0;
7415	}
7416
7417	static int get_bitmap_file(struct mddev mddev, void __user arg)
7418	{
7419	mdu_bitmap_file_t file = NULL; / too big for stack allocation */
7420	char *ptr;
7421	int err;
7422
7423	file = kzalloc_obj(*file, GFP_NOIO);
7424	if (!file)
7425	return -ENOMEM;
7426
7427	err = 0;
7428	spin_lock(&mddev->lock);
7429	/* bitmap enabled */
7430	if (mddev->bitmap_info.file) {
7431	ptr = file_path(mddev->bitmap_info.file, file->pathname,
7432	sizeof(file->pathname));
7433	if (IS_ERR(ptr))
7434	err = PTR_ERR(ptr);
7435	else
7436	memmove(file->pathname, ptr,
7437	sizeof(file->pathname)-(ptr-file->pathname));
7438	}
7439	spin_unlock(&mddev->lock);
7440
7441	if (err == 0 &&
7442	copy_to_user(arg, file, sizeof(*file)))
7443	err = -EFAULT;
7444
7445	kfree(file);
7446	return err;
7447	}
7448
7449	static int get_disk_info(struct mddev mddev, void __user arg)
7450	{
7451	mdu_disk_info_t info;
7452	struct md_rdev *rdev;
7453
7454	if (copy_from_user(&info, arg, sizeof(info)))
7455	return -EFAULT;
7456
7457	rcu_read_lock();
7458	rdev = md_find_rdev_nr_rcu(mddev, info.number);
7459	if (rdev) {
7460	info.major = MAJOR(rdev->bdev->bd_dev);
7461	info.minor = MINOR(rdev->bdev->bd_dev);
7462	info.raid_disk = rdev->raid_disk;
7463	info.state = 0;
7464	if (test_bit(Faulty, &rdev->flags))
7465	info.state \|= (1<<MD_DISK_FAULTY);
7466	else if (test_bit(In_sync, &rdev->flags)) {
7467	info.state \|= (1<<MD_DISK_ACTIVE);
7468	info.state \|= (1<<MD_DISK_SYNC);
7469	}
7470	if (test_bit(Journal, &rdev->flags))
7471	info.state \|= (1<<MD_DISK_JOURNAL);
7472	if (test_bit(WriteMostly, &rdev->flags))
7473	info.state \|= (1<<MD_DISK_WRITEMOSTLY);
7474	if (test_bit(FailFast, &rdev->flags))
7475	info.state \|= (1<<MD_DISK_FAILFAST);
7476	} else {
7477	info.major = info.minor = 0;
7478	info.raid_disk = -1;
7479	info.state = (1<<MD_DISK_REMOVED);
7480	}
7481	rcu_read_unlock();
7482
7483	if (copy_to_user(arg, &info, sizeof(info)))
7484	return -EFAULT;
7485
7486	return 0;
7487	}
7488
7489	int md_add_new_disk(struct mddev mddev, struct mdu_disk_info_s info)
7490	{
7491	struct md_rdev *rdev;
7492	dev_t dev = MKDEV(info->major,info->minor);
7493
7494	if (mddev_is_clustered(mddev) &&
7495	!(info->state & ((1 << MD_DISK_CLUSTER_ADD) \| (1 << MD_DISK_CANDIDATE)))) {
7496	pr_warn("%s: Cannot add to clustered mddev.\n",
7497	mdname(mddev));
7498	return -EINVAL;
7499	}
7500
7501	if (info->major != MAJOR(dev) \|\| info->minor != MINOR(dev))
7502	return -EOVERFLOW;
7503
7504	if (!mddev->raid_disks) {
7505	int err;
7506	/* expecting a device which has a superblock */
7507	rdev = md_import_device(dev, mddev->major_version, mddev->minor_version);
7508	if (IS_ERR(rdev)) {
7509	pr_warn("md: md_import_device returned %ld\n",
7510	PTR_ERR(rdev));
7511	return PTR_ERR(rdev);
7512	}
7513	if (!list_empty(&mddev->disks)) {
7514	struct md_rdev *rdev0
7515	= list_entry(mddev->disks.next,
7516	struct md_rdev, same_set);
7517	err = super_types[mddev->major_version]
7518	.load_super(rdev, rdev0, mddev->minor_version);
7519	if (err < 0) {
7520	pr_warn("md: %pg has different UUID to %pg\n",
7521	rdev->bdev,
7522	rdev0->bdev);
7523	export_rdev(rdev);
7524	return -EINVAL;
7525	}
7526	}
7527	err = bind_rdev_to_array(rdev, mddev);
7528	if (err)
7529	export_rdev(rdev);
7530	return err;
7531	}
7532
7533	/*
7534	* md_add_new_disk can be used once the array is assembled
7535	* to add "hot spares". They must already have a superblock
7536	* written
7537	*/
7538	if (mddev->pers) {
7539	int err;
7540	if (!mddev->pers->hot_add_disk) {
7541	pr_warn("%s: personality does not support diskops!\n",
7542	mdname(mddev));
7543	return -EINVAL;
7544	}
7545	if (mddev->persistent)
7546	rdev = md_import_device(dev, mddev->major_version,
7547	mddev->minor_version);
7548	else
7549	rdev = md_import_device(dev, -1, -1);
7550	if (IS_ERR(rdev)) {
7551	pr_warn("md: md_import_device returned %ld\n",
7552	PTR_ERR(rdev));
7553	return PTR_ERR(rdev);
7554	}
7555	/* set saved_raid_disk if appropriate */
7556	if (!mddev->persistent) {
7557	if (info->state & (1<<MD_DISK_SYNC) &&
7558	info->raid_disk < mddev->raid_disks) {
7559	rdev->raid_disk = info->raid_disk;
7560	clear_bit(Bitmap_sync, &rdev->flags);
7561	} else
7562	rdev->raid_disk = -1;
7563	rdev->saved_raid_disk = rdev->raid_disk;
7564	} else
7565	super_types[mddev->major_version].
7566	validate_super(mddev, NULL/freshest/, rdev);
7567	if ((info->state & (1<<MD_DISK_SYNC)) &&
7568	rdev->raid_disk != info->raid_disk) {
7569	/* This was a hot-add request, but events doesn't
7570	* match, so reject it.
7571	*/
7572	export_rdev(rdev);
7573	return -EINVAL;
7574	}
7575
7576	clear_bit(In_sync, &rdev->flags); /* just to be sure */
7577	if (info->state & (1<<MD_DISK_WRITEMOSTLY))
7578	set_bit(WriteMostly, &rdev->flags);
7579	else
7580	clear_bit(WriteMostly, &rdev->flags);
7581	if (info->state & (1<<MD_DISK_FAILFAST))
7582	set_bit(FailFast, &rdev->flags);
7583	else
7584	clear_bit(FailFast, &rdev->flags);
7585
7586	if (info->state & (1<<MD_DISK_JOURNAL)) {
7587	struct md_rdev *rdev2;
7588	bool has_journal = false;
7589
7590	/* make sure no existing journal disk */
7591	rdev_for_each(rdev2, mddev) {
7592	if (test_bit(Journal, &rdev2->flags)) {
7593	has_journal = true;
7594	break;
7595	}
7596	}
7597	if (has_journal \|\| mddev->bitmap) {
7598	export_rdev(rdev);
7599	return -EBUSY;
7600	}
7601	set_bit(Journal, &rdev->flags);
7602	}
7603	/*
7604	* check whether the device shows up in other nodes
7605	*/
7606	if (mddev_is_clustered(mddev)) {
7607	if (info->state & (1 << MD_DISK_CANDIDATE))
7608	set_bit(Candidate, &rdev->flags);
7609	else if (info->state & (1 << MD_DISK_CLUSTER_ADD)) {
7610	/* --add initiated by this node */
7611	err = mddev->cluster_ops->add_new_disk(mddev, rdev);
7612	if (err) {
7613	export_rdev(rdev);
7614	return err;
7615	}
7616	}
7617	}
7618
7619	rdev->raid_disk = -1;
7620	err = bind_rdev_to_array(rdev, mddev);
7621
7622	if (err)
7623	export_rdev(rdev);
7624
7625	if (mddev_is_clustered(mddev)) {
7626	if (info->state & (1 << MD_DISK_CANDIDATE)) {
7627	if (!err) {
7628	err = mddev->cluster_ops->new_disk_ack(
7629	mddev, err == 0);
7630	if (err)
7631	md_kick_rdev_from_array(rdev);
7632	}
7633	} else {
7634	if (err)
7635	mddev->cluster_ops->add_new_disk_cancel(mddev);
7636	else
7637	err = add_bound_rdev(rdev);
7638	}
7639
7640	} else if (!err)
7641	err = add_bound_rdev(rdev);
7642
7643	return err;
7644	}
7645
7646	/* otherwise, md_add_new_disk is only allowed
7647	* for major_version==0 superblocks
7648	*/
7649	if (mddev->major_version != 0) {
7650	pr_warn("%s: ADD_NEW_DISK not supported\n", mdname(mddev));
7651	return -EINVAL;
7652	}
7653
7654	if (!(info->state & (1<<MD_DISK_FAULTY))) {
7655	int err;
7656	rdev = md_import_device(dev, -1, 0);
7657	if (IS_ERR(rdev)) {
7658	pr_warn("md: error, md_import_device() returned %ld\n",
7659	PTR_ERR(rdev));
7660	return PTR_ERR(rdev);
7661	}
7662	rdev->desc_nr = info->number;
7663	if (info->raid_disk < mddev->raid_disks)
7664	rdev->raid_disk = info->raid_disk;
7665	else
7666	rdev->raid_disk = -1;
7667
7668	if (rdev->raid_disk < mddev->raid_disks)
7669	if (info->state & (1<<MD_DISK_SYNC))
7670	set_bit(In_sync, &rdev->flags);
7671
7672	if (info->state & (1<<MD_DISK_WRITEMOSTLY))
7673	set_bit(WriteMostly, &rdev->flags);
7674	if (info->state & (1<<MD_DISK_FAILFAST))
7675	set_bit(FailFast, &rdev->flags);
7676
7677	if (!mddev->persistent) {
7678	pr_debug("md: nonpersistent superblock ...\n");
7679	rdev->sb_start = bdev_nr_sectors(rdev->bdev);
7680	} else
7681	rdev->sb_start = calc_dev_sboffset(rdev);
7682	rdev->sectors = rdev->sb_start;
7683
7684	err = bind_rdev_to_array(rdev, mddev);
7685	if (err) {
7686	export_rdev(rdev);
7687	return err;
7688	}
7689	}
7690
7691	return 0;
7692	}
7693
7694	static int hot_remove_disk(struct mddev *mddev, dev_t dev)
7695	{
7696	struct md_rdev *rdev;
7697
7698	if (!mddev->pers)
7699	return -ENODEV;
7700
7701	rdev = find_rdev(mddev, dev);
7702	if (!rdev)
7703	return -ENXIO;
7704
7705	if (rdev->raid_disk < 0)
7706	goto kick_rdev;
7707
7708	clear_bit(Blocked, &rdev->flags);
7709	remove_and_add_spares(mddev, rdev);
7710
7711	if (rdev->raid_disk >= 0)
7712	goto busy;
7713
7714	kick_rdev:
7715	if (mddev_is_clustered(mddev) &&
7716	mddev->cluster_ops->remove_disk(mddev, rdev))
7717	goto busy;
7718
7719	md_kick_rdev_from_array(rdev);
7720	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
7721	if (!mddev->thread)
7722	md_update_sb(mddev, 1);
7723	md_new_event();
7724
7725	return 0;
7726	busy:
7727	pr_debug("md: cannot remove active disk %pg from %s ...\n",
7728	rdev->bdev, mdname(mddev));
7729	return -EBUSY;
7730	}
7731
7732	static int hot_add_disk(struct mddev *mddev, dev_t dev)
7733	{
7734	int err;
7735	struct md_rdev *rdev;
7736
7737	if (!mddev->pers)
7738	return -ENODEV;
7739
7740	if (mddev->major_version != 0) {
7741	pr_warn("%s: HOT_ADD may only be used with version-0 superblocks.\n",
7742	mdname(mddev));
7743	return -EINVAL;
7744	}
7745	if (!mddev->pers->hot_add_disk) {
7746	pr_warn("%s: personality does not support diskops!\n",
7747	mdname(mddev));
7748	return -EINVAL;
7749	}
7750
7751	rdev = md_import_device(dev, -1, 0);
7752	if (IS_ERR(rdev)) {
7753	pr_warn("md: error, md_import_device() returned %ld\n",
7754	PTR_ERR(rdev));
7755	return -EINVAL;
7756	}
7757
7758	if (mddev->persistent)
7759	rdev->sb_start = calc_dev_sboffset(rdev);
7760	else
7761	rdev->sb_start = bdev_nr_sectors(rdev->bdev);
7762
7763	rdev->sectors = rdev->sb_start;
7764
7765	if (test_bit(Faulty, &rdev->flags)) {
7766	pr_warn("md: can not hot-add faulty %pg disk to %s!\n",
7767	rdev->bdev, mdname(mddev));
7768	err = -EINVAL;
7769	goto abort_export;
7770	}
7771
7772	clear_bit(In_sync, &rdev->flags);
7773	rdev->desc_nr = -1;
7774	rdev->saved_raid_disk = -1;
7775	err = bind_rdev_to_array(rdev, mddev);
7776	if (err)
7777	goto abort_export;
7778
7779	/*
7780	* The rest should better be atomic, we can have disk failures
7781	* noticed in interrupt contexts ...
7782	*/
7783
7784	rdev->raid_disk = -1;
7785
7786	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
7787	if (!mddev->thread)
7788	md_update_sb(mddev, 1);
7789	/*
7790	* Kick recovery, maybe this spare has to be added to the
7791	* array immediately.
7792	*/
7793	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
7794	md_new_event();
7795	return 0;
7796
7797	abort_export:
7798	export_rdev(rdev);
7799	return err;
7800	}
7801
7802	static int set_bitmap_file(struct mddev *mddev, int fd)
7803	{
7804	int err = 0;
7805
7806	if (!md_bitmap_registered(mddev) \|\|
7807	mddev->bitmap_id == ID_BITMAP_NONE)
7808	return -EINVAL;
7809
7810	if (mddev->pers) {
7811	if (!mddev->pers->quiesce \|\| !mddev->thread)
7812	return -EBUSY;
7813	if (mddev->recovery \|\| mddev->sync_thread)
7814	return -EBUSY;
7815	/* we should be able to change the bitmap.. */
7816	}
7817
7818	if (fd >= 0) {
7819	struct inode *inode;
7820	struct file *f;
7821
7822	if (mddev->bitmap \|\| mddev->bitmap_info.file)
7823	return -EEXIST; /* cannot add when bitmap is present */
7824
7825	if (!IS_ENABLED(CONFIG_MD_BITMAP_FILE)) {
7826	pr_warn("%s: bitmap files not supported by this kernel\n",
7827	mdname(mddev));
7828	return -EINVAL;
7829	}
7830	pr_warn("%s: using deprecated bitmap file support\n",
7831	mdname(mddev));
7832
7833	f = fget(fd);
7834
7835	if (f == NULL) {
7836	pr_warn("%s: error: failed to get bitmap file\n",
7837	mdname(mddev));
7838	return -EBADF;
7839	}
7840
7841	inode = f->f_mapping->host;
7842	if (!S_ISREG(inode->i_mode)) {
7843	pr_warn("%s: error: bitmap file must be a regular file\n",
7844	mdname(mddev));
7845	err = -EBADF;
7846	} else if (!(f->f_mode & FMODE_WRITE)) {
7847	pr_warn("%s: error: bitmap file must open for write\n",
7848	mdname(mddev));
7849	err = -EBADF;
7850	} else if (atomic_read(&inode->i_writecount) != 1) {
7851	pr_warn("%s: error: bitmap file is already in use\n",
7852	mdname(mddev));
7853	err = -EBUSY;
7854	}
7855	if (err) {
7856	fput(f);
7857	return err;
7858	}
7859	mddev->bitmap_info.file = f;
7860	mddev->bitmap_info.offset = 0; /* file overrides offset */
7861	} else if (mddev->bitmap == NULL)
7862	return -ENOENT; /* cannot remove what isn't there */
7863	err = 0;
7864	if (mddev->pers) {
7865	if (fd >= 0) {
7866	err = md_bitmap_create(mddev);
7867	if (!err)
7868	err = mddev->bitmap_ops->load(mddev);
7869
7870	if (err) {
7871	md_bitmap_destroy(mddev);
7872	md_bitmap_set_none(mddev);
7873	fd = -1;
7874	}
7875	} else if (fd < 0) {
7876	md_bitmap_destroy(mddev);
7877	md_bitmap_set_none(mddev);
7878	}
7879	}
7880
7881	if (fd < 0) {
7882	struct file *f = mddev->bitmap_info.file;
7883	if (f) {
7884	spin_lock(&mddev->lock);
7885	mddev->bitmap_info.file = NULL;
7886	spin_unlock(&mddev->lock);
7887	fput(f);
7888	}
7889	}
7890
7891	return err;
7892	}
7893
7894	/*
7895	* md_set_array_info is used two different ways
7896	* The original usage is when creating a new array.
7897	* In this usage, raid_disks is > 0 and it together with
7898	* level, size, not_persistent,layout,chunksize determine the
7899	* shape of the array.
7900	* This will always create an array with a type-0.90.0 superblock.
7901	* The newer usage is when assembling an array.
7902	* In this case raid_disks will be 0, and the major_version field is
7903	* use to determine which style super-blocks are to be found on the devices.
7904	* The minor and patch _version numbers are also kept incase the
7905	* super_block handler wishes to interpret them.
7906	*/
7907	int md_set_array_info(struct mddev mddev, struct mdu_array_info_s info)
7908	{
7909	if (info->raid_disks == 0) {
7910	/* just setting version number for superblock loading */
7911	if (info->major_version < 0 \|\|
7912	info->major_version >= ARRAY_SIZE(super_types) \|\|
7913	super_types[info->major_version].name == NULL) {
7914	/* maybe try to auto-load a module? */
7915	pr_warn("md: superblock version %d not known\n",
7916	info->major_version);
7917	return -EINVAL;
7918	}
7919	mddev->major_version = info->major_version;
7920	mddev->minor_version = info->minor_version;
7921	mddev->patch_version = info->patch_version;
7922	mddev->persistent = !info->not_persistent;
7923	/* ensure mddev_put doesn't delete this now that there
7924	* is some minimal configuration.
7925	*/
7926	mddev->ctime = ktime_get_real_seconds();
7927	return 0;
7928	}
7929	mddev->major_version = MD_MAJOR_VERSION;
7930	mddev->minor_version = MD_MINOR_VERSION;
7931	mddev->patch_version = MD_PATCHLEVEL_VERSION;
7932	mddev->ctime = ktime_get_real_seconds();
7933
7934	mddev->level = info->level;
7935	mddev->clevel[0] = 0;
7936	mddev->dev_sectors = 2 * (sector_t)info->size;
7937	mddev->raid_disks = info->raid_disks;
7938	/* don't set md_minor, it is determined by which /dev/md* was
7939	* openned
7940	*/
7941	if (info->state & (1<<MD_SB_CLEAN))
7942	mddev->resync_offset = MaxSector;
7943	else
7944	mddev->resync_offset = 0;
7945	mddev->persistent = ! info->not_persistent;
7946	mddev->external = 0;
7947
7948	mddev->layout = info->layout;
7949	if (mddev->level == 0)
7950	/* Cannot trust RAID0 layout info here */
7951	mddev->layout = -1;
7952	mddev->chunk_sectors = info->chunk_size >> 9;
7953
7954	if (mddev->persistent) {
7955	mddev->max_disks = MD_SB_DISKS;
7956	mddev->flags = 0;
7957	mddev->sb_flags = 0;
7958	}
7959	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
7960
7961	mddev->bitmap_info.default_offset = MD_SB_BYTES >> 9;
7962	mddev->bitmap_info.default_space = 64*2 - (MD_SB_BYTES >> 9);
7963	mddev->bitmap_info.offset = 0;
7964
7965	mddev->reshape_position = MaxSector;
7966
7967	/*
7968	* Generate a 128 bit UUID
7969	*/
7970	get_random_bytes(mddev->uuid, 16);
7971
7972	mddev->new_level = mddev->level;
7973	mddev->new_chunk_sectors = mddev->chunk_sectors;
7974	mddev->new_layout = mddev->layout;
7975	mddev->delta_disks = 0;
7976	mddev->reshape_backwards = 0;
7977
7978	return 0;
7979	}
7980
7981	void md_set_array_sectors(struct mddev *mddev, sector_t array_sectors)
7982	{
7983	lockdep_assert_held(&mddev->reconfig_mutex);
7984
7985	if (mddev->external_size)
7986	return;
7987
7988	mddev->array_sectors = array_sectors;
7989	}
7990	EXPORT_SYMBOL(md_set_array_sectors);
7991
7992	static int update_size(struct mddev *mddev, sector_t num_sectors)
7993	{
7994	struct md_rdev *rdev;
7995	int rv;
7996	int fit = (num_sectors == 0);
7997	sector_t old_dev_sectors = mddev->dev_sectors;
7998
7999	if (mddev->pers->resize == NULL)
8000	return -EINVAL;
8001	/* The "num_sectors" is the number of sectors of each device that
8002	* is used. This can only make sense for arrays with redundancy.
8003	* linear and raid0 always use whatever space is available. We can only
8004	* consider changing this number if no resync or reconstruction is
8005	* happening, and if the new size is acceptable. It must fit before the
8006	* sb_start or, if that is <data_offset, it must fit before the size
8007	* of each device. If num_sectors is zero, we find the largest size
8008	* that fits.
8009	*/
8010	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
8011	return -EBUSY;
8012	if (!md_is_rdwr(mddev))
8013	return -EROFS;
8014
8015	rdev_for_each(rdev, mddev) {
8016	sector_t avail = rdev->sectors;
8017
8018	if (fit && (num_sectors == 0 \|\| num_sectors > avail))
8019	num_sectors = avail;
8020	if (avail < num_sectors)
8021	return -ENOSPC;
8022	}
8023	rv = mddev->pers->resize(mddev, num_sectors);
8024	if (!rv) {
8025	if (mddev_is_clustered(mddev))
8026	mddev->cluster_ops->update_size(mddev, old_dev_sectors);
8027	else if (!mddev_is_dm(mddev))
8028	set_capacity_and_notify(mddev->gendisk,
8029	mddev->array_sectors);
8030	}
8031	return rv;
8032	}
8033
8034	static int update_raid_disks(struct mddev *mddev, int raid_disks)
8035	{
8036	int rv;
8037	struct md_rdev *rdev;
8038	/* change the number of raid disks */
8039	if (mddev->pers->check_reshape == NULL)
8040	return -EINVAL;
8041	if (!md_is_rdwr(mddev))
8042	return -EROFS;
8043	if (raid_disks <= 0 \|\|
8044	(mddev->max_disks && raid_disks >= mddev->max_disks))
8045	return -EINVAL;
8046	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) \|\|
8047	test_bit(MD_RESYNCING_REMOTE, &mddev->recovery) \|\|
8048	mddev->reshape_position != MaxSector)
8049	return -EBUSY;
8050
8051	rdev_for_each(rdev, mddev) {
8052	if (mddev->raid_disks < raid_disks &&
8053	rdev->data_offset < rdev->new_data_offset)
8054	return -EINVAL;
8055	if (mddev->raid_disks > raid_disks &&
8056	rdev->data_offset > rdev->new_data_offset)
8057	return -EINVAL;
8058	}
8059
8060	mddev->delta_disks = raid_disks - mddev->raid_disks;
8061	if (mddev->delta_disks < 0)
8062	mddev->reshape_backwards = 1;
8063	else if (mddev->delta_disks > 0)
8064	mddev->reshape_backwards = 0;
8065
8066	rv = mddev->pers->check_reshape(mddev);
8067	if (rv < 0) {
8068	mddev->delta_disks = 0;
8069	mddev->reshape_backwards = 0;
8070	}
8071	return rv;
8072	}
8073
8074	static int get_cluster_ops(struct mddev *mddev)
8075	{
8076	xa_lock(&md_submodule);
8077	mddev->cluster_ops = xa_load(&md_submodule, ID_CLUSTER);
8078	if (mddev->cluster_ops &&
8079	!try_module_get(mddev->cluster_ops->head.owner))
8080	mddev->cluster_ops = NULL;
8081	xa_unlock(&md_submodule);
8082
8083	return mddev->cluster_ops == NULL ? -ENOENT : 0;
8084	}
8085
8086	static void put_cluster_ops(struct mddev *mddev)
8087	{
8088	if (!mddev->cluster_ops)
8089	return;
8090
8091	mddev->cluster_ops->leave(mddev);
8092	module_put(mddev->cluster_ops->head.owner);
8093	mddev->cluster_ops = NULL;
8094	}
8095
8096	/*
8097	* update_array_info is used to change the configuration of an
8098	* on-line array.
8099	* The version, ctime,level,size,raid_disks,not_persistent, layout,chunk_size
8100	* fields in the info are checked against the array.
8101	* Any differences that cannot be handled will cause an error.
8102	* Normally, only one change can be managed at a time.
8103	*/
8104	static int update_array_info(struct mddev mddev, mdu_array_info_t info)
8105	{
8106	int rv = 0;
8107	int cnt = 0;
8108	int state = 0;
8109
8110	/* calculate expected state,ignoring low bits */
8111	if (mddev->bitmap && mddev->bitmap_info.offset)
8112	state \|= (1 << MD_SB_BITMAP_PRESENT);
8113
8114	if (mddev->major_version != info->major_version \|\|
8115	mddev->minor_version != info->minor_version \|\|
8116	/* mddev->patch_version != info->patch_version \|\| */
8117	mddev->ctime != info->ctime \|\|
8118	mddev->level != info->level \|\|
8119	/* mddev->layout != info->layout \|\| */
8120	mddev->persistent != !info->not_persistent \|\|
8121	mddev->chunk_sectors != info->chunk_size >> 9 \|\|
8122	/* ignore bottom 8 bits of state, and allow SB_BITMAP_PRESENT to change */
8123	((state^info->state) & 0xfffffe00)
8124	)
8125	return -EINVAL;
8126	/* Check there is only one change */
8127	if (info->size >= 0 && mddev->dev_sectors / 2 != info->size)
8128	cnt++;
8129	if (mddev->raid_disks != info->raid_disks)
8130	cnt++;
8131	if (mddev->layout != info->layout)
8132	cnt++;
8133	if ((state ^ info->state) & (1<<MD_SB_BITMAP_PRESENT))
8134	cnt++;
8135	if (cnt == 0)
8136	return 0;
8137	if (cnt > 1)
8138	return -EINVAL;
8139
8140	if (mddev->layout != info->layout) {
8141	/* Change layout
8142	* we don't need to do anything at the md level, the
8143	* personality will take care of it all.
8144	*/
8145	if (mddev->pers->check_reshape == NULL)
8146	return -EINVAL;
8147	else {
8148	mddev->new_layout = info->layout;
8149	rv = mddev->pers->check_reshape(mddev);
8150	if (rv)
8151	mddev->new_layout = mddev->layout;
8152	return rv;
8153	}
8154	}
8155	if (info->size >= 0 && mddev->dev_sectors / 2 != info->size)
8156	rv = update_size(mddev, (sector_t)info->size * 2);
8157
8158	if (mddev->raid_disks != info->raid_disks)
8159	rv = update_raid_disks(mddev, info->raid_disks);
8160
8161	if ((state ^ info->state) & (1<<MD_SB_BITMAP_PRESENT)) {
8162	if (mddev->pers->quiesce == NULL \|\| mddev->thread == NULL) {
8163	rv = -EINVAL;
8164	goto err;
8165	}
8166	if (mddev->recovery \|\| mddev->sync_thread) {
8167	rv = -EBUSY;
8168	goto err;
8169	}
8170	if (info->state & (1<<MD_SB_BITMAP_PRESENT)) {
8171	/* add the bitmap */
8172	if (mddev->bitmap) {
8173	rv = -EEXIST;
8174	goto err;
8175	}
8176	if (mddev->bitmap_info.default_offset == 0) {
8177	rv = -EINVAL;
8178	goto err;
8179	}
8180	mddev->bitmap_info.offset =
8181	mddev->bitmap_info.default_offset;
8182	mddev->bitmap_info.space =
8183	mddev->bitmap_info.default_space;
8184	mddev->bitmap_id = ID_BITMAP;
8185	rv = md_bitmap_create(mddev);
8186	if (!rv)
8187	rv = mddev->bitmap_ops->load(mddev);
8188
8189	if (rv) {
8190	md_bitmap_destroy(mddev);
8191	mddev->bitmap_info.offset = 0;
8192	md_bitmap_set_none(mddev);
8193	}
8194	} else {
8195	struct md_bitmap_stats stats;
8196
8197	rv = mddev->bitmap_ops->get_stats(mddev->bitmap, &stats);
8198	if (rv)
8199	goto err;
8200
8201	if (stats.file) {
8202	rv = -EINVAL;
8203	goto err;
8204	}
8205
8206	if (mddev->bitmap_info.nodes) {
8207	/* hold PW on all the bitmap lock */
8208	if (mddev->cluster_ops->lock_all_bitmaps(mddev) <= 0) {
8209	pr_warn("md: can't change bitmap to none since the array is in use by more than one node\n");
8210	rv = -EPERM;
8211	mddev->cluster_ops->unlock_all_bitmaps(mddev);
8212	goto err;
8213	}
8214
8215	mddev->bitmap_info.nodes = 0;
8216	put_cluster_ops(mddev);
8217	mddev->safemode_delay = DEFAULT_SAFEMODE_DELAY;
8218	}
8219	md_bitmap_destroy(mddev);
8220	mddev->bitmap_info.offset = 0;
8221	md_bitmap_set_none(mddev);
8222	}
8223	}
8224	md_update_sb(mddev, 1);
8225	return rv;
8226	err:
8227	return rv;
8228	}
8229
8230	static int set_disk_faulty(struct mddev *mddev, dev_t dev)
8231	{
8232	struct md_rdev *rdev;
8233	int err = 0;
8234
8235	if (mddev->pers == NULL)
8236	return -ENODEV;
8237
8238	rcu_read_lock();
8239	rdev = md_find_rdev_rcu(mddev, dev);
8240	if (!rdev)
8241	err = -ENODEV;
8242	else {
8243	md_error(mddev, rdev);
8244	if (test_bit(MD_BROKEN, &mddev->flags))
8245	err = -EBUSY;
8246	}
8247	rcu_read_unlock();
8248	return err;
8249	}
8250
8251	/*
8252	* We have a problem here : there is no easy way to give a CHS
8253	* virtual geometry. We currently pretend that we have a 2 heads
8254	* 4 sectors (with a BIG number of cylinders...). This drives
8255	* dosfs just mad... ;-)
8256	*/
8257	static int md_getgeo(struct gendisk disk, struct hd_geometry geo)
8258	{
8259	struct mddev *mddev = disk->private_data;
8260
8261	geo->heads = 2;
8262	geo->sectors = 4;
8263	geo->cylinders = mddev->array_sectors / 8;
8264	return 0;
8265	}
8266
8267	static inline int md_ioctl_valid(unsigned int cmd)
8268	{
8269	switch (cmd) {
8270	case GET_ARRAY_INFO:
8271	case GET_DISK_INFO:
8272	case RAID_VERSION:
8273	return 0;
8274	case ADD_NEW_DISK:
8275	case GET_BITMAP_FILE:
8276	case HOT_ADD_DISK:
8277	case HOT_REMOVE_DISK:
8278	case RESTART_ARRAY_RW:
8279	case RUN_ARRAY:
8280	case SET_ARRAY_INFO:
8281	case SET_BITMAP_FILE:
8282	case SET_DISK_FAULTY:
8283	case STOP_ARRAY:
8284	case STOP_ARRAY_RO:
8285	case CLUSTERED_DISK_NACK:
8286	if (!capable(CAP_SYS_ADMIN))
8287	return -EACCES;
8288	return 0;
8289	default:
8290	return -ENOTTY;
8291	}
8292	}
8293
8294	static bool md_ioctl_need_suspend(unsigned int cmd)
8295	{
8296	switch (cmd) {
8297	case ADD_NEW_DISK:
8298	case HOT_ADD_DISK:
8299	case HOT_REMOVE_DISK:
8300	case SET_BITMAP_FILE:
8301	case SET_ARRAY_INFO:
8302	return true;
8303	default:
8304	return false;
8305	}
8306	}
8307
8308	static int __md_set_array_info(struct mddev mddev, void __user argp)
8309	{
8310	mdu_array_info_t info;
8311	int err;
8312
8313	if (!argp)
8314	memset(&info, 0, sizeof(info));
8315	else if (copy_from_user(&info, argp, sizeof(info)))
8316	return -EFAULT;
8317
8318	if (mddev->pers) {
8319	err = update_array_info(mddev, &info);
8320	if (err)
8321	pr_warn("md: couldn't update array info. %d\n", err);
8322	return err;
8323	}
8324
8325	if (!list_empty(&mddev->disks)) {
8326	pr_warn("md: array %s already has disks!\n", mdname(mddev));
8327	return -EBUSY;
8328	}
8329
8330	if (mddev->raid_disks) {
8331	pr_warn("md: array %s already initialised!\n", mdname(mddev));
8332	return -EBUSY;
8333	}
8334
8335	err = md_set_array_info(mddev, &info);
8336	if (err)
8337	pr_warn("md: couldn't set array info. %d\n", err);
8338
8339	return err;
8340	}
8341
8342	static int md_ioctl(struct block_device *bdev, blk_mode_t mode,
8343	unsigned int cmd, unsigned long arg)
8344	{
8345	int err = 0;
8346	unsigned int noio_flags = 0;
8347	void __user argp = (void __user )arg;
8348	struct mddev *mddev = NULL;
8349	bool suspend;
8350
8351	err = md_ioctl_valid(cmd);
8352	if (err)
8353	return err;
8354
8355	/*
8356	* Commands dealing with the RAID driver but not any
8357	* particular array:
8358	*/
8359	if (cmd == RAID_VERSION)
8360	return get_version(argp);
8361
8362	/*
8363	* Commands creating/starting a new array:
8364	*/
8365
8366	mddev = bdev->bd_disk->private_data;
8367
8368	/* Some actions do not requires the mutex */
8369	switch (cmd) {
8370	case GET_ARRAY_INFO:
8371	if (!mddev->raid_disks && !mddev->external)
8372	return -ENODEV;
8373	return get_array_info(mddev, argp);
8374
8375	case GET_DISK_INFO:
8376	if (!mddev->raid_disks && !mddev->external)
8377	return -ENODEV;
8378	return get_disk_info(mddev, argp);
8379
8380	case SET_DISK_FAULTY:
8381	return set_disk_faulty(mddev, new_decode_dev(arg));
8382
8383	case GET_BITMAP_FILE:
8384	return get_bitmap_file(mddev, argp);
8385	}
8386
8387	if (cmd == STOP_ARRAY \|\| cmd == STOP_ARRAY_RO) {
8388	/* Need to flush page cache, and ensure no-one else opens
8389	* and writes
8390	*/
8391	err = mddev_set_closing_and_sync_blockdev(mddev, 1);
8392	if (err)
8393	return err;
8394	}
8395
8396	if (!md_is_rdwr(mddev))
8397	flush_work(&mddev->sync_work);
8398
8399	suspend = md_ioctl_need_suspend(cmd);
8400	err = suspend ? mddev_suspend_and_lock(mddev) : mddev_lock(mddev);
8401	if (err) {
8402	pr_debug("md: ioctl lock interrupted, reason %d, cmd %d\n",
8403	err, cmd);
8404	goto out;
8405	}
8406	if (suspend)
8407	noio_flags = memalloc_noio_save();
8408
8409	if (cmd == SET_ARRAY_INFO) {
8410	err = __md_set_array_info(mddev, argp);
8411	goto unlock;
8412	}
8413
8414	/*
8415	* Commands querying/configuring an existing array:
8416	*/
8417	/* if we are not initialised yet, only ADD_NEW_DISK, STOP_ARRAY,
8418	* RUN_ARRAY, and GET_ and SET_BITMAP_FILE are allowed */
8419	if ((!mddev->raid_disks && !mddev->external)
8420	&& cmd != ADD_NEW_DISK && cmd != STOP_ARRAY
8421	&& cmd != RUN_ARRAY && cmd != SET_BITMAP_FILE
8422	&& cmd != GET_BITMAP_FILE) {
8423	err = -ENODEV;
8424	goto unlock;
8425	}
8426
8427	/*
8428	* Commands even a read-only array can execute:
8429	*/
8430	switch (cmd) {
8431	case RESTART_ARRAY_RW:
8432	err = restart_array(mddev);
8433	goto unlock;
8434
8435	case STOP_ARRAY:
8436	err = do_md_stop(mddev, 0);
8437	goto unlock;
8438
8439	case STOP_ARRAY_RO:
8440	if (mddev->pers)
8441	err = md_set_readonly(mddev);
8442	goto unlock;
8443
8444	case HOT_REMOVE_DISK:
8445	err = hot_remove_disk(mddev, new_decode_dev(arg));
8446	goto unlock;
8447
8448	case ADD_NEW_DISK:
8449	/* We can support ADD_NEW_DISK on read-only arrays
8450	* only if we are re-adding a preexisting device.
8451	* So require mddev->pers and MD_DISK_SYNC.
8452	*/
8453	if (mddev->pers) {
8454	mdu_disk_info_t info;
8455	if (copy_from_user(&info, argp, sizeof(info)))
8456	err = -EFAULT;
8457	else if (!(info.state & (1<<MD_DISK_SYNC)))
8458	/* Need to clear read-only for this */
8459	break;
8460	else
8461	err = md_add_new_disk(mddev, &info);
8462	goto unlock;
8463	}
8464	break;
8465	}
8466
8467	/*
8468	* The remaining ioctls are changing the state of the
8469	* superblock, so we do not allow them on read-only arrays.
8470	*/
8471	if (!md_is_rdwr(mddev) && mddev->pers) {
8472	if (mddev->ro != MD_AUTO_READ) {
8473	err = -EROFS;
8474	goto unlock;
8475	}
8476	mddev->ro = MD_RDWR;
8477	sysfs_notify_dirent_safe(mddev->sysfs_state);
8478	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
8479	/* mddev_unlock will wake thread */
8480	/* If a device failed while we were read-only, we
8481	* need to make sure the metadata is updated now.
8482	*/
8483	if (test_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags)) {
8484	mddev_unlock(mddev);
8485	wait_event(mddev->sb_wait,
8486	!test_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags) &&
8487	!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
8488	mddev_lock_nointr(mddev);
8489	}
8490	}
8491
8492	switch (cmd) {
8493	case ADD_NEW_DISK:
8494	{
8495	mdu_disk_info_t info;
8496	if (copy_from_user(&info, argp, sizeof(info)))
8497	err = -EFAULT;
8498	else
8499	err = md_add_new_disk(mddev, &info);
8500	goto unlock;
8501	}
8502
8503	case CLUSTERED_DISK_NACK:
8504	if (mddev_is_clustered(mddev))
8505	mddev->cluster_ops->new_disk_ack(mddev, false);
8506	else
8507	err = -EINVAL;
8508	goto unlock;
8509
8510	case HOT_ADD_DISK:
8511	err = hot_add_disk(mddev, new_decode_dev(arg));
8512	goto unlock;
8513
8514	case RUN_ARRAY:
8515	err = do_md_run(mddev);
8516	goto unlock;
8517
8518	case SET_BITMAP_FILE:
8519	err = set_bitmap_file(mddev, (int)arg);
8520	goto unlock;
8521
8522	default:
8523	err = -EINVAL;
8524	goto unlock;
8525	}
8526
8527	unlock:
8528	if (mddev->hold_active == UNTIL_IOCTL &&
8529	err != -EINVAL)
8530	mddev->hold_active = 0;
8531
8532	if (suspend) {
8533	memalloc_noio_restore(noio_flags);
8534	mddev_unlock_and_resume(mddev);
8535	} else {
8536	mddev_unlock(mddev);
8537	}
8538
8539	out:
8540	if (cmd == STOP_ARRAY_RO \|\| (err && cmd == STOP_ARRAY))
8541	clear_bit(MD_CLOSING, &mddev->flags);
8542	return err;
8543	}
8544	#ifdef CONFIG_COMPAT
8545	static int md_compat_ioctl(struct block_device *bdev, blk_mode_t mode,
8546	unsigned int cmd, unsigned long arg)
8547	{
8548	switch (cmd) {
8549	case HOT_REMOVE_DISK:
8550	case HOT_ADD_DISK:
8551	case SET_DISK_FAULTY:
8552	case SET_BITMAP_FILE:
8553	/* These take in integer arg, do not convert */
8554	break;
8555	default:
8556	arg = (unsigned long)compat_ptr(arg);
8557	break;
8558	}
8559
8560	return md_ioctl(bdev, mode, cmd, arg);
8561	}
8562	#endif /* CONFIG_COMPAT */
8563
8564	static int md_set_read_only(struct block_device *bdev, bool ro)
8565	{
8566	struct mddev *mddev = bdev->bd_disk->private_data;
8567	int err;
8568
8569	err = mddev_lock(mddev);
8570	if (err)
8571	return err;
8572
8573	if (!mddev->raid_disks && !mddev->external) {
8574	err = -ENODEV;
8575	goto out_unlock;
8576	}
8577
8578	/*
8579	* Transitioning to read-auto need only happen for arrays that call
8580	* md_write_start and which are not ready for writes yet.
8581	*/
8582	if (!ro && mddev->ro == MD_RDONLY && mddev->pers) {
8583	err = restart_array(mddev);
8584	if (err)
8585	goto out_unlock;
8586	mddev->ro = MD_AUTO_READ;
8587	}
8588
8589	out_unlock:
8590	mddev_unlock(mddev);
8591	return err;
8592	}
8593
8594	static int md_open(struct gendisk *disk, blk_mode_t mode)
8595	{
8596	struct mddev *mddev;
8597	int err;
8598
8599	spin_lock(&all_mddevs_lock);
8600	mddev = mddev_get(disk->private_data);
8601	spin_unlock(&all_mddevs_lock);
8602	if (!mddev)
8603	return -ENODEV;
8604
8605	err = mutex_lock_interruptible(&mddev->open_mutex);
8606	if (err)
8607	goto out;
8608
8609	err = -ENODEV;
8610	if (test_bit(MD_CLOSING, &mddev->flags))
8611	goto out_unlock;
8612
8613	atomic_inc(&mddev->openers);
8614	mutex_unlock(&mddev->open_mutex);
8615
8616	disk_check_media_change(disk);
8617	return 0;
8618
8619	out_unlock:
8620	mutex_unlock(&mddev->open_mutex);
8621	out:
8622	mddev_put(mddev);
8623	return err;
8624	}
8625
8626	static void md_release(struct gendisk *disk)
8627	{
8628	struct mddev *mddev = disk->private_data;
8629
8630	BUG_ON(!mddev);
8631	atomic_dec(&mddev->openers);
8632	mddev_put(mddev);
8633	}
8634
8635	static unsigned int md_check_events(struct gendisk *disk, unsigned int clearing)
8636	{
8637	struct mddev *mddev = disk->private_data;
8638	unsigned int ret = 0;
8639
8640	if (mddev->changed)
8641	ret = DISK_EVENT_MEDIA_CHANGE;
8642	mddev->changed = 0;
8643	return ret;
8644	}
8645
8646	static void md_free_disk(struct gendisk *disk)
8647	{
8648	struct mddev *mddev = disk->private_data;
8649
8650	mddev_free(mddev);
8651	}
8652
8653	const struct block_device_operations md_fops =
8654	{
8655	.owner = THIS_MODULE,
8656	.submit_bio = md_submit_bio,
8657	.open = md_open,
8658	.release = md_release,
8659	.ioctl = md_ioctl,
8660	#ifdef CONFIG_COMPAT
8661	.compat_ioctl = md_compat_ioctl,
8662	#endif
8663	.getgeo = md_getgeo,
8664	.check_events = md_check_events,
8665	.set_read_only = md_set_read_only,
8666	.free_disk = md_free_disk,
8667	};
8668
8669	static int md_thread(void *arg)
8670	{
8671	struct md_thread *thread = arg;
8672
8673	/*
8674	* md_thread is a 'system-thread', it's priority should be very
8675	* high. We avoid resource deadlocks individually in each
8676	* raid personality. (RAID5 does preallocation) We also use RR and
8677	* the very same RT priority as kswapd, thus we will never get
8678	* into a priority inversion deadlock.
8679	*
8680	* we definitely have to have equal or higher priority than
8681	* bdflush, otherwise bdflush will deadlock if there are too
8682	* many dirty RAID5 blocks.
8683	*/
8684
8685	allow_signal(SIGKILL);
8686	while (!kthread_should_stop()) {
8687
8688	/* We need to wait INTERRUPTIBLE so that
8689	* we don't add to the load-average.
8690	* That means we need to be sure no signals are
8691	* pending
8692	*/
8693	if (signal_pending(current))
8694	flush_signals(current);
8695
8696	wait_event_interruptible_timeout
8697	(thread->wqueue,
8698	test_bit(THREAD_WAKEUP, &thread->flags)
8699	\|\| kthread_should_stop() \|\| kthread_should_park(),
8700	thread->timeout);
8701
8702	clear_bit(THREAD_WAKEUP, &thread->flags);
8703	if (kthread_should_park())
8704	kthread_parkme();
8705	if (!kthread_should_stop())
8706	thread->run(thread);
8707	}
8708
8709	return 0;
8710	}
8711
8712	static void md_wakeup_thread_directly(struct md_thread __rcu **thread)
8713	{
8714	struct md_thread *t;
8715
8716	rcu_read_lock();
8717	t = rcu_dereference(*thread);
8718	if (t)
8719	wake_up_process(t->tsk);
8720	rcu_read_unlock();
8721	}
8722
8723	void __md_wakeup_thread(struct md_thread __rcu *thread)
8724	{
8725	struct md_thread *t;
8726
8727	t = rcu_dereference(thread);
8728	if (t) {
8729	pr_debug("md: waking up MD thread %s.\n", t->tsk->comm);
8730	set_bit(THREAD_WAKEUP, &t->flags);
8731	if (wq_has_sleeper(&t->wqueue))
8732	wake_up(&t->wqueue);
8733	}
8734	}
8735	EXPORT_SYMBOL(__md_wakeup_thread);
8736
8737	struct md_thread md_register_thread(void (run) (struct md_thread *),
8738	struct mddev mddev, const char name)
8739	{
8740	struct md_thread *thread;
8741
8742	thread = kzalloc_obj(struct md_thread);
8743	if (!thread)
8744	return NULL;
8745
8746	init_waitqueue_head(&thread->wqueue);
8747
8748	thread->run = run;
8749	thread->mddev = mddev;
8750	thread->timeout = MAX_SCHEDULE_TIMEOUT;
8751	thread->tsk = kthread_run(md_thread, thread,
8752	"%s_%s",
8753	mdname(thread->mddev),
8754	name);
8755	if (IS_ERR(thread->tsk)) {
8756	kfree(thread);
8757	return NULL;
8758	}
8759	return thread;
8760	}
8761	EXPORT_SYMBOL(md_register_thread);
8762
8763	void md_unregister_thread(struct mddev mddev, struct md_thread __rcu *threadp)
8764	{
8765	struct md_thread thread = rcu_dereference_protected(threadp,
8766	lockdep_is_held(&mddev->reconfig_mutex));
8767
8768	if (!thread)
8769	return;
8770
8771	rcu_assign_pointer(*threadp, NULL);
8772	synchronize_rcu();
8773
8774	pr_debug("interrupting MD-thread pid %d\n", task_pid_nr(thread->tsk));
8775	kthread_stop(thread->tsk);
8776	kfree(thread);
8777	}
8778	EXPORT_SYMBOL(md_unregister_thread);
8779
8780	void md_error(struct mddev mddev, struct md_rdev rdev)
8781	{
8782	if (!rdev \|\| test_bit(Faulty, &rdev->flags))
8783	return;
8784
8785	if (!mddev->pers \|\| !mddev->pers->error_handler)
8786	return;
8787	mddev->pers->error_handler(mddev, rdev);
8788
8789	if (mddev->pers->head.id == ID_RAID0 \|\|
8790	mddev->pers->head.id == ID_LINEAR)
8791	return;
8792
8793	if (mddev->degraded && !test_bit(MD_BROKEN, &mddev->flags))
8794	set_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
8795	sysfs_notify_dirent_safe(rdev->sysfs_state);
8796	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
8797	if (!test_bit(MD_BROKEN, &mddev->flags)) {
8798	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
8799	md_wakeup_thread(mddev->thread);
8800	}
8801	if (mddev->event_work.func)
8802	queue_work(md_misc_wq, &mddev->event_work);
8803	md_new_event();
8804	}
8805	EXPORT_SYMBOL(md_error);
8806
8807	/* seq_file implementation /proc/mdstat */
8808
8809	static void status_unused(struct seq_file *seq)
8810	{
8811	int i = 0;
8812	struct md_rdev *rdev;
8813
8814	seq_printf(seq, "unused devices: ");
8815
8816	list_for_each_entry(rdev, &pending_raid_disks, same_set) {
8817	i++;
8818	seq_printf(seq, "%pg ", rdev->bdev);
8819	}
8820	if (!i)
8821	seq_printf(seq, "<none>");
8822
8823	seq_printf(seq, "\n");
8824	}
8825
8826	static void status_personalities(struct seq_file *seq)
8827	{
8828	struct md_submodule_head *head;
8829	unsigned long i;
8830
8831	seq_puts(seq, "Personalities : ");
8832
8833	xa_lock(&md_submodule);
8834	xa_for_each(&md_submodule, i, head)
8835	if (head->type == MD_PERSONALITY)
8836	seq_printf(seq, "[%s] ", head->name);
8837	xa_unlock(&md_submodule);
8838
8839	seq_puts(seq, "\n");
8840	}
8841
8842	static int status_resync(struct seq_file seq, struct mddev mddev)
8843	{
8844	sector_t max_sectors, resync, res;
8845	unsigned long dt, db = 0;
8846	sector_t rt, curr_mark_cnt, resync_mark_cnt;
8847	int scale, recovery_active;
8848	unsigned int per_milli;
8849
8850	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) \|\|
8851	test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
8852	max_sectors = mddev->resync_max_sectors;
8853	else
8854	max_sectors = mddev->dev_sectors;
8855
8856	resync = mddev->curr_resync;
8857	if (resync < MD_RESYNC_ACTIVE) {
8858	if (test_bit(MD_RECOVERY_DONE, &mddev->recovery))
8859	/* Still cleaning up */
8860	resync = max_sectors;
8861	} else if (resync > max_sectors) {
8862	resync = max_sectors;
8863	} else {
8864	res = atomic_read(&mddev->recovery_active);
8865	/*
8866	* Resync has started, but the subtraction has overflowed or
8867	* yielded one of the special values. Force it to active to
8868	* ensure the status reports an active resync.
8869	*/
8870	if (resync < res \|\| resync - res < MD_RESYNC_ACTIVE)
8871	resync = MD_RESYNC_ACTIVE;
8872	else
8873	resync -= res;
8874	}
8875
8876	if (resync == MD_RESYNC_NONE) {
8877	if (test_bit(MD_RESYNCING_REMOTE, &mddev->recovery)) {
8878	struct md_rdev *rdev;
8879
8880	rdev_for_each(rdev, mddev)
8881	if (rdev->raid_disk >= 0 &&
8882	!test_bit(Faulty, &rdev->flags) &&
8883	rdev->recovery_offset != MaxSector &&
8884	rdev->recovery_offset) {
8885	seq_printf(seq, "\trecover=REMOTE");
8886	return 1;
8887	}
8888	if (mddev->reshape_position != MaxSector)
8889	seq_printf(seq, "\treshape=REMOTE");
8890	else
8891	seq_printf(seq, "\tresync=REMOTE");
8892	return 1;
8893	}
8894	if (mddev->resync_offset < MaxSector) {
8895	seq_printf(seq, "\tresync=PENDING");
8896	return 1;
8897	}
8898	return 0;
8899	}
8900	if (resync < MD_RESYNC_ACTIVE) {
8901	seq_printf(seq, "\tresync=DELAYED");
8902	return 1;
8903	}
8904
8905	WARN_ON(max_sectors == 0);
8906	/* Pick 'scale' such that (resync>>scale)*1000 will fit
8907	* in a sector_t, and (max_sectors>>scale) will fit in a
8908	* u32, as those are the requirements for sector_div.
8909	* Thus 'scale' must be at least 10
8910	*/
8911	scale = 10;
8912	if (sizeof(sector_t) > sizeof(unsigned long)) {
8913	while ( max_sectors/2 > (1ULL<<(scale+32)))
8914	scale++;
8915	}
8916	res = (resync>>scale)*1000;
8917	sector_div(res, (u32)((max_sectors>>scale)+1));
8918
8919	per_milli = res;
8920	{
8921	int i, x = per_milli/50, y = 20-x;
8922	seq_printf(seq, "[");
8923	for (i = 0; i < x; i++)
8924	seq_printf(seq, "=");
8925	seq_printf(seq, ">");
8926	for (i = 0; i < y; i++)
8927	seq_printf(seq, ".");
8928	seq_printf(seq, "] ");
8929	}
8930	seq_printf(seq, " %s =%3u.%u%% (%llu/%llu)",
8931	(test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)?
8932	"reshape" :
8933	(test_bit(MD_RECOVERY_CHECK, &mddev->recovery)?
8934	"check" :
8935	(test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ?
8936	"resync" : "recovery"))),
8937	per_milli/10, per_milli % 10,
8938	(unsigned long long) resync/2,
8939	(unsigned long long) max_sectors/2);
8940
8941	/*
8942	* dt: time from mark until now
8943	* db: blocks written from mark until now
8944	* rt: remaining time
8945	*
8946	* rt is a sector_t, which is always 64bit now. We are keeping
8947	* the original algorithm, but it is not really necessary.
8948	*
8949	* Original algorithm:
8950	* So we divide before multiply in case it is 32bit and close
8951	* to the limit.
8952	* We scale the divisor (db) by 32 to avoid losing precision
8953	* near the end of resync when the number of remaining sectors
8954	* is close to 'db'.
8955	* We then divide rt by 32 after multiplying by db to compensate.
8956	* The '+1' avoids division by zero if db is very small.
8957	*/
8958	dt = ((jiffies - mddev->resync_mark) / HZ);
8959	if (!dt) dt++;
8960
8961	curr_mark_cnt = mddev->curr_mark_cnt;
8962	recovery_active = atomic_read(&mddev->recovery_active);
8963	resync_mark_cnt = mddev->resync_mark_cnt;
8964
8965	if (curr_mark_cnt >= (recovery_active + resync_mark_cnt))
8966	db = curr_mark_cnt - (recovery_active + resync_mark_cnt);
8967
8968	rt = max_sectors - resync; /* number of remaining sectors */
8969	rt = div64_u64(rt, db/32+1);
8970	rt *= dt;
8971	rt >>= 5;
8972
8973	seq_printf(seq, " finish=%lu.%lumin", (unsigned long)rt / 60,
8974	((unsigned long)rt % 60)/6);
8975
8976	seq_printf(seq, " speed=%ldK/sec", db/2/dt);
8977	return 1;
8978	}
8979
8980	static void md_seq_start(struct seq_file seq, loff_t *pos)
8981	__acquires(&all_mddevs_lock)
8982	{
8983	seq->poll_event = atomic_read(&md_event_count);
8984	spin_lock(&all_mddevs_lock);
8985
8986	return seq_list_start_head(&all_mddevs, *pos);
8987	}
8988
8989	static void md_seq_next(struct seq_file seq, void v, loff_t pos)
8990	{
8991	return seq_list_next(v, &all_mddevs, pos);
8992	}
8993
8994	static void md_seq_stop(struct seq_file seq, void v)
8995	__releases(&all_mddevs_lock)
8996	{
8997	spin_unlock(&all_mddevs_lock);
8998	}
8999
9000	static void md_bitmap_status(struct seq_file seq, struct mddev mddev)
9001	{
9002	struct md_bitmap_stats stats;
9003	unsigned long used_pages;
9004	unsigned long chunk_kb;
9005	int err;
9006
9007	if (!md_bitmap_enabled(mddev, false))
9008	return;
9009
9010	err = mddev->bitmap_ops->get_stats(mddev->bitmap, &stats);
9011	if (err)
9012	return;
9013
9014	chunk_kb = mddev->bitmap_info.chunksize >> 10;
9015	used_pages = stats.pages - stats.missing_pages;
9016
9017	seq_printf(seq, "bitmap: %lu/%lu pages [%luKB], %lu%s chunk",
9018	used_pages, stats.pages, used_pages << (PAGE_SHIFT - 10),
9019	chunk_kb ? chunk_kb : mddev->bitmap_info.chunksize,
9020	chunk_kb ? "KB" : "B");
9021
9022	if (stats.file) {
9023	seq_puts(seq, ", file: ");
9024	seq_file_path(seq, stats.file, " \t\n");
9025	}
9026
9027	seq_putc(seq, '\n');
9028	}
9029
9030	static int md_seq_show(struct seq_file seq, void v)
9031	{
9032	struct mddev *mddev;
9033	sector_t sectors;
9034	struct md_rdev *rdev;
9035
9036	if (v == &all_mddevs) {
9037	status_personalities(seq);
9038	if (list_empty(&all_mddevs))
9039	status_unused(seq);
9040	return 0;
9041	}
9042
9043	mddev = list_entry(v, struct mddev, all_mddevs);
9044	if (!mddev_get(mddev))
9045	return 0;
9046
9047	spin_unlock(&all_mddevs_lock);
9048
9049	/* prevent bitmap to be freed after checking */
9050	mutex_lock(&mddev->bitmap_info.mutex);
9051
9052	spin_lock(&mddev->lock);
9053	if (mddev->pers \|\| mddev->raid_disks \|\| !list_empty(&mddev->disks)) {
9054	seq_printf(seq, "%s : ", mdname(mddev));
9055	if (mddev->pers) {
9056	if (test_bit(MD_BROKEN, &mddev->flags))
9057	seq_printf(seq, "broken");
9058	else
9059	seq_printf(seq, "active");
9060	if (mddev->ro == MD_RDONLY)
9061	seq_printf(seq, " (read-only)");
9062	if (mddev->ro == MD_AUTO_READ)
9063	seq_printf(seq, " (auto-read-only)");
9064	seq_printf(seq, " %s", mddev->pers->head.name);
9065	} else {
9066	seq_printf(seq, "inactive");
9067	}
9068
9069	sectors = 0;
9070	rcu_read_lock();
9071	rdev_for_each_rcu(rdev, mddev) {
9072	seq_printf(seq, " %pg[%d]", rdev->bdev, rdev->desc_nr);
9073
9074	if (test_bit(WriteMostly, &rdev->flags))
9075	seq_printf(seq, "(W)");
9076	if (test_bit(Journal, &rdev->flags))
9077	seq_printf(seq, "(J)");
9078	if (test_bit(Faulty, &rdev->flags)) {
9079	seq_printf(seq, "(F)");
9080	continue;
9081	}
9082	if (rdev->raid_disk < 0)
9083	seq_printf(seq, "(S)"); /* spare */
9084	if (test_bit(Replacement, &rdev->flags))
9085	seq_printf(seq, "(R)");
9086	sectors += rdev->sectors;
9087	}
9088	rcu_read_unlock();
9089
9090	if (!list_empty(&mddev->disks)) {
9091	if (mddev->pers)
9092	seq_printf(seq, "\n %llu blocks",
9093	(unsigned long long)
9094	mddev->array_sectors / 2);
9095	else
9096	seq_printf(seq, "\n %llu blocks",
9097	(unsigned long long)sectors / 2);
9098	}
9099	if (mddev->persistent) {
9100	if (mddev->major_version != 0 \|\|
9101	mddev->minor_version != 90) {
9102	seq_printf(seq," super %d.%d",
9103	mddev->major_version,
9104	mddev->minor_version);
9105	}
9106	} else if (mddev->external)
9107	seq_printf(seq, " super external:%s",
9108	mddev->metadata_type);
9109	else
9110	seq_printf(seq, " super non-persistent");
9111
9112	if (mddev->pers) {
9113	mddev->pers->status(seq, mddev);
9114	seq_printf(seq, "\n ");
9115	if (mddev->pers->sync_request) {
9116	if (status_resync(seq, mddev))
9117	seq_printf(seq, "\n ");
9118	}
9119	} else
9120	seq_printf(seq, "\n ");
9121
9122	md_bitmap_status(seq, mddev);
9123
9124	seq_printf(seq, "\n");
9125	}
9126	spin_unlock(&mddev->lock);
9127	mutex_unlock(&mddev->bitmap_info.mutex);
9128	spin_lock(&all_mddevs_lock);
9129
9130	if (mddev == list_last_entry(&all_mddevs, struct mddev, all_mddevs))
9131	status_unused(seq);
9132
9133	mddev_put_locked(mddev);
9134	return 0;
9135	}
9136
9137	static const struct seq_operations md_seq_ops = {
9138	.start = md_seq_start,
9139	.next = md_seq_next,
9140	.stop = md_seq_stop,
9141	.show = md_seq_show,
9142	};
9143
9144	static int md_seq_open(struct inode inode, struct file file)
9145	{
9146	struct seq_file *seq;
9147	int error;
9148
9149	error = seq_open(file, &md_seq_ops);
9150	if (error)
9151	return error;
9152
9153	seq = file->private_data;
9154	seq->poll_event = atomic_read(&md_event_count);
9155	return error;
9156	}
9157
9158	static int md_unloading;
9159	static __poll_t mdstat_poll(struct file filp, poll_table wait)
9160	{
9161	struct seq_file *seq = filp->private_data;
9162	__poll_t mask;
9163
9164	if (md_unloading)
9165	return EPOLLIN\|EPOLLRDNORM\|EPOLLERR\|EPOLLPRI;
9166	poll_wait(filp, &md_event_waiters, wait);
9167
9168	/* always allow read */
9169	mask = EPOLLIN \| EPOLLRDNORM;
9170
9171	if (seq->poll_event != atomic_read(&md_event_count))
9172	mask \|= EPOLLERR \| EPOLLPRI;
9173	return mask;
9174	}
9175
9176	static const struct proc_ops mdstat_proc_ops = {
9177	.proc_open = md_seq_open,
9178	.proc_read = seq_read,
9179	.proc_lseek = seq_lseek,
9180	.proc_release = seq_release,
9181	.proc_poll = mdstat_poll,
9182	};
9183
9184	int register_md_submodule(struct md_submodule_head *msh)
9185	{
9186	return xa_insert(&md_submodule, msh->id, msh, GFP_KERNEL);
9187	}
9188	EXPORT_SYMBOL_GPL(register_md_submodule);
9189
9190	void unregister_md_submodule(struct md_submodule_head *msh)
9191	{
9192	xa_erase(&md_submodule, msh->id);
9193	}
9194	EXPORT_SYMBOL_GPL(unregister_md_submodule);
9195
9196	int md_setup_cluster(struct mddev *mddev, int nodes)
9197	{
9198	int ret = get_cluster_ops(mddev);
9199
9200	if (ret) {
9201	request_module("md-cluster");
9202	ret = get_cluster_ops(mddev);
9203	}
9204
9205	/* ensure module won't be unloaded */
9206	if (ret) {
9207	pr_warn("can't find md-cluster module or get its reference.\n");
9208	return ret;
9209	}
9210
9211	ret = mddev->cluster_ops->join(mddev, nodes);
9212	if (!ret)
9213	mddev->safemode_delay = 0;
9214	return ret;
9215	}
9216
9217	void md_cluster_stop(struct mddev *mddev)
9218	{
9219	put_cluster_ops(mddev);
9220	}
9221
9222	static bool is_rdev_holder_idle(struct md_rdev *rdev, bool init)
9223	{
9224	unsigned long last_events = rdev->last_events;
9225
9226	if (!bdev_is_partition(rdev->bdev))
9227	return true;
9228
9229	/*
9230	* If rdev is partition, and user doesn't issue IO to the array, the
9231	* array is still not idle if user issues IO to other partitions.
9232	*/
9233	rdev->last_events = part_stat_read_accum(rdev->bdev->bd_disk->part0,
9234	sectors) -
9235	part_stat_read_accum(rdev->bdev, sectors);
9236
9237	return init \|\| rdev->last_events <= last_events;
9238	}
9239
9240	/*
9241	* mddev is idle if following conditions are matched since last check:
9242	* 1) mddev doesn't have normal IO completed;
9243	* 2) mddev doesn't have inflight normal IO;
9244	* 3) if any member disk is partition, and other partitions don't have IO
9245	* completed;
9246	*
9247	* Noted this checking rely on IO accounting is enabled.
9248	*/
9249	static bool is_mddev_idle(struct mddev *mddev, int init)
9250	{
9251	unsigned long last_events = mddev->normal_io_events;
9252	struct gendisk *disk;
9253	struct md_rdev *rdev;
9254	bool idle = true;
9255
9256	disk = mddev_is_dm(mddev) ? mddev->dm_gendisk : mddev->gendisk;
9257	if (!disk)
9258	return true;
9259
9260	mddev->normal_io_events = part_stat_read_accum(disk->part0, sectors);
9261	if (!init && (mddev->normal_io_events > last_events \|\|
9262	bdev_count_inflight(disk->part0)))
9263	idle = false;
9264
9265	rcu_read_lock();
9266	rdev_for_each_rcu(rdev, mddev)
9267	if (!is_rdev_holder_idle(rdev, init))
9268	idle = false;
9269	rcu_read_unlock();
9270
9271	return idle;
9272	}
9273
9274	void md_done_sync(struct mddev *mddev, int blocks)
9275	{
9276	/* another "blocks" (512byte) blocks have been synced */
9277	atomic_sub(blocks, &mddev->recovery_active);
9278	wake_up(&mddev->recovery_wait);
9279	}
9280	EXPORT_SYMBOL(md_done_sync);
9281
9282	void md_sync_error(struct mddev *mddev)
9283	{
9284	// stop recovery, signal do_sync ....
9285	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
9286	md_wakeup_thread(mddev->thread);
9287	}
9288	EXPORT_SYMBOL(md_sync_error);
9289
9290	/* md_write_start(mddev, bi)
9291	* If we need to update some array metadata (e.g. 'active' flag
9292	* in superblock) before writing, schedule a superblock update
9293	* and wait for it to complete.
9294	* A return value of 'false' means that the write wasn't recorded
9295	* and cannot proceed as the array is being suspend.
9296	*/
9297	void md_write_start(struct mddev mddev, struct bio bi)
9298	{
9299	int did_change = 0;
9300
9301	if (bio_data_dir(bi) != WRITE)
9302	return;
9303
9304	BUG_ON(mddev->ro == MD_RDONLY);
9305	if (mddev->ro == MD_AUTO_READ) {
9306	/* need to switch to read/write */
9307	mddev->ro = MD_RDWR;
9308	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
9309	md_wakeup_thread(mddev->thread);
9310	md_wakeup_thread(mddev->sync_thread);
9311	did_change = 1;
9312	}
9313	rcu_read_lock();
9314	percpu_ref_get(&mddev->writes_pending);
9315	smp_mb(); /* Match smp_mb in set_in_sync() */
9316	if (mddev->safemode == 1)
9317	mddev->safemode = 0;
9318	/* sync_checkers is always 0 when writes_pending is in per-cpu mode */
9319	if (mddev->in_sync \|\| mddev->sync_checkers) {
9320	spin_lock(&mddev->lock);
9321	if (mddev->in_sync) {
9322	mddev->in_sync = 0;
9323	set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
9324	set_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
9325	md_wakeup_thread(mddev->thread);
9326	did_change = 1;
9327	}
9328	spin_unlock(&mddev->lock);
9329	}
9330	rcu_read_unlock();
9331	if (did_change)
9332	sysfs_notify_dirent_safe(mddev->sysfs_state);
9333	if (!test_bit(MD_HAS_SUPERBLOCK, &mddev->flags))
9334	return;
9335	wait_event(mddev->sb_wait,
9336	!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
9337	}
9338	EXPORT_SYMBOL(md_write_start);
9339
9340	/* md_write_inc can only be called when md_write_start() has
9341	* already been called at least once of the current request.
9342	* It increments the counter and is useful when a single request
9343	* is split into several parts. Each part causes an increment and
9344	* so needs a matching md_write_end().
9345	* Unlike md_write_start(), it is safe to call md_write_inc() inside
9346	* a spinlocked region.
9347	*/
9348	void md_write_inc(struct mddev mddev, struct bio bi)
9349	{
9350	if (bio_data_dir(bi) != WRITE)
9351	return;
9352	WARN_ON_ONCE(mddev->in_sync \|\| !md_is_rdwr(mddev));
9353	percpu_ref_get(&mddev->writes_pending);
9354	}
9355	EXPORT_SYMBOL(md_write_inc);
9356
9357	void md_write_end(struct mddev *mddev)
9358	{
9359	percpu_ref_put(&mddev->writes_pending);
9360
9361	if (mddev->safemode == 2)
9362	md_wakeup_thread(mddev->thread);
9363	else if (mddev->safemode_delay)
9364	/* The roundup() ensures this only performs locking once
9365	* every ->safemode_delay jiffies
9366	*/
9367	mod_timer(&mddev->safemode_timer,
9368	roundup(jiffies, mddev->safemode_delay) +
9369	mddev->safemode_delay);
9370	}
9371
9372	EXPORT_SYMBOL(md_write_end);
9373
9374	/* This is used by raid0 and raid10 */
9375	void md_submit_discard_bio(struct mddev mddev, struct md_rdev rdev,
9376	struct bio *bio, sector_t start, sector_t size)
9377	{
9378	struct bio *discard_bio = NULL;
9379
9380	/* Discard is optional, so silently skip members that do not support it. */
9381	if (unlikely(!bdev_max_discard_sectors(rdev->bdev)))
9382	return;
9383
9384	__blkdev_issue_discard(rdev->bdev, start, size, GFP_NOIO, &discard_bio);
9385	if (!discard_bio)
9386	return;
9387
9388	bio_chain(discard_bio, bio);
9389	bio_clone_blkg_association(discard_bio, bio);
9390	mddev_trace_remap(mddev, discard_bio, bio->bi_iter.bi_sector);
9391	submit_bio_noacct(discard_bio);
9392	}
9393	EXPORT_SYMBOL_GPL(md_submit_discard_bio);
9394
9395	struct bio mddev_bio_split_at_reshape_offset(struct mddev mddev,
9396	struct bio *bio,
9397	unsigned int *max_sectors,
9398	struct bio_set *bs)
9399	{
9400	sector_t boundary;
9401	sector_t start;
9402	sector_t end;
9403	unsigned int split_sectors;
9404
9405	split_sectors = bio_sectors(bio);
9406	if (max_sectors && max_sectors && max_sectors < split_sectors)
9407	split_sectors = *max_sectors;
9408
9409	if (!test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
9410	goto split;
9411
9412	boundary = READ_ONCE(mddev->reshape_position);
9413	start = bio->bi_iter.bi_sector;
9414	end = bio_end_sector(bio);
9415	if (start >= boundary \|\| end <= boundary)
9416	goto split;
9417
9418	if (boundary - start < split_sectors)
9419	split_sectors = boundary - start;
9420
9421	split:
9422	if (max_sectors)
9423	*max_sectors = split_sectors;
9424	if (split_sectors < bio_sectors(bio)) {
9425	bio = bio_submit_split_bioset(bio, split_sectors, bs);
9426	if (bio)
9427	bio->bi_opf \|= REQ_NOMERGE;
9428	}
9429
9430	return bio;
9431	}
9432	EXPORT_SYMBOL_GPL(mddev_bio_split_at_reshape_offset);
9433
9434	static void md_bitmap_prepare_range(struct mddev mddev, sector_t offset,
9435	unsigned long *sectors, bool discard)
9436	{
9437	mddev->bitmap_ops->prepare_range(mddev, offset, sectors, discard);
9438	}
9439
9440	static void md_bitmap_start(struct mddev *mddev,
9441	struct md_io_clone *md_io_clone)
9442	{
9443	bool discard = md_io_clone->rw == STAT_DISCARD;
9444	md_bitmap_fn *fn = discard ? mddev->bitmap_ops->start_discard :
9445	mddev->bitmap_ops->start_write;
9446
9447	md_bitmap_prepare_range(mddev, &md_io_clone->offset,
9448	&md_io_clone->sectors, discard);
9449	if (!md_io_clone->sectors)
9450	return;
9451	fn(mddev, md_io_clone->offset, md_io_clone->sectors);
9452	}
9453
9454	static void md_bitmap_end(struct mddev mddev, struct md_io_clone md_io_clone)
9455	{
9456	md_bitmap_fn *fn = unlikely(md_io_clone->rw == STAT_DISCARD) ?
9457	mddev->bitmap_ops->end_discard :
9458	mddev->bitmap_ops->end_write;
9459
9460	fn(mddev, md_io_clone->offset, md_io_clone->sectors);
9461	}
9462
9463	static void md_end_clone_io(struct bio *bio)
9464	{
9465	struct md_io_clone *md_io_clone = container_of(bio, struct md_io_clone,
9466	bio_clone);
9467	struct bio *orig_bio = md_io_clone->orig_bio;
9468	struct mddev *mddev = md_io_clone->mddev;
9469	struct completion *reshape_completion = bio->bi_private;
9470
9471	if (bio_data_dir(orig_bio) == WRITE && md_io_clone->sectors &&
9472	md_bitmap_enabled(mddev, false))
9473	md_bitmap_end(mddev, md_io_clone);
9474
9475	if (bio->bi_status && !orig_bio->bi_status)
9476	orig_bio->bi_status = bio->bi_status;
9477
9478	if (md_io_clone->start_time)
9479	bio_end_io_acct(orig_bio, md_io_clone->start_time);
9480
9481	bio_put(bio);
9482	if (unlikely(reshape_completion))
9483	complete(reshape_completion);
9484	else
9485	bio_endio(orig_bio);
9486	percpu_ref_put(&mddev->active_io);
9487	}
9488
9489	static void md_clone_bio(struct mddev mddev, struct bio *bio)
9490	{
9491	struct block_device bdev = (bio)->bi_bdev;
9492	struct md_io_clone *md_io_clone;
9493	struct bio *clone =
9494	bio_alloc_clone(bdev, *bio, GFP_NOIO, &mddev->io_clone_set);
9495
9496	md_io_clone = container_of(clone, struct md_io_clone, bio_clone);
9497	md_io_clone->orig_bio = *bio;
9498	md_io_clone->mddev = mddev;
9499	md_io_clone->sectors = 0;
9500	if (blk_queue_io_stat(bdev->bd_disk->queue))
9501	md_io_clone->start_time = bio_start_io_acct(*bio);
9502	else
9503	md_io_clone->start_time = 0;
9504
9505	if (bio_data_dir(bio) == WRITE && bio_sectors(bio) &&
9506	md_bitmap_enabled(mddev, false)) {
9507	md_io_clone->offset = (*bio)->bi_iter.bi_sector;
9508	md_io_clone->sectors = bio_sectors(*bio);
9509	md_io_clone->rw = op_stat_group(bio_op(*bio));
9510	md_bitmap_start(mddev, md_io_clone);
9511	}
9512
9513	clone->bi_end_io = md_end_clone_io;
9514	clone->bi_private = NULL;
9515	*bio = clone;
9516	}
9517
9518	void md_account_bio(struct mddev mddev, struct bio *bio)
9519	{
9520	percpu_ref_get(&mddev->active_io);
9521	md_clone_bio(mddev, bio);
9522	}
9523	EXPORT_SYMBOL_GPL(md_account_bio);
9524
9525	/* md_allow_write(mddev)
9526	* Calling this ensures that the array is marked 'active' so that writes
9527	* may proceed without blocking. It is important to call this before
9528	* attempting a GFP_KERNEL allocation while holding the mddev lock.
9529	* Must be called with mddev_lock held.
9530	*/
9531	void md_allow_write(struct mddev *mddev)
9532	{
9533	if (!mddev->pers)
9534	return;
9535	if (!md_is_rdwr(mddev))
9536	return;
9537	if (!mddev->pers->sync_request)
9538	return;
9539
9540	spin_lock(&mddev->lock);
9541	if (mddev->in_sync) {
9542	mddev->in_sync = 0;
9543	set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
9544	set_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
9545	if (mddev->safemode_delay &&
9546	mddev->safemode == 0)
9547	mddev->safemode = 1;
9548	spin_unlock(&mddev->lock);
9549	md_update_sb(mddev, 0);
9550	sysfs_notify_dirent_safe(mddev->sysfs_state);
9551	/* wait for the dirty state to be recorded in the metadata */
9552	wait_event(mddev->sb_wait,
9553	!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
9554	} else
9555	spin_unlock(&mddev->lock);
9556	}
9557	EXPORT_SYMBOL_GPL(md_allow_write);
9558
9559	static sector_t md_sync_max_sectors(struct mddev *mddev,
9560	enum sync_action action)
9561	{
9562	switch (action) {
9563	case ACTION_RESYNC:
9564	case ACTION_CHECK:
9565	case ACTION_REPAIR:
9566	atomic64_set(&mddev->resync_mismatches, 0);
9567	fallthrough;
9568	case ACTION_RESHAPE:
9569	return mddev->resync_max_sectors;
9570	case ACTION_RECOVER:
9571	return mddev->dev_sectors;
9572	default:
9573	return 0;
9574	}
9575	}
9576
9577	/*
9578	* If lazy recovery is requested and all rdevs are in sync, select the rdev with
9579	* the higest index to perfore recovery to build initial xor data, this is the
9580	* same as old bitmap.
9581	*/
9582	static bool mddev_select_lazy_recover_rdev(struct mddev *mddev)
9583	{
9584	struct md_rdev *recover_rdev = NULL;
9585	struct md_rdev *rdev;
9586	bool ret = false;
9587
9588	rcu_read_lock();
9589	rdev_for_each_rcu(rdev, mddev) {
9590	if (rdev->raid_disk < 0)
9591	continue;
9592
9593	if (test_bit(Faulty, &rdev->flags) \|\|
9594	!test_bit(In_sync, &rdev->flags))
9595	break;
9596
9597	if (!recover_rdev \|\| recover_rdev->raid_disk < rdev->raid_disk)
9598	recover_rdev = rdev;
9599	}
9600
9601	if (recover_rdev) {
9602	clear_bit(In_sync, &recover_rdev->flags);
9603	ret = true;
9604	}
9605
9606	rcu_read_unlock();
9607	return ret;
9608	}
9609
9610	static sector_t md_sync_position(struct mddev *mddev, enum sync_action action)
9611	{
9612	sector_t start = 0;
9613	struct md_rdev *rdev;
9614
9615	switch (action) {
9616	case ACTION_CHECK:
9617	case ACTION_REPAIR:
9618	return mddev->resync_min;
9619	case ACTION_RESYNC:
9620	if (!mddev->bitmap)
9621	return mddev->resync_offset;
9622	return 0;
9623	case ACTION_RESHAPE:
9624	/*
9625	* If the original node aborts reshaping then we continue the
9626	* reshaping, so set again to avoid restart reshape from the
9627	* first beginning
9628	*/
9629	if (mddev_is_clustered(mddev) &&
9630	mddev->reshape_position != MaxSector)
9631	return mddev->reshape_position;
9632	return 0;
9633	case ACTION_RECOVER:
9634	start = MaxSector;
9635	rcu_read_lock();
9636	rdev_for_each_rcu(rdev, mddev)
9637	if (rdev_needs_recovery(rdev, start))
9638	start = rdev->recovery_offset;
9639	rcu_read_unlock();
9640
9641	/*
9642	* If there are no spares, and raid456 lazy initial recover is
9643	* requested.
9644	*/
9645	if (test_bit(MD_RECOVERY_LAZY_RECOVER, &mddev->recovery) &&
9646	start == MaxSector && mddev_select_lazy_recover_rdev(mddev))
9647	start = 0;
9648
9649	/* If there is a bitmap, we need to make sure all
9650	* writes that started before we added a spare
9651	* complete before we start doing a recovery.
9652	* Otherwise the write might complete and (via
9653	* bitmap_endwrite) set a bit in the bitmap after the
9654	* recovery has checked that bit and skipped that
9655	* region.
9656	*/
9657	if (mddev->bitmap) {
9658	mddev->pers->quiesce(mddev, 1);
9659	mddev->pers->quiesce(mddev, 0);
9660	}
9661	return start;
9662	default:
9663	return MaxSector;
9664	}
9665	}
9666
9667	static bool sync_io_within_limit(struct mddev *mddev)
9668	{
9669	/*
9670	* For raid456, sync IO is stripe(4k) per IO, for other levels, it's
9671	* RESYNC_PAGES(64k) per IO.
9672	*/
9673	return atomic_read(&mddev->recovery_active) <
9674	(raid_is_456(mddev) ? 8 : 128) * sync_io_depth(mddev);
9675	}
9676
9677	/*
9678	* Update sync offset and mddev status when sync completes
9679	*/
9680	static void md_finish_sync(struct mddev *mddev, enum sync_action action)
9681	{
9682	struct md_rdev *rdev;
9683
9684	switch (action) {
9685	case ACTION_RESYNC:
9686	case ACTION_REPAIR:
9687	if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery))
9688	mddev->curr_resync = MaxSector;
9689	mddev->resync_offset = mddev->curr_resync;
9690	break;
9691	case ACTION_RECOVER:
9692	if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery))
9693	mddev->curr_resync = MaxSector;
9694	rcu_read_lock();
9695	rdev_for_each_rcu(rdev, mddev)
9696	if (mddev->delta_disks >= 0 &&
9697	rdev_needs_recovery(rdev, mddev->curr_resync))
9698	rdev->recovery_offset = mddev->curr_resync;
9699	rcu_read_unlock();
9700	break;
9701	case ACTION_RESHAPE:
9702	if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery) &&
9703	mddev->delta_disks > 0 &&
9704	mddev->pers->finish_reshape &&
9705	mddev->pers->size &&
9706	!mddev_is_dm(mddev)) {
9707	mddev_lock_nointr(mddev);
9708	md_set_array_sectors(mddev, mddev->pers->size(mddev, 0, 0));
9709	mddev_unlock(mddev);
9710	if (!mddev_is_clustered(mddev))
9711	set_capacity_and_notify(mddev->gendisk,
9712	mddev->array_sectors);
9713	}
9714	if (mddev->pers->finish_reshape)
9715	mddev->pers->finish_reshape(mddev);
9716	break;
9717	/* */
9718	case ACTION_CHECK:
9719	default:
9720	break;
9721	}
9722	}
9723
9724	#define SYNC_MARKS 10
9725	#define SYNC_MARK_STEP (3*HZ)
9726	#define UPDATE_FREQUENCY (560HZ)
9727	void md_do_sync(struct md_thread *thread)
9728	{
9729	struct mddev *mddev = thread->mddev;
9730	struct mddev *mddev2;
9731	unsigned int currspeed = 0, window;
9732	sector_t max_sectors,j, io_sectors, recovery_done;
9733	unsigned long mark[SYNC_MARKS];
9734	unsigned long update_time;
9735	sector_t mark_cnt[SYNC_MARKS];
9736	int last_mark,m;
9737	sector_t last_check;
9738	int skipped = 0;
9739	enum sync_action action;
9740	const char *desc;
9741	struct blk_plug plug;
9742	int ret;
9743
9744	/* just incase thread restarts... */
9745	if (test_bit(MD_RECOVERY_DONE, &mddev->recovery))
9746	return;
9747
9748	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
9749	goto skip;
9750
9751	if (test_bit(MD_RECOVERY_WAIT, &mddev->recovery) \|\|
9752	!md_is_rdwr(mddev)) {/* never try to sync a read-only array */
9753	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
9754	goto skip;
9755	}
9756
9757	if (mddev_is_clustered(mddev)) {
9758	ret = mddev->cluster_ops->resync_start(mddev);
9759	if (ret)
9760	goto skip;
9761
9762	set_bit(MD_CLUSTER_RESYNC_LOCKED, &mddev->flags);
9763	if (!(test_bit(MD_RECOVERY_SYNC, &mddev->recovery) \|\|
9764	test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) \|\|
9765	test_bit(MD_RECOVERY_RECOVER, &mddev->recovery))
9766	&& ((unsigned long long)mddev->curr_resync_completed
9767	< (unsigned long long)mddev->resync_max_sectors))
9768	goto skip;
9769	}
9770
9771	action = md_sync_action(mddev);
9772	if (action == ACTION_FROZEN \|\| action == ACTION_IDLE) {
9773	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
9774	goto skip;
9775	}
9776
9777	desc = md_sync_action_name(action);
9778	mddev->last_sync_action = action;
9779
9780	/*
9781	* Before starting a resync we must have set curr_resync to
9782	* 2, and then checked that every "conflicting" array has curr_resync
9783	* less than ours. When we find one that is the same or higher
9784	* we wait on resync_wait. To avoid deadlock, we reduce curr_resync
9785	* to 1 if we choose to yield (based arbitrarily on address of mddev structure).
9786	* This will mean we have to start checking from the beginning again.
9787	*
9788	*/
9789	if (mddev_is_clustered(mddev))
9790	mddev->cluster_ops->resync_start_notify(mddev);
9791	do {
9792	int mddev2_minor = -1;
9793	mddev->curr_resync = MD_RESYNC_DELAYED;
9794
9795	try_again:
9796	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
9797	goto skip;
9798	spin_lock(&all_mddevs_lock);
9799	list_for_each_entry(mddev2, &all_mddevs, all_mddevs) {
9800	if (test_bit(MD_DELETED, &mddev2->flags))
9801	continue;
9802	if (mddev2 == mddev)
9803	continue;
9804	if (!mddev->parallel_resync
9805	&& mddev2->curr_resync
9806	&& match_mddev_units(mddev, mddev2)) {
9807	DEFINE_WAIT(wq);
9808	if (mddev < mddev2 &&
9809	mddev->curr_resync == MD_RESYNC_DELAYED) {
9810	/* arbitrarily yield */
9811	mddev->curr_resync = MD_RESYNC_YIELDED;
9812	wake_up(&resync_wait);
9813	}
9814	if (mddev > mddev2 &&
9815	mddev->curr_resync == MD_RESYNC_YIELDED)
9816	/* no need to wait here, we can wait the next
9817	* time 'round when curr_resync == 2
9818	*/
9819	continue;
9820	/* We need to wait 'interruptible' so as not to
9821	* contribute to the load average, and not to
9822	* be caught by 'softlockup'
9823	*/
9824	prepare_to_wait(&resync_wait, &wq, TASK_INTERRUPTIBLE);
9825	if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery) &&
9826	mddev2->curr_resync >= mddev->curr_resync) {
9827	if (mddev2_minor != mddev2->md_minor) {
9828	mddev2_minor = mddev2->md_minor;
9829	pr_info("md: delaying %s of %s until %s has finished (they share one or more physical units)\n",
9830	desc, mdname(mddev),
9831	mdname(mddev2));
9832	}
9833	spin_unlock(&all_mddevs_lock);
9834
9835	if (signal_pending(current))
9836	flush_signals(current);
9837	schedule();
9838	finish_wait(&resync_wait, &wq);
9839	goto try_again;
9840	}
9841	finish_wait(&resync_wait, &wq);
9842	}
9843	}
9844	spin_unlock(&all_mddevs_lock);
9845	} while (mddev->curr_resync < MD_RESYNC_DELAYED);
9846
9847	max_sectors = md_sync_max_sectors(mddev, action);
9848	j = md_sync_position(mddev, action);
9849
9850	pr_info("md: %s of RAID array %s\n", desc, mdname(mddev));
9851	pr_debug("md: minimum _guaranteed_ speed: %d KB/sec/disk.\n", speed_min(mddev));
9852	pr_debug("md: using maximum available idle IO bandwidth (but not more than %d KB/sec) for %s.\n",
9853	speed_max(mddev), desc);
9854
9855	is_mddev_idle(mddev, 1); /* this initializes IO event counters */
9856
9857	io_sectors = 0;
9858	for (m = 0; m < SYNC_MARKS; m++) {
9859	mark[m] = jiffies;
9860	mark_cnt[m] = io_sectors;
9861	}
9862	last_mark = 0;
9863	mddev->resync_mark = mark[last_mark];
9864	mddev->resync_mark_cnt = mark_cnt[last_mark];
9865
9866	/*
9867	* Tune reconstruction:
9868	*/
9869	window = 32 * (PAGE_SIZE / 512);
9870	pr_debug("md: using %dk window, over a total of %lluk.\n",
9871	window/2, (unsigned long long)max_sectors/2);
9872
9873	atomic_set(&mddev->recovery_active, 0);
9874	last_check = 0;
9875
9876	if (j >= MD_RESYNC_ACTIVE) {
9877	pr_debug("md: resuming %s of %s from checkpoint.\n",
9878	desc, mdname(mddev));
9879	mddev->curr_resync = j;
9880	} else
9881	mddev->curr_resync = MD_RESYNC_ACTIVE; /* no longer delayed */
9882	mddev->curr_resync_completed = j;
9883	sysfs_notify_dirent_safe(mddev->sysfs_completed);
9884	md_new_event();
9885	update_time = jiffies;
9886
9887	blk_start_plug(&plug);
9888	while (j < max_sectors) {
9889	sector_t sectors;
9890
9891	skipped = 0;
9892
9893	if (!test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
9894	((mddev->curr_resync > mddev->curr_resync_completed &&
9895	(mddev->curr_resync - mddev->curr_resync_completed)
9896	> (max_sectors >> 4)) \|\|
9897	time_after_eq(jiffies, update_time + UPDATE_FREQUENCY) \|\|
9898	(j - mddev->curr_resync_completed)*2
9899	>= mddev->resync_max - mddev->curr_resync_completed \|\|
9900	mddev->curr_resync_completed > mddev->resync_max
9901	)) {
9902	/* time to update curr_resync_completed */
9903	wait_event(mddev->recovery_wait,
9904	atomic_read(&mddev->recovery_active) == 0);
9905	mddev->curr_resync_completed = j;
9906	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) &&
9907	j > mddev->resync_offset)
9908	mddev->resync_offset = j;
9909	update_time = jiffies;
9910	set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
9911	sysfs_notify_dirent_safe(mddev->sysfs_completed);
9912	}
9913
9914	while (j >= mddev->resync_max &&
9915	!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
9916	/* As this condition is controlled by user-space,
9917	* we can block indefinitely, so use '_interruptible'
9918	* to avoid triggering warnings.
9919	*/
9920	flush_signals(current); /* just in case */
9921	wait_event_interruptible(mddev->recovery_wait,
9922	mddev->resync_max > j
9923	\|\| test_bit(MD_RECOVERY_INTR,
9924	&mddev->recovery));
9925	}
9926
9927	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
9928	break;
9929
9930	if (mddev->bitmap_ops && mddev->bitmap_ops->skip_sync_blocks) {
9931	sectors = mddev->bitmap_ops->skip_sync_blocks(mddev, j);
9932	if (sectors)
9933	goto update;
9934	}
9935
9936	sectors = mddev->pers->sync_request(mddev, j, max_sectors,
9937	&skipped);
9938	if (sectors == 0) {
9939	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
9940	break;
9941	}
9942
9943	if (!skipped) { /* actual IO requested */
9944	io_sectors += sectors;
9945	atomic_add(sectors, &mddev->recovery_active);
9946	}
9947
9948	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
9949	break;
9950
9951	update:
9952	j += sectors;
9953	if (j > max_sectors)
9954	/* when skipping, extra large numbers can be returned. */
9955	j = max_sectors;
9956	if (j >= MD_RESYNC_ACTIVE)
9957	mddev->curr_resync = j;
9958	mddev->curr_mark_cnt = io_sectors;
9959	if (last_check == 0)
9960	/* this is the earliest that rebuild will be
9961	* visible in /proc/mdstat
9962	*/
9963	md_new_event();
9964
9965	if (last_check + window > io_sectors \|\| j == max_sectors) {
9966	cond_resched();
9967	continue;
9968	}
9969
9970	last_check = io_sectors;
9971	repeat:
9972	if (time_after_eq(jiffies, mark[last_mark] + SYNC_MARK_STEP )) {
9973	/* step marks */
9974	int next = (last_mark+1) % SYNC_MARKS;
9975
9976	mddev->resync_mark = mark[next];
9977	mddev->resync_mark_cnt = mark_cnt[next];
9978	mark[next] = jiffies;
9979	mark_cnt[next] = io_sectors - atomic_read(&mddev->recovery_active);
9980	last_mark = next;
9981	}
9982
9983	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
9984	break;
9985
9986	/*
9987	* this loop exits only if either when we are slower than
9988	* the 'hard' speed limit, or the system was IO-idle for
9989	* a jiffy.
9990	* the system might be non-idle CPU-wise, but we only care
9991	* about not overloading the IO subsystem. (things like an
9992	* e2fsck being done on the RAID array should execute fast)
9993	*/
9994	cond_resched();
9995
9996	recovery_done = io_sectors - atomic_read(&mddev->recovery_active);
9997	currspeed = ((unsigned long)(recovery_done - mddev->resync_mark_cnt))/2
9998	/((jiffies-mddev->resync_mark)/HZ +1) +1;
9999
10000	if (currspeed > speed_min(mddev)) {
10001	if (currspeed > speed_max(mddev)) {
10002	msleep(500);
10003	goto repeat;
10004	}
10005	if (!sync_io_within_limit(mddev) &&
10006	!is_mddev_idle(mddev, 0)) {
10007	/*
10008	* Give other IO more of a chance.
10009	* The faster the devices, the less we wait.
10010	*/
10011	wait_event(mddev->recovery_wait,
10012	!atomic_read(&mddev->recovery_active));
10013	}
10014	}
10015	}
10016	pr_info("md: %s: %s %s.\n",mdname(mddev), desc,
10017	test_bit(MD_RECOVERY_INTR, &mddev->recovery)
10018	? "interrupted" : "done");
10019	/*
10020	* this also signals 'finished resyncing' to md_stop
10021	*/
10022	blk_finish_plug(&plug);
10023	wait_event(mddev->recovery_wait, !atomic_read(&mddev->recovery_active));
10024
10025	if (!test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
10026	mddev->curr_resync >= MD_RESYNC_ACTIVE) {
10027	/* All sync IO completes after recovery_active becomes 0 */
10028	mddev->curr_resync_completed = mddev->curr_resync;
10029	sysfs_notify_dirent_safe(mddev->sysfs_completed);
10030	}
10031	mddev->pers->sync_request(mddev, max_sectors, max_sectors, &skipped);
10032
10033	if (mddev->curr_resync > MD_RESYNC_ACTIVE)
10034	md_finish_sync(mddev, action);
10035	skip:
10036	/* set CHANGE_PENDING here since maybe another update is needed,
10037	* so other nodes are informed. It should be harmless for normal
10038	* raid */
10039	set_mask_bits(&mddev->sb_flags, 0,
10040	BIT(MD_SB_CHANGE_PENDING) \| BIT(MD_SB_CHANGE_DEVS));
10041	spin_lock(&mddev->lock);
10042	if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
10043	/* We completed so min/max setting can be forgotten if used. */
10044	if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery))
10045	mddev->resync_min = 0;
10046	mddev->resync_max = MaxSector;
10047	} else if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery))
10048	mddev->resync_min = mddev->curr_resync_completed;
10049	set_bit(MD_RECOVERY_DONE, &mddev->recovery);
10050	mddev->curr_resync = MD_RESYNC_NONE;
10051	spin_unlock(&mddev->lock);
10052
10053	wake_up(&resync_wait);
10054	md_wakeup_thread(mddev->thread);
10055	return;
10056	}
10057	EXPORT_SYMBOL_GPL(md_do_sync);
10058
10059	static bool rdev_removeable(struct md_rdev *rdev)
10060	{
10061	/* rdev is not used. */
10062	if (rdev->raid_disk < 0)
10063	return false;
10064
10065	/* There are still inflight io, don't remove this rdev. */
10066	if (atomic_read(&rdev->nr_pending))
10067	return false;
10068
10069	/*
10070	* An error occurred but has not yet been acknowledged by the metadata
10071	* handler, don't remove this rdev.
10072	*/
10073	if (test_bit(Blocked, &rdev->flags))
10074	return false;
10075
10076	/* Fautly rdev is not used, it's safe to remove it. */
10077	if (test_bit(Faulty, &rdev->flags))
10078	return true;
10079
10080	/* Journal disk can only be removed if it's faulty. */
10081	if (test_bit(Journal, &rdev->flags))
10082	return false;
10083
10084	/*
10085	* 'In_sync' is cleared while 'raid_disk' is valid, which means
10086	* replacement has just become active from pers->spare_active(), and
10087	* then pers->hot_remove_disk() will replace this rdev with replacement.
10088	*/
10089	if (!test_bit(In_sync, &rdev->flags))
10090	return true;
10091
10092	return false;
10093	}
10094
10095	static bool rdev_is_spare(struct md_rdev *rdev)
10096	{
10097	return !test_bit(Candidate, &rdev->flags) && rdev->raid_disk >= 0 &&
10098	!test_bit(In_sync, &rdev->flags) &&
10099	!test_bit(Journal, &rdev->flags) &&
10100	!test_bit(Faulty, &rdev->flags);
10101	}
10102
10103	static bool rdev_addable(struct md_rdev *rdev)
10104	{
10105	struct mddev *mddev;
10106
10107	mddev = READ_ONCE(rdev->mddev);
10108	if (!mddev)
10109	return false;
10110
10111	/* rdev is already used, don't add it again. */
10112	if (test_bit(Candidate, &rdev->flags) \|\| rdev->raid_disk >= 0 \|\|
10113	test_bit(Faulty, &rdev->flags))
10114	return false;
10115
10116	/* Allow to add journal disk. */
10117	if (test_bit(Journal, &rdev->flags))
10118	return true;
10119
10120	/* Allow to add if array is read-write. */
10121	if (md_is_rdwr(mddev))
10122	return true;
10123
10124	/*
10125	* For read-only array, only allow to readd a rdev. And if bitmap is
10126	* used, don't allow to readd a rdev that is too old.
10127	*/
10128	if (rdev->saved_raid_disk >= 0 && !test_bit(Bitmap_sync, &rdev->flags))
10129	return true;
10130
10131	return false;
10132	}
10133
10134	static bool md_spares_need_change(struct mddev *mddev)
10135	{
10136	struct md_rdev *rdev;
10137
10138	rcu_read_lock();
10139	rdev_for_each_rcu(rdev, mddev) {
10140	if (rdev_removeable(rdev) \|\| rdev_addable(rdev)) {
10141	rcu_read_unlock();
10142	return true;
10143	}
10144	}
10145	rcu_read_unlock();
10146	return false;
10147	}
10148
10149	static int remove_spares(struct mddev mddev, struct md_rdev this)
10150	{
10151	struct md_rdev *rdev;
10152	int removed = 0;
10153
10154	rdev_for_each(rdev, mddev) {
10155	if ((this == NULL \|\| rdev == this) && rdev_removeable(rdev) &&
10156	!mddev->pers->hot_remove_disk(mddev, rdev)) {
10157	sysfs_unlink_rdev(mddev, rdev);
10158	rdev->saved_raid_disk = rdev->raid_disk;
10159	rdev->raid_disk = -1;
10160	removed++;
10161	}
10162	}
10163
10164	if (removed && mddev->kobj.sd)
10165	sysfs_notify_dirent_safe(mddev->sysfs_degraded);
10166
10167	return removed;
10168	}
10169
10170	static int remove_and_add_spares(struct mddev *mddev,
10171	struct md_rdev *this)
10172	{
10173	struct md_rdev *rdev;
10174	int spares = 0;
10175	int removed = 0;
10176
10177	if (this && test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
10178	/* Mustn't remove devices when resync thread is running */
10179	return 0;
10180
10181	removed = remove_spares(mddev, this);
10182	if (this && removed)
10183	goto no_add;
10184
10185	rdev_for_each(rdev, mddev) {
10186	if (this && this != rdev)
10187	continue;
10188	if (rdev_is_spare(rdev))
10189	spares++;
10190	if (!rdev_addable(rdev))
10191	continue;
10192	if (!test_bit(Journal, &rdev->flags))
10193	rdev->recovery_offset = 0;
10194	if (mddev->pers->hot_add_disk(mddev, rdev) == 0) {
10195	/* failure here is OK */
10196	sysfs_link_rdev(mddev, rdev);
10197	if (!test_bit(Journal, &rdev->flags))
10198	spares++;
10199	md_new_event();
10200	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
10201	}
10202	}
10203	no_add:
10204	if (removed)
10205	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
10206	return spares;
10207	}
10208
10209	static bool md_choose_sync_action(struct mddev mddev, int spares)
10210	{
10211	/* Check if reshape is in progress first. */
10212	if (mddev->reshape_position != MaxSector) {
10213	if (mddev->pers->check_reshape == NULL \|\|
10214	mddev->pers->check_reshape(mddev) != 0)
10215	return false;
10216
10217	set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
10218	clear_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
10219	clear_bit(MD_RECOVERY_LAZY_RECOVER, &mddev->recovery);
10220	return true;
10221	}
10222
10223	/* Check if resync is in progress. */
10224	if (mddev->resync_offset < MaxSector) {
10225	remove_spares(mddev, NULL);
10226	set_bit(MD_RECOVERY_SYNC, &mddev->recovery);
10227	clear_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
10228	clear_bit(MD_RECOVERY_LAZY_RECOVER, &mddev->recovery);
10229	return true;
10230	}
10231
10232	/*
10233	* Remove any failed drives, then add spares if possible. Spares are
10234	* also removed and re-added, to allow the personality to fail the
10235	* re-add.
10236	*/
10237	*spares = remove_and_add_spares(mddev, NULL);
10238	if (*spares \|\| test_bit(MD_RECOVERY_LAZY_RECOVER, &mddev->recovery)) {
10239	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
10240	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
10241	clear_bit(MD_RECOVERY_REQUESTED, &mddev->recovery);
10242
10243	/* Start new recovery. */
10244	set_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
10245	return true;
10246	}
10247
10248	/* Delay to choose resync/check/repair in md_do_sync(). */
10249	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
10250	return true;
10251
10252	/* Nothing to be done */
10253	return false;
10254	}
10255
10256	static void md_start_sync(struct work_struct *ws)
10257	{
10258	struct mddev *mddev = container_of(ws, struct mddev, sync_work);
10259	int spares = 0;
10260	bool suspend = false;
10261	unsigned int noio_flags = 0;
10262	char *name;
10263
10264	/*
10265	* If reshape is still in progress, spares won't be added or removed
10266	* from conf until reshape is done.
10267	*/
10268	if ((mddev->reshape_position == MaxSector \|\| !md_is_rdwr(mddev)) &&
10269	md_spares_need_change(mddev)) {
10270	suspend = true;
10271	mddev_suspend(mddev, false);
10272	noio_flags = memalloc_noio_save();
10273	}
10274
10275	mddev_lock_nointr(mddev);
10276
10277	/*
10278	* The spare configuration can change before reconfig_mutex is acquired.
10279	* Recheck while holding the lock and suspend if needed.
10280	*/
10281	if (!suspend && (mddev->reshape_position == MaxSector \|\| !md_is_rdwr(mddev)) &&
10282	md_spares_need_change(mddev)) {
10283	mddev_unlock(mddev);
10284	mddev_suspend_and_lock_nointr(mddev);
10285	suspend = true;
10286	noio_flags = memalloc_noio_save();
10287	}
10288
10289	if (!md_is_rdwr(mddev)) {
10290	/*
10291	* On a read-only array we can:
10292	* - remove failed devices
10293	* - add already-in_sync devices if the array itself is in-sync.
10294	* As we only add devices that are already in-sync, we can
10295	* activate the spares immediately.
10296	*/
10297	remove_and_add_spares(mddev, NULL);
10298	goto not_running;
10299	}
10300
10301	if (!md_choose_sync_action(mddev, &spares))
10302	goto not_running;
10303
10304	if (!mddev->pers->sync_request)
10305	goto not_running;
10306
10307	/*
10308	* We are adding a device or devices to an array which has the bitmap
10309	* stored on all devices. So make sure all bitmap pages get written.
10310	*/
10311	if (spares && md_bitmap_enabled(mddev, true))
10312	mddev->bitmap_ops->write_all(mddev);
10313
10314	name = test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) ?
10315	"reshape" : "resync";
10316	rcu_assign_pointer(mddev->sync_thread,
10317	md_register_thread(md_do_sync, mddev, name));
10318	if (!mddev->sync_thread) {
10319	pr_warn("%s: could not start resync thread...\n",
10320	mdname(mddev));
10321	/* leave the spares where they are, it shouldn't hurt */
10322	goto not_running;
10323	}
10324
10325	mddev_unlock(mddev);
10326	/*
10327	* md_start_sync was triggered by MD_RECOVERY_NEEDED, so we should
10328	* not set it again. Otherwise, we may cause issue like this one:
10329	* https://bugzilla.kernel.org/show_bug.cgi?id=218200
10330	* Therefore, use __mddev_resume(mddev, false).
10331	*/
10332	if (suspend) {
10333	memalloc_noio_restore(noio_flags);
10334	__mddev_resume(mddev, false);
10335	}
10336	md_wakeup_thread(mddev->sync_thread);
10337	sysfs_notify_dirent_safe(mddev->sysfs_action);
10338	md_new_event();
10339	return;
10340
10341	not_running:
10342	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
10343	clear_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
10344	clear_bit(MD_RECOVERY_REQUESTED, &mddev->recovery);
10345	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
10346	clear_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
10347	mddev_unlock(mddev);
10348	/*
10349	* md_start_sync was triggered by MD_RECOVERY_NEEDED, so we should
10350	* not set it again. Otherwise, we may cause issue like this one:
10351	* https://bugzilla.kernel.org/show_bug.cgi?id=218200
10352	* Therefore, use __mddev_resume(mddev, false).
10353	*/
10354	if (suspend) {
10355	memalloc_noio_restore(noio_flags);
10356	__mddev_resume(mddev, false);
10357	}
10358
10359	wake_up(&resync_wait);
10360	if (test_and_clear_bit(MD_RECOVERY_RECOVER, &mddev->recovery) &&
10361	mddev->sysfs_action)
10362	sysfs_notify_dirent_safe(mddev->sysfs_action);
10363	}
10364
10365	static void unregister_sync_thread(struct mddev *mddev)
10366	{
10367	if (!test_bit(MD_RECOVERY_DONE, &mddev->recovery)) {
10368	/* resync/recovery still happening */
10369	clear_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
10370	return;
10371	}
10372
10373	if (WARN_ON_ONCE(!mddev->sync_thread))
10374	return;
10375
10376	md_reap_sync_thread(mddev);
10377	}
10378
10379	static bool md_should_do_recovery(struct mddev *mddev)
10380	{
10381	/*
10382	* As long as one of the following flags is set,
10383	* recovery needs to do or cleanup.
10384	*/
10385	if (test_bit(MD_RECOVERY_NEEDED, &mddev->recovery) \|\|
10386	test_bit(MD_RECOVERY_DONE, &mddev->recovery))
10387	return true;
10388
10389	/*
10390	* If no flags are set and it is in read-only status,
10391	* there is nothing to do.
10392	*/
10393	if (!md_is_rdwr(mddev))
10394	return false;
10395
10396	/*
10397	* MD_SB_CHANGE_PENDING indicates that the array is switching from clean to
10398	* active, and no action is needed for now.
10399	* All other MD_SB_* flags require to update the superblock.
10400	*/
10401	if (mddev->sb_flags & ~ (1<<MD_SB_CHANGE_PENDING))
10402	return true;
10403
10404	/*
10405	* If the array is not using external metadata and there has been no data
10406	* written for some time, then the array's status needs to be set to
10407	* in_sync.
10408	*/
10409	if (mddev->external == 0 && mddev->safemode == 1)
10410	return true;
10411
10412	/*
10413	* When the system is about to restart or the process receives an signal,
10414	* the array needs to be synchronized as soon as possible.
10415	* Once the data synchronization is completed, need to change the array
10416	* status to in_sync.
10417	*/
10418	if (mddev->safemode == 2 && !mddev->in_sync &&
10419	mddev->resync_offset == MaxSector)
10420	return true;
10421
10422	return false;
10423	}
10424
10425	/*
10426	* This routine is regularly called by all per-raid-array threads to
10427	* deal with generic issues like resync and super-block update.
10428	* Raid personalities that don't have a thread (linear/raid0) do not
10429	* need this as they never do any recovery or update the superblock.
10430	*
10431	* It does not do any resync itself, but rather "forks" off other threads
10432	* to do that as needed.
10433	* When it is determined that resync is needed, we set MD_RECOVERY_RUNNING in
10434	* "->recovery" and create a thread at ->sync_thread.
10435	* When the thread finishes it sets MD_RECOVERY_DONE
10436	* and wakeups up this thread which will reap the thread and finish up.
10437	* This thread also removes any faulty devices (with nr_pending == 0).
10438	*
10439	* The overall approach is:
10440	* 1/ if the superblock needs updating, update it.
10441	* 2/ If a recovery thread is running, don't do anything else.
10442	* 3/ If recovery has finished, clean up, possibly marking spares active.
10443	* 4/ If there are any faulty devices, remove them.
10444	* 5/ If array is degraded, try to add spares devices
10445	* 6/ If array has spares or is not in-sync, start a resync thread.
10446	*/
10447	void md_check_recovery(struct mddev *mddev)
10448	{
10449	if (md_bitmap_enabled(mddev, false) && mddev->bitmap_ops->daemon_work)
10450	mddev->bitmap_ops->daemon_work(mddev);
10451
10452	if (signal_pending(current)) {
10453	if (mddev->pers->sync_request && !mddev->external) {
10454	pr_debug("md: %s in immediate safe mode\n",
10455	mdname(mddev));
10456	mddev->safemode = 2;
10457	}
10458	flush_signals(current);
10459	}
10460
10461	if (!md_should_do_recovery(mddev))
10462	return;
10463
10464	if (mddev_trylock(mddev)) {
10465	bool try_set_sync = mddev->safemode != 0;
10466
10467	if (!mddev->external && mddev->safemode == 1)
10468	mddev->safemode = 0;
10469
10470	if (!md_is_rdwr(mddev)) {
10471	struct md_rdev *rdev;
10472
10473	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) {
10474	unregister_sync_thread(mddev);
10475	goto unlock;
10476	}
10477
10478	if (!mddev->external && mddev->in_sync)
10479	/*
10480	* 'Blocked' flag not needed as failed devices
10481	* will be recorded if array switched to read/write.
10482	* Leaving it set will prevent the device
10483	* from being removed.
10484	*/
10485	rdev_for_each(rdev, mddev)
10486	clear_bit(Blocked, &rdev->flags);
10487
10488	/*
10489	* There is no thread, but we need to call
10490	* ->spare_active and clear saved_raid_disk
10491	*/
10492	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
10493	md_reap_sync_thread(mddev);
10494
10495	/*
10496	* Let md_start_sync() to remove and add rdevs to the
10497	* array.
10498	*/
10499	if (md_spares_need_change(mddev)) {
10500	set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
10501	queue_work(md_misc_wq, &mddev->sync_work);
10502	}
10503
10504	clear_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
10505	clear_bit(MD_RECOVERY_LAZY_RECOVER, &mddev->recovery);
10506	clear_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
10507	clear_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
10508
10509	goto unlock;
10510	}
10511
10512	if (mddev_is_clustered(mddev)) {
10513	struct md_rdev rdev, tmp;
10514	/* kick the device if another node issued a
10515	* remove disk.
10516	*/
10517	rdev_for_each_safe(rdev, tmp, mddev) {
10518	if (rdev->raid_disk < 0 &&
10519	test_and_clear_bit(ClusterRemove, &rdev->flags))
10520	md_kick_rdev_from_array(rdev);
10521	}
10522	}
10523
10524	if (try_set_sync && !mddev->external && !mddev->in_sync) {
10525	spin_lock(&mddev->lock);
10526	set_in_sync(mddev);
10527	spin_unlock(&mddev->lock);
10528	}
10529
10530	if (mddev->sb_flags)
10531	md_update_sb(mddev, 0);
10532
10533	/*
10534	* Never start a new sync thread if MD_RECOVERY_RUNNING is
10535	* still set.
10536	*/
10537	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) {
10538	unregister_sync_thread(mddev);
10539	goto unlock;
10540	}
10541
10542	/* Set RUNNING before clearing NEEDED to avoid
10543	* any transients in the value of "sync_action".
10544	*/
10545	mddev->curr_resync_completed = 0;
10546	spin_lock(&mddev->lock);
10547	set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
10548	spin_unlock(&mddev->lock);
10549	/* Clear some bits that don't mean anything, but
10550	* might be left set
10551	*/
10552	clear_bit(MD_RECOVERY_INTR, &mddev->recovery);
10553	clear_bit(MD_RECOVERY_DONE, &mddev->recovery);
10554
10555	if (test_and_clear_bit(MD_RECOVERY_NEEDED, &mddev->recovery) &&
10556	!test_bit(MD_RECOVERY_FROZEN, &mddev->recovery)) {
10557	queue_work(md_misc_wq, &mddev->sync_work);
10558	} else {
10559	clear_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
10560	wake_up(&resync_wait);
10561	}
10562
10563	unlock:
10564	wake_up(&mddev->sb_wait);
10565	mddev_unlock(mddev);
10566	}
10567	}
10568	EXPORT_SYMBOL(md_check_recovery);
10569
10570	void md_reap_sync_thread(struct mddev *mddev)
10571	{
10572	struct md_rdev *rdev;
10573	sector_t old_dev_sectors = mddev->dev_sectors;
10574	bool is_reshaped = test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
10575
10576	/* resync has finished, collect result */
10577	md_unregister_thread(mddev, &mddev->sync_thread);
10578	atomic_inc(&mddev->sync_seq);
10579
10580	if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery) &&
10581	!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
10582	mddev->degraded != mddev->raid_disks) {
10583	/* success...*/
10584	/* activate any spares */
10585	if (mddev->pers->spare_active(mddev)) {
10586	sysfs_notify_dirent_safe(mddev->sysfs_degraded);
10587	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
10588	}
10589	}
10590
10591	/* If array is no-longer degraded, then any saved_raid_disk
10592	* information must be scrapped.
10593	*/
10594	if (!mddev->degraded)
10595	rdev_for_each(rdev, mddev)
10596	rdev->saved_raid_disk = -1;
10597
10598	md_update_sb(mddev, 1);
10599	/* MD_SB_CHANGE_PENDING should be cleared by md_update_sb, so we can
10600	* call resync_finish here if MD_CLUSTER_RESYNC_LOCKED is set by
10601	* clustered raid */
10602	if (test_and_clear_bit(MD_CLUSTER_RESYNC_LOCKED, &mddev->flags))
10603	mddev->cluster_ops->resync_finish(mddev);
10604	clear_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
10605	clear_bit(MD_RECOVERY_DONE, &mddev->recovery);
10606	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
10607	clear_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
10608	clear_bit(MD_RECOVERY_REQUESTED, &mddev->recovery);
10609	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
10610	clear_bit(MD_RECOVERY_LAZY_RECOVER, &mddev->recovery);
10611	/*
10612	* We call mddev->cluster_ops->update_size here because sync_size could
10613	* be changed by md_update_sb, and MD_RECOVERY_RESHAPE is cleared,
10614	* so it is time to update size across cluster.
10615	*/
10616	if (mddev_is_clustered(mddev) && is_reshaped &&
10617	mddev->pers->finish_reshape &&
10618	!test_bit(MD_CLOSING, &mddev->flags))
10619	mddev->cluster_ops->update_size(mddev, old_dev_sectors);
10620	/* flag recovery needed just to double check */
10621	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
10622	sysfs_notify_dirent_safe(mddev->sysfs_completed);
10623	sysfs_notify_dirent_safe(mddev->sysfs_action);
10624	md_new_event();
10625	if (mddev->event_work.func)
10626	queue_work(md_misc_wq, &mddev->event_work);
10627	wake_up(&resync_wait);
10628	}
10629	EXPORT_SYMBOL(md_reap_sync_thread);
10630
10631	void md_wait_for_blocked_rdev(struct md_rdev rdev, struct mddev mddev)
10632	{
10633	sysfs_notify_dirent_safe(rdev->sysfs_state);
10634	wait_event_timeout(rdev->blocked_wait, !rdev_blocked(rdev),
10635	msecs_to_jiffies(5000));
10636	rdev_dec_pending(rdev, mddev);
10637	}
10638	EXPORT_SYMBOL(md_wait_for_blocked_rdev);
10639
10640	void md_finish_reshape(struct mddev *mddev)
10641	{
10642	/* called be personality module when reshape completes. */
10643	struct md_rdev *rdev;
10644
10645	rdev_for_each(rdev, mddev) {
10646	if (rdev->data_offset > rdev->new_data_offset)
10647	rdev->sectors += rdev->data_offset - rdev->new_data_offset;
10648	else
10649	rdev->sectors -= rdev->new_data_offset - rdev->data_offset;
10650	rdev->data_offset = rdev->new_data_offset;
10651	}
10652	}
10653	EXPORT_SYMBOL(md_finish_reshape);
10654
10655	/* Bad block management */
10656
10657	/* Returns true on success, false on failure */
10658	bool rdev_set_badblocks(struct md_rdev *rdev, sector_t s, sector_t sectors,
10659	int is_new)
10660	{
10661	struct mddev *mddev = rdev->mddev;
10662
10663	/*
10664	* Recording new badblocks for faulty rdev will force unnecessary
10665	* super block updating. This is fragile for external management because
10666	* userspace daemon may trying to remove this device and deadlock may
10667	* occur. This will be probably solved in the mdadm, but it is safer to
10668	* avoid it.
10669	*/
10670	if (test_bit(Faulty, &rdev->flags))
10671	return true;
10672
10673	if (is_new)
10674	s += rdev->new_data_offset;
10675	else
10676	s += rdev->data_offset;
10677
10678	if (!badblocks_set(&rdev->badblocks, s, sectors, 0)) {
10679	/*
10680	* Mark the disk as Faulty when setting badblocks fails,
10681	* otherwise, bad sectors may be read.
10682	*/
10683	md_error(mddev, rdev);
10684	return false;
10685	}
10686
10687	/* Make sure they get written out promptly */
10688	if (test_bit(ExternalBbl, &rdev->flags))
10689	sysfs_notify_dirent_safe(rdev->sysfs_unack_badblocks);
10690	sysfs_notify_dirent_safe(rdev->sysfs_state);
10691	set_mask_bits(&mddev->sb_flags, 0,
10692	BIT(MD_SB_CHANGE_CLEAN) \| BIT(MD_SB_CHANGE_PENDING));
10693	md_wakeup_thread(rdev->mddev->thread);
10694	return true;
10695	}
10696	EXPORT_SYMBOL_GPL(rdev_set_badblocks);
10697
10698	void rdev_clear_badblocks(struct md_rdev *rdev, sector_t s, sector_t sectors,
10699	int is_new)
10700	{
10701	if (is_new)
10702	s += rdev->new_data_offset;
10703	else
10704	s += rdev->data_offset;
10705
10706	if (!badblocks_clear(&rdev->badblocks, s, sectors))
10707	return;
10708
10709	if (test_bit(ExternalBbl, &rdev->flags))
10710	sysfs_notify_dirent_safe(rdev->sysfs_badblocks);
10711	}
10712	EXPORT_SYMBOL_GPL(rdev_clear_badblocks);
10713
10714	static int md_notify_reboot(struct notifier_block *this,
10715	unsigned long code, void *x)
10716	{
10717	struct mddev *mddev;
10718
10719	spin_lock(&all_mddevs_lock);
10720	list_for_each_entry(mddev, &all_mddevs, all_mddevs) {
10721	if (!mddev_get(mddev))
10722	continue;
10723	spin_unlock(&all_mddevs_lock);
10724	if (mddev_trylock(mddev)) {
10725	if (mddev->pers)
10726	__md_stop_writes(mddev);
10727	if (mddev->persistent)
10728	mddev->safemode = 2;
10729	mddev_unlock(mddev);
10730	}
10731	spin_lock(&all_mddevs_lock);
10732	mddev_put_locked(mddev);
10733	}
10734	spin_unlock(&all_mddevs_lock);
10735
10736	return NOTIFY_DONE;
10737	}
10738
10739	static struct notifier_block md_notifier = {
10740	.notifier_call = md_notify_reboot,
10741	.next = NULL,
10742	.priority = INT_MAX, /* before any real devices */
10743	};
10744
10745	static void md_geninit(void)
10746	{
10747	pr_debug("md: sizeof(mdp_super_t) = %d\n", (int)sizeof(mdp_super_t));
10748
10749	proc_create("mdstat", S_IRUGO, NULL, &mdstat_proc_ops);
10750	}
10751
10752	static int __init md_init(void)
10753	{
10754	int ret = md_bitmap_init();
10755
10756	if (ret)
10757	return ret;
10758
10759	ret = md_llbitmap_init();
10760	if (ret)
10761	goto err_bitmap;
10762
10763	ret = -ENOMEM;
10764	md_misc_wq = alloc_workqueue("md_misc", WQ_PERCPU, 0);
10765	if (!md_misc_wq)
10766	goto err_misc_wq;
10767
10768	ret = __register_blkdev(MD_MAJOR, "md", md_probe);
10769	if (ret < 0)
10770	goto err_md;
10771
10772	ret = __register_blkdev(0, "mdp", md_probe);
10773	if (ret < 0)
10774	goto err_mdp;
10775	mdp_major = ret;
10776
10777	register_reboot_notifier(&md_notifier);
10778	raid_table_header = register_sysctl("dev/raid", raid_table);
10779
10780	md_geninit();
10781	return 0;
10782
10783	err_mdp:
10784	unregister_blkdev(MD_MAJOR, "md");
10785	err_md:
10786	destroy_workqueue(md_misc_wq);
10787	err_misc_wq:
10788	md_llbitmap_exit();
10789	err_bitmap:
10790	md_bitmap_exit();
10791	return ret;
10792	}
10793
10794	static void check_sb_changes(struct mddev mddev, struct md_rdev rdev)
10795	{
10796	struct mdp_superblock_1 *sb = page_address(rdev->sb_page);
10797	struct md_rdev rdev2, tmp;
10798	int role, ret;
10799
10800	/*
10801	* If size is changed in another node then we need to
10802	* do resize as well.
10803	*/
10804	if (mddev->dev_sectors != le64_to_cpu(sb->size)) {
10805	ret = mddev->pers->resize(mddev, le64_to_cpu(sb->size));
10806	if (ret)
10807	pr_info("md-cluster: resize failed\n");
10808	else if (md_bitmap_enabled(mddev, false))
10809	mddev->bitmap_ops->update_sb(mddev->bitmap);
10810	}
10811
10812	/* Check for change of roles in the active devices */
10813	rdev_for_each_safe(rdev2, tmp, mddev) {
10814	if (test_bit(Faulty, &rdev2->flags)) {
10815	if (test_bit(ClusterRemove, &rdev2->flags))
10816	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
10817	continue;
10818	}
10819
10820	/* Check if the roles changed */
10821	role = le16_to_cpu(sb->dev_roles[rdev2->desc_nr]);
10822
10823	if (test_bit(Candidate, &rdev2->flags)) {
10824	if (role == MD_DISK_ROLE_FAULTY) {
10825	pr_info("md: Removing Candidate device %pg because add failed\n",
10826	rdev2->bdev);
10827	md_kick_rdev_from_array(rdev2);
10828	continue;
10829	}
10830	else
10831	clear_bit(Candidate, &rdev2->flags);
10832	}
10833
10834	if (role != rdev2->raid_disk) {
10835	/*
10836	* got activated except reshape is happening.
10837	*/
10838	if (rdev2->raid_disk == -1 && role != MD_DISK_ROLE_SPARE &&
10839	!(le32_to_cpu(sb->feature_map) &
10840	MD_FEATURE_RESHAPE_ACTIVE) &&
10841	!mddev->cluster_ops->resync_status_get(mddev)) {
10842	/*
10843	* -1 to make raid1_add_disk() set conf->fullsync
10844	* to 1. This could avoid skipping sync when the
10845	* remote node is down during resyncing.
10846	*/
10847	if ((le32_to_cpu(sb->feature_map)
10848	& MD_FEATURE_RECOVERY_OFFSET))
10849	rdev2->saved_raid_disk = -1;
10850	else
10851	rdev2->saved_raid_disk = role;
10852	ret = remove_and_add_spares(mddev, rdev2);
10853	pr_info("Activated spare: %pg\n",
10854	rdev2->bdev);
10855	/* wakeup mddev->thread here, so array could
10856	* perform resync with the new activated disk */
10857	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
10858	md_wakeup_thread(mddev->thread);
10859	}
10860	/* device faulty
10861	* We just want to do the minimum to mark the disk
10862	* as faulty. The recovery is performed by the
10863	* one who initiated the error.
10864	*/
10865	if (role == MD_DISK_ROLE_FAULTY \|\|
10866	role == MD_DISK_ROLE_JOURNAL) {
10867	md_error(mddev, rdev2);
10868	clear_bit(Blocked, &rdev2->flags);
10869	}
10870	}
10871	}
10872
10873	if (mddev->raid_disks != le32_to_cpu(sb->raid_disks)) {
10874	ret = update_raid_disks(mddev, le32_to_cpu(sb->raid_disks));
10875	if (ret)
10876	pr_warn("md: updating array disks failed. %d\n", ret);
10877	}
10878
10879	/*
10880	* Since mddev->delta_disks has already updated in update_raid_disks,
10881	* so it is time to check reshape.
10882	*/
10883	if (test_bit(MD_RESYNCING_REMOTE, &mddev->recovery) &&
10884	(le32_to_cpu(sb->feature_map) & MD_FEATURE_RESHAPE_ACTIVE)) {
10885	/*
10886	* reshape is happening in the remote node, we need to
10887	* update reshape_position and call start_reshape.
10888	*/
10889	mddev->reshape_position = le64_to_cpu(sb->reshape_position);
10890	if (mddev->pers->update_reshape_pos)
10891	mddev->pers->update_reshape_pos(mddev);
10892	if (mddev->pers->start_reshape)
10893	mddev->pers->start_reshape(mddev);
10894	} else if (test_bit(MD_RESYNCING_REMOTE, &mddev->recovery) &&
10895	mddev->reshape_position != MaxSector &&
10896	!(le32_to_cpu(sb->feature_map) & MD_FEATURE_RESHAPE_ACTIVE)) {
10897	/* reshape is just done in another node. */
10898	mddev->reshape_position = MaxSector;
10899	if (mddev->pers->update_reshape_pos)
10900	mddev->pers->update_reshape_pos(mddev);
10901	}
10902
10903	/* Finally set the event to be up to date */
10904	mddev->events = le64_to_cpu(sb->events);
10905	}
10906
10907	static int read_rdev(struct mddev mddev, struct md_rdev rdev)
10908	{
10909	int err;
10910	struct page *swapout = rdev->sb_page;
10911	struct mdp_superblock_1 *sb;
10912
10913	/* Store the sb page of the rdev in the swapout temporary
10914	* variable in case we err in the future
10915	*/
10916	rdev->sb_page = NULL;
10917	err = alloc_disk_sb(rdev);
10918	if (err == 0) {
10919	ClearPageUptodate(rdev->sb_page);
10920	rdev->sb_loaded = 0;
10921	err = super_types[mddev->major_version].
10922	load_super(rdev, NULL, mddev->minor_version);
10923	}
10924	if (err < 0) {
10925	pr_warn("%s: %d Could not reload rdev(%d) err: %d. Restoring old values\n",
10926	__func__, __LINE__, rdev->desc_nr, err);
10927	if (rdev->sb_page)
10928	put_page(rdev->sb_page);
10929	rdev->sb_page = swapout;
10930	rdev->sb_loaded = 1;
10931	return err;
10932	}
10933
10934	sb = page_address(rdev->sb_page);
10935	/* Read the offset unconditionally, even if MD_FEATURE_RECOVERY_OFFSET
10936	* is not set
10937	*/
10938
10939	if ((le32_to_cpu(sb->feature_map) & MD_FEATURE_RECOVERY_OFFSET))
10940	rdev->recovery_offset = le64_to_cpu(sb->recovery_offset);
10941
10942	/* The other node finished recovery, call spare_active to set
10943	* device In_sync and mddev->degraded
10944	*/
10945	if (rdev->recovery_offset == MaxSector &&
10946	!test_bit(In_sync, &rdev->flags) &&
10947	mddev->pers->spare_active(mddev))
10948	sysfs_notify_dirent_safe(mddev->sysfs_degraded);
10949
10950	put_page(swapout);
10951	return 0;
10952	}
10953
10954	void md_reload_sb(struct mddev *mddev, int nr)
10955	{
10956	struct md_rdev rdev = NULL, iter;
10957	int err;
10958
10959	/* Find the rdev */
10960	rdev_for_each_rcu(iter, mddev) {
10961	if (iter->desc_nr == nr) {
10962	rdev = iter;
10963	break;
10964	}
10965	}
10966
10967	if (!rdev) {
10968	pr_warn("%s: %d Could not find rdev with nr %d\n", __func__, __LINE__, nr);
10969	return;
10970	}
10971
10972	err = read_rdev(mddev, rdev);
10973	if (err < 0)
10974	return;
10975
10976	check_sb_changes(mddev, rdev);
10977
10978	/* Read all rdev's to update recovery_offset */
10979	rdev_for_each_rcu(rdev, mddev) {
10980	if (!test_bit(Faulty, &rdev->flags))
10981	read_rdev(mddev, rdev);
10982	}
10983	}
10984	EXPORT_SYMBOL(md_reload_sb);
10985
10986	#ifndef MODULE
10987
10988	/*
10989	* Searches all registered partitions for autorun RAID arrays
10990	* at boot time.
10991	*/
10992
10993	static DEFINE_MUTEX(detected_devices_mutex);
10994	static LIST_HEAD(all_detected_devices);
10995	struct detected_devices_node {
10996	struct list_head list;
10997	dev_t dev;
10998	};
10999
11000	void md_autodetect_dev(dev_t dev)
11001	{
11002	struct detected_devices_node *node_detected_dev;
11003
11004	node_detected_dev = kzalloc_obj(*node_detected_dev);
11005	if (node_detected_dev) {
11006	node_detected_dev->dev = dev;
11007	mutex_lock(&detected_devices_mutex);
11008	list_add_tail(&node_detected_dev->list, &all_detected_devices);
11009	mutex_unlock(&detected_devices_mutex);
11010	}
11011	}
11012
11013	void md_autostart_arrays(int part)
11014	{
11015	struct md_rdev *rdev;
11016	struct detected_devices_node *node_detected_dev;
11017	dev_t dev;
11018	int i_scanned, i_passed;
11019
11020	i_scanned = 0;
11021	i_passed = 0;
11022
11023	pr_info("md: Autodetecting RAID arrays.\n");
11024
11025	mutex_lock(&detected_devices_mutex);
11026	while (!list_empty(&all_detected_devices) && i_scanned < INT_MAX) {
11027	i_scanned++;
11028	node_detected_dev = list_entry(all_detected_devices.next,
11029	struct detected_devices_node, list);
11030	list_del(&node_detected_dev->list);
11031	dev = node_detected_dev->dev;
11032	kfree(node_detected_dev);
11033	mutex_unlock(&detected_devices_mutex);
11034	rdev = md_import_device(dev,0, 90);
11035	mutex_lock(&detected_devices_mutex);
11036	if (IS_ERR(rdev))
11037	continue;
11038
11039	if (test_bit(Faulty, &rdev->flags))
11040	continue;
11041
11042	set_bit(AutoDetected, &rdev->flags);
11043	list_add(&rdev->same_set, &pending_raid_disks);
11044	i_passed++;
11045	}
11046	mutex_unlock(&detected_devices_mutex);
11047
11048	pr_debug("md: Scanned %d and added %d devices.\n", i_scanned, i_passed);
11049
11050	autorun_devices(part);
11051	}
11052
11053	#endif /* !MODULE */
11054
11055	static __exit void md_exit(void)
11056	{
11057	struct mddev *mddev;
11058	int delay = 1;
11059
11060	unregister_blkdev(MD_MAJOR,"md");
11061	unregister_blkdev(mdp_major, "mdp");
11062	unregister_reboot_notifier(&md_notifier);
11063	unregister_sysctl_table(raid_table_header);
11064
11065	/* We cannot unload the modules while some process is
11066	* waiting for us in select() or poll() - wake them up
11067	*/
11068	md_unloading = 1;
11069	while (waitqueue_active(&md_event_waiters)) {
11070	/* not safe to leave yet */
11071	wake_up(&md_event_waiters);
11072	msleep(delay);
11073	delay += delay;
11074	}
11075	remove_proc_entry("mdstat", NULL);
11076
11077	spin_lock(&all_mddevs_lock);
11078	list_for_each_entry(mddev, &all_mddevs, all_mddevs) {
11079	if (!mddev_get(mddev))
11080	continue;
11081	spin_unlock(&all_mddevs_lock);
11082	export_array(mddev);
11083	mddev->ctime = 0;
11084	mddev->hold_active = 0;
11085	/*
11086	* As the mddev is now fully clear, mddev_put will schedule
11087	* the mddev for destruction by a workqueue, and the
11088	* destroy_workqueue() below will wait for that to complete.
11089	*/
11090	spin_lock(&all_mddevs_lock);
11091	mddev_put_locked(mddev);
11092	}
11093	spin_unlock(&all_mddevs_lock);
11094
11095	destroy_workqueue(md_misc_wq);
11096	md_bitmap_exit();
11097	}
11098
11099	subsys_initcall(md_init);
11100	module_exit(md_exit)
11101
11102	static int get_ro(char buffer, const struct kernel_param kp)
11103	{
11104	return sprintf(buffer, "%d\n", start_readonly);
11105	}
11106	static int set_ro(const char val, const struct kernel_param kp)
11107	{
11108	return kstrtouint(val, 10, (unsigned int *)&start_readonly);
11109	}
11110
11111	module_param_call(start_ro, set_ro, get_ro, NULL, S_IRUSR\|S_IWUSR);
11112	module_param(start_dirty_degraded, int, S_IRUGO\|S_IWUSR);
11113	module_param_call(new_array, add_named_array, NULL, NULL, S_IWUSR);
11114	module_param(create_on_open, bool, S_IRUSR\|S_IWUSR);
11115	module_param(legacy_async_del_gendisk, bool, 0600);
11116	module_param(check_new_feature, bool, 0600);
11117
11118	MODULE_LICENSE("GPL");
11119	MODULE_DESCRIPTION("MD RAID framework");
11120	MODULE_ALIAS("md");
11121	MODULE_ALIAS_BLOCKDEV_MAJOR(MD_MAJOR);