linux - kernel/cgroup.c

6642 lines · 176.3 KiB

1	/*
2	* Generic process-grouping system.
3	*
4	* Based originally on the cpuset system, extracted by Paul Menage
5	* Copyright (C) 2006 Google, Inc
6	*
7	* Notifications support
8	* Copyright (C) 2009 Nokia Corporation
9	* Author: Kirill A. Shutemov
10	*
11	* Copyright notices from the original cpuset code:
12	* --------------------------------------------------
13	* Copyright (C) 2003 BULL SA.
14	* Copyright (C) 2004-2006 Silicon Graphics, Inc.
15	*
16	* Portions derived from Patrick Mochel's sysfs code.
17	* sysfs is Copyright (c) 2001-3 Patrick Mochel
18	*
19	* 2003-10-10 Written by Simon Derr.
20	* 2003-10-22 Updates by Stephen Hemminger.
21	* 2004 May-July Rework by Paul Jackson.
22	* ---------------------------------------------------
23	*
24	* This file is subject to the terms and conditions of the GNU General Public
25	* License. See the file COPYING in the main directory of the Linux
26	* distribution for more details.
27	*/
28
29	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
31	#include <linux/cgroup.h>
32	#include <linux/cred.h>
33	#include <linux/ctype.h>
34	#include <linux/errno.h>
35	#include <linux/init_task.h>
36	#include <linux/kernel.h>
37	#include <linux/list.h>
38	#include <linux/magic.h>
39	#include <linux/mm.h>
40	#include <linux/mutex.h>
41	#include <linux/mount.h>
42	#include <linux/pagemap.h>
43	#include <linux/proc_fs.h>
44	#include <linux/rcupdate.h>
45	#include <linux/sched.h>
46	#include <linux/slab.h>
47	#include <linux/spinlock.h>
48	#include <linux/percpu-rwsem.h>
49	#include <linux/string.h>
50	#include <linux/sort.h>
51	#include <linux/kmod.h>
52	#include <linux/delayacct.h>
53	#include <linux/cgroupstats.h>
54	#include <linux/hashtable.h>
55	#include <linux/pid_namespace.h>
56	#include <linux/idr.h>
57	#include <linux/vmalloc.h> /* TODO: replace with more sophisticated array */
58	#include <linux/kthread.h>
59	#include <linux/delay.h>
60	#include <linux/atomic.h>
61	#include <linux/cpuset.h>
62	#include <linux/proc_ns.h>
63	#include <linux/nsproxy.h>
64	#include <linux/file.h>
65	#include <net/sock.h>
66
67	#define CREATE_TRACE_POINTS
68	#include <trace/events/cgroup.h>
69
70	/*
71	* pidlists linger the following amount before being destroyed. The goal
72	* is avoiding frequent destruction in the middle of consecutive read calls
73	* Expiring in the middle is a performance problem not a correctness one.
74	* 1 sec should be enough.
75	*/
76	#define CGROUP_PIDLIST_DESTROY_DELAY HZ
77
78	#define CGROUP_FILE_NAME_MAX (MAX_CGROUP_TYPE_NAMELEN + \
79	MAX_CFTYPE_NAME + 2)
80
81	/*
82	* cgroup_mutex is the master lock. Any modification to cgroup or its
83	* hierarchy must be performed while holding it.
84	*
85	* css_set_lock protects task->cgroups pointer, the list of css_set
86	* objects, and the chain of tasks off each css_set.
87	*
88	* These locks are exported if CONFIG_PROVE_RCU so that accessors in
89	* cgroup.h can use them for lockdep annotations.
90	*/
91	#ifdef CONFIG_PROVE_RCU
92	DEFINE_MUTEX(cgroup_mutex);
93	DEFINE_SPINLOCK(css_set_lock);
94	EXPORT_SYMBOL_GPL(cgroup_mutex);
95	EXPORT_SYMBOL_GPL(css_set_lock);
96	#else
97	static DEFINE_MUTEX(cgroup_mutex);
98	static DEFINE_SPINLOCK(css_set_lock);
99	#endif
100
101	/*
102	* Protects cgroup_idr and css_idr so that IDs can be released without
103	* grabbing cgroup_mutex.
104	*/
105	static DEFINE_SPINLOCK(cgroup_idr_lock);
106
107	/*
108	* Protects cgroup_file->kn for !self csses. It synchronizes notifications
109	* against file removal/re-creation across css hiding.
110	*/
111	static DEFINE_SPINLOCK(cgroup_file_kn_lock);
112
113	/*
114	* Protects cgroup_subsys->release_agent_path. Modifying it also requires
115	* cgroup_mutex. Reading requires either cgroup_mutex or this spinlock.
116	*/
117	static DEFINE_SPINLOCK(release_agent_path_lock);
118
119	struct percpu_rw_semaphore cgroup_threadgroup_rwsem;
120
121	#define cgroup_assert_mutex_or_rcu_locked() \
122	RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
123	!lockdep_is_held(&cgroup_mutex), \
124	"cgroup_mutex or RCU read lock required");
125
126	/*
127	* cgroup destruction makes heavy use of work items and there can be a lot
128	* of concurrent destructions. Use a separate workqueue so that cgroup
129	* destruction work items don't end up filling up max_active of system_wq
130	* which may lead to deadlock.
131	*/
132	static struct workqueue_struct *cgroup_destroy_wq;
133
134	/*
135	* pidlist destructions need to be flushed on cgroup destruction. Use a
136	* separate workqueue as flush domain.
137	*/
138	static struct workqueue_struct *cgroup_pidlist_destroy_wq;
139
140	/* generate an array of cgroup subsystem pointers */
141	#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys,
142	static struct cgroup_subsys *cgroup_subsys[] = {
143	#include <linux/cgroup_subsys.h>
144	};
145	#undef SUBSYS
146
147	/* array of cgroup subsystem names */
148	#define SUBSYS(_x) [_x ## _cgrp_id] = #_x,
149	static const char *cgroup_subsys_name[] = {
150	#include <linux/cgroup_subsys.h>
151	};
152	#undef SUBSYS
153
154	/* array of static_keys for cgroup_subsys_enabled() and cgroup_subsys_on_dfl() */
155	#define SUBSYS(_x) \
156	DEFINE_STATIC_KEY_TRUE(_x ## _cgrp_subsys_enabled_key); \
157	DEFINE_STATIC_KEY_TRUE(_x ## _cgrp_subsys_on_dfl_key); \
158	EXPORT_SYMBOL_GPL(_x ## _cgrp_subsys_enabled_key); \
159	EXPORT_SYMBOL_GPL(_x ## _cgrp_subsys_on_dfl_key);
160	#include <linux/cgroup_subsys.h>
161	#undef SUBSYS
162
163	#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys_enabled_key,
164	static struct static_key_true *cgroup_subsys_enabled_key[] = {
165	#include <linux/cgroup_subsys.h>
166	};
167	#undef SUBSYS
168
169	#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys_on_dfl_key,
170	static struct static_key_true *cgroup_subsys_on_dfl_key[] = {
171	#include <linux/cgroup_subsys.h>
172	};
173	#undef SUBSYS
174
175	/*
176	* The default hierarchy, reserved for the subsystems that are otherwise
177	* unattached - it never has more than a single cgroup, and all tasks are
178	* part of that cgroup.
179	*/
180	struct cgroup_root cgrp_dfl_root;
181	EXPORT_SYMBOL_GPL(cgrp_dfl_root);
182
183	/*
184	* The default hierarchy always exists but is hidden until mounted for the
185	* first time. This is for backward compatibility.
186	*/
187	static bool cgrp_dfl_visible;
188
189	/* Controllers blocked by the commandline in v1 */
190	static u16 cgroup_no_v1_mask;
191
192	/* some controllers are not supported in the default hierarchy */
193	static u16 cgrp_dfl_inhibit_ss_mask;
194
195	/* some controllers are implicitly enabled on the default hierarchy */
196	static unsigned long cgrp_dfl_implicit_ss_mask;
197
198	/* The list of hierarchy roots */
199
200	static LIST_HEAD(cgroup_roots);
201	static int cgroup_root_count;
202
203	/* hierarchy ID allocation and mapping, protected by cgroup_mutex */
204	static DEFINE_IDR(cgroup_hierarchy_idr);
205
206	/*
207	* Assign a monotonically increasing serial number to csses. It guarantees
208	* cgroups with bigger numbers are newer than those with smaller numbers.
209	* Also, as csses are always appended to the parent's ->children list, it
210	* guarantees that sibling csses are always sorted in the ascending serial
211	* number order on the list. Protected by cgroup_mutex.
212	*/
213	static u64 css_serial_nr_next = 1;
214
215	/*
216	* These bitmask flags indicate whether tasks in the fork and exit paths have
217	* fork/exit handlers to call. This avoids us having to do extra work in the
218	* fork/exit path to check which subsystems have fork/exit callbacks.
219	*/
220	static u16 have_fork_callback __read_mostly;
221	static u16 have_exit_callback __read_mostly;
222	static u16 have_free_callback __read_mostly;
223
224	/* cgroup namespace for init task */
225	struct cgroup_namespace init_cgroup_ns = {
226	.count = { .counter = 2, },
227	.user_ns = &init_user_ns,
228	.ns.ops = &cgroupns_operations,
229	.ns.inum = PROC_CGROUP_INIT_INO,
230	.root_cset = &init_css_set,
231	};
232
233	/* Ditto for the can_fork callback. */
234	static u16 have_canfork_callback __read_mostly;
235
236	static struct file_system_type cgroup2_fs_type;
237	static struct cftype cgroup_dfl_base_files[];
238	static struct cftype cgroup_legacy_base_files[];
239
240	static int rebind_subsystems(struct cgroup_root *dst_root, u16 ss_mask);
241	static void cgroup_lock_and_drain_offline(struct cgroup *cgrp);
242	static int cgroup_apply_control(struct cgroup *cgrp);
243	static void cgroup_finalize_control(struct cgroup *cgrp, int ret);
244	static void css_task_iter_advance(struct css_task_iter *it);
245	static int cgroup_destroy_locked(struct cgroup *cgrp);
246	static struct cgroup_subsys_state css_create(struct cgroup cgrp,
247	struct cgroup_subsys *ss);
248	static void css_release(struct percpu_ref *ref);
249	static void kill_css(struct cgroup_subsys_state *css);
250	static int cgroup_addrm_files(struct cgroup_subsys_state *css,
251	struct cgroup *cgrp, struct cftype cfts[],
252	bool is_add);
253
254	/**
255	* cgroup_ssid_enabled - cgroup subsys enabled test by subsys ID
256	* @ssid: subsys ID of interest
257	*
258	* cgroup_subsys_enabled() can only be used with literal subsys names which
259	* is fine for individual subsystems but unsuitable for cgroup core. This
260	* is slower static_key_enabled() based test indexed by @ssid.
261	*/
262	static bool cgroup_ssid_enabled(int ssid)
263	{
264	if (CGROUP_SUBSYS_COUNT == 0)
265	return false;
266
267	return static_key_enabled(cgroup_subsys_enabled_key[ssid]);
268	}
269
270	static bool cgroup_ssid_no_v1(int ssid)
271	{
272	return cgroup_no_v1_mask & (1 << ssid);
273	}
274
275	/**
276	* cgroup_on_dfl - test whether a cgroup is on the default hierarchy
277	* @cgrp: the cgroup of interest
278	*
279	* The default hierarchy is the v2 interface of cgroup and this function
280	* can be used to test whether a cgroup is on the default hierarchy for
281	* cases where a subsystem should behave differnetly depending on the
282	* interface version.
283	*
284	* The set of behaviors which change on the default hierarchy are still
285	* being determined and the mount option is prefixed with __DEVEL__.
286	*
287	* List of changed behaviors:
288	*
289	* - Mount options "noprefix", "xattr", "clone_children", "release_agent"
290	* and "name" are disallowed.
291	*
292	* - When mounting an existing superblock, mount options should match.
293	*
294	* - Remount is disallowed.
295	*
296	* - rename(2) is disallowed.
297	*
298	* - "tasks" is removed. Everything should be at process granularity. Use
299	* "cgroup.procs" instead.
300	*
301	* - "cgroup.procs" is not sorted. pids will be unique unless they got
302	* recycled inbetween reads.
303	*
304	* - "release_agent" and "notify_on_release" are removed. Replacement
305	* notification mechanism will be implemented.
306	*
307	* - "cgroup.clone_children" is removed.
308	*
309	* - "cgroup.subtree_populated" is available. Its value is 0 if the cgroup
310	* and its descendants contain no task; otherwise, 1. The file also
311	* generates kernfs notification which can be monitored through poll and
312	* [di]notify when the value of the file changes.
313	*
314	* - cpuset: tasks will be kept in empty cpusets when hotplug happens and
315	* take masks of ancestors with non-empty cpus/mems, instead of being
316	* moved to an ancestor.
317	*
318	* - cpuset: a task can be moved into an empty cpuset, and again it takes
319	* masks of ancestors.
320	*
321	* - memcg: use_hierarchy is on by default and the cgroup file for the flag
322	* is not created.
323	*
324	* - blkcg: blk-throttle becomes properly hierarchical.
325	*
326	* - debug: disallowed on the default hierarchy.
327	*/
328	static bool cgroup_on_dfl(const struct cgroup *cgrp)
329	{
330	return cgrp->root == &cgrp_dfl_root;
331	}
332
333	/* IDR wrappers which synchronize using cgroup_idr_lock */
334	static int cgroup_idr_alloc(struct idr idr, void ptr, int start, int end,
335	gfp_t gfp_mask)
336	{
337	int ret;
338
339	idr_preload(gfp_mask);
340	spin_lock_bh(&cgroup_idr_lock);
341	ret = idr_alloc(idr, ptr, start, end, gfp_mask & ~__GFP_DIRECT_RECLAIM);
342	spin_unlock_bh(&cgroup_idr_lock);
343	idr_preload_end();
344	return ret;
345	}
346
347	static void cgroup_idr_replace(struct idr idr, void *ptr, int id)
348	{
349	void *ret;
350
351	spin_lock_bh(&cgroup_idr_lock);
352	ret = idr_replace(idr, ptr, id);
353	spin_unlock_bh(&cgroup_idr_lock);
354	return ret;
355	}
356
357	static void cgroup_idr_remove(struct idr *idr, int id)
358	{
359	spin_lock_bh(&cgroup_idr_lock);
360	idr_remove(idr, id);
361	spin_unlock_bh(&cgroup_idr_lock);
362	}
363
364	static struct cgroup cgroup_parent(struct cgroup cgrp)
365	{
366	struct cgroup_subsys_state *parent_css = cgrp->self.parent;
367
368	if (parent_css)
369	return container_of(parent_css, struct cgroup, self);
370	return NULL;
371	}
372
373	/* subsystems visibly enabled on a cgroup */
374	static u16 cgroup_control(struct cgroup *cgrp)
375	{
376	struct cgroup *parent = cgroup_parent(cgrp);
377	u16 root_ss_mask = cgrp->root->subsys_mask;
378
379	if (parent)
380	return parent->subtree_control;
381
382	if (cgroup_on_dfl(cgrp))
383	root_ss_mask &= ~(cgrp_dfl_inhibit_ss_mask \|
384	cgrp_dfl_implicit_ss_mask);
385	return root_ss_mask;
386	}
387
388	/* subsystems enabled on a cgroup */
389	static u16 cgroup_ss_mask(struct cgroup *cgrp)
390	{
391	struct cgroup *parent = cgroup_parent(cgrp);
392
393	if (parent)
394	return parent->subtree_ss_mask;
395
396	return cgrp->root->subsys_mask;
397	}
398
399	/**
400	* cgroup_css - obtain a cgroup's css for the specified subsystem
401	* @cgrp: the cgroup of interest
402	* @ss: the subsystem of interest (%NULL returns @cgrp->self)
403	*
404	* Return @cgrp's css (cgroup_subsys_state) associated with @ss. This
405	* function must be called either under cgroup_mutex or rcu_read_lock() and
406	* the caller is responsible for pinning the returned css if it wants to
407	* keep accessing it outside the said locks. This function may return
408	* %NULL if @cgrp doesn't have @subsys_id enabled.
409	*/
410	static struct cgroup_subsys_state cgroup_css(struct cgroup cgrp,
411	struct cgroup_subsys *ss)
412	{
413	if (ss)
414	return rcu_dereference_check(cgrp->subsys[ss->id],
415	lockdep_is_held(&cgroup_mutex));
416	else
417	return &cgrp->self;
418	}
419
420	/**
421	* cgroup_e_css - obtain a cgroup's effective css for the specified subsystem
422	* @cgrp: the cgroup of interest
423	* @ss: the subsystem of interest (%NULL returns @cgrp->self)
424	*
425	* Similar to cgroup_css() but returns the effective css, which is defined
426	* as the matching css of the nearest ancestor including self which has @ss
427	* enabled. If @ss is associated with the hierarchy @cgrp is on, this
428	* function is guaranteed to return non-NULL css.
429	*/
430	static struct cgroup_subsys_state cgroup_e_css(struct cgroup cgrp,
431	struct cgroup_subsys *ss)
432	{
433	lockdep_assert_held(&cgroup_mutex);
434
435	if (!ss)
436	return &cgrp->self;
437
438	/*
439	* This function is used while updating css associations and thus
440	* can't test the csses directly. Test ss_mask.
441	*/
442	while (!(cgroup_ss_mask(cgrp) & (1 << ss->id))) {
443	cgrp = cgroup_parent(cgrp);
444	if (!cgrp)
445	return NULL;
446	}
447
448	return cgroup_css(cgrp, ss);
449	}
450
451	/**
452	* cgroup_get_e_css - get a cgroup's effective css for the specified subsystem
453	* @cgrp: the cgroup of interest
454	* @ss: the subsystem of interest
455	*
456	* Find and get the effective css of @cgrp for @ss. The effective css is
457	* defined as the matching css of the nearest ancestor including self which
458	* has @ss enabled. If @ss is not mounted on the hierarchy @cgrp is on,
459	* the root css is returned, so this function always returns a valid css.
460	* The returned css must be put using css_put().
461	*/
462	struct cgroup_subsys_state cgroup_get_e_css(struct cgroup cgrp,
463	struct cgroup_subsys *ss)
464	{
465	struct cgroup_subsys_state *css;
466
467	rcu_read_lock();
468
469	do {
470	css = cgroup_css(cgrp, ss);
471
472	if (css && css_tryget_online(css))
473	goto out_unlock;
474	cgrp = cgroup_parent(cgrp);
475	} while (cgrp);
476
477	css = init_css_set.subsys[ss->id];
478	css_get(css);
479	out_unlock:
480	rcu_read_unlock();
481	return css;
482	}
483
484	/* convenient tests for these bits */
485	static inline bool cgroup_is_dead(const struct cgroup *cgrp)
486	{
487	return !(cgrp->self.flags & CSS_ONLINE);
488	}
489
490	static void cgroup_get(struct cgroup *cgrp)
491	{
492	WARN_ON_ONCE(cgroup_is_dead(cgrp));
493	css_get(&cgrp->self);
494	}
495
496	static bool cgroup_tryget(struct cgroup *cgrp)
497	{
498	return css_tryget(&cgrp->self);
499	}
500
501	struct cgroup_subsys_state of_css(struct kernfs_open_file of)
502	{
503	struct cgroup *cgrp = of->kn->parent->priv;
504	struct cftype *cft = of_cft(of);
505
506	/*
507	* This is open and unprotected implementation of cgroup_css().
508	* seq_css() is only called from a kernfs file operation which has
509	* an active reference on the file. Because all the subsystem
510	* files are drained before a css is disassociated with a cgroup,
511	* the matching css from the cgroup's subsys table is guaranteed to
512	* be and stay valid until the enclosing operation is complete.
513	*/
514	if (cft->ss)
515	return rcu_dereference_raw(cgrp->subsys[cft->ss->id]);
516	else
517	return &cgrp->self;
518	}
519	EXPORT_SYMBOL_GPL(of_css);
520
521	static int notify_on_release(const struct cgroup *cgrp)
522	{
523	return test_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
524	}
525
526	/**
527	* for_each_css - iterate all css's of a cgroup
528	* @css: the iteration cursor
529	* @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
530	* @cgrp: the target cgroup to iterate css's of
531	*
532	* Should be called under cgroup_[tree_]mutex.
533	*/
534	#define for_each_css(css, ssid, cgrp) \
535	for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \
536	if (!((css) = rcu_dereference_check( \
537	(cgrp)->subsys[(ssid)], \
538	lockdep_is_held(&cgroup_mutex)))) { } \
539	else
540
541	/**
542	* for_each_e_css - iterate all effective css's of a cgroup
543	* @css: the iteration cursor
544	* @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
545	* @cgrp: the target cgroup to iterate css's of
546	*
547	* Should be called under cgroup_[tree_]mutex.
548	*/
549	#define for_each_e_css(css, ssid, cgrp) \
550	for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \
551	if (!((css) = cgroup_e_css(cgrp, cgroup_subsys[(ssid)]))) \
552	; \
553	else
554
555	/**
556	* for_each_subsys - iterate all enabled cgroup subsystems
557	* @ss: the iteration cursor
558	* @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end
559	*/
560	#define for_each_subsys(ss, ssid) \
561	for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT && \
562	(((ss) = cgroup_subsys[ssid]) \|\| true); (ssid)++)
563
564	/**
565	* do_each_subsys_mask - filter for_each_subsys with a bitmask
566	* @ss: the iteration cursor
567	* @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end
568	* @ss_mask: the bitmask
569	*
570	* The block will only run for cases where the ssid-th bit (1 << ssid) of
571	* @ss_mask is set.
572	*/
573	#define do_each_subsys_mask(ss, ssid, ss_mask) do { \
574	unsigned long __ss_mask = (ss_mask); \
575	if (!CGROUP_SUBSYS_COUNT) { /* to avoid spurious gcc warning */ \
576	(ssid) = 0; \
577	break; \
578	} \
579	for_each_set_bit(ssid, &__ss_mask, CGROUP_SUBSYS_COUNT) { \
580	(ss) = cgroup_subsys[ssid]; \
581	{
582
583	#define while_each_subsys_mask() \
584	} \
585	} \
586	} while (false)
587
588	/* iterate across the hierarchies */
589	#define for_each_root(root) \
590	list_for_each_entry((root), &cgroup_roots, root_list)
591
592	/* iterate over child cgrps, lock should be held throughout iteration */
593	#define cgroup_for_each_live_child(child, cgrp) \
594	list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \
595	if (({ lockdep_assert_held(&cgroup_mutex); \
596	cgroup_is_dead(child); })) \
597	; \
598	else
599
600	/* walk live descendants in preorder */
601	#define cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) \
602	css_for_each_descendant_pre((d_css), cgroup_css((cgrp), NULL)) \
603	if (({ lockdep_assert_held(&cgroup_mutex); \
604	(dsct) = (d_css)->cgroup; \
605	cgroup_is_dead(dsct); })) \
606	; \
607	else
608
609	/* walk live descendants in postorder */
610	#define cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) \
611	css_for_each_descendant_post((d_css), cgroup_css((cgrp), NULL)) \
612	if (({ lockdep_assert_held(&cgroup_mutex); \
613	(dsct) = (d_css)->cgroup; \
614	cgroup_is_dead(dsct); })) \
615	; \
616	else
617
618	static void cgroup_release_agent(struct work_struct *work);
619	static void check_for_release(struct cgroup *cgrp);
620
621	/*
622	* A cgroup can be associated with multiple css_sets as different tasks may
623	* belong to different cgroups on different hierarchies. In the other
624	* direction, a css_set is naturally associated with multiple cgroups.
625	* This M:N relationship is represented by the following link structure
626	* which exists for each association and allows traversing the associations
627	* from both sides.
628	*/
629	struct cgrp_cset_link {
630	/* the cgroup and css_set this link associates */
631	struct cgroup *cgrp;
632	struct css_set *cset;
633
634	/* list of cgrp_cset_links anchored at cgrp->cset_links */
635	struct list_head cset_link;
636
637	/* list of cgrp_cset_links anchored at css_set->cgrp_links */
638	struct list_head cgrp_link;
639	};
640
641	/*
642	* The default css_set - used by init and its children prior to any
643	* hierarchies being mounted. It contains a pointer to the root state
644	* for each subsystem. Also used to anchor the list of css_sets. Not
645	* reference-counted, to improve performance when child cgroups
646	* haven't been created.
647	*/
648	struct css_set init_css_set = {
649	.refcount = ATOMIC_INIT(1),
650	.cgrp_links = LIST_HEAD_INIT(init_css_set.cgrp_links),
651	.tasks = LIST_HEAD_INIT(init_css_set.tasks),
652	.mg_tasks = LIST_HEAD_INIT(init_css_set.mg_tasks),
653	.mg_preload_node = LIST_HEAD_INIT(init_css_set.mg_preload_node),
654	.mg_node = LIST_HEAD_INIT(init_css_set.mg_node),
655	.task_iters = LIST_HEAD_INIT(init_css_set.task_iters),
656	};
657
658	static int css_set_count = 1; /* 1 for init_css_set */
659
660	/**
661	* css_set_populated - does a css_set contain any tasks?
662	* @cset: target css_set
663	*/
664	static bool css_set_populated(struct css_set *cset)
665	{
666	lockdep_assert_held(&css_set_lock);
667
668	return !list_empty(&cset->tasks) \|\| !list_empty(&cset->mg_tasks);
669	}
670
671	/**
672	* cgroup_update_populated - updated populated count of a cgroup
673	* @cgrp: the target cgroup
674	* @populated: inc or dec populated count
675	*
676	* One of the css_sets associated with @cgrp is either getting its first
677	* task or losing the last. Update @cgrp->populated_cnt accordingly. The
678	* count is propagated towards root so that a given cgroup's populated_cnt
679	* is zero iff the cgroup and all its descendants don't contain any tasks.
680	*
681	* @cgrp's interface file "cgroup.populated" is zero if
682	* @cgrp->populated_cnt is zero and 1 otherwise. When @cgrp->populated_cnt
683	* changes from or to zero, userland is notified that the content of the
684	* interface file has changed. This can be used to detect when @cgrp and
685	* its descendants become populated or empty.
686	*/
687	static void cgroup_update_populated(struct cgroup *cgrp, bool populated)
688	{
689	lockdep_assert_held(&css_set_lock);
690
691	do {
692	bool trigger;
693
694	if (populated)
695	trigger = !cgrp->populated_cnt++;
696	else
697	trigger = !--cgrp->populated_cnt;
698
699	if (!trigger)
700	break;
701
702	check_for_release(cgrp);
703	cgroup_file_notify(&cgrp->events_file);
704
705	cgrp = cgroup_parent(cgrp);
706	} while (cgrp);
707	}
708
709	/**
710	* css_set_update_populated - update populated state of a css_set
711	* @cset: target css_set
712	* @populated: whether @cset is populated or depopulated
713	*
714	* @cset is either getting the first task or losing the last. Update the
715	* ->populated_cnt of all associated cgroups accordingly.
716	*/
717	static void css_set_update_populated(struct css_set *cset, bool populated)
718	{
719	struct cgrp_cset_link *link;
720
721	lockdep_assert_held(&css_set_lock);
722
723	list_for_each_entry(link, &cset->cgrp_links, cgrp_link)
724	cgroup_update_populated(link->cgrp, populated);
725	}
726
727	/**
728	* css_set_move_task - move a task from one css_set to another
729	* @task: task being moved
730	* @from_cset: css_set @task currently belongs to (may be NULL)
731	* @to_cset: new css_set @task is being moved to (may be NULL)
732	* @use_mg_tasks: move to @to_cset->mg_tasks instead of ->tasks
733	*
734	* Move @task from @from_cset to @to_cset. If @task didn't belong to any
735	* css_set, @from_cset can be NULL. If @task is being disassociated
736	* instead of moved, @to_cset can be NULL.
737	*
738	* This function automatically handles populated_cnt updates and
739	* css_task_iter adjustments but the caller is responsible for managing
740	* @from_cset and @to_cset's reference counts.
741	*/
742	static void css_set_move_task(struct task_struct *task,
743	struct css_set from_cset, struct css_set to_cset,
744	bool use_mg_tasks)
745	{
746	lockdep_assert_held(&css_set_lock);
747
748	if (to_cset && !css_set_populated(to_cset))
749	css_set_update_populated(to_cset, true);
750
751	if (from_cset) {
752	struct css_task_iter it, pos;
753
754	WARN_ON_ONCE(list_empty(&task->cg_list));
755
756	/*
757	* @task is leaving, advance task iterators which are
758	* pointing to it so that they can resume at the next
759	* position. Advancing an iterator might remove it from
760	* the list, use safe walk. See css_task_iter_advance*()
761	* for details.
762	*/
763	list_for_each_entry_safe(it, pos, &from_cset->task_iters,
764	iters_node)
765	if (it->task_pos == &task->cg_list)
766	css_task_iter_advance(it);
767
768	list_del_init(&task->cg_list);
769	if (!css_set_populated(from_cset))
770	css_set_update_populated(from_cset, false);
771	} else {
772	WARN_ON_ONCE(!list_empty(&task->cg_list));
773	}
774
775	if (to_cset) {
776	/*
777	* We are synchronized through cgroup_threadgroup_rwsem
778	* against PF_EXITING setting such that we can't race
779	* against cgroup_exit() changing the css_set to
780	* init_css_set and dropping the old one.
781	*/
782	WARN_ON_ONCE(task->flags & PF_EXITING);
783
784	rcu_assign_pointer(task->cgroups, to_cset);
785	list_add_tail(&task->cg_list, use_mg_tasks ? &to_cset->mg_tasks :
786	&to_cset->tasks);
787	}
788	}
789
790	/*
791	* hash table for cgroup groups. This improves the performance to find
792	* an existing css_set. This hash doesn't (currently) take into
793	* account cgroups in empty hierarchies.
794	*/
795	#define CSS_SET_HASH_BITS 7
796	static DEFINE_HASHTABLE(css_set_table, CSS_SET_HASH_BITS);
797
798	static unsigned long css_set_hash(struct cgroup_subsys_state *css[])
799	{
800	unsigned long key = 0UL;
801	struct cgroup_subsys *ss;
802	int i;
803
804	for_each_subsys(ss, i)
805	key += (unsigned long)css[i];
806	key = (key >> 16) ^ key;
807
808	return key;
809	}
810
811	static void put_css_set_locked(struct css_set *cset)
812	{
813	struct cgrp_cset_link link, tmp_link;
814	struct cgroup_subsys *ss;
815	int ssid;
816
817	lockdep_assert_held(&css_set_lock);
818
819	if (!atomic_dec_and_test(&cset->refcount))
820	return;
821
822	/* This css_set is dead. unlink it and release cgroup and css refs */
823	for_each_subsys(ss, ssid) {
824	list_del(&cset->e_cset_node[ssid]);
825	css_put(cset->subsys[ssid]);
826	}
827	hash_del(&cset->hlist);
828	css_set_count--;
829
830	list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {
831	list_del(&link->cset_link);
832	list_del(&link->cgrp_link);
833	if (cgroup_parent(link->cgrp))
834	cgroup_put(link->cgrp);
835	kfree(link);
836	}
837
838	kfree_rcu(cset, rcu_head);
839	}
840
841	static void put_css_set(struct css_set *cset)
842	{
843	unsigned long flags;
844
845	/*
846	* Ensure that the refcount doesn't hit zero while any readers
847	* can see it. Similar to atomic_dec_and_lock(), but for an
848	* rwlock
849	*/
850	if (atomic_add_unless(&cset->refcount, -1, 1))
851	return;
852
853	spin_lock_irqsave(&css_set_lock, flags);
854	put_css_set_locked(cset);
855	spin_unlock_irqrestore(&css_set_lock, flags);
856	}
857
858	/*
859	* refcounted get/put for css_set objects
860	*/
861	static inline void get_css_set(struct css_set *cset)
862	{
863	atomic_inc(&cset->refcount);
864	}
865
866	/**
867	* compare_css_sets - helper function for find_existing_css_set().
868	* @cset: candidate css_set being tested
869	* @old_cset: existing css_set for a task
870	* @new_cgrp: cgroup that's being entered by the task
871	* @template: desired set of css pointers in css_set (pre-calculated)
872	*
873	* Returns true if "cset" matches "old_cset" except for the hierarchy
874	* which "new_cgrp" belongs to, for which it should match "new_cgrp".
875	*/
876	static bool compare_css_sets(struct css_set *cset,
877	struct css_set *old_cset,
878	struct cgroup *new_cgrp,
879	struct cgroup_subsys_state *template[])
880	{
881	struct list_head l1, l2;
882
883	/*
884	* On the default hierarchy, there can be csets which are
885	* associated with the same set of cgroups but different csses.
886	* Let's first ensure that csses match.
887	*/
888	if (memcmp(template, cset->subsys, sizeof(cset->subsys)))
889	return false;
890
891	/*
892	* Compare cgroup pointers in order to distinguish between
893	* different cgroups in hierarchies. As different cgroups may
894	* share the same effective css, this comparison is always
895	* necessary.
896	*/
897	l1 = &cset->cgrp_links;
898	l2 = &old_cset->cgrp_links;
899	while (1) {
900	struct cgrp_cset_link link1, link2;
901	struct cgroup cgrp1, cgrp2;
902
903	l1 = l1->next;
904	l2 = l2->next;
905	/* See if we reached the end - both lists are equal length. */
906	if (l1 == &cset->cgrp_links) {
907	BUG_ON(l2 != &old_cset->cgrp_links);
908	break;
909	} else {
910	BUG_ON(l2 == &old_cset->cgrp_links);
911	}
912	/* Locate the cgroups associated with these links. */
913	link1 = list_entry(l1, struct cgrp_cset_link, cgrp_link);
914	link2 = list_entry(l2, struct cgrp_cset_link, cgrp_link);
915	cgrp1 = link1->cgrp;
916	cgrp2 = link2->cgrp;
917	/* Hierarchies should be linked in the same order. */
918	BUG_ON(cgrp1->root != cgrp2->root);
919
920	/*
921	* If this hierarchy is the hierarchy of the cgroup
922	* that's changing, then we need to check that this
923	* css_set points to the new cgroup; if it's any other
924	* hierarchy, then this css_set should point to the
925	* same cgroup as the old css_set.
926	*/
927	if (cgrp1->root == new_cgrp->root) {
928	if (cgrp1 != new_cgrp)
929	return false;
930	} else {
931	if (cgrp1 != cgrp2)
932	return false;
933	}
934	}
935	return true;
936	}
937
938	/**
939	* find_existing_css_set - init css array and find the matching css_set
940	* @old_cset: the css_set that we're using before the cgroup transition
941	* @cgrp: the cgroup that we're moving into
942	* @template: out param for the new set of csses, should be clear on entry
943	*/
944	static struct css_set find_existing_css_set(struct css_set old_cset,
945	struct cgroup *cgrp,
946	struct cgroup_subsys_state *template[])
947	{
948	struct cgroup_root *root = cgrp->root;
949	struct cgroup_subsys *ss;
950	struct css_set *cset;
951	unsigned long key;
952	int i;
953
954	/*
955	* Build the set of subsystem state objects that we want to see in the
956	* new css_set. while subsystems can change globally, the entries here
957	* won't change, so no need for locking.
958	*/
959	for_each_subsys(ss, i) {
960	if (root->subsys_mask & (1UL << i)) {
961	/*
962	* @ss is in this hierarchy, so we want the
963	* effective css from @cgrp.
964	*/
965	template[i] = cgroup_e_css(cgrp, ss);
966	} else {
967	/*
968	* @ss is not in this hierarchy, so we don't want
969	* to change the css.
970	*/
971	template[i] = old_cset->subsys[i];
972	}
973	}
974
975	key = css_set_hash(template);
976	hash_for_each_possible(css_set_table, cset, hlist, key) {
977	if (!compare_css_sets(cset, old_cset, cgrp, template))
978	continue;
979
980	/* This css_set matches what we need */
981	return cset;
982	}
983
984	/* No existing cgroup group matched */
985	return NULL;
986	}
987
988	static void free_cgrp_cset_links(struct list_head *links_to_free)
989	{
990	struct cgrp_cset_link link, tmp_link;
991
992	list_for_each_entry_safe(link, tmp_link, links_to_free, cset_link) {
993	list_del(&link->cset_link);
994	kfree(link);
995	}
996	}
997
998	/**
999	* allocate_cgrp_cset_links - allocate cgrp_cset_links
1000	* @count: the number of links to allocate
1001	* @tmp_links: list_head the allocated links are put on
1002	*
1003	* Allocate @count cgrp_cset_link structures and chain them on @tmp_links
1004	* through ->cset_link. Returns 0 on success or -errno.
1005	*/
1006	static int allocate_cgrp_cset_links(int count, struct list_head *tmp_links)
1007	{
1008	struct cgrp_cset_link *link;
1009	int i;
1010
1011	INIT_LIST_HEAD(tmp_links);
1012
1013	for (i = 0; i < count; i++) {
1014	link = kzalloc(sizeof(*link), GFP_KERNEL);
1015	if (!link) {
1016	free_cgrp_cset_links(tmp_links);
1017	return -ENOMEM;
1018	}
1019	list_add(&link->cset_link, tmp_links);
1020	}
1021	return 0;
1022	}
1023
1024	/**
1025	* link_css_set - a helper function to link a css_set to a cgroup
1026	* @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links()
1027	* @cset: the css_set to be linked
1028	* @cgrp: the destination cgroup
1029	*/
1030	static void link_css_set(struct list_head tmp_links, struct css_set cset,
1031	struct cgroup *cgrp)
1032	{
1033	struct cgrp_cset_link *link;
1034
1035	BUG_ON(list_empty(tmp_links));
1036
1037	if (cgroup_on_dfl(cgrp))
1038	cset->dfl_cgrp = cgrp;
1039
1040	link = list_first_entry(tmp_links, struct cgrp_cset_link, cset_link);
1041	link->cset = cset;
1042	link->cgrp = cgrp;
1043
1044	/*
1045	* Always add links to the tail of the lists so that the lists are
1046	* in choronological order.
1047	*/
1048	list_move_tail(&link->cset_link, &cgrp->cset_links);
1049	list_add_tail(&link->cgrp_link, &cset->cgrp_links);
1050
1051	if (cgroup_parent(cgrp))
1052	cgroup_get(cgrp);
1053	}
1054
1055	/**
1056	* find_css_set - return a new css_set with one cgroup updated
1057	* @old_cset: the baseline css_set
1058	* @cgrp: the cgroup to be updated
1059	*
1060	* Return a new css_set that's equivalent to @old_cset, but with @cgrp
1061	* substituted into the appropriate hierarchy.
1062	*/
1063	static struct css_set find_css_set(struct css_set old_cset,
1064	struct cgroup *cgrp)
1065	{
1066	struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT] = { };
1067	struct css_set *cset;
1068	struct list_head tmp_links;
1069	struct cgrp_cset_link *link;
1070	struct cgroup_subsys *ss;
1071	unsigned long key;
1072	int ssid;
1073
1074	lockdep_assert_held(&cgroup_mutex);
1075
1076	/* First see if we already have a cgroup group that matches
1077	* the desired set */
1078	spin_lock_irq(&css_set_lock);
1079	cset = find_existing_css_set(old_cset, cgrp, template);
1080	if (cset)
1081	get_css_set(cset);
1082	spin_unlock_irq(&css_set_lock);
1083
1084	if (cset)
1085	return cset;
1086
1087	cset = kzalloc(sizeof(*cset), GFP_KERNEL);
1088	if (!cset)
1089	return NULL;
1090
1091	/* Allocate all the cgrp_cset_link objects that we'll need */
1092	if (allocate_cgrp_cset_links(cgroup_root_count, &tmp_links) < 0) {
1093	kfree(cset);
1094	return NULL;
1095	}
1096
1097	atomic_set(&cset->refcount, 1);
1098	INIT_LIST_HEAD(&cset->cgrp_links);
1099	INIT_LIST_HEAD(&cset->tasks);
1100	INIT_LIST_HEAD(&cset->mg_tasks);
1101	INIT_LIST_HEAD(&cset->mg_preload_node);
1102	INIT_LIST_HEAD(&cset->mg_node);
1103	INIT_LIST_HEAD(&cset->task_iters);
1104	INIT_HLIST_NODE(&cset->hlist);
1105
1106	/* Copy the set of subsystem state objects generated in
1107	* find_existing_css_set() */
1108	memcpy(cset->subsys, template, sizeof(cset->subsys));
1109
1110	spin_lock_irq(&css_set_lock);
1111	/* Add reference counts and links from the new css_set. */
1112	list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) {
1113	struct cgroup *c = link->cgrp;
1114
1115	if (c->root == cgrp->root)
1116	c = cgrp;
1117	link_css_set(&tmp_links, cset, c);
1118	}
1119
1120	BUG_ON(!list_empty(&tmp_links));
1121
1122	css_set_count++;
1123
1124	/* Add @cset to the hash table */
1125	key = css_set_hash(cset->subsys);
1126	hash_add(css_set_table, &cset->hlist, key);
1127
1128	for_each_subsys(ss, ssid) {
1129	struct cgroup_subsys_state *css = cset->subsys[ssid];
1130
1131	list_add_tail(&cset->e_cset_node[ssid],
1132	&css->cgroup->e_csets[ssid]);
1133	css_get(css);
1134	}
1135
1136	spin_unlock_irq(&css_set_lock);
1137
1138	return cset;
1139	}
1140
1141	static struct cgroup_root cgroup_root_from_kf(struct kernfs_root kf_root)
1142	{
1143	struct cgroup *root_cgrp = kf_root->kn->priv;
1144
1145	return root_cgrp->root;
1146	}
1147
1148	static int cgroup_init_root_id(struct cgroup_root *root)
1149	{
1150	int id;
1151
1152	lockdep_assert_held(&cgroup_mutex);
1153
1154	id = idr_alloc_cyclic(&cgroup_hierarchy_idr, root, 0, 0, GFP_KERNEL);
1155	if (id < 0)
1156	return id;
1157
1158	root->hierarchy_id = id;
1159	return 0;
1160	}
1161
1162	static void cgroup_exit_root_id(struct cgroup_root *root)
1163	{
1164	lockdep_assert_held(&cgroup_mutex);
1165
1166	idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id);
1167	}
1168
1169	static void cgroup_free_root(struct cgroup_root *root)
1170	{
1171	if (root) {
1172	idr_destroy(&root->cgroup_idr);
1173	kfree(root);
1174	}
1175	}
1176
1177	static void cgroup_destroy_root(struct cgroup_root *root)
1178	{
1179	struct cgroup *cgrp = &root->cgrp;
1180	struct cgrp_cset_link link, tmp_link;
1181
1182	trace_cgroup_destroy_root(root);
1183
1184	cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp);
1185
1186	BUG_ON(atomic_read(&root->nr_cgrps));
1187	BUG_ON(!list_empty(&cgrp->self.children));
1188
1189	/* Rebind all subsystems back to the default hierarchy */
1190	WARN_ON(rebind_subsystems(&cgrp_dfl_root, root->subsys_mask));
1191
1192	/*
1193	* Release all the links from cset_links to this hierarchy's
1194	* root cgroup
1195	*/
1196	spin_lock_irq(&css_set_lock);
1197
1198	list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) {
1199	list_del(&link->cset_link);
1200	list_del(&link->cgrp_link);
1201	kfree(link);
1202	}
1203
1204	spin_unlock_irq(&css_set_lock);
1205
1206	if (!list_empty(&root->root_list)) {
1207	list_del(&root->root_list);
1208	cgroup_root_count--;
1209	}
1210
1211	cgroup_exit_root_id(root);
1212
1213	mutex_unlock(&cgroup_mutex);
1214
1215	kernfs_destroy_root(root->kf_root);
1216	cgroup_free_root(root);
1217	}
1218
1219	/*
1220	* look up cgroup associated with current task's cgroup namespace on the
1221	* specified hierarchy
1222	*/
1223	static struct cgroup *
1224	current_cgns_cgroup_from_root(struct cgroup_root *root)
1225	{
1226	struct cgroup *res = NULL;
1227	struct css_set *cset;
1228
1229	lockdep_assert_held(&css_set_lock);
1230
1231	rcu_read_lock();
1232
1233	cset = current->nsproxy->cgroup_ns->root_cset;
1234	if (cset == &init_css_set) {
1235	res = &root->cgrp;
1236	} else {
1237	struct cgrp_cset_link *link;
1238
1239	list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
1240	struct cgroup *c = link->cgrp;
1241
1242	if (c->root == root) {
1243	res = c;
1244	break;
1245	}
1246	}
1247	}
1248	rcu_read_unlock();
1249
1250	BUG_ON(!res);
1251	return res;
1252	}
1253
1254	/* look up cgroup associated with given css_set on the specified hierarchy */
1255	static struct cgroup cset_cgroup_from_root(struct css_set cset,
1256	struct cgroup_root *root)
1257	{
1258	struct cgroup *res = NULL;
1259
1260	lockdep_assert_held(&cgroup_mutex);
1261	lockdep_assert_held(&css_set_lock);
1262
1263	if (cset == &init_css_set) {
1264	res = &root->cgrp;
1265	} else {
1266	struct cgrp_cset_link *link;
1267
1268	list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
1269	struct cgroup *c = link->cgrp;
1270
1271	if (c->root == root) {
1272	res = c;
1273	break;
1274	}
1275	}
1276	}
1277
1278	BUG_ON(!res);
1279	return res;
1280	}
1281
1282	/*
1283	* Return the cgroup for "task" from the given hierarchy. Must be
1284	* called with cgroup_mutex and css_set_lock held.
1285	*/
1286	static struct cgroup task_cgroup_from_root(struct task_struct task,
1287	struct cgroup_root *root)
1288	{
1289	/*
1290	* No need to lock the task - since we hold cgroup_mutex the
1291	* task can't change groups, so the only thing that can happen
1292	* is that it exits and its css is set back to init_css_set.
1293	*/
1294	return cset_cgroup_from_root(task_css_set(task), root);
1295	}
1296
1297	/*
1298	* A task must hold cgroup_mutex to modify cgroups.
1299	*
1300	* Any task can increment and decrement the count field without lock.
1301	* So in general, code holding cgroup_mutex can't rely on the count
1302	* field not changing. However, if the count goes to zero, then only
1303	* cgroup_attach_task() can increment it again. Because a count of zero
1304	* means that no tasks are currently attached, therefore there is no
1305	* way a task attached to that cgroup can fork (the other way to
1306	* increment the count). So code holding cgroup_mutex can safely
1307	* assume that if the count is zero, it will stay zero. Similarly, if
1308	* a task holds cgroup_mutex on a cgroup with zero count, it
1309	* knows that the cgroup won't be removed, as cgroup_rmdir()
1310	* needs that mutex.
1311	*
1312	* A cgroup can only be deleted if both its 'count' of using tasks
1313	* is zero, and its list of 'children' cgroups is empty. Since all
1314	* tasks in the system use _some_ cgroup, and since there is always at
1315	* least one task in the system (init, pid == 1), therefore, root cgroup
1316	* always has either children cgroups and/or using tasks. So we don't
1317	* need a special hack to ensure that root cgroup cannot be deleted.
1318	*
1319	* P.S. One more locking exception. RCU is used to guard the
1320	* update of a tasks cgroup pointer by cgroup_attach_task()
1321	*/
1322
1323	static struct kernfs_syscall_ops cgroup_kf_syscall_ops;
1324	static const struct file_operations proc_cgroupstats_operations;
1325
1326	static char cgroup_file_name(struct cgroup cgrp, const struct cftype *cft,
1327	char *buf)
1328	{
1329	struct cgroup_subsys *ss = cft->ss;
1330
1331	if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) &&
1332	!(cgrp->root->flags & CGRP_ROOT_NOPREFIX))
1333	snprintf(buf, CGROUP_FILE_NAME_MAX, "%s.%s",
1334	cgroup_on_dfl(cgrp) ? ss->name : ss->legacy_name,
1335	cft->name);
1336	else
1337	strncpy(buf, cft->name, CGROUP_FILE_NAME_MAX);
1338	return buf;
1339	}
1340
1341	/**
1342	* cgroup_file_mode - deduce file mode of a control file
1343	* @cft: the control file in question
1344	*
1345	* S_IRUGO for read, S_IWUSR for write.
1346	*/
1347	static umode_t cgroup_file_mode(const struct cftype *cft)
1348	{
1349	umode_t mode = 0;
1350
1351	if (cft->read_u64 \|\| cft->read_s64 \|\| cft->seq_show)
1352	mode \|= S_IRUGO;
1353
1354	if (cft->write_u64 \|\| cft->write_s64 \|\| cft->write) {
1355	if (cft->flags & CFTYPE_WORLD_WRITABLE)
1356	mode \|= S_IWUGO;
1357	else
1358	mode \|= S_IWUSR;
1359	}
1360
1361	return mode;
1362	}
1363
1364	/**
1365	* cgroup_calc_subtree_ss_mask - calculate subtree_ss_mask
1366	* @subtree_control: the new subtree_control mask to consider
1367	* @this_ss_mask: available subsystems
1368	*
1369	* On the default hierarchy, a subsystem may request other subsystems to be
1370	* enabled together through its ->depends_on mask. In such cases, more
1371	* subsystems than specified in "cgroup.subtree_control" may be enabled.
1372	*
1373	* This function calculates which subsystems need to be enabled if
1374	* @subtree_control is to be applied while restricted to @this_ss_mask.
1375	*/
1376	static u16 cgroup_calc_subtree_ss_mask(u16 subtree_control, u16 this_ss_mask)
1377	{
1378	u16 cur_ss_mask = subtree_control;
1379	struct cgroup_subsys *ss;
1380	int ssid;
1381
1382	lockdep_assert_held(&cgroup_mutex);
1383
1384	cur_ss_mask \|= cgrp_dfl_implicit_ss_mask;
1385
1386	while (true) {
1387	u16 new_ss_mask = cur_ss_mask;
1388
1389	do_each_subsys_mask(ss, ssid, cur_ss_mask) {
1390	new_ss_mask \|= ss->depends_on;
1391	} while_each_subsys_mask();
1392
1393	/*
1394	* Mask out subsystems which aren't available. This can
1395	* happen only if some depended-upon subsystems were bound
1396	* to non-default hierarchies.
1397	*/
1398	new_ss_mask &= this_ss_mask;
1399
1400	if (new_ss_mask == cur_ss_mask)
1401	break;
1402	cur_ss_mask = new_ss_mask;
1403	}
1404
1405	return cur_ss_mask;
1406	}
1407
1408	/**
1409	* cgroup_kn_unlock - unlocking helper for cgroup kernfs methods
1410	* @kn: the kernfs_node being serviced
1411	*
1412	* This helper undoes cgroup_kn_lock_live() and should be invoked before
1413	* the method finishes if locking succeeded. Note that once this function
1414	* returns the cgroup returned by cgroup_kn_lock_live() may become
1415	* inaccessible any time. If the caller intends to continue to access the
1416	* cgroup, it should pin it before invoking this function.
1417	*/
1418	static void cgroup_kn_unlock(struct kernfs_node *kn)
1419	{
1420	struct cgroup *cgrp;
1421
1422	if (kernfs_type(kn) == KERNFS_DIR)
1423	cgrp = kn->priv;
1424	else
1425	cgrp = kn->parent->priv;
1426
1427	mutex_unlock(&cgroup_mutex);
1428
1429	kernfs_unbreak_active_protection(kn);
1430	cgroup_put(cgrp);
1431	}
1432
1433	/**
1434	* cgroup_kn_lock_live - locking helper for cgroup kernfs methods
1435	* @kn: the kernfs_node being serviced
1436	* @drain_offline: perform offline draining on the cgroup
1437	*
1438	* This helper is to be used by a cgroup kernfs method currently servicing
1439	* @kn. It breaks the active protection, performs cgroup locking and
1440	* verifies that the associated cgroup is alive. Returns the cgroup if
1441	* alive; otherwise, %NULL. A successful return should be undone by a
1442	* matching cgroup_kn_unlock() invocation. If @drain_offline is %true, the
1443	* cgroup is drained of offlining csses before return.
1444	*
1445	* Any cgroup kernfs method implementation which requires locking the
1446	* associated cgroup should use this helper. It avoids nesting cgroup
1447	* locking under kernfs active protection and allows all kernfs operations
1448	* including self-removal.
1449	*/
1450	static struct cgroup cgroup_kn_lock_live(struct kernfs_node kn,
1451	bool drain_offline)
1452	{
1453	struct cgroup *cgrp;
1454
1455	if (kernfs_type(kn) == KERNFS_DIR)
1456	cgrp = kn->priv;
1457	else
1458	cgrp = kn->parent->priv;
1459
1460	/*
1461	* We're gonna grab cgroup_mutex which nests outside kernfs
1462	* active_ref. cgroup liveliness check alone provides enough
1463	* protection against removal. Ensure @cgrp stays accessible and
1464	* break the active_ref protection.
1465	*/
1466	if (!cgroup_tryget(cgrp))
1467	return NULL;
1468	kernfs_break_active_protection(kn);
1469
1470	if (drain_offline)
1471	cgroup_lock_and_drain_offline(cgrp);
1472	else
1473	mutex_lock(&cgroup_mutex);
1474
1475	if (!cgroup_is_dead(cgrp))
1476	return cgrp;
1477
1478	cgroup_kn_unlock(kn);
1479	return NULL;
1480	}
1481
1482	static void cgroup_rm_file(struct cgroup cgrp, const struct cftype cft)
1483	{
1484	char name[CGROUP_FILE_NAME_MAX];
1485
1486	lockdep_assert_held(&cgroup_mutex);
1487
1488	if (cft->file_offset) {
1489	struct cgroup_subsys_state *css = cgroup_css(cgrp, cft->ss);
1490	struct cgroup_file cfile = (void )css + cft->file_offset;
1491
1492	spin_lock_irq(&cgroup_file_kn_lock);
1493	cfile->kn = NULL;
1494	spin_unlock_irq(&cgroup_file_kn_lock);
1495	}
1496
1497	kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name));
1498	}
1499
1500	/**
1501	* css_clear_dir - remove subsys files in a cgroup directory
1502	* @css: taget css
1503	*/
1504	static void css_clear_dir(struct cgroup_subsys_state *css)
1505	{
1506	struct cgroup *cgrp = css->cgroup;
1507	struct cftype *cfts;
1508
1509	if (!(css->flags & CSS_VISIBLE))
1510	return;
1511
1512	css->flags &= ~CSS_VISIBLE;
1513
1514	list_for_each_entry(cfts, &css->ss->cfts, node)
1515	cgroup_addrm_files(css, cgrp, cfts, false);
1516	}
1517
1518	/**
1519	* css_populate_dir - create subsys files in a cgroup directory
1520	* @css: target css
1521	*
1522	* On failure, no file is added.
1523	*/
1524	static int css_populate_dir(struct cgroup_subsys_state *css)
1525	{
1526	struct cgroup *cgrp = css->cgroup;
1527	struct cftype cfts, failed_cfts;
1528	int ret;
1529
1530	if ((css->flags & CSS_VISIBLE) \|\| !cgrp->kn)
1531	return 0;
1532
1533	if (!css->ss) {
1534	if (cgroup_on_dfl(cgrp))
1535	cfts = cgroup_dfl_base_files;
1536	else
1537	cfts = cgroup_legacy_base_files;
1538
1539	return cgroup_addrm_files(&cgrp->self, cgrp, cfts, true);
1540	}
1541
1542	list_for_each_entry(cfts, &css->ss->cfts, node) {
1543	ret = cgroup_addrm_files(css, cgrp, cfts, true);
1544	if (ret < 0) {
1545	failed_cfts = cfts;
1546	goto err;
1547	}
1548	}
1549
1550	css->flags \|= CSS_VISIBLE;
1551
1552	return 0;
1553	err:
1554	list_for_each_entry(cfts, &css->ss->cfts, node) {
1555	if (cfts == failed_cfts)
1556	break;
1557	cgroup_addrm_files(css, cgrp, cfts, false);
1558	}
1559	return ret;
1560	}
1561
1562	static int rebind_subsystems(struct cgroup_root *dst_root, u16 ss_mask)
1563	{
1564	struct cgroup *dcgrp = &dst_root->cgrp;
1565	struct cgroup_subsys *ss;
1566	int ssid, i, ret;
1567
1568	lockdep_assert_held(&cgroup_mutex);
1569
1570	do_each_subsys_mask(ss, ssid, ss_mask) {
1571	/*
1572	* If @ss has non-root csses attached to it, can't move.
1573	* If @ss is an implicit controller, it is exempt from this
1574	* rule and can be stolen.
1575	*/
1576	if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss)) &&
1577	!ss->implicit_on_dfl)
1578	return -EBUSY;
1579
1580	/* can't move between two non-dummy roots either */
1581	if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root)
1582	return -EBUSY;
1583	} while_each_subsys_mask();
1584
1585	do_each_subsys_mask(ss, ssid, ss_mask) {
1586	struct cgroup_root *src_root = ss->root;
1587	struct cgroup *scgrp = &src_root->cgrp;
1588	struct cgroup_subsys_state *css = cgroup_css(scgrp, ss);
1589	struct css_set *cset;
1590
1591	WARN_ON(!css \|\| cgroup_css(dcgrp, ss));
1592
1593	/* disable from the source */
1594	src_root->subsys_mask &= ~(1 << ssid);
1595	WARN_ON(cgroup_apply_control(scgrp));
1596	cgroup_finalize_control(scgrp, 0);
1597
1598	/* rebind */
1599	RCU_INIT_POINTER(scgrp->subsys[ssid], NULL);
1600	rcu_assign_pointer(dcgrp->subsys[ssid], css);
1601	ss->root = dst_root;
1602	css->cgroup = dcgrp;
1603
1604	spin_lock_irq(&css_set_lock);
1605	hash_for_each(css_set_table, i, cset, hlist)
1606	list_move_tail(&cset->e_cset_node[ss->id],
1607	&dcgrp->e_csets[ss->id]);
1608	spin_unlock_irq(&css_set_lock);
1609
1610	/* default hierarchy doesn't enable controllers by default */
1611	dst_root->subsys_mask \|= 1 << ssid;
1612	if (dst_root == &cgrp_dfl_root) {
1613	static_branch_enable(cgroup_subsys_on_dfl_key[ssid]);
1614	} else {
1615	dcgrp->subtree_control \|= 1 << ssid;
1616	static_branch_disable(cgroup_subsys_on_dfl_key[ssid]);
1617	}
1618
1619	ret = cgroup_apply_control(dcgrp);
1620	if (ret)
1621	pr_warn("partial failure to rebind %s controller (err=%d)\n",
1622	ss->name, ret);
1623
1624	if (ss->bind)
1625	ss->bind(css);
1626	} while_each_subsys_mask();
1627
1628	kernfs_activate(dcgrp->kn);
1629	return 0;
1630	}
1631
1632	static int cgroup_show_path(struct seq_file sf, struct kernfs_node kf_node,
1633	struct kernfs_root *kf_root)
1634	{
1635	int len = 0;
1636	char *buf = NULL;
1637	struct cgroup_root *kf_cgroot = cgroup_root_from_kf(kf_root);
1638	struct cgroup *ns_cgroup;
1639
1640	buf = kmalloc(PATH_MAX, GFP_KERNEL);
1641	if (!buf)
1642	return -ENOMEM;
1643
1644	spin_lock_irq(&css_set_lock);
1645	ns_cgroup = current_cgns_cgroup_from_root(kf_cgroot);
1646	len = kernfs_path_from_node(kf_node, ns_cgroup->kn, buf, PATH_MAX);
1647	spin_unlock_irq(&css_set_lock);
1648
1649	if (len >= PATH_MAX)
1650	len = -ERANGE;
1651	else if (len > 0) {
1652	seq_escape(sf, buf, " \t\n\\");
1653	len = 0;
1654	}
1655	kfree(buf);
1656	return len;
1657	}
1658
1659	static int cgroup_show_options(struct seq_file *seq,
1660	struct kernfs_root *kf_root)
1661	{
1662	struct cgroup_root *root = cgroup_root_from_kf(kf_root);
1663	struct cgroup_subsys *ss;
1664	int ssid;
1665
1666	if (root != &cgrp_dfl_root)
1667	for_each_subsys(ss, ssid)
1668	if (root->subsys_mask & (1 << ssid))
1669	seq_show_option(seq, ss->legacy_name, NULL);
1670	if (root->flags & CGRP_ROOT_NOPREFIX)
1671	seq_puts(seq, ",noprefix");
1672	if (root->flags & CGRP_ROOT_XATTR)
1673	seq_puts(seq, ",xattr");
1674
1675	spin_lock(&release_agent_path_lock);
1676	if (strlen(root->release_agent_path))
1677	seq_show_option(seq, "release_agent",
1678	root->release_agent_path);
1679	spin_unlock(&release_agent_path_lock);
1680
1681	if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags))
1682	seq_puts(seq, ",clone_children");
1683	if (strlen(root->name))
1684	seq_show_option(seq, "name", root->name);
1685	return 0;
1686	}
1687
1688	struct cgroup_sb_opts {
1689	u16 subsys_mask;
1690	unsigned int flags;
1691	char *release_agent;
1692	bool cpuset_clone_children;
1693	char *name;
1694	/* User explicitly requested empty subsystem */
1695	bool none;
1696	};
1697
1698	static int parse_cgroupfs_options(char data, struct cgroup_sb_opts opts)
1699	{
1700	char token, o = data;
1701	bool all_ss = false, one_ss = false;
1702	u16 mask = U16_MAX;
1703	struct cgroup_subsys *ss;
1704	int nr_opts = 0;
1705	int i;
1706
1707	#ifdef CONFIG_CPUSETS
1708	mask = ~((u16)1 << cpuset_cgrp_id);
1709	#endif
1710
1711	memset(opts, 0, sizeof(*opts));
1712
1713	while ((token = strsep(&o, ",")) != NULL) {
1714	nr_opts++;
1715
1716	if (!*token)
1717	return -EINVAL;
1718	if (!strcmp(token, "none")) {
1719	/* Explicitly have no subsystems */
1720	opts->none = true;
1721	continue;
1722	}
1723	if (!strcmp(token, "all")) {
1724	/* Mutually exclusive option 'all' + subsystem name */
1725	if (one_ss)
1726	return -EINVAL;
1727	all_ss = true;
1728	continue;
1729	}
1730	if (!strcmp(token, "noprefix")) {
1731	opts->flags \|= CGRP_ROOT_NOPREFIX;
1732	continue;
1733	}
1734	if (!strcmp(token, "clone_children")) {
1735	opts->cpuset_clone_children = true;
1736	continue;
1737	}
1738	if (!strcmp(token, "xattr")) {
1739	opts->flags \|= CGRP_ROOT_XATTR;
1740	continue;
1741	}
1742	if (!strncmp(token, "release_agent=", 14)) {
1743	/* Specifying two release agents is forbidden */
1744	if (opts->release_agent)
1745	return -EINVAL;
1746	opts->release_agent =
1747	kstrndup(token + 14, PATH_MAX - 1, GFP_KERNEL);
1748	if (!opts->release_agent)
1749	return -ENOMEM;
1750	continue;
1751	}
1752	if (!strncmp(token, "name=", 5)) {
1753	const char *name = token + 5;
1754	/* Can't specify an empty name */
1755	if (!strlen(name))
1756	return -EINVAL;
1757	/* Must match [\w.-]+ */
1758	for (i = 0; i < strlen(name); i++) {
1759	char c = name[i];
1760	if (isalnum(c))
1761	continue;
1762	if ((c == '.') \|\| (c == '-') \|\| (c == '_'))
1763	continue;
1764	return -EINVAL;
1765	}
1766	/* Specifying two names is forbidden */
1767	if (opts->name)
1768	return -EINVAL;
1769	opts->name = kstrndup(name,
1770	MAX_CGROUP_ROOT_NAMELEN - 1,
1771	GFP_KERNEL);
1772	if (!opts->name)
1773	return -ENOMEM;
1774
1775	continue;
1776	}
1777
1778	for_each_subsys(ss, i) {
1779	if (strcmp(token, ss->legacy_name))
1780	continue;
1781	if (!cgroup_ssid_enabled(i))
1782	continue;
1783	if (cgroup_ssid_no_v1(i))
1784	continue;
1785
1786	/* Mutually exclusive option 'all' + subsystem name */
1787	if (all_ss)
1788	return -EINVAL;
1789	opts->subsys_mask \|= (1 << i);
1790	one_ss = true;
1791
1792	break;
1793	}
1794	if (i == CGROUP_SUBSYS_COUNT)
1795	return -ENOENT;
1796	}
1797
1798	/*
1799	* If the 'all' option was specified select all the subsystems,
1800	* otherwise if 'none', 'name=' and a subsystem name options were
1801	* not specified, let's default to 'all'
1802	*/
1803	if (all_ss \|\| (!one_ss && !opts->none && !opts->name))
1804	for_each_subsys(ss, i)
1805	if (cgroup_ssid_enabled(i) && !cgroup_ssid_no_v1(i))
1806	opts->subsys_mask \|= (1 << i);
1807
1808	/*
1809	* We either have to specify by name or by subsystems. (So all
1810	* empty hierarchies must have a name).
1811	*/
1812	if (!opts->subsys_mask && !opts->name)
1813	return -EINVAL;
1814
1815	/*
1816	* Option noprefix was introduced just for backward compatibility
1817	* with the old cpuset, so we allow noprefix only if mounting just
1818	* the cpuset subsystem.
1819	*/
1820	if ((opts->flags & CGRP_ROOT_NOPREFIX) && (opts->subsys_mask & mask))
1821	return -EINVAL;
1822
1823	/* Can't specify "none" and some subsystems */
1824	if (opts->subsys_mask && opts->none)
1825	return -EINVAL;
1826
1827	return 0;
1828	}
1829
1830	static int cgroup_remount(struct kernfs_root kf_root, int flags, char *data)
1831	{
1832	int ret = 0;
1833	struct cgroup_root *root = cgroup_root_from_kf(kf_root);
1834	struct cgroup_sb_opts opts;
1835	u16 added_mask, removed_mask;
1836
1837	if (root == &cgrp_dfl_root) {
1838	pr_err("remount is not allowed\n");
1839	return -EINVAL;
1840	}
1841
1842	cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp);
1843
1844	/* See what subsystems are wanted */
1845	ret = parse_cgroupfs_options(data, &opts);
1846	if (ret)
1847	goto out_unlock;
1848
1849	if (opts.subsys_mask != root->subsys_mask \|\| opts.release_agent)
1850	pr_warn("option changes via remount are deprecated (pid=%d comm=%s)\n",
1851	task_tgid_nr(current), current->comm);
1852
1853	added_mask = opts.subsys_mask & ~root->subsys_mask;
1854	removed_mask = root->subsys_mask & ~opts.subsys_mask;
1855
1856	/* Don't allow flags or name to change at remount */
1857	if ((opts.flags ^ root->flags) \|\|
1858	(opts.name && strcmp(opts.name, root->name))) {
1859	pr_err("option or name mismatch, new: 0x%x \"%s\", old: 0x%x \"%s\"\n",
1860	opts.flags, opts.name ?: "", root->flags, root->name);
1861	ret = -EINVAL;
1862	goto out_unlock;
1863	}
1864
1865	/* remounting is not allowed for populated hierarchies */
1866	if (!list_empty(&root->cgrp.self.children)) {
1867	ret = -EBUSY;
1868	goto out_unlock;
1869	}
1870
1871	ret = rebind_subsystems(root, added_mask);
1872	if (ret)
1873	goto out_unlock;
1874
1875	WARN_ON(rebind_subsystems(&cgrp_dfl_root, removed_mask));
1876
1877	if (opts.release_agent) {
1878	spin_lock(&release_agent_path_lock);
1879	strcpy(root->release_agent_path, opts.release_agent);
1880	spin_unlock(&release_agent_path_lock);
1881	}
1882
1883	trace_cgroup_remount(root);
1884
1885	out_unlock:
1886	kfree(opts.release_agent);
1887	kfree(opts.name);
1888	mutex_unlock(&cgroup_mutex);
1889	return ret;
1890	}
1891
1892	/*
1893	* To reduce the fork() overhead for systems that are not actually using
1894	* their cgroups capability, we don't maintain the lists running through
1895	* each css_set to its tasks until we see the list actually used - in other
1896	* words after the first mount.
1897	*/
1898	static bool use_task_css_set_links __read_mostly;
1899
1900	static void cgroup_enable_task_cg_lists(void)
1901	{
1902	struct task_struct p, g;
1903
1904	spin_lock_irq(&css_set_lock);
1905
1906	if (use_task_css_set_links)
1907	goto out_unlock;
1908
1909	use_task_css_set_links = true;
1910
1911	/*
1912	* We need tasklist_lock because RCU is not safe against
1913	* while_each_thread(). Besides, a forking task that has passed
1914	* cgroup_post_fork() without seeing use_task_css_set_links = 1
1915	* is not guaranteed to have its child immediately visible in the
1916	* tasklist if we walk through it with RCU.
1917	*/
1918	read_lock(&tasklist_lock);
1919	do_each_thread(g, p) {
1920	WARN_ON_ONCE(!list_empty(&p->cg_list) \|\|
1921	task_css_set(p) != &init_css_set);
1922
1923	/*
1924	* We should check if the process is exiting, otherwise
1925	* it will race with cgroup_exit() in that the list
1926	* entry won't be deleted though the process has exited.
1927	* Do it while holding siglock so that we don't end up
1928	* racing against cgroup_exit().
1929	*
1930	* Interrupts were already disabled while acquiring
1931	* the css_set_lock, so we do not need to disable it
1932	* again when acquiring the sighand->siglock here.
1933	*/
1934	spin_lock(&p->sighand->siglock);
1935	if (!(p->flags & PF_EXITING)) {
1936	struct css_set *cset = task_css_set(p);
1937
1938	if (!css_set_populated(cset))
1939	css_set_update_populated(cset, true);
1940	list_add_tail(&p->cg_list, &cset->tasks);
1941	get_css_set(cset);
1942	}
1943	spin_unlock(&p->sighand->siglock);
1944	} while_each_thread(g, p);
1945	read_unlock(&tasklist_lock);
1946	out_unlock:
1947	spin_unlock_irq(&css_set_lock);
1948	}
1949
1950	static void init_cgroup_housekeeping(struct cgroup *cgrp)
1951	{
1952	struct cgroup_subsys *ss;
1953	int ssid;
1954
1955	INIT_LIST_HEAD(&cgrp->self.sibling);
1956	INIT_LIST_HEAD(&cgrp->self.children);
1957	INIT_LIST_HEAD(&cgrp->cset_links);
1958	INIT_LIST_HEAD(&cgrp->pidlists);
1959	mutex_init(&cgrp->pidlist_mutex);
1960	cgrp->self.cgroup = cgrp;
1961	cgrp->self.flags \|= CSS_ONLINE;
1962
1963	for_each_subsys(ss, ssid)
1964	INIT_LIST_HEAD(&cgrp->e_csets[ssid]);
1965
1966	init_waitqueue_head(&cgrp->offline_waitq);
1967	INIT_WORK(&cgrp->release_agent_work, cgroup_release_agent);
1968	}
1969
1970	static void init_cgroup_root(struct cgroup_root *root,
1971	struct cgroup_sb_opts *opts)
1972	{
1973	struct cgroup *cgrp = &root->cgrp;
1974
1975	INIT_LIST_HEAD(&root->root_list);
1976	atomic_set(&root->nr_cgrps, 1);
1977	cgrp->root = root;
1978	init_cgroup_housekeeping(cgrp);
1979	idr_init(&root->cgroup_idr);
1980
1981	root->flags = opts->flags;
1982	if (opts->release_agent)
1983	strcpy(root->release_agent_path, opts->release_agent);
1984	if (opts->name)
1985	strcpy(root->name, opts->name);
1986	if (opts->cpuset_clone_children)
1987	set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags);
1988	}
1989
1990	static int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask)
1991	{
1992	LIST_HEAD(tmp_links);
1993	struct cgroup *root_cgrp = &root->cgrp;
1994	struct css_set *cset;
1995	int i, ret;
1996
1997	lockdep_assert_held(&cgroup_mutex);
1998
1999	ret = cgroup_idr_alloc(&root->cgroup_idr, root_cgrp, 1, 2, GFP_KERNEL);
2000	if (ret < 0)
2001	goto out;
2002	root_cgrp->id = ret;
2003	root_cgrp->ancestor_ids[0] = ret;
2004
2005	ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release, 0,
2006	GFP_KERNEL);
2007	if (ret)
2008	goto out;
2009
2010	/*
2011	* We're accessing css_set_count without locking css_set_lock here,
2012	* but that's OK - it can only be increased by someone holding
2013	* cgroup_lock, and that's us. Later rebinding may disable
2014	* controllers on the default hierarchy and thus create new csets,
2015	* which can't be more than the existing ones. Allocate 2x.
2016	*/
2017	ret = allocate_cgrp_cset_links(2 * css_set_count, &tmp_links);
2018	if (ret)
2019	goto cancel_ref;
2020
2021	ret = cgroup_init_root_id(root);
2022	if (ret)
2023	goto cancel_ref;
2024
2025	root->kf_root = kernfs_create_root(&cgroup_kf_syscall_ops,
2026	KERNFS_ROOT_CREATE_DEACTIVATED,
2027	root_cgrp);
2028	if (IS_ERR(root->kf_root)) {
2029	ret = PTR_ERR(root->kf_root);
2030	goto exit_root_id;
2031	}
2032	root_cgrp->kn = root->kf_root->kn;
2033
2034	ret = css_populate_dir(&root_cgrp->self);
2035	if (ret)
2036	goto destroy_root;
2037
2038	ret = rebind_subsystems(root, ss_mask);
2039	if (ret)
2040	goto destroy_root;
2041
2042	trace_cgroup_setup_root(root);
2043
2044	/*
2045	* There must be no failure case after here, since rebinding takes
2046	* care of subsystems' refcounts, which are explicitly dropped in
2047	* the failure exit path.
2048	*/
2049	list_add(&root->root_list, &cgroup_roots);
2050	cgroup_root_count++;
2051
2052	/*
2053	* Link the root cgroup in this hierarchy into all the css_set
2054	* objects.
2055	*/
2056	spin_lock_irq(&css_set_lock);
2057	hash_for_each(css_set_table, i, cset, hlist) {
2058	link_css_set(&tmp_links, cset, root_cgrp);
2059	if (css_set_populated(cset))
2060	cgroup_update_populated(root_cgrp, true);
2061	}
2062	spin_unlock_irq(&css_set_lock);
2063
2064	BUG_ON(!list_empty(&root_cgrp->self.children));
2065	BUG_ON(atomic_read(&root->nr_cgrps) != 1);
2066
2067	kernfs_activate(root_cgrp->kn);
2068	ret = 0;
2069	goto out;
2070
2071	destroy_root:
2072	kernfs_destroy_root(root->kf_root);
2073	root->kf_root = NULL;
2074	exit_root_id:
2075	cgroup_exit_root_id(root);
2076	cancel_ref:
2077	percpu_ref_exit(&root_cgrp->self.refcnt);
2078	out:
2079	free_cgrp_cset_links(&tmp_links);
2080	return ret;
2081	}
2082
2083	static struct dentry cgroup_mount(struct file_system_type fs_type,
2084	int flags, const char *unused_dev_name,
2085	void *data)
2086	{
2087	bool is_v2 = fs_type == &cgroup2_fs_type;
2088	struct super_block *pinned_sb = NULL;
2089	struct cgroup_namespace *ns = current->nsproxy->cgroup_ns;
2090	struct cgroup_subsys *ss;
2091	struct cgroup_root *root;
2092	struct cgroup_sb_opts opts;
2093	struct dentry *dentry;
2094	int ret;
2095	int i;
2096	bool new_sb;
2097
2098	get_cgroup_ns(ns);
2099
2100	/* Check if the caller has permission to mount. */
2101	if (!ns_capable(ns->user_ns, CAP_SYS_ADMIN)) {
2102	put_cgroup_ns(ns);
2103	return ERR_PTR(-EPERM);
2104	}
2105
2106	/*
2107	* The first time anyone tries to mount a cgroup, enable the list
2108	* linking each css_set to its tasks and fix up all existing tasks.
2109	*/
2110	if (!use_task_css_set_links)
2111	cgroup_enable_task_cg_lists();
2112
2113	if (is_v2) {
2114	if (data) {
2115	pr_err("cgroup2: unknown option \"%s\"\n", (char *)data);
2116	put_cgroup_ns(ns);
2117	return ERR_PTR(-EINVAL);
2118	}
2119	cgrp_dfl_visible = true;
2120	root = &cgrp_dfl_root;
2121	cgroup_get(&root->cgrp);
2122	goto out_mount;
2123	}
2124
2125	cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp);
2126
2127	/* First find the desired set of subsystems */
2128	ret = parse_cgroupfs_options(data, &opts);
2129	if (ret)
2130	goto out_unlock;
2131
2132	/*
2133	* Destruction of cgroup root is asynchronous, so subsystems may
2134	* still be dying after the previous unmount. Let's drain the
2135	* dying subsystems. We just need to ensure that the ones
2136	* unmounted previously finish dying and don't care about new ones
2137	* starting. Testing ref liveliness is good enough.
2138	*/
2139	for_each_subsys(ss, i) {
2140	if (!(opts.subsys_mask & (1 << i)) \|\|
2141	ss->root == &cgrp_dfl_root)
2142	continue;
2143
2144	if (!percpu_ref_tryget_live(&ss->root->cgrp.self.refcnt)) {
2145	mutex_unlock(&cgroup_mutex);
2146	msleep(10);
2147	ret = restart_syscall();
2148	goto out_free;
2149	}
2150	cgroup_put(&ss->root->cgrp);
2151	}
2152
2153	for_each_root(root) {
2154	bool name_match = false;
2155
2156	if (root == &cgrp_dfl_root)
2157	continue;
2158
2159	/*
2160	* If we asked for a name then it must match. Also, if
2161	* name matches but sybsys_mask doesn't, we should fail.
2162	* Remember whether name matched.
2163	*/
2164	if (opts.name) {
2165	if (strcmp(opts.name, root->name))
2166	continue;
2167	name_match = true;
2168	}
2169
2170	/*
2171	* If we asked for subsystems (or explicitly for no
2172	* subsystems) then they must match.
2173	*/
2174	if ((opts.subsys_mask \|\| opts.none) &&
2175	(opts.subsys_mask != root->subsys_mask)) {
2176	if (!name_match)
2177	continue;
2178	ret = -EBUSY;
2179	goto out_unlock;
2180	}
2181
2182	if (root->flags ^ opts.flags)
2183	pr_warn("new mount options do not match the existing superblock, will be ignored\n");
2184
2185	/*
2186	* We want to reuse @root whose lifetime is governed by its
2187	* ->cgrp. Let's check whether @root is alive and keep it
2188	* that way. As cgroup_kill_sb() can happen anytime, we
2189	* want to block it by pinning the sb so that @root doesn't
2190	* get killed before mount is complete.
2191	*
2192	* With the sb pinned, tryget_live can reliably indicate
2193	* whether @root can be reused. If it's being killed,
2194	* drain it. We can use wait_queue for the wait but this
2195	* path is super cold. Let's just sleep a bit and retry.
2196	*/
2197	pinned_sb = kernfs_pin_sb(root->kf_root, NULL);
2198	if (IS_ERR(pinned_sb) \|\|
2199	!percpu_ref_tryget_live(&root->cgrp.self.refcnt)) {
2200	mutex_unlock(&cgroup_mutex);
2201	if (!IS_ERR_OR_NULL(pinned_sb))
2202	deactivate_super(pinned_sb);
2203	msleep(10);
2204	ret = restart_syscall();
2205	goto out_free;
2206	}
2207
2208	ret = 0;
2209	goto out_unlock;
2210	}
2211
2212	/*
2213	* No such thing, create a new one. name= matching without subsys
2214	* specification is allowed for already existing hierarchies but we
2215	* can't create new one without subsys specification.
2216	*/
2217	if (!opts.subsys_mask && !opts.none) {
2218	ret = -EINVAL;
2219	goto out_unlock;
2220	}
2221
2222	/* Hierarchies may only be created in the initial cgroup namespace. */
2223	if (ns != &init_cgroup_ns) {
2224	ret = -EPERM;
2225	goto out_unlock;
2226	}
2227
2228	root = kzalloc(sizeof(*root), GFP_KERNEL);
2229	if (!root) {
2230	ret = -ENOMEM;
2231	goto out_unlock;
2232	}
2233
2234	init_cgroup_root(root, &opts);
2235
2236	ret = cgroup_setup_root(root, opts.subsys_mask);
2237	if (ret)
2238	cgroup_free_root(root);
2239
2240	out_unlock:
2241	mutex_unlock(&cgroup_mutex);
2242	out_free:
2243	kfree(opts.release_agent);
2244	kfree(opts.name);
2245
2246	if (ret) {
2247	put_cgroup_ns(ns);
2248	return ERR_PTR(ret);
2249	}
2250	out_mount:
2251	dentry = kernfs_mount(fs_type, flags, root->kf_root,
2252	is_v2 ? CGROUP2_SUPER_MAGIC : CGROUP_SUPER_MAGIC,
2253	&new_sb);
2254
2255	/*
2256	* In non-init cgroup namespace, instead of root cgroup's
2257	* dentry, we return the dentry corresponding to the
2258	* cgroupns->root_cgrp.
2259	*/
2260	if (!IS_ERR(dentry) && ns != &init_cgroup_ns) {
2261	struct dentry *nsdentry;
2262	struct cgroup *cgrp;
2263
2264	mutex_lock(&cgroup_mutex);
2265	spin_lock_irq(&css_set_lock);
2266
2267	cgrp = cset_cgroup_from_root(ns->root_cset, root);
2268
2269	spin_unlock_irq(&css_set_lock);
2270	mutex_unlock(&cgroup_mutex);
2271
2272	nsdentry = kernfs_node_dentry(cgrp->kn, dentry->d_sb);
2273	dput(dentry);
2274	dentry = nsdentry;
2275	}
2276
2277	if (IS_ERR(dentry) \|\| !new_sb)
2278	cgroup_put(&root->cgrp);
2279
2280	/*
2281	* If @pinned_sb, we're reusing an existing root and holding an
2282	* extra ref on its sb. Mount is complete. Put the extra ref.
2283	*/
2284	if (pinned_sb) {
2285	WARN_ON(new_sb);
2286	deactivate_super(pinned_sb);
2287	}
2288
2289	put_cgroup_ns(ns);
2290	return dentry;
2291	}
2292
2293	static void cgroup_kill_sb(struct super_block *sb)
2294	{
2295	struct kernfs_root *kf_root = kernfs_root_from_sb(sb);
2296	struct cgroup_root *root = cgroup_root_from_kf(kf_root);
2297
2298	/*
2299	* If @root doesn't have any mounts or children, start killing it.
2300	* This prevents new mounts by disabling percpu_ref_tryget_live().
2301	* cgroup_mount() may wait for @root's release.
2302	*
2303	* And don't kill the default root.
2304	*/
2305	if (!list_empty(&root->cgrp.self.children) \|\|
2306	root == &cgrp_dfl_root)
2307	cgroup_put(&root->cgrp);
2308	else
2309	percpu_ref_kill(&root->cgrp.self.refcnt);
2310
2311	kernfs_kill_sb(sb);
2312	}
2313
2314	static struct file_system_type cgroup_fs_type = {
2315	.name = "cgroup",
2316	.mount = cgroup_mount,
2317	.kill_sb = cgroup_kill_sb,
2318	.fs_flags = FS_USERNS_MOUNT,
2319	};
2320
2321	static struct file_system_type cgroup2_fs_type = {
2322	.name = "cgroup2",
2323	.mount = cgroup_mount,
2324	.kill_sb = cgroup_kill_sb,
2325	.fs_flags = FS_USERNS_MOUNT,
2326	};
2327
2328	static int cgroup_path_ns_locked(struct cgroup cgrp, char buf, size_t buflen,
2329	struct cgroup_namespace *ns)
2330	{
2331	struct cgroup *root = cset_cgroup_from_root(ns->root_cset, cgrp->root);
2332
2333	return kernfs_path_from_node(cgrp->kn, root->kn, buf, buflen);
2334	}
2335
2336	int cgroup_path_ns(struct cgroup cgrp, char buf, size_t buflen,
2337	struct cgroup_namespace *ns)
2338	{
2339	int ret;
2340
2341	mutex_lock(&cgroup_mutex);
2342	spin_lock_irq(&css_set_lock);
2343
2344	ret = cgroup_path_ns_locked(cgrp, buf, buflen, ns);
2345
2346	spin_unlock_irq(&css_set_lock);
2347	mutex_unlock(&cgroup_mutex);
2348
2349	return ret;
2350	}
2351	EXPORT_SYMBOL_GPL(cgroup_path_ns);
2352
2353	/**
2354	* task_cgroup_path - cgroup path of a task in the first cgroup hierarchy
2355	* @task: target task
2356	* @buf: the buffer to write the path into
2357	* @buflen: the length of the buffer
2358	*
2359	* Determine @task's cgroup on the first (the one with the lowest non-zero
2360	* hierarchy_id) cgroup hierarchy and copy its path into @buf. This
2361	* function grabs cgroup_mutex and shouldn't be used inside locks used by
2362	* cgroup controller callbacks.
2363	*
2364	* Return value is the same as kernfs_path().
2365	*/
2366	int task_cgroup_path(struct task_struct task, char buf, size_t buflen)
2367	{
2368	struct cgroup_root *root;
2369	struct cgroup *cgrp;
2370	int hierarchy_id = 1;
2371	int ret;
2372
2373	mutex_lock(&cgroup_mutex);
2374	spin_lock_irq(&css_set_lock);
2375
2376	root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id);
2377
2378	if (root) {
2379	cgrp = task_cgroup_from_root(task, root);
2380	ret = cgroup_path_ns_locked(cgrp, buf, buflen, &init_cgroup_ns);
2381	} else {
2382	/* if no hierarchy exists, everyone is in "/" */
2383	ret = strlcpy(buf, "/", buflen);
2384	}
2385
2386	spin_unlock_irq(&css_set_lock);
2387	mutex_unlock(&cgroup_mutex);
2388	return ret;
2389	}
2390	EXPORT_SYMBOL_GPL(task_cgroup_path);
2391
2392	/* used to track tasks and other necessary states during migration */
2393	struct cgroup_taskset {
2394	/* the src and dst cset list running through cset->mg_node */
2395	struct list_head src_csets;
2396	struct list_head dst_csets;
2397
2398	/* the subsys currently being processed */
2399	int ssid;
2400
2401	/*
2402	* Fields for cgroup_taskset_*() iteration.
2403	*
2404	* Before migration is committed, the target migration tasks are on
2405	* ->mg_tasks of the csets on ->src_csets. After, on ->mg_tasks of
2406	* the csets on ->dst_csets. ->csets point to either ->src_csets
2407	* or ->dst_csets depending on whether migration is committed.
2408	*
2409	* ->cur_csets and ->cur_task point to the current task position
2410	* during iteration.
2411	*/
2412	struct list_head *csets;
2413	struct css_set *cur_cset;
2414	struct task_struct *cur_task;
2415	};
2416
2417	#define CGROUP_TASKSET_INIT(tset) (struct cgroup_taskset){ \
2418	.src_csets = LIST_HEAD_INIT(tset.src_csets), \
2419	.dst_csets = LIST_HEAD_INIT(tset.dst_csets), \
2420	.csets = &tset.src_csets, \
2421	}
2422
2423	/**
2424	* cgroup_taskset_add - try to add a migration target task to a taskset
2425	* @task: target task
2426	* @tset: target taskset
2427	*
2428	* Add @task, which is a migration target, to @tset. This function becomes
2429	* noop if @task doesn't need to be migrated. @task's css_set should have
2430	* been added as a migration source and @task->cg_list will be moved from
2431	* the css_set's tasks list to mg_tasks one.
2432	*/
2433	static void cgroup_taskset_add(struct task_struct *task,
2434	struct cgroup_taskset *tset)
2435	{
2436	struct css_set *cset;
2437
2438	lockdep_assert_held(&css_set_lock);
2439
2440	/* @task either already exited or can't exit until the end */
2441	if (task->flags & PF_EXITING)
2442	return;
2443
2444	/* leave @task alone if post_fork() hasn't linked it yet */
2445	if (list_empty(&task->cg_list))
2446	return;
2447
2448	cset = task_css_set(task);
2449	if (!cset->mg_src_cgrp)
2450	return;
2451
2452	list_move_tail(&task->cg_list, &cset->mg_tasks);
2453	if (list_empty(&cset->mg_node))
2454	list_add_tail(&cset->mg_node, &tset->src_csets);
2455	if (list_empty(&cset->mg_dst_cset->mg_node))
2456	list_move_tail(&cset->mg_dst_cset->mg_node,
2457	&tset->dst_csets);
2458	}
2459
2460	/**
2461	* cgroup_taskset_first - reset taskset and return the first task
2462	* @tset: taskset of interest
2463	* @dst_cssp: output variable for the destination css
2464	*
2465	* @tset iteration is initialized and the first task is returned.
2466	*/
2467	struct task_struct cgroup_taskset_first(struct cgroup_taskset tset,
2468	struct cgroup_subsys_state **dst_cssp)
2469	{
2470	tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node);
2471	tset->cur_task = NULL;
2472
2473	return cgroup_taskset_next(tset, dst_cssp);
2474	}
2475
2476	/**
2477	* cgroup_taskset_next - iterate to the next task in taskset
2478	* @tset: taskset of interest
2479	* @dst_cssp: output variable for the destination css
2480	*
2481	* Return the next task in @tset. Iteration must have been initialized
2482	* with cgroup_taskset_first().
2483	*/
2484	struct task_struct cgroup_taskset_next(struct cgroup_taskset tset,
2485	struct cgroup_subsys_state **dst_cssp)
2486	{
2487	struct css_set *cset = tset->cur_cset;
2488	struct task_struct *task = tset->cur_task;
2489
2490	while (&cset->mg_node != tset->csets) {
2491	if (!task)
2492	task = list_first_entry(&cset->mg_tasks,
2493	struct task_struct, cg_list);
2494	else
2495	task = list_next_entry(task, cg_list);
2496
2497	if (&task->cg_list != &cset->mg_tasks) {
2498	tset->cur_cset = cset;
2499	tset->cur_task = task;
2500
2501	/*
2502	* This function may be called both before and
2503	* after cgroup_taskset_migrate(). The two cases
2504	* can be distinguished by looking at whether @cset
2505	* has its ->mg_dst_cset set.
2506	*/
2507	if (cset->mg_dst_cset)
2508	*dst_cssp = cset->mg_dst_cset->subsys[tset->ssid];
2509	else
2510	*dst_cssp = cset->subsys[tset->ssid];
2511
2512	return task;
2513	}
2514
2515	cset = list_next_entry(cset, mg_node);
2516	task = NULL;
2517	}
2518
2519	return NULL;
2520	}
2521
2522	/**
2523	* cgroup_taskset_migrate - migrate a taskset
2524	* @tset: taget taskset
2525	* @root: cgroup root the migration is taking place on
2526	*
2527	* Migrate tasks in @tset as setup by migration preparation functions.
2528	* This function fails iff one of the ->can_attach callbacks fails and
2529	* guarantees that either all or none of the tasks in @tset are migrated.
2530	* @tset is consumed regardless of success.
2531	*/
2532	static int cgroup_taskset_migrate(struct cgroup_taskset *tset,
2533	struct cgroup_root *root)
2534	{
2535	struct cgroup_subsys *ss;
2536	struct task_struct task, tmp_task;
2537	struct css_set cset, tmp_cset;
2538	int ssid, failed_ssid, ret;
2539
2540	/* methods shouldn't be called if no task is actually migrating */
2541	if (list_empty(&tset->src_csets))
2542	return 0;
2543
2544	/* check that we can legitimately attach to the cgroup */
2545	do_each_subsys_mask(ss, ssid, root->subsys_mask) {
2546	if (ss->can_attach) {
2547	tset->ssid = ssid;
2548	ret = ss->can_attach(tset);
2549	if (ret) {
2550	failed_ssid = ssid;
2551	goto out_cancel_attach;
2552	}
2553	}
2554	} while_each_subsys_mask();
2555
2556	/*
2557	* Now that we're guaranteed success, proceed to move all tasks to
2558	* the new cgroup. There are no failure cases after here, so this
2559	* is the commit point.
2560	*/
2561	spin_lock_irq(&css_set_lock);
2562	list_for_each_entry(cset, &tset->src_csets, mg_node) {
2563	list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list) {
2564	struct css_set *from_cset = task_css_set(task);
2565	struct css_set *to_cset = cset->mg_dst_cset;
2566
2567	get_css_set(to_cset);
2568	css_set_move_task(task, from_cset, to_cset, true);
2569	put_css_set_locked(from_cset);
2570	}
2571	}
2572	spin_unlock_irq(&css_set_lock);
2573
2574	/*
2575	* Migration is committed, all target tasks are now on dst_csets.
2576	* Nothing is sensitive to fork() after this point. Notify
2577	* controllers that migration is complete.
2578	*/
2579	tset->csets = &tset->dst_csets;
2580
2581	do_each_subsys_mask(ss, ssid, root->subsys_mask) {
2582	if (ss->attach) {
2583	tset->ssid = ssid;
2584	ss->attach(tset);
2585	}
2586	} while_each_subsys_mask();
2587
2588	ret = 0;
2589	goto out_release_tset;
2590
2591	out_cancel_attach:
2592	do_each_subsys_mask(ss, ssid, root->subsys_mask) {
2593	if (ssid == failed_ssid)
2594	break;
2595	if (ss->cancel_attach) {
2596	tset->ssid = ssid;
2597	ss->cancel_attach(tset);
2598	}
2599	} while_each_subsys_mask();
2600	out_release_tset:
2601	spin_lock_irq(&css_set_lock);
2602	list_splice_init(&tset->dst_csets, &tset->src_csets);
2603	list_for_each_entry_safe(cset, tmp_cset, &tset->src_csets, mg_node) {
2604	list_splice_tail_init(&cset->mg_tasks, &cset->tasks);
2605	list_del_init(&cset->mg_node);
2606	}
2607	spin_unlock_irq(&css_set_lock);
2608	return ret;
2609	}
2610
2611	/**
2612	* cgroup_may_migrate_to - verify whether a cgroup can be migration destination
2613	* @dst_cgrp: destination cgroup to test
2614	*
2615	* On the default hierarchy, except for the root, subtree_control must be
2616	* zero for migration destination cgroups with tasks so that child cgroups
2617	* don't compete against tasks.
2618	*/
2619	static bool cgroup_may_migrate_to(struct cgroup *dst_cgrp)
2620	{
2621	return !cgroup_on_dfl(dst_cgrp) \|\| !cgroup_parent(dst_cgrp) \|\|
2622	!dst_cgrp->subtree_control;
2623	}
2624
2625	/**
2626	* cgroup_migrate_finish - cleanup after attach
2627	* @preloaded_csets: list of preloaded css_sets
2628	*
2629	* Undo cgroup_migrate_add_src() and cgroup_migrate_prepare_dst(). See
2630	* those functions for details.
2631	*/
2632	static void cgroup_migrate_finish(struct list_head *preloaded_csets)
2633	{
2634	struct css_set cset, tmp_cset;
2635
2636	lockdep_assert_held(&cgroup_mutex);
2637
2638	spin_lock_irq(&css_set_lock);
2639	list_for_each_entry_safe(cset, tmp_cset, preloaded_csets, mg_preload_node) {
2640	cset->mg_src_cgrp = NULL;
2641	cset->mg_dst_cgrp = NULL;
2642	cset->mg_dst_cset = NULL;
2643	list_del_init(&cset->mg_preload_node);
2644	put_css_set_locked(cset);
2645	}
2646	spin_unlock_irq(&css_set_lock);
2647	}
2648
2649	/**
2650	* cgroup_migrate_add_src - add a migration source css_set
2651	* @src_cset: the source css_set to add
2652	* @dst_cgrp: the destination cgroup
2653	* @preloaded_csets: list of preloaded css_sets
2654	*
2655	* Tasks belonging to @src_cset are about to be migrated to @dst_cgrp. Pin
2656	* @src_cset and add it to @preloaded_csets, which should later be cleaned
2657	* up by cgroup_migrate_finish().
2658	*
2659	* This function may be called without holding cgroup_threadgroup_rwsem
2660	* even if the target is a process. Threads may be created and destroyed
2661	* but as long as cgroup_mutex is not dropped, no new css_set can be put
2662	* into play and the preloaded css_sets are guaranteed to cover all
2663	* migrations.
2664	*/
2665	static void cgroup_migrate_add_src(struct css_set *src_cset,
2666	struct cgroup *dst_cgrp,
2667	struct list_head *preloaded_csets)
2668	{
2669	struct cgroup *src_cgrp;
2670
2671	lockdep_assert_held(&cgroup_mutex);
2672	lockdep_assert_held(&css_set_lock);
2673
2674	/*
2675	* If ->dead, @src_set is associated with one or more dead cgroups
2676	* and doesn't contain any migratable tasks. Ignore it early so
2677	* that the rest of migration path doesn't get confused by it.
2678	*/
2679	if (src_cset->dead)
2680	return;
2681
2682	src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root);
2683
2684	if (!list_empty(&src_cset->mg_preload_node))
2685	return;
2686
2687	WARN_ON(src_cset->mg_src_cgrp);
2688	WARN_ON(src_cset->mg_dst_cgrp);
2689	WARN_ON(!list_empty(&src_cset->mg_tasks));
2690	WARN_ON(!list_empty(&src_cset->mg_node));
2691
2692	src_cset->mg_src_cgrp = src_cgrp;
2693	src_cset->mg_dst_cgrp = dst_cgrp;
2694	get_css_set(src_cset);
2695	list_add(&src_cset->mg_preload_node, preloaded_csets);
2696	}
2697
2698	/**
2699	* cgroup_migrate_prepare_dst - prepare destination css_sets for migration
2700	* @preloaded_csets: list of preloaded source css_sets
2701	*
2702	* Tasks are about to be moved and all the source css_sets have been
2703	* preloaded to @preloaded_csets. This function looks up and pins all
2704	* destination css_sets, links each to its source, and append them to
2705	* @preloaded_csets.
2706	*
2707	* This function must be called after cgroup_migrate_add_src() has been
2708	* called on each migration source css_set. After migration is performed
2709	* using cgroup_migrate(), cgroup_migrate_finish() must be called on
2710	* @preloaded_csets.
2711	*/
2712	static int cgroup_migrate_prepare_dst(struct list_head *preloaded_csets)
2713	{
2714	LIST_HEAD(csets);
2715	struct css_set src_cset, tmp_cset;
2716
2717	lockdep_assert_held(&cgroup_mutex);
2718
2719	/* look up the dst cset for each src cset and link it to src */
2720	list_for_each_entry_safe(src_cset, tmp_cset, preloaded_csets, mg_preload_node) {
2721	struct css_set *dst_cset;
2722
2723	dst_cset = find_css_set(src_cset, src_cset->mg_dst_cgrp);
2724	if (!dst_cset)
2725	goto err;
2726
2727	WARN_ON_ONCE(src_cset->mg_dst_cset \|\| dst_cset->mg_dst_cset);
2728
2729	/*
2730	* If src cset equals dst, it's noop. Drop the src.
2731	* cgroup_migrate() will skip the cset too. Note that we
2732	* can't handle src == dst as some nodes are used by both.
2733	*/
2734	if (src_cset == dst_cset) {
2735	src_cset->mg_src_cgrp = NULL;
2736	src_cset->mg_dst_cgrp = NULL;
2737	list_del_init(&src_cset->mg_preload_node);
2738	put_css_set(src_cset);
2739	put_css_set(dst_cset);
2740	continue;
2741	}
2742
2743	src_cset->mg_dst_cset = dst_cset;
2744
2745	if (list_empty(&dst_cset->mg_preload_node))
2746	list_add(&dst_cset->mg_preload_node, &csets);
2747	else
2748	put_css_set(dst_cset);
2749	}
2750
2751	list_splice_tail(&csets, preloaded_csets);
2752	return 0;
2753	err:
2754	cgroup_migrate_finish(&csets);
2755	return -ENOMEM;
2756	}
2757
2758	/**
2759	* cgroup_migrate - migrate a process or task to a cgroup
2760	* @leader: the leader of the process or the task to migrate
2761	* @threadgroup: whether @leader points to the whole process or a single task
2762	* @root: cgroup root migration is taking place on
2763	*
2764	* Migrate a process or task denoted by @leader. If migrating a process,
2765	* the caller must be holding cgroup_threadgroup_rwsem. The caller is also
2766	* responsible for invoking cgroup_migrate_add_src() and
2767	* cgroup_migrate_prepare_dst() on the targets before invoking this
2768	* function and following up with cgroup_migrate_finish().
2769	*
2770	* As long as a controller's ->can_attach() doesn't fail, this function is
2771	* guaranteed to succeed. This means that, excluding ->can_attach()
2772	* failure, when migrating multiple targets, the success or failure can be
2773	* decided for all targets by invoking group_migrate_prepare_dst() before
2774	* actually starting migrating.
2775	*/
2776	static int cgroup_migrate(struct task_struct *leader, bool threadgroup,
2777	struct cgroup_root *root)
2778	{
2779	struct cgroup_taskset tset = CGROUP_TASKSET_INIT(tset);
2780	struct task_struct *task;
2781
2782	/*
2783	* Prevent freeing of tasks while we take a snapshot. Tasks that are
2784	* already PF_EXITING could be freed from underneath us unless we
2785	* take an rcu_read_lock.
2786	*/
2787	spin_lock_irq(&css_set_lock);
2788	rcu_read_lock();
2789	task = leader;
2790	do {
2791	cgroup_taskset_add(task, &tset);
2792	if (!threadgroup)
2793	break;
2794	} while_each_thread(leader, task);
2795	rcu_read_unlock();
2796	spin_unlock_irq(&css_set_lock);
2797
2798	return cgroup_taskset_migrate(&tset, root);
2799	}
2800
2801	/**
2802	* cgroup_attach_task - attach a task or a whole threadgroup to a cgroup
2803	* @dst_cgrp: the cgroup to attach to
2804	* @leader: the task or the leader of the threadgroup to be attached
2805	* @threadgroup: attach the whole threadgroup?
2806	*
2807	* Call holding cgroup_mutex and cgroup_threadgroup_rwsem.
2808	*/
2809	static int cgroup_attach_task(struct cgroup *dst_cgrp,
2810	struct task_struct *leader, bool threadgroup)
2811	{
2812	LIST_HEAD(preloaded_csets);
2813	struct task_struct *task;
2814	int ret;
2815
2816	if (!cgroup_may_migrate_to(dst_cgrp))
2817	return -EBUSY;
2818
2819	/* look up all src csets */
2820	spin_lock_irq(&css_set_lock);
2821	rcu_read_lock();
2822	task = leader;
2823	do {
2824	cgroup_migrate_add_src(task_css_set(task), dst_cgrp,
2825	&preloaded_csets);
2826	if (!threadgroup)
2827	break;
2828	} while_each_thread(leader, task);
2829	rcu_read_unlock();
2830	spin_unlock_irq(&css_set_lock);
2831
2832	/* prepare dst csets and commit */
2833	ret = cgroup_migrate_prepare_dst(&preloaded_csets);
2834	if (!ret)
2835	ret = cgroup_migrate(leader, threadgroup, dst_cgrp->root);
2836
2837	cgroup_migrate_finish(&preloaded_csets);
2838
2839	if (!ret)
2840	trace_cgroup_attach_task(dst_cgrp, leader, threadgroup);
2841
2842	return ret;
2843	}
2844
2845	static int cgroup_procs_write_permission(struct task_struct *task,
2846	struct cgroup *dst_cgrp,
2847	struct kernfs_open_file *of)
2848	{
2849	const struct cred *cred = current_cred();
2850	const struct cred *tcred = get_task_cred(task);
2851	int ret = 0;
2852
2853	/*
2854	* even if we're attaching all tasks in the thread group, we only
2855	* need to check permissions on one of them.
2856	*/
2857	if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) &&
2858	!uid_eq(cred->euid, tcred->uid) &&
2859	!uid_eq(cred->euid, tcred->suid))
2860	ret = -EACCES;
2861
2862	if (!ret && cgroup_on_dfl(dst_cgrp)) {
2863	struct super_block *sb = of->file->f_path.dentry->d_sb;
2864	struct cgroup *cgrp;
2865	struct inode *inode;
2866
2867	spin_lock_irq(&css_set_lock);
2868	cgrp = task_cgroup_from_root(task, &cgrp_dfl_root);
2869	spin_unlock_irq(&css_set_lock);
2870
2871	while (!cgroup_is_descendant(dst_cgrp, cgrp))
2872	cgrp = cgroup_parent(cgrp);
2873
2874	ret = -ENOMEM;
2875	inode = kernfs_get_inode(sb, cgrp->procs_file.kn);
2876	if (inode) {
2877	ret = inode_permission(inode, MAY_WRITE);
2878	iput(inode);
2879	}
2880	}
2881
2882	put_cred(tcred);
2883	return ret;
2884	}
2885
2886	/*
2887	* Find the task_struct of the task to attach by vpid and pass it along to the
2888	* function to attach either it or all tasks in its threadgroup. Will lock
2889	* cgroup_mutex and threadgroup.
2890	*/
2891	static ssize_t __cgroup_procs_write(struct kernfs_open_file of, char buf,
2892	size_t nbytes, loff_t off, bool threadgroup)
2893	{
2894	struct task_struct *tsk;
2895	struct cgroup_subsys *ss;
2896	struct cgroup *cgrp;
2897	pid_t pid;
2898	int ssid, ret;
2899
2900	if (kstrtoint(strstrip(buf), 0, &pid) \|\| pid < 0)
2901	return -EINVAL;
2902
2903	cgrp = cgroup_kn_lock_live(of->kn, false);
2904	if (!cgrp)
2905	return -ENODEV;
2906
2907	percpu_down_write(&cgroup_threadgroup_rwsem);
2908	rcu_read_lock();
2909	if (pid) {
2910	tsk = find_task_by_vpid(pid);
2911	if (!tsk) {
2912	ret = -ESRCH;
2913	goto out_unlock_rcu;
2914	}
2915	} else {
2916	tsk = current;
2917	}
2918
2919	if (threadgroup)
2920	tsk = tsk->group_leader;
2921
2922	/*
2923	* Workqueue threads may acquire PF_NO_SETAFFINITY and become
2924	* trapped in a cpuset, or RT worker may be born in a cgroup
2925	* with no rt_runtime allocated. Just say no.
2926	*/
2927	if (tsk == kthreadd_task \|\| (tsk->flags & PF_NO_SETAFFINITY)) {
2928	ret = -EINVAL;
2929	goto out_unlock_rcu;
2930	}
2931
2932	get_task_struct(tsk);
2933	rcu_read_unlock();
2934
2935	ret = cgroup_procs_write_permission(tsk, cgrp, of);
2936	if (!ret)
2937	ret = cgroup_attach_task(cgrp, tsk, threadgroup);
2938
2939	put_task_struct(tsk);
2940	goto out_unlock_threadgroup;
2941
2942	out_unlock_rcu:
2943	rcu_read_unlock();
2944	out_unlock_threadgroup:
2945	percpu_up_write(&cgroup_threadgroup_rwsem);
2946	for_each_subsys(ss, ssid)
2947	if (ss->post_attach)
2948	ss->post_attach();
2949	cgroup_kn_unlock(of->kn);
2950	return ret ?: nbytes;
2951	}
2952
2953	/**
2954	* cgroup_attach_task_all - attach task 'tsk' to all cgroups of task 'from'
2955	* @from: attach to all cgroups of a given task
2956	* @tsk: the task to be attached
2957	*/
2958	int cgroup_attach_task_all(struct task_struct from, struct task_struct tsk)
2959	{
2960	struct cgroup_root *root;
2961	int retval = 0;
2962
2963	mutex_lock(&cgroup_mutex);
2964	percpu_down_write(&cgroup_threadgroup_rwsem);
2965	for_each_root(root) {
2966	struct cgroup *from_cgrp;
2967
2968	if (root == &cgrp_dfl_root)
2969	continue;
2970
2971	spin_lock_irq(&css_set_lock);
2972	from_cgrp = task_cgroup_from_root(from, root);
2973	spin_unlock_irq(&css_set_lock);
2974
2975	retval = cgroup_attach_task(from_cgrp, tsk, false);
2976	if (retval)
2977	break;
2978	}
2979	percpu_up_write(&cgroup_threadgroup_rwsem);
2980	mutex_unlock(&cgroup_mutex);
2981
2982	return retval;
2983	}
2984	EXPORT_SYMBOL_GPL(cgroup_attach_task_all);
2985
2986	static ssize_t cgroup_tasks_write(struct kernfs_open_file *of,
2987	char *buf, size_t nbytes, loff_t off)
2988	{
2989	return __cgroup_procs_write(of, buf, nbytes, off, false);
2990	}
2991
2992	static ssize_t cgroup_procs_write(struct kernfs_open_file *of,
2993	char *buf, size_t nbytes, loff_t off)
2994	{
2995	return __cgroup_procs_write(of, buf, nbytes, off, true);
2996	}
2997
2998	static ssize_t cgroup_release_agent_write(struct kernfs_open_file *of,
2999	char *buf, size_t nbytes, loff_t off)
3000	{
3001	struct cgroup *cgrp;
3002
3003	BUILD_BUG_ON(sizeof(cgrp->root->release_agent_path) < PATH_MAX);
3004
3005	cgrp = cgroup_kn_lock_live(of->kn, false);
3006	if (!cgrp)
3007	return -ENODEV;
3008	spin_lock(&release_agent_path_lock);
3009	strlcpy(cgrp->root->release_agent_path, strstrip(buf),
3010	sizeof(cgrp->root->release_agent_path));
3011	spin_unlock(&release_agent_path_lock);
3012	cgroup_kn_unlock(of->kn);
3013	return nbytes;
3014	}
3015
3016	static int cgroup_release_agent_show(struct seq_file seq, void v)
3017	{
3018	struct cgroup *cgrp = seq_css(seq)->cgroup;
3019
3020	spin_lock(&release_agent_path_lock);
3021	seq_puts(seq, cgrp->root->release_agent_path);
3022	spin_unlock(&release_agent_path_lock);
3023	seq_putc(seq, '\n');
3024	return 0;
3025	}
3026
3027	static int cgroup_sane_behavior_show(struct seq_file seq, void v)
3028	{
3029	seq_puts(seq, "0\n");
3030	return 0;
3031	}
3032
3033	static void cgroup_print_ss_mask(struct seq_file *seq, u16 ss_mask)
3034	{
3035	struct cgroup_subsys *ss;
3036	bool printed = false;
3037	int ssid;
3038
3039	do_each_subsys_mask(ss, ssid, ss_mask) {
3040	if (printed)
3041	seq_putc(seq, ' ');
3042	seq_printf(seq, "%s", ss->name);
3043	printed = true;
3044	} while_each_subsys_mask();
3045	if (printed)
3046	seq_putc(seq, '\n');
3047	}
3048
3049	/* show controllers which are enabled from the parent */
3050	static int cgroup_controllers_show(struct seq_file seq, void v)
3051	{
3052	struct cgroup *cgrp = seq_css(seq)->cgroup;
3053
3054	cgroup_print_ss_mask(seq, cgroup_control(cgrp));
3055	return 0;
3056	}
3057
3058	/* show controllers which are enabled for a given cgroup's children */
3059	static int cgroup_subtree_control_show(struct seq_file seq, void v)
3060	{
3061	struct cgroup *cgrp = seq_css(seq)->cgroup;
3062
3063	cgroup_print_ss_mask(seq, cgrp->subtree_control);
3064	return 0;
3065	}
3066
3067	/**
3068	* cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy
3069	* @cgrp: root of the subtree to update csses for
3070	*
3071	* @cgrp's control masks have changed and its subtree's css associations
3072	* need to be updated accordingly. This function looks up all css_sets
3073	* which are attached to the subtree, creates the matching updated css_sets
3074	* and migrates the tasks to the new ones.
3075	*/
3076	static int cgroup_update_dfl_csses(struct cgroup *cgrp)
3077	{
3078	LIST_HEAD(preloaded_csets);
3079	struct cgroup_taskset tset = CGROUP_TASKSET_INIT(tset);
3080	struct cgroup_subsys_state *d_css;
3081	struct cgroup *dsct;
3082	struct css_set *src_cset;
3083	int ret;
3084
3085	lockdep_assert_held(&cgroup_mutex);
3086
3087	percpu_down_write(&cgroup_threadgroup_rwsem);
3088
3089	/* look up all csses currently attached to @cgrp's subtree */
3090	spin_lock_irq(&css_set_lock);
3091	cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) {
3092	struct cgrp_cset_link *link;
3093
3094	list_for_each_entry(link, &dsct->cset_links, cset_link)
3095	cgroup_migrate_add_src(link->cset, dsct,
3096	&preloaded_csets);
3097	}
3098	spin_unlock_irq(&css_set_lock);
3099
3100	/* NULL dst indicates self on default hierarchy */
3101	ret = cgroup_migrate_prepare_dst(&preloaded_csets);
3102	if (ret)
3103	goto out_finish;
3104
3105	spin_lock_irq(&css_set_lock);
3106	list_for_each_entry(src_cset, &preloaded_csets, mg_preload_node) {
3107	struct task_struct task, ntask;
3108
3109	/* src_csets precede dst_csets, break on the first dst_cset */
3110	if (!src_cset->mg_src_cgrp)
3111	break;
3112
3113	/* all tasks in src_csets need to be migrated */
3114	list_for_each_entry_safe(task, ntask, &src_cset->tasks, cg_list)
3115	cgroup_taskset_add(task, &tset);
3116	}
3117	spin_unlock_irq(&css_set_lock);
3118
3119	ret = cgroup_taskset_migrate(&tset, cgrp->root);
3120	out_finish:
3121	cgroup_migrate_finish(&preloaded_csets);
3122	percpu_up_write(&cgroup_threadgroup_rwsem);
3123	return ret;
3124	}
3125
3126	/**
3127	* cgroup_lock_and_drain_offline - lock cgroup_mutex and drain offlined csses
3128	* @cgrp: root of the target subtree
3129	*
3130	* Because css offlining is asynchronous, userland may try to re-enable a
3131	* controller while the previous css is still around. This function grabs
3132	* cgroup_mutex and drains the previous css instances of @cgrp's subtree.
3133	*/
3134	static void cgroup_lock_and_drain_offline(struct cgroup *cgrp)
3135	__acquires(&cgroup_mutex)
3136	{
3137	struct cgroup *dsct;
3138	struct cgroup_subsys_state *d_css;
3139	struct cgroup_subsys *ss;
3140	int ssid;
3141
3142	restart:
3143	mutex_lock(&cgroup_mutex);
3144
3145	cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) {
3146	for_each_subsys(ss, ssid) {
3147	struct cgroup_subsys_state *css = cgroup_css(dsct, ss);
3148	DEFINE_WAIT(wait);
3149
3150	if (!css \|\| !percpu_ref_is_dying(&css->refcnt))
3151	continue;
3152
3153	cgroup_get(dsct);
3154	prepare_to_wait(&dsct->offline_waitq, &wait,
3155	TASK_UNINTERRUPTIBLE);
3156
3157	mutex_unlock(&cgroup_mutex);
3158	schedule();
3159	finish_wait(&dsct->offline_waitq, &wait);
3160
3161	cgroup_put(dsct);
3162	goto restart;
3163	}
3164	}
3165	}
3166
3167	/**
3168	* cgroup_save_control - save control masks of a subtree
3169	* @cgrp: root of the target subtree
3170	*
3171	* Save ->subtree_control and ->subtree_ss_mask to the respective old_
3172	* prefixed fields for @cgrp's subtree including @cgrp itself.
3173	*/
3174	static void cgroup_save_control(struct cgroup *cgrp)
3175	{
3176	struct cgroup *dsct;
3177	struct cgroup_subsys_state *d_css;
3178
3179	cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) {
3180	dsct->old_subtree_control = dsct->subtree_control;
3181	dsct->old_subtree_ss_mask = dsct->subtree_ss_mask;
3182	}
3183	}
3184
3185	/**
3186	* cgroup_propagate_control - refresh control masks of a subtree
3187	* @cgrp: root of the target subtree
3188	*
3189	* For @cgrp and its subtree, ensure ->subtree_ss_mask matches
3190	* ->subtree_control and propagate controller availability through the
3191	* subtree so that descendants don't have unavailable controllers enabled.
3192	*/
3193	static void cgroup_propagate_control(struct cgroup *cgrp)
3194	{
3195	struct cgroup *dsct;
3196	struct cgroup_subsys_state *d_css;
3197
3198	cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) {
3199	dsct->subtree_control &= cgroup_control(dsct);
3200	dsct->subtree_ss_mask =
3201	cgroup_calc_subtree_ss_mask(dsct->subtree_control,
3202	cgroup_ss_mask(dsct));
3203	}
3204	}
3205
3206	/**
3207	* cgroup_restore_control - restore control masks of a subtree
3208	* @cgrp: root of the target subtree
3209	*
3210	* Restore ->subtree_control and ->subtree_ss_mask from the respective old_
3211	* prefixed fields for @cgrp's subtree including @cgrp itself.
3212	*/
3213	static void cgroup_restore_control(struct cgroup *cgrp)
3214	{
3215	struct cgroup *dsct;
3216	struct cgroup_subsys_state *d_css;
3217
3218	cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) {
3219	dsct->subtree_control = dsct->old_subtree_control;
3220	dsct->subtree_ss_mask = dsct->old_subtree_ss_mask;
3221	}
3222	}
3223
3224	static bool css_visible(struct cgroup_subsys_state *css)
3225	{
3226	struct cgroup_subsys *ss = css->ss;
3227	struct cgroup *cgrp = css->cgroup;
3228
3229	if (cgroup_control(cgrp) & (1 << ss->id))
3230	return true;
3231	if (!(cgroup_ss_mask(cgrp) & (1 << ss->id)))
3232	return false;
3233	return cgroup_on_dfl(cgrp) && ss->implicit_on_dfl;
3234	}
3235
3236	/**
3237	* cgroup_apply_control_enable - enable or show csses according to control
3238	* @cgrp: root of the target subtree
3239	*
3240	* Walk @cgrp's subtree and create new csses or make the existing ones
3241	* visible. A css is created invisible if it's being implicitly enabled
3242	* through dependency. An invisible css is made visible when the userland
3243	* explicitly enables it.
3244	*
3245	* Returns 0 on success, -errno on failure. On failure, csses which have
3246	* been processed already aren't cleaned up. The caller is responsible for
3247	* cleaning up with cgroup_apply_control_disble().
3248	*/
3249	static int cgroup_apply_control_enable(struct cgroup *cgrp)
3250	{
3251	struct cgroup *dsct;
3252	struct cgroup_subsys_state *d_css;
3253	struct cgroup_subsys *ss;
3254	int ssid, ret;
3255
3256	cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) {
3257	for_each_subsys(ss, ssid) {
3258	struct cgroup_subsys_state *css = cgroup_css(dsct, ss);
3259
3260	WARN_ON_ONCE(css && percpu_ref_is_dying(&css->refcnt));
3261
3262	if (!(cgroup_ss_mask(dsct) & (1 << ss->id)))
3263	continue;
3264
3265	if (!css) {
3266	css = css_create(dsct, ss);
3267	if (IS_ERR(css))
3268	return PTR_ERR(css);
3269	}
3270
3271	if (css_visible(css)) {
3272	ret = css_populate_dir(css);
3273	if (ret)
3274	return ret;
3275	}
3276	}
3277	}
3278
3279	return 0;
3280	}
3281
3282	/**
3283	* cgroup_apply_control_disable - kill or hide csses according to control
3284	* @cgrp: root of the target subtree
3285	*
3286	* Walk @cgrp's subtree and kill and hide csses so that they match
3287	* cgroup_ss_mask() and cgroup_visible_mask().
3288	*
3289	* A css is hidden when the userland requests it to be disabled while other
3290	* subsystems are still depending on it. The css must not actively control
3291	* resources and be in the vanilla state if it's made visible again later.
3292	* Controllers which may be depended upon should provide ->css_reset() for
3293	* this purpose.
3294	*/
3295	static void cgroup_apply_control_disable(struct cgroup *cgrp)
3296	{
3297	struct cgroup *dsct;
3298	struct cgroup_subsys_state *d_css;
3299	struct cgroup_subsys *ss;
3300	int ssid;
3301
3302	cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) {
3303	for_each_subsys(ss, ssid) {
3304	struct cgroup_subsys_state *css = cgroup_css(dsct, ss);
3305
3306	WARN_ON_ONCE(css && percpu_ref_is_dying(&css->refcnt));
3307
3308	if (!css)
3309	continue;
3310
3311	if (css->parent &&
3312	!(cgroup_ss_mask(dsct) & (1 << ss->id))) {
3313	kill_css(css);
3314	} else if (!css_visible(css)) {
3315	css_clear_dir(css);
3316	if (ss->css_reset)
3317	ss->css_reset(css);
3318	}
3319	}
3320	}
3321	}
3322
3323	/**
3324	* cgroup_apply_control - apply control mask updates to the subtree
3325	* @cgrp: root of the target subtree
3326	*
3327	* subsystems can be enabled and disabled in a subtree using the following
3328	* steps.
3329	*
3330	* 1. Call cgroup_save_control() to stash the current state.
3331	* 2. Update ->subtree_control masks in the subtree as desired.
3332	* 3. Call cgroup_apply_control() to apply the changes.
3333	* 4. Optionally perform other related operations.
3334	* 5. Call cgroup_finalize_control() to finish up.
3335	*
3336	* This function implements step 3 and propagates the mask changes
3337	* throughout @cgrp's subtree, updates csses accordingly and perform
3338	* process migrations.
3339	*/
3340	static int cgroup_apply_control(struct cgroup *cgrp)
3341	{
3342	int ret;
3343
3344	cgroup_propagate_control(cgrp);
3345
3346	ret = cgroup_apply_control_enable(cgrp);
3347	if (ret)
3348	return ret;
3349
3350	/*
3351	* At this point, cgroup_e_css() results reflect the new csses
3352	* making the following cgroup_update_dfl_csses() properly update
3353	* css associations of all tasks in the subtree.
3354	*/
3355	ret = cgroup_update_dfl_csses(cgrp);
3356	if (ret)
3357	return ret;
3358
3359	return 0;
3360	}
3361
3362	/**
3363	* cgroup_finalize_control - finalize control mask update
3364	* @cgrp: root of the target subtree
3365	* @ret: the result of the update
3366	*
3367	* Finalize control mask update. See cgroup_apply_control() for more info.
3368	*/
3369	static void cgroup_finalize_control(struct cgroup *cgrp, int ret)
3370	{
3371	if (ret) {
3372	cgroup_restore_control(cgrp);
3373	cgroup_propagate_control(cgrp);
3374	}
3375
3376	cgroup_apply_control_disable(cgrp);
3377	}
3378
3379	/* change the enabled child controllers for a cgroup in the default hierarchy */
3380	static ssize_t cgroup_subtree_control_write(struct kernfs_open_file *of,
3381	char *buf, size_t nbytes,
3382	loff_t off)
3383	{
3384	u16 enable = 0, disable = 0;
3385	struct cgroup cgrp, child;
3386	struct cgroup_subsys *ss;
3387	char *tok;
3388	int ssid, ret;
3389
3390	/*
3391	* Parse input - space separated list of subsystem names prefixed
3392	* with either + or -.
3393	*/
3394	buf = strstrip(buf);
3395	while ((tok = strsep(&buf, " "))) {
3396	if (tok[0] == '\0')
3397	continue;
3398	do_each_subsys_mask(ss, ssid, ~cgrp_dfl_inhibit_ss_mask) {
3399	if (!cgroup_ssid_enabled(ssid) \|\|
3400	strcmp(tok + 1, ss->name))
3401	continue;
3402
3403	if (*tok == '+') {
3404	enable \|= 1 << ssid;
3405	disable &= ~(1 << ssid);
3406	} else if (*tok == '-') {
3407	disable \|= 1 << ssid;
3408	enable &= ~(1 << ssid);
3409	} else {
3410	return -EINVAL;
3411	}
3412	break;
3413	} while_each_subsys_mask();
3414	if (ssid == CGROUP_SUBSYS_COUNT)
3415	return -EINVAL;
3416	}
3417
3418	cgrp = cgroup_kn_lock_live(of->kn, true);
3419	if (!cgrp)
3420	return -ENODEV;
3421
3422	for_each_subsys(ss, ssid) {
3423	if (enable & (1 << ssid)) {
3424	if (cgrp->subtree_control & (1 << ssid)) {
3425	enable &= ~(1 << ssid);
3426	continue;
3427	}
3428
3429	if (!(cgroup_control(cgrp) & (1 << ssid))) {
3430	ret = -ENOENT;
3431	goto out_unlock;
3432	}
3433	} else if (disable & (1 << ssid)) {
3434	if (!(cgrp->subtree_control & (1 << ssid))) {
3435	disable &= ~(1 << ssid);
3436	continue;
3437	}
3438
3439	/* a child has it enabled? */
3440	cgroup_for_each_live_child(child, cgrp) {
3441	if (child->subtree_control & (1 << ssid)) {
3442	ret = -EBUSY;
3443	goto out_unlock;
3444	}
3445	}
3446	}
3447	}
3448
3449	if (!enable && !disable) {
3450	ret = 0;
3451	goto out_unlock;
3452	}
3453
3454	/*
3455	* Except for the root, subtree_control must be zero for a cgroup
3456	* with tasks so that child cgroups don't compete against tasks.
3457	*/
3458	if (enable && cgroup_parent(cgrp)) {
3459	struct cgrp_cset_link *link;
3460
3461	/*
3462	* Because namespaces pin csets too, @cgrp->cset_links
3463	* might not be empty even when @cgrp is empty. Walk and
3464	* verify each cset.
3465	*/
3466	spin_lock_irq(&css_set_lock);
3467
3468	ret = 0;
3469	list_for_each_entry(link, &cgrp->cset_links, cset_link) {
3470	if (css_set_populated(link->cset)) {
3471	ret = -EBUSY;
3472	break;
3473	}
3474	}
3475
3476	spin_unlock_irq(&css_set_lock);
3477
3478	if (ret)
3479	goto out_unlock;
3480	}
3481
3482	/* save and update control masks and prepare csses */
3483	cgroup_save_control(cgrp);
3484
3485	cgrp->subtree_control \|= enable;
3486	cgrp->subtree_control &= ~disable;
3487
3488	ret = cgroup_apply_control(cgrp);
3489
3490	cgroup_finalize_control(cgrp, ret);
3491
3492	kernfs_activate(cgrp->kn);
3493	ret = 0;
3494	out_unlock:
3495	cgroup_kn_unlock(of->kn);
3496	return ret ?: nbytes;
3497	}
3498
3499	static int cgroup_events_show(struct seq_file seq, void v)
3500	{
3501	seq_printf(seq, "populated %d\n",
3502	cgroup_is_populated(seq_css(seq)->cgroup));
3503	return 0;
3504	}
3505
3506	static ssize_t cgroup_file_write(struct kernfs_open_file of, char buf,
3507	size_t nbytes, loff_t off)
3508	{
3509	struct cgroup *cgrp = of->kn->parent->priv;
3510	struct cftype *cft = of->kn->priv;
3511	struct cgroup_subsys_state *css;
3512	int ret;
3513
3514	if (cft->write)
3515	return cft->write(of, buf, nbytes, off);
3516
3517	/*
3518	* kernfs guarantees that a file isn't deleted with operations in
3519	* flight, which means that the matching css is and stays alive and
3520	* doesn't need to be pinned. The RCU locking is not necessary
3521	* either. It's just for the convenience of using cgroup_css().
3522	*/
3523	rcu_read_lock();
3524	css = cgroup_css(cgrp, cft->ss);
3525	rcu_read_unlock();
3526
3527	if (cft->write_u64) {
3528	unsigned long long v;
3529	ret = kstrtoull(buf, 0, &v);
3530	if (!ret)
3531	ret = cft->write_u64(css, cft, v);
3532	} else if (cft->write_s64) {
3533	long long v;
3534	ret = kstrtoll(buf, 0, &v);
3535	if (!ret)
3536	ret = cft->write_s64(css, cft, v);
3537	} else {
3538	ret = -EINVAL;
3539	}
3540
3541	return ret ?: nbytes;
3542	}
3543
3544	static void cgroup_seqfile_start(struct seq_file seq, loff_t *ppos)
3545	{
3546	return seq_cft(seq)->seq_start(seq, ppos);
3547	}
3548
3549	static void cgroup_seqfile_next(struct seq_file seq, void v, loff_t ppos)
3550	{
3551	return seq_cft(seq)->seq_next(seq, v, ppos);
3552	}
3553
3554	static void cgroup_seqfile_stop(struct seq_file seq, void v)
3555	{
3556	seq_cft(seq)->seq_stop(seq, v);
3557	}
3558
3559	static int cgroup_seqfile_show(struct seq_file m, void arg)
3560	{
3561	struct cftype *cft = seq_cft(m);
3562	struct cgroup_subsys_state *css = seq_css(m);
3563
3564	if (cft->seq_show)
3565	return cft->seq_show(m, arg);
3566
3567	if (cft->read_u64)
3568	seq_printf(m, "%llu\n", cft->read_u64(css, cft));
3569	else if (cft->read_s64)
3570	seq_printf(m, "%lld\n", cft->read_s64(css, cft));
3571	else
3572	return -EINVAL;
3573	return 0;
3574	}
3575
3576	static struct kernfs_ops cgroup_kf_single_ops = {
3577	.atomic_write_len = PAGE_SIZE,
3578	.write = cgroup_file_write,
3579	.seq_show = cgroup_seqfile_show,
3580	};
3581
3582	static struct kernfs_ops cgroup_kf_ops = {
3583	.atomic_write_len = PAGE_SIZE,
3584	.write = cgroup_file_write,
3585	.seq_start = cgroup_seqfile_start,
3586	.seq_next = cgroup_seqfile_next,
3587	.seq_stop = cgroup_seqfile_stop,
3588	.seq_show = cgroup_seqfile_show,
3589	};
3590
3591	/*
3592	* cgroup_rename - Only allow simple rename of directories in place.
3593	*/
3594	static int cgroup_rename(struct kernfs_node kn, struct kernfs_node new_parent,
3595	const char *new_name_str)
3596	{
3597	struct cgroup *cgrp = kn->priv;
3598	int ret;
3599
3600	if (kernfs_type(kn) != KERNFS_DIR)
3601	return -ENOTDIR;
3602	if (kn->parent != new_parent)
3603	return -EIO;
3604
3605	/*
3606	* This isn't a proper migration and its usefulness is very
3607	* limited. Disallow on the default hierarchy.
3608	*/
3609	if (cgroup_on_dfl(cgrp))
3610	return -EPERM;
3611
3612	/*
3613	* We're gonna grab cgroup_mutex which nests outside kernfs
3614	* active_ref. kernfs_rename() doesn't require active_ref
3615	* protection. Break them before grabbing cgroup_mutex.
3616	*/
3617	kernfs_break_active_protection(new_parent);
3618	kernfs_break_active_protection(kn);
3619
3620	mutex_lock(&cgroup_mutex);
3621
3622	ret = kernfs_rename(kn, new_parent, new_name_str);
3623	if (!ret)
3624	trace_cgroup_rename(cgrp);
3625
3626	mutex_unlock(&cgroup_mutex);
3627
3628	kernfs_unbreak_active_protection(kn);
3629	kernfs_unbreak_active_protection(new_parent);
3630	return ret;
3631	}
3632
3633	/* set uid and gid of cgroup dirs and files to that of the creator */
3634	static int cgroup_kn_set_ugid(struct kernfs_node *kn)
3635	{
3636	struct iattr iattr = { .ia_valid = ATTR_UID \| ATTR_GID,
3637	.ia_uid = current_fsuid(),
3638	.ia_gid = current_fsgid(), };
3639
3640	if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) &&
3641	gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID))
3642	return 0;
3643
3644	return kernfs_setattr(kn, &iattr);
3645	}
3646
3647	static int cgroup_add_file(struct cgroup_subsys_state css, struct cgroup cgrp,
3648	struct cftype *cft)
3649	{
3650	char name[CGROUP_FILE_NAME_MAX];
3651	struct kernfs_node *kn;
3652	struct lock_class_key *key = NULL;
3653	int ret;
3654
3655	#ifdef CONFIG_DEBUG_LOCK_ALLOC
3656	key = &cft->lockdep_key;
3657	#endif
3658	kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name),
3659	cgroup_file_mode(cft), 0, cft->kf_ops, cft,
3660	NULL, key);
3661	if (IS_ERR(kn))
3662	return PTR_ERR(kn);
3663
3664	ret = cgroup_kn_set_ugid(kn);
3665	if (ret) {
3666	kernfs_remove(kn);
3667	return ret;
3668	}
3669
3670	if (cft->file_offset) {
3671	struct cgroup_file cfile = (void )css + cft->file_offset;
3672
3673	spin_lock_irq(&cgroup_file_kn_lock);
3674	cfile->kn = kn;
3675	spin_unlock_irq(&cgroup_file_kn_lock);
3676	}
3677
3678	return 0;
3679	}
3680
3681	/**
3682	* cgroup_addrm_files - add or remove files to a cgroup directory
3683	* @css: the target css
3684	* @cgrp: the target cgroup (usually css->cgroup)
3685	* @cfts: array of cftypes to be added
3686	* @is_add: whether to add or remove
3687	*
3688	* Depending on @is_add, add or remove files defined by @cfts on @cgrp.
3689	* For removals, this function never fails.
3690	*/
3691	static int cgroup_addrm_files(struct cgroup_subsys_state *css,
3692	struct cgroup *cgrp, struct cftype cfts[],
3693	bool is_add)
3694	{
3695	struct cftype cft, cft_end = NULL;
3696	int ret = 0;
3697
3698	lockdep_assert_held(&cgroup_mutex);
3699
3700	restart:
3701	for (cft = cfts; cft != cft_end && cft->name[0] != '\0'; cft++) {
3702	/* does cft->flags tell us to skip this file on @cgrp? */
3703	if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp))
3704	continue;
3705	if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp))
3706	continue;
3707	if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp))
3708	continue;
3709	if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp))
3710	continue;
3711
3712	if (is_add) {
3713	ret = cgroup_add_file(css, cgrp, cft);
3714	if (ret) {
3715	pr_warn("%s: failed to add %s, err=%d\n",
3716	__func__, cft->name, ret);
3717	cft_end = cft;
3718	is_add = false;
3719	goto restart;
3720	}
3721	} else {
3722	cgroup_rm_file(cgrp, cft);
3723	}
3724	}
3725	return ret;
3726	}
3727
3728	static int cgroup_apply_cftypes(struct cftype *cfts, bool is_add)
3729	{
3730	LIST_HEAD(pending);
3731	struct cgroup_subsys *ss = cfts[0].ss;
3732	struct cgroup *root = &ss->root->cgrp;
3733	struct cgroup_subsys_state *css;
3734	int ret = 0;
3735
3736	lockdep_assert_held(&cgroup_mutex);
3737
3738	/* add/rm files for all cgroups created before */
3739	css_for_each_descendant_pre(css, cgroup_css(root, ss)) {
3740	struct cgroup *cgrp = css->cgroup;
3741
3742	if (!(css->flags & CSS_VISIBLE))
3743	continue;
3744
3745	ret = cgroup_addrm_files(css, cgrp, cfts, is_add);
3746	if (ret)
3747	break;
3748	}
3749
3750	if (is_add && !ret)
3751	kernfs_activate(root->kn);
3752	return ret;
3753	}
3754
3755	static void cgroup_exit_cftypes(struct cftype *cfts)
3756	{
3757	struct cftype *cft;
3758
3759	for (cft = cfts; cft->name[0] != '\0'; cft++) {
3760	/* free copy for custom atomic_write_len, see init_cftypes() */
3761	if (cft->max_write_len && cft->max_write_len != PAGE_SIZE)
3762	kfree(cft->kf_ops);
3763	cft->kf_ops = NULL;
3764	cft->ss = NULL;
3765
3766	/* revert flags set by cgroup core while adding @cfts */
3767	cft->flags &= ~(__CFTYPE_ONLY_ON_DFL \| __CFTYPE_NOT_ON_DFL);
3768	}
3769	}
3770
3771	static int cgroup_init_cftypes(struct cgroup_subsys ss, struct cftype cfts)
3772	{
3773	struct cftype *cft;
3774
3775	for (cft = cfts; cft->name[0] != '\0'; cft++) {
3776	struct kernfs_ops *kf_ops;
3777
3778	WARN_ON(cft->ss \|\| cft->kf_ops);
3779
3780	if (cft->seq_start)
3781	kf_ops = &cgroup_kf_ops;
3782	else
3783	kf_ops = &cgroup_kf_single_ops;
3784
3785	/*
3786	* Ugh... if @cft wants a custom max_write_len, we need to
3787	* make a copy of kf_ops to set its atomic_write_len.
3788	*/
3789	if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) {
3790	kf_ops = kmemdup(kf_ops, sizeof(*kf_ops), GFP_KERNEL);
3791	if (!kf_ops) {
3792	cgroup_exit_cftypes(cfts);
3793	return -ENOMEM;
3794	}
3795	kf_ops->atomic_write_len = cft->max_write_len;
3796	}
3797
3798	cft->kf_ops = kf_ops;
3799	cft->ss = ss;
3800	}
3801
3802	return 0;
3803	}
3804
3805	static int cgroup_rm_cftypes_locked(struct cftype *cfts)
3806	{
3807	lockdep_assert_held(&cgroup_mutex);
3808
3809	if (!cfts \|\| !cfts[0].ss)
3810	return -ENOENT;
3811
3812	list_del(&cfts->node);
3813	cgroup_apply_cftypes(cfts, false);
3814	cgroup_exit_cftypes(cfts);
3815	return 0;
3816	}
3817
3818	/**
3819	* cgroup_rm_cftypes - remove an array of cftypes from a subsystem
3820	* @cfts: zero-length name terminated array of cftypes
3821	*
3822	* Unregister @cfts. Files described by @cfts are removed from all
3823	* existing cgroups and all future cgroups won't have them either. This
3824	* function can be called anytime whether @cfts' subsys is attached or not.
3825	*
3826	* Returns 0 on successful unregistration, -ENOENT if @cfts is not
3827	* registered.
3828	*/
3829	int cgroup_rm_cftypes(struct cftype *cfts)
3830	{
3831	int ret;
3832
3833	mutex_lock(&cgroup_mutex);
3834	ret = cgroup_rm_cftypes_locked(cfts);
3835	mutex_unlock(&cgroup_mutex);
3836	return ret;
3837	}
3838
3839	/**
3840	* cgroup_add_cftypes - add an array of cftypes to a subsystem
3841	* @ss: target cgroup subsystem
3842	* @cfts: zero-length name terminated array of cftypes
3843	*
3844	* Register @cfts to @ss. Files described by @cfts are created for all
3845	* existing cgroups to which @ss is attached and all future cgroups will
3846	* have them too. This function can be called anytime whether @ss is
3847	* attached or not.
3848	*
3849	* Returns 0 on successful registration, -errno on failure. Note that this
3850	* function currently returns 0 as long as @cfts registration is successful
3851	* even if some file creation attempts on existing cgroups fail.
3852	*/
3853	static int cgroup_add_cftypes(struct cgroup_subsys ss, struct cftype cfts)
3854	{
3855	int ret;
3856
3857	if (!cgroup_ssid_enabled(ss->id))
3858	return 0;
3859
3860	if (!cfts \|\| cfts[0].name[0] == '\0')
3861	return 0;
3862
3863	ret = cgroup_init_cftypes(ss, cfts);
3864	if (ret)
3865	return ret;
3866
3867	mutex_lock(&cgroup_mutex);
3868
3869	list_add_tail(&cfts->node, &ss->cfts);
3870	ret = cgroup_apply_cftypes(cfts, true);
3871	if (ret)
3872	cgroup_rm_cftypes_locked(cfts);
3873
3874	mutex_unlock(&cgroup_mutex);
3875	return ret;
3876	}
3877
3878	/**
3879	* cgroup_add_dfl_cftypes - add an array of cftypes for default hierarchy
3880	* @ss: target cgroup subsystem
3881	* @cfts: zero-length name terminated array of cftypes
3882	*
3883	* Similar to cgroup_add_cftypes() but the added files are only used for
3884	* the default hierarchy.
3885	*/
3886	int cgroup_add_dfl_cftypes(struct cgroup_subsys ss, struct cftype cfts)
3887	{
3888	struct cftype *cft;
3889
3890	for (cft = cfts; cft && cft->name[0] != '\0'; cft++)
3891	cft->flags \|= __CFTYPE_ONLY_ON_DFL;
3892	return cgroup_add_cftypes(ss, cfts);
3893	}
3894
3895	/**
3896	* cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies
3897	* @ss: target cgroup subsystem
3898	* @cfts: zero-length name terminated array of cftypes
3899	*
3900	* Similar to cgroup_add_cftypes() but the added files are only used for
3901	* the legacy hierarchies.
3902	*/
3903	int cgroup_add_legacy_cftypes(struct cgroup_subsys ss, struct cftype cfts)
3904	{
3905	struct cftype *cft;
3906
3907	for (cft = cfts; cft && cft->name[0] != '\0'; cft++)
3908	cft->flags \|= __CFTYPE_NOT_ON_DFL;
3909	return cgroup_add_cftypes(ss, cfts);
3910	}
3911
3912	/**
3913	* cgroup_file_notify - generate a file modified event for a cgroup_file
3914	* @cfile: target cgroup_file
3915	*
3916	* @cfile must have been obtained by setting cftype->file_offset.
3917	*/
3918	void cgroup_file_notify(struct cgroup_file *cfile)
3919	{
3920	unsigned long flags;
3921
3922	spin_lock_irqsave(&cgroup_file_kn_lock, flags);
3923	if (cfile->kn)
3924	kernfs_notify(cfile->kn);
3925	spin_unlock_irqrestore(&cgroup_file_kn_lock, flags);
3926	}
3927
3928	/**
3929	* cgroup_task_count - count the number of tasks in a cgroup.
3930	* @cgrp: the cgroup in question
3931	*
3932	* Return the number of tasks in the cgroup. The returned number can be
3933	* higher than the actual number of tasks due to css_set references from
3934	* namespace roots and temporary usages.
3935	*/
3936	static int cgroup_task_count(const struct cgroup *cgrp)
3937	{
3938	int count = 0;
3939	struct cgrp_cset_link *link;
3940
3941	spin_lock_irq(&css_set_lock);
3942	list_for_each_entry(link, &cgrp->cset_links, cset_link)
3943	count += atomic_read(&link->cset->refcount);
3944	spin_unlock_irq(&css_set_lock);
3945	return count;
3946	}
3947
3948	/**
3949	* css_next_child - find the next child of a given css
3950	* @pos: the current position (%NULL to initiate traversal)
3951	* @parent: css whose children to walk
3952	*
3953	* This function returns the next child of @parent and should be called
3954	* under either cgroup_mutex or RCU read lock. The only requirement is
3955	* that @parent and @pos are accessible. The next sibling is guaranteed to
3956	* be returned regardless of their states.
3957	*
3958	* If a subsystem synchronizes ->css_online() and the start of iteration, a
3959	* css which finished ->css_online() is guaranteed to be visible in the
3960	* future iterations and will stay visible until the last reference is put.
3961	* A css which hasn't finished ->css_online() or already finished
3962	* ->css_offline() may show up during traversal. It's each subsystem's
3963	* responsibility to synchronize against on/offlining.
3964	*/
3965	struct cgroup_subsys_state css_next_child(struct cgroup_subsys_state pos,
3966	struct cgroup_subsys_state *parent)
3967	{
3968	struct cgroup_subsys_state *next;
3969
3970	cgroup_assert_mutex_or_rcu_locked();
3971
3972	/*
3973	* @pos could already have been unlinked from the sibling list.
3974	* Once a cgroup is removed, its ->sibling.next is no longer
3975	* updated when its next sibling changes. CSS_RELEASED is set when
3976	* @pos is taken off list, at which time its next pointer is valid,
3977	* and, as releases are serialized, the one pointed to by the next
3978	* pointer is guaranteed to not have started release yet. This
3979	* implies that if we observe !CSS_RELEASED on @pos in this RCU
3980	* critical section, the one pointed to by its next pointer is
3981	* guaranteed to not have finished its RCU grace period even if we
3982	* have dropped rcu_read_lock() inbetween iterations.
3983	*
3984	* If @pos has CSS_RELEASED set, its next pointer can't be
3985	* dereferenced; however, as each css is given a monotonically
3986	* increasing unique serial number and always appended to the
3987	* sibling list, the next one can be found by walking the parent's
3988	* children until the first css with higher serial number than
3989	* @pos's. While this path can be slower, it happens iff iteration
3990	* races against release and the race window is very small.
3991	*/
3992	if (!pos) {
3993	next = list_entry_rcu(parent->children.next, struct cgroup_subsys_state, sibling);
3994	} else if (likely(!(pos->flags & CSS_RELEASED))) {
3995	next = list_entry_rcu(pos->sibling.next, struct cgroup_subsys_state, sibling);
3996	} else {
3997	list_for_each_entry_rcu(next, &parent->children, sibling)
3998	if (next->serial_nr > pos->serial_nr)
3999	break;
4000	}
4001
4002	/*
4003	* @next, if not pointing to the head, can be dereferenced and is
4004	* the next sibling.
4005	*/
4006	if (&next->sibling != &parent->children)
4007	return next;
4008	return NULL;
4009	}
4010
4011	/**
4012	* css_next_descendant_pre - find the next descendant for pre-order walk
4013	* @pos: the current position (%NULL to initiate traversal)
4014	* @root: css whose descendants to walk
4015	*
4016	* To be used by css_for_each_descendant_pre(). Find the next descendant
4017	* to visit for pre-order traversal of @root's descendants. @root is
4018	* included in the iteration and the first node to be visited.
4019	*
4020	* While this function requires cgroup_mutex or RCU read locking, it
4021	* doesn't require the whole traversal to be contained in a single critical
4022	* section. This function will return the correct next descendant as long
4023	* as both @pos and @root are accessible and @pos is a descendant of @root.
4024	*
4025	* If a subsystem synchronizes ->css_online() and the start of iteration, a
4026	* css which finished ->css_online() is guaranteed to be visible in the
4027	* future iterations and will stay visible until the last reference is put.
4028	* A css which hasn't finished ->css_online() or already finished
4029	* ->css_offline() may show up during traversal. It's each subsystem's
4030	* responsibility to synchronize against on/offlining.
4031	*/
4032	struct cgroup_subsys_state *
4033	css_next_descendant_pre(struct cgroup_subsys_state *pos,
4034	struct cgroup_subsys_state *root)
4035	{
4036	struct cgroup_subsys_state *next;
4037
4038	cgroup_assert_mutex_or_rcu_locked();
4039
4040	/* if first iteration, visit @root */
4041	if (!pos)
4042	return root;
4043
4044	/* visit the first child if exists */
4045	next = css_next_child(NULL, pos);
4046	if (next)
4047	return next;
4048
4049	/* no child, visit my or the closest ancestor's next sibling */
4050	while (pos != root) {
4051	next = css_next_child(pos, pos->parent);
4052	if (next)
4053	return next;
4054	pos = pos->parent;
4055	}
4056
4057	return NULL;
4058	}
4059
4060	/**
4061	* css_rightmost_descendant - return the rightmost descendant of a css
4062	* @pos: css of interest
4063	*
4064	* Return the rightmost descendant of @pos. If there's no descendant, @pos
4065	* is returned. This can be used during pre-order traversal to skip
4066	* subtree of @pos.
4067	*
4068	* While this function requires cgroup_mutex or RCU read locking, it
4069	* doesn't require the whole traversal to be contained in a single critical
4070	* section. This function will return the correct rightmost descendant as
4071	* long as @pos is accessible.
4072	*/
4073	struct cgroup_subsys_state *
4074	css_rightmost_descendant(struct cgroup_subsys_state *pos)
4075	{
4076	struct cgroup_subsys_state last, tmp;
4077
4078	cgroup_assert_mutex_or_rcu_locked();
4079
4080	do {
4081	last = pos;
4082	/* ->prev isn't RCU safe, walk ->next till the end */
4083	pos = NULL;
4084	css_for_each_child(tmp, last)
4085	pos = tmp;
4086	} while (pos);
4087
4088	return last;
4089	}
4090
4091	static struct cgroup_subsys_state *
4092	css_leftmost_descendant(struct cgroup_subsys_state *pos)
4093	{
4094	struct cgroup_subsys_state *last;
4095
4096	do {
4097	last = pos;
4098	pos = css_next_child(NULL, pos);
4099	} while (pos);
4100
4101	return last;
4102	}
4103
4104	/**
4105	* css_next_descendant_post - find the next descendant for post-order walk
4106	* @pos: the current position (%NULL to initiate traversal)
4107	* @root: css whose descendants to walk
4108	*
4109	* To be used by css_for_each_descendant_post(). Find the next descendant
4110	* to visit for post-order traversal of @root's descendants. @root is
4111	* included in the iteration and the last node to be visited.
4112	*
4113	* While this function requires cgroup_mutex or RCU read locking, it
4114	* doesn't require the whole traversal to be contained in a single critical
4115	* section. This function will return the correct next descendant as long
4116	* as both @pos and @cgroup are accessible and @pos is a descendant of
4117	* @cgroup.
4118	*
4119	* If a subsystem synchronizes ->css_online() and the start of iteration, a
4120	* css which finished ->css_online() is guaranteed to be visible in the
4121	* future iterations and will stay visible until the last reference is put.
4122	* A css which hasn't finished ->css_online() or already finished
4123	* ->css_offline() may show up during traversal. It's each subsystem's
4124	* responsibility to synchronize against on/offlining.
4125	*/
4126	struct cgroup_subsys_state *
4127	css_next_descendant_post(struct cgroup_subsys_state *pos,
4128	struct cgroup_subsys_state *root)
4129	{
4130	struct cgroup_subsys_state *next;
4131
4132	cgroup_assert_mutex_or_rcu_locked();
4133
4134	/* if first iteration, visit leftmost descendant which may be @root */
4135	if (!pos)
4136	return css_leftmost_descendant(root);
4137
4138	/* if we visited @root, we're done */
4139	if (pos == root)
4140	return NULL;
4141
4142	/* if there's an unvisited sibling, visit its leftmost descendant */
4143	next = css_next_child(pos, pos->parent);
4144	if (next)
4145	return css_leftmost_descendant(next);
4146
4147	/* no sibling left, visit parent */
4148	return pos->parent;
4149	}
4150
4151	/**
4152	* css_has_online_children - does a css have online children
4153	* @css: the target css
4154	*
4155	* Returns %true if @css has any online children; otherwise, %false. This
4156	* function can be called from any context but the caller is responsible
4157	* for synchronizing against on/offlining as necessary.
4158	*/
4159	bool css_has_online_children(struct cgroup_subsys_state *css)
4160	{
4161	struct cgroup_subsys_state *child;
4162	bool ret = false;
4163
4164	rcu_read_lock();
4165	css_for_each_child(child, css) {
4166	if (child->flags & CSS_ONLINE) {
4167	ret = true;
4168	break;
4169	}
4170	}
4171	rcu_read_unlock();
4172	return ret;
4173	}
4174
4175	/**
4176	* css_task_iter_advance_css_set - advance a task itererator to the next css_set
4177	* @it: the iterator to advance
4178	*
4179	* Advance @it to the next css_set to walk.
4180	*/
4181	static void css_task_iter_advance_css_set(struct css_task_iter *it)
4182	{
4183	struct list_head *l = it->cset_pos;
4184	struct cgrp_cset_link *link;
4185	struct css_set *cset;
4186
4187	lockdep_assert_held(&css_set_lock);
4188
4189	/* Advance to the next non-empty css_set */
4190	do {
4191	l = l->next;
4192	if (l == it->cset_head) {
4193	it->cset_pos = NULL;
4194	it->task_pos = NULL;
4195	return;
4196	}
4197
4198	if (it->ss) {
4199	cset = container_of(l, struct css_set,
4200	e_cset_node[it->ss->id]);
4201	} else {
4202	link = list_entry(l, struct cgrp_cset_link, cset_link);
4203	cset = link->cset;
4204	}
4205	} while (!css_set_populated(cset));
4206
4207	it->cset_pos = l;
4208
4209	if (!list_empty(&cset->tasks))
4210	it->task_pos = cset->tasks.next;
4211	else
4212	it->task_pos = cset->mg_tasks.next;
4213
4214	it->tasks_head = &cset->tasks;
4215	it->mg_tasks_head = &cset->mg_tasks;
4216
4217	/*
4218	* We don't keep css_sets locked across iteration steps and thus
4219	* need to take steps to ensure that iteration can be resumed after
4220	* the lock is re-acquired. Iteration is performed at two levels -
4221	* css_sets and tasks in them.
4222	*
4223	* Once created, a css_set never leaves its cgroup lists, so a
4224	* pinned css_set is guaranteed to stay put and we can resume
4225	* iteration afterwards.
4226	*
4227	* Tasks may leave @cset across iteration steps. This is resolved
4228	* by registering each iterator with the css_set currently being
4229	* walked and making css_set_move_task() advance iterators whose
4230	* next task is leaving.
4231	*/
4232	if (it->cur_cset) {
4233	list_del(&it->iters_node);
4234	put_css_set_locked(it->cur_cset);
4235	}
4236	get_css_set(cset);
4237	it->cur_cset = cset;
4238	list_add(&it->iters_node, &cset->task_iters);
4239	}
4240
4241	static void css_task_iter_advance(struct css_task_iter *it)
4242	{
4243	struct list_head *l = it->task_pos;
4244
4245	lockdep_assert_held(&css_set_lock);
4246	WARN_ON_ONCE(!l);
4247
4248	/*
4249	* Advance iterator to find next entry. cset->tasks is consumed
4250	* first and then ->mg_tasks. After ->mg_tasks, we move onto the
4251	* next cset.
4252	*/
4253	l = l->next;
4254
4255	if (l == it->tasks_head)
4256	l = it->mg_tasks_head->next;
4257
4258	if (l == it->mg_tasks_head)
4259	css_task_iter_advance_css_set(it);
4260	else
4261	it->task_pos = l;
4262	}
4263
4264	/**
4265	* css_task_iter_start - initiate task iteration
4266	* @css: the css to walk tasks of
4267	* @it: the task iterator to use
4268	*
4269	* Initiate iteration through the tasks of @css. The caller can call
4270	* css_task_iter_next() to walk through the tasks until the function
4271	* returns NULL. On completion of iteration, css_task_iter_end() must be
4272	* called.
4273	*/
4274	void css_task_iter_start(struct cgroup_subsys_state *css,
4275	struct css_task_iter *it)
4276	{
4277	/* no one should try to iterate before mounting cgroups */
4278	WARN_ON_ONCE(!use_task_css_set_links);
4279
4280	memset(it, 0, sizeof(*it));
4281
4282	spin_lock_irq(&css_set_lock);
4283
4284	it->ss = css->ss;
4285
4286	if (it->ss)
4287	it->cset_pos = &css->cgroup->e_csets[css->ss->id];
4288	else
4289	it->cset_pos = &css->cgroup->cset_links;
4290
4291	it->cset_head = it->cset_pos;
4292
4293	css_task_iter_advance_css_set(it);
4294
4295	spin_unlock_irq(&css_set_lock);
4296	}
4297
4298	/**
4299	* css_task_iter_next - return the next task for the iterator
4300	* @it: the task iterator being iterated
4301	*
4302	* The "next" function for task iteration. @it should have been
4303	* initialized via css_task_iter_start(). Returns NULL when the iteration
4304	* reaches the end.
4305	*/
4306	struct task_struct css_task_iter_next(struct css_task_iter it)
4307	{
4308	if (it->cur_task) {
4309	put_task_struct(it->cur_task);
4310	it->cur_task = NULL;
4311	}
4312
4313	spin_lock_irq(&css_set_lock);
4314
4315	if (it->task_pos) {
4316	it->cur_task = list_entry(it->task_pos, struct task_struct,
4317	cg_list);
4318	get_task_struct(it->cur_task);
4319	css_task_iter_advance(it);
4320	}
4321
4322	spin_unlock_irq(&css_set_lock);
4323
4324	return it->cur_task;
4325	}
4326
4327	/**
4328	* css_task_iter_end - finish task iteration
4329	* @it: the task iterator to finish
4330	*
4331	* Finish task iteration started by css_task_iter_start().
4332	*/
4333	void css_task_iter_end(struct css_task_iter *it)
4334	{
4335	if (it->cur_cset) {
4336	spin_lock_irq(&css_set_lock);
4337	list_del(&it->iters_node);
4338	put_css_set_locked(it->cur_cset);
4339	spin_unlock_irq(&css_set_lock);
4340	}
4341
4342	if (it->cur_task)
4343	put_task_struct(it->cur_task);
4344	}
4345
4346	/**
4347	* cgroup_trasnsfer_tasks - move tasks from one cgroup to another
4348	* @to: cgroup to which the tasks will be moved
4349	* @from: cgroup in which the tasks currently reside
4350	*
4351	* Locking rules between cgroup_post_fork() and the migration path
4352	* guarantee that, if a task is forking while being migrated, the new child
4353	* is guaranteed to be either visible in the source cgroup after the
4354	* parent's migration is complete or put into the target cgroup. No task
4355	* can slip out of migration through forking.
4356	*/
4357	int cgroup_transfer_tasks(struct cgroup to, struct cgroup from)
4358	{
4359	LIST_HEAD(preloaded_csets);
4360	struct cgrp_cset_link *link;
4361	struct css_task_iter it;
4362	struct task_struct *task;
4363	int ret;
4364
4365	if (!cgroup_may_migrate_to(to))
4366	return -EBUSY;
4367
4368	mutex_lock(&cgroup_mutex);
4369
4370	percpu_down_write(&cgroup_threadgroup_rwsem);
4371
4372	/* all tasks in @from are being moved, all csets are source */
4373	spin_lock_irq(&css_set_lock);
4374	list_for_each_entry(link, &from->cset_links, cset_link)
4375	cgroup_migrate_add_src(link->cset, to, &preloaded_csets);
4376	spin_unlock_irq(&css_set_lock);
4377
4378	ret = cgroup_migrate_prepare_dst(&preloaded_csets);
4379	if (ret)
4380	goto out_err;
4381
4382	/*
4383	* Migrate tasks one-by-one until @from is empty. This fails iff
4384	* ->can_attach() fails.
4385	*/
4386	do {
4387	css_task_iter_start(&from->self, &it);
4388	task = css_task_iter_next(&it);
4389	if (task)
4390	get_task_struct(task);
4391	css_task_iter_end(&it);
4392
4393	if (task) {
4394	ret = cgroup_migrate(task, false, to->root);
4395	if (!ret)
4396	trace_cgroup_transfer_tasks(to, task, false);
4397	put_task_struct(task);
4398	}
4399	} while (task && !ret);
4400	out_err:
4401	cgroup_migrate_finish(&preloaded_csets);
4402	percpu_up_write(&cgroup_threadgroup_rwsem);
4403	mutex_unlock(&cgroup_mutex);
4404	return ret;
4405	}
4406
4407	/*
4408	* Stuff for reading the 'tasks'/'procs' files.
4409	*
4410	* Reading this file can return large amounts of data if a cgroup has
4411	* lots of attached tasks. So it may need several calls to read(),
4412	* but we cannot guarantee that the information we produce is correct
4413	* unless we produce it entirely atomically.
4414	*
4415	*/
4416
4417	/* which pidlist file are we talking about? */
4418	enum cgroup_filetype {
4419	CGROUP_FILE_PROCS,
4420	CGROUP_FILE_TASKS,
4421	};
4422
4423	/*
4424	* A pidlist is a list of pids that virtually represents the contents of one
4425	* of the cgroup files ("procs" or "tasks"). We keep a list of such pidlists,
4426	* a pair (one each for procs, tasks) for each pid namespace that's relevant
4427	* to the cgroup.
4428	*/
4429	struct cgroup_pidlist {
4430	/*
4431	* used to find which pidlist is wanted. doesn't change as long as
4432	* this particular list stays in the list.
4433	*/
4434	struct { enum cgroup_filetype type; struct pid_namespace *ns; } key;
4435	/* array of xids */
4436	pid_t *list;
4437	/* how many elements the above list has */
4438	int length;
4439	/* each of these stored in a list by its cgroup */
4440	struct list_head links;
4441	/* pointer to the cgroup we belong to, for list removal purposes */
4442	struct cgroup *owner;
4443	/* for delayed destruction */
4444	struct delayed_work destroy_dwork;
4445	};
4446
4447	/*
4448	* The following two functions "fix" the issue where there are more pids
4449	* than kmalloc will give memory for; in such cases, we use vmalloc/vfree.
4450	* TODO: replace with a kernel-wide solution to this problem
4451	*/
4452	#define PIDLIST_TOO_LARGE(c) ((c) * sizeof(pid_t) > (PAGE_SIZE * 2))
4453	static void *pidlist_allocate(int count)
4454	{
4455	if (PIDLIST_TOO_LARGE(count))
4456	return vmalloc(count * sizeof(pid_t));
4457	else
4458	return kmalloc(count * sizeof(pid_t), GFP_KERNEL);
4459	}
4460
4461	static void pidlist_free(void *p)
4462	{
4463	kvfree(p);
4464	}
4465
4466	/*
4467	* Used to destroy all pidlists lingering waiting for destroy timer. None
4468	* should be left afterwards.
4469	*/
4470	static void cgroup_pidlist_destroy_all(struct cgroup *cgrp)
4471	{
4472	struct cgroup_pidlist l, tmp_l;
4473
4474	mutex_lock(&cgrp->pidlist_mutex);
4475	list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links)
4476	mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, 0);
4477	mutex_unlock(&cgrp->pidlist_mutex);
4478
4479	flush_workqueue(cgroup_pidlist_destroy_wq);
4480	BUG_ON(!list_empty(&cgrp->pidlists));
4481	}
4482
4483	static void cgroup_pidlist_destroy_work_fn(struct work_struct *work)
4484	{
4485	struct delayed_work *dwork = to_delayed_work(work);
4486	struct cgroup_pidlist *l = container_of(dwork, struct cgroup_pidlist,
4487	destroy_dwork);
4488	struct cgroup_pidlist *tofree = NULL;
4489
4490	mutex_lock(&l->owner->pidlist_mutex);
4491
4492	/*
4493	* Destroy iff we didn't get queued again. The state won't change
4494	* as destroy_dwork can only be queued while locked.
4495	*/
4496	if (!delayed_work_pending(dwork)) {
4497	list_del(&l->links);
4498	pidlist_free(l->list);
4499	put_pid_ns(l->key.ns);
4500	tofree = l;
4501	}
4502
4503	mutex_unlock(&l->owner->pidlist_mutex);
4504	kfree(tofree);
4505	}
4506
4507	/*
4508	* pidlist_uniq - given a kmalloc()ed list, strip out all duplicate entries
4509	* Returns the number of unique elements.
4510	*/
4511	static int pidlist_uniq(pid_t *list, int length)
4512	{
4513	int src, dest = 1;
4514
4515	/*
4516	* we presume the 0th element is unique, so i starts at 1. trivial
4517	* edge cases first; no work needs to be done for either
4518	*/
4519	if (length == 0 \|\| length == 1)
4520	return length;
4521	/* src and dest walk down the list; dest counts unique elements */
4522	for (src = 1; src < length; src++) {
4523	/* find next unique element */
4524	while (list[src] == list[src-1]) {
4525	src++;
4526	if (src == length)
4527	goto after;
4528	}
4529	/* dest always points to where the next unique element goes */
4530	list[dest] = list[src];
4531	dest++;
4532	}
4533	after:
4534	return dest;
4535	}
4536
4537	/*
4538	* The two pid files - task and cgroup.procs - guaranteed that the result
4539	* is sorted, which forced this whole pidlist fiasco. As pid order is
4540	* different per namespace, each namespace needs differently sorted list,
4541	* making it impossible to use, for example, single rbtree of member tasks
4542	* sorted by task pointer. As pidlists can be fairly large, allocating one
4543	* per open file is dangerous, so cgroup had to implement shared pool of
4544	* pidlists keyed by cgroup and namespace.
4545	*
4546	* All this extra complexity was caused by the original implementation
4547	* committing to an entirely unnecessary property. In the long term, we
4548	* want to do away with it. Explicitly scramble sort order if on the
4549	* default hierarchy so that no such expectation exists in the new
4550	* interface.
4551	*
4552	* Scrambling is done by swapping every two consecutive bits, which is
4553	* non-identity one-to-one mapping which disturbs sort order sufficiently.
4554	*/
4555	static pid_t pid_fry(pid_t pid)
4556	{
4557	unsigned a = pid & 0x55555555;
4558	unsigned b = pid & 0xAAAAAAAA;
4559
4560	return (a << 1) \| (b >> 1);
4561	}
4562
4563	static pid_t cgroup_pid_fry(struct cgroup *cgrp, pid_t pid)
4564	{
4565	if (cgroup_on_dfl(cgrp))
4566	return pid_fry(pid);
4567	else
4568	return pid;
4569	}
4570
4571	static int cmppid(const void a, const void b)
4572	{
4573	return (pid_t )a - (pid_t )b;
4574	}
4575
4576	static int fried_cmppid(const void a, const void b)
4577	{
4578	return pid_fry((pid_t )a) - pid_fry((pid_t )b);
4579	}
4580
4581	static struct cgroup_pidlist cgroup_pidlist_find(struct cgroup cgrp,
4582	enum cgroup_filetype type)
4583	{
4584	struct cgroup_pidlist *l;
4585	/* don't need task_nsproxy() if we're looking at ourself */
4586	struct pid_namespace *ns = task_active_pid_ns(current);
4587
4588	lockdep_assert_held(&cgrp->pidlist_mutex);
4589
4590	list_for_each_entry(l, &cgrp->pidlists, links)
4591	if (l->key.type == type && l->key.ns == ns)
4592	return l;
4593	return NULL;
4594	}
4595
4596	/*
4597	* find the appropriate pidlist for our purpose (given procs vs tasks)
4598	* returns with the lock on that pidlist already held, and takes care
4599	* of the use count, or returns NULL with no locks held if we're out of
4600	* memory.
4601	*/
4602	static struct cgroup_pidlist cgroup_pidlist_find_create(struct cgroup cgrp,
4603	enum cgroup_filetype type)
4604	{
4605	struct cgroup_pidlist *l;
4606
4607	lockdep_assert_held(&cgrp->pidlist_mutex);
4608
4609	l = cgroup_pidlist_find(cgrp, type);
4610	if (l)
4611	return l;
4612
4613	/* entry not found; create a new one */
4614	l = kzalloc(sizeof(struct cgroup_pidlist), GFP_KERNEL);
4615	if (!l)
4616	return l;
4617
4618	INIT_DELAYED_WORK(&l->destroy_dwork, cgroup_pidlist_destroy_work_fn);
4619	l->key.type = type;
4620	/* don't need task_nsproxy() if we're looking at ourself */
4621	l->key.ns = get_pid_ns(task_active_pid_ns(current));
4622	l->owner = cgrp;
4623	list_add(&l->links, &cgrp->pidlists);
4624	return l;
4625	}
4626
4627	/*
4628	* Load a cgroup's pidarray with either procs' tgids or tasks' pids
4629	*/
4630	static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type,
4631	struct cgroup_pidlist **lp)
4632	{
4633	pid_t *array;
4634	int length;
4635	int pid, n = 0; /* used for populating the array */
4636	struct css_task_iter it;
4637	struct task_struct *tsk;
4638	struct cgroup_pidlist *l;
4639
4640	lockdep_assert_held(&cgrp->pidlist_mutex);
4641
4642	/*
4643	* If cgroup gets more users after we read count, we won't have
4644	* enough space - tough. This race is indistinguishable to the
4645	* caller from the case that the additional cgroup users didn't
4646	* show up until sometime later on.
4647	*/
4648	length = cgroup_task_count(cgrp);
4649	array = pidlist_allocate(length);
4650	if (!array)
4651	return -ENOMEM;
4652	/* now, populate the array */
4653	css_task_iter_start(&cgrp->self, &it);
4654	while ((tsk = css_task_iter_next(&it))) {
4655	if (unlikely(n == length))
4656	break;
4657	/* get tgid or pid for procs or tasks file respectively */
4658	if (type == CGROUP_FILE_PROCS)
4659	pid = task_tgid_vnr(tsk);
4660	else
4661	pid = task_pid_vnr(tsk);
4662	if (pid > 0) /* make sure to only use valid results */
4663	array[n++] = pid;
4664	}
4665	css_task_iter_end(&it);
4666	length = n;
4667	/* now sort & (if procs) strip out duplicates */
4668	if (cgroup_on_dfl(cgrp))
4669	sort(array, length, sizeof(pid_t), fried_cmppid, NULL);
4670	else
4671	sort(array, length, sizeof(pid_t), cmppid, NULL);
4672	if (type == CGROUP_FILE_PROCS)
4673	length = pidlist_uniq(array, length);
4674
4675	l = cgroup_pidlist_find_create(cgrp, type);
4676	if (!l) {
4677	pidlist_free(array);
4678	return -ENOMEM;
4679	}
4680
4681	/* store array, freeing old if necessary */
4682	pidlist_free(l->list);
4683	l->list = array;
4684	l->length = length;
4685	*lp = l;
4686	return 0;
4687	}
4688
4689	/**
4690	* cgroupstats_build - build and fill cgroupstats
4691	* @stats: cgroupstats to fill information into
4692	* @dentry: A dentry entry belonging to the cgroup for which stats have
4693	* been requested.
4694	*
4695	* Build and fill cgroupstats so that taskstats can export it to user
4696	* space.
4697	*/
4698	int cgroupstats_build(struct cgroupstats stats, struct dentry dentry)
4699	{
4700	struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
4701	struct cgroup *cgrp;
4702	struct css_task_iter it;
4703	struct task_struct *tsk;
4704
4705	/* it should be kernfs_node belonging to cgroupfs and is a directory */
4706	if (dentry->d_sb->s_type != &cgroup_fs_type \|\| !kn \|\|
4707	kernfs_type(kn) != KERNFS_DIR)
4708	return -EINVAL;
4709
4710	mutex_lock(&cgroup_mutex);
4711
4712	/*
4713	* We aren't being called from kernfs and there's no guarantee on
4714	* @kn->priv's validity. For this and css_tryget_online_from_dir(),
4715	* @kn->priv is RCU safe. Let's do the RCU dancing.
4716	*/
4717	rcu_read_lock();
4718	cgrp = rcu_dereference(kn->priv);
4719	if (!cgrp \|\| cgroup_is_dead(cgrp)) {
4720	rcu_read_unlock();
4721	mutex_unlock(&cgroup_mutex);
4722	return -ENOENT;
4723	}
4724	rcu_read_unlock();
4725
4726	css_task_iter_start(&cgrp->self, &it);
4727	while ((tsk = css_task_iter_next(&it))) {
4728	switch (tsk->state) {
4729	case TASK_RUNNING:
4730	stats->nr_running++;
4731	break;
4732	case TASK_INTERRUPTIBLE:
4733	stats->nr_sleeping++;
4734	break;
4735	case TASK_UNINTERRUPTIBLE:
4736	stats->nr_uninterruptible++;
4737	break;
4738	case TASK_STOPPED:
4739	stats->nr_stopped++;
4740	break;
4741	default:
4742	if (delayacct_is_task_waiting_on_io(tsk))
4743	stats->nr_io_wait++;
4744	break;
4745	}
4746	}
4747	css_task_iter_end(&it);
4748
4749	mutex_unlock(&cgroup_mutex);
4750	return 0;
4751	}
4752
4753
4754	/*
4755	* seq_file methods for the tasks/procs files. The seq_file position is the
4756	* next pid to display; the seq_file iterator is a pointer to the pid
4757	* in the cgroup->l->list array.
4758	*/
4759
4760	static void cgroup_pidlist_start(struct seq_file s, loff_t *pos)
4761	{
4762	/*
4763	* Initially we receive a position value that corresponds to
4764	* one more than the last pid shown (or 0 on the first call or
4765	* after a seek to the start). Use a binary-search to find the
4766	* next pid to display, if any
4767	*/
4768	struct kernfs_open_file *of = s->private;
4769	struct cgroup *cgrp = seq_css(s)->cgroup;
4770	struct cgroup_pidlist *l;
4771	enum cgroup_filetype type = seq_cft(s)->private;
4772	int index = 0, pid = *pos;
4773	int *iter, ret;
4774
4775	mutex_lock(&cgrp->pidlist_mutex);
4776
4777	/*
4778	* !NULL @of->priv indicates that this isn't the first start()
4779	* after open. If the matching pidlist is around, we can use that.
4780	* Look for it. Note that @of->priv can't be used directly. It
4781	* could already have been destroyed.
4782	*/
4783	if (of->priv)
4784	of->priv = cgroup_pidlist_find(cgrp, type);
4785
4786	/*
4787	* Either this is the first start() after open or the matching
4788	* pidlist has been destroyed inbetween. Create a new one.
4789	*/
4790	if (!of->priv) {
4791	ret = pidlist_array_load(cgrp, type,
4792	(struct cgroup_pidlist **)&of->priv);
4793	if (ret)
4794	return ERR_PTR(ret);
4795	}
4796	l = of->priv;
4797
4798	if (pid) {
4799	int end = l->length;
4800
4801	while (index < end) {
4802	int mid = (index + end) / 2;
4803	if (cgroup_pid_fry(cgrp, l->list[mid]) == pid) {
4804	index = mid;
4805	break;
4806	} else if (cgroup_pid_fry(cgrp, l->list[mid]) <= pid)
4807	index = mid + 1;
4808	else
4809	end = mid;
4810	}
4811	}
4812	/* If we're off the end of the array, we're done */
4813	if (index >= l->length)
4814	return NULL;
4815	/* Update the abstract position to be the actual pid that we found */
4816	iter = l->list + index;
4817	pos = cgroup_pid_fry(cgrp, iter);
4818	return iter;
4819	}
4820
4821	static void cgroup_pidlist_stop(struct seq_file s, void v)
4822	{
4823	struct kernfs_open_file *of = s->private;
4824	struct cgroup_pidlist *l = of->priv;
4825
4826	if (l)
4827	mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork,
4828	CGROUP_PIDLIST_DESTROY_DELAY);
4829	mutex_unlock(&seq_css(s)->cgroup->pidlist_mutex);
4830	}
4831
4832	static void cgroup_pidlist_next(struct seq_file s, void v, loff_t pos)
4833	{
4834	struct kernfs_open_file *of = s->private;
4835	struct cgroup_pidlist *l = of->priv;
4836	pid_t *p = v;
4837	pid_t *end = l->list + l->length;
4838	/*
4839	* Advance to the next pid in the array. If this goes off the
4840	* end, we're done
4841	*/
4842	p++;
4843	if (p >= end) {
4844	return NULL;
4845	} else {
4846	pos = cgroup_pid_fry(seq_css(s)->cgroup, p);
4847	return p;
4848	}
4849	}
4850
4851	static int cgroup_pidlist_show(struct seq_file s, void v)
4852	{
4853	seq_printf(s, "%d\n", (int )v);
4854
4855	return 0;
4856	}
4857
4858	static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css,
4859	struct cftype *cft)
4860	{
4861	return notify_on_release(css->cgroup);
4862	}
4863
4864	static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css,
4865	struct cftype *cft, u64 val)
4866	{
4867	if (val)
4868	set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
4869	else
4870	clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
4871	return 0;
4872	}
4873
4874	static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css,
4875	struct cftype *cft)
4876	{
4877	return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4878	}
4879
4880	static int cgroup_clone_children_write(struct cgroup_subsys_state *css,
4881	struct cftype *cft, u64 val)
4882	{
4883	if (val)
4884	set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4885	else
4886	clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4887	return 0;
4888	}
4889
4890	/* cgroup core interface files for the default hierarchy */
4891	static struct cftype cgroup_dfl_base_files[] = {
4892	{
4893	.name = "cgroup.procs",
4894	.file_offset = offsetof(struct cgroup, procs_file),
4895	.seq_start = cgroup_pidlist_start,
4896	.seq_next = cgroup_pidlist_next,
4897	.seq_stop = cgroup_pidlist_stop,
4898	.seq_show = cgroup_pidlist_show,
4899	.private = CGROUP_FILE_PROCS,
4900	.write = cgroup_procs_write,
4901	},
4902	{
4903	.name = "cgroup.controllers",
4904	.seq_show = cgroup_controllers_show,
4905	},
4906	{
4907	.name = "cgroup.subtree_control",
4908	.seq_show = cgroup_subtree_control_show,
4909	.write = cgroup_subtree_control_write,
4910	},
4911	{
4912	.name = "cgroup.events",
4913	.flags = CFTYPE_NOT_ON_ROOT,
4914	.file_offset = offsetof(struct cgroup, events_file),
4915	.seq_show = cgroup_events_show,
4916	},
4917	{ } /* terminate */
4918	};
4919
4920	/* cgroup core interface files for the legacy hierarchies */
4921	static struct cftype cgroup_legacy_base_files[] = {
4922	{
4923	.name = "cgroup.procs",
4924	.seq_start = cgroup_pidlist_start,
4925	.seq_next = cgroup_pidlist_next,
4926	.seq_stop = cgroup_pidlist_stop,
4927	.seq_show = cgroup_pidlist_show,
4928	.private = CGROUP_FILE_PROCS,
4929	.write = cgroup_procs_write,
4930	},
4931	{
4932	.name = "cgroup.clone_children",
4933	.read_u64 = cgroup_clone_children_read,
4934	.write_u64 = cgroup_clone_children_write,
4935	},
4936	{
4937	.name = "cgroup.sane_behavior",
4938	.flags = CFTYPE_ONLY_ON_ROOT,
4939	.seq_show = cgroup_sane_behavior_show,
4940	},
4941	{
4942	.name = "tasks",
4943	.seq_start = cgroup_pidlist_start,
4944	.seq_next = cgroup_pidlist_next,
4945	.seq_stop = cgroup_pidlist_stop,
4946	.seq_show = cgroup_pidlist_show,
4947	.private = CGROUP_FILE_TASKS,
4948	.write = cgroup_tasks_write,
4949	},
4950	{
4951	.name = "notify_on_release",
4952	.read_u64 = cgroup_read_notify_on_release,
4953	.write_u64 = cgroup_write_notify_on_release,
4954	},
4955	{
4956	.name = "release_agent",
4957	.flags = CFTYPE_ONLY_ON_ROOT,
4958	.seq_show = cgroup_release_agent_show,
4959	.write = cgroup_release_agent_write,
4960	.max_write_len = PATH_MAX - 1,
4961	},
4962	{ } /* terminate */
4963	};
4964
4965	/*
4966	* css destruction is four-stage process.
4967	*
4968	* 1. Destruction starts. Killing of the percpu_ref is initiated.
4969	* Implemented in kill_css().
4970	*
4971	* 2. When the percpu_ref is confirmed to be visible as killed on all CPUs
4972	* and thus css_tryget_online() is guaranteed to fail, the css can be
4973	* offlined by invoking offline_css(). After offlining, the base ref is
4974	* put. Implemented in css_killed_work_fn().
4975	*
4976	* 3. When the percpu_ref reaches zero, the only possible remaining
4977	* accessors are inside RCU read sections. css_release() schedules the
4978	* RCU callback.
4979	*
4980	* 4. After the grace period, the css can be freed. Implemented in
4981	* css_free_work_fn().
4982	*
4983	* It is actually hairier because both step 2 and 4 require process context
4984	* and thus involve punting to css->destroy_work adding two additional
4985	* steps to the already complex sequence.
4986	*/
4987	static void css_free_work_fn(struct work_struct *work)
4988	{
4989	struct cgroup_subsys_state *css =
4990	container_of(work, struct cgroup_subsys_state, destroy_work);
4991	struct cgroup_subsys *ss = css->ss;
4992	struct cgroup *cgrp = css->cgroup;
4993
4994	percpu_ref_exit(&css->refcnt);
4995
4996	if (ss) {
4997	/* css free path */
4998	struct cgroup_subsys_state *parent = css->parent;
4999	int id = css->id;
5000
5001	ss->css_free(css);
5002	cgroup_idr_remove(&ss->css_idr, id);
5003	cgroup_put(cgrp);
5004
5005	if (parent)
5006	css_put(parent);
5007	} else {
5008	/* cgroup free path */
5009	atomic_dec(&cgrp->root->nr_cgrps);
5010	cgroup_pidlist_destroy_all(cgrp);
5011	cancel_work_sync(&cgrp->release_agent_work);
5012
5013	if (cgroup_parent(cgrp)) {
5014	/*
5015	* We get a ref to the parent, and put the ref when
5016	* this cgroup is being freed, so it's guaranteed
5017	* that the parent won't be destroyed before its
5018	* children.
5019	*/
5020	cgroup_put(cgroup_parent(cgrp));
5021	kernfs_put(cgrp->kn);
5022	kfree(cgrp);
5023	} else {
5024	/*
5025	* This is root cgroup's refcnt reaching zero,
5026	* which indicates that the root should be
5027	* released.
5028	*/
5029	cgroup_destroy_root(cgrp->root);
5030	}
5031	}
5032	}
5033
5034	static void css_free_rcu_fn(struct rcu_head *rcu_head)
5035	{
5036	struct cgroup_subsys_state *css =
5037	container_of(rcu_head, struct cgroup_subsys_state, rcu_head);
5038
5039	INIT_WORK(&css->destroy_work, css_free_work_fn);
5040	queue_work(cgroup_destroy_wq, &css->destroy_work);
5041	}
5042
5043	static void css_release_work_fn(struct work_struct *work)
5044	{
5045	struct cgroup_subsys_state *css =
5046	container_of(work, struct cgroup_subsys_state, destroy_work);
5047	struct cgroup_subsys *ss = css->ss;
5048	struct cgroup *cgrp = css->cgroup;
5049
5050	mutex_lock(&cgroup_mutex);
5051
5052	css->flags \|= CSS_RELEASED;
5053	list_del_rcu(&css->sibling);
5054
5055	if (ss) {
5056	/* css release path */
5057	cgroup_idr_replace(&ss->css_idr, NULL, css->id);
5058	if (ss->css_released)
5059	ss->css_released(css);
5060	} else {
5061	/* cgroup release path */
5062	trace_cgroup_release(cgrp);
5063
5064	cgroup_idr_remove(&cgrp->root->cgroup_idr, cgrp->id);
5065	cgrp->id = -1;
5066
5067	/*
5068	* There are two control paths which try to determine
5069	* cgroup from dentry without going through kernfs -
5070	* cgroupstats_build() and css_tryget_online_from_dir().
5071	* Those are supported by RCU protecting clearing of
5072	* cgrp->kn->priv backpointer.
5073	*/
5074	if (cgrp->kn)
5075	RCU_INIT_POINTER((void __rcu __force *)&cgrp->kn->priv,
5076	NULL);
5077	}
5078
5079	mutex_unlock(&cgroup_mutex);
5080
5081	call_rcu(&css->rcu_head, css_free_rcu_fn);
5082	}
5083
5084	static void css_release(struct percpu_ref *ref)
5085	{
5086	struct cgroup_subsys_state *css =
5087	container_of(ref, struct cgroup_subsys_state, refcnt);
5088
5089	INIT_WORK(&css->destroy_work, css_release_work_fn);
5090	queue_work(cgroup_destroy_wq, &css->destroy_work);
5091	}
5092
5093	static void init_and_link_css(struct cgroup_subsys_state *css,
5094	struct cgroup_subsys ss, struct cgroup cgrp)
5095	{
5096	lockdep_assert_held(&cgroup_mutex);
5097
5098	cgroup_get(cgrp);
5099
5100	memset(css, 0, sizeof(*css));
5101	css->cgroup = cgrp;
5102	css->ss = ss;
5103	css->id = -1;
5104	INIT_LIST_HEAD(&css->sibling);
5105	INIT_LIST_HEAD(&css->children);
5106	css->serial_nr = css_serial_nr_next++;
5107	atomic_set(&css->online_cnt, 0);
5108
5109	if (cgroup_parent(cgrp)) {
5110	css->parent = cgroup_css(cgroup_parent(cgrp), ss);
5111	css_get(css->parent);
5112	}
5113
5114	BUG_ON(cgroup_css(cgrp, ss));
5115	}
5116
5117	/* invoke ->css_online() on a new CSS and mark it online if successful */
5118	static int online_css(struct cgroup_subsys_state *css)
5119	{
5120	struct cgroup_subsys *ss = css->ss;
5121	int ret = 0;
5122
5123	lockdep_assert_held(&cgroup_mutex);
5124
5125	if (ss->css_online)
5126	ret = ss->css_online(css);
5127	if (!ret) {
5128	css->flags \|= CSS_ONLINE;
5129	rcu_assign_pointer(css->cgroup->subsys[ss->id], css);
5130
5131	atomic_inc(&css->online_cnt);
5132	if (css->parent)
5133	atomic_inc(&css->parent->online_cnt);
5134	}
5135	return ret;
5136	}
5137
5138	/* if the CSS is online, invoke ->css_offline() on it and mark it offline */
5139	static void offline_css(struct cgroup_subsys_state *css)
5140	{
5141	struct cgroup_subsys *ss = css->ss;
5142
5143	lockdep_assert_held(&cgroup_mutex);
5144
5145	if (!(css->flags & CSS_ONLINE))
5146	return;
5147
5148	if (ss->css_reset)
5149	ss->css_reset(css);
5150
5151	if (ss->css_offline)
5152	ss->css_offline(css);
5153
5154	css->flags &= ~CSS_ONLINE;
5155	RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL);
5156
5157	wake_up_all(&css->cgroup->offline_waitq);
5158	}
5159
5160	/**
5161	* css_create - create a cgroup_subsys_state
5162	* @cgrp: the cgroup new css will be associated with
5163	* @ss: the subsys of new css
5164	*
5165	* Create a new css associated with @cgrp - @ss pair. On success, the new
5166	* css is online and installed in @cgrp. This function doesn't create the
5167	* interface files. Returns 0 on success, -errno on failure.
5168	*/
5169	static struct cgroup_subsys_state css_create(struct cgroup cgrp,
5170	struct cgroup_subsys *ss)
5171	{
5172	struct cgroup *parent = cgroup_parent(cgrp);
5173	struct cgroup_subsys_state *parent_css = cgroup_css(parent, ss);
5174	struct cgroup_subsys_state *css;
5175	int err;
5176
5177	lockdep_assert_held(&cgroup_mutex);
5178
5179	css = ss->css_alloc(parent_css);
5180	if (!css)
5181	css = ERR_PTR(-ENOMEM);
5182	if (IS_ERR(css))
5183	return css;
5184
5185	init_and_link_css(css, ss, cgrp);
5186
5187	err = percpu_ref_init(&css->refcnt, css_release, 0, GFP_KERNEL);
5188	if (err)
5189	goto err_free_css;
5190
5191	err = cgroup_idr_alloc(&ss->css_idr, NULL, 2, 0, GFP_KERNEL);
5192	if (err < 0)
5193	goto err_free_css;
5194	css->id = err;
5195
5196	/* @css is ready to be brought online now, make it visible */
5197	list_add_tail_rcu(&css->sibling, &parent_css->children);
5198	cgroup_idr_replace(&ss->css_idr, css, css->id);
5199
5200	err = online_css(css);
5201	if (err)
5202	goto err_list_del;
5203
5204	if (ss->broken_hierarchy && !ss->warned_broken_hierarchy &&
5205	cgroup_parent(parent)) {
5206	pr_warn("%s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n",
5207	current->comm, current->pid, ss->name);
5208	if (!strcmp(ss->name, "memory"))
5209	pr_warn("\"memory\" requires setting use_hierarchy to 1 on the root\n");
5210	ss->warned_broken_hierarchy = true;
5211	}
5212
5213	return css;
5214
5215	err_list_del:
5216	list_del_rcu(&css->sibling);
5217	err_free_css:
5218	call_rcu(&css->rcu_head, css_free_rcu_fn);
5219	return ERR_PTR(err);
5220	}
5221
5222	static struct cgroup cgroup_create(struct cgroup parent)
5223	{
5224	struct cgroup_root *root = parent->root;
5225	struct cgroup cgrp, tcgrp;
5226	int level = parent->level + 1;
5227	int ret;
5228
5229	/* allocate the cgroup and its ID, 0 is reserved for the root */
5230	cgrp = kzalloc(sizeof(*cgrp) +
5231	sizeof(cgrp->ancestor_ids[0]) * (level + 1), GFP_KERNEL);
5232	if (!cgrp)
5233	return ERR_PTR(-ENOMEM);
5234
5235	ret = percpu_ref_init(&cgrp->self.refcnt, css_release, 0, GFP_KERNEL);
5236	if (ret)
5237	goto out_free_cgrp;
5238
5239	/*
5240	* Temporarily set the pointer to NULL, so idr_find() won't return
5241	* a half-baked cgroup.
5242	*/
5243	cgrp->id = cgroup_idr_alloc(&root->cgroup_idr, NULL, 2, 0, GFP_KERNEL);
5244	if (cgrp->id < 0) {
5245	ret = -ENOMEM;
5246	goto out_cancel_ref;
5247	}
5248
5249	init_cgroup_housekeeping(cgrp);
5250
5251	cgrp->self.parent = &parent->self;
5252	cgrp->root = root;
5253	cgrp->level = level;
5254
5255	for (tcgrp = cgrp; tcgrp; tcgrp = cgroup_parent(tcgrp))
5256	cgrp->ancestor_ids[tcgrp->level] = tcgrp->id;
5257
5258	if (notify_on_release(parent))
5259	set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
5260
5261	if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags))
5262	set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
5263
5264	cgrp->self.serial_nr = css_serial_nr_next++;
5265
5266	/* allocation complete, commit to creation */
5267	list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children);
5268	atomic_inc(&root->nr_cgrps);
5269	cgroup_get(parent);
5270
5271	/*
5272	* @cgrp is now fully operational. If something fails after this
5273	* point, it'll be released via the normal destruction path.
5274	*/
5275	cgroup_idr_replace(&root->cgroup_idr, cgrp, cgrp->id);
5276
5277	/*
5278	* On the default hierarchy, a child doesn't automatically inherit
5279	* subtree_control from the parent. Each is configured manually.
5280	*/
5281	if (!cgroup_on_dfl(cgrp))
5282	cgrp->subtree_control = cgroup_control(cgrp);
5283
5284	cgroup_propagate_control(cgrp);
5285
5286	/* @cgrp doesn't have dir yet so the following will only create csses */
5287	ret = cgroup_apply_control_enable(cgrp);
5288	if (ret)
5289	goto out_destroy;
5290
5291	return cgrp;
5292
5293	out_cancel_ref:
5294	percpu_ref_exit(&cgrp->self.refcnt);
5295	out_free_cgrp:
5296	kfree(cgrp);
5297	return ERR_PTR(ret);
5298	out_destroy:
5299	cgroup_destroy_locked(cgrp);
5300	return ERR_PTR(ret);
5301	}
5302
5303	static int cgroup_mkdir(struct kernfs_node parent_kn, const char name,
5304	umode_t mode)
5305	{
5306	struct cgroup parent, cgrp;
5307	struct kernfs_node *kn;
5308	int ret;
5309
5310	/* do not accept '\n' to prevent making /proc/<pid>/cgroup unparsable */
5311	if (strchr(name, '\n'))
5312	return -EINVAL;
5313
5314	parent = cgroup_kn_lock_live(parent_kn, false);
5315	if (!parent)
5316	return -ENODEV;
5317
5318	cgrp = cgroup_create(parent);
5319	if (IS_ERR(cgrp)) {
5320	ret = PTR_ERR(cgrp);
5321	goto out_unlock;
5322	}
5323
5324	/* create the directory */
5325	kn = kernfs_create_dir(parent->kn, name, mode, cgrp);
5326	if (IS_ERR(kn)) {
5327	ret = PTR_ERR(kn);
5328	goto out_destroy;
5329	}
5330	cgrp->kn = kn;
5331
5332	/*
5333	* This extra ref will be put in cgroup_free_fn() and guarantees
5334	* that @cgrp->kn is always accessible.
5335	*/
5336	kernfs_get(kn);
5337
5338	ret = cgroup_kn_set_ugid(kn);
5339	if (ret)
5340	goto out_destroy;
5341
5342	ret = css_populate_dir(&cgrp->self);
5343	if (ret)
5344	goto out_destroy;
5345
5346	ret = cgroup_apply_control_enable(cgrp);
5347	if (ret)
5348	goto out_destroy;
5349
5350	trace_cgroup_mkdir(cgrp);
5351
5352	/* let's create and online css's */
5353	kernfs_activate(kn);
5354
5355	ret = 0;
5356	goto out_unlock;
5357
5358	out_destroy:
5359	cgroup_destroy_locked(cgrp);
5360	out_unlock:
5361	cgroup_kn_unlock(parent_kn);
5362	return ret;
5363	}
5364
5365	/*
5366	* This is called when the refcnt of a css is confirmed to be killed.
5367	* css_tryget_online() is now guaranteed to fail. Tell the subsystem to
5368	* initate destruction and put the css ref from kill_css().
5369	*/
5370	static void css_killed_work_fn(struct work_struct *work)
5371	{
5372	struct cgroup_subsys_state *css =
5373	container_of(work, struct cgroup_subsys_state, destroy_work);
5374
5375	mutex_lock(&cgroup_mutex);
5376
5377	do {
5378	offline_css(css);
5379	css_put(css);
5380	/* @css can't go away while we're holding cgroup_mutex */
5381	css = css->parent;
5382	} while (css && atomic_dec_and_test(&css->online_cnt));
5383
5384	mutex_unlock(&cgroup_mutex);
5385	}
5386
5387	/* css kill confirmation processing requires process context, bounce */
5388	static void css_killed_ref_fn(struct percpu_ref *ref)
5389	{
5390	struct cgroup_subsys_state *css =
5391	container_of(ref, struct cgroup_subsys_state, refcnt);
5392
5393	if (atomic_dec_and_test(&css->online_cnt)) {
5394	INIT_WORK(&css->destroy_work, css_killed_work_fn);
5395	queue_work(cgroup_destroy_wq, &css->destroy_work);
5396	}
5397	}
5398
5399	/**
5400	* kill_css - destroy a css
5401	* @css: css to destroy
5402	*
5403	* This function initiates destruction of @css by removing cgroup interface
5404	* files and putting its base reference. ->css_offline() will be invoked
5405	* asynchronously once css_tryget_online() is guaranteed to fail and when
5406	* the reference count reaches zero, @css will be released.
5407	*/
5408	static void kill_css(struct cgroup_subsys_state *css)
5409	{
5410	lockdep_assert_held(&cgroup_mutex);
5411
5412	/*
5413	* This must happen before css is disassociated with its cgroup.
5414	* See seq_css() for details.
5415	*/
5416	css_clear_dir(css);
5417
5418	/*
5419	* Killing would put the base ref, but we need to keep it alive
5420	* until after ->css_offline().
5421	*/
5422	css_get(css);
5423
5424	/*
5425	* cgroup core guarantees that, by the time ->css_offline() is
5426	* invoked, no new css reference will be given out via
5427	* css_tryget_online(). We can't simply call percpu_ref_kill() and
5428	* proceed to offlining css's because percpu_ref_kill() doesn't
5429	* guarantee that the ref is seen as killed on all CPUs on return.
5430	*
5431	* Use percpu_ref_kill_and_confirm() to get notifications as each
5432	* css is confirmed to be seen as killed on all CPUs.
5433	*/
5434	percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);
5435	}
5436
5437	/**
5438	* cgroup_destroy_locked - the first stage of cgroup destruction
5439	* @cgrp: cgroup to be destroyed
5440	*
5441	* css's make use of percpu refcnts whose killing latency shouldn't be
5442	* exposed to userland and are RCU protected. Also, cgroup core needs to
5443	* guarantee that css_tryget_online() won't succeed by the time
5444	* ->css_offline() is invoked. To satisfy all the requirements,
5445	* destruction is implemented in the following two steps.
5446	*
5447	* s1. Verify @cgrp can be destroyed and mark it dying. Remove all
5448	* userland visible parts and start killing the percpu refcnts of
5449	* css's. Set up so that the next stage will be kicked off once all
5450	* the percpu refcnts are confirmed to be killed.
5451	*
5452	* s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the
5453	* rest of destruction. Once all cgroup references are gone, the
5454	* cgroup is RCU-freed.
5455	*
5456	* This function implements s1. After this step, @cgrp is gone as far as
5457	* the userland is concerned and a new cgroup with the same name may be
5458	* created. As cgroup doesn't care about the names internally, this
5459	* doesn't cause any problem.
5460	*/
5461	static int cgroup_destroy_locked(struct cgroup *cgrp)
5462	__releases(&cgroup_mutex) __acquires(&cgroup_mutex)
5463	{
5464	struct cgroup_subsys_state *css;
5465	struct cgrp_cset_link *link;
5466	int ssid;
5467
5468	lockdep_assert_held(&cgroup_mutex);
5469
5470	/*
5471	* Only migration can raise populated from zero and we're already
5472	* holding cgroup_mutex.
5473	*/
5474	if (cgroup_is_populated(cgrp))
5475	return -EBUSY;
5476
5477	/*
5478	* Make sure there's no live children. We can't test emptiness of
5479	* ->self.children as dead children linger on it while being
5480	* drained; otherwise, "rmdir parent/child parent" may fail.
5481	*/
5482	if (css_has_online_children(&cgrp->self))
5483	return -EBUSY;
5484
5485	/*
5486	* Mark @cgrp and the associated csets dead. The former prevents
5487	* further task migration and child creation by disabling
5488	* cgroup_lock_live_group(). The latter makes the csets ignored by
5489	* the migration path.
5490	*/
5491	cgrp->self.flags &= ~CSS_ONLINE;
5492
5493	spin_lock_irq(&css_set_lock);
5494	list_for_each_entry(link, &cgrp->cset_links, cset_link)
5495	link->cset->dead = true;
5496	spin_unlock_irq(&css_set_lock);
5497
5498	/* initiate massacre of all css's */
5499	for_each_css(css, ssid, cgrp)
5500	kill_css(css);
5501
5502	/*
5503	* Remove @cgrp directory along with the base files. @cgrp has an
5504	* extra ref on its kn.
5505	*/
5506	kernfs_remove(cgrp->kn);
5507
5508	check_for_release(cgroup_parent(cgrp));
5509
5510	/* put the base reference */
5511	percpu_ref_kill(&cgrp->self.refcnt);
5512
5513	return 0;
5514	};
5515
5516	static int cgroup_rmdir(struct kernfs_node *kn)
5517	{
5518	struct cgroup *cgrp;
5519	int ret = 0;
5520
5521	cgrp = cgroup_kn_lock_live(kn, false);
5522	if (!cgrp)
5523	return 0;
5524
5525	ret = cgroup_destroy_locked(cgrp);
5526
5527	if (!ret)
5528	trace_cgroup_rmdir(cgrp);
5529
5530	cgroup_kn_unlock(kn);
5531	return ret;
5532	}
5533
5534	static struct kernfs_syscall_ops cgroup_kf_syscall_ops = {
5535	.remount_fs = cgroup_remount,
5536	.show_options = cgroup_show_options,
5537	.mkdir = cgroup_mkdir,
5538	.rmdir = cgroup_rmdir,
5539	.rename = cgroup_rename,
5540	.show_path = cgroup_show_path,
5541	};
5542
5543	static void __init cgroup_init_subsys(struct cgroup_subsys *ss, bool early)
5544	{
5545	struct cgroup_subsys_state *css;
5546
5547	pr_debug("Initializing cgroup subsys %s\n", ss->name);
5548
5549	mutex_lock(&cgroup_mutex);
5550
5551	idr_init(&ss->css_idr);
5552	INIT_LIST_HEAD(&ss->cfts);
5553
5554	/* Create the root cgroup state for this subsystem */
5555	ss->root = &cgrp_dfl_root;
5556	css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss));
5557	/* We don't handle early failures gracefully */
5558	BUG_ON(IS_ERR(css));
5559	init_and_link_css(css, ss, &cgrp_dfl_root.cgrp);
5560
5561	/*
5562	* Root csses are never destroyed and we can't initialize
5563	* percpu_ref during early init. Disable refcnting.
5564	*/
5565	css->flags \|= CSS_NO_REF;
5566
5567	if (early) {
5568	/* allocation can't be done safely during early init */
5569	css->id = 1;
5570	} else {
5571	css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL);
5572	BUG_ON(css->id < 0);
5573	}
5574
5575	/* Update the init_css_set to contain a subsys
5576	* pointer to this state - since the subsystem is
5577	* newly registered, all tasks and hence the
5578	* init_css_set is in the subsystem's root cgroup. */
5579	init_css_set.subsys[ss->id] = css;
5580
5581	have_fork_callback \|= (bool)ss->fork << ss->id;
5582	have_exit_callback \|= (bool)ss->exit << ss->id;
5583	have_free_callback \|= (bool)ss->free << ss->id;
5584	have_canfork_callback \|= (bool)ss->can_fork << ss->id;
5585
5586	/* At system boot, before all subsystems have been
5587	* registered, no tasks have been forked, so we don't
5588	* need to invoke fork callbacks here. */
5589	BUG_ON(!list_empty(&init_task.tasks));
5590
5591	BUG_ON(online_css(css));
5592
5593	mutex_unlock(&cgroup_mutex);
5594	}
5595
5596	/**
5597	* cgroup_init_early - cgroup initialization at system boot
5598	*
5599	* Initialize cgroups at system boot, and initialize any
5600	* subsystems that request early init.
5601	*/
5602	int __init cgroup_init_early(void)
5603	{
5604	static struct cgroup_sb_opts __initdata opts;
5605	struct cgroup_subsys *ss;
5606	int i;
5607
5608	init_cgroup_root(&cgrp_dfl_root, &opts);
5609	cgrp_dfl_root.cgrp.self.flags \|= CSS_NO_REF;
5610
5611	RCU_INIT_POINTER(init_task.cgroups, &init_css_set);
5612
5613	for_each_subsys(ss, i) {
5614	WARN(!ss->css_alloc \|\| !ss->css_free \|\| ss->name \|\| ss->id,
5615	"invalid cgroup_subsys %d:%s css_alloc=%p css_free=%p id:name=%d:%s\n",
5616	i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free,
5617	ss->id, ss->name);
5618	WARN(strlen(cgroup_subsys_name[i]) > MAX_CGROUP_TYPE_NAMELEN,
5619	"cgroup_subsys_name %s too long\n", cgroup_subsys_name[i]);
5620
5621	ss->id = i;
5622	ss->name = cgroup_subsys_name[i];
5623	if (!ss->legacy_name)
5624	ss->legacy_name = cgroup_subsys_name[i];
5625
5626	if (ss->early_init)
5627	cgroup_init_subsys(ss, true);
5628	}
5629	return 0;
5630	}
5631
5632	static u16 cgroup_disable_mask __initdata;
5633
5634	/**
5635	* cgroup_init - cgroup initialization
5636	*
5637	* Register cgroup filesystem and /proc file, and initialize
5638	* any subsystems that didn't request early init.
5639	*/
5640	int __init cgroup_init(void)
5641	{
5642	struct cgroup_subsys *ss;
5643	int ssid;
5644
5645	BUILD_BUG_ON(CGROUP_SUBSYS_COUNT > 16);
5646	BUG_ON(percpu_init_rwsem(&cgroup_threadgroup_rwsem));
5647	BUG_ON(cgroup_init_cftypes(NULL, cgroup_dfl_base_files));
5648	BUG_ON(cgroup_init_cftypes(NULL, cgroup_legacy_base_files));
5649
5650	/*
5651	* The latency of the synchronize_sched() is too high for cgroups,
5652	* avoid it at the cost of forcing all readers into the slow path.
5653	*/
5654	rcu_sync_enter_start(&cgroup_threadgroup_rwsem.rss);
5655
5656	get_user_ns(init_cgroup_ns.user_ns);
5657
5658	mutex_lock(&cgroup_mutex);
5659
5660	/*
5661	* Add init_css_set to the hash table so that dfl_root can link to
5662	* it during init.
5663	*/
5664	hash_add(css_set_table, &init_css_set.hlist,
5665	css_set_hash(init_css_set.subsys));
5666
5667	BUG_ON(cgroup_setup_root(&cgrp_dfl_root, 0));
5668
5669	mutex_unlock(&cgroup_mutex);
5670
5671	for_each_subsys(ss, ssid) {
5672	if (ss->early_init) {
5673	struct cgroup_subsys_state *css =
5674	init_css_set.subsys[ss->id];
5675
5676	css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2,
5677	GFP_KERNEL);
5678	BUG_ON(css->id < 0);
5679	} else {
5680	cgroup_init_subsys(ss, false);
5681	}
5682
5683	list_add_tail(&init_css_set.e_cset_node[ssid],
5684	&cgrp_dfl_root.cgrp.e_csets[ssid]);
5685
5686	/*
5687	* Setting dfl_root subsys_mask needs to consider the
5688	* disabled flag and cftype registration needs kmalloc,
5689	* both of which aren't available during early_init.
5690	*/
5691	if (cgroup_disable_mask & (1 << ssid)) {
5692	static_branch_disable(cgroup_subsys_enabled_key[ssid]);
5693	printk(KERN_INFO "Disabling %s control group subsystem\n",
5694	ss->name);
5695	continue;
5696	}
5697
5698	if (cgroup_ssid_no_v1(ssid))
5699	printk(KERN_INFO "Disabling %s control group subsystem in v1 mounts\n",
5700	ss->name);
5701
5702	cgrp_dfl_root.subsys_mask \|= 1 << ss->id;
5703
5704	if (ss->implicit_on_dfl)
5705	cgrp_dfl_implicit_ss_mask \|= 1 << ss->id;
5706	else if (!ss->dfl_cftypes)
5707	cgrp_dfl_inhibit_ss_mask \|= 1 << ss->id;
5708
5709	if (ss->dfl_cftypes == ss->legacy_cftypes) {
5710	WARN_ON(cgroup_add_cftypes(ss, ss->dfl_cftypes));
5711	} else {
5712	WARN_ON(cgroup_add_dfl_cftypes(ss, ss->dfl_cftypes));
5713	WARN_ON(cgroup_add_legacy_cftypes(ss, ss->legacy_cftypes));
5714	}
5715
5716	if (ss->bind)
5717	ss->bind(init_css_set.subsys[ssid]);
5718	}
5719
5720	/* init_css_set.subsys[] has been updated, re-hash */
5721	hash_del(&init_css_set.hlist);
5722	hash_add(css_set_table, &init_css_set.hlist,
5723	css_set_hash(init_css_set.subsys));
5724
5725	WARN_ON(sysfs_create_mount_point(fs_kobj, "cgroup"));
5726	WARN_ON(register_filesystem(&cgroup_fs_type));
5727	WARN_ON(register_filesystem(&cgroup2_fs_type));
5728	WARN_ON(!proc_create("cgroups", 0, NULL, &proc_cgroupstats_operations));
5729
5730	return 0;
5731	}
5732
5733	static int __init cgroup_wq_init(void)
5734	{
5735	/*
5736	* There isn't much point in executing destruction path in
5737	* parallel. Good chunk is serialized with cgroup_mutex anyway.
5738	* Use 1 for @max_active.
5739	*
5740	* We would prefer to do this in cgroup_init() above, but that
5741	* is called before init_workqueues(): so leave this until after.
5742	*/
5743	cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1);
5744	BUG_ON(!cgroup_destroy_wq);
5745
5746	/*
5747	* Used to destroy pidlists and separate to serve as flush domain.
5748	* Cap @max_active to 1 too.
5749	*/
5750	cgroup_pidlist_destroy_wq = alloc_workqueue("cgroup_pidlist_destroy",
5751	0, 1);
5752	BUG_ON(!cgroup_pidlist_destroy_wq);
5753
5754	return 0;
5755	}
5756	core_initcall(cgroup_wq_init);
5757
5758	/*
5759	* proc_cgroup_show()
5760	* - Print task's cgroup paths into seq_file, one line for each hierarchy
5761	* - Used for /proc/<pid>/cgroup.
5762	*/
5763	int proc_cgroup_show(struct seq_file m, struct pid_namespace ns,
5764	struct pid pid, struct task_struct tsk)
5765	{
5766	char *buf;
5767	int retval;
5768	struct cgroup_root *root;
5769
5770	retval = -ENOMEM;
5771	buf = kmalloc(PATH_MAX, GFP_KERNEL);
5772	if (!buf)
5773	goto out;
5774
5775	mutex_lock(&cgroup_mutex);
5776	spin_lock_irq(&css_set_lock);
5777
5778	for_each_root(root) {
5779	struct cgroup_subsys *ss;
5780	struct cgroup *cgrp;
5781	int ssid, count = 0;
5782
5783	if (root == &cgrp_dfl_root && !cgrp_dfl_visible)
5784	continue;
5785
5786	seq_printf(m, "%d:", root->hierarchy_id);
5787	if (root != &cgrp_dfl_root)
5788	for_each_subsys(ss, ssid)
5789	if (root->subsys_mask & (1 << ssid))
5790	seq_printf(m, "%s%s", count++ ? "," : "",
5791	ss->legacy_name);
5792	if (strlen(root->name))
5793	seq_printf(m, "%sname=%s", count ? "," : "",
5794	root->name);
5795	seq_putc(m, ':');
5796
5797	cgrp = task_cgroup_from_root(tsk, root);
5798
5799	/*
5800	* On traditional hierarchies, all zombie tasks show up as
5801	* belonging to the root cgroup. On the default hierarchy,
5802	* while a zombie doesn't show up in "cgroup.procs" and
5803	* thus can't be migrated, its /proc/PID/cgroup keeps
5804	* reporting the cgroup it belonged to before exiting. If
5805	* the cgroup is removed before the zombie is reaped,
5806	* " (deleted)" is appended to the cgroup path.
5807	*/
5808	if (cgroup_on_dfl(cgrp) \|\| !(tsk->flags & PF_EXITING)) {
5809	retval = cgroup_path_ns_locked(cgrp, buf, PATH_MAX,
5810	current->nsproxy->cgroup_ns);
5811	if (retval >= PATH_MAX)
5812	retval = -ENAMETOOLONG;
5813	if (retval < 0)
5814	goto out_unlock;
5815
5816	seq_puts(m, buf);
5817	} else {
5818	seq_puts(m, "/");
5819	}
5820
5821	if (cgroup_on_dfl(cgrp) && cgroup_is_dead(cgrp))
5822	seq_puts(m, " (deleted)\n");
5823	else
5824	seq_putc(m, '\n');
5825	}
5826
5827	retval = 0;
5828	out_unlock:
5829	spin_unlock_irq(&css_set_lock);
5830	mutex_unlock(&cgroup_mutex);
5831	kfree(buf);
5832	out:
5833	return retval;
5834	}
5835
5836	/* Display information about each subsystem and each hierarchy */
5837	static int proc_cgroupstats_show(struct seq_file m, void v)
5838	{
5839	struct cgroup_subsys *ss;
5840	int i;
5841
5842	seq_puts(m, "#subsys_name\thierarchy\tnum_cgroups\tenabled\n");
5843	/*
5844	* ideally we don't want subsystems moving around while we do this.
5845	* cgroup_mutex is also necessary to guarantee an atomic snapshot of
5846	* subsys/hierarchy state.
5847	*/
5848	mutex_lock(&cgroup_mutex);
5849
5850	for_each_subsys(ss, i)
5851	seq_printf(m, "%s\t%d\t%d\t%d\n",
5852	ss->legacy_name, ss->root->hierarchy_id,
5853	atomic_read(&ss->root->nr_cgrps),
5854	cgroup_ssid_enabled(i));
5855
5856	mutex_unlock(&cgroup_mutex);
5857	return 0;
5858	}
5859
5860	static int cgroupstats_open(struct inode inode, struct file file)
5861	{
5862	return single_open(file, proc_cgroupstats_show, NULL);
5863	}
5864
5865	static const struct file_operations proc_cgroupstats_operations = {
5866	.open = cgroupstats_open,
5867	.read = seq_read,
5868	.llseek = seq_lseek,
5869	.release = single_release,
5870	};
5871
5872	/**
5873	* cgroup_fork - initialize cgroup related fields during copy_process()
5874	* @child: pointer to task_struct of forking parent process.
5875	*
5876	* A task is associated with the init_css_set until cgroup_post_fork()
5877	* attaches it to the parent's css_set. Empty cg_list indicates that
5878	* @child isn't holding reference to its css_set.
5879	*/
5880	void cgroup_fork(struct task_struct *child)
5881	{
5882	RCU_INIT_POINTER(child->cgroups, &init_css_set);
5883	INIT_LIST_HEAD(&child->cg_list);
5884	}
5885
5886	/**
5887	* cgroup_can_fork - called on a new task before the process is exposed
5888	* @child: the task in question.
5889	*
5890	* This calls the subsystem can_fork() callbacks. If the can_fork() callback
5891	* returns an error, the fork aborts with that error code. This allows for
5892	* a cgroup subsystem to conditionally allow or deny new forks.
5893	*/
5894	int cgroup_can_fork(struct task_struct *child)
5895	{
5896	struct cgroup_subsys *ss;
5897	int i, j, ret;
5898
5899	do_each_subsys_mask(ss, i, have_canfork_callback) {
5900	ret = ss->can_fork(child);
5901	if (ret)
5902	goto out_revert;
5903	} while_each_subsys_mask();
5904
5905	return 0;
5906
5907	out_revert:
5908	for_each_subsys(ss, j) {
5909	if (j >= i)
5910	break;
5911	if (ss->cancel_fork)
5912	ss->cancel_fork(child);
5913	}
5914
5915	return ret;
5916	}
5917
5918	/**
5919	* cgroup_cancel_fork - called if a fork failed after cgroup_can_fork()
5920	* @child: the task in question
5921	*
5922	* This calls the cancel_fork() callbacks if a fork failed after
5923	* cgroup_can_fork() succeded.
5924	*/
5925	void cgroup_cancel_fork(struct task_struct *child)
5926	{
5927	struct cgroup_subsys *ss;
5928	int i;
5929
5930	for_each_subsys(ss, i)
5931	if (ss->cancel_fork)
5932	ss->cancel_fork(child);
5933	}
5934
5935	/**
5936	* cgroup_post_fork - called on a new task after adding it to the task list
5937	* @child: the task in question
5938	*
5939	* Adds the task to the list running through its css_set if necessary and
5940	* call the subsystem fork() callbacks. Has to be after the task is
5941	* visible on the task list in case we race with the first call to
5942	* cgroup_task_iter_start() - to guarantee that the new task ends up on its
5943	* list.
5944	*/
5945	void cgroup_post_fork(struct task_struct *child)
5946	{
5947	struct cgroup_subsys *ss;
5948	int i;
5949
5950	/*
5951	* This may race against cgroup_enable_task_cg_lists(). As that
5952	* function sets use_task_css_set_links before grabbing
5953	* tasklist_lock and we just went through tasklist_lock to add
5954	* @child, it's guaranteed that either we see the set
5955	* use_task_css_set_links or cgroup_enable_task_cg_lists() sees
5956	* @child during its iteration.
5957	*
5958	* If we won the race, @child is associated with %current's
5959	* css_set. Grabbing css_set_lock guarantees both that the
5960	* association is stable, and, on completion of the parent's
5961	* migration, @child is visible in the source of migration or
5962	* already in the destination cgroup. This guarantee is necessary
5963	* when implementing operations which need to migrate all tasks of
5964	* a cgroup to another.
5965	*
5966	* Note that if we lose to cgroup_enable_task_cg_lists(), @child
5967	* will remain in init_css_set. This is safe because all tasks are
5968	* in the init_css_set before cg_links is enabled and there's no
5969	* operation which transfers all tasks out of init_css_set.
5970	*/
5971	if (use_task_css_set_links) {
5972	struct css_set *cset;
5973
5974	spin_lock_irq(&css_set_lock);
5975	cset = task_css_set(current);
5976	if (list_empty(&child->cg_list)) {
5977	get_css_set(cset);
5978	css_set_move_task(child, NULL, cset, false);
5979	}
5980	spin_unlock_irq(&css_set_lock);
5981	}
5982
5983	/*
5984	* Call ss->fork(). This must happen after @child is linked on
5985	* css_set; otherwise, @child might change state between ->fork()
5986	* and addition to css_set.
5987	*/
5988	do_each_subsys_mask(ss, i, have_fork_callback) {
5989	ss->fork(child);
5990	} while_each_subsys_mask();
5991	}
5992
5993	/**
5994	* cgroup_exit - detach cgroup from exiting task
5995	* @tsk: pointer to task_struct of exiting process
5996	*
5997	* Description: Detach cgroup from @tsk and release it.
5998	*
5999	* Note that cgroups marked notify_on_release force every task in
6000	* them to take the global cgroup_mutex mutex when exiting.
6001	* This could impact scaling on very large systems. Be reluctant to
6002	* use notify_on_release cgroups where very high task exit scaling
6003	* is required on large systems.
6004	*
6005	* We set the exiting tasks cgroup to the root cgroup (top_cgroup). We
6006	* call cgroup_exit() while the task is still competent to handle
6007	* notify_on_release(), then leave the task attached to the root cgroup in
6008	* each hierarchy for the remainder of its exit. No need to bother with
6009	* init_css_set refcnting. init_css_set never goes away and we can't race
6010	* with migration path - PF_EXITING is visible to migration path.
6011	*/
6012	void cgroup_exit(struct task_struct *tsk)
6013	{
6014	struct cgroup_subsys *ss;
6015	struct css_set *cset;
6016	int i;
6017
6018	/*
6019	* Unlink from @tsk from its css_set. As migration path can't race
6020	* with us, we can check css_set and cg_list without synchronization.
6021	*/
6022	cset = task_css_set(tsk);
6023
6024	if (!list_empty(&tsk->cg_list)) {
6025	spin_lock_irq(&css_set_lock);
6026	css_set_move_task(tsk, cset, NULL, false);
6027	spin_unlock_irq(&css_set_lock);
6028	} else {
6029	get_css_set(cset);
6030	}
6031
6032	/* see cgroup_post_fork() for details */
6033	do_each_subsys_mask(ss, i, have_exit_callback) {
6034	ss->exit(tsk);
6035	} while_each_subsys_mask();
6036	}
6037
6038	void cgroup_free(struct task_struct *task)
6039	{
6040	struct css_set *cset = task_css_set(task);
6041	struct cgroup_subsys *ss;
6042	int ssid;
6043
6044	do_each_subsys_mask(ss, ssid, have_free_callback) {
6045	ss->free(task);
6046	} while_each_subsys_mask();
6047
6048	put_css_set(cset);
6049	}
6050
6051	static void check_for_release(struct cgroup *cgrp)
6052	{
6053	if (notify_on_release(cgrp) && !cgroup_is_populated(cgrp) &&
6054	!css_has_online_children(&cgrp->self) && !cgroup_is_dead(cgrp))
6055	schedule_work(&cgrp->release_agent_work);
6056	}
6057
6058	/*
6059	* Notify userspace when a cgroup is released, by running the
6060	* configured release agent with the name of the cgroup (path
6061	* relative to the root of cgroup file system) as the argument.
6062	*
6063	* Most likely, this user command will try to rmdir this cgroup.
6064	*
6065	* This races with the possibility that some other task will be
6066	* attached to this cgroup before it is removed, or that some other
6067	* user task will 'mkdir' a child cgroup of this cgroup. That's ok.
6068	* The presumed 'rmdir' will fail quietly if this cgroup is no longer
6069	* unused, and this cgroup will be reprieved from its death sentence,
6070	* to continue to serve a useful existence. Next time it's released,
6071	* we will get notified again, if it still has 'notify_on_release' set.
6072	*
6073	* The final arg to call_usermodehelper() is UMH_WAIT_EXEC, which
6074	* means only wait until the task is successfully execve()'d. The
6075	* separate release agent task is forked by call_usermodehelper(),
6076	* then control in this thread returns here, without waiting for the
6077	* release agent task. We don't bother to wait because the caller of
6078	* this routine has no use for the exit status of the release agent
6079	* task, so no sense holding our caller up for that.
6080	*/
6081	static void cgroup_release_agent(struct work_struct *work)
6082	{
6083	struct cgroup *cgrp =
6084	container_of(work, struct cgroup, release_agent_work);
6085	char pathbuf = NULL, agentbuf = NULL;
6086	char argv[3], envp[3];
6087	int ret;
6088
6089	mutex_lock(&cgroup_mutex);
6090
6091	pathbuf = kmalloc(PATH_MAX, GFP_KERNEL);
6092	agentbuf = kstrdup(cgrp->root->release_agent_path, GFP_KERNEL);
6093	if (!pathbuf \|\| !agentbuf)
6094	goto out;
6095
6096	spin_lock_irq(&css_set_lock);
6097	ret = cgroup_path_ns_locked(cgrp, pathbuf, PATH_MAX, &init_cgroup_ns);
6098	spin_unlock_irq(&css_set_lock);
6099	if (ret < 0 \|\| ret >= PATH_MAX)
6100	goto out;
6101
6102	argv[0] = agentbuf;
6103	argv[1] = pathbuf;
6104	argv[2] = NULL;
6105
6106	/* minimal command environment */
6107	envp[0] = "HOME=/";
6108	envp[1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
6109	envp[2] = NULL;
6110
6111	mutex_unlock(&cgroup_mutex);
6112	call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC);
6113	goto out_free;
6114	out:
6115	mutex_unlock(&cgroup_mutex);
6116	out_free:
6117	kfree(agentbuf);
6118	kfree(pathbuf);
6119	}
6120
6121	static int __init cgroup_disable(char *str)
6122	{
6123	struct cgroup_subsys *ss;
6124	char *token;
6125	int i;
6126
6127	while ((token = strsep(&str, ",")) != NULL) {
6128	if (!*token)
6129	continue;
6130
6131	for_each_subsys(ss, i) {
6132	if (strcmp(token, ss->name) &&
6133	strcmp(token, ss->legacy_name))
6134	continue;
6135	cgroup_disable_mask \|= 1 << i;
6136	}
6137	}
6138	return 1;
6139	}
6140	__setup("cgroup_disable=", cgroup_disable);
6141
6142	static int __init cgroup_no_v1(char *str)
6143	{
6144	struct cgroup_subsys *ss;
6145	char *token;
6146	int i;
6147
6148	while ((token = strsep(&str, ",")) != NULL) {
6149	if (!*token)
6150	continue;
6151
6152	if (!strcmp(token, "all")) {
6153	cgroup_no_v1_mask = U16_MAX;
6154	break;
6155	}
6156
6157	for_each_subsys(ss, i) {
6158	if (strcmp(token, ss->name) &&
6159	strcmp(token, ss->legacy_name))
6160	continue;
6161
6162	cgroup_no_v1_mask \|= 1 << i;
6163	}
6164	}
6165	return 1;
6166	}
6167	__setup("cgroup_no_v1=", cgroup_no_v1);
6168
6169	/**
6170	* css_tryget_online_from_dir - get corresponding css from a cgroup dentry
6171	* @dentry: directory dentry of interest
6172	* @ss: subsystem of interest
6173	*
6174	* If @dentry is a directory for a cgroup which has @ss enabled on it, try
6175	* to get the corresponding css and return it. If such css doesn't exist
6176	* or can't be pinned, an ERR_PTR value is returned.
6177	*/
6178	struct cgroup_subsys_state css_tryget_online_from_dir(struct dentry dentry,
6179	struct cgroup_subsys *ss)
6180	{
6181	struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
6182	struct file_system_type *s_type = dentry->d_sb->s_type;
6183	struct cgroup_subsys_state *css = NULL;
6184	struct cgroup *cgrp;
6185
6186	/* is @dentry a cgroup dir? */
6187	if ((s_type != &cgroup_fs_type && s_type != &cgroup2_fs_type) \|\|
6188	!kn \|\| kernfs_type(kn) != KERNFS_DIR)
6189	return ERR_PTR(-EBADF);
6190
6191	rcu_read_lock();
6192
6193	/*
6194	* This path doesn't originate from kernfs and @kn could already
6195	* have been or be removed at any point. @kn->priv is RCU
6196	* protected for this access. See css_release_work_fn() for details.
6197	*/
6198	cgrp = rcu_dereference(kn->priv);
6199	if (cgrp)
6200	css = cgroup_css(cgrp, ss);
6201
6202	if (!css \|\| !css_tryget_online(css))
6203	css = ERR_PTR(-ENOENT);
6204
6205	rcu_read_unlock();
6206	return css;
6207	}
6208
6209	/**
6210	* css_from_id - lookup css by id
6211	* @id: the cgroup id
6212	* @ss: cgroup subsys to be looked into
6213	*
6214	* Returns the css if there's valid one with @id, otherwise returns NULL.
6215	* Should be called under rcu_read_lock().
6216	*/
6217	struct cgroup_subsys_state css_from_id(int id, struct cgroup_subsys ss)
6218	{
6219	WARN_ON_ONCE(!rcu_read_lock_held());
6220	return idr_find(&ss->css_idr, id);
6221	}
6222
6223	/**
6224	* cgroup_get_from_path - lookup and get a cgroup from its default hierarchy path
6225	* @path: path on the default hierarchy
6226	*
6227	* Find the cgroup at @path on the default hierarchy, increment its
6228	* reference count and return it. Returns pointer to the found cgroup on
6229	* success, ERR_PTR(-ENOENT) if @path doens't exist and ERR_PTR(-ENOTDIR)
6230	* if @path points to a non-directory.
6231	*/
6232	struct cgroup cgroup_get_from_path(const char path)
6233	{
6234	struct kernfs_node *kn;
6235	struct cgroup *cgrp;
6236
6237	mutex_lock(&cgroup_mutex);
6238
6239	kn = kernfs_walk_and_get(cgrp_dfl_root.cgrp.kn, path);
6240	if (kn) {
6241	if (kernfs_type(kn) == KERNFS_DIR) {
6242	cgrp = kn->priv;
6243	cgroup_get(cgrp);
6244	} else {
6245	cgrp = ERR_PTR(-ENOTDIR);
6246	}
6247	kernfs_put(kn);
6248	} else {
6249	cgrp = ERR_PTR(-ENOENT);
6250	}
6251
6252	mutex_unlock(&cgroup_mutex);
6253	return cgrp;
6254	}
6255	EXPORT_SYMBOL_GPL(cgroup_get_from_path);
6256
6257	/**
6258	* cgroup_get_from_fd - get a cgroup pointer from a fd
6259	* @fd: fd obtained by open(cgroup2_dir)
6260	*
6261	* Find the cgroup from a fd which should be obtained
6262	* by opening a cgroup directory. Returns a pointer to the
6263	* cgroup on success. ERR_PTR is returned if the cgroup
6264	* cannot be found.
6265	*/
6266	struct cgroup *cgroup_get_from_fd(int fd)
6267	{
6268	struct cgroup_subsys_state *css;
6269	struct cgroup *cgrp;
6270	struct file *f;
6271
6272	f = fget_raw(fd);
6273	if (!f)
6274	return ERR_PTR(-EBADF);
6275
6276	css = css_tryget_online_from_dir(f->f_path.dentry, NULL);
6277	fput(f);
6278	if (IS_ERR(css))
6279	return ERR_CAST(css);
6280
6281	cgrp = css->cgroup;
6282	if (!cgroup_on_dfl(cgrp)) {
6283	cgroup_put(cgrp);
6284	return ERR_PTR(-EBADF);
6285	}
6286
6287	return cgrp;
6288	}
6289	EXPORT_SYMBOL_GPL(cgroup_get_from_fd);
6290
6291	/*
6292	* sock->sk_cgrp_data handling. For more info, see sock_cgroup_data
6293	* definition in cgroup-defs.h.
6294	*/
6295	#ifdef CONFIG_SOCK_CGROUP_DATA
6296
6297	#if defined(CONFIG_CGROUP_NET_PRIO) \|\| defined(CONFIG_CGROUP_NET_CLASSID)
6298
6299	DEFINE_SPINLOCK(cgroup_sk_update_lock);
6300	static bool cgroup_sk_alloc_disabled __read_mostly;
6301
6302	void cgroup_sk_alloc_disable(void)
6303	{
6304	if (cgroup_sk_alloc_disabled)
6305	return;
6306	pr_info("cgroup: disabling cgroup2 socket matching due to net_prio or net_cls activation\n");
6307	cgroup_sk_alloc_disabled = true;
6308	}
6309
6310	#else
6311
6312	#define cgroup_sk_alloc_disabled false
6313
6314	#endif
6315
6316	void cgroup_sk_alloc(struct sock_cgroup_data *skcd)
6317	{
6318	if (cgroup_sk_alloc_disabled)
6319	return;
6320
6321	/* Socket clone path */
6322	if (skcd->val) {
6323	cgroup_get(sock_cgroup_ptr(skcd));
6324	return;
6325	}
6326
6327	rcu_read_lock();
6328
6329	while (true) {
6330	struct css_set *cset;
6331
6332	cset = task_css_set(current);
6333	if (likely(cgroup_tryget(cset->dfl_cgrp))) {
6334	skcd->val = (unsigned long)cset->dfl_cgrp;
6335	break;
6336	}
6337	cpu_relax();
6338	}
6339
6340	rcu_read_unlock();
6341	}
6342
6343	void cgroup_sk_free(struct sock_cgroup_data *skcd)
6344	{
6345	cgroup_put(sock_cgroup_ptr(skcd));
6346	}
6347
6348	#endif /* CONFIG_SOCK_CGROUP_DATA */
6349
6350	/* cgroup namespaces */
6351
6352	static struct ucounts inc_cgroup_namespaces(struct user_namespace ns)
6353	{
6354	return inc_ucount(ns, current_euid(), UCOUNT_CGROUP_NAMESPACES);
6355	}
6356
6357	static void dec_cgroup_namespaces(struct ucounts *ucounts)
6358	{
6359	dec_ucount(ucounts, UCOUNT_CGROUP_NAMESPACES);
6360	}
6361
6362	static struct cgroup_namespace *alloc_cgroup_ns(void)
6363	{
6364	struct cgroup_namespace *new_ns;
6365	int ret;
6366
6367	new_ns = kzalloc(sizeof(struct cgroup_namespace), GFP_KERNEL);
6368	if (!new_ns)
6369	return ERR_PTR(-ENOMEM);
6370	ret = ns_alloc_inum(&new_ns->ns);
6371	if (ret) {
6372	kfree(new_ns);
6373	return ERR_PTR(ret);
6374	}
6375	atomic_set(&new_ns->count, 1);
6376	new_ns->ns.ops = &cgroupns_operations;
6377	return new_ns;
6378	}
6379
6380	void free_cgroup_ns(struct cgroup_namespace *ns)
6381	{
6382	put_css_set(ns->root_cset);
6383	dec_cgroup_namespaces(ns->ucounts);
6384	put_user_ns(ns->user_ns);
6385	ns_free_inum(&ns->ns);
6386	kfree(ns);
6387	}
6388	EXPORT_SYMBOL(free_cgroup_ns);
6389
6390	struct cgroup_namespace *copy_cgroup_ns(unsigned long flags,
6391	struct user_namespace *user_ns,
6392	struct cgroup_namespace *old_ns)
6393	{
6394	struct cgroup_namespace *new_ns;
6395	struct ucounts *ucounts;
6396	struct css_set *cset;
6397
6398	BUG_ON(!old_ns);
6399
6400	if (!(flags & CLONE_NEWCGROUP)) {
6401	get_cgroup_ns(old_ns);
6402	return old_ns;
6403	}
6404
6405	/* Allow only sysadmin to create cgroup namespace. */
6406	if (!ns_capable(user_ns, CAP_SYS_ADMIN))
6407	return ERR_PTR(-EPERM);
6408
6409	ucounts = inc_cgroup_namespaces(user_ns);
6410	if (!ucounts)
6411	return ERR_PTR(-ENOSPC);
6412
6413	/* It is not safe to take cgroup_mutex here */
6414	spin_lock_irq(&css_set_lock);
6415	cset = task_css_set(current);
6416	get_css_set(cset);
6417	spin_unlock_irq(&css_set_lock);
6418
6419	new_ns = alloc_cgroup_ns();
6420	if (IS_ERR(new_ns)) {
6421	put_css_set(cset);
6422	dec_cgroup_namespaces(ucounts);
6423	return new_ns;
6424	}
6425
6426	new_ns->user_ns = get_user_ns(user_ns);
6427	new_ns->ucounts = ucounts;
6428	new_ns->root_cset = cset;
6429
6430	return new_ns;
6431	}
6432
6433	static inline struct cgroup_namespace to_cg_ns(struct ns_common ns)
6434	{
6435	return container_of(ns, struct cgroup_namespace, ns);
6436	}
6437
6438	static int cgroupns_install(struct nsproxy nsproxy, struct ns_common ns)
6439	{
6440	struct cgroup_namespace *cgroup_ns = to_cg_ns(ns);
6441
6442	if (!ns_capable(current_user_ns(), CAP_SYS_ADMIN) \|\|
6443	!ns_capable(cgroup_ns->user_ns, CAP_SYS_ADMIN))
6444	return -EPERM;
6445
6446	/* Don't need to do anything if we are attaching to our own cgroupns. */
6447	if (cgroup_ns == nsproxy->cgroup_ns)
6448	return 0;
6449
6450	get_cgroup_ns(cgroup_ns);
6451	put_cgroup_ns(nsproxy->cgroup_ns);
6452	nsproxy->cgroup_ns = cgroup_ns;
6453
6454	return 0;
6455	}
6456
6457	static struct ns_common cgroupns_get(struct task_struct task)
6458	{
6459	struct cgroup_namespace *ns = NULL;
6460	struct nsproxy *nsproxy;
6461
6462	task_lock(task);
6463	nsproxy = task->nsproxy;
6464	if (nsproxy) {
6465	ns = nsproxy->cgroup_ns;
6466	get_cgroup_ns(ns);
6467	}
6468	task_unlock(task);
6469
6470	return ns ? &ns->ns : NULL;
6471	}
6472
6473	static void cgroupns_put(struct ns_common *ns)
6474	{
6475	put_cgroup_ns(to_cg_ns(ns));
6476	}
6477
6478	static struct user_namespace cgroupns_owner(struct ns_common ns)
6479	{
6480	return to_cg_ns(ns)->user_ns;
6481	}
6482
6483	const struct proc_ns_operations cgroupns_operations = {
6484	.name = "cgroup",
6485	.type = CLONE_NEWCGROUP,
6486	.get = cgroupns_get,
6487	.put = cgroupns_put,
6488	.install = cgroupns_install,
6489	.owner = cgroupns_owner,
6490	};
6491
6492	static __init int cgroup_namespaces_init(void)
6493	{
6494	return 0;
6495	}
6496	subsys_initcall(cgroup_namespaces_init);
6497
6498	#ifdef CONFIG_CGROUP_DEBUG
6499	static struct cgroup_subsys_state *
6500	debug_css_alloc(struct cgroup_subsys_state *parent_css)
6501	{
6502	struct cgroup_subsys_state css = kzalloc(sizeof(css), GFP_KERNEL);
6503
6504	if (!css)
6505	return ERR_PTR(-ENOMEM);
6506
6507	return css;
6508	}
6509
6510	static void debug_css_free(struct cgroup_subsys_state *css)
6511	{
6512	kfree(css);
6513	}
6514
6515	static u64 debug_taskcount_read(struct cgroup_subsys_state *css,
6516	struct cftype *cft)
6517	{
6518	return cgroup_task_count(css->cgroup);
6519	}
6520
6521	static u64 current_css_set_read(struct cgroup_subsys_state *css,
6522	struct cftype *cft)
6523	{
6524	return (u64)(unsigned long)current->cgroups;
6525	}
6526
6527	static u64 current_css_set_refcount_read(struct cgroup_subsys_state *css,
6528	struct cftype *cft)
6529	{
6530	u64 count;
6531
6532	rcu_read_lock();
6533	count = atomic_read(&task_css_set(current)->refcount);
6534	rcu_read_unlock();
6535	return count;
6536	}
6537
6538	static int current_css_set_cg_links_read(struct seq_file seq, void v)
6539	{
6540	struct cgrp_cset_link *link;
6541	struct css_set *cset;
6542	char *name_buf;
6543
6544	name_buf = kmalloc(NAME_MAX + 1, GFP_KERNEL);
6545	if (!name_buf)
6546	return -ENOMEM;
6547
6548	spin_lock_irq(&css_set_lock);
6549	rcu_read_lock();
6550	cset = rcu_dereference(current->cgroups);
6551	list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
6552	struct cgroup *c = link->cgrp;
6553
6554	cgroup_name(c, name_buf, NAME_MAX + 1);
6555	seq_printf(seq, "Root %d group %s\n",
6556	c->root->hierarchy_id, name_buf);
6557	}
6558	rcu_read_unlock();
6559	spin_unlock_irq(&css_set_lock);
6560	kfree(name_buf);
6561	return 0;
6562	}
6563
6564	#define MAX_TASKS_SHOWN_PER_CSS 25
6565	static int cgroup_css_links_read(struct seq_file seq, void v)
6566	{
6567	struct cgroup_subsys_state *css = seq_css(seq);
6568	struct cgrp_cset_link *link;
6569
6570	spin_lock_irq(&css_set_lock);
6571	list_for_each_entry(link, &css->cgroup->cset_links, cset_link) {
6572	struct css_set *cset = link->cset;
6573	struct task_struct *task;
6574	int count = 0;
6575
6576	seq_printf(seq, "css_set %p\n", cset);
6577
6578	list_for_each_entry(task, &cset->tasks, cg_list) {
6579	if (count++ > MAX_TASKS_SHOWN_PER_CSS)
6580	goto overflow;
6581	seq_printf(seq, " task %d\n", task_pid_vnr(task));
6582	}
6583
6584	list_for_each_entry(task, &cset->mg_tasks, cg_list) {
6585	if (count++ > MAX_TASKS_SHOWN_PER_CSS)
6586	goto overflow;
6587	seq_printf(seq, " task %d\n", task_pid_vnr(task));
6588	}
6589	continue;
6590	overflow:
6591	seq_puts(seq, " ...\n");
6592	}
6593	spin_unlock_irq(&css_set_lock);
6594	return 0;
6595	}
6596
6597	static u64 releasable_read(struct cgroup_subsys_state css, struct cftype cft)
6598	{
6599	return (!cgroup_is_populated(css->cgroup) &&
6600	!css_has_online_children(&css->cgroup->self));
6601	}
6602
6603	static struct cftype debug_files[] = {
6604	{
6605	.name = "taskcount",
6606	.read_u64 = debug_taskcount_read,
6607	},
6608
6609	{
6610	.name = "current_css_set",
6611	.read_u64 = current_css_set_read,
6612	},
6613
6614	{
6615	.name = "current_css_set_refcount",
6616	.read_u64 = current_css_set_refcount_read,
6617	},
6618
6619	{
6620	.name = "current_css_set_cg_links",
6621	.seq_show = current_css_set_cg_links_read,
6622	},
6623
6624	{
6625	.name = "cgroup_css_links",
6626	.seq_show = cgroup_css_links_read,
6627	},
6628
6629	{
6630	.name = "releasable",
6631	.read_u64 = releasable_read,
6632	},
6633
6634	{ } /* terminate */
6635	};
6636
6637	struct cgroup_subsys debug_cgrp_subsys = {
6638	.css_alloc = debug_css_alloc,
6639	.css_free = debug_css_free,
6640	.legacy_cftypes = debug_files,
6641	};
6642	#endif /* CONFIG_CGROUP_DEBUG */