linux - ipc/sem.c

2211 lines · 56.1 KiB

1	/*
2	* linux/ipc/sem.c
3	* Copyright (C) 1992 Krishna Balasubramanian
4	* Copyright (C) 1995 Eric Schenk, Bruno Haible
5	*
6	* /proc/sysvipc/sem support (c) 1999 Dragos Acostachioaie <[email protected]>
7	*
8	* SMP-threaded, sysctl's added
9	* (c) 1999 Manfred Spraul <[email protected]>
10	* Enforced range limit on SEM_UNDO
11	* (c) 2001 Red Hat Inc
12	* Lockless wakeup
13	* (c) 2003 Manfred Spraul <[email protected]>
14	* Further wakeup optimizations, documentation
15	* (c) 2010 Manfred Spraul <[email protected]>
16	*
17	* support for audit of ipc object properties and permission changes
18	* Dustin Kirkland <[email protected]>
19	*
20	* namespaces support
21	* OpenVZ, SWsoft Inc.
22	* Pavel Emelianov <[email protected]>
23	*
24	* Implementation notes: (May 2010)
25	* This file implements System V semaphores.
26	*
27	* User space visible behavior:
28	* - FIFO ordering for semop() operations (just FIFO, not starvation
29	* protection)
30	* - multiple semaphore operations that alter the same semaphore in
31	* one semop() are handled.
32	* - sem_ctime (time of last semctl()) is updated in the IPC_SET, SETVAL and
33	* SETALL calls.
34	* - two Linux specific semctl() commands: SEM_STAT, SEM_INFO.
35	* - undo adjustments at process exit are limited to 0..SEMVMX.
36	* - namespace are supported.
37	* - SEMMSL, SEMMNS, SEMOPM and SEMMNI can be configured at runtine by writing
38	* to /proc/sys/kernel/sem.
39	* - statistics about the usage are reported in /proc/sysvipc/sem.
40	*
41	* Internals:
42	* - scalability:
43	* - all global variables are read-mostly.
44	* - semop() calls and semctl(RMID) are synchronized by RCU.
45	* - most operations do write operations (actually: spin_lock calls) to
46	* the per-semaphore array structure.
47	* Thus: Perfect SMP scaling between independent semaphore arrays.
48	* If multiple semaphores in one array are used, then cache line
49	* trashing on the semaphore array spinlock will limit the scaling.
50	* - semncnt and semzcnt are calculated on demand in count_semcnt()
51	* - the task that performs a successful semop() scans the list of all
52	* sleeping tasks and completes any pending operations that can be fulfilled.
53	* Semaphores are actively given to waiting tasks (necessary for FIFO).
54	* (see update_queue())
55	* - To improve the scalability, the actual wake-up calls are performed after
56	* dropping all locks. (see wake_up_sem_queue_prepare(),
57	* wake_up_sem_queue_do())
58	* - All work is done by the waker, the woken up task does not have to do
59	* anything - not even acquiring a lock or dropping a refcount.
60	* - A woken up task may not even touch the semaphore array anymore, it may
61	* have been destroyed already by a semctl(RMID).
62	* - The synchronizations between wake-ups due to a timeout/signal and a
63	* wake-up due to a completed semaphore operation is achieved by using an
64	* intermediate state (IN_WAKEUP).
65	* - UNDO values are stored in an array (one per process and per
66	* semaphore array, lazily allocated). For backwards compatibility, multiple
67	* modes for the UNDO variables are supported (per process, per thread)
68	* (see copy_semundo, CLONE_SYSVSEM)
69	* - There are two lists of the pending operations: a per-array list
70	* and per-semaphore list (stored in the array). This allows to achieve FIFO
71	* ordering without always scanning all pending operations.
72	* The worst-case behavior is nevertheless O(N^2) for N wakeups.
73	*/
74
75	#include <linux/slab.h>
76	#include <linux/spinlock.h>
77	#include <linux/init.h>
78	#include <linux/proc_fs.h>
79	#include <linux/time.h>
80	#include <linux/security.h>
81	#include <linux/syscalls.h>
82	#include <linux/audit.h>
83	#include <linux/capability.h>
84	#include <linux/seq_file.h>
85	#include <linux/rwsem.h>
86	#include <linux/nsproxy.h>
87	#include <linux/ipc_namespace.h>
88
89	#include <linux/uaccess.h>
90	#include "util.h"
91
92	/* One semaphore structure for each semaphore in the system. */
93	struct sem {
94	int semval; /* current value */
95	int sempid; /* pid of last operation */
96	spinlock_t lock; /* spinlock for fine-grained semtimedop */
97	struct list_head pending_alter; /* pending single-sop operations */
98	/* that alter the semaphore */
99	struct list_head pending_const; /* pending single-sop operations */
100	/* that do not alter the semaphore*/
101	time_t sem_otime; /* candidate for sem_otime */
102	} ____cacheline_aligned_in_smp;
103
104	/* One queue for each sleeping process in the system. */
105	struct sem_queue {
106	struct list_head list; /* queue of pending operations */
107	struct task_struct sleeper; / this process */
108	struct sem_undo undo; / undo structure */
109	int pid; /* process id of requesting process */
110	int status; /* completion status of operation */
111	struct sembuf sops; / array of pending operations */
112	struct sembuf blocking; / the operation that blocked */
113	int nsops; /* number of operations */
114	int alter; /* does sops alter the array? /
115	};
116
117	/* Each task has a list of undo requests. They are executed automatically
118	* when the process exits.
119	*/
120	struct sem_undo {
121	struct list_head list_proc; /* per-process list: *
122	* all undos from one process
123	* rcu protected */
124	struct rcu_head rcu; /* rcu struct for sem_undo */
125	struct sem_undo_list ulp; / back ptr to sem_undo_list */
126	struct list_head list_id; /* per semaphore array list:
127	* all undos for one array */
128	int semid; /* semaphore set identifier */
129	short semadj; / array of adjustments */
130	/* one per semaphore */
131	};
132
133	/* sem_undo_list controls shared access to the list of sem_undo structures
134	* that may be shared among all a CLONE_SYSVSEM task group.
135	*/
136	struct sem_undo_list {
137	atomic_t refcnt;
138	spinlock_t lock;
139	struct list_head list_proc;
140	};
141
142
143	#define sem_ids(ns) ((ns)->ids[IPC_SEM_IDS])
144
145	#define sem_checkid(sma, semid) ipc_checkid(&sma->sem_perm, semid)
146
147	static int newary(struct ipc_namespace , struct ipc_params );
148	static void freeary(struct ipc_namespace , struct kern_ipc_perm );
149	#ifdef CONFIG_PROC_FS
150	static int sysvipc_sem_proc_show(struct seq_file s, void it);
151	#endif
152
153	#define SEMMSL_FAST 256 /* 512 bytes on stack */
154	#define SEMOPM_FAST 64 /* ~ 372 bytes on stack */
155
156	/*
157	* Locking:
158	* sem_undo.id_next,
159	* sem_array.complex_count,
160	* sem_array.pending{_alter,_cont},
161	* sem_array.sem_undo: global sem_lock() for read/write
162	* sem_undo.proc_next: only "current" is allowed to read/write that field.
163	*
164	* sem_array.sem_base[i].pending_{const,alter}:
165	* global or semaphore sem_lock() for read/write
166	*/
167
168	#define sc_semmsl sem_ctls[0]
169	#define sc_semmns sem_ctls[1]
170	#define sc_semopm sem_ctls[2]
171	#define sc_semmni sem_ctls[3]
172
173	void sem_init_ns(struct ipc_namespace *ns)
174	{
175	ns->sc_semmsl = SEMMSL;
176	ns->sc_semmns = SEMMNS;
177	ns->sc_semopm = SEMOPM;
178	ns->sc_semmni = SEMMNI;
179	ns->used_sems = 0;
180	ipc_init_ids(&ns->ids[IPC_SEM_IDS]);
181	}
182
183	#ifdef CONFIG_IPC_NS
184	void sem_exit_ns(struct ipc_namespace *ns)
185	{
186	free_ipcs(ns, &sem_ids(ns), freeary);
187	idr_destroy(&ns->ids[IPC_SEM_IDS].ipcs_idr);
188	}
189	#endif
190
191	void __init sem_init(void)
192	{
193	sem_init_ns(&init_ipc_ns);
194	ipc_init_proc_interface("sysvipc/sem",
195	" key semid perms nsems uid gid cuid cgid otime ctime\n",
196	IPC_SEM_IDS, sysvipc_sem_proc_show);
197	}
198
199	/**
200	* unmerge_queues - unmerge queues, if possible.
201	* @sma: semaphore array
202	*
203	* The function unmerges the wait queues if complex_count is 0.
204	* It must be called prior to dropping the global semaphore array lock.
205	*/
206	static void unmerge_queues(struct sem_array *sma)
207	{
208	struct sem_queue q, tq;
209
210	/* complex operations still around? */
211	if (sma->complex_count)
212	return;
213	/*
214	* We will switch back to simple mode.
215	* Move all pending operation back into the per-semaphore
216	* queues.
217	*/
218	list_for_each_entry_safe(q, tq, &sma->pending_alter, list) {
219	struct sem *curr;
220	curr = &sma->sem_base[q->sops[0].sem_num];
221
222	list_add_tail(&q->list, &curr->pending_alter);
223	}
224	INIT_LIST_HEAD(&sma->pending_alter);
225	}
226
227	/**
228	* merge_queues - merge single semop queues into global queue
229	* @sma: semaphore array
230	*
231	* This function merges all per-semaphore queues into the global queue.
232	* It is necessary to achieve FIFO ordering for the pending single-sop
233	* operations when a multi-semop operation must sleep.
234	* Only the alter operations must be moved, the const operations can stay.
235	*/
236	static void merge_queues(struct sem_array *sma)
237	{
238	int i;
239	for (i = 0; i < sma->sem_nsems; i++) {
240	struct sem *sem = sma->sem_base + i;
241
242	list_splice_init(&sem->pending_alter, &sma->pending_alter);
243	}
244	}
245
246	static void sem_rcu_free(struct rcu_head *head)
247	{
248	struct ipc_rcu *p = container_of(head, struct ipc_rcu, rcu);
249	struct sem_array *sma = ipc_rcu_to_struct(p);
250
251	security_sem_free(sma);
252	ipc_rcu_free(head);
253	}
254
255	/*
256	* spin_unlock_wait() and !spin_is_locked() are not memory barriers, they
257	* are only control barriers.
258	* The code must pair with spin_unlock(&sem->lock) or
259	* spin_unlock(&sem_perm.lock), thus just the control barrier is insufficient.
260	*
261	* smp_rmb() is sufficient, as writes cannot pass the control barrier.
262	*/
263	#define ipc_smp_acquire__after_spin_is_unlocked() smp_rmb()
264
265	/*
266	* Wait until all currently ongoing simple ops have completed.
267	* Caller must own sem_perm.lock.
268	* New simple ops cannot start, because simple ops first check
269	* that sem_perm.lock is free.
270	* that a) sem_perm.lock is free and b) complex_count is 0.
271	*/
272	static void sem_wait_array(struct sem_array *sma)
273	{
274	int i;
275	struct sem *sem;
276
277	if (sma->complex_count) {
278	/* The thread that increased sma->complex_count waited on
279	* all sem->lock locks. Thus we don't need to wait again.
280	*/
281	return;
282	}
283
284	for (i = 0; i < sma->sem_nsems; i++) {
285	sem = sma->sem_base + i;
286	spin_unlock_wait(&sem->lock);
287	}
288	ipc_smp_acquire__after_spin_is_unlocked();
289	}
290
291	/*
292	* If the request contains only one semaphore operation, and there are
293	* no complex transactions pending, lock only the semaphore involved.
294	* Otherwise, lock the entire semaphore array, since we either have
295	* multiple semaphores in our own semops, or we need to look at
296	* semaphores from other pending complex operations.
297	*/
298	static inline int sem_lock(struct sem_array sma, struct sembuf sops,
299	int nsops)
300	{
301	struct sem *sem;
302
303	if (nsops != 1) {
304	/* Complex operation - acquire a full lock */
305	ipc_lock_object(&sma->sem_perm);
306
307	/* And wait until all simple ops that are processed
308	* right now have dropped their locks.
309	*/
310	sem_wait_array(sma);
311	return -1;
312	}
313
314	/*
315	* Only one semaphore affected - try to optimize locking.
316	* The rules are:
317	* - optimized locking is possible if no complex operation
318	* is either enqueued or processed right now.
319	* - The test for enqueued complex ops is simple:
320	* sma->complex_count != 0
321	* - Testing for complex ops that are processed right now is
322	* a bit more difficult. Complex ops acquire the full lock
323	* and first wait that the running simple ops have completed.
324	* (see above)
325	* Thus: If we own a simple lock and the global lock is free
326	* and complex_count is now 0, then it will stay 0 and
327	* thus just locking sem->lock is sufficient.
328	*/
329	sem = sma->sem_base + sops->sem_num;
330
331	if (sma->complex_count == 0) {
332	/*
333	* It appears that no complex operation is around.
334	* Acquire the per-semaphore lock.
335	*/
336	spin_lock(&sem->lock);
337
338	/* Then check that the global lock is free */
339	if (!spin_is_locked(&sma->sem_perm.lock)) {
340	/*
341	* We need a memory barrier with acquire semantics,
342	* otherwise we can race with another thread that does:
343	* complex_count++;
344	* spin_unlock(sem_perm.lock);
345	*/
346	ipc_smp_acquire__after_spin_is_unlocked();
347
348	/*
349	* Now repeat the test of complex_count:
350	* It can't change anymore until we drop sem->lock.
351	* Thus: if is now 0, then it will stay 0.
352	*/
353	if (sma->complex_count == 0) {
354	/* fast path successful! */
355	return sops->sem_num;
356	}
357	}
358	spin_unlock(&sem->lock);
359	}
360
361	/* slow path: acquire the full lock */
362	ipc_lock_object(&sma->sem_perm);
363
364	if (sma->complex_count == 0) {
365	/* False alarm:
366	* There is no complex operation, thus we can switch
367	* back to the fast path.
368	*/
369	spin_lock(&sem->lock);
370	ipc_unlock_object(&sma->sem_perm);
371	return sops->sem_num;
372	} else {
373	/* Not a false alarm, thus complete the sequence for a
374	* full lock.
375	*/
376	sem_wait_array(sma);
377	return -1;
378	}
379	}
380
381	static inline void sem_unlock(struct sem_array *sma, int locknum)
382	{
383	if (locknum == -1) {
384	unmerge_queues(sma);
385	ipc_unlock_object(&sma->sem_perm);
386	} else {
387	struct sem *sem = sma->sem_base + locknum;
388	spin_unlock(&sem->lock);
389	}
390	}
391
392	/*
393	* sem_lock_(check_) routines are called in the paths where the rwsem
394	* is not held.
395	*
396	* The caller holds the RCU read lock.
397	*/
398	static inline struct sem_array sem_obtain_lock(struct ipc_namespace ns,
399	int id, struct sembuf sops, int nsops, int locknum)
400	{
401	struct kern_ipc_perm *ipcp;
402	struct sem_array *sma;
403
404	ipcp = ipc_obtain_object_idr(&sem_ids(ns), id);
405	if (IS_ERR(ipcp))
406	return ERR_CAST(ipcp);
407
408	sma = container_of(ipcp, struct sem_array, sem_perm);
409	*locknum = sem_lock(sma, sops, nsops);
410
411	/* ipc_rmid() may have already freed the ID while sem_lock
412	* was spinning: verify that the structure is still valid
413	*/
414	if (ipc_valid_object(ipcp))
415	return container_of(ipcp, struct sem_array, sem_perm);
416
417	sem_unlock(sma, *locknum);
418	return ERR_PTR(-EINVAL);
419	}
420
421	static inline struct sem_array sem_obtain_object(struct ipc_namespace ns, int id)
422	{
423	struct kern_ipc_perm *ipcp = ipc_obtain_object_idr(&sem_ids(ns), id);
424
425	if (IS_ERR(ipcp))
426	return ERR_CAST(ipcp);
427
428	return container_of(ipcp, struct sem_array, sem_perm);
429	}
430
431	static inline struct sem_array sem_obtain_object_check(struct ipc_namespace ns,
432	int id)
433	{
434	struct kern_ipc_perm *ipcp = ipc_obtain_object_check(&sem_ids(ns), id);
435
436	if (IS_ERR(ipcp))
437	return ERR_CAST(ipcp);
438
439	return container_of(ipcp, struct sem_array, sem_perm);
440	}
441
442	static inline void sem_lock_and_putref(struct sem_array *sma)
443	{
444	sem_lock(sma, NULL, -1);
445	ipc_rcu_putref(sma, ipc_rcu_free);
446	}
447
448	static inline void sem_rmid(struct ipc_namespace ns, struct sem_array s)
449	{
450	ipc_rmid(&sem_ids(ns), &s->sem_perm);
451	}
452
453	/*
454	* Lockless wakeup algorithm:
455	* Without the check/retry algorithm a lockless wakeup is possible:
456	* - queue.status is initialized to -EINTR before blocking.
457	* - wakeup is performed by
458	* * unlinking the queue entry from the pending list
459	* * setting queue.status to IN_WAKEUP
460	* This is the notification for the blocked thread that a
461	* result value is imminent.
462	* * call wake_up_process
463	* * set queue.status to the final value.
464	* - the previously blocked thread checks queue.status:
465	* * if it's IN_WAKEUP, then it must wait until the value changes
466	* * if it's not -EINTR, then the operation was completed by
467	* update_queue. semtimedop can return queue.status without
468	* performing any operation on the sem array.
469	* * otherwise it must acquire the spinlock and check what's up.
470	*
471	* The two-stage algorithm is necessary to protect against the following
472	* races:
473	* - if queue.status is set after wake_up_process, then the woken up idle
474	* thread could race forward and try (and fail) to acquire sma->lock
475	* before update_queue had a chance to set queue.status
476	* - if queue.status is written before wake_up_process and if the
477	* blocked process is woken up by a signal between writing
478	* queue.status and the wake_up_process, then the woken up
479	* process could return from semtimedop and die by calling
480	* sys_exit before wake_up_process is called. Then wake_up_process
481	* will oops, because the task structure is already invalid.
482	* (yes, this happened on s390 with sysv msg).
483	*
484	*/
485	#define IN_WAKEUP 1
486
487	/**
488	* newary - Create a new semaphore set
489	* @ns: namespace
490	* @params: ptr to the structure that contains key, semflg and nsems
491	*
492	* Called with sem_ids.rwsem held (as a writer)
493	*/
494	static int newary(struct ipc_namespace ns, struct ipc_params params)
495	{
496	int id;
497	int retval;
498	struct sem_array *sma;
499	int size;
500	key_t key = params->key;
501	int nsems = params->u.nsems;
502	int semflg = params->flg;
503	int i;
504
505	if (!nsems)
506	return -EINVAL;
507	if (ns->used_sems + nsems > ns->sc_semmns)
508	return -ENOSPC;
509
510	size = sizeof(sma) + nsems sizeof(struct sem);
511	sma = ipc_rcu_alloc(size);
512	if (!sma)
513	return -ENOMEM;
514
515	memset(sma, 0, size);
516
517	sma->sem_perm.mode = (semflg & S_IRWXUGO);
518	sma->sem_perm.key = key;
519
520	sma->sem_perm.security = NULL;
521	retval = security_sem_alloc(sma);
522	if (retval) {
523	ipc_rcu_putref(sma, ipc_rcu_free);
524	return retval;
525	}
526
527	sma->sem_base = (struct sem *) &sma[1];
528
529	for (i = 0; i < nsems; i++) {
530	INIT_LIST_HEAD(&sma->sem_base[i].pending_alter);
531	INIT_LIST_HEAD(&sma->sem_base[i].pending_const);
532	spin_lock_init(&sma->sem_base[i].lock);
533	}
534
535	sma->complex_count = 0;
536	INIT_LIST_HEAD(&sma->pending_alter);
537	INIT_LIST_HEAD(&sma->pending_const);
538	INIT_LIST_HEAD(&sma->list_id);
539	sma->sem_nsems = nsems;
540	sma->sem_ctime = get_seconds();
541
542	id = ipc_addid(&sem_ids(ns), &sma->sem_perm, ns->sc_semmni);
543	if (id < 0) {
544	ipc_rcu_putref(sma, sem_rcu_free);
545	return id;
546	}
547	ns->used_sems += nsems;
548
549	sem_unlock(sma, -1);
550	rcu_read_unlock();
551
552	return sma->sem_perm.id;
553	}
554
555
556	/*
557	* Called with sem_ids.rwsem and ipcp locked.
558	*/
559	static inline int sem_security(struct kern_ipc_perm *ipcp, int semflg)
560	{
561	struct sem_array *sma;
562
563	sma = container_of(ipcp, struct sem_array, sem_perm);
564	return security_sem_associate(sma, semflg);
565	}
566
567	/*
568	* Called with sem_ids.rwsem and ipcp locked.
569	*/
570	static inline int sem_more_checks(struct kern_ipc_perm *ipcp,
571	struct ipc_params *params)
572	{
573	struct sem_array *sma;
574
575	sma = container_of(ipcp, struct sem_array, sem_perm);
576	if (params->u.nsems > sma->sem_nsems)
577	return -EINVAL;
578
579	return 0;
580	}
581
582	SYSCALL_DEFINE3(semget, key_t, key, int, nsems, int, semflg)
583	{
584	struct ipc_namespace *ns;
585	static const struct ipc_ops sem_ops = {
586	.getnew = newary,
587	.associate = sem_security,
588	.more_checks = sem_more_checks,
589	};
590	struct ipc_params sem_params;
591
592	ns = current->nsproxy->ipc_ns;
593
594	if (nsems < 0 \|\| nsems > ns->sc_semmsl)
595	return -EINVAL;
596
597	sem_params.key = key;
598	sem_params.flg = semflg;
599	sem_params.u.nsems = nsems;
600
601	return ipcget(ns, &sem_ids(ns), &sem_ops, &sem_params);
602	}
603
604	/**
605	* perform_atomic_semop - Perform (if possible) a semaphore operation
606	* @sma: semaphore array
607	* @q: struct sem_queue that describes the operation
608	*
609	* Returns 0 if the operation was possible.
610	* Returns 1 if the operation is impossible, the caller must sleep.
611	* Negative values are error codes.
612	*/
613	static int perform_atomic_semop(struct sem_array sma, struct sem_queue q)
614	{
615	int result, sem_op, nsops, pid;
616	struct sembuf *sop;
617	struct sem *curr;
618	struct sembuf *sops;
619	struct sem_undo *un;
620
621	sops = q->sops;
622	nsops = q->nsops;
623	un = q->undo;
624
625	for (sop = sops; sop < sops + nsops; sop++) {
626	curr = sma->sem_base + sop->sem_num;
627	sem_op = sop->sem_op;
628	result = curr->semval;
629
630	if (!sem_op && result)
631	goto would_block;
632
633	result += sem_op;
634	if (result < 0)
635	goto would_block;
636	if (result > SEMVMX)
637	goto out_of_range;
638
639	if (sop->sem_flg & SEM_UNDO) {
640	int undo = un->semadj[sop->sem_num] - sem_op;
641	/* Exceeding the undo range is an error. */
642	if (undo < (-SEMAEM - 1) \|\| undo > SEMAEM)
643	goto out_of_range;
644	un->semadj[sop->sem_num] = undo;
645	}
646
647	curr->semval = result;
648	}
649
650	sop--;
651	pid = q->pid;
652	while (sop >= sops) {
653	sma->sem_base[sop->sem_num].sempid = pid;
654	sop--;
655	}
656
657	return 0;
658
659	out_of_range:
660	result = -ERANGE;
661	goto undo;
662
663	would_block:
664	q->blocking = sop;
665
666	if (sop->sem_flg & IPC_NOWAIT)
667	result = -EAGAIN;
668	else
669	result = 1;
670
671	undo:
672	sop--;
673	while (sop >= sops) {
674	sem_op = sop->sem_op;
675	sma->sem_base[sop->sem_num].semval -= sem_op;
676	if (sop->sem_flg & SEM_UNDO)
677	un->semadj[sop->sem_num] += sem_op;
678	sop--;
679	}
680
681	return result;
682	}
683
684	/** wake_up_sem_queue_prepare(q, error): Prepare wake-up
685	* @q: queue entry that must be signaled
686	* @error: Error value for the signal
687	*
688	* Prepare the wake-up of the queue entry q.
689	*/
690	static void wake_up_sem_queue_prepare(struct list_head *pt,
691	struct sem_queue *q, int error)
692	{
693	if (list_empty(pt)) {
694	/*
695	* Hold preempt off so that we don't get preempted and have the
696	* wakee busy-wait until we're scheduled back on.
697	*/
698	preempt_disable();
699	}
700	q->status = IN_WAKEUP;
701	q->pid = error;
702
703	list_add_tail(&q->list, pt);
704	}
705
706	/**
707	* wake_up_sem_queue_do - do the actual wake-up
708	* @pt: list of tasks to be woken up
709	*
710	* Do the actual wake-up.
711	* The function is called without any locks held, thus the semaphore array
712	* could be destroyed already and the tasks can disappear as soon as the
713	* status is set to the actual return code.
714	*/
715	static void wake_up_sem_queue_do(struct list_head *pt)
716	{
717	struct sem_queue q, t;
718	int did_something;
719
720	did_something = !list_empty(pt);
721	list_for_each_entry_safe(q, t, pt, list) {
722	wake_up_process(q->sleeper);
723	/* q can disappear immediately after writing q->status. */
724	smp_wmb();
725	q->status = q->pid;
726	}
727	if (did_something)
728	preempt_enable();
729	}
730
731	static void unlink_queue(struct sem_array sma, struct sem_queue q)
732	{
733	list_del(&q->list);
734	if (q->nsops > 1)
735	sma->complex_count--;
736	}
737
738	/** check_restart(sma, q)
739	* @sma: semaphore array
740	* @q: the operation that just completed
741	*
742	* update_queue is O(N^2) when it restarts scanning the whole queue of
743	* waiting operations. Therefore this function checks if the restart is
744	* really necessary. It is called after a previously waiting operation
745	* modified the array.
746	* Note that wait-for-zero operations are handled without restart.
747	*/
748	static int check_restart(struct sem_array sma, struct sem_queue q)
749	{
750	/* pending complex alter operations are too difficult to analyse */
751	if (!list_empty(&sma->pending_alter))
752	return 1;
753
754	/* we were a sleeping complex operation. Too difficult */
755	if (q->nsops > 1)
756	return 1;
757
758	/* It is impossible that someone waits for the new value:
759	* - complex operations always restart.
760	* - wait-for-zero are handled seperately.
761	* - q is a previously sleeping simple operation that
762	* altered the array. It must be a decrement, because
763	* simple increments never sleep.
764	* - If there are older (higher priority) decrements
765	* in the queue, then they have observed the original
766	* semval value and couldn't proceed. The operation
767	* decremented to value - thus they won't proceed either.
768	*/
769	return 0;
770	}
771
772	/**
773	* wake_const_ops - wake up non-alter tasks
774	* @sma: semaphore array.
775	* @semnum: semaphore that was modified.
776	* @pt: list head for the tasks that must be woken up.
777	*
778	* wake_const_ops must be called after a semaphore in a semaphore array
779	* was set to 0. If complex const operations are pending, wake_const_ops must
780	* be called with semnum = -1, as well as with the number of each modified
781	* semaphore.
782	* The tasks that must be woken up are added to @pt. The return code
783	* is stored in q->pid.
784	* The function returns 1 if at least one operation was completed successfully.
785	*/
786	static int wake_const_ops(struct sem_array *sma, int semnum,
787	struct list_head *pt)
788	{
789	struct sem_queue *q;
790	struct list_head *walk;
791	struct list_head *pending_list;
792	int semop_completed = 0;
793
794	if (semnum == -1)
795	pending_list = &sma->pending_const;
796	else
797	pending_list = &sma->sem_base[semnum].pending_const;
798
799	walk = pending_list->next;
800	while (walk != pending_list) {
801	int error;
802
803	q = container_of(walk, struct sem_queue, list);
804	walk = walk->next;
805
806	error = perform_atomic_semop(sma, q);
807
808	if (error <= 0) {
809	/* operation completed, remove from queue & wakeup */
810
811	unlink_queue(sma, q);
812
813	wake_up_sem_queue_prepare(pt, q, error);
814	if (error == 0)
815	semop_completed = 1;
816	}
817	}
818	return semop_completed;
819	}
820
821	/**
822	* do_smart_wakeup_zero - wakeup all wait for zero tasks
823	* @sma: semaphore array
824	* @sops: operations that were performed
825	* @nsops: number of operations
826	* @pt: list head of the tasks that must be woken up.
827	*
828	* Checks all required queue for wait-for-zero operations, based
829	* on the actual changes that were performed on the semaphore array.
830	* The function returns 1 if at least one operation was completed successfully.
831	*/
832	static int do_smart_wakeup_zero(struct sem_array sma, struct sembuf sops,
833	int nsops, struct list_head *pt)
834	{
835	int i;
836	int semop_completed = 0;
837	int got_zero = 0;
838
839	/* first: the per-semaphore queues, if known */
840	if (sops) {
841	for (i = 0; i < nsops; i++) {
842	int num = sops[i].sem_num;
843
844	if (sma->sem_base[num].semval == 0) {
845	got_zero = 1;
846	semop_completed \|= wake_const_ops(sma, num, pt);
847	}
848	}
849	} else {
850	/*
851	* No sops means modified semaphores not known.
852	* Assume all were changed.
853	*/
854	for (i = 0; i < sma->sem_nsems; i++) {
855	if (sma->sem_base[i].semval == 0) {
856	got_zero = 1;
857	semop_completed \|= wake_const_ops(sma, i, pt);
858	}
859	}
860	}
861	/*
862	* If one of the modified semaphores got 0,
863	* then check the global queue, too.
864	*/
865	if (got_zero)
866	semop_completed \|= wake_const_ops(sma, -1, pt);
867
868	return semop_completed;
869	}
870
871
872	/**
873	* update_queue - look for tasks that can be completed.
874	* @sma: semaphore array.
875	* @semnum: semaphore that was modified.
876	* @pt: list head for the tasks that must be woken up.
877	*
878	* update_queue must be called after a semaphore in a semaphore array
879	* was modified. If multiple semaphores were modified, update_queue must
880	* be called with semnum = -1, as well as with the number of each modified
881	* semaphore.
882	* The tasks that must be woken up are added to @pt. The return code
883	* is stored in q->pid.
884	* The function internally checks if const operations can now succeed.
885	*
886	* The function return 1 if at least one semop was completed successfully.
887	*/
888	static int update_queue(struct sem_array sma, int semnum, struct list_head pt)
889	{
890	struct sem_queue *q;
891	struct list_head *walk;
892	struct list_head *pending_list;
893	int semop_completed = 0;
894
895	if (semnum == -1)
896	pending_list = &sma->pending_alter;
897	else
898	pending_list = &sma->sem_base[semnum].pending_alter;
899
900	again:
901	walk = pending_list->next;
902	while (walk != pending_list) {
903	int error, restart;
904
905	q = container_of(walk, struct sem_queue, list);
906	walk = walk->next;
907
908	/* If we are scanning the single sop, per-semaphore list of
909	* one semaphore and that semaphore is 0, then it is not
910	* necessary to scan further: simple increments
911	* that affect only one entry succeed immediately and cannot
912	* be in the per semaphore pending queue, and decrements
913	* cannot be successful if the value is already 0.
914	*/
915	if (semnum != -1 && sma->sem_base[semnum].semval == 0)
916	break;
917
918	error = perform_atomic_semop(sma, q);
919
920	/* Does q->sleeper still need to sleep? */
921	if (error > 0)
922	continue;
923
924	unlink_queue(sma, q);
925
926	if (error) {
927	restart = 0;
928	} else {
929	semop_completed = 1;
930	do_smart_wakeup_zero(sma, q->sops, q->nsops, pt);
931	restart = check_restart(sma, q);
932	}
933
934	wake_up_sem_queue_prepare(pt, q, error);
935	if (restart)
936	goto again;
937	}
938	return semop_completed;
939	}
940
941	/**
942	* set_semotime - set sem_otime
943	* @sma: semaphore array
944	* @sops: operations that modified the array, may be NULL
945	*
946	* sem_otime is replicated to avoid cache line trashing.
947	* This function sets one instance to the current time.
948	*/
949	static void set_semotime(struct sem_array sma, struct sembuf sops)
950	{
951	if (sops == NULL) {
952	sma->sem_base[0].sem_otime = get_seconds();
953	} else {
954	sma->sem_base[sops[0].sem_num].sem_otime =
955	get_seconds();
956	}
957	}
958
959	/**
960	* do_smart_update - optimized update_queue
961	* @sma: semaphore array
962	* @sops: operations that were performed
963	* @nsops: number of operations
964	* @otime: force setting otime
965	* @pt: list head of the tasks that must be woken up.
966	*
967	* do_smart_update() does the required calls to update_queue and wakeup_zero,
968	* based on the actual changes that were performed on the semaphore array.
969	* Note that the function does not do the actual wake-up: the caller is
970	* responsible for calling wake_up_sem_queue_do(@pt).
971	* It is safe to perform this call after dropping all locks.
972	*/
973	static void do_smart_update(struct sem_array sma, struct sembuf sops, int nsops,
974	int otime, struct list_head *pt)
975	{
976	int i;
977
978	otime \|= do_smart_wakeup_zero(sma, sops, nsops, pt);
979
980	if (!list_empty(&sma->pending_alter)) {
981	/* semaphore array uses the global queue - just process it. */
982	otime \|= update_queue(sma, -1, pt);
983	} else {
984	if (!sops) {
985	/*
986	* No sops, thus the modified semaphores are not
987	* known. Check all.
988	*/
989	for (i = 0; i < sma->sem_nsems; i++)
990	otime \|= update_queue(sma, i, pt);
991	} else {
992	/*
993	* Check the semaphores that were increased:
994	* - No complex ops, thus all sleeping ops are
995	* decrease.
996	* - if we decreased the value, then any sleeping
997	* semaphore ops wont be able to run: If the
998	* previous value was too small, then the new
999	* value will be too small, too.
1000	*/
1001	for (i = 0; i < nsops; i++) {
1002	if (sops[i].sem_op > 0) {
1003	otime \|= update_queue(sma,
1004	sops[i].sem_num, pt);
1005	}
1006	}
1007	}
1008	}
1009	if (otime)
1010	set_semotime(sma, sops);
1011	}
1012
1013	/*
1014	* check_qop: Test if a queued operation sleeps on the semaphore semnum
1015	*/
1016	static int check_qop(struct sem_array sma, int semnum, struct sem_queue q,
1017	bool count_zero)
1018	{
1019	struct sembuf *sop = q->blocking;
1020
1021	/*
1022	* Linux always (since 0.99.10) reported a task as sleeping on all
1023	* semaphores. This violates SUS, therefore it was changed to the
1024	* standard compliant behavior.
1025	* Give the administrators a chance to notice that an application
1026	* might misbehave because it relies on the Linux behavior.
1027	*/
1028	pr_info_once("semctl(GETNCNT/GETZCNT) is since 3.16 Single Unix Specification compliant.\n"
1029	"The task %s (%d) triggered the difference, watch for misbehavior.\n",
1030	current->comm, task_pid_nr(current));
1031
1032	if (sop->sem_num != semnum)
1033	return 0;
1034
1035	if (count_zero && sop->sem_op == 0)
1036	return 1;
1037	if (!count_zero && sop->sem_op < 0)
1038	return 1;
1039
1040	return 0;
1041	}
1042
1043	/* The following counts are associated to each semaphore:
1044	* semncnt number of tasks waiting on semval being nonzero
1045	* semzcnt number of tasks waiting on semval being zero
1046	*
1047	* Per definition, a task waits only on the semaphore of the first semop
1048	* that cannot proceed, even if additional operation would block, too.
1049	*/
1050	static int count_semcnt(struct sem_array *sma, ushort semnum,
1051	bool count_zero)
1052	{
1053	struct list_head *l;
1054	struct sem_queue *q;
1055	int semcnt;
1056
1057	semcnt = 0;
1058	/* First: check the simple operations. They are easy to evaluate */
1059	if (count_zero)
1060	l = &sma->sem_base[semnum].pending_const;
1061	else
1062	l = &sma->sem_base[semnum].pending_alter;
1063
1064	list_for_each_entry(q, l, list) {
1065	/* all task on a per-semaphore list sleep on exactly
1066	* that semaphore
1067	*/
1068	semcnt++;
1069	}
1070
1071	/* Then: check the complex operations. */
1072	list_for_each_entry(q, &sma->pending_alter, list) {
1073	semcnt += check_qop(sma, semnum, q, count_zero);
1074	}
1075	if (count_zero) {
1076	list_for_each_entry(q, &sma->pending_const, list) {
1077	semcnt += check_qop(sma, semnum, q, count_zero);
1078	}
1079	}
1080	return semcnt;
1081	}
1082
1083	/* Free a semaphore set. freeary() is called with sem_ids.rwsem locked
1084	* as a writer and the spinlock for this semaphore set hold. sem_ids.rwsem
1085	* remains locked on exit.
1086	*/
1087	static void freeary(struct ipc_namespace ns, struct kern_ipc_perm ipcp)
1088	{
1089	struct sem_undo un, tu;
1090	struct sem_queue q, tq;
1091	struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm);
1092	struct list_head tasks;
1093	int i;
1094
1095	/* Free the existing undo structures for this semaphore set. */
1096	ipc_assert_locked_object(&sma->sem_perm);
1097	list_for_each_entry_safe(un, tu, &sma->list_id, list_id) {
1098	list_del(&un->list_id);
1099	spin_lock(&un->ulp->lock);
1100	un->semid = -1;
1101	list_del_rcu(&un->list_proc);
1102	spin_unlock(&un->ulp->lock);
1103	kfree_rcu(un, rcu);
1104	}
1105
1106	/* Wake up all pending processes and let them fail with EIDRM. */
1107	INIT_LIST_HEAD(&tasks);
1108	list_for_each_entry_safe(q, tq, &sma->pending_const, list) {
1109	unlink_queue(sma, q);
1110	wake_up_sem_queue_prepare(&tasks, q, -EIDRM);
1111	}
1112
1113	list_for_each_entry_safe(q, tq, &sma->pending_alter, list) {
1114	unlink_queue(sma, q);
1115	wake_up_sem_queue_prepare(&tasks, q, -EIDRM);
1116	}
1117	for (i = 0; i < sma->sem_nsems; i++) {
1118	struct sem *sem = sma->sem_base + i;
1119	list_for_each_entry_safe(q, tq, &sem->pending_const, list) {
1120	unlink_queue(sma, q);
1121	wake_up_sem_queue_prepare(&tasks, q, -EIDRM);
1122	}
1123	list_for_each_entry_safe(q, tq, &sem->pending_alter, list) {
1124	unlink_queue(sma, q);
1125	wake_up_sem_queue_prepare(&tasks, q, -EIDRM);
1126	}
1127	}
1128
1129	/* Remove the semaphore set from the IDR */
1130	sem_rmid(ns, sma);
1131	sem_unlock(sma, -1);
1132	rcu_read_unlock();
1133
1134	wake_up_sem_queue_do(&tasks);
1135	ns->used_sems -= sma->sem_nsems;
1136	ipc_rcu_putref(sma, sem_rcu_free);
1137	}
1138
1139	static unsigned long copy_semid_to_user(void __user buf, struct semid64_ds in, int version)
1140	{
1141	switch (version) {
1142	case IPC_64:
1143	return copy_to_user(buf, in, sizeof(*in));
1144	case IPC_OLD:
1145	{
1146	struct semid_ds out;
1147
1148	memset(&out, 0, sizeof(out));
1149
1150	ipc64_perm_to_ipc_perm(&in->sem_perm, &out.sem_perm);
1151
1152	out.sem_otime = in->sem_otime;
1153	out.sem_ctime = in->sem_ctime;
1154	out.sem_nsems = in->sem_nsems;
1155
1156	return copy_to_user(buf, &out, sizeof(out));
1157	}
1158	default:
1159	return -EINVAL;
1160	}
1161	}
1162
1163	static time_t get_semotime(struct sem_array *sma)
1164	{
1165	int i;
1166	time_t res;
1167
1168	res = sma->sem_base[0].sem_otime;
1169	for (i = 1; i < sma->sem_nsems; i++) {
1170	time_t to = sma->sem_base[i].sem_otime;
1171
1172	if (to > res)
1173	res = to;
1174	}
1175	return res;
1176	}
1177
1178	static int semctl_nolock(struct ipc_namespace *ns, int semid,
1179	int cmd, int version, void __user *p)
1180	{
1181	int err;
1182	struct sem_array *sma;
1183
1184	switch (cmd) {
1185	case IPC_INFO:
1186	case SEM_INFO:
1187	{
1188	struct seminfo seminfo;
1189	int max_id;
1190
1191	err = security_sem_semctl(NULL, cmd);
1192	if (err)
1193	return err;
1194
1195	memset(&seminfo, 0, sizeof(seminfo));
1196	seminfo.semmni = ns->sc_semmni;
1197	seminfo.semmns = ns->sc_semmns;
1198	seminfo.semmsl = ns->sc_semmsl;
1199	seminfo.semopm = ns->sc_semopm;
1200	seminfo.semvmx = SEMVMX;
1201	seminfo.semmnu = SEMMNU;
1202	seminfo.semmap = SEMMAP;
1203	seminfo.semume = SEMUME;
1204	down_read(&sem_ids(ns).rwsem);
1205	if (cmd == SEM_INFO) {
1206	seminfo.semusz = sem_ids(ns).in_use;
1207	seminfo.semaem = ns->used_sems;
1208	} else {
1209	seminfo.semusz = SEMUSZ;
1210	seminfo.semaem = SEMAEM;
1211	}
1212	max_id = ipc_get_maxid(&sem_ids(ns));
1213	up_read(&sem_ids(ns).rwsem);
1214	if (copy_to_user(p, &seminfo, sizeof(struct seminfo)))
1215	return -EFAULT;
1216	return (max_id < 0) ? 0 : max_id;
1217	}
1218	case IPC_STAT:
1219	case SEM_STAT:
1220	{
1221	struct semid64_ds tbuf;
1222	int id = 0;
1223
1224	memset(&tbuf, 0, sizeof(tbuf));
1225
1226	rcu_read_lock();
1227	if (cmd == SEM_STAT) {
1228	sma = sem_obtain_object(ns, semid);
1229	if (IS_ERR(sma)) {
1230	err = PTR_ERR(sma);
1231	goto out_unlock;
1232	}
1233	id = sma->sem_perm.id;
1234	} else {
1235	sma = sem_obtain_object_check(ns, semid);
1236	if (IS_ERR(sma)) {
1237	err = PTR_ERR(sma);
1238	goto out_unlock;
1239	}
1240	}
1241
1242	err = -EACCES;
1243	if (ipcperms(ns, &sma->sem_perm, S_IRUGO))
1244	goto out_unlock;
1245
1246	err = security_sem_semctl(sma, cmd);
1247	if (err)
1248	goto out_unlock;
1249
1250	kernel_to_ipc64_perm(&sma->sem_perm, &tbuf.sem_perm);
1251	tbuf.sem_otime = get_semotime(sma);
1252	tbuf.sem_ctime = sma->sem_ctime;
1253	tbuf.sem_nsems = sma->sem_nsems;
1254	rcu_read_unlock();
1255	if (copy_semid_to_user(p, &tbuf, version))
1256	return -EFAULT;
1257	return id;
1258	}
1259	default:
1260	return -EINVAL;
1261	}
1262	out_unlock:
1263	rcu_read_unlock();
1264	return err;
1265	}
1266
1267	static int semctl_setval(struct ipc_namespace *ns, int semid, int semnum,
1268	unsigned long arg)
1269	{
1270	struct sem_undo *un;
1271	struct sem_array *sma;
1272	struct sem *curr;
1273	int err;
1274	struct list_head tasks;
1275	int val;
1276	#if defined(CONFIG_64BIT) && defined(__BIG_ENDIAN)
1277	/* big-endian 64bit */
1278	val = arg >> 32;
1279	#else
1280	/* 32bit or little-endian 64bit */
1281	val = arg;
1282	#endif
1283
1284	if (val > SEMVMX \|\| val < 0)
1285	return -ERANGE;
1286
1287	INIT_LIST_HEAD(&tasks);
1288
1289	rcu_read_lock();
1290	sma = sem_obtain_object_check(ns, semid);
1291	if (IS_ERR(sma)) {
1292	rcu_read_unlock();
1293	return PTR_ERR(sma);
1294	}
1295
1296	if (semnum < 0 \|\| semnum >= sma->sem_nsems) {
1297	rcu_read_unlock();
1298	return -EINVAL;
1299	}
1300
1301
1302	if (ipcperms(ns, &sma->sem_perm, S_IWUGO)) {
1303	rcu_read_unlock();
1304	return -EACCES;
1305	}
1306
1307	err = security_sem_semctl(sma, SETVAL);
1308	if (err) {
1309	rcu_read_unlock();
1310	return -EACCES;
1311	}
1312
1313	sem_lock(sma, NULL, -1);
1314
1315	if (!ipc_valid_object(&sma->sem_perm)) {
1316	sem_unlock(sma, -1);
1317	rcu_read_unlock();
1318	return -EIDRM;
1319	}
1320
1321	curr = &sma->sem_base[semnum];
1322
1323	ipc_assert_locked_object(&sma->sem_perm);
1324	list_for_each_entry(un, &sma->list_id, list_id)
1325	un->semadj[semnum] = 0;
1326
1327	curr->semval = val;
1328	curr->sempid = task_tgid_vnr(current);
1329	sma->sem_ctime = get_seconds();
1330	/* maybe some queued-up processes were waiting for this */
1331	do_smart_update(sma, NULL, 0, 0, &tasks);
1332	sem_unlock(sma, -1);
1333	rcu_read_unlock();
1334	wake_up_sem_queue_do(&tasks);
1335	return 0;
1336	}
1337
1338	static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
1339	int cmd, void __user *p)
1340	{
1341	struct sem_array *sma;
1342	struct sem *curr;
1343	int err, nsems;
1344	ushort fast_sem_io[SEMMSL_FAST];
1345	ushort *sem_io = fast_sem_io;
1346	struct list_head tasks;
1347
1348	INIT_LIST_HEAD(&tasks);
1349
1350	rcu_read_lock();
1351	sma = sem_obtain_object_check(ns, semid);
1352	if (IS_ERR(sma)) {
1353	rcu_read_unlock();
1354	return PTR_ERR(sma);
1355	}
1356
1357	nsems = sma->sem_nsems;
1358
1359	err = -EACCES;
1360	if (ipcperms(ns, &sma->sem_perm, cmd == SETALL ? S_IWUGO : S_IRUGO))
1361	goto out_rcu_wakeup;
1362
1363	err = security_sem_semctl(sma, cmd);
1364	if (err)
1365	goto out_rcu_wakeup;
1366
1367	err = -EACCES;
1368	switch (cmd) {
1369	case GETALL:
1370	{
1371	ushort __user *array = p;
1372	int i;
1373
1374	sem_lock(sma, NULL, -1);
1375	if (!ipc_valid_object(&sma->sem_perm)) {
1376	err = -EIDRM;
1377	goto out_unlock;
1378	}
1379	if (nsems > SEMMSL_FAST) {
1380	if (!ipc_rcu_getref(sma)) {
1381	err = -EIDRM;
1382	goto out_unlock;
1383	}
1384	sem_unlock(sma, -1);
1385	rcu_read_unlock();
1386	sem_io = ipc_alloc(sizeof(ushort)*nsems);
1387	if (sem_io == NULL) {
1388	ipc_rcu_putref(sma, ipc_rcu_free);
1389	return -ENOMEM;
1390	}
1391
1392	rcu_read_lock();
1393	sem_lock_and_putref(sma);
1394	if (!ipc_valid_object(&sma->sem_perm)) {
1395	err = -EIDRM;
1396	goto out_unlock;
1397	}
1398	}
1399	for (i = 0; i < sma->sem_nsems; i++)
1400	sem_io[i] = sma->sem_base[i].semval;
1401	sem_unlock(sma, -1);
1402	rcu_read_unlock();
1403	err = 0;
1404	if (copy_to_user(array, sem_io, nsems*sizeof(ushort)))
1405	err = -EFAULT;
1406	goto out_free;
1407	}
1408	case SETALL:
1409	{
1410	int i;
1411	struct sem_undo *un;
1412
1413	if (!ipc_rcu_getref(sma)) {
1414	err = -EIDRM;
1415	goto out_rcu_wakeup;
1416	}
1417	rcu_read_unlock();
1418
1419	if (nsems > SEMMSL_FAST) {
1420	sem_io = ipc_alloc(sizeof(ushort)*nsems);
1421	if (sem_io == NULL) {
1422	ipc_rcu_putref(sma, ipc_rcu_free);
1423	return -ENOMEM;
1424	}
1425	}
1426
1427	if (copy_from_user(sem_io, p, nsems*sizeof(ushort))) {
1428	ipc_rcu_putref(sma, ipc_rcu_free);
1429	err = -EFAULT;
1430	goto out_free;
1431	}
1432
1433	for (i = 0; i < nsems; i++) {
1434	if (sem_io[i] > SEMVMX) {
1435	ipc_rcu_putref(sma, ipc_rcu_free);
1436	err = -ERANGE;
1437	goto out_free;
1438	}
1439	}
1440	rcu_read_lock();
1441	sem_lock_and_putref(sma);
1442	if (!ipc_valid_object(&sma->sem_perm)) {
1443	err = -EIDRM;
1444	goto out_unlock;
1445	}
1446
1447	for (i = 0; i < nsems; i++)
1448	sma->sem_base[i].semval = sem_io[i];
1449
1450	ipc_assert_locked_object(&sma->sem_perm);
1451	list_for_each_entry(un, &sma->list_id, list_id) {
1452	for (i = 0; i < nsems; i++)
1453	un->semadj[i] = 0;
1454	}
1455	sma->sem_ctime = get_seconds();
1456	/* maybe some queued-up processes were waiting for this */
1457	do_smart_update(sma, NULL, 0, 0, &tasks);
1458	err = 0;
1459	goto out_unlock;
1460	}
1461	/* GETVAL, GETPID, GETNCTN, GETZCNT: fall-through */
1462	}
1463	err = -EINVAL;
1464	if (semnum < 0 \|\| semnum >= nsems)
1465	goto out_rcu_wakeup;
1466
1467	sem_lock(sma, NULL, -1);
1468	if (!ipc_valid_object(&sma->sem_perm)) {
1469	err = -EIDRM;
1470	goto out_unlock;
1471	}
1472	curr = &sma->sem_base[semnum];
1473
1474	switch (cmd) {
1475	case GETVAL:
1476	err = curr->semval;
1477	goto out_unlock;
1478	case GETPID:
1479	err = curr->sempid;
1480	goto out_unlock;
1481	case GETNCNT:
1482	err = count_semcnt(sma, semnum, 0);
1483	goto out_unlock;
1484	case GETZCNT:
1485	err = count_semcnt(sma, semnum, 1);
1486	goto out_unlock;
1487	}
1488
1489	out_unlock:
1490	sem_unlock(sma, -1);
1491	out_rcu_wakeup:
1492	rcu_read_unlock();
1493	wake_up_sem_queue_do(&tasks);
1494	out_free:
1495	if (sem_io != fast_sem_io)
1496	ipc_free(sem_io, sizeof(ushort)*nsems);
1497	return err;
1498	}
1499
1500	static inline unsigned long
1501	copy_semid_from_user(struct semid64_ds out, void __user buf, int version)
1502	{
1503	switch (version) {
1504	case IPC_64:
1505	if (copy_from_user(out, buf, sizeof(*out)))
1506	return -EFAULT;
1507	return 0;
1508	case IPC_OLD:
1509	{
1510	struct semid_ds tbuf_old;
1511
1512	if (copy_from_user(&tbuf_old, buf, sizeof(tbuf_old)))
1513	return -EFAULT;
1514
1515	out->sem_perm.uid = tbuf_old.sem_perm.uid;
1516	out->sem_perm.gid = tbuf_old.sem_perm.gid;
1517	out->sem_perm.mode = tbuf_old.sem_perm.mode;
1518
1519	return 0;
1520	}
1521	default:
1522	return -EINVAL;
1523	}
1524	}
1525
1526	/*
1527	* This function handles some semctl commands which require the rwsem
1528	* to be held in write mode.
1529	* NOTE: no locks must be held, the rwsem is taken inside this function.
1530	*/
1531	static int semctl_down(struct ipc_namespace *ns, int semid,
1532	int cmd, int version, void __user *p)
1533	{
1534	struct sem_array *sma;
1535	int err;
1536	struct semid64_ds semid64;
1537	struct kern_ipc_perm *ipcp;
1538
1539	if (cmd == IPC_SET) {
1540	if (copy_semid_from_user(&semid64, p, version))
1541	return -EFAULT;
1542	}
1543
1544	down_write(&sem_ids(ns).rwsem);
1545	rcu_read_lock();
1546
1547	ipcp = ipcctl_pre_down_nolock(ns, &sem_ids(ns), semid, cmd,
1548	&semid64.sem_perm, 0);
1549	if (IS_ERR(ipcp)) {
1550	err = PTR_ERR(ipcp);
1551	goto out_unlock1;
1552	}
1553
1554	sma = container_of(ipcp, struct sem_array, sem_perm);
1555
1556	err = security_sem_semctl(sma, cmd);
1557	if (err)
1558	goto out_unlock1;
1559
1560	switch (cmd) {
1561	case IPC_RMID:
1562	sem_lock(sma, NULL, -1);
1563	/* freeary unlocks the ipc object and rcu */
1564	freeary(ns, ipcp);
1565	goto out_up;
1566	case IPC_SET:
1567	sem_lock(sma, NULL, -1);
1568	err = ipc_update_perm(&semid64.sem_perm, ipcp);
1569	if (err)
1570	goto out_unlock0;
1571	sma->sem_ctime = get_seconds();
1572	break;
1573	default:
1574	err = -EINVAL;
1575	goto out_unlock1;
1576	}
1577
1578	out_unlock0:
1579	sem_unlock(sma, -1);
1580	out_unlock1:
1581	rcu_read_unlock();
1582	out_up:
1583	up_write(&sem_ids(ns).rwsem);
1584	return err;
1585	}
1586
1587	SYSCALL_DEFINE4(semctl, int, semid, int, semnum, int, cmd, unsigned long, arg)
1588	{
1589	int version;
1590	struct ipc_namespace *ns;
1591	void __user p = (void __user )arg;
1592
1593	if (semid < 0)
1594	return -EINVAL;
1595
1596	version = ipc_parse_version(&cmd);
1597	ns = current->nsproxy->ipc_ns;
1598
1599	switch (cmd) {
1600	case IPC_INFO:
1601	case SEM_INFO:
1602	case IPC_STAT:
1603	case SEM_STAT:
1604	return semctl_nolock(ns, semid, cmd, version, p);
1605	case GETALL:
1606	case GETVAL:
1607	case GETPID:
1608	case GETNCNT:
1609	case GETZCNT:
1610	case SETALL:
1611	return semctl_main(ns, semid, semnum, cmd, p);
1612	case SETVAL:
1613	return semctl_setval(ns, semid, semnum, arg);
1614	case IPC_RMID:
1615	case IPC_SET:
1616	return semctl_down(ns, semid, cmd, version, p);
1617	default:
1618	return -EINVAL;
1619	}
1620	}
1621
1622	/* If the task doesn't already have a undo_list, then allocate one
1623	* here. We guarantee there is only one thread using this undo list,
1624	* and current is THE ONE
1625	*
1626	* If this allocation and assignment succeeds, but later
1627	* portions of this code fail, there is no need to free the sem_undo_list.
1628	* Just let it stay associated with the task, and it'll be freed later
1629	* at exit time.
1630	*
1631	* This can block, so callers must hold no locks.
1632	*/
1633	static inline int get_undo_list(struct sem_undo_list **undo_listp)
1634	{
1635	struct sem_undo_list *undo_list;
1636
1637	undo_list = current->sysvsem.undo_list;
1638	if (!undo_list) {
1639	undo_list = kzalloc(sizeof(*undo_list), GFP_KERNEL);
1640	if (undo_list == NULL)
1641	return -ENOMEM;
1642	spin_lock_init(&undo_list->lock);
1643	atomic_set(&undo_list->refcnt, 1);
1644	INIT_LIST_HEAD(&undo_list->list_proc);
1645
1646	current->sysvsem.undo_list = undo_list;
1647	}
1648	*undo_listp = undo_list;
1649	return 0;
1650	}
1651
1652	static struct sem_undo __lookup_undo(struct sem_undo_list ulp, int semid)
1653	{
1654	struct sem_undo *un;
1655
1656	list_for_each_entry_rcu(un, &ulp->list_proc, list_proc) {
1657	if (un->semid == semid)
1658	return un;
1659	}
1660	return NULL;
1661	}
1662
1663	static struct sem_undo lookup_undo(struct sem_undo_list ulp, int semid)
1664	{
1665	struct sem_undo *un;
1666
1667	assert_spin_locked(&ulp->lock);
1668
1669	un = __lookup_undo(ulp, semid);
1670	if (un) {
1671	list_del_rcu(&un->list_proc);
1672	list_add_rcu(&un->list_proc, &ulp->list_proc);
1673	}
1674	return un;
1675	}
1676
1677	/**
1678	* find_alloc_undo - lookup (and if not present create) undo array
1679	* @ns: namespace
1680	* @semid: semaphore array id
1681	*
1682	* The function looks up (and if not present creates) the undo structure.
1683	* The size of the undo structure depends on the size of the semaphore
1684	* array, thus the alloc path is not that straightforward.
1685	* Lifetime-rules: sem_undo is rcu-protected, on success, the function
1686	* performs a rcu_read_lock().
1687	*/
1688	static struct sem_undo find_alloc_undo(struct ipc_namespace ns, int semid)
1689	{
1690	struct sem_array *sma;
1691	struct sem_undo_list *ulp;
1692	struct sem_undo un, new;
1693	int nsems, error;
1694
1695	error = get_undo_list(&ulp);
1696	if (error)
1697	return ERR_PTR(error);
1698
1699	rcu_read_lock();
1700	spin_lock(&ulp->lock);
1701	un = lookup_undo(ulp, semid);
1702	spin_unlock(&ulp->lock);
1703	if (likely(un != NULL))
1704	goto out;
1705
1706	/* no undo structure around - allocate one. */
1707	/* step 1: figure out the size of the semaphore array */
1708	sma = sem_obtain_object_check(ns, semid);
1709	if (IS_ERR(sma)) {
1710	rcu_read_unlock();
1711	return ERR_CAST(sma);
1712	}
1713
1714	nsems = sma->sem_nsems;
1715	if (!ipc_rcu_getref(sma)) {
1716	rcu_read_unlock();
1717	un = ERR_PTR(-EIDRM);
1718	goto out;
1719	}
1720	rcu_read_unlock();
1721
1722	/* step 2: allocate new undo structure */
1723	new = kzalloc(sizeof(struct sem_undo) + sizeof(short)*nsems, GFP_KERNEL);
1724	if (!new) {
1725	ipc_rcu_putref(sma, ipc_rcu_free);
1726	return ERR_PTR(-ENOMEM);
1727	}
1728
1729	/* step 3: Acquire the lock on semaphore array */
1730	rcu_read_lock();
1731	sem_lock_and_putref(sma);
1732	if (!ipc_valid_object(&sma->sem_perm)) {
1733	sem_unlock(sma, -1);
1734	rcu_read_unlock();
1735	kfree(new);
1736	un = ERR_PTR(-EIDRM);
1737	goto out;
1738	}
1739	spin_lock(&ulp->lock);
1740
1741	/*
1742	* step 4: check for races: did someone else allocate the undo struct?
1743	*/
1744	un = lookup_undo(ulp, semid);
1745	if (un) {
1746	kfree(new);
1747	goto success;
1748	}
1749	/* step 5: initialize & link new undo structure */
1750	new->semadj = (short *) &new[1];
1751	new->ulp = ulp;
1752	new->semid = semid;
1753	assert_spin_locked(&ulp->lock);
1754	list_add_rcu(&new->list_proc, &ulp->list_proc);
1755	ipc_assert_locked_object(&sma->sem_perm);
1756	list_add(&new->list_id, &sma->list_id);
1757	un = new;
1758
1759	success:
1760	spin_unlock(&ulp->lock);
1761	sem_unlock(sma, -1);
1762	out:
1763	return un;
1764	}
1765
1766
1767	/**
1768	* get_queue_result - retrieve the result code from sem_queue
1769	* @q: Pointer to queue structure
1770	*
1771	* Retrieve the return code from the pending queue. If IN_WAKEUP is found in
1772	* q->status, then we must loop until the value is replaced with the final
1773	* value: This may happen if a task is woken up by an unrelated event (e.g.
1774	* signal) and in parallel the task is woken up by another task because it got
1775	* the requested semaphores.
1776	*
1777	* The function can be called with or without holding the semaphore spinlock.
1778	*/
1779	static int get_queue_result(struct sem_queue *q)
1780	{
1781	int error;
1782
1783	error = q->status;
1784	while (unlikely(error == IN_WAKEUP)) {
1785	cpu_relax();
1786	error = q->status;
1787	}
1788
1789	return error;
1790	}
1791
1792	SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
1793	unsigned, nsops, const struct timespec __user *, timeout)
1794	{
1795	int error = -EINVAL;
1796	struct sem_array *sma;
1797	struct sembuf fast_sops[SEMOPM_FAST];
1798	struct sembuf sops = fast_sops, sop;
1799	struct sem_undo *un;
1800	int undos = 0, alter = 0, max, locknum;
1801	struct sem_queue queue;
1802	unsigned long jiffies_left = 0;
1803	struct ipc_namespace *ns;
1804	struct list_head tasks;
1805
1806	ns = current->nsproxy->ipc_ns;
1807
1808	if (nsops < 1 \|\| semid < 0)
1809	return -EINVAL;
1810	if (nsops > ns->sc_semopm)
1811	return -E2BIG;
1812	if (nsops > SEMOPM_FAST) {
1813	sops = kmalloc(sizeof(sops)nsops, GFP_KERNEL);
1814	if (sops == NULL)
1815	return -ENOMEM;
1816	}
1817	if (copy_from_user(sops, tsops, nsops * sizeof(*tsops))) {
1818	error = -EFAULT;
1819	goto out_free;
1820	}
1821	if (timeout) {
1822	struct timespec _timeout;
1823	if (copy_from_user(&_timeout, timeout, sizeof(*timeout))) {
1824	error = -EFAULT;
1825	goto out_free;
1826	}
1827	if (_timeout.tv_sec < 0 \|\| _timeout.tv_nsec < 0 \|\|
1828	_timeout.tv_nsec >= 1000000000L) {
1829	error = -EINVAL;
1830	goto out_free;
1831	}
1832	jiffies_left = timespec_to_jiffies(&_timeout);
1833	}
1834	max = 0;
1835	for (sop = sops; sop < sops + nsops; sop++) {
1836	if (sop->sem_num >= max)
1837	max = sop->sem_num;
1838	if (sop->sem_flg & SEM_UNDO)
1839	undos = 1;
1840	if (sop->sem_op != 0)
1841	alter = 1;
1842	}
1843
1844	INIT_LIST_HEAD(&tasks);
1845
1846	if (undos) {
1847	/* On success, find_alloc_undo takes the rcu_read_lock */
1848	un = find_alloc_undo(ns, semid);
1849	if (IS_ERR(un)) {
1850	error = PTR_ERR(un);
1851	goto out_free;
1852	}
1853	} else {
1854	un = NULL;
1855	rcu_read_lock();
1856	}
1857
1858	sma = sem_obtain_object_check(ns, semid);
1859	if (IS_ERR(sma)) {
1860	rcu_read_unlock();
1861	error = PTR_ERR(sma);
1862	goto out_free;
1863	}
1864
1865	error = -EFBIG;
1866	if (max >= sma->sem_nsems)
1867	goto out_rcu_wakeup;
1868
1869	error = -EACCES;
1870	if (ipcperms(ns, &sma->sem_perm, alter ? S_IWUGO : S_IRUGO))
1871	goto out_rcu_wakeup;
1872
1873	error = security_sem_semop(sma, sops, nsops, alter);
1874	if (error)
1875	goto out_rcu_wakeup;
1876
1877	error = -EIDRM;
1878	locknum = sem_lock(sma, sops, nsops);
1879	/*
1880	* We eventually might perform the following check in a lockless
1881	* fashion, considering ipc_valid_object() locking constraints.
1882	* If nsops == 1 and there is no contention for sem_perm.lock, then
1883	* only a per-semaphore lock is held and it's OK to proceed with the
1884	* check below. More details on the fine grained locking scheme
1885	* entangled here and why it's RMID race safe on comments at sem_lock()
1886	*/
1887	if (!ipc_valid_object(&sma->sem_perm))
1888	goto out_unlock_free;
1889	/*
1890	* semid identifiers are not unique - find_alloc_undo may have
1891	* allocated an undo structure, it was invalidated by an RMID
1892	* and now a new array with received the same id. Check and fail.
1893	* This case can be detected checking un->semid. The existence of
1894	* "un" itself is guaranteed by rcu.
1895	*/
1896	if (un && un->semid == -1)
1897	goto out_unlock_free;
1898
1899	queue.sops = sops;
1900	queue.nsops = nsops;
1901	queue.undo = un;
1902	queue.pid = task_tgid_vnr(current);
1903	queue.alter = alter;
1904
1905	error = perform_atomic_semop(sma, &queue);
1906	if (error == 0) {
1907	/* If the operation was successful, then do
1908	* the required updates.
1909	*/
1910	if (alter)
1911	do_smart_update(sma, sops, nsops, 1, &tasks);
1912	else
1913	set_semotime(sma, sops);
1914	}
1915	if (error <= 0)
1916	goto out_unlock_free;
1917
1918	/* We need to sleep on this operation, so we put the current
1919	* task into the pending queue and go to sleep.
1920	*/
1921
1922	if (nsops == 1) {
1923	struct sem *curr;
1924	curr = &sma->sem_base[sops->sem_num];
1925
1926	if (alter) {
1927	if (sma->complex_count) {
1928	list_add_tail(&queue.list,
1929	&sma->pending_alter);
1930	} else {
1931
1932	list_add_tail(&queue.list,
1933	&curr->pending_alter);
1934	}
1935	} else {
1936	list_add_tail(&queue.list, &curr->pending_const);
1937	}
1938	} else {
1939	if (!sma->complex_count)
1940	merge_queues(sma);
1941
1942	if (alter)
1943	list_add_tail(&queue.list, &sma->pending_alter);
1944	else
1945	list_add_tail(&queue.list, &sma->pending_const);
1946
1947	sma->complex_count++;
1948	}
1949
1950	queue.status = -EINTR;
1951	queue.sleeper = current;
1952
1953	sleep_again:
1954	__set_current_state(TASK_INTERRUPTIBLE);
1955	sem_unlock(sma, locknum);
1956	rcu_read_unlock();
1957
1958	if (timeout)
1959	jiffies_left = schedule_timeout(jiffies_left);
1960	else
1961	schedule();
1962
1963	error = get_queue_result(&queue);
1964
1965	if (error != -EINTR) {
1966	/* fast path: update_queue already obtained all requested
1967	* resources.
1968	* Perform a smp_mb(): User space could assume that semop()
1969	* is a memory barrier: Without the mb(), the cpu could
1970	* speculatively read in user space stale data that was
1971	* overwritten by the previous owner of the semaphore.
1972	*/
1973	smp_mb();
1974
1975	goto out_free;
1976	}
1977
1978	rcu_read_lock();
1979	sma = sem_obtain_lock(ns, semid, sops, nsops, &locknum);
1980
1981	/*
1982	* Wait until it's guaranteed that no wakeup_sem_queue_do() is ongoing.
1983	*/
1984	error = get_queue_result(&queue);
1985
1986	/*
1987	* Array removed? If yes, leave without sem_unlock().
1988	*/
1989	if (IS_ERR(sma)) {
1990	rcu_read_unlock();
1991	goto out_free;
1992	}
1993
1994
1995	/*
1996	* If queue.status != -EINTR we are woken up by another process.
1997	* Leave without unlink_queue(), but with sem_unlock().
1998	*/
1999	if (error != -EINTR)
2000	goto out_unlock_free;
2001
2002	/*
2003	* If an interrupt occurred we have to clean up the queue
2004	*/
2005	if (timeout && jiffies_left == 0)
2006	error = -EAGAIN;
2007
2008	/*
2009	* If the wakeup was spurious, just retry
2010	*/
2011	if (error == -EINTR && !signal_pending(current))
2012	goto sleep_again;
2013
2014	unlink_queue(sma, &queue);
2015
2016	out_unlock_free:
2017	sem_unlock(sma, locknum);
2018	out_rcu_wakeup:
2019	rcu_read_unlock();
2020	wake_up_sem_queue_do(&tasks);
2021	out_free:
2022	if (sops != fast_sops)
2023	kfree(sops);
2024	return error;
2025	}
2026
2027	SYSCALL_DEFINE3(semop, int, semid, struct sembuf __user *, tsops,
2028	unsigned, nsops)
2029	{
2030	return sys_semtimedop(semid, tsops, nsops, NULL);
2031	}
2032
2033	/* If CLONE_SYSVSEM is set, establish sharing of SEM_UNDO state between
2034	* parent and child tasks.
2035	*/
2036
2037	int copy_semundo(unsigned long clone_flags, struct task_struct *tsk)
2038	{
2039	struct sem_undo_list *undo_list;
2040	int error;
2041
2042	if (clone_flags & CLONE_SYSVSEM) {
2043	error = get_undo_list(&undo_list);
2044	if (error)
2045	return error;
2046	atomic_inc(&undo_list->refcnt);
2047	tsk->sysvsem.undo_list = undo_list;
2048	} else
2049	tsk->sysvsem.undo_list = NULL;
2050
2051	return 0;
2052	}
2053
2054	/*
2055	* add semadj values to semaphores, free undo structures.
2056	* undo structures are not freed when semaphore arrays are destroyed
2057	* so some of them may be out of date.
2058	* IMPLEMENTATION NOTE: There is some confusion over whether the
2059	* set of adjustments that needs to be done should be done in an atomic
2060	* manner or not. That is, if we are attempting to decrement the semval
2061	* should we queue up and wait until we can do so legally?
2062	* The original implementation attempted to do this (queue and wait).
2063	* The current implementation does not do so. The POSIX standard
2064	* and SVID should be consulted to determine what behavior is mandated.
2065	*/
2066	void exit_sem(struct task_struct *tsk)
2067	{
2068	struct sem_undo_list *ulp;
2069
2070	ulp = tsk->sysvsem.undo_list;
2071	if (!ulp)
2072	return;
2073	tsk->sysvsem.undo_list = NULL;
2074
2075	if (!atomic_dec_and_test(&ulp->refcnt))
2076	return;
2077
2078	for (;;) {
2079	struct sem_array *sma;
2080	struct sem_undo *un;
2081	struct list_head tasks;
2082	int semid, i;
2083
2084	rcu_read_lock();
2085	un = list_entry_rcu(ulp->list_proc.next,
2086	struct sem_undo, list_proc);
2087	if (&un->list_proc == &ulp->list_proc) {
2088	/*
2089	* We must wait for freeary() before freeing this ulp,
2090	* in case we raced with last sem_undo. There is a small
2091	* possibility where we exit while freeary() didn't
2092	* finish unlocking sem_undo_list.
2093	*/
2094	spin_unlock_wait(&ulp->lock);
2095	rcu_read_unlock();
2096	break;
2097	}
2098	spin_lock(&ulp->lock);
2099	semid = un->semid;
2100	spin_unlock(&ulp->lock);
2101
2102	/* exit_sem raced with IPC_RMID, nothing to do */
2103	if (semid == -1) {
2104	rcu_read_unlock();
2105	continue;
2106	}
2107
2108	sma = sem_obtain_object_check(tsk->nsproxy->ipc_ns, semid);
2109	/* exit_sem raced with IPC_RMID, nothing to do */
2110	if (IS_ERR(sma)) {
2111	rcu_read_unlock();
2112	continue;
2113	}
2114
2115	sem_lock(sma, NULL, -1);
2116	/* exit_sem raced with IPC_RMID, nothing to do */
2117	if (!ipc_valid_object(&sma->sem_perm)) {
2118	sem_unlock(sma, -1);
2119	rcu_read_unlock();
2120	continue;
2121	}
2122	un = __lookup_undo(ulp, semid);
2123	if (un == NULL) {
2124	/* exit_sem raced with IPC_RMID+semget() that created
2125	* exactly the same semid. Nothing to do.
2126	*/
2127	sem_unlock(sma, -1);
2128	rcu_read_unlock();
2129	continue;
2130	}
2131
2132	/* remove un from the linked lists */
2133	ipc_assert_locked_object(&sma->sem_perm);
2134	list_del(&un->list_id);
2135
2136	/* we are the last process using this ulp, acquiring ulp->lock
2137	* isn't required. Besides that, we are also protected against
2138	* IPC_RMID as we hold sma->sem_perm lock now
2139	*/
2140	list_del_rcu(&un->list_proc);
2141
2142	/* perform adjustments registered in un */
2143	for (i = 0; i < sma->sem_nsems; i++) {
2144	struct sem *semaphore = &sma->sem_base[i];
2145	if (un->semadj[i]) {
2146	semaphore->semval += un->semadj[i];
2147	/*
2148	* Range checks of the new semaphore value,
2149	* not defined by sus:
2150	* - Some unices ignore the undo entirely
2151	* (e.g. HP UX 11i 11.22, Tru64 V5.1)
2152	* - some cap the value (e.g. FreeBSD caps
2153	* at 0, but doesn't enforce SEMVMX)
2154	*
2155	* Linux caps the semaphore value, both at 0
2156	* and at SEMVMX.
2157	*
2158	* Manfred <[email protected]>
2159	*/
2160	if (semaphore->semval < 0)
2161	semaphore->semval = 0;
2162	if (semaphore->semval > SEMVMX)
2163	semaphore->semval = SEMVMX;
2164	semaphore->sempid = task_tgid_vnr(current);
2165	}
2166	}
2167	/* maybe some queued-up processes were waiting for this */
2168	INIT_LIST_HEAD(&tasks);
2169	do_smart_update(sma, NULL, 0, 1, &tasks);
2170	sem_unlock(sma, -1);
2171	rcu_read_unlock();
2172	wake_up_sem_queue_do(&tasks);
2173
2174	kfree_rcu(un, rcu);
2175	}
2176	kfree(ulp);
2177	}
2178
2179	#ifdef CONFIG_PROC_FS
2180	static int sysvipc_sem_proc_show(struct seq_file s, void it)
2181	{
2182	struct user_namespace *user_ns = seq_user_ns(s);
2183	struct sem_array *sma = it;
2184	time_t sem_otime;
2185
2186	/*
2187	* The proc interface isn't aware of sem_lock(), it calls
2188	* ipc_lock_object() directly (in sysvipc_find_ipc).
2189	* In order to stay compatible with sem_lock(), we must wait until
2190	* all simple semop() calls have left their critical regions.
2191	*/
2192	sem_wait_array(sma);
2193
2194	sem_otime = get_semotime(sma);
2195
2196	seq_printf(s,
2197	"%10d %10d %4o %10u %5u %5u %5u %5u %10lu %10lu\n",
2198	sma->sem_perm.key,
2199	sma->sem_perm.id,
2200	sma->sem_perm.mode,
2201	sma->sem_nsems,
2202	from_kuid_munged(user_ns, sma->sem_perm.uid),
2203	from_kgid_munged(user_ns, sma->sem_perm.gid),
2204	from_kuid_munged(user_ns, sma->sem_perm.cuid),
2205	from_kgid_munged(user_ns, sma->sem_perm.cgid),
2206	sem_otime,
2207	sma->sem_ctime);
2208
2209	return 0;
2210	}
2211	#endif